2 #define SYSCALL_NO_TLS 1
20 #include <semaphore.h>
21 #include <sys/membarrier.h>
22 #include "pthread_impl.h"
23 #include "fork_impl.h"
27 static size_t ldso_page_size
;
28 /* libc.h may have defined a macro for dynamic PAGE_SIZE already, but
29 * PAGESIZE is only defined if it's constant for the arch. */
32 #define PAGE_SIZE ldso_page_size
35 #define malloc __libc_malloc
36 #define calloc __libc_calloc
37 #define realloc __libc_realloc
38 #define free __libc_free
40 static void error_impl(const char *, ...);
41 static void error_noop(const char *, ...);
42 static void (*error
)(const char *, ...) = error_noop
;
44 #define MAXP2(a,b) (-(-(a)&-(b)))
45 #define ALIGN(x,y) ((x)+(y)-1 & -(y))
47 #define container_of(p,t,m) ((t*)((char *)(p)-offsetof(t,m)))
48 #define countof(a) ((sizeof (a))/(sizeof (a)[0]))
60 struct td_index
*next
;
65 struct fdpic_loadmap
*loadmap
;
71 struct dso
*next
, *prev
;
81 struct dso
*syms_next
, *lazy_next
;
82 size_t *lazy
, lazy_cnt
;
93 struct dso
**deps
, *needed_by
;
96 pthread_t ctor_visitor
;
97 char *rpath_orig
, *rpath
;
98 struct tls_module tls
;
100 size_t relro_start
, relro_end
;
102 unsigned char *new_tls
;
103 struct td_index
*td_index
;
104 struct dso
*fini_next
;
109 struct fdpic_loadmap
*loadmap
;
124 typedef void (*stage3_func
)(size_t *, size_t *);
126 static struct builtin_tls
{
131 #define MIN_TLS_ALIGN offsetof(struct builtin_tls, pt)
133 #define ADDEND_LIMIT 4096
134 static size_t *saved_addends
, *apply_addends_to
;
136 static struct dso ldso
;
137 static struct dso
*head
, *tail
, *fini_head
, *syms_tail
, *lazy_head
;
138 static char *env_path
, *sys_path
;
139 static unsigned long long gencnt
;
142 static int ldso_fail
;
144 static int shutting_down
;
145 static jmp_buf *rtld_fail
;
146 static pthread_rwlock_t lock
;
147 static struct debug debug
;
148 static struct tls_module
*tls_tail
;
149 static size_t tls_cnt
, tls_offset
, tls_align
= MIN_TLS_ALIGN
;
150 static size_t static_tls_cnt
;
151 static pthread_mutex_t init_fini_lock
;
152 static pthread_cond_t ctor_cond
;
153 static struct dso
*builtin_deps
[2];
154 static struct dso
*const no_deps
[1];
155 static struct dso
*builtin_ctor_queue
[4];
156 static struct dso
**main_ctor_queue
;
157 static struct fdpic_loadmap
*app_loadmap
;
158 static struct fdpic_dummy_loadmap app_dummy_loadmap
;
160 struct debug
*_dl_debug_addr
= &debug
;
162 extern weak hidden
char __ehdr_start
[];
164 extern hidden
int __malloc_replaced
;
166 hidden
void (*const __init_array_start
)(void)=0, (*const __fini_array_start
)(void)=0;
168 extern hidden
void (*const __init_array_end
)(void), (*const __fini_array_end
)(void);
170 weak_alias(__init_array_start
, __init_array_end
);
171 weak_alias(__fini_array_start
, __fini_array_end
);
173 static int dl_strcmp(const char *l
, const char *r
)
175 for (; *l
==*r
&& *l
; l
++, r
++);
176 return *(unsigned char *)l
- *(unsigned char *)r
;
178 #define strcmp(l,r) dl_strcmp(l,r)
180 /* Compute load address for a virtual address in a given dso. */
182 static void *laddr(const struct dso
*p
, size_t v
)
185 if (!p
->loadmap
) return p
->base
+ v
;
186 for (j
=0; v
-p
->loadmap
->segs
[j
].p_vaddr
>= p
->loadmap
->segs
[j
].p_memsz
; j
++);
187 return (void *)(v
- p
->loadmap
->segs
[j
].p_vaddr
+ p
->loadmap
->segs
[j
].addr
);
189 static void *laddr_pg(const struct dso
*p
, size_t v
)
192 size_t pgsz
= PAGE_SIZE
;
193 if (!p
->loadmap
) return p
->base
+ v
;
195 size_t a
= p
->loadmap
->segs
[j
].p_vaddr
;
196 size_t b
= a
+ p
->loadmap
->segs
[j
].p_memsz
;
202 return (void *)(v
- p
->loadmap
->segs
[j
].p_vaddr
+ p
->loadmap
->segs
[j
].addr
);
204 static void (*fdbarrier(void *p
))()
207 __asm__("" : "=r"(fd
) : "0"(p
));
210 #define fpaddr(p, v) fdbarrier((&(struct funcdesc){ \
211 laddr(p, v), (p)->got }))
213 #define laddr(p, v) (void *)((p)->base + (v))
214 #define laddr_pg(p, v) laddr(p, v)
215 #define fpaddr(p, v) ((void (*)())laddr(p, v))
218 static void decode_vec(size_t *v
, size_t *a
, size_t cnt
)
221 for (i
=0; i
<cnt
; i
++) a
[i
] = 0;
222 for (; v
[0]; v
+=2) if (v
[0]-1<cnt
-1) {
223 if (v
[0] < 8*sizeof(long))
229 static int search_vec(size_t *v
, size_t *r
, size_t key
)
231 for (; v
[0]!=key
; v
+=2)
237 static uint32_t sysv_hash(const char *s0
)
239 const unsigned char *s
= (void *)s0
;
245 return h
& 0xfffffff;
248 static uint32_t gnu_hash(const char *s0
)
250 const unsigned char *s
= (void *)s0
;
251 uint_fast32_t h
= 5381;
257 static Sym
*sysv_lookup(const char *s
, uint32_t h
, struct dso
*dso
)
260 Sym
*syms
= dso
->syms
;
261 Elf_Symndx
*hashtab
= dso
->hashtab
;
262 char *strings
= dso
->strings
;
263 for (i
=hashtab
[2+h
%hashtab
[0]]; i
; i
=hashtab
[2+hashtab
[0]+i
]) {
264 if ((!dso
->versym
|| dso
->versym
[i
] >= 0)
265 && (!strcmp(s
, strings
+syms
[i
].st_name
)))
271 static Sym
*gnu_lookup(uint32_t h1
, uint32_t *hashtab
, struct dso
*dso
, const char *s
)
273 uint32_t nbuckets
= hashtab
[0];
274 uint32_t *buckets
= hashtab
+ 4 + hashtab
[2]*(sizeof(size_t)/4);
275 uint32_t i
= buckets
[h1
% nbuckets
];
279 uint32_t *hashval
= buckets
+ nbuckets
+ (i
- hashtab
[1]);
281 for (h1
|= 1; ; i
++) {
282 uint32_t h2
= *hashval
++;
283 if ((h1
== (h2
|1)) && (!dso
->versym
|| dso
->versym
[i
] >= 0)
284 && !strcmp(s
, dso
->strings
+ dso
->syms
[i
].st_name
))
292 static Sym
*gnu_lookup_filtered(uint32_t h1
, uint32_t *hashtab
, struct dso
*dso
, const char *s
, uint32_t fofs
, size_t fmask
)
294 const size_t *bloomwords
= (const void *)(hashtab
+4);
295 size_t f
= bloomwords
[fofs
& (hashtab
[2]-1)];
296 if (!(f
& fmask
)) return 0;
298 f
>>= (h1
>> hashtab
[3]) % (8 * sizeof f
);
299 if (!(f
& 1)) return 0;
301 return gnu_lookup(h1
, hashtab
, dso
, s
);
304 #define OK_TYPES (1<<STT_NOTYPE | 1<<STT_OBJECT | 1<<STT_FUNC | 1<<STT_COMMON | 1<<STT_TLS)
305 #define OK_BINDS (1<<STB_GLOBAL | 1<<STB_WEAK | 1<<STB_GNU_UNIQUE)
307 #ifndef ARCH_SYM_REJECT_UND
308 #define ARCH_SYM_REJECT_UND(s) 0
311 #if defined(__GNUC__)
312 __attribute__((always_inline
))
314 static inline struct symdef
find_sym2(struct dso
*dso
, const char *s
, int need_def
, int use_deps
)
316 uint32_t h
= 0, gh
= gnu_hash(s
), gho
= gh
/ (8*sizeof(size_t)), *ght
;
317 size_t ghm
= 1ul << gh
% (8*sizeof(size_t));
318 struct symdef def
= {0};
319 struct dso
**deps
= use_deps
? dso
->deps
: 0;
320 for (; dso
; dso
=use_deps
? *deps
++ : dso
->syms_next
) {
322 if ((ght
= dso
->ghashtab
)) {
323 sym
= gnu_lookup_filtered(gh
, ght
, dso
, s
, gho
, ghm
);
325 if (!h
) h
= sysv_hash(s
);
326 sym
= sysv_lookup(s
, h
, dso
);
330 if (need_def
|| (sym
->st_info
&0xf) == STT_TLS
331 || ARCH_SYM_REJECT_UND(sym
))
334 if ((sym
->st_info
&0xf) != STT_TLS
)
336 if (!(1<<(sym
->st_info
&0xf) & OK_TYPES
)) continue;
337 if (!(1<<(sym
->st_info
>>4) & OK_BINDS
)) continue;
345 static struct symdef
find_sym(struct dso
*dso
, const char *s
, int need_def
)
347 return find_sym2(dso
, s
, need_def
, 0);
350 static struct symdef
get_lfs64(const char *name
)
353 static const char lfs64_list
[] =
354 "aio_cancel\0aio_error\0aio_fsync\0aio_read\0aio_return\0"
355 "aio_suspend\0aio_write\0alphasort\0creat\0fallocate\0"
356 "fgetpos\0fopen\0freopen\0fseeko\0fsetpos\0fstat\0"
357 "fstatat\0fstatfs\0fstatvfs\0ftello\0ftruncate\0ftw\0"
358 "getdents\0getrlimit\0glob\0globfree\0lio_listio\0"
359 "lockf\0lseek\0lstat\0mkostemp\0mkostemps\0mkstemp\0"
360 "mkstemps\0mmap\0nftw\0open\0openat\0posix_fadvise\0"
361 "posix_fallocate\0pread\0preadv\0prlimit\0pwrite\0"
362 "pwritev\0readdir\0scandir\0sendfile\0setrlimit\0"
363 "stat\0statfs\0statvfs\0tmpfile\0truncate\0versionsort\0"
364 "__fxstat\0__fxstatat\0__lxstat\0__xstat\0";
365 if (!strcmp(name
, "readdir64_r"))
366 return find_sym(&ldso
, "readdir_r", 1);
367 size_t l
= strnlen(name
, 18);
368 if (l
<2 || name
[l
-2]!='6' || name
[l
-1]!='4' || name
[l
])
370 for (p
=lfs64_list
; *p
; p
++) {
371 if (!strncmp(name
, p
, l
-2) && !p
[l
-2])
372 return find_sym(&ldso
, p
, 1);
376 return (struct symdef
){ 0 };
379 static void do_relocs(struct dso
*dso
, size_t *rel
, size_t rel_size
, size_t stride
)
381 unsigned char *base
= dso
->base
;
382 Sym
*syms
= dso
->syms
;
383 char *strings
= dso
->strings
;
394 int skip_relative
= 0, reuse_addends
= 0, save_slot
= 0;
397 /* Only ldso's REL table needs addend saving/reuse. */
398 if (rel
== apply_addends_to
)
403 for (; rel_size
; rel
+=stride
, rel_size
-=stride
*sizeof(size_t)) {
404 if (skip_relative
&& IS_RELATIVE(rel
[1], dso
->syms
)) continue;
405 type
= R_TYPE(rel
[1]);
406 if (type
== REL_NONE
) continue;
407 reloc_addr
= laddr(dso
, rel
[0]);
411 } else if (type
==REL_GOT
|| type
==REL_PLT
|| type
==REL_COPY
) {
413 } else if (reuse_addends
) {
414 /* Save original addend in stage 2 where the dso
415 * chain consists of just ldso; otherwise read back
416 * saved addend since the inline one was clobbered. */
418 saved_addends
[save_slot
] = *reloc_addr
;
419 addend
= saved_addends
[save_slot
++];
421 addend
= *reloc_addr
;
424 sym_index
= R_SYM(rel
[1]);
426 sym
= syms
+ sym_index
;
427 name
= strings
+ sym
->st_name
;
428 ctx
= type
==REL_COPY
? head
->syms_next
: head
;
429 def
= (sym
->st_info
>>4) == STB_LOCAL
430 ? (struct symdef
){ .dso
= dso
, .sym
= sym
}
431 : find_sym(ctx
, name
, type
==REL_PLT
);
432 if (!def
.sym
) def
= get_lfs64(name
);
433 if (!def
.sym
&& (sym
->st_shndx
!= SHN_UNDEF
434 || sym
->st_info
>>4 != STB_WEAK
)) {
435 if (dso
->lazy
&& (type
==REL_PLT
|| type
==REL_GOT
)) {
436 dso
->lazy
[3*dso
->lazy_cnt
+0] = rel
[0];
437 dso
->lazy
[3*dso
->lazy_cnt
+1] = rel
[1];
438 dso
->lazy
[3*dso
->lazy_cnt
+2] = addend
;
442 error("Error relocating %s: %s: symbol not found",
444 if (runtime
) longjmp(*rtld_fail
, 1);
453 sym_val
= def
.sym
? (size_t)laddr(def
.dso
, def
.sym
->st_value
) : 0;
454 tls_val
= def
.sym
? def
.sym
->st_value
: 0;
456 if ((type
== REL_TPOFF
|| type
== REL_TPOFF_NEG
)
457 && def
.dso
->tls_id
> static_tls_cnt
) {
458 error("Error relocating %s: %s: initial-exec TLS "
459 "resolves to dynamic definition in %s",
460 dso
->name
, name
, def
.dso
->name
);
461 longjmp(*rtld_fail
, 1);
466 addend
-= (size_t)reloc_addr
;
470 *reloc_addr
= sym_val
+ addend
;
473 memcpy(reloc_addr
, &(size_t){sym_val
+ addend
}, sizeof(size_t));
476 *reloc_addr
= (size_t)base
+ addend
;
479 if (sym
) *reloc_addr
= sym_val
+ addend
;
480 else *reloc_addr
= (size_t)base
+ addend
;
483 memcpy(reloc_addr
, (void *)sym_val
, sym
->st_size
);
486 *(uint32_t *)reloc_addr
= sym_val
+ addend
487 - (size_t)reloc_addr
;
490 *reloc_addr
= def
.sym
? (size_t)(def
.dso
->funcdescs
491 + (def
.sym
- def
.dso
->syms
)) : 0;
493 case REL_FUNCDESC_VAL
:
494 if ((sym
->st_info
&0xf) == STT_SECTION
) *reloc_addr
+= sym_val
;
495 else *reloc_addr
= sym_val
;
496 reloc_addr
[1] = def
.sym
? (size_t)def
.dso
->got
: 0;
499 *reloc_addr
= def
.dso
->tls_id
;
502 *reloc_addr
= tls_val
+ addend
- DTP_OFFSET
;
506 *reloc_addr
= tls_val
+ def
.dso
->tls
.offset
+ TPOFF_K
+ addend
;
510 *reloc_addr
= tls_val
- def
.dso
->tls
.offset
+ addend
;
513 *reloc_addr
= def
.dso
->tls
.offset
- tls_val
+ addend
;
517 if (stride
<3) addend
= reloc_addr
[!TLSDESC_BACKWARDS
];
518 if (def
.dso
->tls_id
> static_tls_cnt
) {
519 struct td_index
*new = malloc(sizeof *new);
522 "Error relocating %s: cannot allocate TLSDESC for %s",
523 dso
->name
, sym
? name
: "(local)" );
524 longjmp(*rtld_fail
, 1);
526 new->next
= dso
->td_index
;
528 new->args
[0] = def
.dso
->tls_id
;
529 new->args
[1] = tls_val
+ addend
- DTP_OFFSET
;
530 reloc_addr
[0] = (size_t)__tlsdesc_dynamic
;
531 reloc_addr
[1] = (size_t)new;
533 reloc_addr
[0] = (size_t)__tlsdesc_static
;
535 reloc_addr
[1] = tls_val
+ def
.dso
->tls
.offset
538 reloc_addr
[1] = tls_val
- def
.dso
->tls
.offset
542 /* Some archs (32-bit ARM at least) invert the order of
543 * the descriptor members. Fix them up here. */
544 if (TLSDESC_BACKWARDS
) {
545 size_t tmp
= reloc_addr
[0];
546 reloc_addr
[0] = reloc_addr
[1];
551 error("Error relocating %s: unsupported relocation type %d",
553 if (runtime
) longjmp(*rtld_fail
, 1);
559 static void do_relr_relocs(struct dso
*dso
, size_t *relr
, size_t relr_size
)
561 if (dso
== &ldso
) return; /* self-relocation was done in _dlstart */
562 unsigned char *base
= dso
->base
;
564 for (; relr_size
; relr
++, relr_size
-=sizeof(size_t))
565 if ((relr
[0]&1) == 0) {
566 reloc_addr
= laddr(dso
, relr
[0]);
567 *reloc_addr
++ += (size_t)base
;
570 for (size_t bitmap
=relr
[0]; (bitmap
>>=1); i
++)
572 reloc_addr
[i
] += (size_t)base
;
573 reloc_addr
+= 8*sizeof(size_t)-1;
577 static void redo_lazy_relocs()
579 struct dso
*p
= lazy_head
, *next
;
583 size_t size
= p
->lazy_cnt
*3*sizeof(size_t);
585 do_relocs(p
, p
->lazy
, size
, 3);
587 p
->lazy_next
= lazy_head
;
597 /* A huge hack: to make up for the wastefulness of shared libraries
598 * needing at least a page of dirty memory even if they have no global
599 * data, we reclaim the gaps at the beginning and end of writable maps
600 * and "donate" them to the heap. */
602 static void reclaim(struct dso
*dso
, size_t start
, size_t end
)
604 if (start
>= dso
->relro_start
&& start
< dso
->relro_end
) start
= dso
->relro_end
;
605 if (end
>= dso
->relro_start
&& end
< dso
->relro_end
) end
= dso
->relro_start
;
606 if (start
>= end
) return;
607 char *base
= laddr_pg(dso
, start
);
608 __malloc_donate(base
, base
+(end
-start
));
611 static void reclaim_gaps(struct dso
*dso
)
613 Phdr
*ph
= dso
->phdr
;
614 size_t phcnt
= dso
->phnum
;
616 for (; phcnt
--; ph
=(void *)((char *)ph
+dso
->phentsize
)) {
617 if (ph
->p_type
!=PT_LOAD
) continue;
618 if ((ph
->p_flags
&(PF_R
|PF_W
))!=(PF_R
|PF_W
)) continue;
619 reclaim(dso
, ph
->p_vaddr
& -PAGE_SIZE
, ph
->p_vaddr
);
620 reclaim(dso
, ph
->p_vaddr
+ph
->p_memsz
,
621 ph
->p_vaddr
+ph
->p_memsz
+PAGE_SIZE
-1 & -PAGE_SIZE
);
625 static ssize_t
read_loop(int fd
, void *p
, size_t n
)
627 for (size_t i
=0; i
<n
; ) {
628 ssize_t l
= read(fd
, (char *)p
+i
, n
-i
);
630 if (errno
==EINTR
) continue;
639 static void *mmap_fixed(void *p
, size_t n
, int prot
, int flags
, int fd
, off_t off
)
641 static int no_map_fixed
;
645 q
= mmap(p
, n
, prot
, flags
|MAP_FIXED
, fd
, off
);
646 if (!DL_NOMMU_SUPPORT
|| q
!= MAP_FAILED
|| errno
!= EINVAL
)
650 /* Fallbacks for MAP_FIXED failure on NOMMU kernels. */
651 if (flags
& MAP_ANONYMOUS
) {
656 if (lseek(fd
, off
, SEEK_SET
) < 0) return MAP_FAILED
;
657 for (q
=p
; n
; q
+=r
, off
+=r
, n
-=r
) {
659 if (r
< 0 && errno
!= EINTR
) return MAP_FAILED
;
668 static void unmap_library(struct dso
*dso
)
672 for (i
=0; i
<dso
->loadmap
->nsegs
; i
++) {
673 if (!dso
->loadmap
->segs
[i
].p_memsz
)
675 munmap((void *)dso
->loadmap
->segs
[i
].addr
,
676 dso
->loadmap
->segs
[i
].p_memsz
);
679 } else if (dso
->map
&& dso
->map_len
) {
680 munmap(dso
->map
, dso
->map_len
);
684 static void *map_library(int fd
, struct dso
*dso
)
686 Ehdr buf
[(896+sizeof(Ehdr
))/sizeof(Ehdr
)];
687 void *allocated_buf
=0;
689 size_t addr_min
=SIZE_MAX
, addr_max
=0, map_len
;
690 size_t this_min
, this_max
;
696 unsigned char *map
=MAP_FAILED
, *base
;
701 ssize_t l
= read(fd
, buf
, sizeof buf
);
704 if (l
<sizeof *eh
|| (eh
->e_type
!= ET_DYN
&& eh
->e_type
!= ET_EXEC
))
706 phsize
= eh
->e_phentsize
* eh
->e_phnum
;
707 if (phsize
> sizeof buf
- sizeof *eh
) {
708 allocated_buf
= malloc(phsize
);
709 if (!allocated_buf
) return 0;
710 l
= pread(fd
, allocated_buf
, phsize
, eh
->e_phoff
);
711 if (l
< 0) goto error
;
712 if (l
!= phsize
) goto noexec
;
713 ph
= ph0
= allocated_buf
;
714 } else if (eh
->e_phoff
+ phsize
> l
) {
715 l
= pread(fd
, buf
+1, phsize
, eh
->e_phoff
);
716 if (l
< 0) goto error
;
717 if (l
!= phsize
) goto noexec
;
718 ph
= ph0
= (void *)(buf
+ 1);
720 ph
= ph0
= (void *)((char *)buf
+ eh
->e_phoff
);
722 for (i
=eh
->e_phnum
; i
; i
--, ph
=(void *)((char *)ph
+eh
->e_phentsize
)) {
723 if (ph
->p_type
== PT_DYNAMIC
) {
725 } else if (ph
->p_type
== PT_TLS
) {
726 tls_image
= ph
->p_vaddr
;
727 dso
->tls
.align
= ph
->p_align
;
728 dso
->tls
.len
= ph
->p_filesz
;
729 dso
->tls
.size
= ph
->p_memsz
;
730 } else if (ph
->p_type
== PT_GNU_RELRO
) {
731 dso
->relro_start
= ph
->p_vaddr
& -PAGE_SIZE
;
732 dso
->relro_end
= (ph
->p_vaddr
+ ph
->p_memsz
) & -PAGE_SIZE
;
733 } else if (ph
->p_type
== PT_GNU_STACK
) {
734 if (!runtime
&& ph
->p_memsz
> __default_stacksize
) {
735 __default_stacksize
=
736 ph
->p_memsz
< DEFAULT_STACK_MAX
?
737 ph
->p_memsz
: DEFAULT_STACK_MAX
;
740 if (ph
->p_type
!= PT_LOAD
) continue;
742 if (ph
->p_vaddr
< addr_min
) {
743 addr_min
= ph
->p_vaddr
;
744 off_start
= ph
->p_offset
;
745 prot
= (((ph
->p_flags
&PF_R
) ? PROT_READ
: 0) |
746 ((ph
->p_flags
&PF_W
) ? PROT_WRITE
: 0) |
747 ((ph
->p_flags
&PF_X
) ? PROT_EXEC
: 0));
749 if (ph
->p_vaddr
+ph
->p_memsz
> addr_max
) {
750 addr_max
= ph
->p_vaddr
+ph
->p_memsz
;
753 if (!dyn
) goto noexec
;
754 if (DL_FDPIC
&& !(eh
->e_flags
& FDPIC_CONSTDISP_FLAG
)) {
755 dso
->loadmap
= calloc(1, sizeof *dso
->loadmap
756 + nsegs
* sizeof *dso
->loadmap
->segs
);
757 if (!dso
->loadmap
) goto error
;
758 dso
->loadmap
->nsegs
= nsegs
;
759 for (ph
=ph0
, i
=0; i
<nsegs
; ph
=(void *)((char *)ph
+eh
->e_phentsize
)) {
760 if (ph
->p_type
!= PT_LOAD
) continue;
761 prot
= (((ph
->p_flags
&PF_R
) ? PROT_READ
: 0) |
762 ((ph
->p_flags
&PF_W
) ? PROT_WRITE
: 0) |
763 ((ph
->p_flags
&PF_X
) ? PROT_EXEC
: 0));
764 map
= mmap(0, ph
->p_memsz
+ (ph
->p_vaddr
& PAGE_SIZE
-1),
766 fd
, ph
->p_offset
& -PAGE_SIZE
);
767 if (map
== MAP_FAILED
) {
771 dso
->loadmap
->segs
[i
].addr
= (size_t)map
+
772 (ph
->p_vaddr
& PAGE_SIZE
-1);
773 dso
->loadmap
->segs
[i
].p_vaddr
= ph
->p_vaddr
;
774 dso
->loadmap
->segs
[i
].p_memsz
= ph
->p_memsz
;
776 if (prot
& PROT_WRITE
) {
777 size_t brk
= (ph
->p_vaddr
& PAGE_SIZE
-1)
779 size_t pgbrk
= brk
+ PAGE_SIZE
-1 & -PAGE_SIZE
;
780 size_t pgend
= brk
+ ph
->p_memsz
- ph
->p_filesz
781 + PAGE_SIZE
-1 & -PAGE_SIZE
;
782 if (pgend
> pgbrk
&& mmap_fixed(map
+pgbrk
,
784 MAP_PRIVATE
|MAP_FIXED
|MAP_ANONYMOUS
,
785 -1, off_start
) == MAP_FAILED
)
787 memset(map
+ brk
, 0, pgbrk
-brk
);
790 map
= (void *)dso
->loadmap
->segs
[0].addr
;
794 addr_max
+= PAGE_SIZE
-1;
795 addr_max
&= -PAGE_SIZE
;
796 addr_min
&= -PAGE_SIZE
;
797 off_start
&= -PAGE_SIZE
;
798 map_len
= addr_max
- addr_min
+ off_start
;
799 /* The first time, we map too much, possibly even more than
800 * the length of the file. This is okay because we will not
801 * use the invalid part; we just need to reserve the right
802 * amount of virtual address space to map over later. */
803 map
= DL_NOMMU_SUPPORT
804 ? mmap((void *)addr_min
, map_len
, PROT_READ
|PROT_WRITE
|PROT_EXEC
,
805 MAP_PRIVATE
|MAP_ANONYMOUS
, -1, 0)
806 : mmap((void *)addr_min
, map_len
, prot
,
807 MAP_PRIVATE
, fd
, off_start
);
808 if (map
==MAP_FAILED
) goto error
;
810 dso
->map_len
= map_len
;
811 /* If the loaded file is not relocatable and the requested address is
812 * not available, then the load operation must fail. */
813 if (eh
->e_type
!= ET_DYN
&& addr_min
&& map
!=(void *)addr_min
) {
817 base
= map
- addr_min
;
820 for (ph
=ph0
, i
=eh
->e_phnum
; i
; i
--, ph
=(void *)((char *)ph
+eh
->e_phentsize
)) {
821 if (ph
->p_type
!= PT_LOAD
) continue;
822 /* Check if the programs headers are in this load segment, and
823 * if so, record the address for use by dl_iterate_phdr. */
824 if (!dso
->phdr
&& eh
->e_phoff
>= ph
->p_offset
825 && eh
->e_phoff
+phsize
<= ph
->p_offset
+ph
->p_filesz
) {
826 dso
->phdr
= (void *)(base
+ ph
->p_vaddr
827 + (eh
->e_phoff
-ph
->p_offset
));
828 dso
->phnum
= eh
->e_phnum
;
829 dso
->phentsize
= eh
->e_phentsize
;
831 this_min
= ph
->p_vaddr
& -PAGE_SIZE
;
832 this_max
= ph
->p_vaddr
+ph
->p_memsz
+PAGE_SIZE
-1 & -PAGE_SIZE
;
833 off_start
= ph
->p_offset
& -PAGE_SIZE
;
834 prot
= (((ph
->p_flags
&PF_R
) ? PROT_READ
: 0) |
835 ((ph
->p_flags
&PF_W
) ? PROT_WRITE
: 0) |
836 ((ph
->p_flags
&PF_X
) ? PROT_EXEC
: 0));
837 /* Reuse the existing mapping for the lowest-address LOAD */
838 if ((ph
->p_vaddr
& -PAGE_SIZE
) != addr_min
|| DL_NOMMU_SUPPORT
)
839 if (mmap_fixed(base
+this_min
, this_max
-this_min
, prot
, MAP_PRIVATE
|MAP_FIXED
, fd
, off_start
) == MAP_FAILED
)
841 if (ph
->p_memsz
> ph
->p_filesz
&& (ph
->p_flags
&PF_W
)) {
842 size_t brk
= (size_t)base
+ph
->p_vaddr
+ph
->p_filesz
;
843 size_t pgbrk
= brk
+PAGE_SIZE
-1 & -PAGE_SIZE
;
844 memset((void *)brk
, 0, pgbrk
-brk
& PAGE_SIZE
-1);
845 if (pgbrk
-(size_t)base
< this_max
&& mmap_fixed((void *)pgbrk
, (size_t)base
+this_max
-pgbrk
, prot
, MAP_PRIVATE
|MAP_FIXED
|MAP_ANONYMOUS
, -1, 0) == MAP_FAILED
)
849 for (i
=0; ((size_t *)(base
+dyn
))[i
]; i
+=2)
850 if (((size_t *)(base
+dyn
))[i
]==DT_TEXTREL
) {
851 if (mprotect(map
, map_len
, PROT_READ
|PROT_WRITE
|PROT_EXEC
)
858 dso
->dynv
= laddr(dso
, dyn
);
859 if (dso
->tls
.size
) dso
->tls
.image
= laddr(dso
, tls_image
);
865 if (map
!=MAP_FAILED
) unmap_library(dso
);
870 static int path_open(const char *name
, const char *s
, char *buf
, size_t buf_size
)
875 s
+= strspn(s
, ":\n");
876 l
= strcspn(s
, ":\n");
877 if (l
-1 >= INT_MAX
) return -1;
878 if (snprintf(buf
, buf_size
, "%.*s/%s", (int)l
, s
, name
) < buf_size
) {
879 if ((fd
= open(buf
, O_RDONLY
|O_CLOEXEC
))>=0) return fd
;
887 /* Any negative value but -1 will inhibit
888 * futher path search. */
896 static int fixup_rpath(struct dso
*p
, char *buf
, size_t buf_size
)
899 const char *s
, *t
, *origin
;
901 if (p
->rpath
|| !p
->rpath_orig
) return 0;
902 if (!strchr(p
->rpath_orig
, '$')) {
903 p
->rpath
= p
->rpath_orig
;
908 while ((t
=strchr(s
, '$'))) {
909 if (strncmp(t
, "$ORIGIN", 7) && strncmp(t
, "${ORIGIN}", 9))
914 if (n
> SSIZE_MAX
/PATH_MAX
) return 0;
916 if (p
->kernel_mapped
) {
917 /* $ORIGIN searches cannot be performed for the main program
918 * when it is suid/sgid/AT_SECURE. This is because the
919 * pathname is under the control of the caller of execve.
920 * For libraries, however, $ORIGIN can be processed safely
921 * since the library's pathname came from a trusted source
922 * (either system paths or a call to dlopen). */
925 l
= readlink("/proc/self/exe", buf
, buf_size
);
926 if (l
== -1) switch (errno
) {
941 t
= strrchr(origin
, '/');
945 /* Normally p->name will always be an absolute or relative
946 * pathname containing at least one '/' character, but in the
947 * case where ldso was invoked as a command to execute a
948 * program in the working directory, app.name may not. Fix. */
952 /* Disallow non-absolute origins for suid/sgid/AT_SECURE. */
953 if (libc
.secure
&& *origin
!= '/')
955 p
->rpath
= malloc(strlen(p
->rpath_orig
) + n
*l
+ 1);
956 if (!p
->rpath
) return -1;
960 while ((t
=strchr(s
, '$'))) {
963 memcpy(d
, origin
, l
);
965 /* It was determined previously that the '$' is followed
966 * either by "ORIGIN" or "{ORIGIN}". */
967 s
= t
+ 7 + 2*(t
[1]=='{');
973 static void decode_dyn(struct dso
*p
)
976 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
977 p
->syms
= laddr(p
, dyn
[DT_SYMTAB
]);
978 p
->strings
= laddr(p
, dyn
[DT_STRTAB
]);
979 if (dyn
[0]&(1<<DT_HASH
))
980 p
->hashtab
= laddr(p
, dyn
[DT_HASH
]);
981 if (dyn
[0]&(1<<DT_RPATH
))
982 p
->rpath_orig
= p
->strings
+ dyn
[DT_RPATH
];
983 if (dyn
[0]&(1<<DT_RUNPATH
))
984 p
->rpath_orig
= p
->strings
+ dyn
[DT_RUNPATH
];
985 if (dyn
[0]&(1<<DT_PLTGOT
))
986 p
->got
= laddr(p
, dyn
[DT_PLTGOT
]);
987 if (search_vec(p
->dynv
, dyn
, DT_GNU_HASH
))
988 p
->ghashtab
= laddr(p
, *dyn
);
989 if (search_vec(p
->dynv
, dyn
, DT_VERSYM
))
990 p
->versym
= laddr(p
, *dyn
);
993 static size_t count_syms(struct dso
*p
)
995 if (p
->hashtab
) return p
->hashtab
[1];
998 uint32_t *buckets
= p
->ghashtab
+ 4 + (p
->ghashtab
[2]*sizeof(size_t)/4);
1000 for (i
= nsym
= 0; i
< p
->ghashtab
[0]; i
++) {
1001 if (buckets
[i
] > nsym
)
1005 hashval
= buckets
+ p
->ghashtab
[0] + (nsym
- p
->ghashtab
[1]);
1007 while (!(*hashval
++ & 1));
1012 static void *dl_mmap(size_t n
)
1015 int prot
= PROT_READ
|PROT_WRITE
, flags
= MAP_ANONYMOUS
|MAP_PRIVATE
;
1017 p
= (void *)__syscall(SYS_mmap2
, 0, n
, prot
, flags
, -1, 0);
1019 p
= (void *)__syscall(SYS_mmap
, 0, n
, prot
, flags
, -1, 0);
1021 return (unsigned long)p
> -4096UL ? 0 : p
;
1024 static void makefuncdescs(struct dso
*p
)
1026 static int self_done
;
1027 size_t nsym
= count_syms(p
);
1028 size_t i
, size
= nsym
* sizeof(*p
->funcdescs
);
1031 p
->funcdescs
= dl_mmap(size
);
1034 p
->funcdescs
= malloc(size
);
1036 if (!p
->funcdescs
) {
1037 if (!runtime
) a_crash();
1038 error("Error allocating function descriptors for %s", p
->name
);
1039 longjmp(*rtld_fail
, 1);
1041 for (i
=0; i
<nsym
; i
++) {
1042 if ((p
->syms
[i
].st_info
&0xf)==STT_FUNC
&& p
->syms
[i
].st_shndx
) {
1043 p
->funcdescs
[i
].addr
= laddr(p
, p
->syms
[i
].st_value
);
1044 p
->funcdescs
[i
].got
= p
->got
;
1046 p
->funcdescs
[i
].addr
= 0;
1047 p
->funcdescs
[i
].got
= 0;
1052 static struct dso
*load_library(const char *name
, struct dso
*needed_by
)
1054 char buf
[2*NAME_MAX
+2];
1055 const char *pathname
;
1057 struct dso
*p
, temp_dso
= {0};
1069 /* Catch and block attempts to reload the implementation itself */
1070 if (name
[0]=='l' && name
[1]=='i' && name
[2]=='b') {
1071 static const char reserved
[] =
1072 "c.pthread.rt.m.dl.util.xnet.";
1073 const char *rp
, *next
;
1074 for (rp
=reserved
; *rp
; rp
=next
) {
1075 next
= strchr(rp
, '.') + 1;
1076 if (strncmp(name
+3, rp
, next
-rp
) == 0)
1081 /* Track which names have been resolved
1082 * and only report each one once. */
1083 static unsigned reported
;
1084 unsigned mask
= 1U<<(rp
-reserved
);
1085 if (!(reported
& mask
)) {
1087 dprintf(1, "\t%s => %s (%p)\n",
1095 if (!strcmp(name
, ldso
.name
)) is_self
= 1;
1104 if (strchr(name
, '/')) {
1106 fd
= open(name
, O_RDONLY
|O_CLOEXEC
);
1108 /* Search for the name to see if it's already loaded */
1109 for (p
=head
->next
; p
; p
=p
->next
) {
1110 if (p
->shortname
&& !strcmp(p
->shortname
, name
)) {
1114 if (strlen(name
) > NAME_MAX
) return 0;
1116 if (env_path
) fd
= path_open(name
, env_path
, buf
, sizeof buf
);
1117 for (p
=needed_by
; fd
== -1 && p
; p
=p
->needed_by
) {
1118 if (fixup_rpath(p
, buf
, sizeof buf
) < 0)
1119 fd
= -2; /* Inhibit further search. */
1121 fd
= path_open(name
, p
->rpath
, buf
, sizeof buf
);
1127 if (ldso
.name
[0]=='/') {
1129 for (s
=t
=z
=ldso
.name
; *s
; s
++)
1130 if (*s
=='/') z
=t
, t
=s
;
1131 prefix_len
= z
-ldso
.name
;
1132 if (prefix_len
< PATH_MAX
)
1139 char etc_ldso_path
[prefix_len
+ 1
1140 + sizeof "/etc/ld-musl-" LDSO_ARCH
".path"];
1141 snprintf(etc_ldso_path
, sizeof etc_ldso_path
,
1142 "%.*s/etc/ld-musl-" LDSO_ARCH
".path",
1143 (int)prefix_len
, prefix
);
1144 fd
= open(etc_ldso_path
, O_RDONLY
|O_CLOEXEC
);
1147 if (!fstat(fd
, &st
)) n
= st
.st_size
;
1148 if ((sys_path
= malloc(n
+1)))
1150 if (!sys_path
|| read_loop(fd
, sys_path
, n
)<0) {
1155 } else if (errno
!= ENOENT
) {
1159 if (!sys_path
) sys_path
= "/lib:/usr/local/lib:/usr/lib";
1160 fd
= path_open(name
, sys_path
, buf
, sizeof buf
);
1164 if (fd
< 0) return 0;
1165 if (fstat(fd
, &st
) < 0) {
1169 for (p
=head
->next
; p
; p
=p
->next
) {
1170 if (p
->dev
== st
.st_dev
&& p
->ino
== st
.st_ino
) {
1171 /* If this library was previously loaded with a
1172 * pathname but a search found the same inode,
1173 * setup its shortname so it can be found by name. */
1174 if (!p
->shortname
&& pathname
!= name
)
1175 p
->shortname
= strrchr(p
->name
, '/')+1;
1180 map
= noload
? 0 : map_library(fd
, &temp_dso
);
1184 /* Avoid the danger of getting two versions of libc mapped into the
1185 * same process when an absolute pathname was used. The symbols
1186 * checked are chosen to catch both musl and glibc, and to avoid
1187 * false positives from interposition-hack libraries. */
1188 decode_dyn(&temp_dso
);
1189 if (find_sym(&temp_dso
, "__libc_start_main", 1).sym
&&
1190 find_sym(&temp_dso
, "stdin", 1).sym
) {
1191 unmap_library(&temp_dso
);
1192 return load_library("libc.so", needed_by
);
1194 /* Past this point, if we haven't reached runtime yet, ldso has
1195 * committed either to use the mapped library or to abort execution.
1196 * Unmapping is not possible, so we can safely reclaim gaps. */
1197 if (!runtime
) reclaim_gaps(&temp_dso
);
1199 /* Allocate storage for the new DSO. When there is TLS, this
1200 * storage must include a reservation for all pre-existing
1201 * threads to obtain copies of both the new TLS, and an
1202 * extended DTV capable of storing an additional slot for
1203 * the newly-loaded DSO. */
1204 alloc_size
= sizeof *p
+ strlen(pathname
) + 1;
1205 if (runtime
&& temp_dso
.tls
.image
) {
1206 size_t per_th
= temp_dso
.tls
.size
+ temp_dso
.tls
.align
1207 + sizeof(void *) * (tls_cnt
+3);
1208 n_th
= libc
.threads_minus_1
+ 1;
1209 if (n_th
> SSIZE_MAX
/ per_th
) alloc_size
= SIZE_MAX
;
1210 else alloc_size
+= n_th
* per_th
;
1212 p
= calloc(1, alloc_size
);
1214 unmap_library(&temp_dso
);
1217 memcpy(p
, &temp_dso
, sizeof temp_dso
);
1220 p
->needed_by
= needed_by
;
1222 p
->runtime_loaded
= runtime
;
1223 strcpy(p
->name
, pathname
);
1224 /* Add a shortname only if name arg was not an explicit pathname. */
1225 if (pathname
!= name
) p
->shortname
= strrchr(p
->name
, '/')+1;
1227 p
->tls_id
= ++tls_cnt
;
1228 tls_align
= MAXP2(tls_align
, p
->tls
.align
);
1230 p
->tls
.offset
= tls_offset
+ ( (p
->tls
.align
-1) &
1231 (-tls_offset
+ (uintptr_t)p
->tls
.image
) );
1232 tls_offset
= p
->tls
.offset
+ p
->tls
.size
;
1234 tls_offset
+= p
->tls
.size
+ p
->tls
.align
- 1;
1235 tls_offset
-= (tls_offset
+ (uintptr_t)p
->tls
.image
)
1237 p
->tls
.offset
= tls_offset
;
1239 p
->new_dtv
= (void *)(-sizeof(size_t) &
1240 (uintptr_t)(p
->name
+strlen(p
->name
)+sizeof(size_t)));
1241 p
->new_tls
= (void *)(p
->new_dtv
+ n_th
*(tls_cnt
+1));
1242 if (tls_tail
) tls_tail
->next
= &p
->tls
;
1243 else libc
.tls_head
= &p
->tls
;
1251 if (DL_FDPIC
) makefuncdescs(p
);
1253 if (ldd_mode
) dprintf(1, "\t%s => %s (%p)\n", name
, pathname
, p
->base
);
1258 static void load_direct_deps(struct dso
*p
)
1262 if (p
->deps
) return;
1263 /* For head, all preloads are direct pseudo-dependencies.
1264 * Count and include them now to avoid realloc later. */
1265 if (p
==head
) for (struct dso
*q
=p
->next
; q
; q
=q
->next
)
1267 for (i
=0; p
->dynv
[i
]; i
+=2)
1268 if (p
->dynv
[i
] == DT_NEEDED
) cnt
++;
1269 /* Use builtin buffer for apps with no external deps, to
1270 * preserve property of no runtime failure paths. */
1271 p
->deps
= (p
==head
&& cnt
<2) ? builtin_deps
:
1272 calloc(cnt
+1, sizeof *p
->deps
);
1274 error("Error loading dependencies for %s", p
->name
);
1275 if (runtime
) longjmp(*rtld_fail
, 1);
1278 if (p
==head
) for (struct dso
*q
=p
->next
; q
; q
=q
->next
)
1280 for (i
=0; p
->dynv
[i
]; i
+=2) {
1281 if (p
->dynv
[i
] != DT_NEEDED
) continue;
1282 struct dso
*dep
= load_library(p
->strings
+ p
->dynv
[i
+1], p
);
1284 error("Error loading shared library %s: %m (needed by %s)",
1285 p
->strings
+ p
->dynv
[i
+1], p
->name
);
1286 if (runtime
) longjmp(*rtld_fail
, 1);
1289 p
->deps
[cnt
++] = dep
;
1292 p
->ndeps_direct
= cnt
;
1295 static void load_deps(struct dso
*p
)
1297 if (p
->deps
) return;
1298 for (; p
; p
=p
->next
)
1299 load_direct_deps(p
);
1302 static void extend_bfs_deps(struct dso
*p
)
1304 size_t i
, j
, cnt
, ndeps_all
;
1307 /* Can't use realloc if the original p->deps was allocated at
1308 * program entry and malloc has been replaced, or if it's
1309 * the builtin non-allocated trivial main program deps array. */
1310 int no_realloc
= (__malloc_replaced
&& !p
->runtime_loaded
)
1311 || p
->deps
== builtin_deps
;
1313 if (p
->bfs_built
) return;
1314 ndeps_all
= p
->ndeps_direct
;
1316 /* Mark existing (direct) deps so they won't be duplicated. */
1317 for (i
=0; p
->deps
[i
]; i
++)
1318 p
->deps
[i
]->mark
= 1;
1320 /* For each dependency already in the list, copy its list of direct
1321 * dependencies to the list, excluding any items already in the
1322 * list. Note that the list this loop iterates over will grow during
1323 * the loop, but since duplicates are excluded, growth is bounded. */
1324 for (i
=0; p
->deps
[i
]; i
++) {
1325 struct dso
*dep
= p
->deps
[i
];
1326 for (j
=cnt
=0; j
<dep
->ndeps_direct
; j
++)
1327 if (!dep
->deps
[j
]->mark
) cnt
++;
1329 malloc(sizeof(*tmp
) * (ndeps_all
+cnt
+1)) :
1330 realloc(p
->deps
, sizeof(*tmp
) * (ndeps_all
+cnt
+1));
1332 error("Error recording dependencies for %s", p
->name
);
1333 if (runtime
) longjmp(*rtld_fail
, 1);
1337 memcpy(tmp
, p
->deps
, sizeof(*tmp
) * (ndeps_all
+1));
1341 for (j
=0; j
<dep
->ndeps_direct
; j
++) {
1342 if (dep
->deps
[j
]->mark
) continue;
1343 dep
->deps
[j
]->mark
= 1;
1344 p
->deps
[ndeps_all
++] = dep
->deps
[j
];
1346 p
->deps
[ndeps_all
] = 0;
1349 for (p
=head
; p
; p
=p
->next
)
1353 static void load_preload(char *s
)
1357 for (z
=s
; *z
; s
=z
) {
1358 for ( ; *s
&& (isspace(*s
) || *s
==':'); s
++);
1359 for (z
=s
; *z
&& !isspace(*z
) && *z
!=':'; z
++);
1367 static void add_syms(struct dso
*p
)
1369 if (!p
->syms_next
&& syms_tail
!= p
) {
1370 syms_tail
->syms_next
= p
;
1375 static void revert_syms(struct dso
*old_tail
)
1377 struct dso
*p
, *next
;
1378 /* Chop off the tail of the list of dsos that participate in
1379 * the global symbol table, reverting them to RTLD_LOCAL. */
1380 for (p
=old_tail
; p
; p
=next
) {
1381 next
= p
->syms_next
;
1384 syms_tail
= old_tail
;
1387 static void do_mips_relocs(struct dso
*p
, size_t *got
)
1389 size_t i
, j
, rel
[2];
1390 unsigned char *base
= p
->base
;
1391 i
=0; search_vec(p
->dynv
, &i
, DT_MIPS_LOCAL_GOTNO
);
1395 while (i
--) *got
++ += (size_t)base
;
1397 j
=0; search_vec(p
->dynv
, &j
, DT_MIPS_GOTSYM
);
1398 i
=0; search_vec(p
->dynv
, &i
, DT_MIPS_SYMTABNO
);
1399 Sym
*sym
= p
->syms
+ j
;
1400 rel
[0] = (unsigned char *)got
- base
;
1401 for (i
-=j
; i
; i
--, sym
++, rel
[0]+=sizeof(size_t)) {
1402 rel
[1] = R_INFO(sym
-p
->syms
, R_MIPS_JUMP_SLOT
);
1403 do_relocs(p
, rel
, sizeof rel
, 2);
1407 static void reloc_all(struct dso
*p
)
1409 size_t dyn
[DYN_CNT
];
1410 for (; p
; p
=p
->next
) {
1411 if (p
->relocated
) continue;
1412 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
1413 if (NEED_MIPS_GOT_RELOCS
)
1414 do_mips_relocs(p
, laddr(p
, dyn
[DT_PLTGOT
]));
1415 do_relocs(p
, laddr(p
, dyn
[DT_JMPREL
]), dyn
[DT_PLTRELSZ
],
1416 2+(dyn
[DT_PLTREL
]==DT_RELA
));
1417 do_relocs(p
, laddr(p
, dyn
[DT_REL
]), dyn
[DT_RELSZ
], 2);
1418 do_relocs(p
, laddr(p
, dyn
[DT_RELA
]), dyn
[DT_RELASZ
], 3);
1420 do_relr_relocs(p
, laddr(p
, dyn
[DT_RELR
]), dyn
[DT_RELRSZ
]);
1422 if (head
!= &ldso
&& p
->relro_start
!= p
->relro_end
) {
1423 long ret
= __syscall(SYS_mprotect
, laddr(p
, p
->relro_start
),
1424 p
->relro_end
-p
->relro_start
, PROT_READ
);
1425 if (ret
!= 0 && ret
!= -ENOSYS
) {
1426 error("Error relocating %s: RELRO protection failed: %m",
1428 if (runtime
) longjmp(*rtld_fail
, 1);
1436 static void kernel_mapped_dso(struct dso
*p
)
1438 size_t min_addr
= -1, max_addr
= 0, cnt
;
1440 for (cnt
= p
->phnum
; cnt
--; ph
= (void *)((char *)ph
+ p
->phentsize
)) {
1441 if (ph
->p_type
== PT_DYNAMIC
) {
1442 p
->dynv
= laddr(p
, ph
->p_vaddr
);
1443 } else if (ph
->p_type
== PT_GNU_RELRO
) {
1444 p
->relro_start
= ph
->p_vaddr
& -PAGE_SIZE
;
1445 p
->relro_end
= (ph
->p_vaddr
+ ph
->p_memsz
) & -PAGE_SIZE
;
1446 } else if (ph
->p_type
== PT_GNU_STACK
) {
1447 if (!runtime
&& ph
->p_memsz
> __default_stacksize
) {
1448 __default_stacksize
=
1449 ph
->p_memsz
< DEFAULT_STACK_MAX
?
1450 ph
->p_memsz
: DEFAULT_STACK_MAX
;
1453 if (ph
->p_type
!= PT_LOAD
) continue;
1454 if (ph
->p_vaddr
< min_addr
)
1455 min_addr
= ph
->p_vaddr
;
1456 if (ph
->p_vaddr
+ph
->p_memsz
> max_addr
)
1457 max_addr
= ph
->p_vaddr
+ph
->p_memsz
;
1459 min_addr
&= -PAGE_SIZE
;
1460 max_addr
= (max_addr
+ PAGE_SIZE
-1) & -PAGE_SIZE
;
1461 p
->map
= p
->base
+ min_addr
;
1462 p
->map_len
= max_addr
- min_addr
;
1463 p
->kernel_mapped
= 1;
1466 void __libc_exit_fini()
1469 size_t dyn
[DYN_CNT
];
1470 pthread_t self
= __pthread_self();
1472 /* Take both locks before setting shutting_down, so that
1473 * either lock is sufficient to read its value. The lock
1474 * order matches that in dlopen to avoid deadlock. */
1475 pthread_rwlock_wrlock(&lock
);
1476 pthread_mutex_lock(&init_fini_lock
);
1478 pthread_rwlock_unlock(&lock
);
1479 for (p
=fini_head
; p
; p
=p
->fini_next
) {
1480 while (p
->ctor_visitor
&& p
->ctor_visitor
!=self
)
1481 pthread_cond_wait(&ctor_cond
, &init_fini_lock
);
1482 if (!p
->constructed
) continue;
1483 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
1484 if (dyn
[0] & (1<<DT_FINI_ARRAY
)) {
1485 size_t n
= dyn
[DT_FINI_ARRAYSZ
]/sizeof(size_t);
1486 size_t *fn
= (size_t *)laddr(p
, dyn
[DT_FINI_ARRAY
])+n
;
1487 while (n
--) ((void (*)(void))*--fn
)();
1489 #ifndef NO_LEGACY_INITFINI
1490 if ((dyn
[0] & (1<<DT_FINI
)) && dyn
[DT_FINI
])
1491 fpaddr(p
, dyn
[DT_FINI
])();
1496 void __ldso_atfork(int who
)
1499 pthread_rwlock_wrlock(&lock
);
1500 pthread_mutex_lock(&init_fini_lock
);
1502 pthread_mutex_unlock(&init_fini_lock
);
1503 pthread_rwlock_unlock(&lock
);
1507 static struct dso
**queue_ctors(struct dso
*dso
)
1509 size_t cnt
, qpos
, spos
, i
;
1510 struct dso
*p
, **queue
, **stack
;
1512 if (ldd_mode
) return 0;
1514 /* Bound on queue size is the total number of indirect deps.
1515 * If a bfs deps list was built, we can use it. Otherwise,
1516 * bound by the total number of DSOs, which is always safe and
1517 * is reasonable we use it (for main app at startup). */
1518 if (dso
->bfs_built
) {
1519 for (cnt
=0; dso
->deps
[cnt
]; cnt
++)
1520 dso
->deps
[cnt
]->mark
= 0;
1521 cnt
++; /* self, not included in deps */
1523 for (cnt
=0, p
=head
; p
; cnt
++, p
=p
->next
)
1526 cnt
++; /* termination slot */
1527 if (dso
==head
&& cnt
<= countof(builtin_ctor_queue
))
1528 queue
= builtin_ctor_queue
;
1530 queue
= calloc(cnt
, sizeof *queue
);
1533 error("Error allocating constructor queue: %m\n");
1534 if (runtime
) longjmp(*rtld_fail
, 1);
1538 /* Opposite ends of the allocated buffer serve as an output queue
1539 * and a working stack. Setup initial stack with just the argument
1540 * dso and initial queue empty... */
1544 stack
[--spos
] = dso
;
1548 /* Then perform pseudo-DFS sort, but ignoring circular deps. */
1551 while (p
->next_dep
< p
->ndeps_direct
) {
1552 if (p
->deps
[p
->next_dep
]->mark
) {
1556 p
= p
->deps
[p
->next_dep
];
1564 for (i
=0; i
<qpos
; i
++) queue
[i
]->mark
= 0;
1565 for (i
=0; i
<qpos
; i
++)
1566 if (queue
[i
]->ctor_visitor
&& queue
[i
]->ctor_visitor
->tid
< 0) {
1567 error("State of %s is inconsistent due to multithreaded fork\n",
1570 if (runtime
) longjmp(*rtld_fail
, 1);
1576 static void do_init_fini(struct dso
**queue
)
1579 size_t dyn
[DYN_CNT
], i
;
1580 pthread_t self
= __pthread_self();
1582 pthread_mutex_lock(&init_fini_lock
);
1583 for (i
=0; (p
=queue
[i
]); i
++) {
1584 while ((p
->ctor_visitor
&& p
->ctor_visitor
!=self
) || shutting_down
)
1585 pthread_cond_wait(&ctor_cond
, &init_fini_lock
);
1586 if (p
->ctor_visitor
|| p
->constructed
)
1588 p
->ctor_visitor
= self
;
1590 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
1591 if (dyn
[0] & ((1<<DT_FINI
) | (1<<DT_FINI_ARRAY
))) {
1592 p
->fini_next
= fini_head
;
1596 pthread_mutex_unlock(&init_fini_lock
);
1598 #ifndef NO_LEGACY_INITFINI
1599 if ((dyn
[0] & (1<<DT_INIT
)) && dyn
[DT_INIT
])
1600 fpaddr(p
, dyn
[DT_INIT
])();
1602 if (dyn
[0] & (1<<DT_INIT_ARRAY
)) {
1603 size_t n
= dyn
[DT_INIT_ARRAYSZ
]/sizeof(size_t);
1604 size_t *fn
= laddr(p
, dyn
[DT_INIT_ARRAY
]);
1605 while (n
--) ((void (*)(void))*fn
++)();
1608 pthread_mutex_lock(&init_fini_lock
);
1609 p
->ctor_visitor
= 0;
1611 pthread_cond_broadcast(&ctor_cond
);
1613 pthread_mutex_unlock(&init_fini_lock
);
1616 void __libc_start_init(void)
1618 do_init_fini(main_ctor_queue
);
1619 if (!__malloc_replaced
&& main_ctor_queue
!= builtin_ctor_queue
)
1620 free(main_ctor_queue
);
1621 main_ctor_queue
= 0;
1624 static void dl_debug_state(void)
1628 weak_alias(dl_debug_state
, _dl_debug_state
);
1630 void __init_tls(size_t *auxv
)
1634 static void update_tls_size()
1636 libc
.tls_cnt
= tls_cnt
;
1637 libc
.tls_align
= tls_align
;
1638 libc
.tls_size
= ALIGN(
1639 (1+tls_cnt
) * sizeof(void *) +
1641 sizeof(struct pthread
) +
1646 static void install_new_tls(void)
1649 pthread_t self
= __pthread_self(), td
;
1650 struct dso
*dtv_provider
= container_of(tls_tail
, struct dso
, tls
);
1651 uintptr_t (*newdtv
)[tls_cnt
+1] = (void *)dtv_provider
->new_dtv
;
1654 size_t old_cnt
= self
->dtv
[0];
1656 __block_app_sigs(&set
);
1658 /* Copy existing dtv contents from all existing threads. */
1659 for (i
=0, td
=self
; !i
|| td
!=self
; i
++, td
=td
->next
) {
1660 memcpy(newdtv
+i
, td
->dtv
,
1661 (old_cnt
+1)*sizeof(uintptr_t));
1662 newdtv
[i
][0] = tls_cnt
;
1664 /* Install new dtls into the enlarged, uninstalled dtv copies. */
1665 for (p
=head
; ; p
=p
->next
) {
1666 if (p
->tls_id
<= old_cnt
) continue;
1667 unsigned char *mem
= p
->new_tls
;
1668 for (j
=0; j
<i
; j
++) {
1669 unsigned char *new = mem
;
1670 new += ((uintptr_t)p
->tls
.image
- (uintptr_t)mem
)
1672 memcpy(new, p
->tls
.image
, p
->tls
.len
);
1673 newdtv
[j
][p
->tls_id
] =
1674 (uintptr_t)new + DTP_OFFSET
;
1675 mem
+= p
->tls
.size
+ p
->tls
.align
;
1677 if (p
->tls_id
== tls_cnt
) break;
1680 /* Broadcast barrier to ensure contents of new dtv is visible
1681 * if the new dtv pointer is. The __membarrier function has a
1682 * fallback emulation using signals for kernels that lack the
1683 * feature at the syscall level. */
1685 __membarrier(MEMBARRIER_CMD_PRIVATE_EXPEDITED
, 0);
1687 /* Install new dtv for each thread. */
1688 for (j
=0, td
=self
; !j
|| td
!=self
; j
++, td
=td
->next
) {
1689 td
->dtv
= newdtv
[j
];
1693 __restore_sigs(&set
);
1696 /* Stage 1 of the dynamic linker is defined in dlstart.c. It calls the
1697 * following stage 2 and stage 3 functions via primitive symbolic lookup
1698 * since it does not have access to their addresses to begin with. */
1700 /* Stage 2 of the dynamic linker is called after relative relocations
1701 * have been processed. It can make function calls to static functions
1702 * and access string literals and static data, but cannot use extern
1703 * symbols. Its job is to perform symbolic relocations on the dynamic
1704 * linker itself, but some of the relocations performed may need to be
1705 * replaced later due to copy relocations in the main program. */
1707 hidden
void __dls2(unsigned char *base
, size_t *sp
)
1710 for (auxv
=sp
+1+*sp
+1; *auxv
; auxv
++);
1713 void *p1
= (void *)sp
[-2];
1714 void *p2
= (void *)sp
[-1];
1716 size_t aux
[AUX_CNT
];
1717 decode_vec(auxv
, aux
, AUX_CNT
);
1718 if (aux
[AT_BASE
]) ldso
.base
= (void *)aux
[AT_BASE
];
1719 else ldso
.base
= (void *)(aux
[AT_PHDR
] & -4096);
1721 app_loadmap
= p2
? p1
: 0;
1722 ldso
.loadmap
= p2
? p2
: p1
;
1723 ldso
.base
= laddr(&ldso
, 0);
1727 Ehdr
*ehdr
= __ehdr_start
? (void *)__ehdr_start
: (void *)ldso
.base
;
1728 ldso
.name
= ldso
.shortname
= "libc.so";
1729 ldso
.phnum
= ehdr
->e_phnum
;
1730 ldso
.phdr
= laddr(&ldso
, ehdr
->e_phoff
);
1731 ldso
.phentsize
= ehdr
->e_phentsize
;
1732 search_vec(auxv
, &ldso_page_size
, AT_PAGESZ
);
1733 kernel_mapped_dso(&ldso
);
1736 if (DL_FDPIC
) makefuncdescs(&ldso
);
1738 /* Prepare storage for to save clobbered REL addends so they
1739 * can be reused in stage 3. There should be very few. If
1740 * something goes wrong and there are a huge number, abort
1741 * instead of risking stack overflow. */
1742 size_t dyn
[DYN_CNT
];
1743 decode_vec(ldso
.dynv
, dyn
, DYN_CNT
);
1744 size_t *rel
= laddr(&ldso
, dyn
[DT_REL
]);
1745 size_t rel_size
= dyn
[DT_RELSZ
];
1746 size_t symbolic_rel_cnt
= 0;
1747 apply_addends_to
= rel
;
1748 for (; rel_size
; rel
+=2, rel_size
-=2*sizeof(size_t))
1749 if (!IS_RELATIVE(rel
[1], ldso
.syms
)) symbolic_rel_cnt
++;
1750 if (symbolic_rel_cnt
>= ADDEND_LIMIT
) a_crash();
1751 size_t addends
[symbolic_rel_cnt
+1];
1752 saved_addends
= addends
;
1759 /* Call dynamic linker stage-2b, __dls2b, looking it up
1760 * symbolically as a barrier against moving the address
1761 * load across the above relocation processing. */
1762 struct symdef dls2b_def
= find_sym(&ldso
, "__dls2b", 0);
1763 if (DL_FDPIC
) ((stage3_func
)&ldso
.funcdescs
[dls2b_def
.sym
-ldso
.syms
])(sp
, auxv
);
1764 else ((stage3_func
)laddr(&ldso
, dls2b_def
.sym
->st_value
))(sp
, auxv
);
1767 /* Stage 2b sets up a valid thread pointer, which requires relocations
1768 * completed in stage 2, and on which stage 3 is permitted to depend.
1769 * This is done as a separate stage, with symbolic lookup as a barrier,
1770 * so that loads of the thread pointer and &errno can be pure/const and
1771 * thereby hoistable. */
1773 void __dls2b(size_t *sp
, size_t *auxv
)
1775 /* Setup early thread pointer in builtin_tls for ldso/libc itself to
1776 * use during dynamic linking. If possible it will also serve as the
1777 * thread pointer at runtime. */
1778 search_vec(auxv
, &__hwcap
, AT_HWCAP
);
1780 libc
.tls_size
= sizeof builtin_tls
;
1781 libc
.tls_align
= tls_align
;
1782 if (__init_tp(__copy_tls((void *)builtin_tls
)) < 0) {
1786 struct symdef dls3_def
= find_sym(&ldso
, "__dls3", 0);
1787 if (DL_FDPIC
) ((stage3_func
)&ldso
.funcdescs
[dls3_def
.sym
-ldso
.syms
])(sp
, auxv
);
1788 else ((stage3_func
)laddr(&ldso
, dls3_def
.sym
->st_value
))(sp
, auxv
);
1791 /* Stage 3 of the dynamic linker is called with the dynamic linker/libc
1792 * fully functional. Its job is to load (if not already loaded) and
1793 * process dependencies and relocations for the main application and
1794 * transfer control to its entry point. */
1796 void __dls3(size_t *sp
, size_t *auxv
)
1798 static struct dso app
, vdso
;
1799 size_t aux
[AUX_CNT
];
1801 char *env_preload
=0;
1802 char *replace_argv0
=0;
1805 char **argv
= (void *)(sp
+1);
1806 char **argv_orig
= argv
;
1807 char **envp
= argv
+argc
+1;
1809 /* Find aux vector just past environ[] and use it to initialize
1810 * global data that may be needed before we can make syscalls. */
1812 decode_vec(auxv
, aux
, AUX_CNT
);
1813 search_vec(auxv
, &__sysinfo
, AT_SYSINFO
);
1814 __pthread_self()->sysinfo
= __sysinfo
;
1815 libc
.page_size
= aux
[AT_PAGESZ
];
1816 libc
.secure
= ((aux
[0]&0x7800)!=0x7800 || aux
[AT_UID
]!=aux
[AT_EUID
]
1817 || aux
[AT_GID
]!=aux
[AT_EGID
] || aux
[AT_SECURE
]);
1819 /* Only trust user/env if kernel says we're not suid/sgid */
1821 env_path
= getenv("LD_LIBRARY_PATH");
1822 env_preload
= getenv("LD_PRELOAD");
1825 /* Activate error handler function */
1828 /* If the main program was already loaded by the kernel,
1829 * AT_PHDR will point to some location other than the dynamic
1830 * linker's program headers. */
1831 if (aux
[AT_PHDR
] != (size_t)ldso
.phdr
) {
1832 size_t interp_off
= 0;
1833 size_t tls_image
= 0;
1834 /* Find load address of the main program, via AT_PHDR vs PT_PHDR. */
1835 Phdr
*phdr
= app
.phdr
= (void *)aux
[AT_PHDR
];
1836 app
.phnum
= aux
[AT_PHNUM
];
1837 app
.phentsize
= aux
[AT_PHENT
];
1838 for (i
=aux
[AT_PHNUM
]; i
; i
--, phdr
=(void *)((char *)phdr
+ aux
[AT_PHENT
])) {
1839 if (phdr
->p_type
== PT_PHDR
)
1840 app
.base
= (void *)(aux
[AT_PHDR
] - phdr
->p_vaddr
);
1841 else if (phdr
->p_type
== PT_INTERP
)
1842 interp_off
= (size_t)phdr
->p_vaddr
;
1843 else if (phdr
->p_type
== PT_TLS
) {
1844 tls_image
= phdr
->p_vaddr
;
1845 app
.tls
.len
= phdr
->p_filesz
;
1846 app
.tls
.size
= phdr
->p_memsz
;
1847 app
.tls
.align
= phdr
->p_align
;
1850 if (DL_FDPIC
) app
.loadmap
= app_loadmap
;
1851 if (app
.tls
.size
) app
.tls
.image
= laddr(&app
, tls_image
);
1852 if (interp_off
) ldso
.name
= laddr(&app
, interp_off
);
1853 if ((aux
[0] & (1UL<<AT_EXECFN
))
1854 && strncmp((char *)aux
[AT_EXECFN
], "/proc/", 6))
1855 app
.name
= (char *)aux
[AT_EXECFN
];
1858 kernel_mapped_dso(&app
);
1861 char *ldname
= argv
[0];
1862 size_t l
= strlen(ldname
);
1863 if (l
>= 3 && !strcmp(ldname
+l
-3, "ldd")) ldd_mode
= 1;
1865 while (argv
[0] && argv
[0][0]=='-' && argv
[0][1]=='-') {
1866 char *opt
= argv
[0]+2;
1867 *argv
++ = (void *)-1;
1870 } else if (!memcmp(opt
, "list", 5)) {
1872 } else if (!memcmp(opt
, "library-path", 12)) {
1873 if (opt
[12]=='=') env_path
= opt
+13;
1874 else if (opt
[12]) *argv
= 0;
1875 else if (*argv
) env_path
= *argv
++;
1876 } else if (!memcmp(opt
, "preload", 7)) {
1877 if (opt
[7]=='=') env_preload
= opt
+8;
1878 else if (opt
[7]) *argv
= 0;
1879 else if (*argv
) env_preload
= *argv
++;
1880 } else if (!memcmp(opt
, "argv0", 5)) {
1881 if (opt
[5]=='=') replace_argv0
= opt
+6;
1882 else if (opt
[5]) *argv
= 0;
1883 else if (*argv
) replace_argv0
= *argv
++;
1888 argv
[-1] = (void *)(argc
- (argv
-argv_orig
));
1890 dprintf(2, "musl libc (" LDSO_ARCH
")\n"
1892 "Dynamic Program Loader\n"
1893 "Usage: %s [options] [--] pathname%s\n",
1894 __libc_version
, ldname
,
1895 ldd_mode
? "" : " [args]");
1898 fd
= open(argv
[0], O_RDONLY
);
1900 dprintf(2, "%s: cannot load %s: %s\n", ldname
, argv
[0], strerror(errno
));
1903 Ehdr
*ehdr
= map_library(fd
, &app
);
1905 dprintf(2, "%s: %s: Not a valid dynamic program\n", ldname
, argv
[0]);
1911 aux
[AT_ENTRY
] = (size_t)laddr(&app
, ehdr
->e_entry
);
1912 /* Find the name that would have been used for the dynamic
1913 * linker had ldd not taken its place. */
1915 for (i
=0; i
<app
.phnum
; i
++) {
1916 if (app
.phdr
[i
].p_type
== PT_INTERP
)
1917 ldso
.name
= laddr(&app
, app
.phdr
[i
].p_vaddr
);
1919 dprintf(1, "\t%s (%p)\n", ldso
.name
, ldso
.base
);
1923 libc
.tls_head
= tls_tail
= &app
.tls
;
1924 app
.tls_id
= tls_cnt
= 1;
1926 app
.tls
.offset
= GAP_ABOVE_TP
;
1927 app
.tls
.offset
+= (-GAP_ABOVE_TP
+ (uintptr_t)app
.tls
.image
)
1928 & (app
.tls
.align
-1);
1929 tls_offset
= app
.tls
.offset
+ app
.tls
.size
;
1931 tls_offset
= app
.tls
.offset
= app
.tls
.size
1932 + ( -((uintptr_t)app
.tls
.image
+ app
.tls
.size
)
1933 & (app
.tls
.align
-1) );
1935 tls_align
= MAXP2(tls_align
, app
.tls
.align
);
1939 makefuncdescs(&app
);
1941 app
.loadmap
= (void *)&app_dummy_loadmap
;
1942 app
.loadmap
->nsegs
= 1;
1943 app
.loadmap
->segs
[0].addr
= (size_t)app
.map
;
1944 app
.loadmap
->segs
[0].p_vaddr
= (size_t)app
.map
1946 app
.loadmap
->segs
[0].p_memsz
= app
.map_len
;
1948 argv
[-3] = (void *)app
.loadmap
;
1951 /* Initial dso chain consists only of the app. */
1952 head
= tail
= syms_tail
= &app
;
1954 /* Donate unused parts of app and library mapping to malloc */
1956 reclaim_gaps(&ldso
);
1958 /* Load preload/needed libraries, add symbols to global namespace. */
1959 ldso
.deps
= (struct dso
**)no_deps
;
1960 if (env_preload
) load_preload(env_preload
);
1962 for (struct dso
*p
=head
; p
; p
=p
->next
)
1965 /* Attach to vdso, if provided by the kernel, last so that it does
1966 * not become part of the global namespace. */
1967 if (search_vec(auxv
, &vdso_base
, AT_SYSINFO_EHDR
) && vdso_base
) {
1968 Ehdr
*ehdr
= (void *)vdso_base
;
1969 Phdr
*phdr
= vdso
.phdr
= (void *)(vdso_base
+ ehdr
->e_phoff
);
1970 vdso
.phnum
= ehdr
->e_phnum
;
1971 vdso
.phentsize
= ehdr
->e_phentsize
;
1972 for (i
=ehdr
->e_phnum
; i
; i
--, phdr
=(void *)((char *)phdr
+ ehdr
->e_phentsize
)) {
1973 if (phdr
->p_type
== PT_DYNAMIC
)
1974 vdso
.dynv
= (void *)(vdso_base
+ phdr
->p_offset
);
1975 if (phdr
->p_type
== PT_LOAD
)
1976 vdso
.base
= (void *)(vdso_base
- phdr
->p_vaddr
+ phdr
->p_offset
);
1979 vdso
.shortname
= "linux-gate.so.1";
1981 vdso
.deps
= (struct dso
**)no_deps
;
1988 for (i
=0; app
.dynv
[i
]; i
+=2) {
1989 if (!DT_DEBUG_INDIRECT
&& app
.dynv
[i
]==DT_DEBUG
)
1990 app
.dynv
[i
+1] = (size_t)&debug
;
1991 if (DT_DEBUG_INDIRECT
&& app
.dynv
[i
]==DT_DEBUG_INDIRECT
) {
1992 size_t *ptr
= (size_t *) app
.dynv
[i
+1];
1993 *ptr
= (size_t)&debug
;
1995 if (app
.dynv
[i
]==DT_DEBUG_INDIRECT_REL
) {
1996 size_t *ptr
= (size_t *)((size_t)&app
.dynv
[i
] + app
.dynv
[i
+1]);
1997 *ptr
= (size_t)&debug
;
2001 /* This must be done before final relocations, since it calls
2002 * malloc, which may be provided by the application. Calling any
2003 * application code prior to the jump to its entry point is not
2004 * valid in our model and does not work with FDPIC, where there
2005 * are additional relocation-like fixups that only the entry point
2006 * code can see to perform. */
2007 main_ctor_queue
= queue_ctors(&app
);
2009 /* Initial TLS must also be allocated before final relocations
2010 * might result in calloc being a call to application code. */
2012 void *initial_tls
= builtin_tls
;
2013 if (libc
.tls_size
> sizeof builtin_tls
|| tls_align
> MIN_TLS_ALIGN
) {
2014 initial_tls
= calloc(libc
.tls_size
, 1);
2016 dprintf(2, "%s: Error getting %zu bytes thread-local storage: %m\n",
2017 argv
[0], libc
.tls_size
);
2021 static_tls_cnt
= tls_cnt
;
2023 /* The main program must be relocated LAST since it may contain
2024 * copy relocations which depend on libraries' relocations. */
2025 reloc_all(app
.next
);
2028 /* Actual copying to new TLS needs to happen after relocations,
2029 * since the TLS images might have contained relocated addresses. */
2030 if (initial_tls
!= builtin_tls
) {
2031 if (__init_tp(__copy_tls(initial_tls
)) < 0) {
2035 size_t tmp_tls_size
= libc
.tls_size
;
2036 pthread_t self
= __pthread_self();
2037 /* Temporarily set the tls size to the full size of
2038 * builtin_tls so that __copy_tls will use the same layout
2039 * as it did for before. Then check, just to be safe. */
2040 libc
.tls_size
= sizeof builtin_tls
;
2041 if (__copy_tls((void*)builtin_tls
) != self
) a_crash();
2042 libc
.tls_size
= tmp_tls_size
;
2045 if (ldso_fail
) _exit(127);
2046 if (ldd_mode
) _exit(0);
2048 /* Determine if malloc was interposed by a replacement implementation
2049 * so that calloc and the memalign family can harden against the
2050 * possibility of incomplete replacement. */
2051 if (find_sym(head
, "malloc", 1).dso
!= &ldso
)
2052 __malloc_replaced
= 1;
2053 if (find_sym(head
, "aligned_alloc", 1).dso
!= &ldso
)
2054 __aligned_alloc_replaced
= 1;
2056 /* Switch to runtime mode: any further failures in the dynamic
2057 * linker are a reportable failure rather than a fatal startup
2062 debug
.bp
= dl_debug_state
;
2064 debug
.base
= ldso
.base
;
2065 debug
.state
= RT_CONSISTENT
;
2068 if (replace_argv0
) argv
[0] = replace_argv0
;
2072 CRTJMP((void *)aux
[AT_ENTRY
], argv
-1);
2076 static void prepare_lazy(struct dso
*p
)
2078 size_t dyn
[DYN_CNT
], n
, flags1
=0;
2079 decode_vec(p
->dynv
, dyn
, DYN_CNT
);
2080 search_vec(p
->dynv
, &flags1
, DT_FLAGS_1
);
2081 if (dyn
[DT_BIND_NOW
] || (dyn
[DT_FLAGS
] & DF_BIND_NOW
) || (flags1
& DF_1_NOW
))
2083 n
= dyn
[DT_RELSZ
]/2 + dyn
[DT_RELASZ
]/3 + dyn
[DT_PLTRELSZ
]/2 + 1;
2084 if (NEED_MIPS_GOT_RELOCS
) {
2085 size_t j
=0; search_vec(p
->dynv
, &j
, DT_MIPS_GOTSYM
);
2086 size_t i
=0; search_vec(p
->dynv
, &i
, DT_MIPS_SYMTABNO
);
2089 p
->lazy
= calloc(n
, 3*sizeof(size_t));
2091 error("Error preparing lazy relocation for %s: %m", p
->name
);
2092 longjmp(*rtld_fail
, 1);
2094 p
->lazy_next
= lazy_head
;
2098 void *dlopen(const char *file
, int mode
)
2100 struct dso
*volatile p
, *orig_tail
, *orig_syms_tail
, *orig_lazy_head
, *next
;
2101 struct tls_module
*orig_tls_tail
;
2102 size_t orig_tls_cnt
, orig_tls_offset
, orig_tls_align
;
2106 struct dso
**volatile ctor_queue
= 0;
2108 if (!file
) return head
;
2110 pthread_setcancelstate(PTHREAD_CANCEL_DISABLE
, &cs
);
2111 pthread_rwlock_wrlock(&lock
);
2114 debug
.state
= RT_ADD
;
2118 if (shutting_down
) {
2119 error("Cannot dlopen while program is exiting.");
2122 orig_tls_tail
= tls_tail
;
2123 orig_tls_cnt
= tls_cnt
;
2124 orig_tls_offset
= tls_offset
;
2125 orig_tls_align
= tls_align
;
2126 orig_lazy_head
= lazy_head
;
2127 orig_syms_tail
= syms_tail
;
2129 noload
= mode
& RTLD_NOLOAD
;
2132 if (setjmp(*rtld_fail
)) {
2133 /* Clean up anything new that was (partially) loaded */
2134 revert_syms(orig_syms_tail
);
2135 for (p
=orig_tail
->next
; p
; p
=next
) {
2137 while (p
->td_index
) {
2138 void *tmp
= p
->td_index
->next
;
2143 if (p
->rpath
!= p
->rpath_orig
)
2151 if (!orig_tls_tail
) libc
.tls_head
= 0;
2152 tls_tail
= orig_tls_tail
;
2153 if (tls_tail
) tls_tail
->next
= 0;
2154 tls_cnt
= orig_tls_cnt
;
2155 tls_offset
= orig_tls_offset
;
2156 tls_align
= orig_tls_align
;
2157 lazy_head
= orig_lazy_head
;
2162 } else p
= load_library(file
, head
);
2166 "Library %s is not already loaded" :
2167 "Error loading shared library %s: %m",
2172 /* First load handling */
2175 pthread_mutex_lock(&init_fini_lock
);
2176 int constructed
= p
->constructed
;
2177 pthread_mutex_unlock(&init_fini_lock
);
2178 if (!constructed
) ctor_queue
= queue_ctors(p
);
2179 if (!p
->relocated
&& (mode
& RTLD_LAZY
)) {
2181 for (i
=0; p
->deps
[i
]; i
++)
2182 if (!p
->deps
[i
]->relocated
)
2183 prepare_lazy(p
->deps
[i
]);
2185 if (!p
->relocated
|| (mode
& RTLD_GLOBAL
)) {
2186 /* Make new symbols global, at least temporarily, so we can do
2187 * relocations. If not RTLD_GLOBAL, this is reverted below. */
2189 for (i
=0; p
->deps
[i
]; i
++)
2190 add_syms(p
->deps
[i
]);
2192 if (!p
->relocated
) {
2196 /* If RTLD_GLOBAL was not specified, undo any new additions
2197 * to the global symbol table. This is a nop if the library was
2198 * previously loaded and already global. */
2199 if (!(mode
& RTLD_GLOBAL
))
2200 revert_syms(orig_syms_tail
);
2202 /* Processing of deferred lazy relocations must not happen until
2203 * the new libraries are committed; otherwise we could end up with
2204 * relocations resolved to symbol definitions that get removed. */
2208 if (tls_cnt
!= orig_tls_cnt
)
2212 debug
.state
= RT_CONSISTENT
;
2216 pthread_rwlock_unlock(&lock
);
2218 do_init_fini(ctor_queue
);
2221 pthread_setcancelstate(cs
, 0);
2225 hidden
int __dl_invalid_handle(void *h
)
2228 for (p
=head
; p
; p
=p
->next
) if (h
==p
) return 0;
2229 error("Invalid library handle %p", (void *)h
);
2233 static void *addr2dso(size_t a
)
2237 if (DL_FDPIC
) for (p
=head
; p
; p
=p
->next
) {
2239 if (a
-(size_t)p
->funcdescs
< i
*sizeof(*p
->funcdescs
))
2242 for (p
=head
; p
; p
=p
->next
) {
2243 if (DL_FDPIC
&& p
->loadmap
) {
2244 for (i
=0; i
<p
->loadmap
->nsegs
; i
++) {
2245 if (a
-p
->loadmap
->segs
[i
].p_vaddr
2246 < p
->loadmap
->segs
[i
].p_memsz
)
2251 size_t phcnt
= p
->phnum
;
2252 size_t entsz
= p
->phentsize
;
2253 size_t base
= (size_t)p
->base
;
2254 for (; phcnt
--; ph
=(void *)((char *)ph
+entsz
)) {
2255 if (ph
->p_type
!= PT_LOAD
) continue;
2256 if (a
-base
-ph
->p_vaddr
< ph
->p_memsz
)
2259 if (a
-(size_t)p
->map
< p
->map_len
)
2266 static void *do_dlsym(struct dso
*p
, const char *s
, void *ra
)
2269 if (p
== head
|| p
== RTLD_DEFAULT
) {
2271 } else if (p
== RTLD_NEXT
) {
2272 p
= addr2dso((size_t)ra
);
2275 } else if (__dl_invalid_handle(p
)) {
2279 struct symdef def
= find_sym2(p
, s
, 0, use_deps
);
2281 error("Symbol not found: %s", s
);
2284 if ((def
.sym
->st_info
&0xf) == STT_TLS
)
2285 return __tls_get_addr((tls_mod_off_t
[]){def
.dso
->tls_id
, def
.sym
->st_value
-DTP_OFFSET
});
2286 if (DL_FDPIC
&& (def
.sym
->st_info
&0xf) == STT_FUNC
)
2287 return def
.dso
->funcdescs
+ (def
.sym
- def
.dso
->syms
);
2288 return laddr(def
.dso
, def
.sym
->st_value
);
2291 int dladdr(const void *addr_arg
, Dl_info
*info
)
2293 size_t addr
= (size_t)addr_arg
;
2299 size_t besterr
= -1;
2301 pthread_rwlock_rdlock(&lock
);
2303 pthread_rwlock_unlock(&lock
);
2308 strings
= p
->strings
;
2309 nsym
= count_syms(p
);
2312 size_t idx
= (addr
-(size_t)p
->funcdescs
)
2313 / sizeof(*p
->funcdescs
);
2314 if (idx
< nsym
&& (sym
[idx
].st_info
&0xf) == STT_FUNC
) {
2315 best
= (size_t)(p
->funcdescs
+ idx
);
2316 bestsym
= sym
+ idx
;
2321 if (!best
) for (; nsym
; nsym
--, sym
++) {
2323 && (1<<(sym
->st_info
&0xf) & OK_TYPES
)
2324 && (1<<(sym
->st_info
>>4) & OK_BINDS
)) {
2325 size_t symaddr
= (size_t)laddr(p
, sym
->st_value
);
2326 if (symaddr
> addr
|| symaddr
<= best
)
2330 besterr
= addr
- symaddr
;
2331 if (addr
== symaddr
)
2336 if (best
&& besterr
> bestsym
->st_size
-1) {
2341 info
->dli_fname
= p
->name
;
2342 info
->dli_fbase
= p
->map
;
2345 info
->dli_sname
= 0;
2346 info
->dli_saddr
= 0;
2350 if (DL_FDPIC
&& (bestsym
->st_info
&0xf) == STT_FUNC
)
2351 best
= (size_t)(p
->funcdescs
+ (bestsym
- p
->syms
));
2352 info
->dli_sname
= strings
+ bestsym
->st_name
;
2353 info
->dli_saddr
= (void *)best
;
2358 hidden
void *__dlsym(void *restrict p
, const char *restrict s
, void *restrict ra
)
2361 pthread_rwlock_rdlock(&lock
);
2362 res
= do_dlsym(p
, s
, ra
);
2363 pthread_rwlock_unlock(&lock
);
2367 hidden
void *__dlsym_redir_time64(void *restrict p
, const char *restrict s
, void *restrict ra
)
2370 const char *suffix
, *suffix2
= "";
2373 /* Map the symbol name to a time64 version of itself according to the
2374 * pattern used for naming the redirected time64 symbols. */
2375 size_t l
= strnlen(s
, sizeof redir
);
2376 if (l
<4 || l
==sizeof redir
) goto no_redir
;
2377 if (s
[l
-2]=='_' && s
[l
-1]=='r') {
2381 if (l
<4) goto no_redir
;
2382 if (!strcmp(s
+l
-4, "time")) suffix
= "64";
2383 else suffix
= "_time64";
2385 /* Use the presence of the remapped symbol name in libc to determine
2386 * whether it's one that requires time64 redirection; replace if so. */
2387 snprintf(redir
, sizeof redir
, "__%.*s%s%s", (int)l
, s
, suffix
, suffix2
);
2388 if (find_sym(&ldso
, redir
, 1).sym
) s
= redir
;
2391 return __dlsym(p
, s
, ra
);
2394 int dl_iterate_phdr(int(*callback
)(struct dl_phdr_info
*info
, size_t size
, void *data
), void *data
)
2396 struct dso
*current
;
2397 struct dl_phdr_info info
;
2399 for(current
= head
; current
;) {
2400 info
.dlpi_addr
= (uintptr_t)current
->base
;
2401 info
.dlpi_name
= current
->name
;
2402 info
.dlpi_phdr
= current
->phdr
;
2403 info
.dlpi_phnum
= current
->phnum
;
2404 info
.dlpi_adds
= gencnt
;
2406 info
.dlpi_tls_modid
= current
->tls_id
;
2407 info
.dlpi_tls_data
= !current
->tls_id
? 0 :
2408 __tls_get_addr((tls_mod_off_t
[]){current
->tls_id
,0});
2410 ret
= (callback
)(&info
, sizeof (info
), data
);
2412 if (ret
!= 0) break;
2414 pthread_rwlock_rdlock(&lock
);
2415 current
= current
->next
;
2416 pthread_rwlock_unlock(&lock
);
2421 static void error_impl(const char *fmt
, ...)
2426 vdprintf(2, fmt
, ap
);
2432 __dl_vseterr(fmt
, ap
);
2436 static void error_noop(const char *fmt
, ...)