1 /* Machine-dependent ELF dynamic relocation inline functions. x86-64 version.
2 Copyright (C) 2001-2024 Free Software Foundation, Inc.
3 This file is part of the GNU C Library.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
22 #define ELF_MACHINE_NAME "x86_64"
26 #include <sys/param.h>
29 #include <dl-tlsdesc.h>
30 #include <dl-static-tls.h>
31 #include <dl-machine-rel.h>
32 #include <isa-level.h>
36 # define RTLD_START_ENABLE_X86_FEATURES
39 /* Translate a processor specific dynamic tag to the index in l_info array. */
40 #define DT_X86_64(x) (DT_X86_64_##x - DT_LOPROC + DT_NUM)
42 /* Return nonzero iff ELF header is compatible with the running host. */
43 static inline int __attribute__ ((unused
))
44 elf_machine_matches_host (const ElfW(Ehdr
) *ehdr
)
46 return ehdr
->e_machine
== EM_X86_64
;
50 /* Return the run-time load address of the shared object. */
51 static inline ElfW(Addr
) __attribute__ ((unused
))
52 elf_machine_load_address (void)
54 extern const ElfW(Ehdr
) __ehdr_start attribute_hidden
;
55 return (ElfW(Addr
)) &__ehdr_start
;
58 /* Return the link-time address of _DYNAMIC. */
59 static inline ElfW(Addr
) __attribute__ ((unused
))
60 elf_machine_dynamic (void)
62 extern ElfW(Dyn
) _DYNAMIC
[] attribute_hidden
;
63 return (ElfW(Addr
)) _DYNAMIC
- elf_machine_load_address ();
66 /* Set up the loaded object described by L so its unrelocated PLT
67 entries will jump to the on-demand fixup code in dl-runtime.c. */
69 static inline int __attribute__ ((unused
, always_inline
))
70 elf_machine_runtime_setup (struct link_map
*l
, struct r_scope_elem
*scope
[],
71 int lazy
, int profile
)
74 extern void _dl_runtime_profile_sse (ElfW(Word
)) attribute_hidden
;
75 extern void _dl_runtime_profile_avx (ElfW(Word
)) attribute_hidden
;
76 extern void _dl_runtime_profile_avx512 (ElfW(Word
)) attribute_hidden
;
78 if (l
->l_info
[DT_JMPREL
] && lazy
)
80 /* The GOT entries for functions in the PLT have not yet been filled
81 in. Their initial contents will arrange when called to push an
82 offset into the .rel.plt section, push _GLOBAL_OFFSET_TABLE_[1],
83 and then jump to _GLOBAL_OFFSET_TABLE_[2]. */
84 got
= (Elf64_Addr
*) D_PTR (l
, l_info
[DT_PLTGOT
]);
85 /* If a library is prelinked but we have to relocate anyway,
86 we have to be able to undo the prelinking of .got.plt.
87 The prelinker saved us here address of .plt + 0x16. */
90 l
->l_mach
.plt
= got
[1] + l
->l_addr
;
91 l
->l_mach
.gotplt
= (ElfW(Addr
)) &got
[3];
93 /* Identify this shared object. */
94 *(ElfW(Addr
) *) (got
+ 1) = (ElfW(Addr
)) l
;
97 /* The got[2] entry contains the address of a function which gets
98 called to get the address of a so far unresolved function and
99 jump to it. The profiling extension of the dynamic linker allows
100 to intercept the calls to collect information. In this case we
101 don't store the address in the GOT so that all future calls also
102 end in this function. */
103 if (__glibc_unlikely (profile
))
105 const struct cpu_features
* cpu_features
= __get_cpu_features ();
106 if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features
, AVX512F
))
107 *(ElfW(Addr
) *) (got
+ 2) = (ElfW(Addr
)) &_dl_runtime_profile_avx512
;
108 else if (X86_ISA_CPU_FEATURE_USABLE_P (cpu_features
, AVX
))
109 *(ElfW(Addr
) *) (got
+ 2) = (ElfW(Addr
)) &_dl_runtime_profile_avx
;
111 *(ElfW(Addr
) *) (got
+ 2) = (ElfW(Addr
)) &_dl_runtime_profile_sse
;
113 if (GLRO(dl_profile
) != NULL
114 && _dl_name_match_p (GLRO(dl_profile
), l
))
115 /* This is the object we are looking for. Say that we really
116 want profiling and the timers are started. */
117 GL(dl_profile_map
) = l
;
122 /* This function will get called to fix up the GOT entry
123 indicated by the offset on the stack, and then jump to
124 the resolved address. */
125 *(ElfW(Addr
) *) (got
+ 2)
126 = (ElfW(Addr
)) GLRO(dl_x86_64_runtime_resolve
);
133 /* Initial entry point code for the dynamic linker.
134 The C function `_dl_start' is the real entry point;
135 its return value is the user program's entry point. */
136 #define RTLD_START asm ("\n\
140 .globl _dl_start_user\n\
142 mov %" RSP_LP ", %" RDI_LP "\n\
145 # Save the user entry point address in %r12.\n\
146 mov %" RAX_LP ", %" R12_LP "\n\
147 # Save %rsp value in %r13.\n\
148 mov %" RSP_LP ", % " R13_LP "\n\
150 RTLD_START_ENABLE_X86_FEATURES \
152 # Read the original argument count.\n\
153 mov (%rsp), %" RDX_LP "\n\
154 # Call _dl_init (struct link_map *main_map, int argc, char **argv, char **env)\n\
156 mov %" RDX_LP ", %" RSI_LP "\n\
157 # And align stack for the _dl_init call. \n\
158 and $-16, %" RSP_LP "\n\
159 # _dl_loaded -> rdi\n\
160 mov _rtld_local(%rip), %" RDI_LP "\n\
162 lea 2*" LP_SIZE "(%r13,%rdx," LP_SIZE "), %" RCX_LP "\n\
164 lea " LP_SIZE "(%r13), %" RDX_LP "\n\
165 # Clear %rbp to mark outermost frame obviously even for constructors.\n\
167 # Call the function to run the initializers.\n\
169 # Pass our finalizer function to the user in %rdx, as per ELF ABI.\n\
170 lea _dl_fini(%rip), %" RDX_LP "\n\
171 # And make sure %rsp points to argc stored on the stack.\n\
172 mov %" R13_LP ", %" RSP_LP "\n\
173 # Jump to the user's entry point.\n\
178 /* ELF_RTYPE_CLASS_PLT iff TYPE describes relocation of a PLT entry or
179 TLS variable, so undefined references should not be allowed to
181 ELF_RTYPE_CLASS_COPY iff TYPE should not be allowed to resolve to one
182 of the main executable's symbols, as for a COPY reloc. */
183 #define elf_machine_type_class(type) \
184 ((((type) == R_X86_64_JUMP_SLOT \
185 || (type) == R_X86_64_DTPMOD64 \
186 || (type) == R_X86_64_DTPOFF64 \
187 || (type) == R_X86_64_TPOFF64 \
188 || (type) == R_X86_64_TLSDESC) \
189 * ELF_RTYPE_CLASS_PLT) \
190 | (((type) == R_X86_64_COPY) * ELF_RTYPE_CLASS_COPY))
192 /* A reloc type used for ld.so cmdline arg lookups to reject PLT entries. */
193 #define ELF_MACHINE_JMP_SLOT R_X86_64_JUMP_SLOT
195 /* The relative ifunc relocation. */
196 // XXX This is a work-around for a broken linker. Remove!
197 #define ELF_MACHINE_IRELATIVE R_X86_64_IRELATIVE
199 /* We define an initialization function. This is called very early in
201 #define DL_PLATFORM_INIT dl_platform_init ()
203 static inline void __attribute__ ((unused
))
204 dl_platform_init (void)
207 /* _dl_x86_init_cpu_features is a wrapper for init_cpu_features which
208 has been called early from __libc_start_main in static executable. */
209 _dl_x86_init_cpu_features ();
211 if (GLRO(dl_platform
) != NULL
&& *GLRO(dl_platform
) == '\0')
212 /* Avoid an empty string which would disturb us. */
213 GLRO(dl_platform
) = NULL
;
217 static inline ElfW(Addr
)
218 elf_machine_fixup_plt (struct link_map
*map
, lookup_t t
,
219 const ElfW(Sym
) *refsym
, const ElfW(Sym
) *sym
,
220 const ElfW(Rela
) *reloc
,
221 ElfW(Addr
) *reloc_addr
, ElfW(Addr
) value
)
223 return *reloc_addr
= value
;
226 /* Return the final value of a PLT relocation. On x86-64 the
227 JUMP_SLOT relocation ignores the addend. */
228 static inline ElfW(Addr
)
229 elf_machine_plt_value (struct link_map
*map
, const ElfW(Rela
) *reloc
,
236 /* Names of the architecture-specific auditing callback functions. */
238 #define ARCH_LA_PLTENTER x86_64_gnu_pltenter
239 #define ARCH_LA_PLTEXIT x86_64_gnu_pltexit
241 #define ARCH_LA_PLTENTER x32_gnu_pltenter
242 #define ARCH_LA_PLTEXIT x32_gnu_pltexit
245 #endif /* !dl_machine_h */
249 /* Perform the relocation specified by RELOC and SYM (which is fully resolved).
250 MAP is the object containing the reloc. */
252 static inline void __attribute__((always_inline
))
253 elf_machine_rela (struct link_map
*map
, struct r_scope_elem
*scope
[],
254 const ElfW(Rela
) *reloc
, const ElfW(Sym
) *sym
,
255 const struct r_found_version
*version
,
256 void *const reloc_addr_arg
, int skip_ifunc
)
258 ElfW(Addr
) *const reloc_addr
= reloc_addr_arg
;
259 const unsigned long int r_type
= ELFW(R_TYPE
) (reloc
->r_info
);
261 # if !defined RTLD_BOOTSTRAP
262 if (__glibc_unlikely (r_type
== R_X86_64_RELATIVE
))
263 *reloc_addr
= map
->l_addr
+ reloc
->r_addend
;
266 # if !defined RTLD_BOOTSTRAP
267 /* l_addr + r_addend may be > 0xffffffff and R_X86_64_RELATIVE64
268 relocation updates the whole 64-bit entry. */
269 if (__glibc_unlikely (r_type
== R_X86_64_RELATIVE64
))
270 *(Elf64_Addr
*) reloc_addr
= (Elf64_Addr
) map
->l_addr
+ reloc
->r_addend
;
273 if (__glibc_unlikely (r_type
== R_X86_64_NONE
))
277 # ifndef RTLD_BOOTSTRAP
278 const ElfW(Sym
) *const refsym
= sym
;
280 struct link_map
*sym_map
= RESOLVE_MAP (map
, scope
, &sym
, version
,
282 ElfW(Addr
) value
= SYMBOL_ADDRESS (sym_map
, sym
, true);
285 && __glibc_unlikely (ELFW(ST_TYPE
) (sym
->st_info
) == STT_GNU_IFUNC
)
286 && __glibc_likely (sym
->st_shndx
!= SHN_UNDEF
)
287 && __glibc_likely (!skip_ifunc
))
289 # ifndef RTLD_BOOTSTRAP
291 && !sym_map
->l_relocated
)
294 = (const char *) D_PTR (map
, l_info
[DT_STRTAB
]);
295 if (sym_map
->l_type
== lt_executable
)
297 %s: IFUNC symbol '%s' referenced in '%s' is defined in the executable \
298 and creates an unsatisfiable circular dependency.\n",
299 RTLD_PROGNAME
, strtab
+ refsym
->st_name
,
303 %s: Relink `%s' with `%s' for IFUNC symbol `%s'\n",
304 RTLD_PROGNAME
, map
->l_name
,
306 strtab
+ refsym
->st_name
);
309 value
= ((ElfW(Addr
) (*) (void)) value
) ();
314 case R_X86_64_JUMP_SLOT
:
315 map
->l_has_jump_slot_reloc
= true;
317 case R_X86_64_GLOB_DAT
:
321 # ifndef RTLD_BOOTSTRAP
323 case R_X86_64_SIZE64
:
324 /* Set to symbol size plus addend. */
325 *(Elf64_Addr
*) (uintptr_t) reloc_addr
326 = (Elf64_Addr
) sym
->st_size
+ reloc
->r_addend
;
329 case R_X86_64_SIZE32
:
331 case R_X86_64_SIZE64
:
333 /* Set to symbol size plus addend. */
334 value
= sym
->st_size
;
335 *reloc_addr
= value
+ reloc
->r_addend
;
338 case R_X86_64_DTPMOD64
:
339 /* Get the information from the link map returned by the
342 *reloc_addr
= sym_map
->l_tls_modid
;
344 case R_X86_64_DTPOFF64
:
345 /* During relocation all TLS symbols are defined and used.
346 Therefore the offset is already correct. */
349 value
= sym
->st_value
+ reloc
->r_addend
;
351 /* This relocation type computes a signed offset that is
352 usually negative. The symbol and addend values are 32
353 bits but the GOT entry is 64 bits wide and the whole
354 64-bit entry is used as a signed quantity, so we need
355 to sign-extend the computed value to 64 bits. */
356 *(Elf64_Sxword
*) reloc_addr
= (Elf64_Sxword
) (Elf32_Sword
) value
;
362 case R_X86_64_TLSDESC
:
364 struct tlsdesc
volatile *td
=
365 (struct tlsdesc
volatile *)reloc_addr
;
369 td
->arg
= (void*)reloc
->r_addend
;
370 td
->entry
= _dl_tlsdesc_undefweak
;
375 CHECK_STATIC_TLS (map
, sym_map
);
377 if (!TRY_STATIC_TLS (map
, sym_map
))
379 td
->arg
= _dl_make_tlsdesc_dynamic
380 (sym_map
, sym
->st_value
+ reloc
->r_addend
);
381 td
->entry
= GLRO(dl_x86_tlsdesc_dynamic
);
386 td
->arg
= (void*)(sym
->st_value
- sym_map
->l_tls_offset
388 td
->entry
= _dl_tlsdesc_return
;
393 case R_X86_64_TPOFF64
:
394 /* The offset is negative, forward from the thread pointer. */
397 CHECK_STATIC_TLS (map
, sym_map
);
398 /* We know the offset of the object the symbol is contained in.
399 It is a negative value which will be added to the
401 value
= (sym
->st_value
+ reloc
->r_addend
402 - sym_map
->l_tls_offset
);
404 /* The symbol and addend values are 32 bits but the GOT
405 entry is 64 bits wide and the whole 64-bit entry is used
406 as a signed quantity, so we need to sign-extend the
407 computed value to 64 bits. */
408 *(Elf64_Sxword
*) reloc_addr
= (Elf64_Sxword
) (Elf32_Sword
) value
;
416 /* value + r_addend may be > 0xffffffff and R_X86_64_64
417 relocation updates the whole 64-bit entry. */
418 *(Elf64_Addr
*) reloc_addr
= (Elf64_Addr
) value
+ reloc
->r_addend
;
421 case R_X86_64_SIZE32
:
422 /* Set to symbol size plus addend. */
423 value
= sym
->st_size
;
427 value
+= reloc
->r_addend
;
428 *(unsigned int *) reloc_addr
= value
;
431 if (__glibc_unlikely (value
> UINT_MAX
))
436 %s: Symbol `%s' causes overflow in R_X86_64_32 relocation\n";
438 strtab
= (const char *) D_PTR (map
, l_info
[DT_STRTAB
]);
440 _dl_error_printf (fmt
, RTLD_PROGNAME
, strtab
+ refsym
->st_name
);
443 /* Not needed for dl-conflict.c. */
445 value
+= reloc
->r_addend
- (ElfW(Addr
)) reloc_addr
;
446 *(unsigned int *) reloc_addr
= value
;
447 if (__glibc_unlikely (value
!= (int) value
))
450 %s: Symbol `%s' causes overflow in R_X86_64_PC32 relocation\n";
456 /* This can happen in trace mode if an object could not be
459 memcpy (reloc_addr_arg
, (void *) value
,
460 MIN (sym
->st_size
, refsym
->st_size
));
461 if (__glibc_unlikely (sym
->st_size
> refsym
->st_size
)
462 || (__glibc_unlikely (sym
->st_size
< refsym
->st_size
)
463 && GLRO(dl_verbose
)))
466 %s: Symbol `%s' has different size in shared object, consider re-linking\n";
470 case R_X86_64_IRELATIVE
:
471 value
= map
->l_addr
+ reloc
->r_addend
;
472 if (__glibc_likely (!skip_ifunc
))
473 value
= ((ElfW(Addr
) (*) (void)) value
) ();
477 _dl_reloc_bad_type (map
, r_type
, 0);
479 # endif /* !RTLD_BOOTSTRAP */
485 __attribute ((always_inline
))
486 elf_machine_rela_relative (ElfW(Addr
) l_addr
, const ElfW(Rela
) *reloc
,
487 void *const reloc_addr_arg
)
489 ElfW(Addr
) *const reloc_addr
= reloc_addr_arg
;
490 #if !defined RTLD_BOOTSTRAP
491 /* l_addr + r_addend may be > 0xffffffff and R_X86_64_RELATIVE64
492 relocation updates the whole 64-bit entry. */
493 if (__glibc_unlikely (ELFW(R_TYPE
) (reloc
->r_info
) == R_X86_64_RELATIVE64
))
494 *(Elf64_Addr
*) reloc_addr
= (Elf64_Addr
) l_addr
+ reloc
->r_addend
;
498 assert (ELFW(R_TYPE
) (reloc
->r_info
) == R_X86_64_RELATIVE
);
499 *reloc_addr
= l_addr
+ reloc
->r_addend
;
504 __attribute ((always_inline
))
505 elf_machine_lazy_rel (struct link_map
*map
, struct r_scope_elem
*scope
[],
506 ElfW(Addr
) l_addr
, const ElfW(Rela
) *reloc
,
509 ElfW(Addr
) *const reloc_addr
= (void *) (l_addr
+ reloc
->r_offset
);
510 const unsigned long int r_type
= ELFW(R_TYPE
) (reloc
->r_info
);
512 /* Check for unexpected PLT reloc type. */
513 if (__glibc_likely (r_type
== R_X86_64_JUMP_SLOT
))
515 /* Prelink has been deprecated. */
516 if (__glibc_likely (map
->l_mach
.plt
== 0))
517 *reloc_addr
+= l_addr
;
521 + (((ElfW(Addr
)) reloc_addr
) - map
->l_mach
.gotplt
) * 2;
523 else if (__glibc_likely (r_type
== R_X86_64_TLSDESC
))
525 const Elf_Symndx symndx
= ELFW (R_SYM
) (reloc
->r_info
);
526 const ElfW (Sym
) *symtab
= (const void *)D_PTR (map
, l_info
[DT_SYMTAB
]);
527 const ElfW (Sym
) *sym
= &symtab
[symndx
];
528 const struct r_found_version
*version
= NULL
;
530 if (map
->l_info
[VERSYMIDX (DT_VERSYM
)] != NULL
)
532 const ElfW (Half
) *vernum
=
533 (const void *)D_PTR (map
, l_info
[VERSYMIDX (DT_VERSYM
)]);
534 version
= &map
->l_versions
[vernum
[symndx
] & 0x7fff];
537 /* Always initialize TLS descriptors completely at load time, in
538 case static TLS is allocated for it that requires locking. */
539 elf_machine_rela (map
, scope
, reloc
, sym
, version
, reloc_addr
, skip_ifunc
);
541 else if (__glibc_unlikely (r_type
== R_X86_64_IRELATIVE
))
543 ElfW(Addr
) value
= map
->l_addr
+ reloc
->r_addend
;
544 if (__glibc_likely (!skip_ifunc
))
545 value
= ((ElfW(Addr
) (*) (void)) value
) ();
549 _dl_reloc_bad_type (map
, r_type
, 1);
552 #endif /* RESOLVE_MAP */
554 #if !defined ELF_DYNAMIC_AFTER_RELOC && !defined RTLD_BOOTSTRAP \
556 # define ELF_DYNAMIC_AFTER_RELOC(map, lazy) \
557 x86_64_dynamic_after_reloc (map, (lazy))
559 # define JMP32_INSN_OPCODE 0xe9
560 # define JMP32_INSN_SIZE 5
561 # define JMPABS_INSN_OPCODE 0xa100d5
562 # define JMPABS_INSN_SIZE 11
563 # define INT3_INSN_OPCODE 0xcc
566 x86_64_reloc_symbol_name (struct link_map
*map
, const ElfW(Rela
) *reloc
)
568 const ElfW(Sym
) *const symtab
569 = (const void *) map
->l_info
[DT_SYMTAB
]->d_un
.d_ptr
;
570 const ElfW(Sym
) *const refsym
= &symtab
[ELFW (R_SYM
) (reloc
->r_info
)];
571 const char *strtab
= (const char *) map
->l_info
[DT_STRTAB
]->d_un
.d_ptr
;
572 return strtab
+ refsym
->st_name
;
576 x86_64_rewrite_plt (struct link_map
*map
, ElfW(Addr
) plt_rewrite
)
578 ElfW(Addr
) l_addr
= map
->l_addr
;
579 ElfW(Addr
) pltent
= map
->l_info
[DT_X86_64 (PLTENT
)]->d_un
.d_val
;
580 ElfW(Addr
) start
= map
->l_info
[DT_JMPREL
]->d_un
.d_ptr
;
581 ElfW(Addr
) size
= map
->l_info
[DT_PLTRELSZ
]->d_un
.d_val
;
582 const ElfW(Rela
) *reloc
= (const void *) start
;
583 const ElfW(Rela
) *reloc_end
= (const void *) (start
+ size
);
586 bool ibt_enabled_p
= dl_cet_ibt_enabled ();
588 bool ibt_enabled_p
= false;
591 if (__glibc_unlikely (GLRO(dl_debug_mask
) & DL_DEBUG_FILES
))
592 _dl_debug_printf ("\nchanging PLT in '%s' to direct branch\n",
593 DSO_FILENAME (map
->l_name
));
595 for (; reloc
< reloc_end
; reloc
++)
596 if (ELFW(R_TYPE
) (reloc
->r_info
) == R_X86_64_JUMP_SLOT
)
598 /* Get the value from the GOT entry. */
599 ElfW(Addr
) value
= *(ElfW(Addr
) *) (l_addr
+ reloc
->r_offset
);
601 /* Get the corresponding PLT entry from r_addend. */
602 ElfW(Addr
) branch_start
= l_addr
+ reloc
->r_addend
;
603 /* Skip ENDBR64 if IBT isn't enabled. */
605 branch_start
= ALIGN_DOWN (branch_start
, pltent
);
606 /* Get the displacement from the branch target. NB: We must use
607 64-bit integer on x32 to avoid overflow. */
608 uint64_t disp
= (uint64_t) value
- branch_start
- JMP32_INSN_SIZE
;
612 plt_end
= (branch_start
| (pltent
- 1)) + 1;
614 /* Update the PLT entry. */
615 if (((uint64_t) disp
+ (uint64_t) ((uint32_t) INT32_MIN
))
616 <= (uint64_t) UINT32_MAX
)
618 pad
= branch_start
+ JMP32_INSN_SIZE
;
620 if (__glibc_unlikely (pad
> plt_end
))
623 /* If the target branch can be reached with a direct branch,
624 rewrite the PLT entry with a direct branch. */
625 if (__glibc_unlikely (GLRO(dl_debug_mask
) & DL_DEBUG_BINDINGS
))
627 const char *sym_name
= x86_64_reloc_symbol_name (map
,
629 _dl_debug_printf ("changing '%s' PLT entry in '%s' to "
630 "direct branch\n", sym_name
,
631 DSO_FILENAME (map
->l_name
));
634 /* Write out direct branch. */
635 *(uint8_t *) branch_start
= JMP32_INSN_OPCODE
;
636 *(uint32_t *) (branch_start
+ 1) = disp
;
640 if (GL(dl_x86_feature_control
).plt_rewrite
641 != plt_rewrite_jmpabs
)
643 if (__glibc_unlikely (GLRO(dl_debug_mask
)
644 & DL_DEBUG_BINDINGS
))
647 = x86_64_reloc_symbol_name (map
, reloc
);
648 _dl_debug_printf ("skipping '%s' PLT entry in '%s'\n",
650 DSO_FILENAME (map
->l_name
));
655 pad
= branch_start
+ JMPABS_INSN_SIZE
;
657 if (__glibc_unlikely (pad
> plt_end
))
660 /* Rewrite the PLT entry with JMPABS. */
661 if (__glibc_unlikely (GLRO(dl_debug_mask
) & DL_DEBUG_BINDINGS
))
663 const char *sym_name
= x86_64_reloc_symbol_name (map
,
665 _dl_debug_printf ("changing '%s' PLT entry in '%s' to "
666 "JMPABS\n", sym_name
,
667 DSO_FILENAME (map
->l_name
));
670 /* "jmpabs $target" for 64-bit displacement. NB: JMPABS has
671 a 3-byte opcode + 64bit address. There is a 1-byte overlap
672 between 4-byte write and 8-byte write. */
673 *(uint32_t *) (branch_start
) = JMPABS_INSN_OPCODE
;
674 *(uint64_t *) (branch_start
+ 3) = value
;
677 /* Fill the unused part of the PLT entry with INT3. */
678 for (; pad
< plt_end
; pad
++)
679 *(uint8_t *) pad
= INT3_INSN_OPCODE
;
684 x86_64_rewrite_plt_in_place (struct link_map
*map
)
686 /* Adjust DT_X86_64_PLT address and DT_X86_64_PLTSZ values. */
687 ElfW(Addr
) plt
= (map
->l_info
[DT_X86_64 (PLT
)]->d_un
.d_ptr
689 size_t pagesize
= GLRO(dl_pagesize
);
690 ElfW(Addr
) plt_aligned
= ALIGN_DOWN (plt
, pagesize
);
691 size_t pltsz
= (map
->l_info
[DT_X86_64 (PLTSZ
)]->d_un
.d_val
692 + plt
- plt_aligned
);
694 if (__glibc_unlikely (GLRO(dl_debug_mask
) & DL_DEBUG_FILES
))
695 _dl_debug_printf ("\nchanging PLT in '%s' to writable\n",
696 DSO_FILENAME (map
->l_name
));
698 if (__glibc_unlikely (__mprotect ((void *) plt_aligned
, pltsz
,
699 PROT_WRITE
| PROT_READ
) < 0))
701 if (__glibc_unlikely (GLRO(dl_debug_mask
) & DL_DEBUG_FILES
))
702 _dl_debug_printf ("\nfailed to change PLT in '%s' to writable\n",
703 DSO_FILENAME (map
->l_name
));
707 x86_64_rewrite_plt (map
, plt_aligned
);
709 if (__glibc_unlikely (GLRO(dl_debug_mask
) & DL_DEBUG_FILES
))
710 _dl_debug_printf ("\nchanging PLT in '%s' back to read-only\n",
711 DSO_FILENAME (map
->l_name
));
713 if (__glibc_unlikely (__mprotect ((void *) plt_aligned
, pltsz
,
714 PROT_EXEC
| PROT_READ
) < 0))
715 _dl_signal_error (0, DSO_FILENAME (map
->l_name
), NULL
,
716 "failed to change PLT back to read-only");
719 /* Rewrite PLT entries to direct branch if possible. */
722 x86_64_dynamic_after_reloc (struct link_map
*map
, int lazy
)
724 /* Ignore DT_X86_64_PLT if the lazy binding is enabled. */
728 /* Ignore DT_X86_64_PLT if PLT rewrite isn't enabled. */
729 if (__glibc_likely (GL(dl_x86_feature_control
).plt_rewrite
730 == plt_rewrite_none
))
733 if (__glibc_likely (map
->l_info
[DT_X86_64 (PLT
)] == NULL
))
736 /* Ignore DT_X86_64_PLT if there is no R_X86_64_JUMP_SLOT. */
737 if (map
->l_has_jump_slot_reloc
== 0)
740 /* Ignore DT_X86_64_PLT if
741 1. DT_JMPREL isn't available or its value is 0.
743 3. DT_X86_64_PLTENT isn't available or its value is smaller than
745 4. DT_X86_64_PLTSZ isn't available or its value is smaller than
746 DT_X86_64_PLTENT's value or isn't a multiple of DT_X86_64_PLTENT's
748 if (map
->l_info
[DT_JMPREL
] == NULL
749 || map
->l_info
[DT_JMPREL
]->d_un
.d_ptr
== 0
750 || map
->l_info
[DT_PLTRELSZ
]->d_un
.d_val
== 0
751 || map
->l_info
[DT_X86_64 (PLTSZ
)] == NULL
752 || map
->l_info
[DT_X86_64 (PLTENT
)] == NULL
753 || map
->l_info
[DT_X86_64 (PLTENT
)]->d_un
.d_val
< 16
754 || (map
->l_info
[DT_X86_64 (PLTSZ
)]->d_un
.d_val
755 < map
->l_info
[DT_X86_64 (PLTENT
)]->d_un
.d_val
)
756 || (map
->l_info
[DT_X86_64 (PLTSZ
)]->d_un
.d_val
757 % map
->l_info
[DT_X86_64 (PLTENT
)]->d_un
.d_val
) != 0)
760 x86_64_rewrite_plt_in_place (map
);