sysdeps/x86_64/dl-trampoline.h

   1 /* PLT trampolines.  x86-64 version.
   2    Copyright (C) 2009-2017 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #undef REGISTER_SAVE_AREA_RAW
  20 #ifdef __ILP32__
  21 /* X32 saves RCX, RDX, RSI, RDI, R8 and R9 plus RAX as well as VEC0 to
  22    VEC7.  */
  23 # define REGISTER_SAVE_AREA_RAW (8 * 7 + VEC_SIZE * 8)
  24 #else
  25 /* X86-64 saves RCX, RDX, RSI, RDI, R8 and R9 plus RAX as well as
  26    BND0, BND1, BND2, BND3 and VEC0 to VEC7. */
  27 # define REGISTER_SAVE_AREA_RAW (8 * 7 + 16 * 4 + VEC_SIZE * 8)
  28 #endif
  29
  30 #undef REGISTER_SAVE_AREA
  31 #undef LOCAL_STORAGE_AREA
  32 #undef BASE
  33 #if DL_RUNTIME_RESOLVE_REALIGN_STACK
  34 # define REGISTER_SAVE_AREA     (REGISTER_SAVE_AREA_RAW + 8)
  35 /* Local stack area before jumping to function address: RBX.  */
  36 # define LOCAL_STORAGE_AREA     8
  37 # define BASE                   rbx
  38 # if (REGISTER_SAVE_AREA % VEC_SIZE) != 0
  39 #  error REGISTER_SAVE_AREA must be multples of VEC_SIZE
  40 # endif
  41 #else
  42 # define REGISTER_SAVE_AREA     REGISTER_SAVE_AREA_RAW
  43 /* Local stack area before jumping to function address:  All saved
  44    registers.  */
  45 # define LOCAL_STORAGE_AREA     REGISTER_SAVE_AREA
  46 # define BASE                   rsp
  47 # if (REGISTER_SAVE_AREA % 16) != 8
  48 #  error REGISTER_SAVE_AREA must be odd multples of 8
  49 # endif
  50 #endif
  51
  52         .text
  53 #ifdef _dl_runtime_resolve_opt
  54 /* Use the smallest vector registers to preserve the full YMM/ZMM
  55    registers to avoid SSE transition penalty.  */
  56
  57 # if VEC_SIZE == 32
  58 /* Check if the upper 128 bits in %ymm0 - %ymm7 registers are non-zero
  59    and preserve %xmm0 - %xmm7 registers with the zero upper bits.  Since
  60    there is no SSE transition penalty on AVX512 processors which don't
  61    support XGETBV with ECX == 1, _dl_runtime_resolve_avx512_slow isn't
  62    provided.   */
  63         .globl _dl_runtime_resolve_avx_slow
  64         .hidden _dl_runtime_resolve_avx_slow
  65         .type _dl_runtime_resolve_avx_slow, @function
  66         .align 16
  67 _dl_runtime_resolve_avx_slow:
  68         cfi_startproc
  69         cfi_adjust_cfa_offset(16) # Incorporate PLT
  70         vorpd %ymm0, %ymm1, %ymm8
  71         vorpd %ymm2, %ymm3, %ymm9
  72         vorpd %ymm4, %ymm5, %ymm10
  73         vorpd %ymm6, %ymm7, %ymm11
  74         vorpd %ymm8, %ymm9, %ymm9
  75         vorpd %ymm10, %ymm11, %ymm10
  76         vpcmpeqd %xmm8, %xmm8, %xmm8
  77         vorpd %ymm9, %ymm10, %ymm10
  78         vptest %ymm10, %ymm8
  79         # Preserve %ymm0 - %ymm7 registers if the upper 128 bits of any
  80         # %ymm0 - %ymm7 registers aren't zero.
  81         PRESERVE_BND_REGS_PREFIX
  82         jnc _dl_runtime_resolve_avx
  83         # Use vzeroupper to avoid SSE transition penalty.
  84         vzeroupper
  85         # Preserve %xmm0 - %xmm7 registers with the zero upper 128 bits
  86         # when the upper 128 bits of %ymm0 - %ymm7 registers are zero.
  87         PRESERVE_BND_REGS_PREFIX
  88         jmp _dl_runtime_resolve_sse_vex
  89         cfi_adjust_cfa_offset(-16) # Restore PLT adjustment
  90         cfi_endproc
  91         .size _dl_runtime_resolve_avx_slow, .-_dl_runtime_resolve_avx_slow
  92 # endif
  93
  94 /* Use XGETBV with ECX == 1 to check which bits in vector registers are
  95    non-zero and only preserve the non-zero lower bits with zero upper
  96    bits.  */
  97         .globl _dl_runtime_resolve_opt
  98         .hidden _dl_runtime_resolve_opt
  99         .type _dl_runtime_resolve_opt, @function
 100         .align 16
 101 _dl_runtime_resolve_opt:
 102         cfi_startproc
 103         cfi_adjust_cfa_offset(16) # Incorporate PLT
 104         pushq %rax
 105         cfi_adjust_cfa_offset(8)
 106         cfi_rel_offset(%rax, 0)
 107         pushq %rcx
 108         cfi_adjust_cfa_offset(8)
 109         cfi_rel_offset(%rcx, 0)
 110         pushq %rdx
 111         cfi_adjust_cfa_offset(8)
 112         cfi_rel_offset(%rdx, 0)
 113         movl $1, %ecx
 114         xgetbv
 115         movl %eax, %r11d
 116         popq %rdx
 117         cfi_adjust_cfa_offset(-8)
 118         cfi_restore (%rdx)
 119         popq %rcx
 120         cfi_adjust_cfa_offset(-8)
 121         cfi_restore (%rcx)
 122         popq %rax
 123         cfi_adjust_cfa_offset(-8)
 124         cfi_restore (%rax)
 125 # if VEC_SIZE == 32
 126         # For YMM registers, check if YMM state is in use.
 127         andl $bit_YMM_state, %r11d
 128         # Preserve %xmm0 - %xmm7 registers with the zero upper 128 bits if
 129         # YMM state isn't in use.
 130         PRESERVE_BND_REGS_PREFIX
 131         jz _dl_runtime_resolve_sse_vex
 132 # elif VEC_SIZE == 64
 133         # For ZMM registers, check if YMM state and ZMM state are in
 134         # use.
 135         andl $(bit_YMM_state | bit_ZMM0_15_state), %r11d
 136         cmpl $bit_YMM_state, %r11d
 137         # Preserve %xmm0 - %xmm7 registers with the zero upper 384 bits if
 138         # neither YMM state nor ZMM state are in use.
 139         PRESERVE_BND_REGS_PREFIX
 140         jl _dl_runtime_resolve_sse_vex
 141         # Preserve %ymm0 - %ymm7 registers with the zero upper 256 bits if
 142         # ZMM state isn't in use.
 143         PRESERVE_BND_REGS_PREFIX
 144         je _dl_runtime_resolve_avx
 145 # else
 146 #  error Unsupported VEC_SIZE!
 147 # endif
 148         cfi_adjust_cfa_offset(-16) # Restore PLT adjustment
 149         cfi_endproc
 150         .size _dl_runtime_resolve_opt, .-_dl_runtime_resolve_opt
 151 #endif
 152         .globl _dl_runtime_resolve
 153         .hidden _dl_runtime_resolve
 154         .type _dl_runtime_resolve, @function
 155         .align 16
 156         cfi_startproc
 157 _dl_runtime_resolve:
 158         cfi_adjust_cfa_offset(16) # Incorporate PLT
 159 #if DL_RUNTIME_RESOLVE_REALIGN_STACK
 160 # if LOCAL_STORAGE_AREA != 8
 161 #  error LOCAL_STORAGE_AREA must be 8
 162 # endif
 163         pushq %rbx                      # push subtracts stack by 8.
 164         cfi_adjust_cfa_offset(8)
 165         cfi_rel_offset(%rbx, 0)
 166         mov %RSP_LP, %RBX_LP
 167         cfi_def_cfa_register(%rbx)
 168         and $-VEC_SIZE, %RSP_LP
 169 #endif
 170         sub $REGISTER_SAVE_AREA, %RSP_LP
 171 #if !DL_RUNTIME_RESOLVE_REALIGN_STACK
 172         cfi_adjust_cfa_offset(REGISTER_SAVE_AREA)
 173 #endif
 174         # Preserve registers otherwise clobbered.
 175         movq %rax, REGISTER_SAVE_RAX(%rsp)
 176         movq %rcx, REGISTER_SAVE_RCX(%rsp)
 177         movq %rdx, REGISTER_SAVE_RDX(%rsp)
 178         movq %rsi, REGISTER_SAVE_RSI(%rsp)
 179         movq %rdi, REGISTER_SAVE_RDI(%rsp)
 180         movq %r8, REGISTER_SAVE_R8(%rsp)
 181         movq %r9, REGISTER_SAVE_R9(%rsp)
 182         VMOV %VEC(0), (REGISTER_SAVE_VEC_OFF)(%rsp)
 183         VMOV %VEC(1), (REGISTER_SAVE_VEC_OFF + VEC_SIZE)(%rsp)
 184         VMOV %VEC(2), (REGISTER_SAVE_VEC_OFF + VEC_SIZE * 2)(%rsp)
 185         VMOV %VEC(3), (REGISTER_SAVE_VEC_OFF + VEC_SIZE * 3)(%rsp)
 186         VMOV %VEC(4), (REGISTER_SAVE_VEC_OFF + VEC_SIZE * 4)(%rsp)
 187         VMOV %VEC(5), (REGISTER_SAVE_VEC_OFF + VEC_SIZE * 5)(%rsp)
 188         VMOV %VEC(6), (REGISTER_SAVE_VEC_OFF + VEC_SIZE * 6)(%rsp)
 189         VMOV %VEC(7), (REGISTER_SAVE_VEC_OFF + VEC_SIZE * 7)(%rsp)
 190 #ifndef __ILP32__
 191         # We also have to preserve bound registers.  These are nops if
 192         # Intel MPX isn't available or disabled.
 193 # ifdef HAVE_MPX_SUPPORT
 194         bndmov %bnd0, REGISTER_SAVE_BND0(%rsp)
 195         bndmov %bnd1, REGISTER_SAVE_BND1(%rsp)
 196         bndmov %bnd2, REGISTER_SAVE_BND2(%rsp)
 197         bndmov %bnd3, REGISTER_SAVE_BND3(%rsp)
 198 # else
 199 #  if REGISTER_SAVE_BND0 == 0
 200         .byte 0x66,0x0f,0x1b,0x04,0x24
 201 #  else
 202         .byte 0x66,0x0f,0x1b,0x44,0x24,REGISTER_SAVE_BND0
 203 #  endif
 204         .byte 0x66,0x0f,0x1b,0x4c,0x24,REGISTER_SAVE_BND1
 205         .byte 0x66,0x0f,0x1b,0x54,0x24,REGISTER_SAVE_BND2
 206         .byte 0x66,0x0f,0x1b,0x5c,0x24,REGISTER_SAVE_BND3
 207 # endif
 208 #endif
 209         # Copy args pushed by PLT in register.
 210         # %rdi: link_map, %rsi: reloc_index
 211         mov (LOCAL_STORAGE_AREA + 8)(%BASE), %RSI_LP
 212         mov LOCAL_STORAGE_AREA(%BASE), %RDI_LP
 213         call _dl_fixup          # Call resolver.
 214         mov %RAX_LP, %R11_LP    # Save return value
 215 #ifndef __ILP32__
 216         # Restore bound registers.  These are nops if Intel MPX isn't
 217         # avaiable or disabled.
 218 # ifdef HAVE_MPX_SUPPORT
 219         bndmov REGISTER_SAVE_BND3(%rsp), %bnd3
 220         bndmov REGISTER_SAVE_BND2(%rsp), %bnd2
 221         bndmov REGISTER_SAVE_BND1(%rsp), %bnd1
 222         bndmov REGISTER_SAVE_BND0(%rsp), %bnd0
 223 # else
 224         .byte 0x66,0x0f,0x1a,0x5c,0x24,REGISTER_SAVE_BND3
 225         .byte 0x66,0x0f,0x1a,0x54,0x24,REGISTER_SAVE_BND2
 226         .byte 0x66,0x0f,0x1a,0x4c,0x24,REGISTER_SAVE_BND1
 227 #  if REGISTER_SAVE_BND0 == 0
 228         .byte 0x66,0x0f,0x1a,0x04,0x24
 229 #  else
 230         .byte 0x66,0x0f,0x1a,0x44,0x24,REGISTER_SAVE_BND0
 231 #  endif
 232 # endif
 233 #endif
 234         # Get register content back.
 235         movq REGISTER_SAVE_R9(%rsp), %r9
 236         movq REGISTER_SAVE_R8(%rsp), %r8
 237         movq REGISTER_SAVE_RDI(%rsp), %rdi
 238         movq REGISTER_SAVE_RSI(%rsp), %rsi
 239         movq REGISTER_SAVE_RDX(%rsp), %rdx
 240         movq REGISTER_SAVE_RCX(%rsp), %rcx
 241         movq REGISTER_SAVE_RAX(%rsp), %rax
 242         VMOV (REGISTER_SAVE_VEC_OFF)(%rsp), %VEC(0)
 243         VMOV (REGISTER_SAVE_VEC_OFF + VEC_SIZE)(%rsp), %VEC(1)
 244         VMOV (REGISTER_SAVE_VEC_OFF + VEC_SIZE * 2)(%rsp), %VEC(2)
 245         VMOV (REGISTER_SAVE_VEC_OFF + VEC_SIZE * 3)(%rsp), %VEC(3)
 246         VMOV (REGISTER_SAVE_VEC_OFF + VEC_SIZE * 4)(%rsp), %VEC(4)
 247         VMOV (REGISTER_SAVE_VEC_OFF + VEC_SIZE * 5)(%rsp), %VEC(5)
 248         VMOV (REGISTER_SAVE_VEC_OFF + VEC_SIZE * 6)(%rsp), %VEC(6)
 249         VMOV (REGISTER_SAVE_VEC_OFF + VEC_SIZE * 7)(%rsp), %VEC(7)
 250 #if DL_RUNTIME_RESOLVE_REALIGN_STACK
 251         mov %RBX_LP, %RSP_LP
 252         cfi_def_cfa_register(%rsp)
 253         movq (%rsp), %rbx
 254         cfi_restore(%rbx)
 255 #endif
 256         # Adjust stack(PLT did 2 pushes)
 257         add $(LOCAL_STORAGE_AREA + 16), %RSP_LP
 258         cfi_adjust_cfa_offset(-(LOCAL_STORAGE_AREA + 16))
 259         # Preserve bound registers.
 260         PRESERVE_BND_REGS_PREFIX
 261         jmp *%r11               # Jump to function address.
 262         cfi_endproc
 263         .size _dl_runtime_resolve, .-_dl_runtime_resolve
 264
 265
 266 /* To preserve %xmm0 - %xmm7 registers, dl-trampoline.h is included
 267    twice, for _dl_runtime_resolve_sse and _dl_runtime_resolve_sse_vex.
 268    But we don't need another _dl_runtime_profile for XMM registers.  */
 269 #if !defined PROF && defined _dl_runtime_profile
 270 # if (LR_VECTOR_OFFSET % VEC_SIZE) != 0
 271 #  error LR_VECTOR_OFFSET must be multples of VEC_SIZE
 272 # endif
 273
 274         .globl _dl_runtime_profile
 275         .hidden _dl_runtime_profile
 276         .type _dl_runtime_profile, @function
 277         .align 16
 278 _dl_runtime_profile:
 279         cfi_startproc
 280         cfi_adjust_cfa_offset(16) # Incorporate PLT
 281         /* The La_x86_64_regs data structure pointed to by the
 282            fourth paramater must be VEC_SIZE-byte aligned.  This must
 283            be explicitly enforced.  We have the set up a dynamically
 284            sized stack frame.  %rbx points to the top half which
 285            has a fixed size and preserves the original stack pointer.  */
 286
 287         sub $32, %RSP_LP        # Allocate the local storage.
 288         cfi_adjust_cfa_offset(32)
 289         movq %rbx, (%rsp)
 290         cfi_rel_offset(%rbx, 0)
 291
 292         /* On the stack:
 293                 56(%rbx)        parameter #1
 294                 48(%rbx)        return address
 295
 296                 40(%rbx)        reloc index
 297                 32(%rbx)        link_map
 298
 299                 24(%rbx)        La_x86_64_regs pointer
 300                 16(%rbx)        framesize
 301                  8(%rbx)        rax
 302                   (%rbx)        rbx
 303         */
 304
 305         movq %rax, 8(%rsp)
 306         mov %RSP_LP, %RBX_LP
 307         cfi_def_cfa_register(%rbx)
 308
 309         /* Actively align the La_x86_64_regs structure.  */
 310         and $-VEC_SIZE, %RSP_LP
 311         /* sizeof(La_x86_64_regs).  Need extra space for 8 SSE registers
 312            to detect if any xmm0-xmm7 registers are changed by audit
 313            module.  */
 314         sub $(LR_SIZE + XMM_SIZE*8), %RSP_LP
 315         movq %rsp, 24(%rbx)
 316
 317         /* Fill the La_x86_64_regs structure.  */
 318         movq %rdx, LR_RDX_OFFSET(%rsp)
 319         movq %r8,  LR_R8_OFFSET(%rsp)
 320         movq %r9,  LR_R9_OFFSET(%rsp)
 321         movq %rcx, LR_RCX_OFFSET(%rsp)
 322         movq %rsi, LR_RSI_OFFSET(%rsp)
 323         movq %rdi, LR_RDI_OFFSET(%rsp)
 324         movq %rbp, LR_RBP_OFFSET(%rsp)
 325
 326         lea 48(%rbx), %RAX_LP
 327         movq %rax, LR_RSP_OFFSET(%rsp)
 328
 329         /* We always store the XMM registers even if AVX is available.
 330            This is to provide backward binary compatibility for existing
 331            audit modules.  */
 332         movaps %xmm0,              (LR_XMM_OFFSET)(%rsp)
 333         movaps %xmm1, (LR_XMM_OFFSET +   XMM_SIZE)(%rsp)
 334         movaps %xmm2, (LR_XMM_OFFSET + XMM_SIZE*2)(%rsp)
 335         movaps %xmm3, (LR_XMM_OFFSET + XMM_SIZE*3)(%rsp)
 336         movaps %xmm4, (LR_XMM_OFFSET + XMM_SIZE*4)(%rsp)
 337         movaps %xmm5, (LR_XMM_OFFSET + XMM_SIZE*5)(%rsp)
 338         movaps %xmm6, (LR_XMM_OFFSET + XMM_SIZE*6)(%rsp)
 339         movaps %xmm7, (LR_XMM_OFFSET + XMM_SIZE*7)(%rsp)
 340
 341 # ifndef __ILP32__
 342 #  ifdef HAVE_MPX_SUPPORT
 343         bndmov %bnd0,              (LR_BND_OFFSET)(%rsp)  # Preserve bound
 344         bndmov %bnd1, (LR_BND_OFFSET +   BND_SIZE)(%rsp)  # registers. Nops if
 345         bndmov %bnd2, (LR_BND_OFFSET + BND_SIZE*2)(%rsp)  # MPX not available
 346         bndmov %bnd3, (LR_BND_OFFSET + BND_SIZE*3)(%rsp)  # or disabled.
 347 #  else
 348         .byte 0x66,0x0f,0x1b,0x84,0x24;.long (LR_BND_OFFSET)
 349         .byte 0x66,0x0f,0x1b,0x8c,0x24;.long (LR_BND_OFFSET + BND_SIZE)
 350         .byte 0x66,0x0f,0x1b,0x94,0x24;.long (LR_BND_OFFSET + BND_SIZE*2)
 351         .byte 0x66,0x0f,0x1b,0x9c,0x24;.long (LR_BND_OFFSET + BND_SIZE*3)
 352 #  endif
 353 # endif
 354
 355 # ifdef RESTORE_AVX
 356         /* This is to support AVX audit modules.  */
 357         VMOVA %VEC(0),                (LR_VECTOR_OFFSET)(%rsp)
 358         VMOVA %VEC(1), (LR_VECTOR_OFFSET +   VECTOR_SIZE)(%rsp)
 359         VMOVA %VEC(2), (LR_VECTOR_OFFSET + VECTOR_SIZE*2)(%rsp)
 360         VMOVA %VEC(3), (LR_VECTOR_OFFSET + VECTOR_SIZE*3)(%rsp)
 361         VMOVA %VEC(4), (LR_VECTOR_OFFSET + VECTOR_SIZE*4)(%rsp)
 362         VMOVA %VEC(5), (LR_VECTOR_OFFSET + VECTOR_SIZE*5)(%rsp)
 363         VMOVA %VEC(6), (LR_VECTOR_OFFSET + VECTOR_SIZE*6)(%rsp)
 364         VMOVA %VEC(7), (LR_VECTOR_OFFSET + VECTOR_SIZE*7)(%rsp)
 365
 366         /* Save xmm0-xmm7 registers to detect if any of them are
 367            changed by audit module.  */
 368         vmovdqa %xmm0,              (LR_SIZE)(%rsp)
 369         vmovdqa %xmm1, (LR_SIZE +   XMM_SIZE)(%rsp)
 370         vmovdqa %xmm2, (LR_SIZE + XMM_SIZE*2)(%rsp)
 371         vmovdqa %xmm3, (LR_SIZE + XMM_SIZE*3)(%rsp)
 372         vmovdqa %xmm4, (LR_SIZE + XMM_SIZE*4)(%rsp)
 373         vmovdqa %xmm5, (LR_SIZE + XMM_SIZE*5)(%rsp)
 374         vmovdqa %xmm6, (LR_SIZE + XMM_SIZE*6)(%rsp)
 375         vmovdqa %xmm7, (LR_SIZE + XMM_SIZE*7)(%rsp)
 376 # endif
 377
 378         mov %RSP_LP, %RCX_LP    # La_x86_64_regs pointer to %rcx.
 379         mov 48(%rbx), %RDX_LP   # Load return address if needed.
 380         mov 40(%rbx), %RSI_LP   # Copy args pushed by PLT in register.
 381         mov 32(%rbx), %RDI_LP   # %rdi: link_map, %rsi: reloc_index
 382         lea 16(%rbx), %R8_LP    # Address of framesize
 383         call _dl_profile_fixup  # Call resolver.
 384
 385         mov %RAX_LP, %R11_LP    # Save return value.
 386
 387         movq 8(%rbx), %rax      # Get back register content.
 388         movq LR_RDX_OFFSET(%rsp), %rdx
 389         movq  LR_R8_OFFSET(%rsp), %r8
 390         movq  LR_R9_OFFSET(%rsp), %r9
 391
 392         movaps              (LR_XMM_OFFSET)(%rsp), %xmm0
 393         movaps   (LR_XMM_OFFSET + XMM_SIZE)(%rsp), %xmm1
 394         movaps (LR_XMM_OFFSET + XMM_SIZE*2)(%rsp), %xmm2
 395         movaps (LR_XMM_OFFSET + XMM_SIZE*3)(%rsp), %xmm3
 396         movaps (LR_XMM_OFFSET + XMM_SIZE*4)(%rsp), %xmm4
 397         movaps (LR_XMM_OFFSET + XMM_SIZE*5)(%rsp), %xmm5
 398         movaps (LR_XMM_OFFSET + XMM_SIZE*6)(%rsp), %xmm6
 399         movaps (LR_XMM_OFFSET + XMM_SIZE*7)(%rsp), %xmm7
 400
 401 # ifdef RESTORE_AVX
 402         /* Check if any xmm0-xmm7 registers are changed by audit
 403            module.  */
 404         vpcmpeqq (LR_SIZE)(%rsp), %xmm0, %xmm8
 405         vpmovmskb %xmm8, %esi
 406         cmpl $0xffff, %esi
 407         je 2f
 408         vmovdqa %xmm0, (LR_VECTOR_OFFSET)(%rsp)
 409         jmp 1f
 410 2:      VMOVA (LR_VECTOR_OFFSET)(%rsp), %VEC(0)
 411         vmovdqa %xmm0, (LR_XMM_OFFSET)(%rsp)
 412
 413 1:      vpcmpeqq (LR_SIZE + XMM_SIZE)(%rsp), %xmm1, %xmm8
 414         vpmovmskb %xmm8, %esi
 415         cmpl $0xffff, %esi
 416         je 2f
 417         vmovdqa %xmm1, (LR_VECTOR_OFFSET + VECTOR_SIZE)(%rsp)
 418         jmp 1f
 419 2:      VMOVA (LR_VECTOR_OFFSET + VECTOR_SIZE)(%rsp), %VEC(1)
 420         vmovdqa %xmm1, (LR_XMM_OFFSET + XMM_SIZE)(%rsp)
 421
 422 1:      vpcmpeqq (LR_SIZE + XMM_SIZE*2)(%rsp), %xmm2, %xmm8
 423         vpmovmskb %xmm8, %esi
 424         cmpl $0xffff, %esi
 425         je 2f
 426         vmovdqa %xmm2, (LR_VECTOR_OFFSET + VECTOR_SIZE*2)(%rsp)
 427         jmp 1f
 428 2:      VMOVA (LR_VECTOR_OFFSET + VECTOR_SIZE*2)(%rsp), %VEC(2)
 429         vmovdqa %xmm2, (LR_XMM_OFFSET + XMM_SIZE*2)(%rsp)
 430
 431 1:      vpcmpeqq (LR_SIZE + XMM_SIZE*3)(%rsp), %xmm3, %xmm8
 432         vpmovmskb %xmm8, %esi
 433         cmpl $0xffff, %esi
 434         je 2f
 435         vmovdqa %xmm3, (LR_VECTOR_OFFSET + VECTOR_SIZE*3)(%rsp)
 436         jmp 1f
 437 2:      VMOVA (LR_VECTOR_OFFSET + VECTOR_SIZE*3)(%rsp), %VEC(3)
 438         vmovdqa %xmm3, (LR_XMM_OFFSET + XMM_SIZE*3)(%rsp)
 439
 440 1:      vpcmpeqq (LR_SIZE + XMM_SIZE*4)(%rsp), %xmm4, %xmm8
 441         vpmovmskb %xmm8, %esi
 442         cmpl $0xffff, %esi
 443         je 2f
 444         vmovdqa %xmm4, (LR_VECTOR_OFFSET + VECTOR_SIZE*4)(%rsp)
 445         jmp 1f
 446 2:      VMOVA (LR_VECTOR_OFFSET + VECTOR_SIZE*4)(%rsp), %VEC(4)
 447         vmovdqa %xmm4, (LR_XMM_OFFSET + XMM_SIZE*4)(%rsp)
 448
 449 1:      vpcmpeqq (LR_SIZE + XMM_SIZE*5)(%rsp), %xmm5, %xmm8
 450         vpmovmskb %xmm8, %esi
 451         cmpl $0xffff, %esi
 452         je 2f
 453         vmovdqa %xmm5, (LR_VECTOR_OFFSET + VECTOR_SIZE*5)(%rsp)
 454         jmp 1f
 455 2:      VMOVA (LR_VECTOR_OFFSET + VECTOR_SIZE*5)(%rsp), %VEC(5)
 456         vmovdqa %xmm5, (LR_XMM_OFFSET + XMM_SIZE*5)(%rsp)
 457
 458 1:      vpcmpeqq (LR_SIZE + XMM_SIZE*6)(%rsp), %xmm6, %xmm8
 459         vpmovmskb %xmm8, %esi
 460         cmpl $0xffff, %esi
 461         je 2f
 462         vmovdqa %xmm6, (LR_VECTOR_OFFSET + VECTOR_SIZE*6)(%rsp)
 463         jmp 1f
 464 2:      VMOVA (LR_VECTOR_OFFSET + VECTOR_SIZE*6)(%rsp), %VEC(6)
 465         vmovdqa %xmm6, (LR_XMM_OFFSET + XMM_SIZE*6)(%rsp)
 466
 467 1:      vpcmpeqq (LR_SIZE + XMM_SIZE*7)(%rsp), %xmm7, %xmm8
 468         vpmovmskb %xmm8, %esi
 469         cmpl $0xffff, %esi
 470         je 2f
 471         vmovdqa %xmm7, (LR_VECTOR_OFFSET + VECTOR_SIZE*7)(%rsp)
 472         jmp 1f
 473 2:      VMOVA (LR_VECTOR_OFFSET + VECTOR_SIZE*7)(%rsp), %VEC(7)
 474         vmovdqa %xmm7, (LR_XMM_OFFSET + XMM_SIZE*7)(%rsp)
 475
 476 1:
 477 # endif
 478
 479 # ifndef __ILP32__
 480 #  ifdef HAVE_MPX_SUPPORT
 481         bndmov              (LR_BND_OFFSET)(%rsp), %bnd0  # Restore bound
 482         bndmov (LR_BND_OFFSET +   BND_SIZE)(%rsp), %bnd1  # registers.
 483         bndmov (LR_BND_OFFSET + BND_SIZE*2)(%rsp), %bnd2
 484         bndmov (LR_BND_OFFSET + BND_SIZE*3)(%rsp), %bnd3
 485 #  else
 486         .byte 0x66,0x0f,0x1a,0x84,0x24;.long (LR_BND_OFFSET)
 487         .byte 0x66,0x0f,0x1a,0x8c,0x24;.long (LR_BND_OFFSET + BND_SIZE)
 488         .byte 0x66,0x0f,0x1a,0x94,0x24;.long (LR_BND_OFFSET + BND_SIZE*2)
 489         .byte 0x66,0x0f,0x1a,0x9c,0x24;.long (LR_BND_OFFSET + BND_SIZE*3)
 490 #  endif
 491 # endif
 492
 493         mov  16(%rbx), %R10_LP  # Anything in framesize?
 494         test %R10_LP, %R10_LP
 495         PRESERVE_BND_REGS_PREFIX
 496         jns 3f
 497
 498         /* There's nothing in the frame size, so there
 499            will be no call to the _dl_call_pltexit. */
 500
 501         /* Get back registers content.  */
 502         movq LR_RCX_OFFSET(%rsp), %rcx
 503         movq LR_RSI_OFFSET(%rsp), %rsi
 504         movq LR_RDI_OFFSET(%rsp), %rdi
 505
 506         mov %RBX_LP, %RSP_LP
 507         movq (%rsp), %rbx
 508         cfi_restore(%rbx)
 509         cfi_def_cfa_register(%rsp)
 510
 511         add $48, %RSP_LP        # Adjust the stack to the return value
 512                                 # (eats the reloc index and link_map)
 513         cfi_adjust_cfa_offset(-48)
 514         PRESERVE_BND_REGS_PREFIX
 515         jmp *%r11               # Jump to function address.
 516
 517 3:
 518         cfi_adjust_cfa_offset(48)
 519         cfi_rel_offset(%rbx, 0)
 520         cfi_def_cfa_register(%rbx)
 521
 522         /* At this point we need to prepare new stack for the function
 523            which has to be called.  We copy the original stack to a
 524            temporary buffer of the size specified by the 'framesize'
 525            returned from _dl_profile_fixup */
 526
 527         lea LR_RSP_OFFSET(%rbx), %RSI_LP # stack
 528         add $8, %R10_LP
 529         and $-16, %R10_LP
 530         mov %R10_LP, %RCX_LP
 531         sub %R10_LP, %RSP_LP
 532         mov %RSP_LP, %RDI_LP
 533         shr $3, %RCX_LP
 534         rep
 535         movsq
 536
 537         movq 24(%rdi), %rcx     # Get back register content.
 538         movq 32(%rdi), %rsi
 539         movq 40(%rdi), %rdi
 540
 541         PRESERVE_BND_REGS_PREFIX
 542         call *%r11
 543
 544         mov 24(%rbx), %RSP_LP   # Drop the copied stack content
 545
 546         /* Now we have to prepare the La_x86_64_retval structure for the
 547            _dl_call_pltexit.  The La_x86_64_regs is being pointed by rsp now,
 548            so we just need to allocate the sizeof(La_x86_64_retval) space on
 549            the stack, since the alignment has already been taken care of. */
 550 # ifdef RESTORE_AVX
 551         /* sizeof(La_x86_64_retval).  Need extra space for 2 SSE
 552            registers to detect if xmm0/xmm1 registers are changed
 553            by audit module.  */
 554         sub $(LRV_SIZE + XMM_SIZE*2), %RSP_LP
 555 # else
 556         sub $LRV_SIZE, %RSP_LP  # sizeof(La_x86_64_retval)
 557 # endif
 558         mov %RSP_LP, %RCX_LP    # La_x86_64_retval argument to %rcx.
 559
 560         /* Fill in the La_x86_64_retval structure.  */
 561         movq %rax, LRV_RAX_OFFSET(%rcx)
 562         movq %rdx, LRV_RDX_OFFSET(%rcx)
 563
 564         movaps %xmm0, LRV_XMM0_OFFSET(%rcx)
 565         movaps %xmm1, LRV_XMM1_OFFSET(%rcx)
 566
 567 # ifdef RESTORE_AVX
 568         /* This is to support AVX audit modules.  */
 569         VMOVA %VEC(0), LRV_VECTOR0_OFFSET(%rcx)
 570         VMOVA %VEC(1), LRV_VECTOR1_OFFSET(%rcx)
 571
 572         /* Save xmm0/xmm1 registers to detect if they are changed
 573            by audit module.  */
 574         vmovdqa %xmm0,            (LRV_SIZE)(%rcx)
 575         vmovdqa %xmm1, (LRV_SIZE + XMM_SIZE)(%rcx)
 576 # endif
 577
 578 # ifndef __ILP32__
 579 #  ifdef HAVE_MPX_SUPPORT
 580         bndmov %bnd0, LRV_BND0_OFFSET(%rcx)  # Preserve returned bounds.
 581         bndmov %bnd1, LRV_BND1_OFFSET(%rcx)
 582 #  else
 583         .byte  0x66,0x0f,0x1b,0x81;.long (LRV_BND0_OFFSET)
 584         .byte  0x66,0x0f,0x1b,0x89;.long (LRV_BND1_OFFSET)
 585 #  endif
 586 # endif
 587
 588         fstpt LRV_ST0_OFFSET(%rcx)
 589         fstpt LRV_ST1_OFFSET(%rcx)
 590
 591         movq 24(%rbx), %rdx     # La_x86_64_regs argument to %rdx.
 592         movq 40(%rbx), %rsi     # Copy args pushed by PLT in register.
 593         movq 32(%rbx), %rdi     # %rdi: link_map, %rsi: reloc_index
 594         call _dl_call_pltexit
 595
 596         /* Restore return registers.  */
 597         movq LRV_RAX_OFFSET(%rsp), %rax
 598         movq LRV_RDX_OFFSET(%rsp), %rdx
 599
 600         movaps LRV_XMM0_OFFSET(%rsp), %xmm0
 601         movaps LRV_XMM1_OFFSET(%rsp), %xmm1
 602
 603 # ifdef RESTORE_AVX
 604         /* Check if xmm0/xmm1 registers are changed by audit module.  */
 605         vpcmpeqq (LRV_SIZE)(%rsp), %xmm0, %xmm2
 606         vpmovmskb %xmm2, %esi
 607         cmpl $0xffff, %esi
 608         jne 1f
 609         VMOVA LRV_VECTOR0_OFFSET(%rsp), %VEC(0)
 610
 611 1:      vpcmpeqq (LRV_SIZE + XMM_SIZE)(%rsp), %xmm1, %xmm2
 612         vpmovmskb %xmm2, %esi
 613         cmpl $0xffff, %esi
 614         jne 1f
 615         VMOVA LRV_VECTOR1_OFFSET(%rsp), %VEC(1)
 616
 617 1:
 618 # endif
 619
 620 # ifndef __ILP32__
 621 #  ifdef HAVE_MPX_SUPPORT
 622         bndmov LRV_BND0_OFFSET(%rsp), %bnd0  # Restore bound registers.
 623         bndmov LRV_BND1_OFFSET(%rsp), %bnd1
 624 #  else
 625         .byte  0x66,0x0f,0x1a,0x84,0x24;.long (LRV_BND0_OFFSET)
 626         .byte  0x66,0x0f,0x1a,0x8c,0x24;.long (LRV_BND1_OFFSET)
 627 #  endif
 628 # endif
 629
 630         fldt LRV_ST1_OFFSET(%rsp)
 631         fldt LRV_ST0_OFFSET(%rsp)
 632
 633         mov %RBX_LP, %RSP_LP
 634         movq (%rsp), %rbx
 635         cfi_restore(%rbx)
 636         cfi_def_cfa_register(%rsp)
 637
 638         add $48, %RSP_LP        # Adjust the stack to the return value
 639                                 # (eats the reloc index and link_map)
 640         cfi_adjust_cfa_offset(-48)
 641         PRESERVE_BND_REGS_PREFIX
 642         retq
 643
 644         cfi_endproc
 645         .size _dl_runtime_profile, .-_dl_runtime_profile
 646 #endif