sysdeps/powerpc/powerpc64/sysdep.h

   1 /* Assembly macros for 64-bit PowerPC.
   2    Copyright (C) 2002-2024 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 #include <sysdeps/powerpc/sysdep.h>
  20 #include <tls.h>
  21
  22 #ifdef __ASSEMBLER__
  23
  24 /* Stack frame offsets.  */
  25 #define FRAME_BACKCHAIN         0
  26 #define FRAME_CR_SAVE           8
  27 #define FRAME_LR_SAVE           16
  28 #if _CALL_ELF != 2
  29 #define FRAME_MIN_SIZE          112
  30 #define FRAME_MIN_SIZE_PARM     112
  31 #define FRAME_TOC_SAVE          40
  32 #define FRAME_PARM_SAVE         48
  33 #else
  34 #define FRAME_MIN_SIZE          32
  35 #define FRAME_MIN_SIZE_PARM     96
  36 #define FRAME_TOC_SAVE          24
  37 #define FRAME_PARM_SAVE         32
  38 #endif
  39
  40 /* Support macros for CALL_MCOUNT.  */
  41         .macro SAVE_ARG NARG
  42         .if \NARG
  43         SAVE_ARG \NARG-1
  44         std     2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG)(1)
  45         .endif
  46         .endm
  47
  48         .macro REST_ARG NARG
  49         .if \NARG
  50         REST_ARG \NARG-1
  51         ld      2+\NARG,FRAME_PARM_SAVE-8+8*(\NARG)(1)
  52         .endif
  53         .endm
  54
  55         .macro CFI_SAVE_ARG NARG
  56         .if \NARG
  57         CFI_SAVE_ARG \NARG-1
  58         cfi_offset(2+\NARG,-FRAME_MIN_SIZE_PARM+FRAME_PARM_SAVE-8+8*(\NARG))
  59         .endif
  60         .endm
  61
  62         .macro CFI_REST_ARG NARG
  63         .if \NARG
  64         CFI_REST_ARG \NARG-1
  65         cfi_restore(2+\NARG)
  66         .endif
  67         .endm
  68
  69 /* If compiled for profiling, call `_mcount' at the start of each function.
  70    see ppc-mcount.S for more details.  */
  71         .macro CALL_MCOUNT NARG
  72 #ifdef  PROF
  73         mflr    r0
  74         SAVE_ARG \NARG
  75         std     r0,FRAME_LR_SAVE(r1)
  76         stdu    r1,-FRAME_MIN_SIZE_PARM(r1)
  77         cfi_adjust_cfa_offset(FRAME_MIN_SIZE_PARM)
  78         cfi_offset(lr,FRAME_LR_SAVE)
  79         CFI_SAVE_ARG \NARG
  80         bl      JUMPTARGET (_mcount)
  81 #ifndef SHARED
  82         nop
  83 #endif
  84         ld      r0,FRAME_MIN_SIZE_PARM+FRAME_LR_SAVE(r1)
  85         REST_ARG \NARG
  86         mtlr    r0
  87         addi    r1,r1,FRAME_MIN_SIZE_PARM
  88         cfi_adjust_cfa_offset(-FRAME_MIN_SIZE_PARM)
  89         cfi_restore(lr)
  90         CFI_REST_ARG \NARG
  91 #endif
  92         .endm
  93
  94 #if _CALL_ELF != 2
  95
  96 /* Macro to prepare for calling via a function pointer.  */
  97         .macro PPC64_LOAD_FUNCPTR PTR
  98         ld      r12,0(\PTR)
  99         ld      r2,8(\PTR)
 100         mtctr   r12
 101         ld      r11,16(\PTR)
 102         .endm
 103
 104 #ifdef USE_PPC64_OVERLAPPING_OPD
 105 # define OPD_ENT(name)  .quad BODY_LABEL (name), .TOC.@tocbase
 106 #else
 107 # define OPD_ENT(name)  .quad BODY_LABEL (name), .TOC.@tocbase, 0
 108 #endif
 109
 110 #define ENTRY_1(name)                           \
 111         .type BODY_LABEL(name),@function;       \
 112         .globl name;                            \
 113         .section ".opd","aw";                   \
 114         .p2align 3;FUNC_LABEL(name):            \
 115         OPD_ENT (name);                         \
 116         .previous
 117
 118 #define FUNC_LABEL(X) X
 119 #define BODY_LABEL(X) .LY##X
 120 #define ENTRY_2(name)                           \
 121         .type name,@function;                   \
 122         ENTRY_1(name)
 123 #define END_2(name)                             \
 124         .size name,.-BODY_LABEL(name);          \
 125         .size BODY_LABEL(name),.-BODY_LABEL(name)
 126 #define LOCALENTRY(name)
 127
 128 #else /* _CALL_ELF == 2 */
 129
 130 /* Macro to prepare for calling via a function pointer.  */
 131         .macro PPC64_LOAD_FUNCPTR PTR
 132         mr      r12,\PTR
 133         mtctr   r12
 134         .endm
 135
 136 #define FUNC_LABEL(X) X
 137 #define BODY_LABEL(X) X
 138 #define ENTRY_2(name)                           \
 139         .globl name;                            \
 140         .type name,@function
 141 #define END_2(name)                             \
 142         .size name,.-name
 143 #define LOCALENTRY(name)                        \
 144 1:      addis   r2,r12,.TOC.-1b@ha;             \
 145         addi    r2,r2,.TOC.-1b@l;               \
 146         .localentry name,.-name
 147
 148 #endif /* _CALL_ELF */
 149
 150         .macro NOPS NARG
 151         .if \NARG
 152         NOPS \NARG-1
 153         nop
 154         .endif
 155         .endm
 156
 157         .macro ENTRY_3 name, alignp2=2, nopwords=0
 158         .text
 159         ENTRY_2(\name)
 160         .p2align \alignp2
 161         NOPS \nopwords
 162 BODY_LABEL(\name):
 163         .endm
 164
 165 /* Use ENTRY_TOCLESS for functions that make no use of r2 and
 166    guarantee r2 is unchanged on exit.  Any function that has @toc or
 167    @got relocs uses r2.  Functions that call other functions via the
 168    PLT use r2.  Use ENTRY for functions that may use or change r2.
 169    The first argument is the function name.
 170    The optional second argument specifies alignment of the function's
 171    code, as the logarithm base two of the byte alignment.  For
 172    example, a value of four aligns to a sixteen byte boundary.
 173    The optional third argument specifies the number of NOPs to emit
 174    before the start of the function's code.   */
 175 #ifndef PROF
 176 #define ENTRY_TOCLESS(name, ...)                \
 177         ENTRY_3 name, ## __VA_ARGS__;           \
 178         cfi_startproc
 179
 180 #define ENTRY(name, ...)                        \
 181         ENTRY_TOCLESS(name, ## __VA_ARGS__);    \
 182         LOCALENTRY(name)
 183 #else
 184 /* The call to _mcount is potentially via the plt, so profiling code
 185    is never free of an r2 use.  */
 186 #define ENTRY_TOCLESS(name, ...)                \
 187         ENTRY_3 name, ## __VA_ARGS__;           \
 188         cfi_startproc;                          \
 189         LOCALENTRY(name)
 190
 191 #define ENTRY(name, ...)                        \
 192         ENTRY_TOCLESS(name, ## __VA_ARGS__)
 193 #endif
 194
 195 /* Local labels stripped out by the linker.  */
 196 #undef L
 197 #define L(x) .L##x
 198
 199 #define tostring(s) #s
 200 #define stringify(s) tostring(s)
 201 #define XGLUE(a,b) a##b
 202 #define GLUE(a,b) XGLUE(a,b)
 203 #define LT_LABEL(name) GLUE(.LT,name)
 204 #define LT_LABELSUFFIX(name,suffix) GLUE(GLUE(.LT,name),suffix)
 205
 206 /* Support Traceback tables */
 207 #define TB_ASM                  0x000c000000000000
 208 #define TB_GLOBALLINK           0x0000800000000000
 209 #define TB_IS_EPROL             0x0000400000000000
 210 #define TB_HAS_TBOFF            0x0000200000000000
 211 #define TB_INT_PROC             0x0000100000000000
 212 #define TB_HAS_CTL              0x0000080000000000
 213 #define TB_TOCLESS              0x0000040000000000
 214 #define TB_FP_PRESENT           0x0000020000000000
 215 #define TB_LOG_ABORT            0x0000010000000000
 216 #define TB_INT_HANDL            0x0000008000000000
 217 #define TB_NAME_PRESENT         0x0000004000000000
 218 #define TB_USES_ALLOCA          0x0000002000000000
 219 #define TB_SAVES_CR             0x0000000200000000
 220 #define TB_SAVES_LR             0x0000000100000000
 221 #define TB_STORES_BC            0x0000000080000000
 222 #define TB_FIXUP                0x0000000040000000
 223 #define TB_FP_SAVED(fprs)       (((fprs) & 0x3f) << 24)
 224 #define TB_GPR_SAVED(gprs)      (((fprs) & 0x3f) << 16)
 225 #define TB_FIXEDPARMS(parms)    (((parms) & 0xff) << 8)
 226 #define TB_FLOATPARMS(parms)    (((parms) & 0x7f) << 1)
 227 #define TB_PARMSONSTK           0x0000000000000001
 228
 229 #define PPC_HIGHER(v)           (((v) >> 32) & 0xffff)
 230 #define TB_DEFAULT              TB_ASM | TB_HAS_TBOFF | TB_NAME_PRESENT
 231
 232 #define TRACEBACK(name) \
 233 LT_LABEL(name): ; \
 234         .long   0 ; \
 235         .quad   TB_DEFAULT ; \
 236         .long   LT_LABEL(name)-BODY_LABEL(name) ; \
 237         .short  LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
 238 LT_LABELSUFFIX(name,_name_start): ;\
 239         .ascii  stringify(name) ; \
 240 LT_LABELSUFFIX(name,_name_end): ; \
 241         .p2align 2
 242
 243 #define TRACEBACK_MASK(name,mask) \
 244 LT_LABEL(name): ; \
 245         .long   0 ; \
 246         .quad   TB_DEFAULT | mask ; \
 247         .long   LT_LABEL(name)-BODY_LABEL(name) ; \
 248         .short  LT_LABELSUFFIX(name,_name_end)-LT_LABELSUFFIX(name,_name_start) ; \
 249 LT_LABELSUFFIX(name,_name_start): ;\
 250         .ascii  stringify(name) ; \
 251 LT_LABELSUFFIX(name,_name_end): ; \
 252         .p2align 2
 253
 254 /* END generates Traceback tables */
 255 #undef  END
 256 #define END(name) \
 257   cfi_endproc;                  \
 258   TRACEBACK(name);              \
 259   END_2(name)
 260
 261 /* This form supports more informative traceback tables */
 262 #define END_GEN_TB(name,mask)   \
 263   cfi_endproc;                  \
 264   TRACEBACK_MASK(name,mask);    \
 265   END_2(name)
 266
 267 /* We will allocate a new frame to save LR and the non-volatile register used to
 268    read the TCB when checking for scv support on syscall code.  We actually just
 269    need the minimum frame size plus room for 1 reg (8 bytes).  But the ABI
 270    mandates stack frames should be aligned at 16 Bytes, so we end up allocating
 271    a bit more space then what will actually be used.  */
 272 #define SCV_FRAME_SIZE (FRAME_MIN_SIZE+16)
 273 #define SCV_FRAME_NVOLREG_SAVE FRAME_MIN_SIZE
 274
 275 /* Allocate frame and save register */
 276 #define NVOLREG_SAVE \
 277     stdu r1,-SCV_FRAME_SIZE(r1); \
 278     cfi_adjust_cfa_offset(SCV_FRAME_SIZE); \
 279     std r31,SCV_FRAME_NVOLREG_SAVE(r1); \
 280     cfi_rel_offset(r31,SCV_FRAME_NVOLREG_SAVE);
 281
 282 /* Restore register and destroy frame */
 283 #define NVOLREG_RESTORE \
 284     ld r31,SCV_FRAME_NVOLREG_SAVE(r1); \
 285     cfi_restore(r31); \
 286     addi r1,r1,SCV_FRAME_SIZE; \
 287     cfi_adjust_cfa_offset(-SCV_FRAME_SIZE);
 288
 289 /* Check PPC_FEATURE2_SCV bit from hwcap2 in the TCB.  If it is not set, scv is
 290    not available, then go to JUMPFALSE (label given by the macro's caller).  We
 291    save the value we read from the TCB in a non-volatile register so we can
 292    reuse it later when exiting from the syscall in PSEUDO_RET.  Note that for
 293    the static case we need an extra check to guarantee the thread pointer has
 294    already been initialized, otherwise we may try to access an invalid address
 295    if a syscall is called before the TLS has been setup.  */
 296     .macro CHECK_SCV_SUPPORT REG JUMPFALSE
 297
 298 #ifndef SHARED
 299     /* Check if thread pointer has already been setup.  */
 300     cmpdi r13,0
 301     beq \JUMPFALSE
 302 #endif
 303
 304     /* Read PPC_FEATURE2_SCV from TCB and store it in REG */
 305     ld \REG,TCB_HWCAP(PT_THREAD_POINTER)
 306     andis. \REG,\REG,PPC_FEATURE2_SCV>>16
 307
 308     beq \JUMPFALSE
 309     .endm
 310
 311 #if !defined(USE_PPC_SCV) || IS_IN(rtld)
 312 # define DO_CALL(syscall) \
 313     li r0,syscall; \
 314     DO_CALL_SC
 315 #else
 316 /* Before doing the syscall, check if we can use scv.  scv is supported by P9
 317    and later with Linux v5.9 and later.  If so, use it.  Otherwise, fallback to
 318    sc.  We use a non-volatile register to save hwcap2 from the TCB, so we need
 319    to save its content beforehand.  */
 320 # define DO_CALL(syscall) \
 321     li r0,syscall; \
 322     NVOLREG_SAVE; \
 323     CHECK_SCV_SUPPORT r31 0f; \
 324     DO_CALL_SCV; \
 325     b 1f; \
 326 0:  DO_CALL_SC; \
 327 1:
 328 #endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */
 329
 330 /* DO_CALL_SC and DO_CALL_SCV expect the syscall number to be in r0.  */
 331 #define DO_CALL_SC \
 332     sc
 333
 334 #define DO_CALL_SCV \
 335     mflr r9; \
 336     std r9,SCV_FRAME_SIZE+FRAME_LR_SAVE(r1);   \
 337     cfi_rel_offset(lr,SCV_FRAME_SIZE+FRAME_LR_SAVE); \
 338     .machine "push"; \
 339     .machine "power9"; \
 340     scv 0; \
 341     .machine "pop"; \
 342     ld r9,SCV_FRAME_SIZE+FRAME_LR_SAVE(r1);      \
 343     mtlr r9; \
 344     cfi_restore(lr);
 345
 346 /* ppc64 is always PIC */
 347 #undef JUMPTARGET
 348 #define JUMPTARGET(name) FUNC_LABEL(name)
 349
 350 #define PSEUDO(name, syscall_name, args) \
 351   .section ".text";                             \
 352   ENTRY (name);                                 \
 353   DO_CALL (SYS_ify (syscall_name))
 354
 355 #ifdef SHARED
 356 #define TAIL_CALL_SYSCALL_ERROR \
 357     b JUMPTARGET (NOTOC (__syscall_error))
 358 #else
 359 /* Static version might be linked into a large app with a toc exceeding
 360    64k.  We can't put a toc adjusting stub on a plain branch, so can't
 361    tail call __syscall_error.  */
 362 #define TAIL_CALL_SYSCALL_ERROR \
 363     .ifdef .Local_syscall_error; \
 364     b .Local_syscall_error; \
 365     .else; \
 366 .Local_syscall_error: \
 367     mflr 0; \
 368     std 0,FRAME_LR_SAVE(1); \
 369     stdu 1,-FRAME_MIN_SIZE(1); \
 370     cfi_adjust_cfa_offset(FRAME_MIN_SIZE); \
 371     cfi_offset(lr,FRAME_LR_SAVE); \
 372     bl JUMPTARGET(__syscall_error); \
 373     nop; \
 374     ld 0,FRAME_MIN_SIZE+FRAME_LR_SAVE(1); \
 375     addi 1,1,FRAME_MIN_SIZE; \
 376     cfi_adjust_cfa_offset(-FRAME_MIN_SIZE); \
 377     mtlr 0; \
 378     cfi_restore(lr); \
 379     blr; \
 380     .endif
 381 #endif
 382
 383 #if !defined(USE_PPC_SCV) || IS_IN(rtld)
 384 # define PSEUDO_RET \
 385     RET_SC; \
 386     TAIL_CALL_SYSCALL_ERROR
 387 #else
 388 /* This should only be called after a DO_CALL.  In such cases, r31 contains the
 389    value of PPC_FEATURE2_SCV read from hwcap2 by CHECK_SCV_SUPPORT.  If it is
 390    set, we know we have entered the kernel using scv, so handle the return code
 391    accordingly.  */
 392 # define PSEUDO_RET \
 393     cmpdi cr5,r31,0; \
 394     NVOLREG_RESTORE; \
 395     beq cr5,0f; \
 396     RET_SCV; \
 397     b 1f; \
 398 0:  RET_SC; \
 399 1:  TAIL_CALL_SYSCALL_ERROR
 400 #endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */
 401
 402 #define RET_SCV \
 403     li r9,-4095; \
 404     cmpld r3,r9; \
 405     bltlr+; \
 406     neg r3,r3;
 407
 408 #define RET_SC \
 409     bnslr+;
 410
 411 #define ret PSEUDO_RET
 412
 413 #undef  PSEUDO_END
 414 #define PSEUDO_END(name) \
 415   END (name)
 416
 417 #define PSEUDO_NOERRNO(name, syscall_name, args) \
 418   .section ".text";                                     \
 419   ENTRY (name);                                         \
 420   DO_CALL (SYS_ify (syscall_name))
 421
 422 #if !defined(USE_PPC_SCV) || IS_IN(rtld)
 423 # define PSEUDO_RET_NOERRNO \
 424     blr
 425 #else
 426 /* This should only be called after a DO_CALL.  */
 427 # define PSEUDO_RET_NOERRNO \
 428     NVOLREG_RESTORE; \
 429     blr
 430 #endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */
 431
 432 #define ret_NOERRNO PSEUDO_RET_NOERRNO
 433
 434 #undef  PSEUDO_END_NOERRNO
 435 #define PSEUDO_END_NOERRNO(name) \
 436   END (name)
 437
 438 #define PSEUDO_ERRVAL(name, syscall_name, args) \
 439   .section ".text";                                     \
 440   ENTRY (name);                                         \
 441   DO_CALL (SYS_ify (syscall_name))
 442
 443 #if !defined(USE_PPC_SCV) || IS_IN(rtld)
 444 # define PSEUDO_RET_ERRVAL \
 445     blr
 446 #else
 447 /* This should only be called after a DO_CALL.  */
 448 # define PSEUDO_RET_ERRVAL \
 449     NVOLREG_RESTORE; \
 450     blr
 451 #endif /* !defined(USE_PPC_SCV) || IS_IN(rtld) */
 452
 453 #define ret_ERRVAL PSEUDO_RET_ERRVAL
 454
 455 #undef  PSEUDO_END_ERRVAL
 456 #define PSEUDO_END_ERRVAL(name) \
 457   END (name)
 458
 459 #ifdef SHARED
 460 # if IS_IN (rtld)
 461          /* Inside ld.so we use the local alias to avoid runtime GOT
 462             relocations.  */
 463 #  define __GLRO_DEF(var)                               \
 464 .LC__ ## var:                                           \
 465         .tc _rtld_local_ro[TC],_rtld_local_ro
 466 # else
 467 #  define __GLRO_DEF(var)                               \
 468 .LC__ ## var:                                           \
 469         .tc _rtld_global_ro[TC],_rtld_global_ro
 470 # endif
 471 # define __GLRO(rOUT, var, offset)              \
 472         addis   rOUT,r2,.LC__ ## var@toc@ha;    \
 473         ld      rOUT,.LC__ ## var@toc@l(rOUT);  \
 474         lwz     rOUT,offset(rOUT)
 475 #else
 476 # define __GLRO_DEF(var)                        \
 477 .LC__ ## var:                                   \
 478         .tc _ ## var[TC],_ ## var
 479 # define __GLRO(rOUT, var, offset)              \
 480         addis   rOUT,r2,.LC__ ## var@toc@ha;    \
 481         ld      rOUT,.LC__ ## var@toc@l(rOUT);  \
 482         lwz     rOUT,0(rOUT)
 483 #endif
 484
 485 #ifdef USE_PPC64_NOTOC
 486 # define NOTOC(l) l@notoc
 487 #else
 488 # define NOTOC(l) l
 489 #endif
 490
 491 #else /* !__ASSEMBLER__ */
 492
 493 #if _CALL_ELF != 2
 494
 495 #define PPC64_LOAD_FUNCPTR(ptr) \
 496         "ld     12,0(" #ptr ")\n"                                       \
 497         "ld     2,8(" #ptr ")\n"                                        \
 498         "mtctr  12\n"                                                   \
 499         "ld     11,16(" #ptr ")"
 500
 501 #ifdef USE_PPC64_OVERLAPPING_OPD
 502 # define OPD_ENT(name)  ".quad " BODY_PREFIX #name ", .TOC.@tocbase"
 503 #else
 504 # define OPD_ENT(name)  ".quad " BODY_PREFIX #name ", .TOC.@tocbase, 0"
 505 #endif
 506
 507 #define ENTRY_1(name)   \
 508         ".type   " BODY_PREFIX #name ",@function\n"                     \
 509         ".globl " #name "\n"                                            \
 510         ".pushsection \".opd\",\"aw\"\n"                                \
 511         ".p2align 3\n"                                                  \
 512 #name ":\n"                                                             \
 513         OPD_ENT (name) "\n"                                             \
 514         ".popsection"
 515
 516 #define DOT_PREFIX ""
 517 #define BODY_PREFIX ".LY"
 518 #define ENTRY_2(name)   \
 519         ".type " #name ",@function\n"                                   \
 520         ENTRY_1(name)
 521 #define END_2(name)     \
 522         ".size " #name ",.-" BODY_PREFIX #name "\n"                     \
 523         ".size " BODY_PREFIX #name ",.-" BODY_PREFIX #name
 524 #define LOCALENTRY(name)
 525
 526 #else /* _CALL_ELF */
 527
 528 #define PPC64_LOAD_FUNCPTR(ptr) \
 529         "mr     12," #ptr "\n"                                          \
 530         "mtctr  12"
 531
 532 #define DOT_PREFIX ""
 533 #define BODY_PREFIX ""
 534 #define ENTRY_2(name)   \
 535         ".type " #name ",@function\n"                                   \
 536         ".globl " #name
 537 #define END_2(name)     \
 538         ".size " #name ",.-" #name
 539 #define LOCALENTRY(name)        \
 540         "1: addis 2,12,.TOC.-1b@ha\n"                                   \
 541         "addi   2,2,.TOC.-1b@l\n"                                       \
 542         ".localentry " #name ",.-" #name
 543
 544 #endif /* _CALL_ELF */
 545
 546 #endif  /* __ASSEMBLER__ */