libatomic/libatomic_i.h

   1 /* Copyright (C) 2012-2024 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Atomic Library (libatomic).
   5
   6    Libatomic is free software; you can redistribute it and/or modify it
   7    under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3, or (at your option)
   9    any later version.
  10
  11    Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
  12    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  13    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14    more details.
  15
  16    Under Section 7 of GPL version 3, you are granted additional
  17    permissions described in the GCC Runtime Library Exception, version
  18    3.1, as published by the Free Software Foundation.
  19
  20    You should have received a copy of the GNU General Public License and
  21    a copy of the GCC Runtime Library Exception along with this program;
  22    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  23    <http://www.gnu.org/licenses/>.  */
  24
  25 /* This file contains data types and function declarations that are
  26    private to the implementation of libatomic.  */
  27
  28 #ifndef LIBATOMIC_H
  29 #define LIBATOMIC_H 1
  30
  31 #include "auto-config.h"
  32 #include <stdbool.h>
  33 #include <stdint.h>
  34 #include <stddef.h>
  35 #include <limits.h>
  36 #include <string.h>
  37
  38
  39 /* Symbol concatenation macros.  */
  40 #define C2_(X,Y)        X ## Y
  41 #define C2(X,Y)         C2_(X,Y)
  42 #define C3_(X,Y,Z)      X ## Y ## Z
  43 #define C3(X,Y,Z)       C3_(X,Y,Z)
  44 #define C4_(W,X,Y,Z)    W ## X ## Y ## Z
  45 #define C4(W,X,Y,Z)     C4_(W,X,Y,Z)
  46
  47 /* Stringification macros.  */
  48 #define S2(X)           #X
  49 #define S(X)            S2(X)
  50
  51 /* All of the primitive types on which we operate.  */
  52 typedef unsigned U_1 __attribute__((mode(QI)));
  53 #if HAVE_INT2
  54 typedef unsigned U_2 __attribute__((mode(HI)));
  55 #endif
  56 #if HAVE_INT4
  57 typedef unsigned U_4 __attribute__((mode(SI)));
  58 #endif
  59 #if HAVE_INT8
  60 typedef unsigned U_8 __attribute__((mode(DI)));
  61 #endif
  62 #if HAVE_INT16
  63 typedef unsigned U_16 __attribute__((mode(TI)));
  64 #endif
  65
  66 /* The widest type that we support.  */
  67 #if HAVE_INT16
  68 # define MAX_SIZE       16
  69 #elif HAVE_INT8
  70 # define MAX_SIZE       8
  71 #elif HAVE_INT4
  72 # define MAX_SIZE       4
  73 #elif HAVE_INT2
  74 # define MAX_SIZE       2
  75 #else
  76 # define MAX_SIZE       1
  77 #endif
  78 typedef C2(U_,MAX_SIZE) U_MAX;
  79
  80 /* Provide dummy fallback types so that stuff is syntactically correct
  81    without having to overdo the ifdefs.  The code using these should
  82    always be protected with the HAVE_INT{n} macros.  */
  83 #if !HAVE_INT2
  84 typedef U_MAX U_2;
  85 #endif
  86 #if !HAVE_INT4
  87 typedef U_MAX U_4;
  88 #endif
  89 #if !HAVE_INT8
  90 typedef U_MAX U_8;
  91 #endif
  92 #if !HAVE_INT16
  93 typedef U_MAX U_16;
  94 #endif
  95
  96 union max_size_u
  97 {
  98   U_1 b[MAX_SIZE];
  99   U_2 i2;
 100   U_4 i4;
 101   U_8 i8;
 102   U_16 i16;
 103 };
 104
 105 /* The "word" size of the machine.  */
 106 typedef unsigned UWORD __attribute__((mode(word)));
 107
 108 /* Macros for handing sub-word sized quantities.  */
 109 #define MASK_1          ((UWORD)0xff)
 110 #define MASK_2          ((UWORD)0xffff)
 111 #define MASK_4          ((UWORD)0xffffffff)
 112 #define MASK_8          ((UWORD)0xffffffffffffffff)
 113 #define INVERT_MASK_1   ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 1) * CHAR_BIT))
 114 #define INVERT_MASK_2   ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 2) * CHAR_BIT))
 115 #define INVERT_MASK_4   ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 4) * CHAR_BIT))
 116 #define INVERT_MASK_8   ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 8) * CHAR_BIT))
 117
 118 /* Most of the files in this library are compiled multiple times with
 119    N defined to be a power of 2 between 1 and 16.  The SIZE macro is
 120    then used to append _N to the symbol being manipulated.  */
 121 #define SIZE(X)         C3(X,_,N)
 122 #define WSIZE(X)        C3(X,_,WORDSIZE)
 123 #define PTR(N,X)        ((C2(U_,N) *)X)
 124
 125 /* And thus, the type on which this compilation will be operating.  */
 126 #define ITYPE           SIZE(I)
 127 #define UTYPE           SIZE(U)
 128
 129 /* Utility macros for GCC attributes.  */
 130 #define UNUSED          __attribute__((unused))
 131 #ifdef HAVE_ATTRIBUTE_VISIBILITY
 132 # define HIDDEN         __attribute__((visibility("hidden")))
 133 #else
 134 # define HIDDEN
 135 #endif
 136
 137 /* Occasionally we have to play games with internal and external symbol
 138    names, in order to work around builtin functions of the same name.
 139    This macro sets the external name of the function appropriately.  */
 140 #define ASMNAME(X)      __asm__(S(C2(__USER_LABEL_PREFIX__,X)))
 141
 142 /* Locking for a "small" operation.  In the bare-metal single processor
 143    cases this could be implemented by disabling interrupts.  Thus the extra
 144    word passed between the two functions, saving the interrupt level.
 145    It is assumed that the object being locked does not cross the locking
 146    granularity.
 147
 148    Not actually declared here so that they can be defined static inline
 149    in a target-specfic <host-config.h>.
 150
 151 UWORD protect_start (void *ptr);
 152 void protect_end (void *ptr, UWORD);
 153 */
 154
 155 /* Locking for a "large' operation.  This should always be some sort of
 156    test-and-set operation, as we assume that the interrupt latency would
 157    be unreasonably large.  */
 158 void libat_lock_n (void *ptr, size_t n);
 159 void libat_unlock_n (void *ptr, size_t n);
 160
 161 /* We'll need to declare all of the sized functions a few times...  */
 162 #define DECLARE_ALL_SIZED(N)  DECLARE_ALL_SIZED_(N,C2(U_,N))
 163 #define DECLARE_ALL_SIZED_(N,T)                                         \
 164   DECLARE_1(T,    C2(load_,N), (T *mptr, int));                         \
 165   DECLARE_1(void, C2(store_,N), (T *mptr, T val, int));                 \
 166   DECLARE_1(T,    C2(exchange_,N), (T *mptr, T, int));                  \
 167   DECLARE_1(bool, C2(compare_exchange_,N), (T *mptr, T *, T, int, int)); \
 168   DECLARE_1(bool, C2(test_and_set_,N), (T *mptr, int));                 \
 169   DECLARE_1(T,    C2(fetch_add_,N), (T *mptr, T, int));                 \
 170   DECLARE_1(T,    C2(fetch_sub_,N), (T *mptr, T, int));                 \
 171   DECLARE_1(T,    C2(fetch_and_,N), (T *mptr, T, int));                 \
 172   DECLARE_1(T,    C2(fetch_xor_,N), (T *mptr, T, int));                 \
 173   DECLARE_1(T,    C2(fetch_or_,N), (T *mptr, T, int));                  \
 174   DECLARE_1(T,    C2(fetch_nand_,N), (T *mptr, T, int));                \
 175   DECLARE_1(T,    C2(add_fetch_,N), (T *mptr, T, int));                 \
 176   DECLARE_1(T,    C2(sub_fetch_,N), (T *mptr, T, int));                 \
 177   DECLARE_1(T,    C2(and_fetch_,N), (T *mptr, T, int));                 \
 178   DECLARE_1(T,    C2(xor_fetch_,N), (T *mptr, T, int));                 \
 179   DECLARE_1(T,    C2(or_fetch_,N), (T *mptr, T, int));                  \
 180   DECLARE_1(T,    C2(nand_fetch_,N), (T *mptr, T, int))
 181
 182 /* All sized operations are implemented in hidden functions prefixed with
 183    "libat_".  These are either renamed or aliased to the expected prefix
 184    of "__atomic".  Some amount of renaming is required to avoid hiding or
 185    conflicting with the builtins of the same name, but this additional
 186    use of hidden symbols (where appropriate) avoids unnecessary PLT entries
 187    on relevant targets.  */
 188
 189 #if IFUNC_ALT
 190 # define MAN(X)                 ASMNAME(C4(libat_,X,_i,IFUNC_ALT)) HIDDEN
 191 #elif defined(HAVE_ATTRIBUTE_ALIAS)
 192 # define MAN(X)                 HIDDEN
 193 #else
 194 # define MAN(X)                 ASMNAME(C2(__atomic_,X))
 195 #endif
 196
 197 #if !defined(N) && HAVE_IFUNC
 198 # define DECLARE_1(RET,NAME,ARGS) \
 199         RET C2(libat_,NAME) ARGS MAN(NAME); \
 200         RET C2(ifunc_,NAME) ARGS ASMNAME(C2(__atomic_,NAME))
 201 #else
 202 # define DECLARE_1(RET,NAME,ARGS)       RET C2(libat_,NAME) ARGS MAN(NAME)
 203 #endif
 204
 205 /* Prefix to use when calling internal, possibly ifunc'ed functions.  */
 206 #if HAVE_IFUNC
 207 # define local_ ifunc_
 208 #else
 209 # define local_ libat_
 210 #endif
 211
 212 DECLARE_ALL_SIZED(1);
 213 DECLARE_ALL_SIZED(2);
 214 DECLARE_ALL_SIZED(4);
 215 DECLARE_ALL_SIZED(8);
 216 DECLARE_ALL_SIZED(16);
 217
 218 #undef DECLARE_1
 219 #undef DECLARE_ALL_SIZED
 220 #undef DECLARE_ALL_SIZED_
 221
 222 /* And the generic sized versions.  */
 223 void libat_load (size_t, void *, void *, int) MAN(load);
 224 void libat_store (size_t, void *, void *, int) MAN(store);
 225 void libat_exchange (size_t, void *, void *, void *, int) MAN(exchange);
 226 bool libat_compare_exchange (size_t, void *, void *, void *, int, int)
 227         MAN(compare_exchange);
 228 bool libat_is_lock_free (size_t, void *) MAN(is_lock_free);
 229
 230 #undef MAN
 231
 232 #include <host-config.h>
 233
 234 /* We don't have IFUNC_NCOND until after host-config.h.  */
 235 #if !HAVE_IFUNC
 236 # define IFUNC_NCOND(N) 0
 237 #endif
 238
 239 #if IFUNC_ALT
 240 # define EXPORT_ALIAS(X)        /* exported symbol in non-alternate file */
 241 #elif defined(N) && IFUNC_NCOND(N)
 242 # if IFUNC_NCOND(N) == 1
 243 #  define GEN_SELECTOR(X)                                       \
 244         extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN;    \
 245         static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \
 246         {                                                       \
 247           if (IFUNC_COND_1)                                     \
 248             return C3(libat_,X,_i1);                            \
 249           return C2(libat_,X);                                  \
 250         }
 251 # elif IFUNC_NCOND(N) == 2
 252 #  define GEN_SELECTOR(X)                                       \
 253         extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN;    \
 254         extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN;    \
 255         static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \
 256         {                                                       \
 257           if (IFUNC_COND_1)                                     \
 258             return C3(libat_,X,_i1);                            \
 259           if (IFUNC_COND_2)                                     \
 260             return C3(libat_,X,_i2);                            \
 261           return C2(libat_,X);                                  \
 262         }
 263 # elif IFUNC_NCOND(N) == 3
 264 #  define GEN_SELECTOR(X)                                       \
 265         extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN;    \
 266         extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN;    \
 267         extern typeof(C2(libat_,X)) C3(libat_,X,_i3) HIDDEN;    \
 268         static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \
 269         {                                                       \
 270           if (IFUNC_COND_1)                                     \
 271             return C3(libat_,X,_i1);                            \
 272           if (IFUNC_COND_2)                                     \
 273             return C3(libat_,X,_i2);                            \
 274           if (IFUNC_COND_3)                                     \
 275             return C3(libat_,X,_i3);                            \
 276           return C2(libat_,X);                                  \
 277         }
 278 # else
 279 #  error "Unsupported number of ifunc alternatives."
 280 # endif
 281 # define EXPORT_ALIAS(X)                                        \
 282         GEN_SELECTOR(X)                                         \
 283         typeof(C2(libat_,X)) C2(ifunc_,X)                       \
 284           ASMNAME(C2(__atomic_,X))                              \
 285           __attribute__((ifunc(S(C2(select_,X)))))
 286 #elif defined(HAVE_ATTRIBUTE_ALIAS)
 287 # define EXPORT_ALIAS(X)                                        \
 288         extern typeof(C2(libat_,X)) C2(export_,X)               \
 289           ASMNAME(C2(__atomic_,X))                              \
 290           __attribute__((alias(S(C2(libat_,X)))))
 291 #else
 292 # define EXPORT_ALIAS(X)        /* original symbol is exported */
 293 #endif
 294
 295 #endif /* LIBATOMIC_H */