libatomic/libatomic_i.h

   1 /* Copyright (C) 2012-2018 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Atomic Library (libatomic).
   5
   6    Libatomic is free software; you can redistribute it and/or modify it
   7    under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3, or (at your option)
   9    any later version.
  10
  11    Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
  12    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  13    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14    more details.
  15
  16    Under Section 7 of GPL version 3, you are granted additional
  17    permissions described in the GCC Runtime Library Exception, version
  18    3.1, as published by the Free Software Foundation.
  19
  20    You should have received a copy of the GNU General Public License and
  21    a copy of the GCC Runtime Library Exception along with this program;
  22    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  23    <http://www.gnu.org/licenses/>.  */
  24
  25 /* This file contains data types and function declarations that are
  26    private to the implementation of libatomic.  */
  27
  28 #ifndef LIBATOMIC_H
  29 #define LIBATOMIC_H 1
  30
  31 #include "auto-config.h"
  32 #include <stdbool.h>
  33 #include <stdint.h>
  34 #include <stddef.h>
  35 #include <limits.h>
  36 #include <string.h>
  37
  38
  39 /* Symbol concatenation macros.  */
  40 #define C2_(X,Y)        X ## Y
  41 #define C2(X,Y)         C2_(X,Y)
  42 #define C3_(X,Y,Z)      X ## Y ## Z
  43 #define C3(X,Y,Z)       C3_(X,Y,Z)
  44 #define C4_(W,X,Y,Z)    W ## X ## Y ## Z
  45 #define C4(W,X,Y,Z)     C4_(W,X,Y,Z)
  46
  47 /* Stringification macros.  */
  48 #define S2(X)           #X
  49 #define S(X)            S2(X)
  50
  51 /* All of the primitive types on which we operate.  */
  52 typedef unsigned U_1 __attribute__((mode(QI)));
  53 #if HAVE_INT2
  54 typedef unsigned U_2 __attribute__((mode(HI)));
  55 #endif
  56 #if HAVE_INT4
  57 typedef unsigned U_4 __attribute__((mode(SI)));
  58 #endif
  59 #if HAVE_INT8
  60 typedef unsigned U_8 __attribute__((mode(DI)));
  61 #endif
  62 #if HAVE_INT16
  63 typedef unsigned U_16 __attribute__((mode(TI)));
  64 #endif
  65
  66 /* The widest type that we support.  */
  67 #if HAVE_INT16
  68 # define MAX_SIZE       16
  69 #elif HAVE_INT8
  70 # define MAX_SIZE       8
  71 #elif HAVE_INT4
  72 # define MAX_SIZE       4
  73 #elif HAVE_INT2
  74 # define MAX_SIZE       2
  75 #else
  76 # define MAX_SIZE       1
  77 #endif
  78 typedef C2(U_,MAX_SIZE) U_MAX;
  79
  80 /* Provide dummy fallback types so that stuff is syntactically correct
  81    without having to overdo the ifdefs.  The code using these should
  82    always be protected with the HAVE_INT{n} macros.  */
  83 #if !HAVE_INT2
  84 typedef U_MAX U_2;
  85 #endif
  86 #if !HAVE_INT4
  87 typedef U_MAX U_4;
  88 #endif
  89 #if !HAVE_INT8
  90 typedef U_MAX U_8;
  91 #endif
  92 #if !HAVE_INT16
  93 typedef U_MAX U_16;
  94 #endif
  95
  96 union max_size_u
  97 {
  98   U_1 b[MAX_SIZE];
  99   U_2 i2;
 100   U_4 i4;
 101   U_8 i8;
 102   U_16 i16;
 103 };
 104
 105 /* The "word" size of the machine.  */
 106 typedef unsigned UWORD __attribute__((mode(word)));
 107
 108 /* Macros for handing sub-word sized quantities.  */
 109 #define MASK_1          ((UWORD)0xff)
 110 #define MASK_2          ((UWORD)0xffff)
 111 #define MASK_4          ((UWORD)0xffffffff)
 112 #define INVERT_MASK_1   ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 1) * CHAR_BIT))
 113 #define INVERT_MASK_2   ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 2) * CHAR_BIT))
 114 #define INVERT_MASK_4   ((UWORD)WORDS_BIGENDIAN << ((WORDSIZE - 4) * CHAR_BIT))
 115
 116 /* Most of the files in this library are compiled multiple times with
 117    N defined to be a power of 2 between 1 and 16.  The SIZE macro is
 118    then used to append _N to the symbol being manipulated.  */
 119 #define SIZE(X)         C3(X,_,N)
 120 #define WSIZE(X)        C3(X,_,WORDSIZE)
 121 #define PTR(N,X)        ((C2(U_,N) *)X)
 122
 123 /* And thus, the type on which this compilation will be operating.  */
 124 #define ITYPE           SIZE(I)
 125 #define UTYPE           SIZE(U)
 126
 127 /* Utility macros for GCC attributes.  */
 128 #define UNUSED          __attribute__((unused))
 129 #ifdef HAVE_ATTRIBUTE_VISIBILITY
 130 # define HIDDEN         __attribute__((visibility("hidden")))
 131 #else
 132 # define HIDDEN
 133 #endif
 134
 135 /* Occasionally we have to play games with internal and external symbol
 136    names, in order to work around builtin functions of the same name.
 137    This macro sets the external name of the function appropriately.  */
 138 #define ASMNAME(X)      __asm__(S(C2(__USER_LABEL_PREFIX__,X)))
 139
 140 /* Locking for a "small" operation.  In the bare-metal single processor
 141    cases this could be implemented by disabling interrupts.  Thus the extra
 142    word passed between the two functions, saving the interrupt level.
 143    It is assumed that the object being locked does not cross the locking
 144    granularity.
 145
 146    Not actually declared here so that they can be defined static inline
 147    in a target-specfic <host-config.h>.
 148
 149 UWORD protect_start (void *ptr);
 150 void protect_end (void *ptr, UWORD);
 151 */
 152
 153 /* Locking for a "large' operation.  This should always be some sort of
 154    test-and-set operation, as we assume that the interrupt latency would
 155    be unreasonably large.  */
 156 void libat_lock_n (void *ptr, size_t n);
 157 void libat_unlock_n (void *ptr, size_t n);
 158
 159 /* We'll need to declare all of the sized functions a few times...  */
 160 #define DECLARE_ALL_SIZED(N)  DECLARE_ALL_SIZED_(N,C2(U_,N))
 161 #define DECLARE_ALL_SIZED_(N,T)                                         \
 162   DECLARE_1(T,    C2(load_,N), (T *mptr, int));                         \
 163   DECLARE_1(void, C2(store_,N), (T *mptr, T val, int));                 \
 164   DECLARE_1(T,    C2(exchange_,N), (T *mptr, T, int));                  \
 165   DECLARE_1(bool, C2(compare_exchange_,N), (T *mptr, T *, T, int, int)); \
 166   DECLARE_1(bool, C2(test_and_set_,N), (T *mptr, int));                 \
 167   DECLARE_1(T,    C2(fetch_add_,N), (T *mptr, T, int));                 \
 168   DECLARE_1(T,    C2(fetch_sub_,N), (T *mptr, T, int));                 \
 169   DECLARE_1(T,    C2(fetch_and_,N), (T *mptr, T, int));                 \
 170   DECLARE_1(T,    C2(fetch_xor_,N), (T *mptr, T, int));                 \
 171   DECLARE_1(T,    C2(fetch_or_,N), (T *mptr, T, int));                  \
 172   DECLARE_1(T,    C2(fetch_nand_,N), (T *mptr, T, int));                \
 173   DECLARE_1(T,    C2(add_fetch_,N), (T *mptr, T, int));                 \
 174   DECLARE_1(T,    C2(sub_fetch_,N), (T *mptr, T, int));                 \
 175   DECLARE_1(T,    C2(and_fetch_,N), (T *mptr, T, int));                 \
 176   DECLARE_1(T,    C2(xor_fetch_,N), (T *mptr, T, int));                 \
 177   DECLARE_1(T,    C2(or_fetch_,N), (T *mptr, T, int));                  \
 178   DECLARE_1(T,    C2(nand_fetch_,N), (T *mptr, T, int))
 179
 180 /* All sized operations are implemented in hidden functions prefixed with
 181    "libat_".  These are either renamed or aliased to the expected prefix
 182    of "__atomic".  Some amount of renaming is required to avoid hiding or
 183    conflicting with the builtins of the same name, but this additional
 184    use of hidden symbols (where appropriate) avoids unnecessary PLT entries
 185    on relevant targets.  */
 186
 187 #if IFUNC_ALT
 188 # define MAN(X)                 ASMNAME(C4(libat_,X,_i,IFUNC_ALT)) HIDDEN
 189 #elif defined(HAVE_ATTRIBUTE_ALIAS)
 190 # define MAN(X)                 HIDDEN
 191 #else
 192 # define MAN(X)                 ASMNAME(C2(__atomic_,X))
 193 #endif
 194
 195 #if !defined(N) && HAVE_IFUNC
 196 # define DECLARE_1(RET,NAME,ARGS) \
 197         RET C2(libat_,NAME) ARGS MAN(NAME); \
 198         RET C2(ifunc_,NAME) ARGS ASMNAME(C2(__atomic_,NAME))
 199 #else
 200 # define DECLARE_1(RET,NAME,ARGS)       RET C2(libat_,NAME) ARGS MAN(NAME)
 201 #endif
 202
 203 /* Prefix to use when calling internal, possibly ifunc'ed functions.  */
 204 #if HAVE_IFUNC
 205 # define local_ ifunc_
 206 #else
 207 # define local_ libat_
 208 #endif
 209
 210 DECLARE_ALL_SIZED(1);
 211 DECLARE_ALL_SIZED(2);
 212 DECLARE_ALL_SIZED(4);
 213 DECLARE_ALL_SIZED(8);
 214 DECLARE_ALL_SIZED(16);
 215
 216 #undef DECLARE_1
 217 #undef DECLARE_ALL_SIZED
 218 #undef DECLARE_ALL_SIZED_
 219
 220 /* And the generic sized versions.  */
 221 void libat_load (size_t, void *, void *, int) MAN(load);
 222 void libat_store (size_t, void *, void *, int) MAN(store);
 223 void libat_exchange (size_t, void *, void *, void *, int) MAN(exchange);
 224 bool libat_compare_exchange (size_t, void *, void *, void *, int, int)
 225         MAN(compare_exchange);
 226 bool libat_is_lock_free (size_t, void *) MAN(is_lock_free);
 227
 228 #undef MAN
 229
 230 #include <host-config.h>
 231
 232 /* We don't have IFUNC_NCOND until after host-config.h.  */
 233 #if !HAVE_IFUNC
 234 # define IFUNC_NCOND(N) 0
 235 #endif
 236
 237 #if IFUNC_ALT
 238 # define EXPORT_ALIAS(X)        /* exported symbol in non-alternate file */
 239 #elif defined(N) && IFUNC_NCOND(N)
 240 # if IFUNC_NCOND(N) == 1
 241 #  define GEN_SELECTOR(X)                                       \
 242         extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN;    \
 243         static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \
 244         {                                                       \
 245           if (IFUNC_COND_1)                                     \
 246             return C3(libat_,X,_i1);                            \
 247           return C2(libat_,X);                                  \
 248         }
 249 # elif IFUNC_NCOND(N) == 2
 250 #  define GEN_SELECTOR(X)                                       \
 251         extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN;    \
 252         extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN;    \
 253         static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \
 254         {                                                       \
 255           if (IFUNC_COND_1)                                     \
 256             return C3(libat_,X,_i1);                            \
 257           if (IFUNC_COND_2)                                     \
 258             return C3(libat_,X,_i2);                            \
 259           return C2(libat_,X);                                  \
 260         }
 261 # elif IFUNC_NCOND(N) == 3
 262 #  define GEN_SELECTOR(X)                                       \
 263         extern typeof(C2(libat_,X)) C3(libat_,X,_i1) HIDDEN;    \
 264         extern typeof(C2(libat_,X)) C3(libat_,X,_i2) HIDDEN;    \
 265         extern typeof(C2(libat_,X)) C3(libat_,X,_i3) HIDDEN;    \
 266         static typeof(C2(libat_,X)) * C2(select_,X) (IFUNC_RESOLVER_ARGS) \
 267         {                                                       \
 268           if (IFUNC_COND_1)                                     \
 269             return C3(libat_,X,_i1);                            \
 270           if (IFUNC_COND_2)                                     \
 271             return C3(libat_,X,_i2);                            \
 272           if (IFUNC_COND_3)                                     \
 273             return C3(libat_,X,_i3);                            \
 274           return C2(libat_,X);                                  \
 275         }
 276 # else
 277 #  error "Unsupported number of ifunc alternatives."
 278 # endif
 279 # define EXPORT_ALIAS(X)                                        \
 280         GEN_SELECTOR(X)                                         \
 281         typeof(C2(libat_,X)) C2(ifunc_,X)                       \
 282           ASMNAME(C2(__atomic_,X))                              \
 283           __attribute__((ifunc(S(C2(select_,X)))))
 284 #elif defined(HAVE_ATTRIBUTE_ALIAS)
 285 # define EXPORT_ALIAS(X)                                        \
 286         extern typeof(C2(libat_,X)) C2(export_,X)               \
 287           ASMNAME(C2(__atomic_,X))                              \
 288           __attribute__((alias(S(C2(libat_,X)))))
 289 #else
 290 # define EXPORT_ALIAS(X)        /* original symbol is exported */
 291 #endif
 292
 293 #endif /* LIBATOMIC_H */