libgcc/config/pa/linux-atomic.c

   1 /* Linux-specific atomic operations for PA Linux.
   2    Copyright (C) 2008-2022 Free Software Foundation, Inc.
   3    Based on code contributed by CodeSourcery for ARM EABI Linux.
   4    Modifications for PA Linux by Helge Deller <deller@gmx.de>
   5
   6 This file is part of GCC.
   7
   8 GCC is free software; you can redistribute it and/or modify it under
   9 the terms of the GNU General Public License as published by the Free
  10 Software Foundation; either version 3, or (at your option) any later
  11 version.
  12
  13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
  14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
  15 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  16 for more details.
  17
  18 Under Section 7 of GPL version 3, you are granted additional
  19 permissions described in the GCC Runtime Library Exception, version
  20 3.1, as published by the Free Software Foundation.
  21
  22 You should have received a copy of the GNU General Public License and
  23 a copy of the GCC Runtime Library Exception along with this program;
  24 see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  25 <http://www.gnu.org/licenses/>.  */
  26
  27 #define EFAULT  14
  28 #define EBUSY   16
  29 #define ENOSYS 251
  30
  31 #define _ASM_EFAULT "-14"
  32
  33 typedef unsigned char u8;
  34 typedef short unsigned int u16;
  35 #ifdef __LP64__
  36 typedef long unsigned int u64;
  37 #else
  38 typedef long long unsigned int u64;
  39 #endif
  40
  41 /* PA-RISC 2.0 supports out-of-order execution for loads and stores.
  42    Thus, we need to synchonize memory accesses.  For more info, see:
  43    "Advanced Performance Features of the 64-bit PA-8000" by Doug Hunt.
  44
  45    We implement byte, short and int versions of each atomic operation
  46    using the kernel helper defined below.  There is no support for
  47    64-bit operations yet.  */
  48
  49 /* Determine kernel LWS function call (0=32-bit, 1=64-bit userspace).  */
  50 #define LWS_CAS (sizeof(long) == 4 ? 0 : 1)
  51
  52 /* Kernel helper for compare-and-exchange a 32-bit value.  */
  53 static inline long
  54 __kernel_cmpxchg (volatile void *mem, int oldval, int newval)
  55 {
  56   register unsigned long lws_mem asm("r26") = (unsigned long) (mem);
  57   register int lws_old asm("r25") = oldval;
  58   register int lws_new asm("r24") = newval;
  59   register long lws_ret   asm("r28");
  60   register long lws_errno asm("r21");
  61   asm volatile (        "ble    0xb0(%%sr2, %%r0)       \n\t"
  62                         "ldi    %2, %%r20               \n\t"
  63                         "cmpiclr,<> " _ASM_EFAULT ", %%r21, %%r0\n\t"
  64                         "iitlbp %%r0,(%%sr0, %%r0)      \n\t"
  65         : "=r" (lws_ret), "=r" (lws_errno)
  66         : "i" (LWS_CAS), "r" (lws_mem), "r" (lws_old), "r" (lws_new)
  67         : "r1", "r20", "r22", "r23", "r29", "r31", "memory"
  68   );
  69
  70   /* If the kernel LWS call succeeded (lws_errno == 0), lws_ret contains
  71      the old value from memory.  If this value is equal to OLDVAL, the
  72      new value was written to memory.  If not, return -EBUSY.  */
  73   if (!lws_errno && lws_ret != oldval)
  74     return -EBUSY;
  75
  76   return lws_errno;
  77 }
  78
  79 static inline long
  80 __kernel_cmpxchg2 (volatile void *mem, const void *oldval, const void *newval,
  81                    int val_size)
  82 {
  83   register unsigned long lws_mem asm("r26") = (unsigned long) (mem);
  84   register unsigned long lws_old asm("r25") = (unsigned long) oldval;
  85   register unsigned long lws_new asm("r24") = (unsigned long) newval;
  86   register int lws_size asm("r23") = val_size;
  87   register long lws_ret   asm("r28");
  88   register long lws_errno asm("r21");
  89   asm volatile (        "ble    0xb0(%%sr2, %%r0)       \n\t"
  90                         "ldi    %6, %%r20               \n\t"
  91                         "cmpiclr,<> " _ASM_EFAULT ", %%r21, %%r0\n\t"
  92                         "iitlbp %%r0,(%%sr0, %%r0)      \n\t"
  93         : "=r" (lws_ret), "=r" (lws_errno), "+r" (lws_mem),
  94           "+r" (lws_old), "+r" (lws_new), "+r" (lws_size)
  95         : "i" (2)
  96         : "r1", "r20", "r22", "r29", "r31", "fr4", "memory"
  97   );
  98
  99   /* If the kernel LWS call is successful, lws_ret contains 0.  */
 100   if (__builtin_expect (lws_ret == 0, 1))
 101     return 0;
 102
 103   /* If the kernel LWS call fails with no error, return -EBUSY */
 104   if (__builtin_expect (!lws_errno, 0))
 105     return -EBUSY;
 106
 107   return lws_errno;
 108 }
 109 #define HIDDEN __attribute__ ((visibility ("hidden")))
 110
 111 /* Big endian masks  */
 112 #define INVERT_MASK_1 24
 113 #define INVERT_MASK_2 16
 114
 115 #define MASK_1 0xffu
 116 #define MASK_2 0xffffu
 117
 118 #define FETCH_AND_OP_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX)          \
 119   TYPE HIDDEN                                                           \
 120   __sync_fetch_and_##OP##_##WIDTH (volatile void *ptr, TYPE val)        \
 121   {                                                                     \
 122     TYPE tmp, newval;                                                   \
 123     long failure;                                                       \
 124                                                                         \
 125     do {                                                                \
 126       tmp = __atomic_load_n ((volatile TYPE *)ptr, __ATOMIC_RELAXED);   \
 127       newval = PFX_OP (tmp INF_OP val);                                 \
 128       failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX);          \
 129     } while (failure != 0);                                             \
 130                                                                         \
 131     return tmp;                                                         \
 132   }
 133
 134 FETCH_AND_OP_2 (add,   , +, u64, 8, 3)
 135 FETCH_AND_OP_2 (sub,   , -, u64, 8, 3)
 136 FETCH_AND_OP_2 (or,    , |, u64, 8, 3)
 137 FETCH_AND_OP_2 (and,   , &, u64, 8, 3)
 138 FETCH_AND_OP_2 (xor,   , ^, u64, 8, 3)
 139 FETCH_AND_OP_2 (nand, ~, &, u64, 8, 3)
 140
 141 FETCH_AND_OP_2 (add,   , +, u16, 2, 1)
 142 FETCH_AND_OP_2 (sub,   , -, u16, 2, 1)
 143 FETCH_AND_OP_2 (or,    , |, u16, 2, 1)
 144 FETCH_AND_OP_2 (and,   , &, u16, 2, 1)
 145 FETCH_AND_OP_2 (xor,   , ^, u16, 2, 1)
 146 FETCH_AND_OP_2 (nand, ~, &, u16, 2, 1)
 147
 148 FETCH_AND_OP_2 (add,   , +, u8, 1, 0)
 149 FETCH_AND_OP_2 (sub,   , -, u8, 1, 0)
 150 FETCH_AND_OP_2 (or,    , |, u8, 1, 0)
 151 FETCH_AND_OP_2 (and,   , &, u8, 1, 0)
 152 FETCH_AND_OP_2 (xor,   , ^, u8, 1, 0)
 153 FETCH_AND_OP_2 (nand, ~, &, u8, 1, 0)
 154
 155 #define OP_AND_FETCH_2(OP, PFX_OP, INF_OP, TYPE, WIDTH, INDEX)          \
 156   TYPE HIDDEN                                                           \
 157   __sync_##OP##_and_fetch_##WIDTH (volatile void *ptr, TYPE val)        \
 158   {                                                                     \
 159     TYPE tmp, newval;                                                   \
 160     long failure;                                                       \
 161                                                                         \
 162     do {                                                                \
 163       tmp = __atomic_load_n ((volatile TYPE *)ptr, __ATOMIC_RELAXED);   \
 164       newval = PFX_OP (tmp INF_OP val);                                 \
 165       failure = __kernel_cmpxchg2 (ptr, &tmp, &newval, INDEX);          \
 166     } while (failure != 0);                                             \
 167                                                                         \
 168     return PFX_OP (tmp INF_OP val);                                     \
 169   }
 170
 171 OP_AND_FETCH_2 (add,   , +, u64, 8, 3)
 172 OP_AND_FETCH_2 (sub,   , -, u64, 8, 3)
 173 OP_AND_FETCH_2 (or,    , |, u64, 8, 3)
 174 OP_AND_FETCH_2 (and,   , &, u64, 8, 3)
 175 OP_AND_FETCH_2 (xor,   , ^, u64, 8, 3)
 176 OP_AND_FETCH_2 (nand, ~, &, u64, 8, 3)
 177
 178 OP_AND_FETCH_2 (add,   , +, u16, 2, 1)
 179 OP_AND_FETCH_2 (sub,   , -, u16, 2, 1)
 180 OP_AND_FETCH_2 (or,    , |, u16, 2, 1)
 181 OP_AND_FETCH_2 (and,   , &, u16, 2, 1)
 182 OP_AND_FETCH_2 (xor,   , ^, u16, 2, 1)
 183 OP_AND_FETCH_2 (nand, ~, &, u16, 2, 1)
 184
 185 OP_AND_FETCH_2 (add,   , +, u8, 1, 0)
 186 OP_AND_FETCH_2 (sub,   , -, u8, 1, 0)
 187 OP_AND_FETCH_2 (or,    , |, u8, 1, 0)
 188 OP_AND_FETCH_2 (and,   , &, u8, 1, 0)
 189 OP_AND_FETCH_2 (xor,   , ^, u8, 1, 0)
 190 OP_AND_FETCH_2 (nand, ~, &, u8, 1, 0)
 191
 192 #define FETCH_AND_OP_WORD(OP, PFX_OP, INF_OP)                           \
 193   unsigned int HIDDEN                                                   \
 194   __sync_fetch_and_##OP##_4 (volatile void *ptr, unsigned int val)      \
 195   {                                                                     \
 196     unsigned int tmp;                                                   \
 197     long failure;                                                       \
 198                                                                         \
 199     do {                                                                \
 200       tmp = __atomic_load_n ((volatile unsigned int *)ptr,              \
 201                              __ATOMIC_RELAXED);                         \
 202       failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val));   \
 203     } while (failure != 0);                                             \
 204                                                                         \
 205     return tmp;                                                         \
 206   }
 207
 208 FETCH_AND_OP_WORD (add,   , +)
 209 FETCH_AND_OP_WORD (sub,   , -)
 210 FETCH_AND_OP_WORD (or,    , |)
 211 FETCH_AND_OP_WORD (and,   , &)
 212 FETCH_AND_OP_WORD (xor,   , ^)
 213 FETCH_AND_OP_WORD (nand, ~, &)
 214
 215 #define OP_AND_FETCH_WORD(OP, PFX_OP, INF_OP)                           \
 216   unsigned int HIDDEN                                                   \
 217   __sync_##OP##_and_fetch_4 (volatile void *ptr, unsigned int val)      \
 218   {                                                                     \
 219     unsigned int tmp;                                                   \
 220     long failure;                                                       \
 221                                                                         \
 222     do {                                                                \
 223       tmp = __atomic_load_n ((volatile unsigned int *)ptr,              \
 224                              __ATOMIC_RELAXED);                         \
 225       failure = __kernel_cmpxchg (ptr, tmp, PFX_OP (tmp INF_OP val));   \
 226     } while (failure != 0);                                             \
 227                                                                         \
 228     return PFX_OP (tmp INF_OP val);                                     \
 229   }
 230
 231 OP_AND_FETCH_WORD (add,   , +)
 232 OP_AND_FETCH_WORD (sub,   , -)
 233 OP_AND_FETCH_WORD (or,    , |)
 234 OP_AND_FETCH_WORD (and,   , &)
 235 OP_AND_FETCH_WORD (xor,   , ^)
 236 OP_AND_FETCH_WORD (nand, ~, &)
 237
 238 typedef unsigned char bool;
 239
 240 #define COMPARE_AND_SWAP_2(TYPE, WIDTH, INDEX)                          \
 241   TYPE HIDDEN                                                           \
 242   __sync_val_compare_and_swap_##WIDTH (volatile void *ptr, TYPE oldval, \
 243                                        TYPE newval)                     \
 244   {                                                                     \
 245     TYPE actual_oldval;                                                 \
 246     long fail;                                                          \
 247                                                                         \
 248     while (1)                                                           \
 249       {                                                                 \
 250         actual_oldval = __atomic_load_n ((volatile TYPE *)ptr,          \
 251                                          __ATOMIC_RELAXED);             \
 252                                                                         \
 253         if (__builtin_expect (oldval != actual_oldval, 0))              \
 254           return actual_oldval;                                         \
 255                                                                         \
 256         fail = __kernel_cmpxchg2 (ptr, &actual_oldval, &newval, INDEX); \
 257                                                                         \
 258         if (__builtin_expect (!fail, 1))                                \
 259           return actual_oldval;                                         \
 260       }                                                                 \
 261   }                                                                     \
 262                                                                         \
 263   _Bool HIDDEN                                                          \
 264   __sync_bool_compare_and_swap_##WIDTH (volatile void *ptr,             \
 265                                         TYPE oldval, TYPE newval)       \
 266   {                                                                     \
 267     long failure = __kernel_cmpxchg2 (ptr, &oldval, &newval, INDEX);    \
 268     return (failure == 0);                                              \
 269   }
 270
 271 COMPARE_AND_SWAP_2 (u64, 8, 3)
 272 COMPARE_AND_SWAP_2 (u16, 2, 1)
 273 COMPARE_AND_SWAP_2 (u8, 1, 0)
 274
 275 unsigned int HIDDEN
 276 __sync_val_compare_and_swap_4 (volatile void *ptr, unsigned int oldval,
 277                                unsigned int newval)
 278 {
 279   long fail;
 280   unsigned int actual_oldval;
 281
 282   while (1)
 283     {
 284       actual_oldval = __atomic_load_n ((volatile unsigned int *)ptr,
 285                                        __ATOMIC_RELAXED);
 286
 287       if (__builtin_expect (oldval != actual_oldval, 0))
 288         return actual_oldval;
 289
 290       fail = __kernel_cmpxchg (ptr, actual_oldval, newval);
 291
 292       if (__builtin_expect (!fail, 1))
 293         return actual_oldval;
 294     }
 295 }
 296
 297 _Bool HIDDEN
 298 __sync_bool_compare_and_swap_4 (volatile void *ptr, unsigned int oldval,
 299                                 unsigned int newval)
 300 {
 301   long failure = __kernel_cmpxchg (ptr, oldval, newval);
 302   return (failure == 0);
 303 }
 304
 305 #define SYNC_LOCK_TEST_AND_SET_2(TYPE, WIDTH, INDEX)                    \
 306 TYPE HIDDEN                                                             \
 307   __sync_lock_test_and_set_##WIDTH (volatile void *ptr, TYPE val)       \
 308   {                                                                     \
 309     TYPE oldval;                                                        \
 310     long failure;                                                       \
 311                                                                         \
 312     do {                                                                \
 313       oldval = __atomic_load_n ((volatile TYPE *)ptr,                   \
 314                                 __ATOMIC_RELAXED);                      \
 315       failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX);          \
 316     } while (failure != 0);                                             \
 317                                                                         \
 318     return oldval;                                                      \
 319   }
 320
 321 SYNC_LOCK_TEST_AND_SET_2 (u64, 8, 3)
 322 SYNC_LOCK_TEST_AND_SET_2 (u16, 2, 1)
 323 SYNC_LOCK_TEST_AND_SET_2 (u8, 1, 0)
 324
 325 unsigned int HIDDEN
 326 __sync_lock_test_and_set_4 (volatile void *ptr, unsigned int val)
 327 {
 328   long failure;
 329   unsigned int oldval;
 330
 331   do {
 332     oldval = __atomic_load_n ((volatile unsigned int *)ptr, __ATOMIC_RELAXED);
 333     failure = __kernel_cmpxchg (ptr, oldval, val);
 334   } while (failure != 0);
 335
 336   return oldval;
 337 }
 338
 339 #define SYNC_LOCK_RELEASE_1(TYPE, WIDTH, INDEX)                 \
 340   void HIDDEN                                                   \
 341   __sync_lock_release_##WIDTH (volatile void *ptr)              \
 342   {                                                             \
 343     TYPE oldval, val = 0;                                       \
 344     long failure;                                               \
 345                                                                 \
 346     do {                                                        \
 347       oldval = __atomic_load_n ((volatile TYPE *)ptr,           \
 348                                 __ATOMIC_RELAXED);              \
 349       failure = __kernel_cmpxchg2 (ptr, &oldval, &val, INDEX);  \
 350     } while (failure != 0);                                     \
 351   }
 352
 353 SYNC_LOCK_RELEASE_1 (u64, 8, 3)
 354 SYNC_LOCK_RELEASE_1 (u16, 2, 1)
 355 SYNC_LOCK_RELEASE_1 (u8, 1, 0)
 356
 357 void HIDDEN
 358 __sync_lock_release_4 (volatile void *ptr)
 359 {
 360   long failure;
 361   unsigned int oldval;
 362
 363   do {
 364     oldval = __atomic_load_n ((volatile unsigned int *)ptr, __ATOMIC_RELAXED);
 365     failure = __kernel_cmpxchg (ptr, oldval, 0);
 366   } while (failure != 0);
 367 }