libatomic/fop_n.c

   1 /* Copyright (C) 2012-2021 Free Software Foundation, Inc.
   2    Contributed by Richard Henderson <rth@redhat.com>.
   3
   4    This file is part of the GNU Atomic Library (libatomic).
   5
   6    Libatomic is free software; you can redistribute it and/or modify it
   7    under the terms of the GNU General Public License as published by
   8    the Free Software Foundation; either version 3 of the License, or
   9    (at your option) any later version.
  10
  11    Libatomic is distributed in the hope that it will be useful, but WITHOUT ANY
  12    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  13    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14    more details.
  15
  16    Under Section 7 of GPL version 3, you are granted additional
  17    permissions described in the GCC Runtime Library Exception, version
  18    3.1, as published by the Free Software Foundation.
  19
  20    You should have received a copy of the GNU General Public License and
  21    a copy of the GCC Runtime Library Exception along with this program;
  22    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  23    <http://www.gnu.org/licenses/>.  */
  24
  25 #include <libatomic_i.h>
  26
  27
  28 /* This file is included multiple times with required defines:
  29      NAME       the name of the operation that we're implementing;
  30      OP         a two-operand functional macro the implements the operation.
  31 */
  32
  33
  34 /* If we support the builtin, just use it.  */
  35 #if !DONE && SIZE(HAVE_ATOMIC_FETCH_OP)
  36 UTYPE
  37 SIZE(C2(libat_fetch_,NAME)) (UTYPE *mptr, UTYPE opval, int smodel)
  38 {
  39   if (maybe_specialcase_relaxed(smodel))
  40     return C2(__atomic_fetch_,NAME) (mptr, opval, __ATOMIC_RELAXED);
  41   else if (maybe_specialcase_acqrel(smodel))
  42     return C2(__atomic_fetch_,NAME) (mptr, opval, __ATOMIC_ACQ_REL);
  43   else
  44     return C2(__atomic_fetch_,NAME) (mptr, opval, __ATOMIC_SEQ_CST);
  45 }
  46
  47 UTYPE
  48 SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel)
  49 {
  50   if (maybe_specialcase_relaxed(smodel))
  51     return C3(__atomic_,NAME,_fetch) (mptr, opval, __ATOMIC_RELAXED);
  52   else if (maybe_specialcase_acqrel(smodel))
  53     return C3(__atomic_,NAME,_fetch) (mptr, opval, __ATOMIC_ACQ_REL);
  54   else
  55     return C3(__atomic_,NAME,_fetch) (mptr, opval, __ATOMIC_SEQ_CST);
  56 }
  57
  58 #define DONE 1
  59 #endif /* HAVE_ATOMIC_FETCH_OP */
  60
  61
  62 #if !DONE && defined(atomic_compare_exchange_n)
  63 UTYPE
  64 SIZE(C2(libat_fetch_,NAME)) (UTYPE *mptr, UTYPE opval, int smodel)
  65 {
  66   UTYPE oldval, t;
  67
  68   pre_barrier (smodel);
  69
  70   oldval = *mptr;
  71   do
  72     {
  73       t = OP(oldval, opval);
  74     }
  75   while (!atomic_compare_exchange_n (mptr, &oldval, t, true,
  76                                      __ATOMIC_RELAXED, __ATOMIC_RELAXED));
  77
  78   post_barrier (smodel);
  79   return oldval;
  80 }
  81
  82 UTYPE
  83 SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel)
  84 {
  85   UTYPE oldval, t;
  86
  87   pre_barrier (smodel);
  88
  89   oldval = *mptr;
  90   do
  91     {
  92       t = OP(oldval, opval);
  93     }
  94   while (!atomic_compare_exchange_n (mptr, &oldval, t, true,
  95                                      __ATOMIC_RELAXED, __ATOMIC_RELAXED));
  96
  97   post_barrier (smodel);
  98   return t;
  99 }
 100
 101 #define DONE 1
 102 #endif /* atomic_compare_exchange_n */
 103
 104
 105 /* If this type is no larger than word-sized, fall back to a word-sized
 106    compare-and-swap loop.  */
 107 #if !DONE && N < WORDSIZE && defined(atomic_compare_exchange_w)
 108 UTYPE
 109 SIZE(C2(libat_fetch_,NAME)) (UTYPE *mptr, UTYPE opval, int smodel)
 110 {
 111   UWORD mask, shift, woldval, wopval, t, *wptr;
 112
 113   pre_barrier (smodel);
 114
 115   wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE);
 116   shift = (((uintptr_t)mptr % WORDSIZE) * CHAR_BIT) ^ SIZE(INVERT_MASK);
 117   mask = SIZE(MASK) << shift;
 118
 119   wopval = (UWORD)opval << shift;
 120   woldval = __atomic_load_n (wptr, __ATOMIC_RELAXED);
 121   do
 122     {
 123       t = (woldval & ~mask) | (OP(woldval, wopval) & mask);
 124     }
 125   while (!atomic_compare_exchange_w (wptr, &woldval, t, true,
 126                                      __ATOMIC_RELAXED, __ATOMIC_RELAXED));
 127
 128   post_barrier (smodel);
 129   return woldval >> shift;
 130 }
 131
 132 UTYPE
 133 SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel)
 134 {
 135   UWORD mask, shift, woldval, wopval, t, *wptr;
 136
 137   pre_barrier (smodel);
 138
 139   wptr = (UWORD *)((uintptr_t)mptr & -WORDSIZE);
 140   shift = (((uintptr_t)mptr % WORDSIZE) * CHAR_BIT) ^ SIZE(INVERT_MASK);
 141   mask = SIZE(MASK) << shift;
 142
 143   wopval = (UWORD)opval << shift;
 144   woldval = __atomic_load_n (wptr, __ATOMIC_RELAXED);
 145   do
 146     {
 147       t = (woldval & ~mask) | (OP(woldval, wopval) & mask);
 148     }
 149   while (!atomic_compare_exchange_w (wptr, &woldval, t, true,
 150                                      __ATOMIC_RELAXED, __ATOMIC_RELAXED));
 151
 152   post_barrier (smodel);
 153   return t >> shift;
 154 }
 155
 156 #define DONE 1
 157 #endif /* atomic_compare_exchange_w */
 158
 159
 160 /* Otherwise, fall back to some sort of protection mechanism.  */
 161 #if !DONE
 162 UTYPE
 163 SIZE(C2(libat_fetch_,NAME)) (UTYPE *mptr, UTYPE opval, int smodel UNUSED)
 164 {
 165   UTYPE ret;
 166   UWORD magic;
 167
 168   pre_seq_barrier (smodel);
 169   magic = protect_start (mptr);
 170
 171   ret = *mptr;
 172   *mptr = OP(ret, opval);
 173
 174   protect_end (mptr, magic);
 175   post_seq_barrier (smodel);
 176
 177   return ret;
 178 }
 179
 180 UTYPE
 181 SIZE(C3(libat_,NAME,_fetch)) (UTYPE *mptr, UTYPE opval, int smodel UNUSED)
 182 {
 183   UTYPE ret;
 184   UWORD magic;
 185
 186   pre_seq_barrier (smodel);
 187   magic = protect_start (mptr);
 188
 189   ret = OP (*mptr, opval);
 190   *mptr = ret;
 191
 192   protect_end (mptr, magic);
 193   post_seq_barrier (smodel);
 194
 195   return ret;
 196 }
 197 #endif
 198
 199 EXPORT_ALIAS (SIZE(C2(fetch_,NAME)));
 200 EXPORT_ALIAS (SIZE(C2(NAME,_fetch)));