libsanitizer/sanitizer_common/sanitizer_atomic_msvc.h

   1 //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer/AddressSanitizer runtime.
   9 // Not intended for direct inclusion. Include sanitizer_atomic.h.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #ifndef SANITIZER_ATOMIC_MSVC_H
  14 #define SANITIZER_ATOMIC_MSVC_H
  15
  16 extern "C" void _ReadWriteBarrier();
  17 #pragma intrinsic(_ReadWriteBarrier)
  18 extern "C" void _mm_mfence();
  19 #pragma intrinsic(_mm_mfence)
  20 extern "C" void _mm_pause();
  21 #pragma intrinsic(_mm_pause)
  22 extern "C" long _InterlockedExchangeAdd(  // NOLINT
  23     long volatile * Addend, long Value);  // NOLINT
  24 #pragma intrinsic(_InterlockedExchangeAdd)
  25 extern "C" short _InterlockedCompareExchange16(  // NOLINT
  26     short volatile *Destination,                 // NOLINT
  27     short Exchange, short Comparand);            // NOLINT
  28 #pragma intrinsic(_InterlockedCompareExchange16)
  29 extern "C"
  30 long long _InterlockedCompareExchange64(  // NOLINT
  31     long long volatile *Destination,              // NOLINT
  32     long long Exchange, long long Comparand);     // NOLINT
  33 #pragma intrinsic(_InterlockedCompareExchange64)
  34 extern "C" void *_InterlockedCompareExchangePointer(
  35     void *volatile *Destination,
  36     void *Exchange, void *Comparand);
  37 #pragma intrinsic(_InterlockedCompareExchangePointer)
  38 extern "C"
  39 long __cdecl _InterlockedCompareExchange(  // NOLINT
  40     long volatile *Destination,            // NOLINT
  41     long Exchange, long Comparand);        // NOLINT
  42 #pragma intrinsic(_InterlockedCompareExchange)
  43
  44 #ifdef _WIN64
  45 extern "C" long long _InterlockedExchangeAdd64(     // NOLINT
  46     long long volatile * Addend, long long Value);  // NOLINT
  47 #pragma intrinsic(_InterlockedExchangeAdd64)
  48 #endif
  49
  50 namespace __sanitizer {
  51
  52 INLINE void atomic_signal_fence(memory_order) {
  53   _ReadWriteBarrier();
  54 }
  55
  56 INLINE void atomic_thread_fence(memory_order) {
  57   _mm_mfence();
  58 }
  59
  60 INLINE void proc_yield(int cnt) {
  61   for (int i = 0; i < cnt; i++)
  62     _mm_pause();
  63 }
  64
  65 template<typename T>
  66 INLINE typename T::Type atomic_load(
  67     const volatile T *a, memory_order mo) {
  68   DCHECK(mo & (memory_order_relaxed | memory_order_consume
  69       | memory_order_acquire | memory_order_seq_cst));
  70   DCHECK(!((uptr)a % sizeof(*a)));
  71   typename T::Type v;
  72   // FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
  73   if (mo == memory_order_relaxed) {
  74     v = a->val_dont_use;
  75   } else {
  76     atomic_signal_fence(memory_order_seq_cst);
  77     v = a->val_dont_use;
  78     atomic_signal_fence(memory_order_seq_cst);
  79   }
  80   return v;
  81 }
  82
  83 template<typename T>
  84 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
  85   DCHECK(mo & (memory_order_relaxed | memory_order_release
  86       | memory_order_seq_cst));
  87   DCHECK(!((uptr)a % sizeof(*a)));
  88   // FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
  89   if (mo == memory_order_relaxed) {
  90     a->val_dont_use = v;
  91   } else {
  92     atomic_signal_fence(memory_order_seq_cst);
  93     a->val_dont_use = v;
  94     atomic_signal_fence(memory_order_seq_cst);
  95   }
  96   if (mo == memory_order_seq_cst)
  97     atomic_thread_fence(memory_order_seq_cst);
  98 }
  99
 100 INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
 101     u32 v, memory_order mo) {
 102   (void)mo;
 103   DCHECK(!((uptr)a % sizeof(*a)));
 104   return (u32)_InterlockedExchangeAdd(
 105       (volatile long*)&a->val_dont_use, (long)v);  // NOLINT
 106 }
 107
 108 INLINE uptr atomic_fetch_add(volatile atomic_uintptr_t *a,
 109     uptr v, memory_order mo) {
 110   (void)mo;
 111   DCHECK(!((uptr)a % sizeof(*a)));
 112 #ifdef _WIN64
 113   return (uptr)_InterlockedExchangeAdd64(
 114       (volatile long long*)&a->val_dont_use, (long long)v);  // NOLINT
 115 #else
 116   return (uptr)_InterlockedExchangeAdd(
 117       (volatile long*)&a->val_dont_use, (long)v);  // NOLINT
 118 #endif
 119 }
 120
 121 INLINE u32 atomic_fetch_sub(volatile atomic_uint32_t *a,
 122     u32 v, memory_order mo) {
 123   (void)mo;
 124   DCHECK(!((uptr)a % sizeof(*a)));
 125   return (u32)_InterlockedExchangeAdd(
 126       (volatile long*)&a->val_dont_use, -(long)v);  // NOLINT
 127 }
 128
 129 INLINE uptr atomic_fetch_sub(volatile atomic_uintptr_t *a,
 130     uptr v, memory_order mo) {
 131   (void)mo;
 132   DCHECK(!((uptr)a % sizeof(*a)));
 133 #ifdef _WIN64
 134   return (uptr)_InterlockedExchangeAdd64(
 135       (volatile long long*)&a->val_dont_use, -(long long)v);  // NOLINT
 136 #else
 137   return (uptr)_InterlockedExchangeAdd(
 138       (volatile long*)&a->val_dont_use, -(long)v);  // NOLINT
 139 #endif
 140 }
 141
 142 INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
 143     u8 v, memory_order mo) {
 144   (void)mo;
 145   DCHECK(!((uptr)a % sizeof(*a)));
 146   __asm {
 147     mov eax, a
 148     mov cl, v
 149     xchg [eax], cl  // NOLINT
 150     mov v, cl
 151   }
 152   return v;
 153 }
 154
 155 INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
 156     u16 v, memory_order mo) {
 157   (void)mo;
 158   DCHECK(!((uptr)a % sizeof(*a)));
 159   __asm {
 160     mov eax, a
 161     mov cx, v
 162     xchg [eax], cx  // NOLINT
 163     mov v, cx
 164   }
 165   return v;
 166 }
 167
 168 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint8_t *a,
 169                                            u8 *cmp,
 170                                            u8 xchgv,
 171                                            memory_order mo) {
 172   (void)mo;
 173   DCHECK(!((uptr)a % sizeof(*a)));
 174   u8 cmpv = *cmp;
 175   u8 prev;
 176   __asm {
 177     mov al, cmpv
 178     mov ecx, a
 179     mov dl, xchgv
 180     lock cmpxchg [ecx], dl
 181     mov prev, al
 182   }
 183   if (prev == cmpv)
 184     return true;
 185   *cmp = prev;
 186   return false;
 187 }
 188
 189 INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
 190                                            uptr *cmp,
 191                                            uptr xchg,
 192                                            memory_order mo) {
 193   uptr cmpv = *cmp;
 194   uptr prev = (uptr)_InterlockedCompareExchangePointer(
 195       (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
 196   if (prev == cmpv)
 197     return true;
 198   *cmp = prev;
 199   return false;
 200 }
 201
 202 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint16_t *a,
 203                                            u16 *cmp,
 204                                            u16 xchg,
 205                                            memory_order mo) {
 206   u16 cmpv = *cmp;
 207   u16 prev = (u16)_InterlockedCompareExchange16(
 208       (volatile short*)&a->val_dont_use, (short)xchg, (short)cmpv);
 209   if (prev == cmpv)
 210     return true;
 211   *cmp = prev;
 212   return false;
 213 }
 214
 215 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint32_t *a,
 216                                            u32 *cmp,
 217                                            u32 xchg,
 218                                            memory_order mo) {
 219   u32 cmpv = *cmp;
 220   u32 prev = (u32)_InterlockedCompareExchange(
 221       (volatile long*)&a->val_dont_use, (long)xchg, (long)cmpv);
 222   if (prev == cmpv)
 223     return true;
 224   *cmp = prev;
 225   return false;
 226 }
 227
 228 INLINE bool atomic_compare_exchange_strong(volatile atomic_uint64_t *a,
 229                                            u64 *cmp,
 230                                            u64 xchg,
 231                                            memory_order mo) {
 232   u64 cmpv = *cmp;
 233   u64 prev = (u64)_InterlockedCompareExchange64(
 234       (volatile long long*)&a->val_dont_use, (long long)xchg, (long long)cmpv);
 235   if (prev == cmpv)
 236     return true;
 237   *cmp = prev;
 238   return false;
 239 }
 240
 241 template<typename T>
 242 INLINE bool atomic_compare_exchange_weak(volatile T *a,
 243                                          typename T::Type *cmp,
 244                                          typename T::Type xchg,
 245                                          memory_order mo) {
 246   return atomic_compare_exchange_strong(a, cmp, xchg, mo);
 247 }
 248
 249 }  // namespace __sanitizer
 250
 251 #endif  // SANITIZER_ATOMIC_CLANG_H