lib/sanitizer_common/sanitizer_atomic_msvc.h

   1 //===-- sanitizer_atomic_msvc.h ---------------------------------*- C++ -*-===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file is a part of ThreadSanitizer/AddressSanitizer runtime.
  11 // Not intended for direct inclusion. Include sanitizer_atomic.h.
  12 //
  13 //===----------------------------------------------------------------------===//
  14
  15 #ifndef SANITIZER_ATOMIC_MSVC_H
  16 #define SANITIZER_ATOMIC_MSVC_H
  17
  18 extern "C" void _ReadWriteBarrier();
  19 #pragma intrinsic(_ReadWriteBarrier)
  20 extern "C" void _mm_mfence();
  21 #pragma intrinsic(_mm_mfence)
  22 extern "C" void _mm_pause();
  23 #pragma intrinsic(_mm_pause)
  24 extern "C" long _InterlockedExchangeAdd(  // NOLINT
  25     long volatile * Addend, long Value);  // NOLINT
  26 #pragma intrinsic(_InterlockedExchangeAdd)
  27
  28 #ifdef _WIN64
  29 extern "C" void *_InterlockedCompareExchangePointer(
  30     void *volatile *Destination,
  31     void *Exchange, void *Comparand);
  32 #pragma intrinsic(_InterlockedCompareExchangePointer)
  33 #else
  34 // There's no _InterlockedCompareExchangePointer intrinsic on x86,
  35 // so call _InterlockedCompareExchange instead.
  36 extern "C"
  37 long __cdecl _InterlockedCompareExchange(  // NOLINT
  38     long volatile *Destination,            // NOLINT
  39     long Exchange, long Comparand);        // NOLINT
  40 #pragma intrinsic(_InterlockedCompareExchange)
  41
  42 inline static void *_InterlockedCompareExchangePointer(
  43     void *volatile *Destination,
  44     void *Exchange, void *Comparand) {
  45   return reinterpret_cast<void*>(
  46       _InterlockedCompareExchange(
  47           reinterpret_cast<long volatile*>(Destination),  // NOLINT
  48           reinterpret_cast<long>(Exchange),               // NOLINT
  49           reinterpret_cast<long>(Comparand)));            // NOLINT
  50 }
  51 #endif
  52
  53 namespace __sanitizer {
  54
  55 INLINE void atomic_signal_fence(memory_order) {
  56   _ReadWriteBarrier();
  57 }
  58
  59 INLINE void atomic_thread_fence(memory_order) {
  60   _mm_mfence();
  61 }
  62
  63 INLINE void proc_yield(int cnt) {
  64   for (int i = 0; i < cnt; i++)
  65     _mm_pause();
  66 }
  67
  68 template<typename T>
  69 INLINE typename T::Type atomic_load(
  70     const volatile T *a, memory_order mo) {
  71   DCHECK(mo & (memory_order_relaxed | memory_order_consume
  72       | memory_order_acquire | memory_order_seq_cst));
  73   DCHECK(!((uptr)a % sizeof(*a)));
  74   typename T::Type v;
  75   // FIXME(dvyukov): 64-bit load is not atomic on 32-bits.
  76   if (mo == memory_order_relaxed) {
  77     v = a->val_dont_use;
  78   } else {
  79     atomic_signal_fence(memory_order_seq_cst);
  80     v = a->val_dont_use;
  81     atomic_signal_fence(memory_order_seq_cst);
  82   }
  83   return v;
  84 }
  85
  86 template<typename T>
  87 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
  88   DCHECK(mo & (memory_order_relaxed | memory_order_release
  89       | memory_order_seq_cst));
  90   DCHECK(!((uptr)a % sizeof(*a)));
  91   // FIXME(dvyukov): 64-bit store is not atomic on 32-bits.
  92   if (mo == memory_order_relaxed) {
  93     a->val_dont_use = v;
  94   } else {
  95     atomic_signal_fence(memory_order_seq_cst);
  96     a->val_dont_use = v;
  97     atomic_signal_fence(memory_order_seq_cst);
  98   }
  99   if (mo == memory_order_seq_cst)
 100     atomic_thread_fence(memory_order_seq_cst);
 101 }
 102
 103 INLINE u32 atomic_fetch_add(volatile atomic_uint32_t *a,
 104     u32 v, memory_order mo) {
 105   (void)mo;
 106   DCHECK(!((uptr)a % sizeof(*a)));
 107   return (u32)_InterlockedExchangeAdd(
 108       (volatile long*)&a->val_dont_use, (long)v);  // NOLINT
 109 }
 110
 111 INLINE u8 atomic_exchange(volatile atomic_uint8_t *a,
 112     u8 v, memory_order mo) {
 113   (void)mo;
 114   DCHECK(!((uptr)a % sizeof(*a)));
 115   __asm {
 116     mov eax, a
 117     mov cl, v
 118     xchg [eax], cl  // NOLINT
 119     mov v, cl
 120   }
 121   return v;
 122 }
 123
 124 INLINE u16 atomic_exchange(volatile atomic_uint16_t *a,
 125     u16 v, memory_order mo) {
 126   (void)mo;
 127   DCHECK(!((uptr)a % sizeof(*a)));
 128   __asm {
 129     mov eax, a
 130     mov cx, v
 131     xchg [eax], cx  // NOLINT
 132     mov v, cx
 133   }
 134   return v;
 135 }
 136
 137 INLINE bool atomic_compare_exchange_strong(volatile atomic_uintptr_t *a,
 138                                            uptr *cmp,
 139                                            uptr xchg,
 140                                            memory_order mo) {
 141   uptr cmpv = *cmp;
 142   uptr prev = (uptr)_InterlockedCompareExchangePointer(
 143       (void*volatile*)&a->val_dont_use, (void*)xchg, (void*)cmpv);
 144   if (prev == cmpv)
 145     return true;
 146   *cmp = prev;
 147   return false;
 148 }
 149
 150 template<typename T>
 151 INLINE bool atomic_compare_exchange_weak(volatile T *a,
 152                                            typename T::Type *cmp,
 153                                            typename T::Type xchg,
 154                                            memory_order mo) {
 155   return atomic_compare_exchange_strong(a, cmp, xchg, mo);
 156 }
 157
 158 }  // namespace __sanitizer
 159
 160 #endif  // SANITIZER_ATOMIC_CLANG_H