libsanitizer/sanitizer_common/sanitizer_atomic_clang.h

   1 //===-- sanitizer_atomic_clang.h --------------------------------*- C++ -*-===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer/AddressSanitizer runtime.
   9 // Not intended for direct inclusion. Include sanitizer_atomic.h.
  10 //
  11 //===----------------------------------------------------------------------===//
  12
  13 #ifndef SANITIZER_ATOMIC_CLANG_H
  14 #define SANITIZER_ATOMIC_CLANG_H
  15
  16 namespace __sanitizer {
  17
  18 INLINE void atomic_signal_fence(memory_order) {
  19   __asm__ __volatile__("" ::: "memory");
  20 }
  21
  22 INLINE void atomic_thread_fence(memory_order) {
  23   __sync_synchronize();
  24 }
  25
  26 INLINE void proc_yield(int cnt) {
  27   __asm__ __volatile__("" ::: "memory");
  28 #if defined(__i386__) || defined(__x86_64__)
  29   for (int i = 0; i < cnt; i++)
  30     __asm__ __volatile__("pause");
  31 #endif
  32   __asm__ __volatile__("" ::: "memory");
  33 }
  34
  35 template<typename T>
  36 INLINE typename T::Type atomic_load(
  37     const volatile T *a, memory_order mo) {
  38   DCHECK(mo & (memory_order_relaxed | memory_order_consume
  39       | memory_order_acquire | memory_order_seq_cst));
  40   DCHECK(!((uptr)a % sizeof(*a)));
  41   typename T::Type v;
  42   // FIXME:
  43   // 64-bit atomic operations are not atomic on 32-bit platforms.
  44   // The implementation lacks necessary memory fences on ARM/PPC.
  45   // We would like to use compiler builtin atomic operations,
  46   // but they are mostly broken:
  47   // - they lead to vastly inefficient code generation
  48   // (http://llvm.org/bugs/show_bug.cgi?id=17281)
  49   // - 64-bit atomic operations are not implemented on x86_32
  50   // (http://llvm.org/bugs/show_bug.cgi?id=15034)
  51   // - they are not implemented on ARM
  52   // error: undefined reference to '__atomic_load_4'
  53   if (mo == memory_order_relaxed) {
  54     v = a->val_dont_use;
  55   } else {
  56     atomic_signal_fence(memory_order_seq_cst);
  57     v = a->val_dont_use;
  58     atomic_signal_fence(memory_order_seq_cst);
  59   }
  60   return v;
  61 }
  62
  63 template<typename T>
  64 INLINE void atomic_store(volatile T *a, typename T::Type v, memory_order mo) {
  65   DCHECK(mo & (memory_order_relaxed | memory_order_release
  66       | memory_order_seq_cst));
  67   DCHECK(!((uptr)a % sizeof(*a)));
  68   if (mo == memory_order_relaxed) {
  69     a->val_dont_use = v;
  70   } else {
  71     atomic_signal_fence(memory_order_seq_cst);
  72     a->val_dont_use = v;
  73     atomic_signal_fence(memory_order_seq_cst);
  74   }
  75   if (mo == memory_order_seq_cst)
  76     atomic_thread_fence(memory_order_seq_cst);
  77 }
  78
  79 template<typename T>
  80 INLINE typename T::Type atomic_fetch_add(volatile T *a,
  81     typename T::Type v, memory_order mo) {
  82   (void)mo;
  83   DCHECK(!((uptr)a % sizeof(*a)));
  84   return __sync_fetch_and_add(&a->val_dont_use, v);
  85 }
  86
  87 template<typename T>
  88 INLINE typename T::Type atomic_fetch_sub(volatile T *a,
  89     typename T::Type v, memory_order mo) {
  90   (void)mo;
  91   DCHECK(!((uptr)a % sizeof(*a)));
  92   return __sync_fetch_and_add(&a->val_dont_use, -v);
  93 }
  94
  95 template<typename T>
  96 INLINE typename T::Type atomic_exchange(volatile T *a,
  97     typename T::Type v, memory_order mo) {
  98   DCHECK(!((uptr)a % sizeof(*a)));
  99   if (mo & (memory_order_release | memory_order_acq_rel | memory_order_seq_cst))
 100     __sync_synchronize();
 101   v = __sync_lock_test_and_set(&a->val_dont_use, v);
 102   if (mo == memory_order_seq_cst)
 103     __sync_synchronize();
 104   return v;
 105 }
 106
 107 template<typename T>
 108 INLINE bool atomic_compare_exchange_strong(volatile T *a,
 109                                            typename T::Type *cmp,
 110                                            typename T::Type xchg,
 111                                            memory_order mo) {
 112   typedef typename T::Type Type;
 113   Type cmpv = *cmp;
 114   Type prev = __sync_val_compare_and_swap(&a->val_dont_use, cmpv, xchg);
 115   if (prev == cmpv)
 116     return true;
 117   *cmp = prev;
 118   return false;
 119 }
 120
 121 template<typename T>
 122 INLINE bool atomic_compare_exchange_weak(volatile T *a,
 123                                          typename T::Type *cmp,
 124                                          typename T::Type xchg,
 125                                          memory_order mo) {
 126   return atomic_compare_exchange_strong(a, cmp, xchg, mo);
 127 }
 128
 129 }  // namespace __sanitizer
 130
 131 #undef ATOMIC_ORDER
 132
 133 #endif  // SANITIZER_ATOMIC_CLANG_H