libsanitizer/tsan/tsan_interface_atomic.cc

   1 //===-- tsan_interface_atomic.cc ------------------------------------------===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer (TSan), a race detector.
   9 //
  10 //===----------------------------------------------------------------------===//
  11
  12 // ThreadSanitizer atomic operations are based on C++11/C1x standards.
  13 // For background see C++11 standard.  A slightly older, publicly
  14 // available draft of the standard (not entirely up-to-date, but close enough
  15 // for casual browsing) is available here:
  16 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
  17 // The following page contains more background information:
  18 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
  19
  20 #include "sanitizer_common/sanitizer_placement_new.h"
  21 #include "sanitizer_common/sanitizer_stacktrace.h"
  22 #include "sanitizer_common/sanitizer_mutex.h"
  23 #include "tsan_flags.h"
  24 #include "tsan_interface.h"
  25 #include "tsan_rtl.h"
  26
  27 using namespace __tsan;  // NOLINT
  28
  29 #if !SANITIZER_GO && __TSAN_HAS_INT128
  30 // Protects emulation of 128-bit atomic operations.
  31 static StaticSpinMutex mutex128;
  32 #endif
  33
  34 static bool IsLoadOrder(morder mo) {
  35   return mo == mo_relaxed || mo == mo_consume
  36       || mo == mo_acquire || mo == mo_seq_cst;
  37 }
  38
  39 static bool IsStoreOrder(morder mo) {
  40   return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
  41 }
  42
  43 static bool IsReleaseOrder(morder mo) {
  44   return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
  45 }
  46
  47 static bool IsAcquireOrder(morder mo) {
  48   return mo == mo_consume || mo == mo_acquire
  49       || mo == mo_acq_rel || mo == mo_seq_cst;
  50 }
  51
  52 static bool IsAcqRelOrder(morder mo) {
  53   return mo == mo_acq_rel || mo == mo_seq_cst;
  54 }
  55
  56 template<typename T> T func_xchg(volatile T *v, T op) {
  57   T res = __sync_lock_test_and_set(v, op);
  58   // __sync_lock_test_and_set does not contain full barrier.
  59   __sync_synchronize();
  60   return res;
  61 }
  62
  63 template<typename T> T func_add(volatile T *v, T op) {
  64   return __sync_fetch_and_add(v, op);
  65 }
  66
  67 template<typename T> T func_sub(volatile T *v, T op) {
  68   return __sync_fetch_and_sub(v, op);
  69 }
  70
  71 template<typename T> T func_and(volatile T *v, T op) {
  72   return __sync_fetch_and_and(v, op);
  73 }
  74
  75 template<typename T> T func_or(volatile T *v, T op) {
  76   return __sync_fetch_and_or(v, op);
  77 }
  78
  79 template<typename T> T func_xor(volatile T *v, T op) {
  80   return __sync_fetch_and_xor(v, op);
  81 }
  82
  83 template<typename T> T func_nand(volatile T *v, T op) {
  84   // clang does not support __sync_fetch_and_nand.
  85   T cmp = *v;
  86   for (;;) {
  87     T newv = ~(cmp & op);
  88     T cur = __sync_val_compare_and_swap(v, cmp, newv);
  89     if (cmp == cur)
  90       return cmp;
  91     cmp = cur;
  92   }
  93 }
  94
  95 template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
  96   return __sync_val_compare_and_swap(v, cmp, xch);
  97 }
  98
  99 // clang does not support 128-bit atomic ops.
 100 // Atomic ops are executed under tsan internal mutex,
 101 // here we assume that the atomic variables are not accessed
 102 // from non-instrumented code.
 103 #if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \
 104     && __TSAN_HAS_INT128
 105 a128 func_xchg(volatile a128 *v, a128 op) {
 106   SpinMutexLock lock(&mutex128);
 107   a128 cmp = *v;
 108   *v = op;
 109   return cmp;
 110 }
 111
 112 a128 func_add(volatile a128 *v, a128 op) {
 113   SpinMutexLock lock(&mutex128);
 114   a128 cmp = *v;
 115   *v = cmp + op;
 116   return cmp;
 117 }
 118
 119 a128 func_sub(volatile a128 *v, a128 op) {
 120   SpinMutexLock lock(&mutex128);
 121   a128 cmp = *v;
 122   *v = cmp - op;
 123   return cmp;
 124 }
 125
 126 a128 func_and(volatile a128 *v, a128 op) {
 127   SpinMutexLock lock(&mutex128);
 128   a128 cmp = *v;
 129   *v = cmp & op;
 130   return cmp;
 131 }
 132
 133 a128 func_or(volatile a128 *v, a128 op) {
 134   SpinMutexLock lock(&mutex128);
 135   a128 cmp = *v;
 136   *v = cmp | op;
 137   return cmp;
 138 }
 139
 140 a128 func_xor(volatile a128 *v, a128 op) {
 141   SpinMutexLock lock(&mutex128);
 142   a128 cmp = *v;
 143   *v = cmp ^ op;
 144   return cmp;
 145 }
 146
 147 a128 func_nand(volatile a128 *v, a128 op) {
 148   SpinMutexLock lock(&mutex128);
 149   a128 cmp = *v;
 150   *v = ~(cmp & op);
 151   return cmp;
 152 }
 153
 154 a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
 155   SpinMutexLock lock(&mutex128);
 156   a128 cur = *v;
 157   if (cur == cmp)
 158     *v = xch;
 159   return cur;
 160 }
 161 #endif
 162
 163 template<typename T>
 164 static int SizeLog() {
 165   if (sizeof(T) <= 1)
 166     return kSizeLog1;
 167   else if (sizeof(T) <= 2)
 168     return kSizeLog2;
 169   else if (sizeof(T) <= 4)
 170     return kSizeLog4;
 171   else
 172     return kSizeLog8;
 173   // For 16-byte atomics we also use 8-byte memory access,
 174   // this leads to false negatives only in very obscure cases.
 175 }
 176
 177 #if !SANITIZER_GO
 178 static atomic_uint8_t *to_atomic(const volatile a8 *a) {
 179   return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a));
 180 }
 181
 182 static atomic_uint16_t *to_atomic(const volatile a16 *a) {
 183   return reinterpret_cast<atomic_uint16_t *>(const_cast<a16 *>(a));
 184 }
 185 #endif
 186
 187 static atomic_uint32_t *to_atomic(const volatile a32 *a) {
 188   return reinterpret_cast<atomic_uint32_t *>(const_cast<a32 *>(a));
 189 }
 190
 191 static atomic_uint64_t *to_atomic(const volatile a64 *a) {
 192   return reinterpret_cast<atomic_uint64_t *>(const_cast<a64 *>(a));
 193 }
 194
 195 static memory_order to_mo(morder mo) {
 196   switch (mo) {
 197   case mo_relaxed: return memory_order_relaxed;
 198   case mo_consume: return memory_order_consume;
 199   case mo_acquire: return memory_order_acquire;
 200   case mo_release: return memory_order_release;
 201   case mo_acq_rel: return memory_order_acq_rel;
 202   case mo_seq_cst: return memory_order_seq_cst;
 203   }
 204   CHECK(0);
 205   return memory_order_seq_cst;
 206 }
 207
 208 template<typename T>
 209 static T NoTsanAtomicLoad(const volatile T *a, morder mo) {
 210   return atomic_load(to_atomic(a), to_mo(mo));
 211 }
 212
 213 #if __TSAN_HAS_INT128 && !SANITIZER_GO
 214 static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
 215   SpinMutexLock lock(&mutex128);
 216   return *a;
 217 }
 218 #endif
 219
 220 template<typename T>
 221 static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a, morder mo) {
 222   CHECK(IsLoadOrder(mo));
 223   // This fast-path is critical for performance.
 224   // Assume the access is atomic.
 225   if (!IsAcquireOrder(mo)) {
 226     MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
 227     return NoTsanAtomicLoad(a, mo);
 228   }
 229   // Don't create sync object if it does not exist yet. For example, an atomic
 230   // pointer is initialized to nullptr and then periodically acquire-loaded.
 231   T v = NoTsanAtomicLoad(a, mo);
 232   SyncVar *s = ctx->metamap.GetIfExistsAndLock((uptr)a, false);
 233   if (s) {
 234     AcquireImpl(thr, pc, &s->clock);
 235     // Re-read under sync mutex because we need a consistent snapshot
 236     // of the value and the clock we acquire.
 237     v = NoTsanAtomicLoad(a, mo);
 238     s->mtx.ReadUnlock();
 239   }
 240   MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
 241   return v;
 242 }
 243
 244 template<typename T>
 245 static void NoTsanAtomicStore(volatile T *a, T v, morder mo) {
 246   atomic_store(to_atomic(a), v, to_mo(mo));
 247 }
 248
 249 #if __TSAN_HAS_INT128 && !SANITIZER_GO
 250 static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
 251   SpinMutexLock lock(&mutex128);
 252   *a = v;
 253 }
 254 #endif
 255
 256 template<typename T>
 257 static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
 258     morder mo) {
 259   CHECK(IsStoreOrder(mo));
 260   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
 261   // This fast-path is critical for performance.
 262   // Assume the access is atomic.
 263   // Strictly saying even relaxed store cuts off release sequence,
 264   // so must reset the clock.
 265   if (!IsReleaseOrder(mo)) {
 266     NoTsanAtomicStore(a, v, mo);
 267     return;
 268   }
 269   __sync_synchronize();
 270   SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
 271   thr->fast_state.IncrementEpoch();
 272   // Can't increment epoch w/o writing to the trace as well.
 273   TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
 274   ReleaseStoreImpl(thr, pc, &s->clock);
 275   NoTsanAtomicStore(a, v, mo);
 276   s->mtx.Unlock();
 277 }
 278
 279 template<typename T, T (*F)(volatile T *v, T op)>
 280 static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
 281   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
 282   SyncVar *s = 0;
 283   if (mo != mo_relaxed) {
 284     s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
 285     thr->fast_state.IncrementEpoch();
 286     // Can't increment epoch w/o writing to the trace as well.
 287     TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
 288     if (IsAcqRelOrder(mo))
 289       AcquireReleaseImpl(thr, pc, &s->clock);
 290     else if (IsReleaseOrder(mo))
 291       ReleaseImpl(thr, pc, &s->clock);
 292     else if (IsAcquireOrder(mo))
 293       AcquireImpl(thr, pc, &s->clock);
 294   }
 295   v = F(a, v);
 296   if (s)
 297     s->mtx.Unlock();
 298   return v;
 299 }
 300
 301 template<typename T>
 302 static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) {
 303   return func_xchg(a, v);
 304 }
 305
 306 template<typename T>
 307 static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) {
 308   return func_add(a, v);
 309 }
 310
 311 template<typename T>
 312 static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) {
 313   return func_sub(a, v);
 314 }
 315
 316 template<typename T>
 317 static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) {
 318   return func_and(a, v);
 319 }
 320
 321 template<typename T>
 322 static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) {
 323   return func_or(a, v);
 324 }
 325
 326 template<typename T>
 327 static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) {
 328   return func_xor(a, v);
 329 }
 330
 331 template<typename T>
 332 static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) {
 333   return func_nand(a, v);
 334 }
 335
 336 template<typename T>
 337 static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v,
 338     morder mo) {
 339   return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
 340 }
 341
 342 template<typename T>
 343 static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v,
 344     morder mo) {
 345   return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
 346 }
 347
 348 template<typename T>
 349 static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v,
 350     morder mo) {
 351   return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
 352 }
 353
 354 template<typename T>
 355 static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v,
 356     morder mo) {
 357   return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
 358 }
 359
 360 template<typename T>
 361 static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v,
 362     morder mo) {
 363   return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
 364 }
 365
 366 template<typename T>
 367 static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v,
 368     morder mo) {
 369   return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
 370 }
 371
 372 template<typename T>
 373 static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v,
 374     morder mo) {
 375   return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
 376 }
 377
 378 template<typename T>
 379 static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) {
 380   return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
 381 }
 382
 383 #if __TSAN_HAS_INT128
 384 static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v,
 385     morder mo, morder fmo) {
 386   a128 old = *c;
 387   a128 cur = func_cas(a, old, v);
 388   if (cur == old)
 389     return true;
 390   *c = cur;
 391   return false;
 392 }
 393 #endif
 394
 395 template<typename T>
 396 static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
 397   NoTsanAtomicCAS(a, &c, v, mo, fmo);
 398   return c;
 399 }
 400
 401 template<typename T>
 402 static bool AtomicCAS(ThreadState *thr, uptr pc,
 403     volatile T *a, T *c, T v, morder mo, morder fmo) {
 404   (void)fmo;  // Unused because llvm does not pass it yet.
 405   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
 406   SyncVar *s = 0;
 407   bool write_lock = mo != mo_acquire && mo != mo_consume;
 408   if (mo != mo_relaxed) {
 409     s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock);
 410     thr->fast_state.IncrementEpoch();
 411     // Can't increment epoch w/o writing to the trace as well.
 412     TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
 413     if (IsAcqRelOrder(mo))
 414       AcquireReleaseImpl(thr, pc, &s->clock);
 415     else if (IsReleaseOrder(mo))
 416       ReleaseImpl(thr, pc, &s->clock);
 417     else if (IsAcquireOrder(mo))
 418       AcquireImpl(thr, pc, &s->clock);
 419   }
 420   T cc = *c;
 421   T pr = func_cas(a, cc, v);
 422   if (s) {
 423     if (write_lock)
 424       s->mtx.Unlock();
 425     else
 426       s->mtx.ReadUnlock();
 427   }
 428   if (pr == cc)
 429     return true;
 430   *c = pr;
 431   return false;
 432 }
 433
 434 template<typename T>
 435 static T AtomicCAS(ThreadState *thr, uptr pc,
 436     volatile T *a, T c, T v, morder mo, morder fmo) {
 437   AtomicCAS(thr, pc, a, &c, v, mo, fmo);
 438   return c;
 439 }
 440
 441 #if !SANITIZER_GO
 442 static void NoTsanAtomicFence(morder mo) {
 443   __sync_synchronize();
 444 }
 445
 446 static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
 447   // FIXME(dvyukov): not implemented.
 448   __sync_synchronize();
 449 }
 450 #endif
 451
 452 // Interface functions follow.
 453 #if !SANITIZER_GO
 454
 455 // C/C++
 456
 457 static morder convert_morder(morder mo) {
 458   if (flags()->force_seq_cst_atomics)
 459     return (morder)mo_seq_cst;
 460
 461   // Filter out additional memory order flags:
 462   // MEMMODEL_SYNC        = 1 << 15
 463   // __ATOMIC_HLE_ACQUIRE = 1 << 16
 464   // __ATOMIC_HLE_RELEASE = 1 << 17
 465   //
 466   // HLE is an optimization, and we pretend that elision always fails.
 467   // MEMMODEL_SYNC is used when lowering __sync_ atomics,
 468   // since we use __sync_ atomics for actual atomic operations,
 469   // we can safely ignore it as well. It also subtly affects semantics,
 470   // but we don't model the difference.
 471   return (morder)(mo & 0x7fff);
 472 }
 473
 474 #define SCOPED_ATOMIC(func, ...) \
 475     ThreadState *const thr = cur_thread(); \
 476     if (thr->ignore_sync || thr->ignore_interceptors) { \
 477       ProcessPendingSignals(thr); \
 478       return NoTsanAtomic##func(__VA_ARGS__); \
 479     } \
 480     const uptr callpc = (uptr)__builtin_return_address(0); \
 481     uptr pc = StackTrace::GetCurrentPc(); \
 482     mo = convert_morder(mo); \
 483     AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \
 484     ScopedAtomic sa(thr, callpc, a, mo, __func__); \
 485     return Atomic##func(thr, pc, __VA_ARGS__); \
 486 /**/
 487
 488 class ScopedAtomic {
 489  public:
 490   ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a,
 491                morder mo, const char *func)
 492       : thr_(thr) {
 493     FuncEntry(thr_, pc);
 494     DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo);
 495   }
 496   ~ScopedAtomic() {
 497     ProcessPendingSignals(thr_);
 498     FuncExit(thr_);
 499   }
 500  private:
 501   ThreadState *thr_;
 502 };
 503
 504 static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) {
 505   StatInc(thr, StatAtomic);
 506   StatInc(thr, t);
 507   StatInc(thr, size == 1 ? StatAtomic1
 508              : size == 2 ? StatAtomic2
 509              : size == 4 ? StatAtomic4
 510              : size == 8 ? StatAtomic8
 511              :             StatAtomic16);
 512   StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed
 513              : mo == mo_consume ? StatAtomicConsume
 514              : mo == mo_acquire ? StatAtomicAcquire
 515              : mo == mo_release ? StatAtomicRelease
 516              : mo == mo_acq_rel ? StatAtomicAcq_Rel
 517              :                    StatAtomicSeq_Cst);
 518 }
 519
 520 extern "C" {
 521 SANITIZER_INTERFACE_ATTRIBUTE
 522 a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
 523   SCOPED_ATOMIC(Load, a, mo);
 524 }
 525
 526 SANITIZER_INTERFACE_ATTRIBUTE
 527 a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
 528   SCOPED_ATOMIC(Load, a, mo);
 529 }
 530
 531 SANITIZER_INTERFACE_ATTRIBUTE
 532 a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
 533   SCOPED_ATOMIC(Load, a, mo);
 534 }
 535
 536 SANITIZER_INTERFACE_ATTRIBUTE
 537 a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
 538   SCOPED_ATOMIC(Load, a, mo);
 539 }
 540
 541 #if __TSAN_HAS_INT128
 542 SANITIZER_INTERFACE_ATTRIBUTE
 543 a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
 544   SCOPED_ATOMIC(Load, a, mo);
 545 }
 546 #endif
 547
 548 SANITIZER_INTERFACE_ATTRIBUTE
 549 void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
 550   SCOPED_ATOMIC(Store, a, v, mo);
 551 }
 552
 553 SANITIZER_INTERFACE_ATTRIBUTE
 554 void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
 555   SCOPED_ATOMIC(Store, a, v, mo);
 556 }
 557
 558 SANITIZER_INTERFACE_ATTRIBUTE
 559 void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
 560   SCOPED_ATOMIC(Store, a, v, mo);
 561 }
 562
 563 SANITIZER_INTERFACE_ATTRIBUTE
 564 void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
 565   SCOPED_ATOMIC(Store, a, v, mo);
 566 }
 567
 568 #if __TSAN_HAS_INT128
 569 SANITIZER_INTERFACE_ATTRIBUTE
 570 void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
 571   SCOPED_ATOMIC(Store, a, v, mo);
 572 }
 573 #endif
 574
 575 SANITIZER_INTERFACE_ATTRIBUTE
 576 a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
 577   SCOPED_ATOMIC(Exchange, a, v, mo);
 578 }
 579
 580 SANITIZER_INTERFACE_ATTRIBUTE
 581 a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
 582   SCOPED_ATOMIC(Exchange, a, v, mo);
 583 }
 584
 585 SANITIZER_INTERFACE_ATTRIBUTE
 586 a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
 587   SCOPED_ATOMIC(Exchange, a, v, mo);
 588 }
 589
 590 SANITIZER_INTERFACE_ATTRIBUTE
 591 a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
 592   SCOPED_ATOMIC(Exchange, a, v, mo);
 593 }
 594
 595 #if __TSAN_HAS_INT128
 596 SANITIZER_INTERFACE_ATTRIBUTE
 597 a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
 598   SCOPED_ATOMIC(Exchange, a, v, mo);
 599 }
 600 #endif
 601
 602 SANITIZER_INTERFACE_ATTRIBUTE
 603 a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
 604   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 605 }
 606
 607 SANITIZER_INTERFACE_ATTRIBUTE
 608 a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
 609   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 610 }
 611
 612 SANITIZER_INTERFACE_ATTRIBUTE
 613 a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
 614   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 615 }
 616
 617 SANITIZER_INTERFACE_ATTRIBUTE
 618 a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
 619   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 620 }
 621
 622 #if __TSAN_HAS_INT128
 623 SANITIZER_INTERFACE_ATTRIBUTE
 624 a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
 625   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 626 }
 627 #endif
 628
 629 SANITIZER_INTERFACE_ATTRIBUTE
 630 a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
 631   SCOPED_ATOMIC(FetchSub, a, v, mo);
 632 }
 633
 634 SANITIZER_INTERFACE_ATTRIBUTE
 635 a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
 636   SCOPED_ATOMIC(FetchSub, a, v, mo);
 637 }
 638
 639 SANITIZER_INTERFACE_ATTRIBUTE
 640 a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
 641   SCOPED_ATOMIC(FetchSub, a, v, mo);
 642 }
 643
 644 SANITIZER_INTERFACE_ATTRIBUTE
 645 a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
 646   SCOPED_ATOMIC(FetchSub, a, v, mo);
 647 }
 648
 649 #if __TSAN_HAS_INT128
 650 SANITIZER_INTERFACE_ATTRIBUTE
 651 a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
 652   SCOPED_ATOMIC(FetchSub, a, v, mo);
 653 }
 654 #endif
 655
 656 SANITIZER_INTERFACE_ATTRIBUTE
 657 a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
 658   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 659 }
 660
 661 SANITIZER_INTERFACE_ATTRIBUTE
 662 a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
 663   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 664 }
 665
 666 SANITIZER_INTERFACE_ATTRIBUTE
 667 a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
 668   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 669 }
 670
 671 SANITIZER_INTERFACE_ATTRIBUTE
 672 a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
 673   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 674 }
 675
 676 #if __TSAN_HAS_INT128
 677 SANITIZER_INTERFACE_ATTRIBUTE
 678 a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
 679   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 680 }
 681 #endif
 682
 683 SANITIZER_INTERFACE_ATTRIBUTE
 684 a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
 685   SCOPED_ATOMIC(FetchOr, a, v, mo);
 686 }
 687
 688 SANITIZER_INTERFACE_ATTRIBUTE
 689 a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
 690   SCOPED_ATOMIC(FetchOr, a, v, mo);
 691 }
 692
 693 SANITIZER_INTERFACE_ATTRIBUTE
 694 a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
 695   SCOPED_ATOMIC(FetchOr, a, v, mo);
 696 }
 697
 698 SANITIZER_INTERFACE_ATTRIBUTE
 699 a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
 700   SCOPED_ATOMIC(FetchOr, a, v, mo);
 701 }
 702
 703 #if __TSAN_HAS_INT128
 704 SANITIZER_INTERFACE_ATTRIBUTE
 705 a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
 706   SCOPED_ATOMIC(FetchOr, a, v, mo);
 707 }
 708 #endif
 709
 710 SANITIZER_INTERFACE_ATTRIBUTE
 711 a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
 712   SCOPED_ATOMIC(FetchXor, a, v, mo);
 713 }
 714
 715 SANITIZER_INTERFACE_ATTRIBUTE
 716 a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
 717   SCOPED_ATOMIC(FetchXor, a, v, mo);
 718 }
 719
 720 SANITIZER_INTERFACE_ATTRIBUTE
 721 a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
 722   SCOPED_ATOMIC(FetchXor, a, v, mo);
 723 }
 724
 725 SANITIZER_INTERFACE_ATTRIBUTE
 726 a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
 727   SCOPED_ATOMIC(FetchXor, a, v, mo);
 728 }
 729
 730 #if __TSAN_HAS_INT128
 731 SANITIZER_INTERFACE_ATTRIBUTE
 732 a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
 733   SCOPED_ATOMIC(FetchXor, a, v, mo);
 734 }
 735 #endif
 736
 737 SANITIZER_INTERFACE_ATTRIBUTE
 738 a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
 739   SCOPED_ATOMIC(FetchNand, a, v, mo);
 740 }
 741
 742 SANITIZER_INTERFACE_ATTRIBUTE
 743 a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
 744   SCOPED_ATOMIC(FetchNand, a, v, mo);
 745 }
 746
 747 SANITIZER_INTERFACE_ATTRIBUTE
 748 a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
 749   SCOPED_ATOMIC(FetchNand, a, v, mo);
 750 }
 751
 752 SANITIZER_INTERFACE_ATTRIBUTE
 753 a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
 754   SCOPED_ATOMIC(FetchNand, a, v, mo);
 755 }
 756
 757 #if __TSAN_HAS_INT128
 758 SANITIZER_INTERFACE_ATTRIBUTE
 759 a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
 760   SCOPED_ATOMIC(FetchNand, a, v, mo);
 761 }
 762 #endif
 763
 764 SANITIZER_INTERFACE_ATTRIBUTE
 765 int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
 766     morder mo, morder fmo) {
 767   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 768 }
 769
 770 SANITIZER_INTERFACE_ATTRIBUTE
 771 int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
 772     morder mo, morder fmo) {
 773   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 774 }
 775
 776 SANITIZER_INTERFACE_ATTRIBUTE
 777 int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
 778     morder mo, morder fmo) {
 779   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 780 }
 781
 782 SANITIZER_INTERFACE_ATTRIBUTE
 783 int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
 784     morder mo, morder fmo) {
 785   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 786 }
 787
 788 #if __TSAN_HAS_INT128
 789 SANITIZER_INTERFACE_ATTRIBUTE
 790 int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
 791     morder mo, morder fmo) {
 792   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 793 }
 794 #endif
 795
 796 SANITIZER_INTERFACE_ATTRIBUTE
 797 int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v,
 798     morder mo, morder fmo) {
 799   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 800 }
 801
 802 SANITIZER_INTERFACE_ATTRIBUTE
 803 int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
 804     morder mo, morder fmo) {
 805   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 806 }
 807
 808 SANITIZER_INTERFACE_ATTRIBUTE
 809 int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
 810     morder mo, morder fmo) {
 811   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 812 }
 813
 814 SANITIZER_INTERFACE_ATTRIBUTE
 815 int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
 816     morder mo, morder fmo) {
 817   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 818 }
 819
 820 #if __TSAN_HAS_INT128
 821 SANITIZER_INTERFACE_ATTRIBUTE
 822 int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
 823     morder mo, morder fmo) {
 824   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 825 }
 826 #endif
 827
 828 SANITIZER_INTERFACE_ATTRIBUTE
 829 a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
 830     morder mo, morder fmo) {
 831   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 832 }
 833
 834 SANITIZER_INTERFACE_ATTRIBUTE
 835 a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
 836     morder mo, morder fmo) {
 837   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 838 }
 839
 840 SANITIZER_INTERFACE_ATTRIBUTE
 841 a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
 842     morder mo, morder fmo) {
 843   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 844 }
 845
 846 SANITIZER_INTERFACE_ATTRIBUTE
 847 a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
 848     morder mo, morder fmo) {
 849   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 850 }
 851
 852 #if __TSAN_HAS_INT128
 853 SANITIZER_INTERFACE_ATTRIBUTE
 854 a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
 855     morder mo, morder fmo) {
 856   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 857 }
 858 #endif
 859
 860 SANITIZER_INTERFACE_ATTRIBUTE
 861 void __tsan_atomic_thread_fence(morder mo) {
 862   char* a = 0;
 863   SCOPED_ATOMIC(Fence, mo);
 864 }
 865
 866 SANITIZER_INTERFACE_ATTRIBUTE
 867 void __tsan_atomic_signal_fence(morder mo) {
 868 }
 869 }  // extern "C"
 870
 871 #else  // #if !SANITIZER_GO
 872
 873 // Go
 874
 875 #define ATOMIC(func, ...) \
 876     if (thr->ignore_sync) { \
 877       NoTsanAtomic##func(__VA_ARGS__); \
 878     } else { \
 879       FuncEntry(thr, cpc); \
 880       Atomic##func(thr, pc, __VA_ARGS__); \
 881       FuncExit(thr); \
 882     } \
 883 /**/
 884
 885 #define ATOMIC_RET(func, ret, ...) \
 886     if (thr->ignore_sync) { \
 887       (ret) = NoTsanAtomic##func(__VA_ARGS__); \
 888     } else { \
 889       FuncEntry(thr, cpc); \
 890       (ret) = Atomic##func(thr, pc, __VA_ARGS__); \
 891       FuncExit(thr); \
 892     } \
 893 /**/
 894
 895 extern "C" {
 896 SANITIZER_INTERFACE_ATTRIBUTE
 897 void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 898   ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire);
 899 }
 900
 901 SANITIZER_INTERFACE_ATTRIBUTE
 902 void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 903   ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire);
 904 }
 905
 906 SANITIZER_INTERFACE_ATTRIBUTE
 907 void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 908   ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release);
 909 }
 910
 911 SANITIZER_INTERFACE_ATTRIBUTE
 912 void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 913   ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release);
 914 }
 915
 916 SANITIZER_INTERFACE_ATTRIBUTE
 917 void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 918   ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
 919 }
 920
 921 SANITIZER_INTERFACE_ATTRIBUTE
 922 void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 923   ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
 924 }
 925
 926 SANITIZER_INTERFACE_ATTRIBUTE
 927 void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 928   ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
 929 }
 930
 931 SANITIZER_INTERFACE_ATTRIBUTE
 932 void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 933   ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
 934 }
 935
 936 SANITIZER_INTERFACE_ATTRIBUTE
 937 void __tsan_go_atomic32_compare_exchange(
 938     ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 939   a32 cur = 0;
 940   a32 cmp = *(a32*)(a+8);
 941   ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire);
 942   *(bool*)(a+16) = (cur == cmp);
 943 }
 944
 945 SANITIZER_INTERFACE_ATTRIBUTE
 946 void __tsan_go_atomic64_compare_exchange(
 947     ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 948   a64 cur = 0;
 949   a64 cmp = *(a64*)(a+8);
 950   ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire);
 951   *(bool*)(a+24) = (cur == cmp);
 952 }
 953 }  // extern "C"
 954 #endif  // #if !SANITIZER_GO