libsanitizer/tsan/tsan_interface_atomic.cc

   1 //===-- tsan_interface_atomic.cc ------------------------------------------===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer (TSan), a race detector.
   9 //
  10 //===----------------------------------------------------------------------===//
  11
  12 // ThreadSanitizer atomic operations are based on C++11/C1x standards.
  13 // For background see C++11 standard.  A slightly older, publicly
  14 // available draft of the standard (not entirely up-to-date, but close enough
  15 // for casual browsing) is available here:
  16 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
  17 // The following page contains more background information:
  18 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
  19
  20 #include "sanitizer_common/sanitizer_placement_new.h"
  21 #include "sanitizer_common/sanitizer_stacktrace.h"
  22 #include "sanitizer_common/sanitizer_mutex.h"
  23 #include "tsan_flags.h"
  24 #include "tsan_interface.h"
  25 #include "tsan_rtl.h"
  26
  27 using namespace __tsan;  // NOLINT
  28
  29 #if !SANITIZER_GO && __TSAN_HAS_INT128
  30 // Protects emulation of 128-bit atomic operations.
  31 static StaticSpinMutex mutex128;
  32 #endif
  33
  34 static bool IsLoadOrder(morder mo) {
  35   return mo == mo_relaxed || mo == mo_consume
  36       || mo == mo_acquire || mo == mo_seq_cst;
  37 }
  38
  39 static bool IsStoreOrder(morder mo) {
  40   return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
  41 }
  42
  43 static bool IsReleaseOrder(morder mo) {
  44   return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
  45 }
  46
  47 static bool IsAcquireOrder(morder mo) {
  48   return mo == mo_consume || mo == mo_acquire
  49       || mo == mo_acq_rel || mo == mo_seq_cst;
  50 }
  51
  52 static bool IsAcqRelOrder(morder mo) {
  53   return mo == mo_acq_rel || mo == mo_seq_cst;
  54 }
  55
  56 template<typename T> T func_xchg(volatile T *v, T op) {
  57   T res = __sync_lock_test_and_set(v, op);
  58   // __sync_lock_test_and_set does not contain full barrier.
  59   __sync_synchronize();
  60   return res;
  61 }
  62
  63 template<typename T> T func_add(volatile T *v, T op) {
  64   return __sync_fetch_and_add(v, op);
  65 }
  66
  67 template<typename T> T func_sub(volatile T *v, T op) {
  68   return __sync_fetch_and_sub(v, op);
  69 }
  70
  71 template<typename T> T func_and(volatile T *v, T op) {
  72   return __sync_fetch_and_and(v, op);
  73 }
  74
  75 template<typename T> T func_or(volatile T *v, T op) {
  76   return __sync_fetch_and_or(v, op);
  77 }
  78
  79 template<typename T> T func_xor(volatile T *v, T op) {
  80   return __sync_fetch_and_xor(v, op);
  81 }
  82
  83 template<typename T> T func_nand(volatile T *v, T op) {
  84   // clang does not support __sync_fetch_and_nand.
  85   T cmp = *v;
  86   for (;;) {
  87     T newv = ~(cmp & op);
  88     T cur = __sync_val_compare_and_swap(v, cmp, newv);
  89     if (cmp == cur)
  90       return cmp;
  91     cmp = cur;
  92   }
  93 }
  94
  95 template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
  96   return __sync_val_compare_and_swap(v, cmp, xch);
  97 }
  98
  99 // clang does not support 128-bit atomic ops.
 100 // Atomic ops are executed under tsan internal mutex,
 101 // here we assume that the atomic variables are not accessed
 102 // from non-instrumented code.
 103 #if !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16) && !SANITIZER_GO \
 104     && __TSAN_HAS_INT128
 105 a128 func_xchg(volatile a128 *v, a128 op) {
 106   SpinMutexLock lock(&mutex128);
 107   a128 cmp = *v;
 108   *v = op;
 109   return cmp;
 110 }
 111
 112 a128 func_add(volatile a128 *v, a128 op) {
 113   SpinMutexLock lock(&mutex128);
 114   a128 cmp = *v;
 115   *v = cmp + op;
 116   return cmp;
 117 }
 118
 119 a128 func_sub(volatile a128 *v, a128 op) {
 120   SpinMutexLock lock(&mutex128);
 121   a128 cmp = *v;
 122   *v = cmp - op;
 123   return cmp;
 124 }
 125
 126 a128 func_and(volatile a128 *v, a128 op) {
 127   SpinMutexLock lock(&mutex128);
 128   a128 cmp = *v;
 129   *v = cmp & op;
 130   return cmp;
 131 }
 132
 133 a128 func_or(volatile a128 *v, a128 op) {
 134   SpinMutexLock lock(&mutex128);
 135   a128 cmp = *v;
 136   *v = cmp | op;
 137   return cmp;
 138 }
 139
 140 a128 func_xor(volatile a128 *v, a128 op) {
 141   SpinMutexLock lock(&mutex128);
 142   a128 cmp = *v;
 143   *v = cmp ^ op;
 144   return cmp;
 145 }
 146
 147 a128 func_nand(volatile a128 *v, a128 op) {
 148   SpinMutexLock lock(&mutex128);
 149   a128 cmp = *v;
 150   *v = ~(cmp & op);
 151   return cmp;
 152 }
 153
 154 a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
 155   SpinMutexLock lock(&mutex128);
 156   a128 cur = *v;
 157   if (cur == cmp)
 158     *v = xch;
 159   return cur;
 160 }
 161 #endif
 162
 163 template<typename T>
 164 static int SizeLog() {
 165   if (sizeof(T) <= 1)
 166     return kSizeLog1;
 167   else if (sizeof(T) <= 2)
 168     return kSizeLog2;
 169   else if (sizeof(T) <= 4)
 170     return kSizeLog4;
 171   else
 172     return kSizeLog8;
 173   // For 16-byte atomics we also use 8-byte memory access,
 174   // this leads to false negatives only in very obscure cases.
 175 }
 176
 177 #if !SANITIZER_GO
 178 static atomic_uint8_t *to_atomic(const volatile a8 *a) {
 179   return reinterpret_cast<atomic_uint8_t *>(const_cast<a8 *>(a));
 180 }
 181
 182 static atomic_uint16_t *to_atomic(const volatile a16 *a) {
 183   return reinterpret_cast<atomic_uint16_t *>(const_cast<a16 *>(a));
 184 }
 185 #endif
 186
 187 static atomic_uint32_t *to_atomic(const volatile a32 *a) {
 188   return reinterpret_cast<atomic_uint32_t *>(const_cast<a32 *>(a));
 189 }
 190
 191 static atomic_uint64_t *to_atomic(const volatile a64 *a) {
 192   return reinterpret_cast<atomic_uint64_t *>(const_cast<a64 *>(a));
 193 }
 194
 195 static memory_order to_mo(morder mo) {
 196   switch (mo) {
 197   case mo_relaxed: return memory_order_relaxed;
 198   case mo_consume: return memory_order_consume;
 199   case mo_acquire: return memory_order_acquire;
 200   case mo_release: return memory_order_release;
 201   case mo_acq_rel: return memory_order_acq_rel;
 202   case mo_seq_cst: return memory_order_seq_cst;
 203   }
 204   CHECK(0);
 205   return memory_order_seq_cst;
 206 }
 207
 208 template<typename T>
 209 static T NoTsanAtomicLoad(const volatile T *a, morder mo) {
 210   return atomic_load(to_atomic(a), to_mo(mo));
 211 }
 212
 213 #if __TSAN_HAS_INT128 && !SANITIZER_GO
 214 static a128 NoTsanAtomicLoad(const volatile a128 *a, morder mo) {
 215   SpinMutexLock lock(&mutex128);
 216   return *a;
 217 }
 218 #endif
 219
 220 template<typename T>
 221 static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
 222     morder mo) {
 223   CHECK(IsLoadOrder(mo));
 224   // This fast-path is critical for performance.
 225   // Assume the access is atomic.
 226   if (!IsAcquireOrder(mo)) {
 227     MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
 228     return NoTsanAtomicLoad(a, mo);
 229   }
 230   SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, false);
 231   AcquireImpl(thr, pc, &s->clock);
 232   T v = NoTsanAtomicLoad(a, mo);
 233   s->mtx.ReadUnlock();
 234   MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
 235   return v;
 236 }
 237
 238 template<typename T>
 239 static void NoTsanAtomicStore(volatile T *a, T v, morder mo) {
 240   atomic_store(to_atomic(a), v, to_mo(mo));
 241 }
 242
 243 #if __TSAN_HAS_INT128 && !SANITIZER_GO
 244 static void NoTsanAtomicStore(volatile a128 *a, a128 v, morder mo) {
 245   SpinMutexLock lock(&mutex128);
 246   *a = v;
 247 }
 248 #endif
 249
 250 template<typename T>
 251 static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
 252     morder mo) {
 253   CHECK(IsStoreOrder(mo));
 254   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
 255   // This fast-path is critical for performance.
 256   // Assume the access is atomic.
 257   // Strictly saying even relaxed store cuts off release sequence,
 258   // so must reset the clock.
 259   if (!IsReleaseOrder(mo)) {
 260     NoTsanAtomicStore(a, v, mo);
 261     return;
 262   }
 263   __sync_synchronize();
 264   SyncVar *s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
 265   thr->fast_state.IncrementEpoch();
 266   // Can't increment epoch w/o writing to the trace as well.
 267   TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
 268   ReleaseImpl(thr, pc, &s->clock);
 269   NoTsanAtomicStore(a, v, mo);
 270   s->mtx.Unlock();
 271 }
 272
 273 template<typename T, T (*F)(volatile T *v, T op)>
 274 static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
 275   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
 276   SyncVar *s = 0;
 277   if (mo != mo_relaxed) {
 278     s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, true);
 279     thr->fast_state.IncrementEpoch();
 280     // Can't increment epoch w/o writing to the trace as well.
 281     TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
 282     if (IsAcqRelOrder(mo))
 283       AcquireReleaseImpl(thr, pc, &s->clock);
 284     else if (IsReleaseOrder(mo))
 285       ReleaseImpl(thr, pc, &s->clock);
 286     else if (IsAcquireOrder(mo))
 287       AcquireImpl(thr, pc, &s->clock);
 288   }
 289   v = F(a, v);
 290   if (s)
 291     s->mtx.Unlock();
 292   return v;
 293 }
 294
 295 template<typename T>
 296 static T NoTsanAtomicExchange(volatile T *a, T v, morder mo) {
 297   return func_xchg(a, v);
 298 }
 299
 300 template<typename T>
 301 static T NoTsanAtomicFetchAdd(volatile T *a, T v, morder mo) {
 302   return func_add(a, v);
 303 }
 304
 305 template<typename T>
 306 static T NoTsanAtomicFetchSub(volatile T *a, T v, morder mo) {
 307   return func_sub(a, v);
 308 }
 309
 310 template<typename T>
 311 static T NoTsanAtomicFetchAnd(volatile T *a, T v, morder mo) {
 312   return func_and(a, v);
 313 }
 314
 315 template<typename T>
 316 static T NoTsanAtomicFetchOr(volatile T *a, T v, morder mo) {
 317   return func_or(a, v);
 318 }
 319
 320 template<typename T>
 321 static T NoTsanAtomicFetchXor(volatile T *a, T v, morder mo) {
 322   return func_xor(a, v);
 323 }
 324
 325 template<typename T>
 326 static T NoTsanAtomicFetchNand(volatile T *a, T v, morder mo) {
 327   return func_nand(a, v);
 328 }
 329
 330 template<typename T>
 331 static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v,
 332     morder mo) {
 333   return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
 334 }
 335
 336 template<typename T>
 337 static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v,
 338     morder mo) {
 339   return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
 340 }
 341
 342 template<typename T>
 343 static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v,
 344     morder mo) {
 345   return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
 346 }
 347
 348 template<typename T>
 349 static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v,
 350     morder mo) {
 351   return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
 352 }
 353
 354 template<typename T>
 355 static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v,
 356     morder mo) {
 357   return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
 358 }
 359
 360 template<typename T>
 361 static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v,
 362     morder mo) {
 363   return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
 364 }
 365
 366 template<typename T>
 367 static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v,
 368     morder mo) {
 369   return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
 370 }
 371
 372 template<typename T>
 373 static bool NoTsanAtomicCAS(volatile T *a, T *c, T v, morder mo, morder fmo) {
 374   return atomic_compare_exchange_strong(to_atomic(a), c, v, to_mo(mo));
 375 }
 376
 377 #if __TSAN_HAS_INT128
 378 static bool NoTsanAtomicCAS(volatile a128 *a, a128 *c, a128 v,
 379     morder mo, morder fmo) {
 380   a128 old = *c;
 381   a128 cur = func_cas(a, old, v);
 382   if (cur == old)
 383     return true;
 384   *c = cur;
 385   return false;
 386 }
 387 #endif
 388
 389 template<typename T>
 390 static T NoTsanAtomicCAS(volatile T *a, T c, T v, morder mo, morder fmo) {
 391   NoTsanAtomicCAS(a, &c, v, mo, fmo);
 392   return c;
 393 }
 394
 395 template<typename T>
 396 static bool AtomicCAS(ThreadState *thr, uptr pc,
 397     volatile T *a, T *c, T v, morder mo, morder fmo) {
 398   (void)fmo;  // Unused because llvm does not pass it yet.
 399   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
 400   SyncVar *s = 0;
 401   bool write_lock = mo != mo_acquire && mo != mo_consume;
 402   if (mo != mo_relaxed) {
 403     s = ctx->metamap.GetOrCreateAndLock(thr, pc, (uptr)a, write_lock);
 404     thr->fast_state.IncrementEpoch();
 405     // Can't increment epoch w/o writing to the trace as well.
 406     TraceAddEvent(thr, thr->fast_state, EventTypeMop, 0);
 407     if (IsAcqRelOrder(mo))
 408       AcquireReleaseImpl(thr, pc, &s->clock);
 409     else if (IsReleaseOrder(mo))
 410       ReleaseImpl(thr, pc, &s->clock);
 411     else if (IsAcquireOrder(mo))
 412       AcquireImpl(thr, pc, &s->clock);
 413   }
 414   T cc = *c;
 415   T pr = func_cas(a, cc, v);
 416   if (s) {
 417     if (write_lock)
 418       s->mtx.Unlock();
 419     else
 420       s->mtx.ReadUnlock();
 421   }
 422   if (pr == cc)
 423     return true;
 424   *c = pr;
 425   return false;
 426 }
 427
 428 template<typename T>
 429 static T AtomicCAS(ThreadState *thr, uptr pc,
 430     volatile T *a, T c, T v, morder mo, morder fmo) {
 431   AtomicCAS(thr, pc, a, &c, v, mo, fmo);
 432   return c;
 433 }
 434
 435 #if !SANITIZER_GO
 436 static void NoTsanAtomicFence(morder mo) {
 437   __sync_synchronize();
 438 }
 439
 440 static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
 441   // FIXME(dvyukov): not implemented.
 442   __sync_synchronize();
 443 }
 444 #endif
 445
 446 // Interface functions follow.
 447 #if !SANITIZER_GO
 448
 449 // C/C++
 450
 451 static morder covert_morder(morder mo) {
 452   if (flags()->force_seq_cst_atomics)
 453     return (morder)mo_seq_cst;
 454
 455   // Filter out additional memory order flags:
 456   // MEMMODEL_SYNC        = 1 << 15
 457   // __ATOMIC_HLE_ACQUIRE = 1 << 16
 458   // __ATOMIC_HLE_RELEASE = 1 << 17
 459   //
 460   // HLE is an optimization, and we pretend that elision always fails.
 461   // MEMMODEL_SYNC is used when lowering __sync_ atomics,
 462   // since we use __sync_ atomics for actual atomic operations,
 463   // we can safely ignore it as well. It also subtly affects semantics,
 464   // but we don't model the difference.
 465   return (morder)(mo & 0x7fff);
 466 }
 467
 468 #define SCOPED_ATOMIC(func, ...) \
 469     const uptr callpc = (uptr)__builtin_return_address(0); \
 470     uptr pc = StackTrace::GetCurrentPc(); \
 471     mo = covert_morder(mo); \
 472     ThreadState *const thr = cur_thread(); \
 473     if (thr->ignore_interceptors) \
 474       return NoTsanAtomic##func(__VA_ARGS__); \
 475     AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \
 476     ScopedAtomic sa(thr, callpc, a, mo, __func__); \
 477     return Atomic##func(thr, pc, __VA_ARGS__); \
 478 /**/
 479
 480 class ScopedAtomic {
 481  public:
 482   ScopedAtomic(ThreadState *thr, uptr pc, const volatile void *a,
 483                morder mo, const char *func)
 484       : thr_(thr) {
 485     FuncEntry(thr_, pc);
 486     DPrintf("#%d: %s(%p, %d)\n", thr_->tid, func, a, mo);
 487   }
 488   ~ScopedAtomic() {
 489     ProcessPendingSignals(thr_);
 490     FuncExit(thr_);
 491   }
 492  private:
 493   ThreadState *thr_;
 494 };
 495
 496 static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) {
 497   StatInc(thr, StatAtomic);
 498   StatInc(thr, t);
 499   StatInc(thr, size == 1 ? StatAtomic1
 500              : size == 2 ? StatAtomic2
 501              : size == 4 ? StatAtomic4
 502              : size == 8 ? StatAtomic8
 503              :             StatAtomic16);
 504   StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed
 505              : mo == mo_consume ? StatAtomicConsume
 506              : mo == mo_acquire ? StatAtomicAcquire
 507              : mo == mo_release ? StatAtomicRelease
 508              : mo == mo_acq_rel ? StatAtomicAcq_Rel
 509              :                    StatAtomicSeq_Cst);
 510 }
 511
 512 extern "C" {
 513 SANITIZER_INTERFACE_ATTRIBUTE
 514 a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
 515   SCOPED_ATOMIC(Load, a, mo);
 516 }
 517
 518 SANITIZER_INTERFACE_ATTRIBUTE
 519 a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
 520   SCOPED_ATOMIC(Load, a, mo);
 521 }
 522
 523 SANITIZER_INTERFACE_ATTRIBUTE
 524 a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
 525   SCOPED_ATOMIC(Load, a, mo);
 526 }
 527
 528 SANITIZER_INTERFACE_ATTRIBUTE
 529 a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
 530   SCOPED_ATOMIC(Load, a, mo);
 531 }
 532
 533 #if __TSAN_HAS_INT128
 534 SANITIZER_INTERFACE_ATTRIBUTE
 535 a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
 536   SCOPED_ATOMIC(Load, a, mo);
 537 }
 538 #endif
 539
 540 SANITIZER_INTERFACE_ATTRIBUTE
 541 void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
 542   SCOPED_ATOMIC(Store, a, v, mo);
 543 }
 544
 545 SANITIZER_INTERFACE_ATTRIBUTE
 546 void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
 547   SCOPED_ATOMIC(Store, a, v, mo);
 548 }
 549
 550 SANITIZER_INTERFACE_ATTRIBUTE
 551 void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
 552   SCOPED_ATOMIC(Store, a, v, mo);
 553 }
 554
 555 SANITIZER_INTERFACE_ATTRIBUTE
 556 void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
 557   SCOPED_ATOMIC(Store, a, v, mo);
 558 }
 559
 560 #if __TSAN_HAS_INT128
 561 SANITIZER_INTERFACE_ATTRIBUTE
 562 void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
 563   SCOPED_ATOMIC(Store, a, v, mo);
 564 }
 565 #endif
 566
 567 SANITIZER_INTERFACE_ATTRIBUTE
 568 a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
 569   SCOPED_ATOMIC(Exchange, a, v, mo);
 570 }
 571
 572 SANITIZER_INTERFACE_ATTRIBUTE
 573 a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
 574   SCOPED_ATOMIC(Exchange, a, v, mo);
 575 }
 576
 577 SANITIZER_INTERFACE_ATTRIBUTE
 578 a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
 579   SCOPED_ATOMIC(Exchange, a, v, mo);
 580 }
 581
 582 SANITIZER_INTERFACE_ATTRIBUTE
 583 a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
 584   SCOPED_ATOMIC(Exchange, a, v, mo);
 585 }
 586
 587 #if __TSAN_HAS_INT128
 588 SANITIZER_INTERFACE_ATTRIBUTE
 589 a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
 590   SCOPED_ATOMIC(Exchange, a, v, mo);
 591 }
 592 #endif
 593
 594 SANITIZER_INTERFACE_ATTRIBUTE
 595 a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
 596   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 597 }
 598
 599 SANITIZER_INTERFACE_ATTRIBUTE
 600 a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
 601   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 602 }
 603
 604 SANITIZER_INTERFACE_ATTRIBUTE
 605 a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
 606   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 607 }
 608
 609 SANITIZER_INTERFACE_ATTRIBUTE
 610 a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
 611   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 612 }
 613
 614 #if __TSAN_HAS_INT128
 615 SANITIZER_INTERFACE_ATTRIBUTE
 616 a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
 617   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 618 }
 619 #endif
 620
 621 SANITIZER_INTERFACE_ATTRIBUTE
 622 a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
 623   SCOPED_ATOMIC(FetchSub, a, v, mo);
 624 }
 625
 626 SANITIZER_INTERFACE_ATTRIBUTE
 627 a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
 628   SCOPED_ATOMIC(FetchSub, a, v, mo);
 629 }
 630
 631 SANITIZER_INTERFACE_ATTRIBUTE
 632 a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
 633   SCOPED_ATOMIC(FetchSub, a, v, mo);
 634 }
 635
 636 SANITIZER_INTERFACE_ATTRIBUTE
 637 a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
 638   SCOPED_ATOMIC(FetchSub, a, v, mo);
 639 }
 640
 641 #if __TSAN_HAS_INT128
 642 SANITIZER_INTERFACE_ATTRIBUTE
 643 a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
 644   SCOPED_ATOMIC(FetchSub, a, v, mo);
 645 }
 646 #endif
 647
 648 SANITIZER_INTERFACE_ATTRIBUTE
 649 a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
 650   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 651 }
 652
 653 SANITIZER_INTERFACE_ATTRIBUTE
 654 a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
 655   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 656 }
 657
 658 SANITIZER_INTERFACE_ATTRIBUTE
 659 a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
 660   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 661 }
 662
 663 SANITIZER_INTERFACE_ATTRIBUTE
 664 a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
 665   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 666 }
 667
 668 #if __TSAN_HAS_INT128
 669 SANITIZER_INTERFACE_ATTRIBUTE
 670 a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
 671   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 672 }
 673 #endif
 674
 675 SANITIZER_INTERFACE_ATTRIBUTE
 676 a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
 677   SCOPED_ATOMIC(FetchOr, a, v, mo);
 678 }
 679
 680 SANITIZER_INTERFACE_ATTRIBUTE
 681 a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
 682   SCOPED_ATOMIC(FetchOr, a, v, mo);
 683 }
 684
 685 SANITIZER_INTERFACE_ATTRIBUTE
 686 a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
 687   SCOPED_ATOMIC(FetchOr, a, v, mo);
 688 }
 689
 690 SANITIZER_INTERFACE_ATTRIBUTE
 691 a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
 692   SCOPED_ATOMIC(FetchOr, a, v, mo);
 693 }
 694
 695 #if __TSAN_HAS_INT128
 696 SANITIZER_INTERFACE_ATTRIBUTE
 697 a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
 698   SCOPED_ATOMIC(FetchOr, a, v, mo);
 699 }
 700 #endif
 701
 702 SANITIZER_INTERFACE_ATTRIBUTE
 703 a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
 704   SCOPED_ATOMIC(FetchXor, a, v, mo);
 705 }
 706
 707 SANITIZER_INTERFACE_ATTRIBUTE
 708 a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
 709   SCOPED_ATOMIC(FetchXor, a, v, mo);
 710 }
 711
 712 SANITIZER_INTERFACE_ATTRIBUTE
 713 a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
 714   SCOPED_ATOMIC(FetchXor, a, v, mo);
 715 }
 716
 717 SANITIZER_INTERFACE_ATTRIBUTE
 718 a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
 719   SCOPED_ATOMIC(FetchXor, a, v, mo);
 720 }
 721
 722 #if __TSAN_HAS_INT128
 723 SANITIZER_INTERFACE_ATTRIBUTE
 724 a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
 725   SCOPED_ATOMIC(FetchXor, a, v, mo);
 726 }
 727 #endif
 728
 729 SANITIZER_INTERFACE_ATTRIBUTE
 730 a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
 731   SCOPED_ATOMIC(FetchNand, a, v, mo);
 732 }
 733
 734 SANITIZER_INTERFACE_ATTRIBUTE
 735 a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
 736   SCOPED_ATOMIC(FetchNand, a, v, mo);
 737 }
 738
 739 SANITIZER_INTERFACE_ATTRIBUTE
 740 a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
 741   SCOPED_ATOMIC(FetchNand, a, v, mo);
 742 }
 743
 744 SANITIZER_INTERFACE_ATTRIBUTE
 745 a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
 746   SCOPED_ATOMIC(FetchNand, a, v, mo);
 747 }
 748
 749 #if __TSAN_HAS_INT128
 750 SANITIZER_INTERFACE_ATTRIBUTE
 751 a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
 752   SCOPED_ATOMIC(FetchNand, a, v, mo);
 753 }
 754 #endif
 755
 756 SANITIZER_INTERFACE_ATTRIBUTE
 757 int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
 758     morder mo, morder fmo) {
 759   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 760 }
 761
 762 SANITIZER_INTERFACE_ATTRIBUTE
 763 int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
 764     morder mo, morder fmo) {
 765   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 766 }
 767
 768 SANITIZER_INTERFACE_ATTRIBUTE
 769 int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
 770     morder mo, morder fmo) {
 771   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 772 }
 773
 774 SANITIZER_INTERFACE_ATTRIBUTE
 775 int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
 776     morder mo, morder fmo) {
 777   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 778 }
 779
 780 #if __TSAN_HAS_INT128
 781 SANITIZER_INTERFACE_ATTRIBUTE
 782 int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
 783     morder mo, morder fmo) {
 784   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 785 }
 786 #endif
 787
 788 SANITIZER_INTERFACE_ATTRIBUTE
 789 int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v,
 790     morder mo, morder fmo) {
 791   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 792 }
 793
 794 SANITIZER_INTERFACE_ATTRIBUTE
 795 int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
 796     morder mo, morder fmo) {
 797   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 798 }
 799
 800 SANITIZER_INTERFACE_ATTRIBUTE
 801 int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
 802     morder mo, morder fmo) {
 803   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 804 }
 805
 806 SANITIZER_INTERFACE_ATTRIBUTE
 807 int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
 808     morder mo, morder fmo) {
 809   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 810 }
 811
 812 #if __TSAN_HAS_INT128
 813 SANITIZER_INTERFACE_ATTRIBUTE
 814 int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
 815     morder mo, morder fmo) {
 816   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 817 }
 818 #endif
 819
 820 SANITIZER_INTERFACE_ATTRIBUTE
 821 a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
 822     morder mo, morder fmo) {
 823   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 824 }
 825
 826 SANITIZER_INTERFACE_ATTRIBUTE
 827 a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
 828     morder mo, morder fmo) {
 829   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 830 }
 831
 832 SANITIZER_INTERFACE_ATTRIBUTE
 833 a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
 834     morder mo, morder fmo) {
 835   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 836 }
 837
 838 SANITIZER_INTERFACE_ATTRIBUTE
 839 a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
 840     morder mo, morder fmo) {
 841   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 842 }
 843
 844 #if __TSAN_HAS_INT128
 845 SANITIZER_INTERFACE_ATTRIBUTE
 846 a128 __tsan_atomic128_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
 847     morder mo, morder fmo) {
 848   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 849 }
 850 #endif
 851
 852 SANITIZER_INTERFACE_ATTRIBUTE
 853 void __tsan_atomic_thread_fence(morder mo) {
 854   char* a = 0;
 855   SCOPED_ATOMIC(Fence, mo);
 856 }
 857
 858 SANITIZER_INTERFACE_ATTRIBUTE
 859 void __tsan_atomic_signal_fence(morder mo) {
 860 }
 861 }  // extern "C"
 862
 863 #else  // #if !SANITIZER_GO
 864
 865 // Go
 866
 867 #define ATOMIC(func, ...) \
 868     if (thr->ignore_sync) { \
 869       NoTsanAtomic##func(__VA_ARGS__); \
 870     } else { \
 871       FuncEntry(thr, cpc); \
 872       Atomic##func(thr, pc, __VA_ARGS__); \
 873       FuncExit(thr); \
 874     } \
 875 /**/
 876
 877 #define ATOMIC_RET(func, ret, ...) \
 878     if (thr->ignore_sync) { \
 879       (ret) = NoTsanAtomic##func(__VA_ARGS__); \
 880     } else { \
 881       FuncEntry(thr, cpc); \
 882       (ret) = Atomic##func(thr, pc, __VA_ARGS__); \
 883       FuncExit(thr); \
 884     } \
 885 /**/
 886
 887 extern "C" {
 888 SANITIZER_INTERFACE_ATTRIBUTE
 889 void __tsan_go_atomic32_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 890   ATOMIC_RET(Load, *(a32*)(a+8), *(a32**)a, mo_acquire);
 891 }
 892
 893 SANITIZER_INTERFACE_ATTRIBUTE
 894 void __tsan_go_atomic64_load(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 895   ATOMIC_RET(Load, *(a64*)(a+8), *(a64**)a, mo_acquire);
 896 }
 897
 898 SANITIZER_INTERFACE_ATTRIBUTE
 899 void __tsan_go_atomic32_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 900   ATOMIC(Store, *(a32**)a, *(a32*)(a+8), mo_release);
 901 }
 902
 903 SANITIZER_INTERFACE_ATTRIBUTE
 904 void __tsan_go_atomic64_store(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 905   ATOMIC(Store, *(a64**)a, *(a64*)(a+8), mo_release);
 906 }
 907
 908 SANITIZER_INTERFACE_ATTRIBUTE
 909 void __tsan_go_atomic32_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 910   ATOMIC_RET(FetchAdd, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
 911 }
 912
 913 SANITIZER_INTERFACE_ATTRIBUTE
 914 void __tsan_go_atomic64_fetch_add(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 915   ATOMIC_RET(FetchAdd, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
 916 }
 917
 918 SANITIZER_INTERFACE_ATTRIBUTE
 919 void __tsan_go_atomic32_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 920   ATOMIC_RET(Exchange, *(a32*)(a+16), *(a32**)a, *(a32*)(a+8), mo_acq_rel);
 921 }
 922
 923 SANITIZER_INTERFACE_ATTRIBUTE
 924 void __tsan_go_atomic64_exchange(ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 925   ATOMIC_RET(Exchange, *(a64*)(a+16), *(a64**)a, *(a64*)(a+8), mo_acq_rel);
 926 }
 927
 928 SANITIZER_INTERFACE_ATTRIBUTE
 929 void __tsan_go_atomic32_compare_exchange(
 930     ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 931   a32 cur = 0;
 932   a32 cmp = *(a32*)(a+8);
 933   ATOMIC_RET(CAS, cur, *(a32**)a, cmp, *(a32*)(a+12), mo_acq_rel, mo_acquire);
 934   *(bool*)(a+16) = (cur == cmp);
 935 }
 936
 937 SANITIZER_INTERFACE_ATTRIBUTE
 938 void __tsan_go_atomic64_compare_exchange(
 939     ThreadState *thr, uptr cpc, uptr pc, u8 *a) {
 940   a64 cur = 0;
 941   a64 cmp = *(a64*)(a+8);
 942   ATOMIC_RET(CAS, cur, *(a64**)a, cmp, *(a64*)(a+16), mo_acq_rel, mo_acquire);
 943   *(bool*)(a+24) = (cur == cmp);
 944 }
 945 }  // extern "C"
 946 #endif  // #if !SANITIZER_GO