libsanitizer/tsan/tsan_interface_atomic.cc

   1 //===-- tsan_interface_atomic.cc ------------------------------------------===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer (TSan), a race detector.
   9 //
  10 //===----------------------------------------------------------------------===//
  11
  12 // ThreadSanitizer atomic operations are based on C++11/C1x standards.
  13 // For background see C++11 standard.  A slightly older, publically
  14 // available draft of the standard (not entirely up-to-date, but close enough
  15 // for casual browsing) is available here:
  16 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
  17 // The following page contains more background information:
  18 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
  19
  20 #include "sanitizer_common/sanitizer_placement_new.h"
  21 #include "sanitizer_common/sanitizer_stacktrace.h"
  22 #include "tsan_interface_atomic.h"
  23 #include "tsan_flags.h"
  24 #include "tsan_rtl.h"
  25
  26 using namespace __tsan;  // NOLINT
  27
  28 #define SCOPED_ATOMIC(func, ...) \
  29     const uptr callpc = (uptr)__builtin_return_address(0); \
  30     uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \
  31     pc = __sanitizer::StackTrace::GetPreviousInstructionPc(pc); \
  32     mo = ConvertOrder(mo); \
  33     mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \
  34     ThreadState *const thr = cur_thread(); \
  35     AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \
  36     ScopedAtomic sa(thr, callpc, __FUNCTION__); \
  37     return Atomic##func(thr, pc, __VA_ARGS__); \
  38 /**/
  39
  40 class ScopedAtomic {
  41  public:
  42   ScopedAtomic(ThreadState *thr, uptr pc, const char *func)
  43       : thr_(thr) {
  44     CHECK_EQ(thr_->in_rtl, 0);
  45     ProcessPendingSignals(thr);
  46     FuncEntry(thr_, pc);
  47     DPrintf("#%d: %s\n", thr_->tid, func);
  48     thr_->in_rtl++;
  49   }
  50   ~ScopedAtomic() {
  51     thr_->in_rtl--;
  52     CHECK_EQ(thr_->in_rtl, 0);
  53     FuncExit(thr_);
  54   }
  55  private:
  56   ThreadState *thr_;
  57 };
  58
  59 // Some shortcuts.
  60 typedef __tsan_memory_order morder;
  61 typedef __tsan_atomic8 a8;
  62 typedef __tsan_atomic16 a16;
  63 typedef __tsan_atomic32 a32;
  64 typedef __tsan_atomic64 a64;
  65 typedef __tsan_atomic128 a128;
  66 const morder mo_relaxed = __tsan_memory_order_relaxed;
  67 const morder mo_consume = __tsan_memory_order_consume;
  68 const morder mo_acquire = __tsan_memory_order_acquire;
  69 const morder mo_release = __tsan_memory_order_release;
  70 const morder mo_acq_rel = __tsan_memory_order_acq_rel;
  71 const morder mo_seq_cst = __tsan_memory_order_seq_cst;
  72
  73 static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) {
  74   StatInc(thr, StatAtomic);
  75   StatInc(thr, t);
  76   StatInc(thr, size == 1 ? StatAtomic1
  77              : size == 2 ? StatAtomic2
  78              : size == 4 ? StatAtomic4
  79              : size == 8 ? StatAtomic8
  80              :             StatAtomic16);
  81   StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed
  82              : mo == mo_consume ? StatAtomicConsume
  83              : mo == mo_acquire ? StatAtomicAcquire
  84              : mo == mo_release ? StatAtomicRelease
  85              : mo == mo_acq_rel ? StatAtomicAcq_Rel
  86              :                    StatAtomicSeq_Cst);
  87 }
  88
  89 static bool IsLoadOrder(morder mo) {
  90   return mo == mo_relaxed || mo == mo_consume
  91       || mo == mo_acquire || mo == mo_seq_cst;
  92 }
  93
  94 static bool IsStoreOrder(morder mo) {
  95   return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
  96 }
  97
  98 static bool IsReleaseOrder(morder mo) {
  99   return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
 100 }
 101
 102 static bool IsAcquireOrder(morder mo) {
 103   return mo == mo_consume || mo == mo_acquire
 104       || mo == mo_acq_rel || mo == mo_seq_cst;
 105 }
 106
 107 static bool IsAcqRelOrder(morder mo) {
 108   return mo == mo_acq_rel || mo == mo_seq_cst;
 109 }
 110
 111 static morder ConvertOrder(morder mo) {
 112   if (mo > (morder)100500) {
 113     mo = morder(mo - 100500);
 114     if (mo ==  morder(1 << 0))
 115       mo = mo_relaxed;
 116     else if (mo == morder(1 << 1))
 117       mo = mo_consume;
 118     else if (mo == morder(1 << 2))
 119       mo = mo_acquire;
 120     else if (mo == morder(1 << 3))
 121       mo = mo_release;
 122     else if (mo == morder(1 << 4))
 123       mo = mo_acq_rel;
 124     else if (mo == morder(1 << 5))
 125       mo = mo_seq_cst;
 126   }
 127   CHECK_GE(mo, mo_relaxed);
 128   CHECK_LE(mo, mo_seq_cst);
 129   return mo;
 130 }
 131
 132 template<typename T> T func_xchg(volatile T *v, T op) {
 133   T res = __sync_lock_test_and_set(v, op);
 134   // __sync_lock_test_and_set does not contain full barrier.
 135   __sync_synchronize();
 136   return res;
 137 }
 138
 139 template<typename T> T func_add(volatile T *v, T op) {
 140   return __sync_fetch_and_add(v, op);
 141 }
 142
 143 template<typename T> T func_sub(volatile T *v, T op) {
 144   return __sync_fetch_and_sub(v, op);
 145 }
 146
 147 template<typename T> T func_and(volatile T *v, T op) {
 148   return __sync_fetch_and_and(v, op);
 149 }
 150
 151 template<typename T> T func_or(volatile T *v, T op) {
 152   return __sync_fetch_and_or(v, op);
 153 }
 154
 155 template<typename T> T func_xor(volatile T *v, T op) {
 156   return __sync_fetch_and_xor(v, op);
 157 }
 158
 159 template<typename T> T func_nand(volatile T *v, T op) {
 160   // clang does not support __sync_fetch_and_nand.
 161   T cmp = *v;
 162   for (;;) {
 163     T newv = ~(cmp & op);
 164     T cur = __sync_val_compare_and_swap(v, cmp, newv);
 165     if (cmp == cur)
 166       return cmp;
 167     cmp = cur;
 168   }
 169 }
 170
 171 template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
 172   return __sync_val_compare_and_swap(v, cmp, xch);
 173 }
 174
 175 // clang does not support 128-bit atomic ops.
 176 // Atomic ops are executed under tsan internal mutex,
 177 // here we assume that the atomic variables are not accessed
 178 // from non-instrumented code.
 179 #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16
 180 a128 func_xchg(volatile a128 *v, a128 op) {
 181   a128 cmp = *v;
 182   *v = op;
 183   return cmp;
 184 }
 185
 186 a128 func_add(volatile a128 *v, a128 op) {
 187   a128 cmp = *v;
 188   *v = cmp + op;
 189   return cmp;
 190 }
 191
 192 a128 func_sub(volatile a128 *v, a128 op) {
 193   a128 cmp = *v;
 194   *v = cmp - op;
 195   return cmp;
 196 }
 197
 198 a128 func_and(volatile a128 *v, a128 op) {
 199   a128 cmp = *v;
 200   *v = cmp & op;
 201   return cmp;
 202 }
 203
 204 a128 func_or(volatile a128 *v, a128 op) {
 205   a128 cmp = *v;
 206   *v = cmp | op;
 207   return cmp;
 208 }
 209
 210 a128 func_xor(volatile a128 *v, a128 op) {
 211   a128 cmp = *v;
 212   *v = cmp ^ op;
 213   return cmp;
 214 }
 215
 216 a128 func_nand(volatile a128 *v, a128 op) {
 217   a128 cmp = *v;
 218   *v = ~(cmp & op);
 219   return cmp;
 220 }
 221
 222 a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
 223   a128 cur = *v;
 224   if (cur == cmp)
 225     *v = xch;
 226   return cur;
 227 }
 228 #endif
 229
 230 template<typename T>
 231 static int SizeLog() {
 232   if (sizeof(T) <= 1)
 233     return kSizeLog1;
 234   else if (sizeof(T) <= 2)
 235     return kSizeLog2;
 236   else if (sizeof(T) <= 4)
 237     return kSizeLog4;
 238   else
 239     return kSizeLog8;
 240   // For 16-byte atomics we also use 8-byte memory access,
 241   // this leads to false negatives only in very obscure cases.
 242 }
 243
 244 template<typename T>
 245 static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
 246     morder mo) {
 247   CHECK(IsLoadOrder(mo));
 248   // This fast-path is critical for performance.
 249   // Assume the access is atomic.
 250   if (!IsAcquireOrder(mo) && sizeof(T) <= sizeof(a)) {
 251     MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
 252     return *a;
 253   }
 254   SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, false);
 255   thr->clock.set(thr->tid, thr->fast_state.epoch());
 256   thr->clock.acquire(&s->clock);
 257   T v = *a;
 258   s->mtx.ReadUnlock();
 259   __sync_synchronize();
 260   MemoryReadAtomic(thr, pc, (uptr)a, SizeLog<T>());
 261   return v;
 262 }
 263
 264 template<typename T>
 265 static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
 266     morder mo) {
 267   CHECK(IsStoreOrder(mo));
 268   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
 269   // This fast-path is critical for performance.
 270   // Assume the access is atomic.
 271   // Strictly saying even relaxed store cuts off release sequence,
 272   // so must reset the clock.
 273   if (!IsReleaseOrder(mo) && sizeof(T) <= sizeof(a)) {
 274     *a = v;
 275     return;
 276   }
 277   __sync_synchronize();
 278   SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true);
 279   thr->clock.set(thr->tid, thr->fast_state.epoch());
 280   thr->clock.ReleaseStore(&s->clock);
 281   *a = v;
 282   s->mtx.Unlock();
 283   // Trainling memory barrier to provide sequential consistency
 284   // for Dekker-like store-load synchronization.
 285   __sync_synchronize();
 286 }
 287
 288 template<typename T, T (*F)(volatile T *v, T op)>
 289 static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
 290   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
 291   SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true);
 292   thr->clock.set(thr->tid, thr->fast_state.epoch());
 293   if (IsAcqRelOrder(mo))
 294     thr->clock.acq_rel(&s->clock);
 295   else if (IsReleaseOrder(mo))
 296     thr->clock.release(&s->clock);
 297   else if (IsAcquireOrder(mo))
 298     thr->clock.acquire(&s->clock);
 299   v = F(a, v);
 300   s->mtx.Unlock();
 301   return v;
 302 }
 303
 304 template<typename T>
 305 static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v,
 306     morder mo) {
 307   return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
 308 }
 309
 310 template<typename T>
 311 static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v,
 312     morder mo) {
 313   return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
 314 }
 315
 316 template<typename T>
 317 static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v,
 318     morder mo) {
 319   return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
 320 }
 321
 322 template<typename T>
 323 static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v,
 324     morder mo) {
 325   return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
 326 }
 327
 328 template<typename T>
 329 static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v,
 330     morder mo) {
 331   return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
 332 }
 333
 334 template<typename T>
 335 static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v,
 336     morder mo) {
 337   return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
 338 }
 339
 340 template<typename T>
 341 static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v,
 342     morder mo) {
 343   return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
 344 }
 345
 346 template<typename T>
 347 static bool AtomicCAS(ThreadState *thr, uptr pc,
 348     volatile T *a, T *c, T v, morder mo, morder fmo) {
 349   (void)fmo;  // Unused because llvm does not pass it yet.
 350   MemoryWriteAtomic(thr, pc, (uptr)a, SizeLog<T>());
 351   SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true);
 352   thr->clock.set(thr->tid, thr->fast_state.epoch());
 353   if (IsAcqRelOrder(mo))
 354     thr->clock.acq_rel(&s->clock);
 355   else if (IsReleaseOrder(mo))
 356     thr->clock.release(&s->clock);
 357   else if (IsAcquireOrder(mo))
 358     thr->clock.acquire(&s->clock);
 359   T cc = *c;
 360   T pr = func_cas(a, cc, v);
 361   s->mtx.Unlock();
 362   if (pr == cc)
 363     return true;
 364   *c = pr;
 365   return false;
 366 }
 367
 368 template<typename T>
 369 static T AtomicCAS(ThreadState *thr, uptr pc,
 370     volatile T *a, T c, T v, morder mo, morder fmo) {
 371   AtomicCAS(thr, pc, a, &c, v, mo, fmo);
 372   return c;
 373 }
 374
 375 static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
 376   // FIXME(dvyukov): not implemented.
 377   __sync_synchronize();
 378 }
 379
 380 a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
 381   SCOPED_ATOMIC(Load, a, mo);
 382 }
 383
 384 a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
 385   SCOPED_ATOMIC(Load, a, mo);
 386 }
 387
 388 a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
 389   SCOPED_ATOMIC(Load, a, mo);
 390 }
 391
 392 a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
 393   SCOPED_ATOMIC(Load, a, mo);
 394 }
 395
 396 #if __TSAN_HAS_INT128
 397 a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
 398   SCOPED_ATOMIC(Load, a, mo);
 399 }
 400 #endif
 401
 402 void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
 403   SCOPED_ATOMIC(Store, a, v, mo);
 404 }
 405
 406 void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
 407   SCOPED_ATOMIC(Store, a, v, mo);
 408 }
 409
 410 void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
 411   SCOPED_ATOMIC(Store, a, v, mo);
 412 }
 413
 414 void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
 415   SCOPED_ATOMIC(Store, a, v, mo);
 416 }
 417
 418 #if __TSAN_HAS_INT128
 419 void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
 420   SCOPED_ATOMIC(Store, a, v, mo);
 421 }
 422 #endif
 423
 424 a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
 425   SCOPED_ATOMIC(Exchange, a, v, mo);
 426 }
 427
 428 a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
 429   SCOPED_ATOMIC(Exchange, a, v, mo);
 430 }
 431
 432 a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
 433   SCOPED_ATOMIC(Exchange, a, v, mo);
 434 }
 435
 436 a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
 437   SCOPED_ATOMIC(Exchange, a, v, mo);
 438 }
 439
 440 #if __TSAN_HAS_INT128
 441 a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
 442   SCOPED_ATOMIC(Exchange, a, v, mo);
 443 }
 444 #endif
 445
 446 a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
 447   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 448 }
 449
 450 a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
 451   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 452 }
 453
 454 a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
 455   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 456 }
 457
 458 a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
 459   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 460 }
 461
 462 #if __TSAN_HAS_INT128
 463 a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
 464   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 465 }
 466 #endif
 467
 468 a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
 469   SCOPED_ATOMIC(FetchSub, a, v, mo);
 470 }
 471
 472 a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
 473   SCOPED_ATOMIC(FetchSub, a, v, mo);
 474 }
 475
 476 a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
 477   SCOPED_ATOMIC(FetchSub, a, v, mo);
 478 }
 479
 480 a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
 481   SCOPED_ATOMIC(FetchSub, a, v, mo);
 482 }
 483
 484 #if __TSAN_HAS_INT128
 485 a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
 486   SCOPED_ATOMIC(FetchSub, a, v, mo);
 487 }
 488 #endif
 489
 490 a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
 491   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 492 }
 493
 494 a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
 495   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 496 }
 497
 498 a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
 499   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 500 }
 501
 502 a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
 503   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 504 }
 505
 506 #if __TSAN_HAS_INT128
 507 a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
 508   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 509 }
 510 #endif
 511
 512 a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
 513   SCOPED_ATOMIC(FetchOr, a, v, mo);
 514 }
 515
 516 a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
 517   SCOPED_ATOMIC(FetchOr, a, v, mo);
 518 }
 519
 520 a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
 521   SCOPED_ATOMIC(FetchOr, a, v, mo);
 522 }
 523
 524 a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
 525   SCOPED_ATOMIC(FetchOr, a, v, mo);
 526 }
 527
 528 #if __TSAN_HAS_INT128
 529 a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
 530   SCOPED_ATOMIC(FetchOr, a, v, mo);
 531 }
 532 #endif
 533
 534 a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
 535   SCOPED_ATOMIC(FetchXor, a, v, mo);
 536 }
 537
 538 a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
 539   SCOPED_ATOMIC(FetchXor, a, v, mo);
 540 }
 541
 542 a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
 543   SCOPED_ATOMIC(FetchXor, a, v, mo);
 544 }
 545
 546 a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
 547   SCOPED_ATOMIC(FetchXor, a, v, mo);
 548 }
 549
 550 #if __TSAN_HAS_INT128
 551 a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
 552   SCOPED_ATOMIC(FetchXor, a, v, mo);
 553 }
 554 #endif
 555
 556 a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
 557   SCOPED_ATOMIC(FetchNand, a, v, mo);
 558 }
 559
 560 a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
 561   SCOPED_ATOMIC(FetchNand, a, v, mo);
 562 }
 563
 564 a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
 565   SCOPED_ATOMIC(FetchNand, a, v, mo);
 566 }
 567
 568 a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
 569   SCOPED_ATOMIC(FetchNand, a, v, mo);
 570 }
 571
 572 #if __TSAN_HAS_INT128
 573 a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
 574   SCOPED_ATOMIC(FetchNand, a, v, mo);
 575 }
 576 #endif
 577
 578 int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
 579     morder mo, morder fmo) {
 580   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 581 }
 582
 583 int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
 584     morder mo, morder fmo) {
 585   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 586 }
 587
 588 int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
 589     morder mo, morder fmo) {
 590   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 591 }
 592
 593 int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
 594     morder mo, morder fmo) {
 595   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 596 }
 597
 598 #if __TSAN_HAS_INT128
 599 int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
 600     morder mo, morder fmo) {
 601   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 602 }
 603 #endif
 604
 605 int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v,
 606     morder mo, morder fmo) {
 607   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 608 }
 609
 610 int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
 611     morder mo, morder fmo) {
 612   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 613 }
 614
 615 int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
 616     morder mo, morder fmo) {
 617   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 618 }
 619
 620 int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
 621     morder mo, morder fmo) {
 622   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 623 }
 624
 625 #if __TSAN_HAS_INT128
 626 int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
 627     morder mo, morder fmo) {
 628   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 629 }
 630 #endif
 631
 632 a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
 633     morder mo, morder fmo) {
 634   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 635 }
 636 a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
 637     morder mo, morder fmo) {
 638   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 639 }
 640
 641 a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
 642     morder mo, morder fmo) {
 643   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 644 }
 645
 646 a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
 647     morder mo, morder fmo) {
 648   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 649 }
 650
 651 #if __TSAN_HAS_INT128
 652 a128 __tsan_atomic64_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
 653     morder mo, morder fmo) {
 654   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 655 }
 656 #endif
 657
 658 void __tsan_atomic_thread_fence(morder mo) {
 659   char* a;
 660   SCOPED_ATOMIC(Fence, mo);
 661 }
 662
 663 void __tsan_atomic_signal_fence(morder mo) {
 664 }