libsanitizer/tsan/tsan_interface_atomic.cc

   1 //===-- tsan_interface_atomic.cc ------------------------------------------===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer (TSan), a race detector.
   9 //
  10 //===----------------------------------------------------------------------===//
  11
  12 // ThreadSanitizer atomic operations are based on C++11/C1x standards.
  13 // For background see C++11 standard.  A slightly older, publically
  14 // available draft of the standard (not entirely up-to-date, but close enough
  15 // for casual browsing) is available here:
  16 // http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2011/n3242.pdf
  17 // The following page contains more background information:
  18 // http://www.hpl.hp.com/personal/Hans_Boehm/c++mm/
  19
  20 #include "sanitizer_common/sanitizer_placement_new.h"
  21 #include "tsan_interface_atomic.h"
  22 #include "tsan_flags.h"
  23 #include "tsan_rtl.h"
  24
  25 using namespace __tsan;  // NOLINT
  26
  27 class ScopedAtomic {
  28  public:
  29   ScopedAtomic(ThreadState *thr, uptr pc, const char *func)
  30       : thr_(thr) {
  31     CHECK_EQ(thr_->in_rtl, 1);  // 1 due to our own ScopedInRtl member.
  32     DPrintf("#%d: %s\n", thr_->tid, func);
  33   }
  34   ~ScopedAtomic() {
  35     CHECK_EQ(thr_->in_rtl, 1);
  36   }
  37  private:
  38   ThreadState *thr_;
  39   ScopedInRtl in_rtl_;
  40 };
  41
  42 // Some shortcuts.
  43 typedef __tsan_memory_order morder;
  44 typedef __tsan_atomic8 a8;
  45 typedef __tsan_atomic16 a16;
  46 typedef __tsan_atomic32 a32;
  47 typedef __tsan_atomic64 a64;
  48 typedef __tsan_atomic128 a128;
  49 const morder mo_relaxed = __tsan_memory_order_relaxed;
  50 const morder mo_consume = __tsan_memory_order_consume;
  51 const morder mo_acquire = __tsan_memory_order_acquire;
  52 const morder mo_release = __tsan_memory_order_release;
  53 const morder mo_acq_rel = __tsan_memory_order_acq_rel;
  54 const morder mo_seq_cst = __tsan_memory_order_seq_cst;
  55
  56 static void AtomicStatInc(ThreadState *thr, uptr size, morder mo, StatType t) {
  57   StatInc(thr, StatAtomic);
  58   StatInc(thr, t);
  59   StatInc(thr, size == 1 ? StatAtomic1
  60              : size == 2 ? StatAtomic2
  61              : size == 4 ? StatAtomic4
  62              : size == 8 ? StatAtomic8
  63              :             StatAtomic16);
  64   StatInc(thr, mo == mo_relaxed ? StatAtomicRelaxed
  65              : mo == mo_consume ? StatAtomicConsume
  66              : mo == mo_acquire ? StatAtomicAcquire
  67              : mo == mo_release ? StatAtomicRelease
  68              : mo == mo_acq_rel ? StatAtomicAcq_Rel
  69              :                    StatAtomicSeq_Cst);
  70 }
  71
  72 static bool IsLoadOrder(morder mo) {
  73   return mo == mo_relaxed || mo == mo_consume
  74       || mo == mo_acquire || mo == mo_seq_cst;
  75 }
  76
  77 static bool IsStoreOrder(morder mo) {
  78   return mo == mo_relaxed || mo == mo_release || mo == mo_seq_cst;
  79 }
  80
  81 static bool IsReleaseOrder(morder mo) {
  82   return mo == mo_release || mo == mo_acq_rel || mo == mo_seq_cst;
  83 }
  84
  85 static bool IsAcquireOrder(morder mo) {
  86   return mo == mo_consume || mo == mo_acquire
  87       || mo == mo_acq_rel || mo == mo_seq_cst;
  88 }
  89
  90 static bool IsAcqRelOrder(morder mo) {
  91   return mo == mo_acq_rel || mo == mo_seq_cst;
  92 }
  93
  94 static morder ConvertOrder(morder mo) {
  95   if (mo > (morder)100500) {
  96     mo = morder(mo - 100500);
  97     if (mo ==  morder(1 << 0))
  98       mo = mo_relaxed;
  99     else if (mo == morder(1 << 1))
 100       mo = mo_consume;
 101     else if (mo == morder(1 << 2))
 102       mo = mo_acquire;
 103     else if (mo == morder(1 << 3))
 104       mo = mo_release;
 105     else if (mo == morder(1 << 4))
 106       mo = mo_acq_rel;
 107     else if (mo == morder(1 << 5))
 108       mo = mo_seq_cst;
 109   }
 110   CHECK_GE(mo, mo_relaxed);
 111   CHECK_LE(mo, mo_seq_cst);
 112   return mo;
 113 }
 114
 115 template<typename T> T func_xchg(volatile T *v, T op) {
 116   T res = __sync_lock_test_and_set(v, op);
 117   // __sync_lock_test_and_set does not contain full barrier.
 118   __sync_synchronize();
 119   return res;
 120 }
 121
 122 template<typename T> T func_add(volatile T *v, T op) {
 123   return __sync_fetch_and_add(v, op);
 124 }
 125
 126 template<typename T> T func_sub(volatile T *v, T op) {
 127   return __sync_fetch_and_sub(v, op);
 128 }
 129
 130 template<typename T> T func_and(volatile T *v, T op) {
 131   return __sync_fetch_and_and(v, op);
 132 }
 133
 134 template<typename T> T func_or(volatile T *v, T op) {
 135   return __sync_fetch_and_or(v, op);
 136 }
 137
 138 template<typename T> T func_xor(volatile T *v, T op) {
 139   return __sync_fetch_and_xor(v, op);
 140 }
 141
 142 template<typename T> T func_nand(volatile T *v, T op) {
 143   // clang does not support __sync_fetch_and_nand.
 144   T cmp = *v;
 145   for (;;) {
 146     T newv = ~(cmp & op);
 147     T cur = __sync_val_compare_and_swap(v, cmp, newv);
 148     if (cmp == cur)
 149       return cmp;
 150     cmp = cur;
 151   }
 152 }
 153
 154 template<typename T> T func_cas(volatile T *v, T cmp, T xch) {
 155   return __sync_val_compare_and_swap(v, cmp, xch);
 156 }
 157
 158 // clang does not support 128-bit atomic ops.
 159 // Atomic ops are executed under tsan internal mutex,
 160 // here we assume that the atomic variables are not accessed
 161 // from non-instrumented code.
 162 #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_16
 163 a128 func_xchg(volatile a128 *v, a128 op) {
 164   a128 cmp = *v;
 165   *v = op;
 166   return cmp;
 167 }
 168
 169 a128 func_add(volatile a128 *v, a128 op) {
 170   a128 cmp = *v;
 171   *v = cmp + op;
 172   return cmp;
 173 }
 174
 175 a128 func_sub(volatile a128 *v, a128 op) {
 176   a128 cmp = *v;
 177   *v = cmp - op;
 178   return cmp;
 179 }
 180
 181 a128 func_and(volatile a128 *v, a128 op) {
 182   a128 cmp = *v;
 183   *v = cmp & op;
 184   return cmp;
 185 }
 186
 187 a128 func_or(volatile a128 *v, a128 op) {
 188   a128 cmp = *v;
 189   *v = cmp | op;
 190   return cmp;
 191 }
 192
 193 a128 func_xor(volatile a128 *v, a128 op) {
 194   a128 cmp = *v;
 195   *v = cmp ^ op;
 196   return cmp;
 197 }
 198
 199 a128 func_nand(volatile a128 *v, a128 op) {
 200   a128 cmp = *v;
 201   *v = ~(cmp & op);
 202   return cmp;
 203 }
 204
 205 a128 func_cas(volatile a128 *v, a128 cmp, a128 xch) {
 206   a128 cur = *v;
 207   if (cur == cmp)
 208     *v = xch;
 209   return cur;
 210 }
 211 #endif
 212
 213 #define SCOPED_ATOMIC(func, ...) \
 214     mo = ConvertOrder(mo); \
 215     mo = flags()->force_seq_cst_atomics ? (morder)mo_seq_cst : mo; \
 216     ThreadState *const thr = cur_thread(); \
 217     ProcessPendingSignals(thr); \
 218     const uptr pc = (uptr)__builtin_return_address(0); \
 219     AtomicStatInc(thr, sizeof(*a), mo, StatAtomic##func); \
 220     ScopedAtomic sa(thr, pc, __FUNCTION__); \
 221     return Atomic##func(thr, pc, __VA_ARGS__); \
 222 /**/
 223
 224 template<typename T>
 225 static T AtomicLoad(ThreadState *thr, uptr pc, const volatile T *a,
 226     morder mo) {
 227   CHECK(IsLoadOrder(mo));
 228   // This fast-path is critical for performance.
 229   // Assume the access is atomic.
 230   if (!IsAcquireOrder(mo) && sizeof(T) <= sizeof(a))
 231     return *a;
 232   SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, false);
 233   thr->clock.set(thr->tid, thr->fast_state.epoch());
 234   thr->clock.acquire(&s->clock);
 235   T v = *a;
 236   s->mtx.ReadUnlock();
 237   __sync_synchronize();
 238   return v;
 239 }
 240
 241 template<typename T>
 242 static void AtomicStore(ThreadState *thr, uptr pc, volatile T *a, T v,
 243     morder mo) {
 244   CHECK(IsStoreOrder(mo));
 245   // This fast-path is critical for performance.
 246   // Assume the access is atomic.
 247   // Strictly saying even relaxed store cuts off release sequence,
 248   // so must reset the clock.
 249   if (!IsReleaseOrder(mo) && sizeof(T) <= sizeof(a)) {
 250     *a = v;
 251     return;
 252   }
 253   __sync_synchronize();
 254   SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true);
 255   thr->clock.set(thr->tid, thr->fast_state.epoch());
 256   thr->clock.ReleaseStore(&s->clock);
 257   *a = v;
 258   s->mtx.Unlock();
 259   // Trainling memory barrier to provide sequential consistency
 260   // for Dekker-like store-load synchronization.
 261   __sync_synchronize();
 262 }
 263
 264 template<typename T, T (*F)(volatile T *v, T op)>
 265 static T AtomicRMW(ThreadState *thr, uptr pc, volatile T *a, T v, morder mo) {
 266   SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true);
 267   thr->clock.set(thr->tid, thr->fast_state.epoch());
 268   if (IsAcqRelOrder(mo))
 269     thr->clock.acq_rel(&s->clock);
 270   else if (IsReleaseOrder(mo))
 271     thr->clock.release(&s->clock);
 272   else if (IsAcquireOrder(mo))
 273     thr->clock.acquire(&s->clock);
 274   v = F(a, v);
 275   s->mtx.Unlock();
 276   return v;
 277 }
 278
 279 template<typename T>
 280 static T AtomicExchange(ThreadState *thr, uptr pc, volatile T *a, T v,
 281     morder mo) {
 282   return AtomicRMW<T, func_xchg>(thr, pc, a, v, mo);
 283 }
 284
 285 template<typename T>
 286 static T AtomicFetchAdd(ThreadState *thr, uptr pc, volatile T *a, T v,
 287     morder mo) {
 288   return AtomicRMW<T, func_add>(thr, pc, a, v, mo);
 289 }
 290
 291 template<typename T>
 292 static T AtomicFetchSub(ThreadState *thr, uptr pc, volatile T *a, T v,
 293     morder mo) {
 294   return AtomicRMW<T, func_sub>(thr, pc, a, v, mo);
 295 }
 296
 297 template<typename T>
 298 static T AtomicFetchAnd(ThreadState *thr, uptr pc, volatile T *a, T v,
 299     morder mo) {
 300   return AtomicRMW<T, func_and>(thr, pc, a, v, mo);
 301 }
 302
 303 template<typename T>
 304 static T AtomicFetchOr(ThreadState *thr, uptr pc, volatile T *a, T v,
 305     morder mo) {
 306   return AtomicRMW<T, func_or>(thr, pc, a, v, mo);
 307 }
 308
 309 template<typename T>
 310 static T AtomicFetchXor(ThreadState *thr, uptr pc, volatile T *a, T v,
 311     morder mo) {
 312   return AtomicRMW<T, func_xor>(thr, pc, a, v, mo);
 313 }
 314
 315 template<typename T>
 316 static T AtomicFetchNand(ThreadState *thr, uptr pc, volatile T *a, T v,
 317     morder mo) {
 318   return AtomicRMW<T, func_nand>(thr, pc, a, v, mo);
 319 }
 320
 321 template<typename T>
 322 static bool AtomicCAS(ThreadState *thr, uptr pc,
 323     volatile T *a, T *c, T v, morder mo, morder fmo) {
 324   (void)fmo;  // Unused because llvm does not pass it yet.
 325   SyncVar *s = CTX()->synctab.GetOrCreateAndLock(thr, pc, (uptr)a, true);
 326   thr->clock.set(thr->tid, thr->fast_state.epoch());
 327   if (IsAcqRelOrder(mo))
 328     thr->clock.acq_rel(&s->clock);
 329   else if (IsReleaseOrder(mo))
 330     thr->clock.release(&s->clock);
 331   else if (IsAcquireOrder(mo))
 332     thr->clock.acquire(&s->clock);
 333   T cc = *c;
 334   T pr = func_cas(a, cc, v);
 335   s->mtx.Unlock();
 336   if (pr == cc)
 337     return true;
 338   *c = pr;
 339   return false;
 340 }
 341
 342 template<typename T>
 343 static T AtomicCAS(ThreadState *thr, uptr pc,
 344     volatile T *a, T c, T v, morder mo, morder fmo) {
 345   AtomicCAS(thr, pc, a, &c, v, mo, fmo);
 346   return c;
 347 }
 348
 349 static void AtomicFence(ThreadState *thr, uptr pc, morder mo) {
 350   // FIXME(dvyukov): not implemented.
 351   __sync_synchronize();
 352 }
 353
 354 a8 __tsan_atomic8_load(const volatile a8 *a, morder mo) {
 355   SCOPED_ATOMIC(Load, a, mo);
 356 }
 357
 358 a16 __tsan_atomic16_load(const volatile a16 *a, morder mo) {
 359   SCOPED_ATOMIC(Load, a, mo);
 360 }
 361
 362 a32 __tsan_atomic32_load(const volatile a32 *a, morder mo) {
 363   SCOPED_ATOMIC(Load, a, mo);
 364 }
 365
 366 a64 __tsan_atomic64_load(const volatile a64 *a, morder mo) {
 367   SCOPED_ATOMIC(Load, a, mo);
 368 }
 369
 370 #if __TSAN_HAS_INT128
 371 a128 __tsan_atomic128_load(const volatile a128 *a, morder mo) {
 372   SCOPED_ATOMIC(Load, a, mo);
 373 }
 374 #endif
 375
 376 void __tsan_atomic8_store(volatile a8 *a, a8 v, morder mo) {
 377   SCOPED_ATOMIC(Store, a, v, mo);
 378 }
 379
 380 void __tsan_atomic16_store(volatile a16 *a, a16 v, morder mo) {
 381   SCOPED_ATOMIC(Store, a, v, mo);
 382 }
 383
 384 void __tsan_atomic32_store(volatile a32 *a, a32 v, morder mo) {
 385   SCOPED_ATOMIC(Store, a, v, mo);
 386 }
 387
 388 void __tsan_atomic64_store(volatile a64 *a, a64 v, morder mo) {
 389   SCOPED_ATOMIC(Store, a, v, mo);
 390 }
 391
 392 #if __TSAN_HAS_INT128
 393 void __tsan_atomic128_store(volatile a128 *a, a128 v, morder mo) {
 394   SCOPED_ATOMIC(Store, a, v, mo);
 395 }
 396 #endif
 397
 398 a8 __tsan_atomic8_exchange(volatile a8 *a, a8 v, morder mo) {
 399   SCOPED_ATOMIC(Exchange, a, v, mo);
 400 }
 401
 402 a16 __tsan_atomic16_exchange(volatile a16 *a, a16 v, morder mo) {
 403   SCOPED_ATOMIC(Exchange, a, v, mo);
 404 }
 405
 406 a32 __tsan_atomic32_exchange(volatile a32 *a, a32 v, morder mo) {
 407   SCOPED_ATOMIC(Exchange, a, v, mo);
 408 }
 409
 410 a64 __tsan_atomic64_exchange(volatile a64 *a, a64 v, morder mo) {
 411   SCOPED_ATOMIC(Exchange, a, v, mo);
 412 }
 413
 414 #if __TSAN_HAS_INT128
 415 a128 __tsan_atomic128_exchange(volatile a128 *a, a128 v, morder mo) {
 416   SCOPED_ATOMIC(Exchange, a, v, mo);
 417 }
 418 #endif
 419
 420 a8 __tsan_atomic8_fetch_add(volatile a8 *a, a8 v, morder mo) {
 421   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 422 }
 423
 424 a16 __tsan_atomic16_fetch_add(volatile a16 *a, a16 v, morder mo) {
 425   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 426 }
 427
 428 a32 __tsan_atomic32_fetch_add(volatile a32 *a, a32 v, morder mo) {
 429   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 430 }
 431
 432 a64 __tsan_atomic64_fetch_add(volatile a64 *a, a64 v, morder mo) {
 433   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 434 }
 435
 436 #if __TSAN_HAS_INT128
 437 a128 __tsan_atomic128_fetch_add(volatile a128 *a, a128 v, morder mo) {
 438   SCOPED_ATOMIC(FetchAdd, a, v, mo);
 439 }
 440 #endif
 441
 442 a8 __tsan_atomic8_fetch_sub(volatile a8 *a, a8 v, morder mo) {
 443   SCOPED_ATOMIC(FetchSub, a, v, mo);
 444 }
 445
 446 a16 __tsan_atomic16_fetch_sub(volatile a16 *a, a16 v, morder mo) {
 447   SCOPED_ATOMIC(FetchSub, a, v, mo);
 448 }
 449
 450 a32 __tsan_atomic32_fetch_sub(volatile a32 *a, a32 v, morder mo) {
 451   SCOPED_ATOMIC(FetchSub, a, v, mo);
 452 }
 453
 454 a64 __tsan_atomic64_fetch_sub(volatile a64 *a, a64 v, morder mo) {
 455   SCOPED_ATOMIC(FetchSub, a, v, mo);
 456 }
 457
 458 #if __TSAN_HAS_INT128
 459 a128 __tsan_atomic128_fetch_sub(volatile a128 *a, a128 v, morder mo) {
 460   SCOPED_ATOMIC(FetchSub, a, v, mo);
 461 }
 462 #endif
 463
 464 a8 __tsan_atomic8_fetch_and(volatile a8 *a, a8 v, morder mo) {
 465   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 466 }
 467
 468 a16 __tsan_atomic16_fetch_and(volatile a16 *a, a16 v, morder mo) {
 469   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 470 }
 471
 472 a32 __tsan_atomic32_fetch_and(volatile a32 *a, a32 v, morder mo) {
 473   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 474 }
 475
 476 a64 __tsan_atomic64_fetch_and(volatile a64 *a, a64 v, morder mo) {
 477   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 478 }
 479
 480 #if __TSAN_HAS_INT128
 481 a128 __tsan_atomic128_fetch_and(volatile a128 *a, a128 v, morder mo) {
 482   SCOPED_ATOMIC(FetchAnd, a, v, mo);
 483 }
 484 #endif
 485
 486 a8 __tsan_atomic8_fetch_or(volatile a8 *a, a8 v, morder mo) {
 487   SCOPED_ATOMIC(FetchOr, a, v, mo);
 488 }
 489
 490 a16 __tsan_atomic16_fetch_or(volatile a16 *a, a16 v, morder mo) {
 491   SCOPED_ATOMIC(FetchOr, a, v, mo);
 492 }
 493
 494 a32 __tsan_atomic32_fetch_or(volatile a32 *a, a32 v, morder mo) {
 495   SCOPED_ATOMIC(FetchOr, a, v, mo);
 496 }
 497
 498 a64 __tsan_atomic64_fetch_or(volatile a64 *a, a64 v, morder mo) {
 499   SCOPED_ATOMIC(FetchOr, a, v, mo);
 500 }
 501
 502 #if __TSAN_HAS_INT128
 503 a128 __tsan_atomic128_fetch_or(volatile a128 *a, a128 v, morder mo) {
 504   SCOPED_ATOMIC(FetchOr, a, v, mo);
 505 }
 506 #endif
 507
 508 a8 __tsan_atomic8_fetch_xor(volatile a8 *a, a8 v, morder mo) {
 509   SCOPED_ATOMIC(FetchXor, a, v, mo);
 510 }
 511
 512 a16 __tsan_atomic16_fetch_xor(volatile a16 *a, a16 v, morder mo) {
 513   SCOPED_ATOMIC(FetchXor, a, v, mo);
 514 }
 515
 516 a32 __tsan_atomic32_fetch_xor(volatile a32 *a, a32 v, morder mo) {
 517   SCOPED_ATOMIC(FetchXor, a, v, mo);
 518 }
 519
 520 a64 __tsan_atomic64_fetch_xor(volatile a64 *a, a64 v, morder mo) {
 521   SCOPED_ATOMIC(FetchXor, a, v, mo);
 522 }
 523
 524 #if __TSAN_HAS_INT128
 525 a128 __tsan_atomic128_fetch_xor(volatile a128 *a, a128 v, morder mo) {
 526   SCOPED_ATOMIC(FetchXor, a, v, mo);
 527 }
 528 #endif
 529
 530 a8 __tsan_atomic8_fetch_nand(volatile a8 *a, a8 v, morder mo) {
 531   SCOPED_ATOMIC(FetchNand, a, v, mo);
 532 }
 533
 534 a16 __tsan_atomic16_fetch_nand(volatile a16 *a, a16 v, morder mo) {
 535   SCOPED_ATOMIC(FetchNand, a, v, mo);
 536 }
 537
 538 a32 __tsan_atomic32_fetch_nand(volatile a32 *a, a32 v, morder mo) {
 539   SCOPED_ATOMIC(FetchNand, a, v, mo);
 540 }
 541
 542 a64 __tsan_atomic64_fetch_nand(volatile a64 *a, a64 v, morder mo) {
 543   SCOPED_ATOMIC(FetchNand, a, v, mo);
 544 }
 545
 546 #if __TSAN_HAS_INT128
 547 a128 __tsan_atomic128_fetch_nand(volatile a128 *a, a128 v, morder mo) {
 548   SCOPED_ATOMIC(FetchNand, a, v, mo);
 549 }
 550 #endif
 551
 552 int __tsan_atomic8_compare_exchange_strong(volatile a8 *a, a8 *c, a8 v,
 553     morder mo, morder fmo) {
 554   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 555 }
 556
 557 int __tsan_atomic16_compare_exchange_strong(volatile a16 *a, a16 *c, a16 v,
 558     morder mo, morder fmo) {
 559   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 560 }
 561
 562 int __tsan_atomic32_compare_exchange_strong(volatile a32 *a, a32 *c, a32 v,
 563     morder mo, morder fmo) {
 564   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 565 }
 566
 567 int __tsan_atomic64_compare_exchange_strong(volatile a64 *a, a64 *c, a64 v,
 568     morder mo, morder fmo) {
 569   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 570 }
 571
 572 #if __TSAN_HAS_INT128
 573 int __tsan_atomic128_compare_exchange_strong(volatile a128 *a, a128 *c, a128 v,
 574     morder mo, morder fmo) {
 575   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 576 }
 577 #endif
 578
 579 int __tsan_atomic8_compare_exchange_weak(volatile a8 *a, a8 *c, a8 v,
 580     morder mo, morder fmo) {
 581   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 582 }
 583
 584 int __tsan_atomic16_compare_exchange_weak(volatile a16 *a, a16 *c, a16 v,
 585     morder mo, morder fmo) {
 586   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 587 }
 588
 589 int __tsan_atomic32_compare_exchange_weak(volatile a32 *a, a32 *c, a32 v,
 590     morder mo, morder fmo) {
 591   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 592 }
 593
 594 int __tsan_atomic64_compare_exchange_weak(volatile a64 *a, a64 *c, a64 v,
 595     morder mo, morder fmo) {
 596   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 597 }
 598
 599 #if __TSAN_HAS_INT128
 600 int __tsan_atomic128_compare_exchange_weak(volatile a128 *a, a128 *c, a128 v,
 601     morder mo, morder fmo) {
 602   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 603 }
 604 #endif
 605
 606 a8 __tsan_atomic8_compare_exchange_val(volatile a8 *a, a8 c, a8 v,
 607     morder mo, morder fmo) {
 608   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 609 }
 610 a16 __tsan_atomic16_compare_exchange_val(volatile a16 *a, a16 c, a16 v,
 611     morder mo, morder fmo) {
 612   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 613 }
 614
 615 a32 __tsan_atomic32_compare_exchange_val(volatile a32 *a, a32 c, a32 v,
 616     morder mo, morder fmo) {
 617   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 618 }
 619
 620 a64 __tsan_atomic64_compare_exchange_val(volatile a64 *a, a64 c, a64 v,
 621     morder mo, morder fmo) {
 622   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 623 }
 624
 625 #if __TSAN_HAS_INT128
 626 a128 __tsan_atomic64_compare_exchange_val(volatile a128 *a, a128 c, a128 v,
 627     morder mo, morder fmo) {
 628   SCOPED_ATOMIC(CAS, a, c, v, mo, fmo);
 629 }
 630 #endif
 631
 632 void __tsan_atomic_thread_fence(morder mo) {
 633   char* a;
 634   SCOPED_ATOMIC(Fence, mo);
 635 }
 636
 637 void __tsan_atomic_signal_fence(morder mo) {
 638 }