libsanitizer/tsan/tsan_update_shadow_word_inl.h

   1 //===-- tsan_update_shadow_word_inl.h ---------------------------*- C++ -*-===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer (TSan), a race detector.
   9 //
  10 // Body of the hottest inner loop.
  11 // If we wrap this body into a function, compilers (both gcc and clang)
  12 // produce sligtly less efficient code.
  13 //===----------------------------------------------------------------------===//
  14 do {
  15   StatInc(thr, StatShadowProcessed);
  16   const unsigned kAccessSize = 1 << kAccessSizeLog;
  17   u64 *sp = &shadow_mem[idx];
  18   old = LoadShadow(sp);
  19   if (old.IsZero()) {
  20     StatInc(thr, StatShadowZero);
  21     if (store_word)
  22       StoreIfNotYetStored(sp, &store_word);
  23     // The above StoreIfNotYetStored could be done unconditionally
  24     // and it even shows 4% gain on synthetic benchmarks (r4307).
  25     break;
  26   }
  27   // is the memory access equal to the previous?
  28   if (Shadow::Addr0AndSizeAreEqual(cur, old)) {
  29     StatInc(thr, StatShadowSameSize);
  30     // same thread?
  31     if (Shadow::TidsAreEqual(old, cur)) {
  32       StatInc(thr, StatShadowSameThread);
  33       if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))
  34         StoreIfNotYetStored(sp, &store_word);
  35       break;
  36     }
  37     StatInc(thr, StatShadowAnotherThread);
  38     if (HappensBefore(old, thr)) {
  39       if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))
  40         StoreIfNotYetStored(sp, &store_word);
  41       break;
  42     }
  43     if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
  44       break;
  45     goto RACE;
  46   }
  47   // Do the memory access intersect?
  48   if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) {
  49     StatInc(thr, StatShadowIntersect);
  50     if (Shadow::TidsAreEqual(old, cur)) {
  51       StatInc(thr, StatShadowSameThread);
  52       break;
  53     }
  54     StatInc(thr, StatShadowAnotherThread);
  55     if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
  56       break;
  57     if (HappensBefore(old, thr))
  58       break;
  59     goto RACE;
  60   }
  61   // The accesses do not intersect.
  62   StatInc(thr, StatShadowNotIntersect);
  63   break;
  64 } while (0);