libsanitizer/tsan/tsan_update_shadow_word_inl.h

   1 //===-- tsan_update_shadow_word_inl.h ---------------------------*- C++ -*-===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer (TSan), a race detector.
   9 //
  10 // Body of the hottest inner loop.
  11 // If we wrap this body into a function, compilers (both gcc and clang)
  12 // produce sligtly less efficient code.
  13 //===----------------------------------------------------------------------===//
  14 do {
  15   StatInc(thr, StatShadowProcessed);
  16   const unsigned kAccessSize = 1 << kAccessSizeLog;
  17   u64 *sp = &shadow_mem[idx];
  18   old = LoadShadow(sp);
  19   if (old.IsZero()) {
  20     StatInc(thr, StatShadowZero);
  21     if (store_word)
  22       StoreIfNotYetStored(sp, &store_word);
  23     // The above StoreIfNotYetStored could be done unconditionally
  24     // and it even shows 4% gain on synthetic benchmarks (r4307).
  25     break;
  26   }
  27   // is the memory access equal to the previous?
  28   if (Shadow::Addr0AndSizeAreEqual(cur, old)) {
  29     StatInc(thr, StatShadowSameSize);
  30     // same thread?
  31     if (Shadow::TidsAreEqual(old, cur)) {
  32       StatInc(thr, StatShadowSameThread);
  33       if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))
  34         StoreIfNotYetStored(sp, &store_word);
  35       break;
  36     }
  37     StatInc(thr, StatShadowAnotherThread);
  38     if (HappensBefore(old, thr)) {
  39       StoreIfNotYetStored(sp, &store_word);
  40       break;
  41     }
  42     if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
  43       break;
  44     goto RACE;
  45   }
  46   // Do the memory access intersect?
  47   if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) {
  48     StatInc(thr, StatShadowIntersect);
  49     if (Shadow::TidsAreEqual(old, cur)) {
  50       StatInc(thr, StatShadowSameThread);
  51       break;
  52     }
  53     StatInc(thr, StatShadowAnotherThread);
  54     if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
  55       break;
  56     if (HappensBefore(old, thr))
  57       break;
  58     goto RACE;
  59   }
  60   // The accesses do not intersect.
  61   StatInc(thr, StatShadowNotIntersect);
  62   break;
  63 } while (0);