libsanitizer/tsan/tsan_update_shadow_word_inl.h

   1 //===-- tsan_update_shadow_word_inl.h ---------------------------*- C++ -*-===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer (TSan), a race detector.
   9 //
  10 // Body of the hottest inner loop.
  11 // If we wrap this body into a function, compilers (both gcc and clang)
  12 // produce sligtly less efficient code.
  13 //===----------------------------------------------------------------------===//
  14 do {
  15   StatInc(thr, StatShadowProcessed);
  16   const unsigned kAccessSize = 1 << kAccessSizeLog;
  17   unsigned off = cur.ComputeSearchOffset();
  18   u64 *sp = &shadow_mem[(idx + off) % kShadowCnt];
  19   old = LoadShadow(sp);
  20   if (old.IsZero()) {
  21     StatInc(thr, StatShadowZero);
  22     if (store_word)
  23       StoreIfNotYetStored(sp, &store_word);
  24     // The above StoreIfNotYetStored could be done unconditionally
  25     // and it even shows 4% gain on synthetic benchmarks (r4307).
  26     break;
  27   }
  28   // is the memory access equal to the previous?
  29   if (Shadow::Addr0AndSizeAreEqual(cur, old)) {
  30     StatInc(thr, StatShadowSameSize);
  31     // same thread?
  32     if (Shadow::TidsAreEqual(old, cur)) {
  33       StatInc(thr, StatShadowSameThread);
  34       if (OldIsInSameSynchEpoch(old, thr)) {
  35         if (old.IsRWNotWeaker(kAccessIsWrite, kIsAtomic)) {
  36           // found a slot that holds effectively the same info
  37           // (that is, same tid, same sync epoch and same size)
  38           StatInc(thr, StatMopSame);
  39           return;
  40         }
  41         StoreIfNotYetStored(sp, &store_word);
  42         break;
  43       }
  44       if (old.IsRWWeakerOrEqual(kAccessIsWrite, kIsAtomic))
  45         StoreIfNotYetStored(sp, &store_word);
  46       break;
  47     }
  48     StatInc(thr, StatShadowAnotherThread);
  49     if (HappensBefore(old, thr)) {
  50       StoreIfNotYetStored(sp, &store_word);
  51       break;
  52     }
  53     if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
  54       break;
  55     goto RACE;
  56   }
  57   // Do the memory access intersect?
  58   if (Shadow::TwoRangesIntersect(old, cur, kAccessSize)) {
  59     StatInc(thr, StatShadowIntersect);
  60     if (Shadow::TidsAreEqual(old, cur)) {
  61       StatInc(thr, StatShadowSameThread);
  62       break;
  63     }
  64     StatInc(thr, StatShadowAnotherThread);
  65     if (old.IsBothReadsOrAtomic(kAccessIsWrite, kIsAtomic))
  66       break;
  67     if (HappensBefore(old, thr))
  68       break;
  69     goto RACE;
  70   }
  71   // The accesses do not intersect.
  72   StatInc(thr, StatShadowNotIntersect);
  73   break;
  74 } while (0);