libsanitizer/lsan/lsan_common.cpp

   1 //=-- lsan_common.cpp -----------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of LeakSanitizer.
  10 // Implementation of common leak checking functionality.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "lsan_common.h"
  15
  16 #include "sanitizer_common/sanitizer_common.h"
  17 #include "sanitizer_common/sanitizer_flag_parser.h"
  18 #include "sanitizer_common/sanitizer_flags.h"
  19 #include "sanitizer_common/sanitizer_placement_new.h"
  20 #include "sanitizer_common/sanitizer_procmaps.h"
  21 #include "sanitizer_common/sanitizer_report_decorator.h"
  22 #include "sanitizer_common/sanitizer_stackdepot.h"
  23 #include "sanitizer_common/sanitizer_stacktrace.h"
  24 #include "sanitizer_common/sanitizer_suppressions.h"
  25 #include "sanitizer_common/sanitizer_thread_registry.h"
  26 #include "sanitizer_common/sanitizer_tls_get_addr.h"
  27
  28 #if CAN_SANITIZE_LEAKS
  29 namespace __lsan {
  30
  31 // This mutex is used to prevent races between DoLeakCheck and IgnoreObject, and
  32 // also to protect the global list of root regions.
  33 Mutex global_mutex;
  34
  35 Flags lsan_flags;
  36
  37 void DisableCounterUnderflow() {
  38   if (common_flags()->detect_leaks) {
  39     Report("Unmatched call to __lsan_enable().\n");
  40     Die();
  41   }
  42 }
  43
  44 void Flags::SetDefaults() {
  45 #  define LSAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
  46 #  include "lsan_flags.inc"
  47 #  undef LSAN_FLAG
  48 }
  49
  50 void RegisterLsanFlags(FlagParser *parser, Flags *f) {
  51 #  define LSAN_FLAG(Type, Name, DefaultValue, Description) \
  52     RegisterFlag(parser, #Name, Description, &f->Name);
  53 #  include "lsan_flags.inc"
  54 #  undef LSAN_FLAG
  55 }
  56
  57 #  define LOG_POINTERS(...)      \
  58     do {                         \
  59       if (flags()->log_pointers) \
  60         Report(__VA_ARGS__);     \
  61     } while (0)
  62
  63 #  define LOG_THREADS(...)      \
  64     do {                        \
  65       if (flags()->log_threads) \
  66         Report(__VA_ARGS__);    \
  67     } while (0)
  68
  69 class LeakSuppressionContext {
  70   bool parsed = false;
  71   SuppressionContext context;
  72   bool suppressed_stacks_sorted = true;
  73   InternalMmapVector<u32> suppressed_stacks;
  74   const LoadedModule *suppress_module = nullptr;
  75
  76   void LazyInit();
  77   Suppression *GetSuppressionForAddr(uptr addr);
  78   bool SuppressInvalid(const StackTrace &stack);
  79   bool SuppressByRule(const StackTrace &stack, uptr hit_count, uptr total_size);
  80
  81  public:
  82   LeakSuppressionContext(const char *supprression_types[],
  83                          int suppression_types_num)
  84       : context(supprression_types, suppression_types_num) {}
  85
  86   bool Suppress(u32 stack_trace_id, uptr hit_count, uptr total_size);
  87
  88   const InternalMmapVector<u32> &GetSortedSuppressedStacks() {
  89     if (!suppressed_stacks_sorted) {
  90       suppressed_stacks_sorted = true;
  91       SortAndDedup(suppressed_stacks);
  92     }
  93     return suppressed_stacks;
  94   }
  95   void PrintMatchedSuppressions();
  96 };
  97
  98 ALIGNED(64) static char suppression_placeholder[sizeof(LeakSuppressionContext)];
  99 static LeakSuppressionContext *suppression_ctx = nullptr;
 100 static const char kSuppressionLeak[] = "leak";
 101 static const char *kSuppressionTypes[] = {kSuppressionLeak};
 102 static const char kStdSuppressions[] =
 103 #  if SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
 104     // For more details refer to the SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
 105     // definition.
 106     "leak:*pthread_exit*\n"
 107 #  endif  // SANITIZER_SUPPRESS_LEAK_ON_PTHREAD_EXIT
 108 #  if SANITIZER_MAC
 109     // For Darwin and os_log/os_trace: https://reviews.llvm.org/D35173
 110     "leak:*_os_trace*\n"
 111 #  endif
 112     // TLS leak in some glibc versions, described in
 113     // https://sourceware.org/bugzilla/show_bug.cgi?id=12650.
 114     "leak:*tls_get_addr*\n";
 115
 116 void InitializeSuppressions() {
 117   CHECK_EQ(nullptr, suppression_ctx);
 118   suppression_ctx = new (suppression_placeholder)
 119       LeakSuppressionContext(kSuppressionTypes, ARRAY_SIZE(kSuppressionTypes));
 120 }
 121
 122 void LeakSuppressionContext::LazyInit() {
 123   if (!parsed) {
 124     parsed = true;
 125     context.ParseFromFile(flags()->suppressions);
 126     if (&__lsan_default_suppressions)
 127       context.Parse(__lsan_default_suppressions());
 128     context.Parse(kStdSuppressions);
 129     if (flags()->use_tls && flags()->use_ld_allocations)
 130       suppress_module = GetLinker();
 131   }
 132 }
 133
 134 Suppression *LeakSuppressionContext::GetSuppressionForAddr(uptr addr) {
 135   Suppression *s = nullptr;
 136
 137   // Suppress by module name.
 138   const char *module_name = Symbolizer::GetOrInit()->GetModuleNameForPc(addr);
 139   if (!module_name)
 140     module_name = "<unknown module>";
 141   if (context.Match(module_name, kSuppressionLeak, &s))
 142     return s;
 143
 144   // Suppress by file or function name.
 145   SymbolizedStack *frames = Symbolizer::GetOrInit()->SymbolizePC(addr);
 146   for (SymbolizedStack *cur = frames; cur; cur = cur->next) {
 147     if (context.Match(cur->info.function, kSuppressionLeak, &s) ||
 148         context.Match(cur->info.file, kSuppressionLeak, &s)) {
 149       break;
 150     }
 151   }
 152   frames->ClearAll();
 153   return s;
 154 }
 155
 156 static uptr GetCallerPC(const StackTrace &stack) {
 157   // The top frame is our malloc/calloc/etc. The next frame is the caller.
 158   if (stack.size >= 2)
 159     return stack.trace[1];
 160   return 0;
 161 }
 162
 163 // On Linux, treats all chunks allocated from ld-linux.so as reachable, which
 164 // covers dynamically allocated TLS blocks, internal dynamic loader's loaded
 165 // modules accounting etc.
 166 // Dynamic TLS blocks contain the TLS variables of dynamically loaded modules.
 167 // They are allocated with a __libc_memalign() call in allocate_and_init()
 168 // (elf/dl-tls.c). Glibc won't tell us the address ranges occupied by those
 169 // blocks, but we can make sure they come from our own allocator by intercepting
 170 // __libc_memalign(). On top of that, there is no easy way to reach them. Their
 171 // addresses are stored in a dynamically allocated array (the DTV) which is
 172 // referenced from the static TLS. Unfortunately, we can't just rely on the DTV
 173 // being reachable from the static TLS, and the dynamic TLS being reachable from
 174 // the DTV. This is because the initial DTV is allocated before our interception
 175 // mechanism kicks in, and thus we don't recognize it as allocated memory. We
 176 // can't special-case it either, since we don't know its size.
 177 // Our solution is to include in the root set all allocations made from
 178 // ld-linux.so (which is where allocate_and_init() is implemented). This is
 179 // guaranteed to include all dynamic TLS blocks (and possibly other allocations
 180 // which we don't care about).
 181 // On all other platforms, this simply checks to ensure that the caller pc is
 182 // valid before reporting chunks as leaked.
 183 bool LeakSuppressionContext::SuppressInvalid(const StackTrace &stack) {
 184   uptr caller_pc = GetCallerPC(stack);
 185   // If caller_pc is unknown, this chunk may be allocated in a coroutine. Mark
 186   // it as reachable, as we can't properly report its allocation stack anyway.
 187   return !caller_pc ||
 188          (suppress_module && suppress_module->containsAddress(caller_pc));
 189 }
 190
 191 bool LeakSuppressionContext::SuppressByRule(const StackTrace &stack,
 192                                             uptr hit_count, uptr total_size) {
 193   for (uptr i = 0; i < stack.size; i++) {
 194     Suppression *s = GetSuppressionForAddr(
 195         StackTrace::GetPreviousInstructionPc(stack.trace[i]));
 196     if (s) {
 197       s->weight += total_size;
 198       atomic_fetch_add(&s->hit_count, hit_count, memory_order_relaxed);
 199       return true;
 200     }
 201   }
 202   return false;
 203 }
 204
 205 bool LeakSuppressionContext::Suppress(u32 stack_trace_id, uptr hit_count,
 206                                       uptr total_size) {
 207   LazyInit();
 208   StackTrace stack = StackDepotGet(stack_trace_id);
 209   if (!SuppressInvalid(stack) && !SuppressByRule(stack, hit_count, total_size))
 210     return false;
 211   suppressed_stacks_sorted = false;
 212   suppressed_stacks.push_back(stack_trace_id);
 213   return true;
 214 }
 215
 216 static LeakSuppressionContext *GetSuppressionContext() {
 217   CHECK(suppression_ctx);
 218   return suppression_ctx;
 219 }
 220
 221 static InternalMmapVectorNoCtor<RootRegion> root_regions;
 222
 223 InternalMmapVectorNoCtor<RootRegion> const *GetRootRegions() {
 224   return &root_regions;
 225 }
 226
 227 void InitCommonLsan() {
 228   if (common_flags()->detect_leaks) {
 229     // Initialization which can fail or print warnings should only be done if
 230     // LSan is actually enabled.
 231     InitializeSuppressions();
 232     InitializePlatformSpecificModules();
 233   }
 234 }
 235
 236 class Decorator : public __sanitizer::SanitizerCommonDecorator {
 237  public:
 238   Decorator() : SanitizerCommonDecorator() {}
 239   const char *Error() { return Red(); }
 240   const char *Leak() { return Blue(); }
 241 };
 242
 243 static inline bool CanBeAHeapPointer(uptr p) {
 244   // Since our heap is located in mmap-ed memory, we can assume a sensible lower
 245   // bound on heap addresses.
 246   const uptr kMinAddress = 4 * 4096;
 247   if (p < kMinAddress)
 248     return false;
 249 #  if defined(__x86_64__)
 250   // Accept only canonical form user-space addresses.
 251   return ((p >> 47) == 0);
 252 #  elif defined(__mips64)
 253   return ((p >> 40) == 0);
 254 #  elif defined(__aarch64__)
 255   unsigned runtimeVMA = (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
 256   return ((p >> runtimeVMA) == 0);
 257 #  else
 258   return true;
 259 #  endif
 260 }
 261
 262 // Scans the memory range, looking for byte patterns that point into allocator
 263 // chunks. Marks those chunks with |tag| and adds them to |frontier|.
 264 // There are two usage modes for this function: finding reachable chunks
 265 // (|tag| = kReachable) and finding indirectly leaked chunks
 266 // (|tag| = kIndirectlyLeaked). In the second case, there's no flood fill,
 267 // so |frontier| = 0.
 268 void ScanRangeForPointers(uptr begin, uptr end, Frontier *frontier,
 269                           const char *region_type, ChunkTag tag) {
 270   CHECK(tag == kReachable || tag == kIndirectlyLeaked);
 271   const uptr alignment = flags()->pointer_alignment();
 272   LOG_POINTERS("Scanning %s range %p-%p.\n", region_type, (void *)begin,
 273                (void *)end);
 274   uptr pp = begin;
 275   if (pp % alignment)
 276     pp = pp + alignment - pp % alignment;
 277   for (; pp + sizeof(void *) <= end; pp += alignment) {
 278     void *p = *reinterpret_cast<void **>(pp);
 279     if (!CanBeAHeapPointer(reinterpret_cast<uptr>(p)))
 280       continue;
 281     uptr chunk = PointsIntoChunk(p);
 282     if (!chunk)
 283       continue;
 284     // Pointers to self don't count. This matters when tag == kIndirectlyLeaked.
 285     if (chunk == begin)
 286       continue;
 287     LsanMetadata m(chunk);
 288     if (m.tag() == kReachable || m.tag() == kIgnored)
 289       continue;
 290
 291     // Do this check relatively late so we can log only the interesting cases.
 292     if (!flags()->use_poisoned && WordIsPoisoned(pp)) {
 293       LOG_POINTERS(
 294           "%p is poisoned: ignoring %p pointing into chunk %p-%p of size "
 295           "%zu.\n",
 296           (void *)pp, p, (void *)chunk, (void *)(chunk + m.requested_size()),
 297           m.requested_size());
 298       continue;
 299     }
 300
 301     m.set_tag(tag);
 302     LOG_POINTERS("%p: found %p pointing into chunk %p-%p of size %zu.\n",
 303                  (void *)pp, p, (void *)chunk,
 304                  (void *)(chunk + m.requested_size()), m.requested_size());
 305     if (frontier)
 306       frontier->push_back(chunk);
 307   }
 308 }
 309
 310 // Scans a global range for pointers
 311 void ScanGlobalRange(uptr begin, uptr end, Frontier *frontier) {
 312   uptr allocator_begin = 0, allocator_end = 0;
 313   GetAllocatorGlobalRange(&allocator_begin, &allocator_end);
 314   if (begin <= allocator_begin && allocator_begin < end) {
 315     CHECK_LE(allocator_begin, allocator_end);
 316     CHECK_LE(allocator_end, end);
 317     if (begin < allocator_begin)
 318       ScanRangeForPointers(begin, allocator_begin, frontier, "GLOBAL",
 319                            kReachable);
 320     if (allocator_end < end)
 321       ScanRangeForPointers(allocator_end, end, frontier, "GLOBAL", kReachable);
 322   } else {
 323     ScanRangeForPointers(begin, end, frontier, "GLOBAL", kReachable);
 324   }
 325 }
 326
 327 void ForEachExtraStackRangeCb(uptr begin, uptr end, void *arg) {
 328   Frontier *frontier = reinterpret_cast<Frontier *>(arg);
 329   ScanRangeForPointers(begin, end, frontier, "FAKE STACK", kReachable);
 330 }
 331
 332 #  if SANITIZER_FUCHSIA
 333
 334 // Fuchsia handles all threads together with its own callback.
 335 static void ProcessThreads(SuspendedThreadsList const &, Frontier *) {}
 336
 337 #  else
 338
 339 #    if SANITIZER_ANDROID
 340 // FIXME: Move this out into *libcdep.cpp
 341 extern "C" SANITIZER_WEAK_ATTRIBUTE void __libc_iterate_dynamic_tls(
 342     pid_t, void (*cb)(void *, void *, uptr, void *), void *);
 343 #    endif
 344
 345 static void ProcessThreadRegistry(Frontier *frontier) {
 346   InternalMmapVector<uptr> ptrs;
 347   GetThreadRegistryLocked()->RunCallbackForEachThreadLocked(
 348       GetAdditionalThreadContextPtrs, &ptrs);
 349
 350   for (uptr i = 0; i < ptrs.size(); ++i) {
 351     void *ptr = reinterpret_cast<void *>(ptrs[i]);
 352     uptr chunk = PointsIntoChunk(ptr);
 353     if (!chunk)
 354       continue;
 355     LsanMetadata m(chunk);
 356     if (!m.allocated())
 357       continue;
 358
 359     // Mark as reachable and add to frontier.
 360     LOG_POINTERS("Treating pointer %p from ThreadContext as reachable\n", ptr);
 361     m.set_tag(kReachable);
 362     frontier->push_back(chunk);
 363   }
 364 }
 365
 366 // Scans thread data (stacks and TLS) for heap pointers.
 367 static void ProcessThreads(SuspendedThreadsList const &suspended_threads,
 368                            Frontier *frontier) {
 369   InternalMmapVector<uptr> registers;
 370   for (uptr i = 0; i < suspended_threads.ThreadCount(); i++) {
 371     tid_t os_id = static_cast<tid_t>(suspended_threads.GetThreadID(i));
 372     LOG_THREADS("Processing thread %llu.\n", os_id);
 373     uptr stack_begin, stack_end, tls_begin, tls_end, cache_begin, cache_end;
 374     DTLS *dtls;
 375     bool thread_found =
 376         GetThreadRangesLocked(os_id, &stack_begin, &stack_end, &tls_begin,
 377                               &tls_end, &cache_begin, &cache_end, &dtls);
 378     if (!thread_found) {
 379       // If a thread can't be found in the thread registry, it's probably in the
 380       // process of destruction. Log this event and move on.
 381       LOG_THREADS("Thread %llu not found in registry.\n", os_id);
 382       continue;
 383     }
 384     uptr sp;
 385     PtraceRegistersStatus have_registers =
 386         suspended_threads.GetRegistersAndSP(i, &registers, &sp);
 387     if (have_registers != REGISTERS_AVAILABLE) {
 388       Report("Unable to get registers from thread %llu.\n", os_id);
 389       // If unable to get SP, consider the entire stack to be reachable unless
 390       // GetRegistersAndSP failed with ESRCH.
 391       if (have_registers == REGISTERS_UNAVAILABLE_FATAL)
 392         continue;
 393       sp = stack_begin;
 394     }
 395
 396     if (flags()->use_registers && have_registers) {
 397       uptr registers_begin = reinterpret_cast<uptr>(registers.data());
 398       uptr registers_end =
 399           reinterpret_cast<uptr>(registers.data() + registers.size());
 400       ScanRangeForPointers(registers_begin, registers_end, frontier,
 401                            "REGISTERS", kReachable);
 402     }
 403
 404     if (flags()->use_stacks) {
 405       LOG_THREADS("Stack at %p-%p (SP = %p).\n", (void *)stack_begin,
 406                   (void *)stack_end, (void *)sp);
 407       if (sp < stack_begin || sp >= stack_end) {
 408         // SP is outside the recorded stack range (e.g. the thread is running a
 409         // signal handler on alternate stack, or swapcontext was used).
 410         // Again, consider the entire stack range to be reachable.
 411         LOG_THREADS("WARNING: stack pointer not in stack range.\n");
 412         uptr page_size = GetPageSizeCached();
 413         int skipped = 0;
 414         while (stack_begin < stack_end &&
 415                !IsAccessibleMemoryRange(stack_begin, 1)) {
 416           skipped++;
 417           stack_begin += page_size;
 418         }
 419         LOG_THREADS("Skipped %d guard page(s) to obtain stack %p-%p.\n",
 420                     skipped, (void *)stack_begin, (void *)stack_end);
 421       } else {
 422         // Shrink the stack range to ignore out-of-scope values.
 423         stack_begin = sp;
 424       }
 425       ScanRangeForPointers(stack_begin, stack_end, frontier, "STACK",
 426                            kReachable);
 427       ForEachExtraStackRange(os_id, ForEachExtraStackRangeCb, frontier);
 428     }
 429
 430     if (flags()->use_tls) {
 431       if (tls_begin) {
 432         LOG_THREADS("TLS at %p-%p.\n", (void *)tls_begin, (void *)tls_end);
 433         // If the tls and cache ranges don't overlap, scan full tls range,
 434         // otherwise, only scan the non-overlapping portions
 435         if (cache_begin == cache_end || tls_end < cache_begin ||
 436             tls_begin > cache_end) {
 437           ScanRangeForPointers(tls_begin, tls_end, frontier, "TLS", kReachable);
 438         } else {
 439           if (tls_begin < cache_begin)
 440             ScanRangeForPointers(tls_begin, cache_begin, frontier, "TLS",
 441                                  kReachable);
 442           if (tls_end > cache_end)
 443             ScanRangeForPointers(cache_end, tls_end, frontier, "TLS",
 444                                  kReachable);
 445         }
 446       }
 447 #    if SANITIZER_ANDROID
 448       auto *cb = +[](void *dtls_begin, void *dtls_end, uptr /*dso_idd*/,
 449                      void *arg) -> void {
 450         ScanRangeForPointers(reinterpret_cast<uptr>(dtls_begin),
 451                              reinterpret_cast<uptr>(dtls_end),
 452                              reinterpret_cast<Frontier *>(arg), "DTLS",
 453                              kReachable);
 454       };
 455
 456       // FIXME: There might be a race-condition here (and in Bionic) if the
 457       // thread is suspended in the middle of updating its DTLS. IOWs, we
 458       // could scan already freed memory. (probably fine for now)
 459       __libc_iterate_dynamic_tls(os_id, cb, frontier);
 460 #    else
 461       if (dtls && !DTLSInDestruction(dtls)) {
 462         ForEachDVT(dtls, [&](const DTLS::DTV &dtv, int id) {
 463           uptr dtls_beg = dtv.beg;
 464           uptr dtls_end = dtls_beg + dtv.size;
 465           if (dtls_beg < dtls_end) {
 466             LOG_THREADS("DTLS %d at %p-%p.\n", id, (void *)dtls_beg,
 467                         (void *)dtls_end);
 468             ScanRangeForPointers(dtls_beg, dtls_end, frontier, "DTLS",
 469                                  kReachable);
 470           }
 471         });
 472       } else {
 473         // We are handling a thread with DTLS under destruction. Log about
 474         // this and continue.
 475         LOG_THREADS("Thread %llu has DTLS under destruction.\n", os_id);
 476       }
 477 #    endif
 478     }
 479   }
 480
 481   // Add pointers reachable from ThreadContexts
 482   ProcessThreadRegistry(frontier);
 483 }
 484
 485 #  endif  // SANITIZER_FUCHSIA
 486
 487 void ScanRootRegion(Frontier *frontier, const RootRegion &root_region,
 488                     uptr region_begin, uptr region_end, bool is_readable) {
 489   uptr intersection_begin = Max(root_region.begin, region_begin);
 490   uptr intersection_end = Min(region_end, root_region.begin + root_region.size);
 491   if (intersection_begin >= intersection_end)
 492     return;
 493   LOG_POINTERS("Root region %p-%p intersects with mapped region %p-%p (%s)\n",
 494                (void *)root_region.begin,
 495                (void *)(root_region.begin + root_region.size),
 496                (void *)region_begin, (void *)region_end,
 497                is_readable ? "readable" : "unreadable");
 498   if (is_readable)
 499     ScanRangeForPointers(intersection_begin, intersection_end, frontier, "ROOT",
 500                          kReachable);
 501 }
 502
 503 static void ProcessRootRegion(Frontier *frontier,
 504                               const RootRegion &root_region) {
 505   MemoryMappingLayout proc_maps(/*cache_enabled*/ true);
 506   MemoryMappedSegment segment;
 507   while (proc_maps.Next(&segment)) {
 508     ScanRootRegion(frontier, root_region, segment.start, segment.end,
 509                    segment.IsReadable());
 510   }
 511 }
 512
 513 // Scans root regions for heap pointers.
 514 static void ProcessRootRegions(Frontier *frontier) {
 515   if (!flags()->use_root_regions)
 516     return;
 517   for (uptr i = 0; i < root_regions.size(); i++)
 518     ProcessRootRegion(frontier, root_regions[i]);
 519 }
 520
 521 static void FloodFillTag(Frontier *frontier, ChunkTag tag) {
 522   while (frontier->size()) {
 523     uptr next_chunk = frontier->back();
 524     frontier->pop_back();
 525     LsanMetadata m(next_chunk);
 526     ScanRangeForPointers(next_chunk, next_chunk + m.requested_size(), frontier,
 527                          "HEAP", tag);
 528   }
 529 }
 530
 531 // ForEachChunk callback. If the chunk is marked as leaked, marks all chunks
 532 // which are reachable from it as indirectly leaked.
 533 static void MarkIndirectlyLeakedCb(uptr chunk, void *arg) {
 534   chunk = GetUserBegin(chunk);
 535   LsanMetadata m(chunk);
 536   if (m.allocated() && m.tag() != kReachable) {
 537     ScanRangeForPointers(chunk, chunk + m.requested_size(),
 538                          /* frontier */ nullptr, "HEAP", kIndirectlyLeaked);
 539   }
 540 }
 541
 542 static void IgnoredSuppressedCb(uptr chunk, void *arg) {
 543   CHECK(arg);
 544   chunk = GetUserBegin(chunk);
 545   LsanMetadata m(chunk);
 546   if (!m.allocated() || m.tag() == kIgnored)
 547     return;
 548
 549   const InternalMmapVector<u32> &suppressed =
 550       *static_cast<const InternalMmapVector<u32> *>(arg);
 551   uptr idx = InternalLowerBound(suppressed, m.stack_trace_id());
 552   if (idx >= suppressed.size() || m.stack_trace_id() != suppressed[idx])
 553     return;
 554
 555   LOG_POINTERS("Suppressed: chunk %p-%p of size %zu.\n", (void *)chunk,
 556                (void *)(chunk + m.requested_size()), m.requested_size());
 557   m.set_tag(kIgnored);
 558 }
 559
 560 // ForEachChunk callback. If chunk is marked as ignored, adds its address to
 561 // frontier.
 562 static void CollectIgnoredCb(uptr chunk, void *arg) {
 563   CHECK(arg);
 564   chunk = GetUserBegin(chunk);
 565   LsanMetadata m(chunk);
 566   if (m.allocated() && m.tag() == kIgnored) {
 567     LOG_POINTERS("Ignored: chunk %p-%p of size %zu.\n", (void *)chunk,
 568                  (void *)(chunk + m.requested_size()), m.requested_size());
 569     reinterpret_cast<Frontier *>(arg)->push_back(chunk);
 570   }
 571 }
 572
 573 // Sets the appropriate tag on each chunk.
 574 static void ClassifyAllChunks(SuspendedThreadsList const &suspended_threads,
 575                               Frontier *frontier) {
 576   const InternalMmapVector<u32> &suppressed_stacks =
 577       GetSuppressionContext()->GetSortedSuppressedStacks();
 578   if (!suppressed_stacks.empty()) {
 579     ForEachChunk(IgnoredSuppressedCb,
 580                  const_cast<InternalMmapVector<u32> *>(&suppressed_stacks));
 581   }
 582   ForEachChunk(CollectIgnoredCb, frontier);
 583   ProcessGlobalRegions(frontier);
 584   ProcessThreads(suspended_threads, frontier);
 585   ProcessRootRegions(frontier);
 586   FloodFillTag(frontier, kReachable);
 587
 588   // The check here is relatively expensive, so we do this in a separate flood
 589   // fill. That way we can skip the check for chunks that are reachable
 590   // otherwise.
 591   LOG_POINTERS("Processing platform-specific allocations.\n");
 592   ProcessPlatformSpecificAllocations(frontier);
 593   FloodFillTag(frontier, kReachable);
 594
 595   // Iterate over leaked chunks and mark those that are reachable from other
 596   // leaked chunks.
 597   LOG_POINTERS("Scanning leaked chunks.\n");
 598   ForEachChunk(MarkIndirectlyLeakedCb, nullptr);
 599 }
 600
 601 // ForEachChunk callback. Resets the tags to pre-leak-check state.
 602 static void ResetTagsCb(uptr chunk, void *arg) {
 603   (void)arg;
 604   chunk = GetUserBegin(chunk);
 605   LsanMetadata m(chunk);
 606   if (m.allocated() && m.tag() != kIgnored)
 607     m.set_tag(kDirectlyLeaked);
 608 }
 609
 610 // ForEachChunk callback. Aggregates information about unreachable chunks into
 611 // a LeakReport.
 612 static void CollectLeaksCb(uptr chunk, void *arg) {
 613   CHECK(arg);
 614   LeakedChunks *leaks = reinterpret_cast<LeakedChunks *>(arg);
 615   chunk = GetUserBegin(chunk);
 616   LsanMetadata m(chunk);
 617   if (!m.allocated())
 618     return;
 619   if (m.tag() == kDirectlyLeaked || m.tag() == kIndirectlyLeaked)
 620     leaks->push_back({chunk, m.stack_trace_id(), m.requested_size(), m.tag()});
 621 }
 622
 623 void LeakSuppressionContext::PrintMatchedSuppressions() {
 624   InternalMmapVector<Suppression *> matched;
 625   context.GetMatched(&matched);
 626   if (!matched.size())
 627     return;
 628   const char *line = "-----------------------------------------------------";
 629   Printf("%s\n", line);
 630   Printf("Suppressions used:\n");
 631   Printf("  count      bytes template\n");
 632   for (uptr i = 0; i < matched.size(); i++) {
 633     Printf("%7zu %10zu %s\n",
 634            static_cast<uptr>(atomic_load_relaxed(&matched[i]->hit_count)),
 635            matched[i]->weight, matched[i]->templ);
 636   }
 637   Printf("%s\n\n", line);
 638 }
 639
 640 static void ReportIfNotSuspended(ThreadContextBase *tctx, void *arg) {
 641   const InternalMmapVector<tid_t> &suspended_threads =
 642       *(const InternalMmapVector<tid_t> *)arg;
 643   if (tctx->status == ThreadStatusRunning) {
 644     uptr i = InternalLowerBound(suspended_threads, tctx->os_id);
 645     if (i >= suspended_threads.size() || suspended_threads[i] != tctx->os_id)
 646       Report(
 647           "Running thread %llu was not suspended. False leaks are possible.\n",
 648           tctx->os_id);
 649   }
 650 }
 651
 652 #  if SANITIZER_FUCHSIA
 653
 654 // Fuchsia provides a libc interface that guarantees all threads are
 655 // covered, and SuspendedThreadList is never really used.
 656 static void ReportUnsuspendedThreads(const SuspendedThreadsList &) {}
 657
 658 #  else  // !SANITIZER_FUCHSIA
 659
 660 static void ReportUnsuspendedThreads(
 661     const SuspendedThreadsList &suspended_threads) {
 662   InternalMmapVector<tid_t> threads(suspended_threads.ThreadCount());
 663   for (uptr i = 0; i < suspended_threads.ThreadCount(); ++i)
 664     threads[i] = suspended_threads.GetThreadID(i);
 665
 666   Sort(threads.data(), threads.size());
 667
 668   GetThreadRegistryLocked()->RunCallbackForEachThreadLocked(
 669       &ReportIfNotSuspended, &threads);
 670 }
 671
 672 #  endif  // !SANITIZER_FUCHSIA
 673
 674 static void CheckForLeaksCallback(const SuspendedThreadsList &suspended_threads,
 675                                   void *arg) {
 676   CheckForLeaksParam *param = reinterpret_cast<CheckForLeaksParam *>(arg);
 677   CHECK(param);
 678   CHECK(!param->success);
 679   ReportUnsuspendedThreads(suspended_threads);
 680   ClassifyAllChunks(suspended_threads, &param->frontier);
 681   ForEachChunk(CollectLeaksCb, &param->leaks);
 682   // Clean up for subsequent leak checks. This assumes we did not overwrite any
 683   // kIgnored tags.
 684   ForEachChunk(ResetTagsCb, nullptr);
 685   param->success = true;
 686 }
 687
 688 static bool PrintResults(LeakReport &report) {
 689   uptr unsuppressed_count = report.UnsuppressedLeakCount();
 690   if (unsuppressed_count) {
 691     Decorator d;
 692     Printf(
 693         "\n"
 694         "================================================================="
 695         "\n");
 696     Printf("%s", d.Error());
 697     Report("ERROR: LeakSanitizer: detected memory leaks\n");
 698     Printf("%s", d.Default());
 699     report.ReportTopLeaks(flags()->max_leaks);
 700   }
 701   if (common_flags()->print_suppressions)
 702     GetSuppressionContext()->PrintMatchedSuppressions();
 703   if (unsuppressed_count > 0) {
 704     report.PrintSummary();
 705     return true;
 706   }
 707   return false;
 708 }
 709
 710 static bool CheckForLeaks() {
 711   if (&__lsan_is_turned_off && __lsan_is_turned_off())
 712     return false;
 713   // Inside LockStuffAndStopTheWorld we can't run symbolizer, so we can't match
 714   // suppressions. However if a stack id was previously suppressed, it should be
 715   // suppressed in future checks as well.
 716   for (int i = 0;; ++i) {
 717     EnsureMainThreadIDIsCorrect();
 718     CheckForLeaksParam param;
 719     LockStuffAndStopTheWorld(CheckForLeaksCallback, &param);
 720     if (!param.success) {
 721       Report("LeakSanitizer has encountered a fatal error.\n");
 722       Report(
 723           "HINT: For debugging, try setting environment variable "
 724           "LSAN_OPTIONS=verbosity=1:log_threads=1\n");
 725       Report(
 726           "HINT: LeakSanitizer does not work under ptrace (strace, gdb, "
 727           "etc)\n");
 728       Die();
 729     }
 730     LeakReport leak_report;
 731     leak_report.AddLeakedChunks(param.leaks);
 732
 733     // No new suppressions stacks, so rerun will not help and we can report.
 734     if (!leak_report.ApplySuppressions())
 735       return PrintResults(leak_report);
 736
 737     // No indirect leaks to report, so we are done here.
 738     if (!leak_report.IndirectUnsuppressedLeakCount())
 739       return PrintResults(leak_report);
 740
 741     if (i >= 8) {
 742       Report("WARNING: LeakSanitizer gave up on indirect leaks suppression.\n");
 743       return PrintResults(leak_report);
 744     }
 745
 746     // We found a new previously unseen suppressed call stack. Rerun to make
 747     // sure it does not hold indirect leaks.
 748     VReport(1, "Rerun with %zu suppressed stacks.",
 749             GetSuppressionContext()->GetSortedSuppressedStacks().size());
 750   }
 751 }
 752
 753 static bool has_reported_leaks = false;
 754 bool HasReportedLeaks() { return has_reported_leaks; }
 755
 756 void DoLeakCheck() {
 757   Lock l(&global_mutex);
 758   static bool already_done;
 759   if (already_done)
 760     return;
 761   already_done = true;
 762   has_reported_leaks = CheckForLeaks();
 763   if (has_reported_leaks)
 764     HandleLeaks();
 765 }
 766
 767 static int DoRecoverableLeakCheck() {
 768   Lock l(&global_mutex);
 769   bool have_leaks = CheckForLeaks();
 770   return have_leaks ? 1 : 0;
 771 }
 772
 773 void DoRecoverableLeakCheckVoid() { DoRecoverableLeakCheck(); }
 774
 775 ///// LeakReport implementation. /////
 776
 777 // A hard limit on the number of distinct leaks, to avoid quadratic complexity
 778 // in LeakReport::AddLeakedChunk(). We don't expect to ever see this many leaks
 779 // in real-world applications.
 780 // FIXME: Get rid of this limit by moving logic into DedupLeaks.
 781 const uptr kMaxLeaksConsidered = 5000;
 782
 783 void LeakReport::AddLeakedChunks(const LeakedChunks &chunks) {
 784   for (const LeakedChunk &leak : chunks) {
 785     uptr chunk = leak.chunk;
 786     u32 stack_trace_id = leak.stack_trace_id;
 787     uptr leaked_size = leak.leaked_size;
 788     ChunkTag tag = leak.tag;
 789     CHECK(tag == kDirectlyLeaked || tag == kIndirectlyLeaked);
 790
 791     if (u32 resolution = flags()->resolution) {
 792       StackTrace stack = StackDepotGet(stack_trace_id);
 793       stack.size = Min(stack.size, resolution);
 794       stack_trace_id = StackDepotPut(stack);
 795     }
 796
 797     bool is_directly_leaked = (tag == kDirectlyLeaked);
 798     uptr i;
 799     for (i = 0; i < leaks_.size(); i++) {
 800       if (leaks_[i].stack_trace_id == stack_trace_id &&
 801           leaks_[i].is_directly_leaked == is_directly_leaked) {
 802         leaks_[i].hit_count++;
 803         leaks_[i].total_size += leaked_size;
 804         break;
 805       }
 806     }
 807     if (i == leaks_.size()) {
 808       if (leaks_.size() == kMaxLeaksConsidered)
 809         return;
 810       Leak leak = {next_id_++,         /* hit_count */ 1,
 811                    leaked_size,        stack_trace_id,
 812                    is_directly_leaked, /* is_suppressed */ false};
 813       leaks_.push_back(leak);
 814     }
 815     if (flags()->report_objects) {
 816       LeakedObject obj = {leaks_[i].id, chunk, leaked_size};
 817       leaked_objects_.push_back(obj);
 818     }
 819   }
 820 }
 821
 822 static bool LeakComparator(const Leak &leak1, const Leak &leak2) {
 823   if (leak1.is_directly_leaked == leak2.is_directly_leaked)
 824     return leak1.total_size > leak2.total_size;
 825   else
 826     return leak1.is_directly_leaked;
 827 }
 828
 829 void LeakReport::ReportTopLeaks(uptr num_leaks_to_report) {
 830   CHECK(leaks_.size() <= kMaxLeaksConsidered);
 831   Printf("\n");
 832   if (leaks_.size() == kMaxLeaksConsidered)
 833     Printf(
 834         "Too many leaks! Only the first %zu leaks encountered will be "
 835         "reported.\n",
 836         kMaxLeaksConsidered);
 837
 838   uptr unsuppressed_count = UnsuppressedLeakCount();
 839   if (num_leaks_to_report > 0 && num_leaks_to_report < unsuppressed_count)
 840     Printf("The %zu top leak(s):\n", num_leaks_to_report);
 841   Sort(leaks_.data(), leaks_.size(), &LeakComparator);
 842   uptr leaks_reported = 0;
 843   for (uptr i = 0; i < leaks_.size(); i++) {
 844     if (leaks_[i].is_suppressed)
 845       continue;
 846     PrintReportForLeak(i);
 847     leaks_reported++;
 848     if (leaks_reported == num_leaks_to_report)
 849       break;
 850   }
 851   if (leaks_reported < unsuppressed_count) {
 852     uptr remaining = unsuppressed_count - leaks_reported;
 853     Printf("Omitting %zu more leak(s).\n", remaining);
 854   }
 855 }
 856
 857 void LeakReport::PrintReportForLeak(uptr index) {
 858   Decorator d;
 859   Printf("%s", d.Leak());
 860   Printf("%s leak of %zu byte(s) in %zu object(s) allocated from:\n",
 861          leaks_[index].is_directly_leaked ? "Direct" : "Indirect",
 862          leaks_[index].total_size, leaks_[index].hit_count);
 863   Printf("%s", d.Default());
 864
 865   CHECK(leaks_[index].stack_trace_id);
 866   StackDepotGet(leaks_[index].stack_trace_id).Print();
 867
 868   if (flags()->report_objects) {
 869     Printf("Objects leaked above:\n");
 870     PrintLeakedObjectsForLeak(index);
 871     Printf("\n");
 872   }
 873 }
 874
 875 void LeakReport::PrintLeakedObjectsForLeak(uptr index) {
 876   u32 leak_id = leaks_[index].id;
 877   for (uptr j = 0; j < leaked_objects_.size(); j++) {
 878     if (leaked_objects_[j].leak_id == leak_id)
 879       Printf("%p (%zu bytes)\n", (void *)leaked_objects_[j].addr,
 880              leaked_objects_[j].size);
 881   }
 882 }
 883
 884 void LeakReport::PrintSummary() {
 885   CHECK(leaks_.size() <= kMaxLeaksConsidered);
 886   uptr bytes = 0, allocations = 0;
 887   for (uptr i = 0; i < leaks_.size(); i++) {
 888     if (leaks_[i].is_suppressed)
 889       continue;
 890     bytes += leaks_[i].total_size;
 891     allocations += leaks_[i].hit_count;
 892   }
 893   InternalScopedString summary;
 894   summary.append("%zu byte(s) leaked in %zu allocation(s).", bytes,
 895                  allocations);
 896   ReportErrorSummary(summary.data());
 897 }
 898
 899 uptr LeakReport::ApplySuppressions() {
 900   LeakSuppressionContext *suppressions = GetSuppressionContext();
 901   uptr new_suppressions = false;
 902   for (uptr i = 0; i < leaks_.size(); i++) {
 903     if (suppressions->Suppress(leaks_[i].stack_trace_id, leaks_[i].hit_count,
 904                                leaks_[i].total_size)) {
 905       leaks_[i].is_suppressed = true;
 906       ++new_suppressions;
 907     }
 908   }
 909   return new_suppressions;
 910 }
 911
 912 uptr LeakReport::UnsuppressedLeakCount() {
 913   uptr result = 0;
 914   for (uptr i = 0; i < leaks_.size(); i++)
 915     if (!leaks_[i].is_suppressed)
 916       result++;
 917   return result;
 918 }
 919
 920 uptr LeakReport::IndirectUnsuppressedLeakCount() {
 921   uptr result = 0;
 922   for (uptr i = 0; i < leaks_.size(); i++)
 923     if (!leaks_[i].is_suppressed && !leaks_[i].is_directly_leaked)
 924       result++;
 925   return result;
 926 }
 927
 928 }  // namespace __lsan
 929 #else   // CAN_SANITIZE_LEAKS
 930 namespace __lsan {
 931 void InitCommonLsan() {}
 932 void DoLeakCheck() {}
 933 void DoRecoverableLeakCheckVoid() {}
 934 void DisableInThisThread() {}
 935 void EnableInThisThread() {}
 936 }  // namespace __lsan
 937 #endif  // CAN_SANITIZE_LEAKS
 938
 939 using namespace __lsan;
 940
 941 extern "C" {
 942 SANITIZER_INTERFACE_ATTRIBUTE
 943 void __lsan_ignore_object(const void *p) {
 944 #if CAN_SANITIZE_LEAKS
 945   if (!common_flags()->detect_leaks)
 946     return;
 947   // Cannot use PointsIntoChunk or LsanMetadata here, since the allocator is not
 948   // locked.
 949   Lock l(&global_mutex);
 950   IgnoreObjectResult res = IgnoreObjectLocked(p);
 951   if (res == kIgnoreObjectInvalid)
 952     VReport(1, "__lsan_ignore_object(): no heap object found at %p", p);
 953   if (res == kIgnoreObjectAlreadyIgnored)
 954     VReport(1,
 955             "__lsan_ignore_object(): "
 956             "heap object at %p is already being ignored\n",
 957             p);
 958   if (res == kIgnoreObjectSuccess)
 959     VReport(1, "__lsan_ignore_object(): ignoring heap object at %p\n", p);
 960 #endif  // CAN_SANITIZE_LEAKS
 961 }
 962
 963 SANITIZER_INTERFACE_ATTRIBUTE
 964 void __lsan_register_root_region(const void *begin, uptr size) {
 965 #if CAN_SANITIZE_LEAKS
 966   Lock l(&global_mutex);
 967   RootRegion region = {reinterpret_cast<uptr>(begin), size};
 968   root_regions.push_back(region);
 969   VReport(1, "Registered root region at %p of size %zu\n", begin, size);
 970 #endif  // CAN_SANITIZE_LEAKS
 971 }
 972
 973 SANITIZER_INTERFACE_ATTRIBUTE
 974 void __lsan_unregister_root_region(const void *begin, uptr size) {
 975 #if CAN_SANITIZE_LEAKS
 976   Lock l(&global_mutex);
 977   bool removed = false;
 978   for (uptr i = 0; i < root_regions.size(); i++) {
 979     RootRegion region = root_regions[i];
 980     if (region.begin == reinterpret_cast<uptr>(begin) && region.size == size) {
 981       removed = true;
 982       uptr last_index = root_regions.size() - 1;
 983       root_regions[i] = root_regions[last_index];
 984       root_regions.pop_back();
 985       VReport(1, "Unregistered root region at %p of size %zu\n", begin, size);
 986       break;
 987     }
 988   }
 989   if (!removed) {
 990     Report(
 991         "__lsan_unregister_root_region(): region at %p of size %zu has not "
 992         "been registered.\n",
 993         begin, size);
 994     Die();
 995   }
 996 #endif  // CAN_SANITIZE_LEAKS
 997 }
 998
 999 SANITIZER_INTERFACE_ATTRIBUTE
1000 void __lsan_disable() {
1001 #if CAN_SANITIZE_LEAKS
1002   __lsan::DisableInThisThread();
1003 #endif
1004 }
1005
1006 SANITIZER_INTERFACE_ATTRIBUTE
1007 void __lsan_enable() {
1008 #if CAN_SANITIZE_LEAKS
1009   __lsan::EnableInThisThread();
1010 #endif
1011 }
1012
1013 SANITIZER_INTERFACE_ATTRIBUTE
1014 void __lsan_do_leak_check() {
1015 #if CAN_SANITIZE_LEAKS
1016   if (common_flags()->detect_leaks)
1017     __lsan::DoLeakCheck();
1018 #endif  // CAN_SANITIZE_LEAKS
1019 }
1020
1021 SANITIZER_INTERFACE_ATTRIBUTE
1022 int __lsan_do_recoverable_leak_check() {
1023 #if CAN_SANITIZE_LEAKS
1024   if (common_flags()->detect_leaks)
1025     return __lsan::DoRecoverableLeakCheck();
1026 #endif  // CAN_SANITIZE_LEAKS
1027   return 0;
1028 }
1029
1030 SANITIZER_INTERFACE_WEAK_DEF(const char *, __lsan_default_options, void) {
1031   return "";
1032 }
1033
1034 #if !SANITIZER_SUPPORTS_WEAK_HOOKS
1035 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE int
1036 __lsan_is_turned_off() {
1037   return 0;
1038 }
1039
1040 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE const char *
1041 __lsan_default_suppressions() {
1042   return "";
1043 }
1044 #endif
1045 }  // extern "C"