libsanitizer/lsan/lsan_allocator.cpp

   1 //=-- lsan_allocator.cpp --------------------------------------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of LeakSanitizer.
  10 // See lsan_allocator.h for details.
  11 //
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "lsan_allocator.h"
  15
  16 #include "sanitizer_common/sanitizer_allocator.h"
  17 #include "sanitizer_common/sanitizer_allocator_checks.h"
  18 #include "sanitizer_common/sanitizer_allocator_interface.h"
  19 #include "sanitizer_common/sanitizer_allocator_report.h"
  20 #include "sanitizer_common/sanitizer_errno.h"
  21 #include "sanitizer_common/sanitizer_internal_defs.h"
  22 #include "sanitizer_common/sanitizer_stackdepot.h"
  23 #include "sanitizer_common/sanitizer_stacktrace.h"
  24 #include "lsan_common.h"
  25
  26 extern "C" void *memset(void *ptr, int value, uptr num);
  27
  28 namespace __lsan {
  29 #if defined(__i386__) || defined(__arm__)
  30 static const uptr kMaxAllowedMallocSize = 1UL << 30;
  31 #elif defined(__mips64) || defined(__aarch64__)
  32 static const uptr kMaxAllowedMallocSize = 4UL << 30;
  33 #else
  34 static const uptr kMaxAllowedMallocSize = 8UL << 30;
  35 #endif
  36
  37 static Allocator allocator;
  38
  39 static uptr max_malloc_size;
  40
  41 void InitializeAllocator() {
  42   SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
  43   allocator.InitLinkerInitialized(
  44       common_flags()->allocator_release_to_os_interval_ms);
  45   if (common_flags()->max_allocation_size_mb)
  46     max_malloc_size = Min(common_flags()->max_allocation_size_mb << 20,
  47                           kMaxAllowedMallocSize);
  48   else
  49     max_malloc_size = kMaxAllowedMallocSize;
  50 }
  51
  52 void AllocatorThreadFinish() {
  53   allocator.SwallowCache(GetAllocatorCache());
  54 }
  55
  56 static ChunkMetadata *Metadata(const void *p) {
  57   return reinterpret_cast<ChunkMetadata *>(allocator.GetMetaData(p));
  58 }
  59
  60 static void RegisterAllocation(const StackTrace &stack, void *p, uptr size) {
  61   if (!p) return;
  62   ChunkMetadata *m = Metadata(p);
  63   CHECK(m);
  64   m->tag = DisabledInThisThread() ? kIgnored : kDirectlyLeaked;
  65   m->stack_trace_id = StackDepotPut(stack);
  66   m->requested_size = size;
  67   atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 1, memory_order_relaxed);
  68 }
  69
  70 static void RegisterDeallocation(void *p) {
  71   if (!p) return;
  72   ChunkMetadata *m = Metadata(p);
  73   CHECK(m);
  74   atomic_store(reinterpret_cast<atomic_uint8_t *>(m), 0, memory_order_relaxed);
  75 }
  76
  77 static void *ReportAllocationSizeTooBig(uptr size, const StackTrace &stack) {
  78   if (AllocatorMayReturnNull()) {
  79     Report("WARNING: LeakSanitizer failed to allocate 0x%zx bytes\n", size);
  80     return nullptr;
  81   }
  82   ReportAllocationSizeTooBig(size, max_malloc_size, &stack);
  83 }
  84
  85 void *Allocate(const StackTrace &stack, uptr size, uptr alignment,
  86                bool cleared) {
  87   if (size == 0)
  88     size = 1;
  89   if (size > max_malloc_size)
  90     return ReportAllocationSizeTooBig(size, stack);
  91   void *p = allocator.Allocate(GetAllocatorCache(), size, alignment);
  92   if (UNLIKELY(!p)) {
  93     SetAllocatorOutOfMemory();
  94     if (AllocatorMayReturnNull())
  95       return nullptr;
  96     ReportOutOfMemory(size, &stack);
  97   }
  98   // Do not rely on the allocator to clear the memory (it's slow).
  99   if (cleared && allocator.FromPrimary(p))
 100     memset(p, 0, size);
 101   RegisterAllocation(stack, p, size);
 102   if (&__sanitizer_malloc_hook) __sanitizer_malloc_hook(p, size);
 103   RunMallocHooks(p, size);
 104   return p;
 105 }
 106
 107 static void *Calloc(uptr nmemb, uptr size, const StackTrace &stack) {
 108   if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
 109     if (AllocatorMayReturnNull())
 110       return nullptr;
 111     ReportCallocOverflow(nmemb, size, &stack);
 112   }
 113   size *= nmemb;
 114   return Allocate(stack, size, 1, true);
 115 }
 116
 117 void Deallocate(void *p) {
 118   if (&__sanitizer_free_hook) __sanitizer_free_hook(p);
 119   RunFreeHooks(p);
 120   RegisterDeallocation(p);
 121   allocator.Deallocate(GetAllocatorCache(), p);
 122 }
 123
 124 void *Reallocate(const StackTrace &stack, void *p, uptr new_size,
 125                  uptr alignment) {
 126   if (new_size > max_malloc_size) {
 127     ReportAllocationSizeTooBig(new_size, stack);
 128     return nullptr;
 129   }
 130   RegisterDeallocation(p);
 131   void *new_p =
 132       allocator.Reallocate(GetAllocatorCache(), p, new_size, alignment);
 133   if (new_p)
 134     RegisterAllocation(stack, new_p, new_size);
 135   else if (new_size != 0)
 136     RegisterAllocation(stack, p, new_size);
 137   return new_p;
 138 }
 139
 140 void GetAllocatorCacheRange(uptr *begin, uptr *end) {
 141   *begin = (uptr)GetAllocatorCache();
 142   *end = *begin + sizeof(AllocatorCache);
 143 }
 144
 145 uptr GetMallocUsableSize(const void *p) {
 146   ChunkMetadata *m = Metadata(p);
 147   if (!m) return 0;
 148   return m->requested_size;
 149 }
 150
 151 int lsan_posix_memalign(void **memptr, uptr alignment, uptr size,
 152                         const StackTrace &stack) {
 153   if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
 154     if (AllocatorMayReturnNull())
 155       return errno_EINVAL;
 156     ReportInvalidPosixMemalignAlignment(alignment, &stack);
 157   }
 158   void *ptr = Allocate(stack, size, alignment, kAlwaysClearMemory);
 159   if (UNLIKELY(!ptr))
 160     // OOM error is already taken care of by Allocate.
 161     return errno_ENOMEM;
 162   CHECK(IsAligned((uptr)ptr, alignment));
 163   *memptr = ptr;
 164   return 0;
 165 }
 166
 167 void *lsan_aligned_alloc(uptr alignment, uptr size, const StackTrace &stack) {
 168   if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
 169     errno = errno_EINVAL;
 170     if (AllocatorMayReturnNull())
 171       return nullptr;
 172     ReportInvalidAlignedAllocAlignment(size, alignment, &stack);
 173   }
 174   return SetErrnoOnNull(Allocate(stack, size, alignment, kAlwaysClearMemory));
 175 }
 176
 177 void *lsan_memalign(uptr alignment, uptr size, const StackTrace &stack) {
 178   if (UNLIKELY(!IsPowerOfTwo(alignment))) {
 179     errno = errno_EINVAL;
 180     if (AllocatorMayReturnNull())
 181       return nullptr;
 182     ReportInvalidAllocationAlignment(alignment, &stack);
 183   }
 184   return SetErrnoOnNull(Allocate(stack, size, alignment, kAlwaysClearMemory));
 185 }
 186
 187 void *lsan_malloc(uptr size, const StackTrace &stack) {
 188   return SetErrnoOnNull(Allocate(stack, size, 1, kAlwaysClearMemory));
 189 }
 190
 191 void lsan_free(void *p) {
 192   Deallocate(p);
 193 }
 194
 195 void *lsan_realloc(void *p, uptr size, const StackTrace &stack) {
 196   return SetErrnoOnNull(Reallocate(stack, p, size, 1));
 197 }
 198
 199 void *lsan_reallocarray(void *ptr, uptr nmemb, uptr size,
 200                         const StackTrace &stack) {
 201   if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
 202     errno = errno_ENOMEM;
 203     if (AllocatorMayReturnNull())
 204       return nullptr;
 205     ReportReallocArrayOverflow(nmemb, size, &stack);
 206   }
 207   return lsan_realloc(ptr, nmemb * size, stack);
 208 }
 209
 210 void *lsan_calloc(uptr nmemb, uptr size, const StackTrace &stack) {
 211   return SetErrnoOnNull(Calloc(nmemb, size, stack));
 212 }
 213
 214 void *lsan_valloc(uptr size, const StackTrace &stack) {
 215   return SetErrnoOnNull(
 216       Allocate(stack, size, GetPageSizeCached(), kAlwaysClearMemory));
 217 }
 218
 219 void *lsan_pvalloc(uptr size, const StackTrace &stack) {
 220   uptr PageSize = GetPageSizeCached();
 221   if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
 222     errno = errno_ENOMEM;
 223     if (AllocatorMayReturnNull())
 224       return nullptr;
 225     ReportPvallocOverflow(size, &stack);
 226   }
 227   // pvalloc(0) should allocate one page.
 228   size = size ? RoundUpTo(size, PageSize) : PageSize;
 229   return SetErrnoOnNull(Allocate(stack, size, PageSize, kAlwaysClearMemory));
 230 }
 231
 232 uptr lsan_mz_size(const void *p) {
 233   return GetMallocUsableSize(p);
 234 }
 235
 236 ///// Interface to the common LSan module. /////
 237
 238 void LockAllocator() {
 239   allocator.ForceLock();
 240 }
 241
 242 void UnlockAllocator() {
 243   allocator.ForceUnlock();
 244 }
 245
 246 void GetAllocatorGlobalRange(uptr *begin, uptr *end) {
 247   *begin = (uptr)&allocator;
 248   *end = *begin + sizeof(allocator);
 249 }
 250
 251 uptr PointsIntoChunk(void* p) {
 252   uptr addr = reinterpret_cast<uptr>(p);
 253   uptr chunk = reinterpret_cast<uptr>(allocator.GetBlockBeginFastLocked(p));
 254   if (!chunk) return 0;
 255   // LargeMmapAllocator considers pointers to the meta-region of a chunk to be
 256   // valid, but we don't want that.
 257   if (addr < chunk) return 0;
 258   ChunkMetadata *m = Metadata(reinterpret_cast<void *>(chunk));
 259   CHECK(m);
 260   if (!m->allocated)
 261     return 0;
 262   if (addr < chunk + m->requested_size)
 263     return chunk;
 264   if (IsSpecialCaseOfOperatorNew0(chunk, m->requested_size, addr))
 265     return chunk;
 266   return 0;
 267 }
 268
 269 uptr GetUserBegin(uptr chunk) {
 270   return chunk;
 271 }
 272
 273 LsanMetadata::LsanMetadata(uptr chunk) {
 274   metadata_ = Metadata(reinterpret_cast<void *>(chunk));
 275   CHECK(metadata_);
 276 }
 277
 278 bool LsanMetadata::allocated() const {
 279   return reinterpret_cast<ChunkMetadata *>(metadata_)->allocated;
 280 }
 281
 282 ChunkTag LsanMetadata::tag() const {
 283   return reinterpret_cast<ChunkMetadata *>(metadata_)->tag;
 284 }
 285
 286 void LsanMetadata::set_tag(ChunkTag value) {
 287   reinterpret_cast<ChunkMetadata *>(metadata_)->tag = value;
 288 }
 289
 290 uptr LsanMetadata::requested_size() const {
 291   return reinterpret_cast<ChunkMetadata *>(metadata_)->requested_size;
 292 }
 293
 294 u32 LsanMetadata::stack_trace_id() const {
 295   return reinterpret_cast<ChunkMetadata *>(metadata_)->stack_trace_id;
 296 }
 297
 298 void ForEachChunk(ForEachChunkCallback callback, void *arg) {
 299   allocator.ForEachChunk(callback, arg);
 300 }
 301
 302 IgnoreObjectResult IgnoreObjectLocked(const void *p) {
 303   void *chunk = allocator.GetBlockBegin(p);
 304   if (!chunk || p < chunk) return kIgnoreObjectInvalid;
 305   ChunkMetadata *m = Metadata(chunk);
 306   CHECK(m);
 307   if (m->allocated && (uptr)p < (uptr)chunk + m->requested_size) {
 308     if (m->tag == kIgnored)
 309       return kIgnoreObjectAlreadyIgnored;
 310     m->tag = kIgnored;
 311     return kIgnoreObjectSuccess;
 312   } else {
 313     return kIgnoreObjectInvalid;
 314   }
 315 }
 316
 317 void GetAdditionalThreadContextPtrs(ThreadContextBase *tctx, void *ptrs) {
 318   // This function can be used to treat memory reachable from `tctx` as live.
 319   // This is useful for threads that have been created but not yet started.
 320
 321   // This is currently a no-op because the LSan `pthread_create()` interceptor
 322   // blocks until the child thread starts which keeps the thread's `arg` pointer
 323   // live.
 324 }
 325
 326 } // namespace __lsan
 327
 328 using namespace __lsan;
 329
 330 extern "C" {
 331 SANITIZER_INTERFACE_ATTRIBUTE
 332 uptr __sanitizer_get_current_allocated_bytes() {
 333   uptr stats[AllocatorStatCount];
 334   allocator.GetStats(stats);
 335   return stats[AllocatorStatAllocated];
 336 }
 337
 338 SANITIZER_INTERFACE_ATTRIBUTE
 339 uptr __sanitizer_get_heap_size() {
 340   uptr stats[AllocatorStatCount];
 341   allocator.GetStats(stats);
 342   return stats[AllocatorStatMapped];
 343 }
 344
 345 SANITIZER_INTERFACE_ATTRIBUTE
 346 uptr __sanitizer_get_free_bytes() { return 0; }
 347
 348 SANITIZER_INTERFACE_ATTRIBUTE
 349 uptr __sanitizer_get_unmapped_bytes() { return 0; }
 350
 351 SANITIZER_INTERFACE_ATTRIBUTE
 352 uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
 353
 354 SANITIZER_INTERFACE_ATTRIBUTE
 355 int __sanitizer_get_ownership(const void *p) { return Metadata(p) != nullptr; }
 356
 357 SANITIZER_INTERFACE_ATTRIBUTE
 358 uptr __sanitizer_get_allocated_size(const void *p) {
 359   return GetMallocUsableSize(p);
 360 }
 361
 362 #if !SANITIZER_SUPPORTS_WEAK_HOOKS
 363 // Provide default (no-op) implementation of malloc hooks.
 364 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
 365 void __sanitizer_malloc_hook(void *ptr, uptr size) {
 366   (void)ptr;
 367   (void)size;
 368 }
 369 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
 370 void __sanitizer_free_hook(void *ptr) {
 371   (void)ptr;
 372 }
 373 #endif
 374 } // extern "C"