libsanitizer/hwasan/hwasan_allocator.cpp

   1 //===-- hwasan_allocator.cpp ------------------------ ---------------------===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of HWAddressSanitizer.
  10 //
  11 // HWAddressSanitizer allocator.
  12 //===----------------------------------------------------------------------===//
  13
  14 #include "sanitizer_common/sanitizer_atomic.h"
  15 #include "sanitizer_common/sanitizer_errno.h"
  16 #include "sanitizer_common/sanitizer_stackdepot.h"
  17 #include "hwasan.h"
  18 #include "hwasan_allocator.h"
  19 #include "hwasan_checks.h"
  20 #include "hwasan_mapping.h"
  21 #include "hwasan_malloc_bisect.h"
  22 #include "hwasan_thread.h"
  23 #include "hwasan_report.h"
  24 #include "lsan/lsan_common.h"
  25
  26 namespace __hwasan {
  27
  28 static Allocator allocator;
  29 static AllocatorCache fallback_allocator_cache;
  30 static SpinMutex fallback_mutex;
  31 static atomic_uint8_t hwasan_allocator_tagging_enabled;
  32
  33 static constexpr tag_t kFallbackAllocTag = 0xBB & kTagMask;
  34 static constexpr tag_t kFallbackFreeTag = 0xBC;
  35
  36 enum {
  37   // Either just allocated by underlying allocator, but AsanChunk is not yet
  38   // ready, or almost returned to undelying allocator and AsanChunk is already
  39   // meaningless.
  40   CHUNK_INVALID = 0,
  41   // The chunk is allocated and not yet freed.
  42   CHUNK_ALLOCATED = 1,
  43 };
  44
  45
  46 // Initialized in HwasanAllocatorInit, an never changed.
  47 static ALIGNED(16) u8 tail_magic[kShadowAlignment - 1];
  48 static uptr max_malloc_size;
  49
  50 bool HwasanChunkView::IsAllocated() const {
  51   return metadata_ && metadata_->IsAllocated();
  52 }
  53
  54 uptr HwasanChunkView::Beg() const {
  55   return block_;
  56 }
  57 uptr HwasanChunkView::End() const {
  58   return Beg() + UsedSize();
  59 }
  60 uptr HwasanChunkView::UsedSize() const {
  61   return metadata_->GetRequestedSize();
  62 }
  63 u32 HwasanChunkView::GetAllocStackId() const {
  64   return metadata_->GetAllocStackId();
  65 }
  66
  67 u32 HwasanChunkView::GetAllocThreadId() const {
  68   return metadata_->GetAllocThreadId();
  69 }
  70
  71 uptr HwasanChunkView::ActualSize() const {
  72   return allocator.GetActuallyAllocatedSize(reinterpret_cast<void *>(block_));
  73 }
  74
  75 bool HwasanChunkView::FromSmallHeap() const {
  76   return allocator.FromPrimary(reinterpret_cast<void *>(block_));
  77 }
  78
  79 bool HwasanChunkView::AddrIsInside(uptr addr) const {
  80   return (addr >= Beg()) && (addr < Beg() + UsedSize());
  81 }
  82
  83 inline void Metadata::SetAllocated(u32 stack, u64 size) {
  84   Thread *t = GetCurrentThread();
  85   u64 context = t ? t->unique_id() : kMainTid;
  86   context <<= 32;
  87   context += stack;
  88   requested_size_low = size & ((1ul << 32) - 1);
  89   requested_size_high = size >> 32;
  90   atomic_store(&alloc_context_id, context, memory_order_relaxed);
  91   atomic_store(&chunk_state, CHUNK_ALLOCATED, memory_order_release);
  92 }
  93
  94 inline void Metadata::SetUnallocated() {
  95   atomic_store(&chunk_state, CHUNK_INVALID, memory_order_release);
  96   requested_size_low = 0;
  97   requested_size_high = 0;
  98   atomic_store(&alloc_context_id, 0, memory_order_relaxed);
  99 }
 100
 101 inline bool Metadata::IsAllocated() const {
 102   return atomic_load(&chunk_state, memory_order_relaxed) == CHUNK_ALLOCATED;
 103 }
 104
 105 inline u64 Metadata::GetRequestedSize() const {
 106   return (static_cast<u64>(requested_size_high) << 32) + requested_size_low;
 107 }
 108
 109 inline u32 Metadata::GetAllocStackId() const {
 110   return atomic_load(&alloc_context_id, memory_order_relaxed);
 111 }
 112
 113 inline u32 Metadata::GetAllocThreadId() const {
 114   u64 context = atomic_load(&alloc_context_id, memory_order_relaxed);
 115   u32 tid = context >> 32;
 116   return tid;
 117 }
 118
 119 void GetAllocatorStats(AllocatorStatCounters s) {
 120   allocator.GetStats(s);
 121 }
 122
 123 inline void Metadata::SetLsanTag(__lsan::ChunkTag tag) {
 124   lsan_tag = tag;
 125 }
 126
 127 inline __lsan::ChunkTag Metadata::GetLsanTag() const {
 128   return static_cast<__lsan::ChunkTag>(lsan_tag);
 129 }
 130
 131 uptr GetAliasRegionStart() {
 132 #if defined(HWASAN_ALIASING_MODE)
 133   constexpr uptr kAliasRegionOffset = 1ULL << (kTaggableRegionCheckShift - 1);
 134   uptr AliasRegionStart =
 135       __hwasan_shadow_memory_dynamic_address + kAliasRegionOffset;
 136
 137   CHECK_EQ(AliasRegionStart >> kTaggableRegionCheckShift,
 138            __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
 139   CHECK_EQ(
 140       (AliasRegionStart + kAliasRegionOffset - 1) >> kTaggableRegionCheckShift,
 141       __hwasan_shadow_memory_dynamic_address >> kTaggableRegionCheckShift);
 142   return AliasRegionStart;
 143 #else
 144   return 0;
 145 #endif
 146 }
 147
 148 void HwasanAllocatorInit() {
 149   atomic_store_relaxed(&hwasan_allocator_tagging_enabled,
 150                        !flags()->disable_allocator_tagging);
 151   SetAllocatorMayReturnNull(common_flags()->allocator_may_return_null);
 152   allocator.Init(common_flags()->allocator_release_to_os_interval_ms,
 153                  GetAliasRegionStart());
 154   for (uptr i = 0; i < sizeof(tail_magic); i++)
 155     tail_magic[i] = GetCurrentThread()->GenerateRandomTag();
 156   if (common_flags()->max_allocation_size_mb) {
 157     max_malloc_size = common_flags()->max_allocation_size_mb << 20;
 158     max_malloc_size = Min(max_malloc_size, kMaxAllowedMallocSize);
 159   } else {
 160     max_malloc_size = kMaxAllowedMallocSize;
 161   }
 162 }
 163
 164 void HwasanAllocatorLock() { allocator.ForceLock(); }
 165
 166 void HwasanAllocatorUnlock() { allocator.ForceUnlock(); }
 167
 168 void AllocatorSwallowThreadLocalCache(AllocatorCache *cache) {
 169   allocator.SwallowCache(cache);
 170 }
 171
 172 static uptr TaggedSize(uptr size) {
 173   if (!size) size = 1;
 174   uptr new_size = RoundUpTo(size, kShadowAlignment);
 175   CHECK_GE(new_size, size);
 176   return new_size;
 177 }
 178
 179 static void *HwasanAllocate(StackTrace *stack, uptr orig_size, uptr alignment,
 180                             bool zeroise) {
 181   // Keep this consistent with LSAN and ASAN behavior.
 182   if (UNLIKELY(orig_size == 0))
 183     orig_size = 1;
 184   if (UNLIKELY(orig_size > max_malloc_size)) {
 185     if (AllocatorMayReturnNull()) {
 186       Report("WARNING: HWAddressSanitizer failed to allocate 0x%zx bytes\n",
 187              orig_size);
 188       return nullptr;
 189     }
 190     ReportAllocationSizeTooBig(orig_size, max_malloc_size, stack);
 191   }
 192   if (UNLIKELY(IsRssLimitExceeded())) {
 193     if (AllocatorMayReturnNull())
 194       return nullptr;
 195     ReportRssLimitExceeded(stack);
 196   }
 197
 198   alignment = Max(alignment, kShadowAlignment);
 199   uptr size = TaggedSize(orig_size);
 200   Thread *t = GetCurrentThread();
 201   void *allocated;
 202   if (t) {
 203     allocated = allocator.Allocate(t->allocator_cache(), size, alignment);
 204   } else {
 205     SpinMutexLock l(&fallback_mutex);
 206     AllocatorCache *cache = &fallback_allocator_cache;
 207     allocated = allocator.Allocate(cache, size, alignment);
 208   }
 209   if (UNLIKELY(!allocated)) {
 210     SetAllocatorOutOfMemory();
 211     if (AllocatorMayReturnNull())
 212       return nullptr;
 213     ReportOutOfMemory(size, stack);
 214   }
 215   if (zeroise) {
 216     // The secondary allocator mmaps memory, which should be zero-inited so we
 217     // don't need to explicitly clear it.
 218     if (allocator.FromPrimary(allocated))
 219       internal_memset(allocated, 0, size);
 220   } else if (flags()->max_malloc_fill_size > 0) {
 221     uptr fill_size = Min(size, (uptr)flags()->max_malloc_fill_size);
 222     internal_memset(allocated, flags()->malloc_fill_byte, fill_size);
 223   }
 224   if (size != orig_size) {
 225     u8 *tail = reinterpret_cast<u8 *>(allocated) + orig_size;
 226     uptr tail_length = size - orig_size;
 227     internal_memcpy(tail, tail_magic, tail_length - 1);
 228     // Short granule is excluded from magic tail, so we explicitly untag.
 229     tail[tail_length - 1] = 0;
 230   }
 231
 232   void *user_ptr = allocated;
 233   // Tagging can only be skipped when both tag_in_malloc and tag_in_free are
 234   // false. When tag_in_malloc = false and tag_in_free = true malloc needs to
 235   // retag to 0.
 236   if (InTaggableRegion(reinterpret_cast<uptr>(user_ptr)) &&
 237       (flags()->tag_in_malloc || flags()->tag_in_free) &&
 238       atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) {
 239     if (flags()->tag_in_malloc && malloc_bisect(stack, orig_size)) {
 240       tag_t tag = t ? t->GenerateRandomTag() : kFallbackAllocTag;
 241       uptr tag_size = orig_size ? orig_size : 1;
 242       uptr full_granule_size = RoundDownTo(tag_size, kShadowAlignment);
 243       user_ptr =
 244           (void *)TagMemoryAligned((uptr)user_ptr, full_granule_size, tag);
 245       if (full_granule_size != tag_size) {
 246         u8 *short_granule =
 247             reinterpret_cast<u8 *>(allocated) + full_granule_size;
 248         TagMemoryAligned((uptr)short_granule, kShadowAlignment,
 249                          tag_size % kShadowAlignment);
 250         short_granule[kShadowAlignment - 1] = tag;
 251       }
 252     } else {
 253       user_ptr = (void *)TagMemoryAligned((uptr)user_ptr, size, 0);
 254     }
 255   }
 256
 257   Metadata *meta =
 258       reinterpret_cast<Metadata *>(allocator.GetMetaData(allocated));
 259 #if CAN_SANITIZE_LEAKS
 260   meta->SetLsanTag(__lsan::DisabledInThisThread() ? __lsan::kIgnored
 261                                                   : __lsan::kDirectlyLeaked);
 262 #endif
 263   meta->SetAllocated(StackDepotPut(*stack), orig_size);
 264   RunMallocHooks(user_ptr, size);
 265   return user_ptr;
 266 }
 267
 268 static bool PointerAndMemoryTagsMatch(void *tagged_ptr) {
 269   CHECK(tagged_ptr);
 270   uptr tagged_uptr = reinterpret_cast<uptr>(tagged_ptr);
 271   if (!InTaggableRegion(tagged_uptr))
 272     return true;
 273   tag_t mem_tag = *reinterpret_cast<tag_t *>(
 274       MemToShadow(reinterpret_cast<uptr>(UntagPtr(tagged_ptr))));
 275   return PossiblyShortTagMatches(mem_tag, tagged_uptr, 1);
 276 }
 277
 278 static bool CheckInvalidFree(StackTrace *stack, void *untagged_ptr,
 279                              void *tagged_ptr) {
 280   // This function can return true if halt_on_error is false.
 281   if (!MemIsApp(reinterpret_cast<uptr>(untagged_ptr)) ||
 282       !PointerAndMemoryTagsMatch(tagged_ptr)) {
 283     ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr));
 284     return true;
 285   }
 286   return false;
 287 }
 288
 289 static void HwasanDeallocate(StackTrace *stack, void *tagged_ptr) {
 290   CHECK(tagged_ptr);
 291   RunFreeHooks(tagged_ptr);
 292
 293   void *untagged_ptr = UntagPtr(tagged_ptr);
 294
 295   if (CheckInvalidFree(stack, untagged_ptr, tagged_ptr))
 296     return;
 297
 298   void *aligned_ptr = reinterpret_cast<void *>(
 299       RoundDownTo(reinterpret_cast<uptr>(untagged_ptr), kShadowAlignment));
 300   tag_t pointer_tag = GetTagFromPointer(reinterpret_cast<uptr>(tagged_ptr));
 301   Metadata *meta =
 302       reinterpret_cast<Metadata *>(allocator.GetMetaData(aligned_ptr));
 303   if (!meta) {
 304     ReportInvalidFree(stack, reinterpret_cast<uptr>(tagged_ptr));
 305     return;
 306   }
 307   uptr orig_size = meta->GetRequestedSize();
 308   u32 free_context_id = StackDepotPut(*stack);
 309   u32 alloc_context_id = meta->GetAllocStackId();
 310   u32 alloc_thread_id = meta->GetAllocThreadId();
 311
 312   bool in_taggable_region =
 313       InTaggableRegion(reinterpret_cast<uptr>(tagged_ptr));
 314
 315   // Check tail magic.
 316   uptr tagged_size = TaggedSize(orig_size);
 317   if (flags()->free_checks_tail_magic && orig_size &&
 318       tagged_size != orig_size) {
 319     uptr tail_size = tagged_size - orig_size - 1;
 320     CHECK_LT(tail_size, kShadowAlignment);
 321     void *tail_beg = reinterpret_cast<void *>(
 322         reinterpret_cast<uptr>(aligned_ptr) + orig_size);
 323     tag_t short_granule_memtag = *(reinterpret_cast<tag_t *>(
 324         reinterpret_cast<uptr>(tail_beg) + tail_size));
 325     if (tail_size &&
 326         (internal_memcmp(tail_beg, tail_magic, tail_size) ||
 327          (in_taggable_region && pointer_tag != short_granule_memtag)))
 328       ReportTailOverwritten(stack, reinterpret_cast<uptr>(tagged_ptr),
 329                             orig_size, tail_magic);
 330   }
 331
 332   // TODO(kstoimenov): consider meta->SetUnallocated(free_context_id).
 333   meta->SetUnallocated();
 334   // This memory will not be reused by anyone else, so we are free to keep it
 335   // poisoned.
 336   Thread *t = GetCurrentThread();
 337   if (flags()->max_free_fill_size > 0) {
 338     uptr fill_size =
 339         Min(TaggedSize(orig_size), (uptr)flags()->max_free_fill_size);
 340     internal_memset(aligned_ptr, flags()->free_fill_byte, fill_size);
 341   }
 342   if (in_taggable_region && flags()->tag_in_free && malloc_bisect(stack, 0) &&
 343       atomic_load_relaxed(&hwasan_allocator_tagging_enabled)) {
 344     // Always store full 8-bit tags on free to maximize UAF detection.
 345     tag_t tag;
 346     if (t) {
 347       // Make sure we are not using a short granule tag as a poison tag. This
 348       // would make us attempt to read the memory on a UaF.
 349       // The tag can be zero if tagging is disabled on this thread.
 350       do {
 351         tag = t->GenerateRandomTag(/*num_bits=*/8);
 352       } while (
 353           UNLIKELY((tag < kShadowAlignment || tag == pointer_tag) && tag != 0));
 354     } else {
 355       static_assert(kFallbackFreeTag >= kShadowAlignment,
 356                     "fallback tag must not be a short granule tag.");
 357       tag = kFallbackFreeTag;
 358     }
 359     TagMemoryAligned(reinterpret_cast<uptr>(aligned_ptr), TaggedSize(orig_size),
 360                      tag);
 361   }
 362   if (t) {
 363     allocator.Deallocate(t->allocator_cache(), aligned_ptr);
 364     if (auto *ha = t->heap_allocations())
 365       ha->push({reinterpret_cast<uptr>(tagged_ptr), alloc_thread_id,
 366                 alloc_context_id, free_context_id,
 367                 static_cast<u32>(orig_size)});
 368   } else {
 369     SpinMutexLock l(&fallback_mutex);
 370     AllocatorCache *cache = &fallback_allocator_cache;
 371     allocator.Deallocate(cache, aligned_ptr);
 372   }
 373 }
 374
 375 static void *HwasanReallocate(StackTrace *stack, void *tagged_ptr_old,
 376                               uptr new_size, uptr alignment) {
 377   void *untagged_ptr_old = UntagPtr(tagged_ptr_old);
 378   if (CheckInvalidFree(stack, untagged_ptr_old, tagged_ptr_old))
 379     return nullptr;
 380   void *tagged_ptr_new =
 381       HwasanAllocate(stack, new_size, alignment, false /*zeroise*/);
 382   if (tagged_ptr_old && tagged_ptr_new) {
 383     Metadata *meta =
 384         reinterpret_cast<Metadata *>(allocator.GetMetaData(untagged_ptr_old));
 385     void *untagged_ptr_new = UntagPtr(tagged_ptr_new);
 386     internal_memcpy(untagged_ptr_new, untagged_ptr_old,
 387                     Min(new_size, static_cast<uptr>(meta->GetRequestedSize())));
 388     HwasanDeallocate(stack, tagged_ptr_old);
 389   }
 390   return tagged_ptr_new;
 391 }
 392
 393 static void *HwasanCalloc(StackTrace *stack, uptr nmemb, uptr size) {
 394   if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
 395     if (AllocatorMayReturnNull())
 396       return nullptr;
 397     ReportCallocOverflow(nmemb, size, stack);
 398   }
 399   return HwasanAllocate(stack, nmemb * size, sizeof(u64), true);
 400 }
 401
 402 HwasanChunkView FindHeapChunkByAddress(uptr address) {
 403   if (!allocator.PointerIsMine(reinterpret_cast<void *>(address)))
 404     return HwasanChunkView();
 405   void *block = allocator.GetBlockBegin(reinterpret_cast<void*>(address));
 406   if (!block)
 407     return HwasanChunkView();
 408   Metadata *metadata =
 409       reinterpret_cast<Metadata*>(allocator.GetMetaData(block));
 410   return HwasanChunkView(reinterpret_cast<uptr>(block), metadata);
 411 }
 412
 413 static const void *AllocationBegin(const void *p) {
 414   const void *untagged_ptr = UntagPtr(p);
 415   if (!untagged_ptr)
 416     return nullptr;
 417
 418   const void *beg = allocator.GetBlockBegin(untagged_ptr);
 419   if (!beg)
 420     return nullptr;
 421
 422   Metadata *b = (Metadata *)allocator.GetMetaData(beg);
 423   if (b->GetRequestedSize() == 0)
 424     return nullptr;
 425
 426   tag_t tag = GetTagFromPointer((uptr)p);
 427   return (const void *)AddTagToPointer((uptr)beg, tag);
 428 }
 429
 430 static uptr AllocationSize(const void *p) {
 431   const void *untagged_ptr = UntagPtr(p);
 432   if (!untagged_ptr) return 0;
 433   const void *beg = allocator.GetBlockBegin(untagged_ptr);
 434   if (!beg)
 435     return 0;
 436   Metadata *b = (Metadata *)allocator.GetMetaData(beg);
 437   return b->GetRequestedSize();
 438 }
 439
 440 void *hwasan_malloc(uptr size, StackTrace *stack) {
 441   return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
 442 }
 443
 444 void *hwasan_calloc(uptr nmemb, uptr size, StackTrace *stack) {
 445   return SetErrnoOnNull(HwasanCalloc(stack, nmemb, size));
 446 }
 447
 448 void *hwasan_realloc(void *ptr, uptr size, StackTrace *stack) {
 449   if (!ptr)
 450     return SetErrnoOnNull(HwasanAllocate(stack, size, sizeof(u64), false));
 451   if (size == 0) {
 452     HwasanDeallocate(stack, ptr);
 453     return nullptr;
 454   }
 455   return SetErrnoOnNull(HwasanReallocate(stack, ptr, size, sizeof(u64)));
 456 }
 457
 458 void *hwasan_reallocarray(void *ptr, uptr nmemb, uptr size, StackTrace *stack) {
 459   if (UNLIKELY(CheckForCallocOverflow(size, nmemb))) {
 460     errno = errno_ENOMEM;
 461     if (AllocatorMayReturnNull())
 462       return nullptr;
 463     ReportReallocArrayOverflow(nmemb, size, stack);
 464   }
 465   return hwasan_realloc(ptr, nmemb * size, stack);
 466 }
 467
 468 void *hwasan_valloc(uptr size, StackTrace *stack) {
 469   return SetErrnoOnNull(
 470       HwasanAllocate(stack, size, GetPageSizeCached(), false));
 471 }
 472
 473 void *hwasan_pvalloc(uptr size, StackTrace *stack) {
 474   uptr PageSize = GetPageSizeCached();
 475   if (UNLIKELY(CheckForPvallocOverflow(size, PageSize))) {
 476     errno = errno_ENOMEM;
 477     if (AllocatorMayReturnNull())
 478       return nullptr;
 479     ReportPvallocOverflow(size, stack);
 480   }
 481   // pvalloc(0) should allocate one page.
 482   size = size ? RoundUpTo(size, PageSize) : PageSize;
 483   return SetErrnoOnNull(HwasanAllocate(stack, size, PageSize, false));
 484 }
 485
 486 void *hwasan_aligned_alloc(uptr alignment, uptr size, StackTrace *stack) {
 487   if (UNLIKELY(!CheckAlignedAllocAlignmentAndSize(alignment, size))) {
 488     errno = errno_EINVAL;
 489     if (AllocatorMayReturnNull())
 490       return nullptr;
 491     ReportInvalidAlignedAllocAlignment(size, alignment, stack);
 492   }
 493   return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
 494 }
 495
 496 void *hwasan_memalign(uptr alignment, uptr size, StackTrace *stack) {
 497   if (UNLIKELY(!IsPowerOfTwo(alignment))) {
 498     errno = errno_EINVAL;
 499     if (AllocatorMayReturnNull())
 500       return nullptr;
 501     ReportInvalidAllocationAlignment(alignment, stack);
 502   }
 503   return SetErrnoOnNull(HwasanAllocate(stack, size, alignment, false));
 504 }
 505
 506 int hwasan_posix_memalign(void **memptr, uptr alignment, uptr size,
 507                         StackTrace *stack) {
 508   if (UNLIKELY(!CheckPosixMemalignAlignment(alignment))) {
 509     if (AllocatorMayReturnNull())
 510       return errno_EINVAL;
 511     ReportInvalidPosixMemalignAlignment(alignment, stack);
 512   }
 513   void *ptr = HwasanAllocate(stack, size, alignment, false);
 514   if (UNLIKELY(!ptr))
 515     // OOM error is already taken care of by HwasanAllocate.
 516     return errno_ENOMEM;
 517   CHECK(IsAligned((uptr)ptr, alignment));
 518   *memptr = ptr;
 519   return 0;
 520 }
 521
 522 void hwasan_free(void *ptr, StackTrace *stack) {
 523   return HwasanDeallocate(stack, ptr);
 524 }
 525
 526 }  // namespace __hwasan
 527
 528 // --- Implementation of LSan-specific functions --- {{{1
 529 namespace __lsan {
 530
 531 void LockAllocator() {
 532   __hwasan::HwasanAllocatorLock();
 533 }
 534
 535 void UnlockAllocator() {
 536   __hwasan::HwasanAllocatorUnlock();
 537 }
 538
 539 void GetAllocatorGlobalRange(uptr *begin, uptr *end) {
 540   *begin = (uptr)&__hwasan::allocator;
 541   *end = *begin + sizeof(__hwasan::allocator);
 542 }
 543
 544 uptr PointsIntoChunk(void *p) {
 545   p = UntagPtr(p);
 546   uptr addr = reinterpret_cast<uptr>(p);
 547   uptr chunk =
 548       reinterpret_cast<uptr>(__hwasan::allocator.GetBlockBeginFastLocked(p));
 549   if (!chunk)
 550     return 0;
 551   __hwasan::Metadata *metadata = reinterpret_cast<__hwasan::Metadata *>(
 552       __hwasan::allocator.GetMetaData(reinterpret_cast<void *>(chunk)));
 553   if (!metadata || !metadata->IsAllocated())
 554     return 0;
 555   if (addr < chunk + metadata->GetRequestedSize())
 556     return chunk;
 557   if (IsSpecialCaseOfOperatorNew0(chunk, metadata->GetRequestedSize(), addr))
 558     return chunk;
 559   return 0;
 560 }
 561
 562 uptr GetUserBegin(uptr chunk) {
 563   CHECK_EQ(UntagAddr(chunk), chunk);
 564   void *block = __hwasan::allocator.GetBlockBeginFastLocked(
 565       reinterpret_cast<void *>(chunk));
 566   if (!block)
 567     return 0;
 568   __hwasan::Metadata *metadata = reinterpret_cast<__hwasan::Metadata *>(
 569       __hwasan::allocator.GetMetaData(block));
 570   if (!metadata || !metadata->IsAllocated())
 571     return 0;
 572
 573   return reinterpret_cast<uptr>(block);
 574 }
 575
 576 uptr GetUserAddr(uptr chunk) {
 577   if (!InTaggableRegion(chunk))
 578     return chunk;
 579   tag_t mem_tag = *(tag_t *)__hwasan::MemToShadow(chunk);
 580   return AddTagToPointer(chunk, mem_tag);
 581 }
 582
 583 LsanMetadata::LsanMetadata(uptr chunk) {
 584   CHECK_EQ(UntagAddr(chunk), chunk);
 585   metadata_ =
 586       chunk ? __hwasan::allocator.GetMetaData(reinterpret_cast<void *>(chunk))
 587             : nullptr;
 588 }
 589
 590 bool LsanMetadata::allocated() const {
 591   if (!metadata_)
 592     return false;
 593   __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
 594   return m->IsAllocated();
 595 }
 596
 597 ChunkTag LsanMetadata::tag() const {
 598   __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
 599   return m->GetLsanTag();
 600 }
 601
 602 void LsanMetadata::set_tag(ChunkTag value) {
 603   __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
 604   m->SetLsanTag(value);
 605 }
 606
 607 uptr LsanMetadata::requested_size() const {
 608   __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
 609   return m->GetRequestedSize();
 610 }
 611
 612 u32 LsanMetadata::stack_trace_id() const {
 613   __hwasan::Metadata *m = reinterpret_cast<__hwasan::Metadata *>(metadata_);
 614   return m->GetAllocStackId();
 615 }
 616
 617 void ForEachChunk(ForEachChunkCallback callback, void *arg) {
 618   __hwasan::allocator.ForEachChunk(callback, arg);
 619 }
 620
 621 IgnoreObjectResult IgnoreObject(const void *p) {
 622   p = UntagPtr(p);
 623   uptr addr = reinterpret_cast<uptr>(p);
 624   uptr chunk = reinterpret_cast<uptr>(__hwasan::allocator.GetBlockBegin(p));
 625   if (!chunk)
 626     return kIgnoreObjectInvalid;
 627   __hwasan::Metadata *metadata = reinterpret_cast<__hwasan::Metadata *>(
 628       __hwasan::allocator.GetMetaData(reinterpret_cast<void *>(chunk)));
 629   if (!metadata || !metadata->IsAllocated())
 630     return kIgnoreObjectInvalid;
 631   if (addr >= chunk + metadata->GetRequestedSize())
 632     return kIgnoreObjectInvalid;
 633   if (metadata->GetLsanTag() == kIgnored)
 634     return kIgnoreObjectAlreadyIgnored;
 635
 636   metadata->SetLsanTag(kIgnored);
 637   return kIgnoreObjectSuccess;
 638 }
 639
 640 }  // namespace __lsan
 641
 642 using namespace __hwasan;
 643
 644 void __hwasan_enable_allocator_tagging() {
 645   atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 1);
 646 }
 647
 648 void __hwasan_disable_allocator_tagging() {
 649   atomic_store_relaxed(&hwasan_allocator_tagging_enabled, 0);
 650 }
 651
 652 uptr __sanitizer_get_current_allocated_bytes() {
 653   uptr stats[AllocatorStatCount];
 654   allocator.GetStats(stats);
 655   return stats[AllocatorStatAllocated];
 656 }
 657
 658 uptr __sanitizer_get_heap_size() {
 659   uptr stats[AllocatorStatCount];
 660   allocator.GetStats(stats);
 661   return stats[AllocatorStatMapped];
 662 }
 663
 664 uptr __sanitizer_get_free_bytes() { return 1; }
 665
 666 uptr __sanitizer_get_unmapped_bytes() { return 1; }
 667
 668 uptr __sanitizer_get_estimated_allocated_size(uptr size) { return size; }
 669
 670 int __sanitizer_get_ownership(const void *p) { return AllocationSize(p) != 0; }
 671
 672 const void *__sanitizer_get_allocated_begin(const void *p) {
 673   return AllocationBegin(p);
 674 }
 675
 676 uptr __sanitizer_get_allocated_size(const void *p) { return AllocationSize(p); }
 677
 678 void __sanitizer_purge_allocator() { allocator.ForceReleaseToOS(); }