libsanitizer/hwasan/hwasan_linux.cpp

   1 //===-- hwasan_linux.cpp ----------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 ///
   9 /// \file
  10 /// This file is a part of HWAddressSanitizer and contains Linux-, NetBSD- and
  11 /// FreeBSD-specific code.
  12 ///
  13 //===----------------------------------------------------------------------===//
  14
  15 #include "sanitizer_common/sanitizer_platform.h"
  16 #if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
  17
  18 #  include <dlfcn.h>
  19 #  include <elf.h>
  20 #  include <errno.h>
  21 #  include <link.h>
  22 #  include <pthread.h>
  23 #  include <signal.h>
  24 #  include <stdio.h>
  25 #  include <stdlib.h>
  26 #  include <sys/prctl.h>
  27 #  include <sys/resource.h>
  28 #  include <sys/time.h>
  29 #  include <unistd.h>
  30 #  include <unwind.h>
  31
  32 #  include "hwasan.h"
  33 #  include "hwasan_dynamic_shadow.h"
  34 #  include "hwasan_interface_internal.h"
  35 #  include "hwasan_mapping.h"
  36 #  include "hwasan_report.h"
  37 #  include "hwasan_thread.h"
  38 #  include "hwasan_thread_list.h"
  39 #  include "sanitizer_common/sanitizer_common.h"
  40 #  include "sanitizer_common/sanitizer_procmaps.h"
  41 #  include "sanitizer_common/sanitizer_stackdepot.h"
  42
  43 // Configurations of HWASAN_WITH_INTERCEPTORS and SANITIZER_ANDROID.
  44 //
  45 // HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=OFF
  46 //   Not currently tested.
  47 // HWASAN_WITH_INTERCEPTORS=OFF, SANITIZER_ANDROID=ON
  48 //   Integration tests downstream exist.
  49 // HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=OFF
  50 //    Tested with check-hwasan on x86_64-linux.
  51 // HWASAN_WITH_INTERCEPTORS=ON, SANITIZER_ANDROID=ON
  52 //    Tested with check-hwasan on aarch64-linux-android.
  53 #  if !SANITIZER_ANDROID
  54 SANITIZER_INTERFACE_ATTRIBUTE
  55 THREADLOCAL uptr __hwasan_tls;
  56 #  endif
  57
  58 namespace __hwasan {
  59
  60 // With the zero shadow base we can not actually map pages starting from 0.
  61 // This constant is somewhat arbitrary.
  62 constexpr uptr kZeroBaseShadowStart = 0;
  63 constexpr uptr kZeroBaseMaxShadowStart = 1 << 18;
  64
  65 static void ProtectGap(uptr addr, uptr size) {
  66   __sanitizer::ProtectGap(addr, size, kZeroBaseShadowStart,
  67                           kZeroBaseMaxShadowStart);
  68 }
  69
  70 uptr kLowMemStart;
  71 uptr kLowMemEnd;
  72 uptr kHighMemStart;
  73 uptr kHighMemEnd;
  74
  75 static void PrintRange(uptr start, uptr end, const char *name) {
  76   Printf("|| [%p, %p] || %.*s ||\n", (void *)start, (void *)end, 10, name);
  77 }
  78
  79 static void PrintAddressSpaceLayout() {
  80   PrintRange(kHighMemStart, kHighMemEnd, "HighMem");
  81   if (kHighShadowEnd + 1 < kHighMemStart)
  82     PrintRange(kHighShadowEnd + 1, kHighMemStart - 1, "ShadowGap");
  83   else
  84     CHECK_EQ(kHighShadowEnd + 1, kHighMemStart);
  85   PrintRange(kHighShadowStart, kHighShadowEnd, "HighShadow");
  86   if (kLowShadowEnd + 1 < kHighShadowStart)
  87     PrintRange(kLowShadowEnd + 1, kHighShadowStart - 1, "ShadowGap");
  88   else
  89     CHECK_EQ(kLowMemEnd + 1, kHighShadowStart);
  90   PrintRange(kLowShadowStart, kLowShadowEnd, "LowShadow");
  91   if (kLowMemEnd + 1 < kLowShadowStart)
  92     PrintRange(kLowMemEnd + 1, kLowShadowStart - 1, "ShadowGap");
  93   else
  94     CHECK_EQ(kLowMemEnd + 1, kLowShadowStart);
  95   PrintRange(kLowMemStart, kLowMemEnd, "LowMem");
  96   CHECK_EQ(0, kLowMemStart);
  97 }
  98
  99 static uptr GetHighMemEnd() {
 100   // HighMem covers the upper part of the address space.
 101   uptr max_address = GetMaxUserVirtualAddress();
 102   // Adjust max address to make sure that kHighMemEnd and kHighMemStart are
 103   // properly aligned:
 104   max_address |= (GetMmapGranularity() << kShadowScale) - 1;
 105   return max_address;
 106 }
 107
 108 static void InitializeShadowBaseAddress(uptr shadow_size_bytes) {
 109   __hwasan_shadow_memory_dynamic_address =
 110       FindDynamicShadowStart(shadow_size_bytes);
 111 }
 112
 113 static void MaybeDieIfNoTaggingAbi(const char *message) {
 114   if (!flags()->fail_without_syscall_abi)
 115     return;
 116   Printf("FATAL: %s\n", message);
 117   Die();
 118 }
 119
 120 #  define PR_SET_TAGGED_ADDR_CTRL 55
 121 #  define PR_GET_TAGGED_ADDR_CTRL 56
 122 #  define PR_TAGGED_ADDR_ENABLE (1UL << 0)
 123 #  define ARCH_GET_UNTAG_MASK 0x4001
 124 #  define ARCH_ENABLE_TAGGED_ADDR 0x4002
 125 #  define ARCH_GET_MAX_TAG_BITS 0x4003
 126
 127 static bool CanUseTaggingAbi() {
 128 #  if defined(__x86_64__)
 129   unsigned long num_bits = 0;
 130   // Check for x86 LAM support. This API is based on a currently unsubmitted
 131   // patch to the Linux kernel (as of August 2022) and is thus subject to
 132   // change. The patch is here:
 133   // https://lore.kernel.org/all/20220815041803.17954-1-kirill.shutemov@linux.intel.com/
 134   //
 135   // arch_prctl(ARCH_GET_MAX_TAG_BITS, &bits) returns the maximum number of tag
 136   // bits the user can request, or zero if LAM is not supported by the hardware.
 137   if (internal_iserror(internal_arch_prctl(ARCH_GET_MAX_TAG_BITS,
 138                                            reinterpret_cast<uptr>(&num_bits))))
 139     return false;
 140   // The platform must provide enough bits for HWASan tags.
 141   if (num_bits < kTagBits)
 142     return false;
 143   return true;
 144 #  else
 145   // Check for ARM TBI support.
 146   return !internal_iserror(internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0));
 147 #  endif // __x86_64__
 148 }
 149
 150 static bool EnableTaggingAbi() {
 151 #  if defined(__x86_64__)
 152   // Enable x86 LAM tagging for the process.
 153   //
 154   // arch_prctl(ARCH_ENABLE_TAGGED_ADDR, bits) enables tagging if the number of
 155   // tag bits requested by the user does not exceed that provided by the system.
 156   // arch_prctl(ARCH_GET_UNTAG_MASK, &mask) returns the mask of significant
 157   // address bits. It is ~0ULL if either LAM is disabled for the process or LAM
 158   // is not supported by the hardware.
 159   if (internal_iserror(internal_arch_prctl(ARCH_ENABLE_TAGGED_ADDR, kTagBits)))
 160     return false;
 161   unsigned long mask = 0;
 162   // Make sure the tag bits are where we expect them to be.
 163   if (internal_iserror(internal_arch_prctl(ARCH_GET_UNTAG_MASK,
 164                                            reinterpret_cast<uptr>(&mask))))
 165     return false;
 166   // @mask has ones for non-tag bits, whereas @kAddressTagMask has ones for tag
 167   // bits. Therefore these masks must not overlap.
 168   if (mask & kAddressTagMask)
 169     return false;
 170   return true;
 171 #  else
 172   // Enable ARM TBI tagging for the process. If for some reason tagging is not
 173   // supported, prctl(PR_SET_TAGGED_ADDR_CTRL, PR_TAGGED_ADDR_ENABLE) returns
 174   // -EINVAL.
 175   if (internal_iserror(internal_prctl(PR_SET_TAGGED_ADDR_CTRL,
 176                                       PR_TAGGED_ADDR_ENABLE, 0, 0, 0)))
 177     return false;
 178   // Ensure that TBI is enabled.
 179   if (internal_prctl(PR_GET_TAGGED_ADDR_CTRL, 0, 0, 0, 0) !=
 180       PR_TAGGED_ADDR_ENABLE)
 181     return false;
 182   return true;
 183 #  endif // __x86_64__
 184 }
 185
 186 void InitializeOsSupport() {
 187   // Check we're running on a kernel that can use the tagged address ABI.
 188   bool has_abi = CanUseTaggingAbi();
 189
 190   if (!has_abi) {
 191 #  if SANITIZER_ANDROID || defined(HWASAN_ALIASING_MODE)
 192     // Some older Android kernels have the tagged pointer ABI on
 193     // unconditionally, and hence don't have the tagged-addr prctl while still
 194     // allow the ABI.
 195     // If targeting Android and the prctl is not around we assume this is the
 196     // case.
 197     return;
 198 #  else
 199     MaybeDieIfNoTaggingAbi(
 200         "HWAddressSanitizer requires a kernel with tagged address ABI.");
 201 #  endif
 202   }
 203
 204   if (EnableTaggingAbi())
 205     return;
 206
 207 #  if SANITIZER_ANDROID
 208   MaybeDieIfNoTaggingAbi(
 209       "HWAddressSanitizer failed to enable tagged address syscall ABI.\n"
 210       "Check the `sysctl abi.tagged_addr_disabled` configuration.");
 211 #  else
 212   MaybeDieIfNoTaggingAbi(
 213       "HWAddressSanitizer failed to enable tagged address syscall ABI.\n");
 214 #  endif
 215 }
 216
 217 bool InitShadow() {
 218   // Define the entire memory range.
 219   kHighMemEnd = GetHighMemEnd();
 220
 221   // Determine shadow memory base offset.
 222   InitializeShadowBaseAddress(MemToShadowSize(kHighMemEnd));
 223
 224   // Place the low memory first.
 225   kLowMemEnd = __hwasan_shadow_memory_dynamic_address - 1;
 226   kLowMemStart = 0;
 227
 228   // Define the low shadow based on the already placed low memory.
 229   kLowShadowEnd = MemToShadow(kLowMemEnd);
 230   kLowShadowStart = __hwasan_shadow_memory_dynamic_address;
 231
 232   // High shadow takes whatever memory is left up there (making sure it is not
 233   // interfering with low memory in the fixed case).
 234   kHighShadowEnd = MemToShadow(kHighMemEnd);
 235   kHighShadowStart = Max(kLowMemEnd, MemToShadow(kHighShadowEnd)) + 1;
 236
 237   // High memory starts where allocated shadow allows.
 238   kHighMemStart = ShadowToMem(kHighShadowStart);
 239
 240   // Check the sanity of the defined memory ranges (there might be gaps).
 241   CHECK_EQ(kHighMemStart % GetMmapGranularity(), 0);
 242   CHECK_GT(kHighMemStart, kHighShadowEnd);
 243   CHECK_GT(kHighShadowEnd, kHighShadowStart);
 244   CHECK_GT(kHighShadowStart, kLowMemEnd);
 245   CHECK_GT(kLowMemEnd, kLowMemStart);
 246   CHECK_GT(kLowShadowEnd, kLowShadowStart);
 247   CHECK_GT(kLowShadowStart, kLowMemEnd);
 248
 249   if (Verbosity())
 250     PrintAddressSpaceLayout();
 251
 252   // Reserve shadow memory.
 253   ReserveShadowMemoryRange(kLowShadowStart, kLowShadowEnd, "low shadow");
 254   ReserveShadowMemoryRange(kHighShadowStart, kHighShadowEnd, "high shadow");
 255
 256   // Protect all the gaps.
 257   ProtectGap(0, Min(kLowMemStart, kLowShadowStart));
 258   if (kLowMemEnd + 1 < kLowShadowStart)
 259     ProtectGap(kLowMemEnd + 1, kLowShadowStart - kLowMemEnd - 1);
 260   if (kLowShadowEnd + 1 < kHighShadowStart)
 261     ProtectGap(kLowShadowEnd + 1, kHighShadowStart - kLowShadowEnd - 1);
 262   if (kHighShadowEnd + 1 < kHighMemStart)
 263     ProtectGap(kHighShadowEnd + 1, kHighMemStart - kHighShadowEnd - 1);
 264
 265   return true;
 266 }
 267
 268 void InitThreads() {
 269   CHECK(__hwasan_shadow_memory_dynamic_address);
 270   uptr guard_page_size = GetMmapGranularity();
 271   uptr thread_space_start =
 272       __hwasan_shadow_memory_dynamic_address - (1ULL << kShadowBaseAlignment);
 273   uptr thread_space_end =
 274       __hwasan_shadow_memory_dynamic_address - guard_page_size;
 275   ReserveShadowMemoryRange(thread_space_start, thread_space_end - 1,
 276                            "hwasan threads", /*madvise_shadow*/ false);
 277   ProtectGap(thread_space_end,
 278              __hwasan_shadow_memory_dynamic_address - thread_space_end);
 279   InitThreadList(thread_space_start, thread_space_end - thread_space_start);
 280   hwasanThreadList().CreateCurrentThread();
 281 }
 282
 283 bool MemIsApp(uptr p) {
 284 // Memory outside the alias range has non-zero tags.
 285 #  if !defined(HWASAN_ALIASING_MODE)
 286   CHECK_EQ(GetTagFromPointer(p), 0);
 287 #  endif
 288
 289   return (p >= kHighMemStart && p <= kHighMemEnd) ||
 290          (p >= kLowMemStart && p <= kLowMemEnd);
 291 }
 292
 293 void InstallAtExitHandler() { atexit(HwasanAtExit); }
 294
 295 // ---------------------- TSD ---------------- {{{1
 296
 297 extern "C" void __hwasan_thread_enter() {
 298   hwasanThreadList().CreateCurrentThread()->EnsureRandomStateInited();
 299 }
 300
 301 extern "C" void __hwasan_thread_exit() {
 302   Thread *t = GetCurrentThread();
 303   // Make sure that signal handler can not see a stale current thread pointer.
 304   atomic_signal_fence(memory_order_seq_cst);
 305   if (t) {
 306     // Block async signals on the thread as the handler can be instrumented.
 307     // After this point instrumented code can't access essential data from TLS
 308     // and will crash.
 309     // Bionic already calls __hwasan_thread_exit with blocked signals.
 310     if (SANITIZER_GLIBC)
 311       BlockSignals();
 312     hwasanThreadList().ReleaseThread(t);
 313   }
 314 }
 315
 316 #  if HWASAN_WITH_INTERCEPTORS
 317 static pthread_key_t tsd_key;
 318 static bool tsd_key_inited = false;
 319
 320 void HwasanTSDThreadInit() {
 321   if (tsd_key_inited)
 322     CHECK_EQ(0, pthread_setspecific(tsd_key,
 323                                     (void *)GetPthreadDestructorIterations()));
 324 }
 325
 326 void HwasanTSDDtor(void *tsd) {
 327   uptr iterations = (uptr)tsd;
 328   if (iterations > 1) {
 329     CHECK_EQ(0, pthread_setspecific(tsd_key, (void *)(iterations - 1)));
 330     return;
 331   }
 332   __hwasan_thread_exit();
 333 }
 334
 335 void HwasanTSDInit() {
 336   CHECK(!tsd_key_inited);
 337   tsd_key_inited = true;
 338   CHECK_EQ(0, pthread_key_create(&tsd_key, HwasanTSDDtor));
 339 }
 340 #  else
 341 void HwasanTSDInit() {}
 342 void HwasanTSDThreadInit() {}
 343 #  endif
 344
 345 #  if SANITIZER_ANDROID
 346 uptr *GetCurrentThreadLongPtr() { return (uptr *)get_android_tls_ptr(); }
 347 #  else
 348 uptr *GetCurrentThreadLongPtr() { return &__hwasan_tls; }
 349 #  endif
 350
 351 #  if SANITIZER_ANDROID
 352 void AndroidTestTlsSlot() {
 353   uptr kMagicValue = 0x010203040A0B0C0D;
 354   uptr *tls_ptr = GetCurrentThreadLongPtr();
 355   uptr old_value = *tls_ptr;
 356   *tls_ptr = kMagicValue;
 357   dlerror();
 358   if (*(uptr *)get_android_tls_ptr() != kMagicValue) {
 359     Printf(
 360         "ERROR: Incompatible version of Android: TLS_SLOT_SANITIZER(6) is used "
 361         "for dlerror().\n");
 362     Die();
 363   }
 364   *tls_ptr = old_value;
 365 }
 366 #  else
 367 void AndroidTestTlsSlot() {}
 368 #  endif
 369
 370 static AccessInfo GetAccessInfo(siginfo_t *info, ucontext_t *uc) {
 371   // Access type is passed in a platform dependent way (see below) and encoded
 372   // as 0xXY, where X&1 is 1 for store, 0 for load, and X&2 is 1 if the error is
 373   // recoverable. Valid values of Y are 0 to 4, which are interpreted as
 374   // log2(access_size), and 0xF, which means that access size is passed via
 375   // platform dependent register (see below).
 376 #  if defined(__aarch64__)
 377   // Access type is encoded in BRK immediate as 0x900 + 0xXY. For Y == 0xF,
 378   // access size is stored in X1 register. Access address is always in X0
 379   // register.
 380   uptr pc = (uptr)info->si_addr;
 381   const unsigned code = ((*(u32 *)pc) >> 5) & 0xffff;
 382   if ((code & 0xff00) != 0x900)
 383     return AccessInfo{};  // Not ours.
 384
 385   const bool is_store = code & 0x10;
 386   const bool recover = code & 0x20;
 387   const uptr addr = uc->uc_mcontext.regs[0];
 388   const unsigned size_log = code & 0xf;
 389   if (size_log > 4 && size_log != 0xf)
 390     return AccessInfo{};  // Not ours.
 391   const uptr size = size_log == 0xf ? uc->uc_mcontext.regs[1] : 1U << size_log;
 392
 393 #  elif defined(__x86_64__)
 394   // Access type is encoded in the instruction following INT3 as
 395   // NOP DWORD ptr [EAX + 0x40 + 0xXY]. For Y == 0xF, access size is stored in
 396   // RSI register. Access address is always in RDI register.
 397   uptr pc = (uptr)uc->uc_mcontext.gregs[REG_RIP];
 398   uint8_t *nop = (uint8_t *)pc;
 399   if (*nop != 0x0f || *(nop + 1) != 0x1f || *(nop + 2) != 0x40 ||
 400       *(nop + 3) < 0x40)
 401     return AccessInfo{};  // Not ours.
 402   const unsigned code = *(nop + 3);
 403
 404   const bool is_store = code & 0x10;
 405   const bool recover = code & 0x20;
 406   const uptr addr = uc->uc_mcontext.gregs[REG_RDI];
 407   const unsigned size_log = code & 0xf;
 408   if (size_log > 4 && size_log != 0xf)
 409     return AccessInfo{};  // Not ours.
 410   const uptr size =
 411       size_log == 0xf ? uc->uc_mcontext.gregs[REG_RSI] : 1U << size_log;
 412
 413 #  elif SANITIZER_RISCV64
 414   // Access type is encoded in the instruction following EBREAK as
 415   // ADDI x0, x0, [0x40 + 0xXY]. For Y == 0xF, access size is stored in
 416   // X11 register. Access address is always in X10 register.
 417   uptr pc = (uptr)uc->uc_mcontext.__gregs[REG_PC];
 418   uint8_t byte1 = *((u8 *)(pc + 0));
 419   uint8_t byte2 = *((u8 *)(pc + 1));
 420   uint8_t byte3 = *((u8 *)(pc + 2));
 421   uint8_t byte4 = *((u8 *)(pc + 3));
 422   uint32_t ebreak = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24));
 423   bool isFaultShort = false;
 424   bool isEbreak = (ebreak == 0x100073);
 425   bool isShortEbreak = false;
 426 #    if defined(__riscv_compressed)
 427   isFaultShort = ((ebreak & 0x3) != 0x3);
 428   isShortEbreak = ((ebreak & 0xffff) == 0x9002);
 429 #    endif
 430   // faulted insn is not ebreak, not our case
 431   if (!(isEbreak || isShortEbreak))
 432     return AccessInfo{};
 433   // advance pc to point after ebreak and reconstruct addi instruction
 434   pc += isFaultShort ? 2 : 4;
 435   byte1 = *((u8 *)(pc + 0));
 436   byte2 = *((u8 *)(pc + 1));
 437   byte3 = *((u8 *)(pc + 2));
 438   byte4 = *((u8 *)(pc + 3));
 439   // reconstruct instruction
 440   uint32_t instr = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24));
 441   // check if this is really 32 bit instruction
 442   // code is encoded in top 12 bits, since instruction is supposed to be with
 443   // imm
 444   const unsigned code = (instr >> 20) & 0xffff;
 445   const uptr addr = uc->uc_mcontext.__gregs[10];
 446   const bool is_store = code & 0x10;
 447   const bool recover = code & 0x20;
 448   const unsigned size_log = code & 0xf;
 449   if (size_log > 4 && size_log != 0xf)
 450     return AccessInfo{};  // Not our case
 451   const uptr size =
 452       size_log == 0xf ? uc->uc_mcontext.__gregs[11] : 1U << size_log;
 453
 454 #  else
 455 #    error Unsupported architecture
 456 #  endif
 457
 458   return AccessInfo{addr, size, is_store, !is_store, recover};
 459 }
 460
 461 static bool HwasanOnSIGTRAP(int signo, siginfo_t *info, ucontext_t *uc) {
 462   AccessInfo ai = GetAccessInfo(info, uc);
 463   if (!ai.is_store && !ai.is_load)
 464     return false;
 465
 466   SignalContext sig{info, uc};
 467   HandleTagMismatch(ai, StackTrace::GetNextInstructionPc(sig.pc), sig.bp, uc);
 468
 469 #  if defined(__aarch64__)
 470   uc->uc_mcontext.pc += 4;
 471 #  elif defined(__x86_64__)
 472 #  elif SANITIZER_RISCV64
 473   // pc points to EBREAK which is 2 bytes long
 474   uint8_t *exception_source = (uint8_t *)(uc->uc_mcontext.__gregs[REG_PC]);
 475   uint8_t byte1 = (uint8_t)(*(exception_source + 0));
 476   uint8_t byte2 = (uint8_t)(*(exception_source + 1));
 477   uint8_t byte3 = (uint8_t)(*(exception_source + 2));
 478   uint8_t byte4 = (uint8_t)(*(exception_source + 3));
 479   uint32_t faulted = (byte1 | (byte2 << 8) | (byte3 << 16) | (byte4 << 24));
 480   bool isFaultShort = false;
 481 #    if defined(__riscv_compressed)
 482   isFaultShort = ((faulted & 0x3) != 0x3);
 483 #    endif
 484   uc->uc_mcontext.__gregs[REG_PC] += isFaultShort ? 2 : 4;
 485 #  else
 486 #    error Unsupported architecture
 487 #  endif
 488   return true;
 489 }
 490
 491 static void OnStackUnwind(const SignalContext &sig, const void *,
 492                           BufferedStackTrace *stack) {
 493   stack->Unwind(StackTrace::GetNextInstructionPc(sig.pc), sig.bp, sig.context,
 494                 common_flags()->fast_unwind_on_fatal);
 495 }
 496
 497 void HwasanOnDeadlySignal(int signo, void *info, void *context) {
 498   // Probably a tag mismatch.
 499   if (signo == SIGTRAP)
 500     if (HwasanOnSIGTRAP(signo, (siginfo_t *)info, (ucontext_t *)context))
 501       return;
 502
 503   HandleDeadlySignal(info, context, GetTid(), &OnStackUnwind, nullptr);
 504 }
 505
 506 void Thread::InitStackAndTls(const InitState *) {
 507   uptr tls_size;
 508   uptr stack_size;
 509   GetThreadStackAndTls(IsMainThread(), &stack_bottom_, &stack_size, &tls_begin_,
 510                        &tls_size);
 511   stack_top_ = stack_bottom_ + stack_size;
 512   tls_end_ = tls_begin_ + tls_size;
 513 }
 514
 515 uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
 516   CHECK(IsAligned(p, kShadowAlignment));
 517   CHECK(IsAligned(size, kShadowAlignment));
 518   uptr shadow_start = MemToShadow(p);
 519   uptr shadow_size = MemToShadowSize(size);
 520
 521   uptr page_size = GetPageSizeCached();
 522   uptr page_start = RoundUpTo(shadow_start, page_size);
 523   uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
 524   uptr threshold = common_flags()->clear_shadow_mmap_threshold;
 525   if (SANITIZER_LINUX &&
 526       UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
 527     internal_memset((void *)shadow_start, tag, page_start - shadow_start);
 528     internal_memset((void *)page_end, tag,
 529                     shadow_start + shadow_size - page_end);
 530     // For an anonymous private mapping MADV_DONTNEED will return a zero page on
 531     // Linux.
 532     ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
 533   } else {
 534     internal_memset((void *)shadow_start, tag, shadow_size);
 535   }
 536   return AddTagToPointer(p, tag);
 537 }
 538
 539 void HwasanInstallAtForkHandler() {
 540   auto before = []() {
 541     HwasanAllocatorLock();
 542     StackDepotLockAll();
 543   };
 544   auto after = []() {
 545     StackDepotUnlockAll();
 546     HwasanAllocatorUnlock();
 547   };
 548   pthread_atfork(before, after, after);
 549 }
 550
 551 void InstallAtExitCheckLeaks() {
 552   if (CAN_SANITIZE_LEAKS) {
 553     if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) {
 554       if (flags()->halt_on_error)
 555         Atexit(__lsan::DoLeakCheck);
 556       else
 557         Atexit(__lsan::DoRecoverableLeakCheckVoid);
 558     }
 559   }
 560 }
 561
 562 }  // namespace __hwasan
 563
 564 #endif  // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD