1 //===-- hwasan.cpp --------------------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is a part of HWAddressSanitizer.
11 // HWAddressSanitizer runtime.
12 //===----------------------------------------------------------------------===//
16 #include "hwasan_checks.h"
17 #include "hwasan_dynamic_shadow.h"
18 #include "hwasan_globals.h"
19 #include "hwasan_mapping.h"
20 #include "hwasan_poisoning.h"
21 #include "hwasan_report.h"
22 #include "hwasan_thread.h"
23 #include "hwasan_thread_list.h"
24 #include "sanitizer_common/sanitizer_atomic.h"
25 #include "sanitizer_common/sanitizer_common.h"
26 #include "sanitizer_common/sanitizer_flag_parser.h"
27 #include "sanitizer_common/sanitizer_flags.h"
28 #include "sanitizer_common/sanitizer_interface_internal.h"
29 #include "sanitizer_common/sanitizer_libc.h"
30 #include "sanitizer_common/sanitizer_procmaps.h"
31 #include "sanitizer_common/sanitizer_stackdepot.h"
32 #include "sanitizer_common/sanitizer_stacktrace.h"
33 #include "sanitizer_common/sanitizer_symbolizer.h"
34 #include "ubsan/ubsan_flags.h"
35 #include "ubsan/ubsan_init.h"
37 // ACHTUNG! No system header includes in this file.
39 using namespace __sanitizer
;
43 static Flags hwasan_flags
;
49 int hwasan_inited
= 0;
50 int hwasan_instrumentation_inited
= 0;
51 bool hwasan_init_is_running
;
53 int hwasan_report_count
= 0;
57 uptr kHighShadowStart
;
60 void Flags::SetDefaults() {
61 #define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
62 #include "hwasan_flags.inc"
66 static void RegisterHwasanFlags(FlagParser
*parser
, Flags
*f
) {
67 #define HWASAN_FLAG(Type, Name, DefaultValue, Description) \
68 RegisterFlag(parser, #Name, Description, &f->Name);
69 #include "hwasan_flags.inc"
73 static void InitializeFlags() {
74 SetCommonFlagsDefaults();
77 cf
.CopyFrom(*common_flags());
78 cf
.external_symbolizer_path
= GetEnv("HWASAN_SYMBOLIZER_PATH");
79 cf
.malloc_context_size
= 20;
80 cf
.handle_ioctl
= true;
81 // FIXME: test and enable.
82 cf
.check_printf
= false;
83 cf
.intercept_tls_get_addr
= true;
85 // 8 shadow pages ~512kB, small enough to cover common stack sizes.
86 cf
.clear_shadow_mmap_threshold
= 4096 * (SANITIZER_ANDROID
? 2 : 8);
87 // Sigtrap is used in error reporting.
88 cf
.handle_sigtrap
= kHandleSignalExclusive
;
89 // For now only tested on Linux. Other plantforms can be turned on as they
91 cf
.detect_leaks
= cf
.detect_leaks
&& SANITIZER_LINUX
&& !SANITIZER_ANDROID
;
94 // Let platform handle other signals. It is better at reporting them then we
96 cf
.handle_segv
= kHandleSignalNo
;
97 cf
.handle_sigbus
= kHandleSignalNo
;
98 cf
.handle_abort
= kHandleSignalNo
;
99 cf
.handle_sigill
= kHandleSignalNo
;
100 cf
.handle_sigfpe
= kHandleSignalNo
;
102 OverrideCommonFlags(cf
);
109 RegisterHwasanFlags(&parser
, f
);
110 RegisterCommonFlags(&parser
);
112 #if CAN_SANITIZE_LEAKS
113 __lsan::Flags
*lf
= __lsan::flags();
116 FlagParser lsan_parser
;
117 __lsan::RegisterLsanFlags(&lsan_parser
, lf
);
118 RegisterCommonFlags(&lsan_parser
);
121 #if HWASAN_CONTAINS_UBSAN
122 __ubsan::Flags
*uf
= __ubsan::flags();
125 FlagParser ubsan_parser
;
126 __ubsan::RegisterUbsanFlags(&ubsan_parser
, uf
);
127 RegisterCommonFlags(&ubsan_parser
);
130 // Override from user-specified string.
131 if (__hwasan_default_options
)
132 parser
.ParseString(__hwasan_default_options());
133 #if CAN_SANITIZE_LEAKS
134 lsan_parser
.ParseString(__lsan_default_options());
136 #if HWASAN_CONTAINS_UBSAN
137 const char *ubsan_default_options
= __ubsan_default_options();
138 ubsan_parser
.ParseString(ubsan_default_options
);
141 parser
.ParseStringFromEnv("HWASAN_OPTIONS");
142 #if CAN_SANITIZE_LEAKS
143 lsan_parser
.ParseStringFromEnv("LSAN_OPTIONS");
145 #if HWASAN_CONTAINS_UBSAN
146 ubsan_parser
.ParseStringFromEnv("UBSAN_OPTIONS");
149 InitializeCommonFlags();
151 if (Verbosity()) ReportUnrecognizedFlags();
153 if (common_flags()->help
) parser
.PrintFlagDescriptions();
155 if (!CAN_SANITIZE_LEAKS
&& common_flags()->detect_leaks
) {
156 Report("%s: detect_leaks is not supported on this platform.\n",
162 static void CheckUnwind() {
163 GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
167 static void HwasanFormatMemoryUsage(InternalScopedString
&s
) {
168 HwasanThreadList
&thread_list
= hwasanThreadList();
169 auto thread_stats
= thread_list
.GetThreadStats();
170 auto sds
= StackDepotGetStats();
171 AllocatorStatCounters asc
;
172 GetAllocatorStats(asc
);
174 "HWASAN pid: %d rss: %zd threads: %zd stacks: %zd"
175 " thr_aux: %zd stack_depot: %zd uniq_stacks: %zd"
177 internal_getpid(), GetRSS(), thread_stats
.n_live_threads
,
178 thread_stats
.total_stack_size
,
179 thread_stats
.n_live_threads
* thread_list
.MemoryUsedPerThread(),
180 sds
.allocated
, sds
.n_uniq_ids
, asc
[AllocatorStatMapped
]);
183 #if SANITIZER_ANDROID
184 static constexpr uptr kMemoryUsageBufferSize
= 4096;
186 static char *memory_usage_buffer
= nullptr;
188 static void InitMemoryUsage() {
189 memory_usage_buffer
=
190 (char *)MmapOrDie(kMemoryUsageBufferSize
, "memory usage string");
191 CHECK(memory_usage_buffer
);
192 memory_usage_buffer
[0] = '\0';
193 DecorateMapping((uptr
)memory_usage_buffer
, kMemoryUsageBufferSize
,
194 memory_usage_buffer
);
197 void UpdateMemoryUsage() {
198 if (!flags()->export_memory_stats
)
200 if (!memory_usage_buffer
)
202 InternalScopedString s
;
203 HwasanFormatMemoryUsage(s
);
204 internal_strncpy(memory_usage_buffer
, s
.data(), kMemoryUsageBufferSize
- 1);
205 memory_usage_buffer
[kMemoryUsageBufferSize
- 1] = '\0';
208 void UpdateMemoryUsage() {}
211 void HwasanAtExit() {
212 if (common_flags()->print_module_map
)
214 if (flags()->print_stats
&& (flags()->atexit
|| hwasan_report_count
> 0))
216 if (hwasan_report_count
> 0) {
217 // ReportAtExitStatistics();
218 if (common_flags()->exitcode
)
219 internal__exit(common_flags()->exitcode
);
223 void HandleTagMismatch(AccessInfo ai
, uptr pc
, uptr frame
, void *uc
,
224 uptr
*registers_frame
) {
225 InternalMmapVector
<BufferedStackTrace
> stack_buffer(1);
226 BufferedStackTrace
*stack
= stack_buffer
.data();
228 stack
->Unwind(pc
, frame
, uc
, common_flags()->fast_unwind_on_fatal
);
230 // The second stack frame contains the failure __hwasan_check function, as
231 // we have a stack frame for the registers saved in __hwasan_tag_mismatch that
232 // we wish to ignore. This (currently) only occurs on AArch64, as x64
233 // implementations use SIGTRAP to implement the failure, and thus do not go
234 // through the stack saver.
235 if (registers_frame
&& stack
->trace
&& stack
->size
> 0) {
240 bool fatal
= flags()->halt_on_error
|| !ai
.recover
;
241 ReportTagMismatch(stack
, ai
.addr
, ai
.size
, ai
.is_store
, fatal
,
245 void HwasanTagMismatch(uptr addr
, uptr pc
, uptr frame
, uptr access_info
,
246 uptr
*registers_frame
, size_t outsize
) {
247 __hwasan::AccessInfo ai
;
248 ai
.is_store
= access_info
& 0x10;
249 ai
.is_load
= !ai
.is_store
;
250 ai
.recover
= access_info
& 0x20;
252 if ((access_info
& 0xf) == 0xf)
255 ai
.size
= 1 << (access_info
& 0xf);
257 HandleTagMismatch(ai
, pc
, frame
, nullptr, registers_frame
);
260 Thread
*GetCurrentThread() {
261 uptr
*ThreadLongPtr
= GetCurrentThreadLongPtr();
262 if (UNLIKELY(*ThreadLongPtr
== 0))
264 auto *R
= (StackAllocationsRingBuffer
*)ThreadLongPtr
;
265 return hwasanThreadList().GetThreadByBufferAddress((uptr
)R
->Next());
268 } // namespace __hwasan
270 using namespace __hwasan
;
272 void __sanitizer::BufferedStackTrace::UnwindImpl(
273 uptr pc
, uptr bp
, void *context
, bool request_fast
, u32 max_depth
) {
274 Thread
*t
= GetCurrentThread();
276 // The thread is still being created, or has already been destroyed.
280 Unwind(max_depth
, pc
, bp
, context
, t
->stack_top(), t
->stack_bottom(),
284 static bool InitializeSingleGlobal(const hwasan_global
&global
) {
285 uptr full_granule_size
= RoundDownTo(global
.size(), 16);
286 TagMemoryAligned(global
.addr(), full_granule_size
, global
.tag());
287 if (global
.size() % 16)
288 TagMemoryAligned(global
.addr() + full_granule_size
, 16, global
.size() % 16);
292 static void InitLoadedGlobals() {
294 [](dl_phdr_info
*info
, size_t /* size */, void * /* data */) -> int {
295 for (const hwasan_global
&global
: HwasanGlobalsFor(
296 info
->dlpi_addr
, info
->dlpi_phdr
, info
->dlpi_phnum
))
297 InitializeSingleGlobal(global
);
303 // Prepare to run instrumented code on the main thread.
304 static void InitInstrumentation() {
305 if (hwasan_instrumentation_inited
) return;
307 InitializeOsSupport();
310 Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n");
317 hwasan_instrumentation_inited
= 1;
322 uptr __hwasan_shadow_memory_dynamic_address
; // Global interface symbol.
324 // This function was used by the old frame descriptor mechanism. We keep it
325 // around to avoid breaking ABI.
326 void __hwasan_init_frames(uptr beg
, uptr end
) {}
328 void __hwasan_init_static() {
330 InitInstrumentation();
332 // In the non-static code path we call dl_iterate_phdr here. But at this point
333 // libc might not have been initialized enough for dl_iterate_phdr to work.
334 // Fortunately, since this is a statically linked executable we can use the
335 // linker-defined symbol __ehdr_start to find the only relevant set of phdrs.
336 extern ElfW(Ehdr
) __ehdr_start
;
337 for (const hwasan_global
&global
: HwasanGlobalsFor(
339 reinterpret_cast<const ElfW(Phdr
) *>(
340 reinterpret_cast<const char *>(&__ehdr_start
) +
341 __ehdr_start
.e_phoff
),
342 __ehdr_start
.e_phnum
))
343 InitializeSingleGlobal(global
);
346 __attribute__((constructor(0))) void __hwasan_init() {
347 CHECK(!hwasan_init_is_running
);
348 if (hwasan_inited
) return;
349 hwasan_init_is_running
= 1;
350 SanitizerToolName
= "HWAddressSanitizer";
357 // Install tool-specific callbacks in sanitizer_common.
358 SetCheckUnwindCallback(CheckUnwind
);
360 __sanitizer_set_report_path(common_flags()->log_path
);
362 AndroidTestTlsSlot();
364 DisableCoreDumperIfNecessary();
366 InitInstrumentation();
367 if constexpr (!SANITIZER_FUCHSIA
) {
368 // Fuchsia's libc provides a hook (__sanitizer_module_loaded) that runs on
369 // the startup path which calls into __hwasan_library_loaded on all
370 // initially loaded modules, so explicitly registering the globals here
375 // Needs to be called here because flags()->random_tags might not have been
376 // initialized when InitInstrumentation() was called.
377 GetCurrentThread()->EnsureRandomStateInited();
379 SetPrintfAndReportCallback(AppendToErrorMessageBuffer
);
380 // This may call libc -> needs initialized shadow.
383 InitializeInterceptors();
384 InstallDeadlySignalHandlers(HwasanOnDeadlySignal
);
385 InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
387 InitializeCoverage(common_flags()->coverage
, common_flags()->coverage_dir
);
390 HwasanTSDThreadInit();
392 HwasanAllocatorInit();
393 HwasanInstallAtForkHandler();
395 if (CAN_SANITIZE_LEAKS
) {
396 __lsan::InitCommonLsan();
397 InstallAtExitCheckLeaks();
400 #if HWASAN_CONTAINS_UBSAN
401 __ubsan::InitAsPlugin();
404 if (CAN_SANITIZE_LEAKS
&& common_flags()->detect_leaks
) {
405 __lsan::ScopedInterceptorDisabler disabler
;
406 Symbolizer::LateInitialize();
409 VPrintf(1, "HWAddressSanitizer init done\n");
411 hwasan_init_is_running
= 0;
415 void __hwasan_library_loaded(ElfW(Addr
) base
, const ElfW(Phdr
) * phdr
,
417 for (const hwasan_global
&global
: HwasanGlobalsFor(base
, phdr
, phnum
))
418 InitializeSingleGlobal(global
);
421 void __hwasan_library_unloaded(ElfW(Addr
) base
, const ElfW(Phdr
) * phdr
,
423 for (; phnum
!= 0; ++phdr
, --phnum
)
424 if (phdr
->p_type
== PT_LOAD
)
425 TagMemory(base
+ phdr
->p_vaddr
, phdr
->p_memsz
, 0);
428 void __hwasan_print_shadow(const void *p
, uptr sz
) {
429 uptr ptr_raw
= UntagAddr(reinterpret_cast<uptr
>(p
));
430 uptr shadow_first
= MemToShadow(ptr_raw
);
431 uptr shadow_last
= MemToShadow(ptr_raw
+ sz
- 1);
432 Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw
,
433 ptr_raw
+ sz
, GetTagFromPointer((uptr
)p
));
434 for (uptr s
= shadow_first
; s
<= shadow_last
; ++s
) {
435 tag_t mem_tag
= *reinterpret_cast<tag_t
*>(s
);
436 uptr granule_addr
= ShadowToMem(s
);
437 if (mem_tag
&& mem_tag
< kShadowAlignment
)
438 Printf(" %zx: %02x(%02x)\n", granule_addr
, mem_tag
,
439 *reinterpret_cast<tag_t
*>(granule_addr
+ kShadowAlignment
- 1));
441 Printf(" %zx: %02x\n", granule_addr
, mem_tag
);
445 sptr
__hwasan_test_shadow(const void *p
, uptr sz
) {
448 uptr ptr
= reinterpret_cast<uptr
>(p
);
449 tag_t ptr_tag
= GetTagFromPointer(ptr
);
450 uptr ptr_raw
= UntagAddr(ptr
);
451 uptr shadow_first
= MemToShadow(ptr_raw
);
452 uptr shadow_last
= MemToShadow(ptr_raw
+ sz
);
453 for (uptr s
= shadow_first
; s
< shadow_last
; ++s
) {
454 if (UNLIKELY(*(tag_t
*)s
!= ptr_tag
)) {
456 ShortTagSize(*(tag_t
*)s
, AddTagToPointer(ShadowToMem(s
), ptr_tag
));
457 sptr offset
= ShadowToMem(s
) - ptr_raw
+ short_size
;
458 return offset
< 0 ? 0 : offset
;
463 uptr tail_sz
= end
& (kShadowAlignment
- 1);
468 ShortTagSize(*(tag_t
*)shadow_last
, end
& ~(kShadowAlignment
- 1));
469 if (LIKELY(tail_sz
<= short_size
))
472 sptr offset
= sz
- tail_sz
+ short_size
;
473 return offset
< 0 ? 0 : offset
;
476 u16
__sanitizer_unaligned_load16(const uu16
*p
) {
479 u32
__sanitizer_unaligned_load32(const uu32
*p
) {
482 u64
__sanitizer_unaligned_load64(const uu64
*p
) {
485 void __sanitizer_unaligned_store16(uu16
*p
, u16 x
) {
488 void __sanitizer_unaligned_store32(uu32
*p
, u32 x
) {
491 void __sanitizer_unaligned_store64(uu64
*p
, u64 x
) {
495 void __hwasan_loadN(uptr p
, uptr sz
) {
496 CheckAddressSized
<ErrorAction::Abort
, AccessType::Load
>(p
, sz
);
498 void __hwasan_load1(uptr p
) {
499 CheckAddress
<ErrorAction::Abort
, AccessType::Load
, 0>(p
);
501 void __hwasan_load2(uptr p
) {
502 CheckAddress
<ErrorAction::Abort
, AccessType::Load
, 1>(p
);
504 void __hwasan_load4(uptr p
) {
505 CheckAddress
<ErrorAction::Abort
, AccessType::Load
, 2>(p
);
507 void __hwasan_load8(uptr p
) {
508 CheckAddress
<ErrorAction::Abort
, AccessType::Load
, 3>(p
);
510 void __hwasan_load16(uptr p
) {
511 CheckAddress
<ErrorAction::Abort
, AccessType::Load
, 4>(p
);
514 void __hwasan_loadN_noabort(uptr p
, uptr sz
) {
515 CheckAddressSized
<ErrorAction::Recover
, AccessType::Load
>(p
, sz
);
517 void __hwasan_load1_noabort(uptr p
) {
518 CheckAddress
<ErrorAction::Recover
, AccessType::Load
, 0>(p
);
520 void __hwasan_load2_noabort(uptr p
) {
521 CheckAddress
<ErrorAction::Recover
, AccessType::Load
, 1>(p
);
523 void __hwasan_load4_noabort(uptr p
) {
524 CheckAddress
<ErrorAction::Recover
, AccessType::Load
, 2>(p
);
526 void __hwasan_load8_noabort(uptr p
) {
527 CheckAddress
<ErrorAction::Recover
, AccessType::Load
, 3>(p
);
529 void __hwasan_load16_noabort(uptr p
) {
530 CheckAddress
<ErrorAction::Recover
, AccessType::Load
, 4>(p
);
533 void __hwasan_storeN(uptr p
, uptr sz
) {
534 CheckAddressSized
<ErrorAction::Abort
, AccessType::Store
>(p
, sz
);
536 void __hwasan_store1(uptr p
) {
537 CheckAddress
<ErrorAction::Abort
, AccessType::Store
, 0>(p
);
539 void __hwasan_store2(uptr p
) {
540 CheckAddress
<ErrorAction::Abort
, AccessType::Store
, 1>(p
);
542 void __hwasan_store4(uptr p
) {
543 CheckAddress
<ErrorAction::Abort
, AccessType::Store
, 2>(p
);
545 void __hwasan_store8(uptr p
) {
546 CheckAddress
<ErrorAction::Abort
, AccessType::Store
, 3>(p
);
548 void __hwasan_store16(uptr p
) {
549 CheckAddress
<ErrorAction::Abort
, AccessType::Store
, 4>(p
);
552 void __hwasan_storeN_noabort(uptr p
, uptr sz
) {
553 CheckAddressSized
<ErrorAction::Recover
, AccessType::Store
>(p
, sz
);
555 void __hwasan_store1_noabort(uptr p
) {
556 CheckAddress
<ErrorAction::Recover
, AccessType::Store
, 0>(p
);
558 void __hwasan_store2_noabort(uptr p
) {
559 CheckAddress
<ErrorAction::Recover
, AccessType::Store
, 1>(p
);
561 void __hwasan_store4_noabort(uptr p
) {
562 CheckAddress
<ErrorAction::Recover
, AccessType::Store
, 2>(p
);
564 void __hwasan_store8_noabort(uptr p
) {
565 CheckAddress
<ErrorAction::Recover
, AccessType::Store
, 3>(p
);
567 void __hwasan_store16_noabort(uptr p
) {
568 CheckAddress
<ErrorAction::Recover
, AccessType::Store
, 4>(p
);
571 void __hwasan_tag_memory(uptr p
, u8 tag
, uptr sz
) {
572 TagMemoryAligned(UntagAddr(p
), sz
, tag
);
575 uptr
__hwasan_tag_pointer(uptr p
, u8 tag
) {
576 return AddTagToPointer(p
, tag
);
579 void __hwasan_handle_longjmp(const void *sp_dst
) {
580 uptr dst
= (uptr
)sp_dst
;
581 // HWASan does not support tagged SP.
582 CHECK(GetTagFromPointer(dst
) == 0);
584 uptr sp
= (uptr
)__builtin_frame_address(0);
585 static const uptr kMaxExpectedCleanupSize
= 64 << 20; // 64M
586 if (dst
< sp
|| dst
- sp
> kMaxExpectedCleanupSize
) {
588 "WARNING: HWASan is ignoring requested __hwasan_handle_longjmp: "
589 "stack top: %p; target %p; distance: %p (%zd)\n"
590 "False positive error reports may follow\n",
591 (void *)sp
, (void *)dst
, dst
- sp
);
594 TagMemory(sp
, dst
- sp
, 0);
597 void __hwasan_handle_vfork(const void *sp_dst
) {
598 uptr sp
= (uptr
)sp_dst
;
599 Thread
*t
= GetCurrentThread();
601 uptr top
= t
->stack_top();
602 uptr bottom
= t
->stack_bottom();
603 if (top
== 0 || bottom
== 0 || sp
< bottom
|| sp
>= top
) {
605 "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: "
606 "stack top: %zx; current %zx; bottom: %zx \n"
607 "False positive error reports may follow\n",
611 TagMemory(bottom
, sp
- bottom
, 0);
614 extern "C" void *__hwasan_extra_spill_area() {
615 Thread
*t
= GetCurrentThread();
616 return &t
->vfork_spill();
619 void __hwasan_print_memory_usage() {
620 InternalScopedString s
;
621 HwasanFormatMemoryUsage(s
);
622 Printf("%s\n", s
.data());
625 static const u8 kFallbackTag
= 0xBB & kTagMask
;
627 u8
__hwasan_generate_tag() {
628 Thread
*t
= GetCurrentThread();
629 if (!t
) return kFallbackTag
;
630 return t
->GenerateRandomTag();
633 void __hwasan_add_frame_record(u64 frame_record_info
) {
634 Thread
*t
= GetCurrentThread();
636 t
->stack_allocations()->push(frame_record_info
);
639 #if !SANITIZER_SUPPORTS_WEAK_HOOKS
641 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
642 const char* __hwasan_default_options() { return ""; }
647 SANITIZER_INTERFACE_ATTRIBUTE
648 void __sanitizer_print_stack_trace() {
649 GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
653 // Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the
654 // rest of the mismatch handling code (C++).
655 void __hwasan_tag_mismatch4(uptr addr
, uptr access_info
, uptr
*registers_frame
,
657 __hwasan::HwasanTagMismatch(addr
, (uptr
)__builtin_return_address(0),
658 (uptr
)__builtin_frame_address(0), access_info
,
659 registers_frame
, outsize
);