1 //===-- hwasan.cpp --------------------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is a part of HWAddressSanitizer.
11 // HWAddressSanitizer runtime.
12 //===----------------------------------------------------------------------===//
16 #include "hwasan_checks.h"
17 #include "hwasan_dynamic_shadow.h"
18 #include "hwasan_globals.h"
19 #include "hwasan_mapping.h"
20 #include "hwasan_poisoning.h"
21 #include "hwasan_report.h"
22 #include "hwasan_thread.h"
23 #include "hwasan_thread_list.h"
24 #include "sanitizer_common/sanitizer_atomic.h"
25 #include "sanitizer_common/sanitizer_common.h"
26 #include "sanitizer_common/sanitizer_flag_parser.h"
27 #include "sanitizer_common/sanitizer_flags.h"
28 #include "sanitizer_common/sanitizer_interface_internal.h"
29 #include "sanitizer_common/sanitizer_libc.h"
30 #include "sanitizer_common/sanitizer_procmaps.h"
31 #include "sanitizer_common/sanitizer_stackdepot.h"
32 #include "sanitizer_common/sanitizer_stacktrace.h"
33 #include "sanitizer_common/sanitizer_symbolizer.h"
34 #include "ubsan/ubsan_flags.h"
35 #include "ubsan/ubsan_init.h"
37 // ACHTUNG! No system header includes in this file.
39 using namespace __sanitizer
;
43 static Flags hwasan_flags
;
49 int hwasan_inited
= 0;
50 int hwasan_instrumentation_inited
= 0;
51 bool hwasan_init_is_running
;
53 int hwasan_report_count
= 0;
57 uptr kHighShadowStart
;
60 void Flags::SetDefaults() {
61 #define HWASAN_FLAG(Type, Name, DefaultValue, Description) Name = DefaultValue;
62 #include "hwasan_flags.inc"
66 static void RegisterHwasanFlags(FlagParser
*parser
, Flags
*f
) {
67 #define HWASAN_FLAG(Type, Name, DefaultValue, Description) \
68 RegisterFlag(parser, #Name, Description, &f->Name);
69 #include "hwasan_flags.inc"
73 static void InitializeFlags() {
74 SetCommonFlagsDefaults();
77 cf
.CopyFrom(*common_flags());
78 cf
.external_symbolizer_path
= GetEnv("HWASAN_SYMBOLIZER_PATH");
79 cf
.malloc_context_size
= 20;
80 cf
.handle_ioctl
= true;
81 // FIXME: test and enable.
82 cf
.check_printf
= false;
83 cf
.intercept_tls_get_addr
= true;
85 // 8 shadow pages ~512kB, small enough to cover common stack sizes.
86 cf
.clear_shadow_mmap_threshold
= 4096 * (SANITIZER_ANDROID
? 2 : 8);
87 // Sigtrap is used in error reporting.
88 cf
.handle_sigtrap
= kHandleSignalExclusive
;
91 // Let platform handle other signals. It is better at reporting them then we
93 cf
.handle_segv
= kHandleSignalNo
;
94 cf
.handle_sigbus
= kHandleSignalNo
;
95 cf
.handle_abort
= kHandleSignalNo
;
96 cf
.handle_sigill
= kHandleSignalNo
;
97 cf
.handle_sigfpe
= kHandleSignalNo
;
99 OverrideCommonFlags(cf
);
106 RegisterHwasanFlags(&parser
, f
);
107 RegisterCommonFlags(&parser
);
109 #if HWASAN_CONTAINS_UBSAN
110 __ubsan::Flags
*uf
= __ubsan::flags();
113 FlagParser ubsan_parser
;
114 __ubsan::RegisterUbsanFlags(&ubsan_parser
, uf
);
115 RegisterCommonFlags(&ubsan_parser
);
118 // Override from user-specified string.
119 if (__hwasan_default_options
)
120 parser
.ParseString(__hwasan_default_options());
121 #if HWASAN_CONTAINS_UBSAN
122 const char *ubsan_default_options
= __ubsan_default_options();
123 ubsan_parser
.ParseString(ubsan_default_options
);
126 parser
.ParseStringFromEnv("HWASAN_OPTIONS");
127 #if HWASAN_CONTAINS_UBSAN
128 ubsan_parser
.ParseStringFromEnv("UBSAN_OPTIONS");
131 InitializeCommonFlags();
133 if (Verbosity()) ReportUnrecognizedFlags();
135 if (common_flags()->help
) parser
.PrintFlagDescriptions();
138 static void CheckUnwind() {
139 GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
143 static void HwasanFormatMemoryUsage(InternalScopedString
&s
) {
144 HwasanThreadList
&thread_list
= hwasanThreadList();
145 auto thread_stats
= thread_list
.GetThreadStats();
146 auto sds
= StackDepotGetStats();
147 AllocatorStatCounters asc
;
148 GetAllocatorStats(asc
);
150 "HWASAN pid: %d rss: %zd threads: %zd stacks: %zd"
151 " thr_aux: %zd stack_depot: %zd uniq_stacks: %zd"
153 internal_getpid(), GetRSS(), thread_stats
.n_live_threads
,
154 thread_stats
.total_stack_size
,
155 thread_stats
.n_live_threads
* thread_list
.MemoryUsedPerThread(),
156 sds
.allocated
, sds
.n_uniq_ids
, asc
[AllocatorStatMapped
]);
159 #if SANITIZER_ANDROID
160 static constexpr uptr kMemoryUsageBufferSize
= 4096;
162 static char *memory_usage_buffer
= nullptr;
164 static void InitMemoryUsage() {
165 memory_usage_buffer
=
166 (char *)MmapOrDie(kMemoryUsageBufferSize
, "memory usage string");
167 CHECK(memory_usage_buffer
);
168 memory_usage_buffer
[0] = '\0';
169 DecorateMapping((uptr
)memory_usage_buffer
, kMemoryUsageBufferSize
,
170 memory_usage_buffer
);
173 void UpdateMemoryUsage() {
174 if (!flags()->export_memory_stats
)
176 if (!memory_usage_buffer
)
178 InternalScopedString s
;
179 HwasanFormatMemoryUsage(s
);
180 internal_strncpy(memory_usage_buffer
, s
.data(), kMemoryUsageBufferSize
- 1);
181 memory_usage_buffer
[kMemoryUsageBufferSize
- 1] = '\0';
184 void UpdateMemoryUsage() {}
187 void HwasanAtExit() {
188 if (common_flags()->print_module_map
)
190 if (flags()->print_stats
&& (flags()->atexit
|| hwasan_report_count
> 0))
192 if (hwasan_report_count
> 0) {
193 // ReportAtExitStatistics();
194 if (common_flags()->exitcode
)
195 internal__exit(common_flags()->exitcode
);
199 void HandleTagMismatch(AccessInfo ai
, uptr pc
, uptr frame
, void *uc
,
200 uptr
*registers_frame
) {
201 InternalMmapVector
<BufferedStackTrace
> stack_buffer(1);
202 BufferedStackTrace
*stack
= stack_buffer
.data();
204 stack
->Unwind(pc
, frame
, uc
, common_flags()->fast_unwind_on_fatal
);
206 // The second stack frame contains the failure __hwasan_check function, as
207 // we have a stack frame for the registers saved in __hwasan_tag_mismatch that
208 // we wish to ignore. This (currently) only occurs on AArch64, as x64
209 // implementations use SIGTRAP to implement the failure, and thus do not go
210 // through the stack saver.
211 if (registers_frame
&& stack
->trace
&& stack
->size
> 0) {
216 bool fatal
= flags()->halt_on_error
|| !ai
.recover
;
217 ReportTagMismatch(stack
, ai
.addr
, ai
.size
, ai
.is_store
, fatal
,
221 void HwasanTagMismatch(uptr addr
, uptr pc
, uptr frame
, uptr access_info
,
222 uptr
*registers_frame
, size_t outsize
) {
223 __hwasan::AccessInfo ai
;
224 ai
.is_store
= access_info
& 0x10;
225 ai
.is_load
= !ai
.is_store
;
226 ai
.recover
= access_info
& 0x20;
228 if ((access_info
& 0xf) == 0xf)
231 ai
.size
= 1 << (access_info
& 0xf);
233 HandleTagMismatch(ai
, pc
, frame
, nullptr, registers_frame
);
236 Thread
*GetCurrentThread() {
237 uptr
*ThreadLongPtr
= GetCurrentThreadLongPtr();
238 if (UNLIKELY(*ThreadLongPtr
== 0))
240 auto *R
= (StackAllocationsRingBuffer
*)ThreadLongPtr
;
241 return hwasanThreadList().GetThreadByBufferAddress((uptr
)R
->Next());
244 } // namespace __hwasan
246 using namespace __hwasan
;
248 void __sanitizer::BufferedStackTrace::UnwindImpl(
249 uptr pc
, uptr bp
, void *context
, bool request_fast
, u32 max_depth
) {
250 Thread
*t
= GetCurrentThread();
252 // The thread is still being created, or has already been destroyed.
256 Unwind(max_depth
, pc
, bp
, context
, t
->stack_top(), t
->stack_bottom(),
260 static bool InitializeSingleGlobal(const hwasan_global
&global
) {
261 uptr full_granule_size
= RoundDownTo(global
.size(), 16);
262 TagMemoryAligned(global
.addr(), full_granule_size
, global
.tag());
263 if (global
.size() % 16)
264 TagMemoryAligned(global
.addr() + full_granule_size
, 16, global
.size() % 16);
268 static void InitLoadedGlobals() {
270 [](dl_phdr_info
*info
, size_t /* size */, void * /* data */) -> int {
271 for (const hwasan_global
&global
: HwasanGlobalsFor(
272 info
->dlpi_addr
, info
->dlpi_phdr
, info
->dlpi_phnum
))
273 InitializeSingleGlobal(global
);
279 // Prepare to run instrumented code on the main thread.
280 static void InitInstrumentation() {
281 if (hwasan_instrumentation_inited
) return;
283 InitializeOsSupport();
286 Printf("FATAL: HWAddressSanitizer cannot mmap the shadow memory.\n");
293 hwasan_instrumentation_inited
= 1;
298 uptr __hwasan_shadow_memory_dynamic_address
; // Global interface symbol.
300 // This function was used by the old frame descriptor mechanism. We keep it
301 // around to avoid breaking ABI.
302 void __hwasan_init_frames(uptr beg
, uptr end
) {}
304 void __hwasan_init_static() {
306 InitInstrumentation();
308 // In the non-static code path we call dl_iterate_phdr here. But at this point
309 // libc might not have been initialized enough for dl_iterate_phdr to work.
310 // Fortunately, since this is a statically linked executable we can use the
311 // linker-defined symbol __ehdr_start to find the only relevant set of phdrs.
312 extern ElfW(Ehdr
) __ehdr_start
;
313 for (const hwasan_global
&global
: HwasanGlobalsFor(
315 reinterpret_cast<const ElfW(Phdr
) *>(
316 reinterpret_cast<const char *>(&__ehdr_start
) +
317 __ehdr_start
.e_phoff
),
318 __ehdr_start
.e_phnum
))
319 InitializeSingleGlobal(global
);
322 __attribute__((constructor(0))) void __hwasan_init() {
323 CHECK(!hwasan_init_is_running
);
324 if (hwasan_inited
) return;
325 hwasan_init_is_running
= 1;
326 SanitizerToolName
= "HWAddressSanitizer";
333 // Install tool-specific callbacks in sanitizer_common.
334 SetCheckUnwindCallback(CheckUnwind
);
336 __sanitizer_set_report_path(common_flags()->log_path
);
338 AndroidTestTlsSlot();
340 DisableCoreDumperIfNecessary();
342 InitInstrumentation();
343 if constexpr (!SANITIZER_FUCHSIA
) {
344 // Fuchsia's libc provides a hook (__sanitizer_module_loaded) that runs on
345 // the startup path which calls into __hwasan_library_loaded on all
346 // initially loaded modules, so explicitly registering the globals here
351 // Needs to be called here because flags()->random_tags might not have been
352 // initialized when InitInstrumentation() was called.
353 GetCurrentThread()->EnsureRandomStateInited();
355 SetPrintfAndReportCallback(AppendToErrorMessageBuffer
);
356 // This may call libc -> needs initialized shadow.
359 InitializeInterceptors();
360 InstallDeadlySignalHandlers(HwasanOnDeadlySignal
);
361 InstallAtExitHandler(); // Needs __cxa_atexit interceptor.
363 InitializeCoverage(common_flags()->coverage
, common_flags()->coverage_dir
);
366 HwasanTSDThreadInit();
368 HwasanAllocatorInit();
369 HwasanInstallAtForkHandler();
371 #if HWASAN_CONTAINS_UBSAN
372 __ubsan::InitAsPlugin();
375 VPrintf(1, "HWAddressSanitizer init done\n");
377 hwasan_init_is_running
= 0;
381 void __hwasan_library_loaded(ElfW(Addr
) base
, const ElfW(Phdr
) * phdr
,
383 for (const hwasan_global
&global
: HwasanGlobalsFor(base
, phdr
, phnum
))
384 InitializeSingleGlobal(global
);
387 void __hwasan_library_unloaded(ElfW(Addr
) base
, const ElfW(Phdr
) * phdr
,
389 for (; phnum
!= 0; ++phdr
, --phnum
)
390 if (phdr
->p_type
== PT_LOAD
)
391 TagMemory(base
+ phdr
->p_vaddr
, phdr
->p_memsz
, 0);
394 void __hwasan_print_shadow(const void *p
, uptr sz
) {
395 uptr ptr_raw
= UntagAddr(reinterpret_cast<uptr
>(p
));
396 uptr shadow_first
= MemToShadow(ptr_raw
);
397 uptr shadow_last
= MemToShadow(ptr_raw
+ sz
- 1);
398 Printf("HWASan shadow map for %zx .. %zx (pointer tag %x)\n", ptr_raw
,
399 ptr_raw
+ sz
, GetTagFromPointer((uptr
)p
));
400 for (uptr s
= shadow_first
; s
<= shadow_last
; ++s
) {
401 tag_t mem_tag
= *reinterpret_cast<tag_t
*>(s
);
402 uptr granule_addr
= ShadowToMem(s
);
403 if (mem_tag
&& mem_tag
< kShadowAlignment
)
404 Printf(" %zx: %02x(%02x)\n", granule_addr
, mem_tag
,
405 *reinterpret_cast<tag_t
*>(granule_addr
+ kShadowAlignment
- 1));
407 Printf(" %zx: %02x\n", granule_addr
, mem_tag
);
411 sptr
__hwasan_test_shadow(const void *p
, uptr sz
) {
414 tag_t ptr_tag
= GetTagFromPointer((uptr
)p
);
415 uptr ptr_raw
= UntagAddr(reinterpret_cast<uptr
>(p
));
416 uptr shadow_first
= MemToShadow(ptr_raw
);
417 uptr shadow_last
= MemToShadow(ptr_raw
+ sz
- 1);
418 for (uptr s
= shadow_first
; s
<= shadow_last
; ++s
)
419 if (*(tag_t
*)s
!= ptr_tag
) {
420 sptr offset
= ShadowToMem(s
) - ptr_raw
;
421 return offset
< 0 ? 0 : offset
;
426 u16
__sanitizer_unaligned_load16(const uu16
*p
) {
429 u32
__sanitizer_unaligned_load32(const uu32
*p
) {
432 u64
__sanitizer_unaligned_load64(const uu64
*p
) {
435 void __sanitizer_unaligned_store16(uu16
*p
, u16 x
) {
438 void __sanitizer_unaligned_store32(uu32
*p
, u32 x
) {
441 void __sanitizer_unaligned_store64(uu64
*p
, u64 x
) {
445 void __hwasan_loadN(uptr p
, uptr sz
) {
446 CheckAddressSized
<ErrorAction::Abort
, AccessType::Load
>(p
, sz
);
448 void __hwasan_load1(uptr p
) {
449 CheckAddress
<ErrorAction::Abort
, AccessType::Load
, 0>(p
);
451 void __hwasan_load2(uptr p
) {
452 CheckAddress
<ErrorAction::Abort
, AccessType::Load
, 1>(p
);
454 void __hwasan_load4(uptr p
) {
455 CheckAddress
<ErrorAction::Abort
, AccessType::Load
, 2>(p
);
457 void __hwasan_load8(uptr p
) {
458 CheckAddress
<ErrorAction::Abort
, AccessType::Load
, 3>(p
);
460 void __hwasan_load16(uptr p
) {
461 CheckAddress
<ErrorAction::Abort
, AccessType::Load
, 4>(p
);
464 void __hwasan_loadN_noabort(uptr p
, uptr sz
) {
465 CheckAddressSized
<ErrorAction::Recover
, AccessType::Load
>(p
, sz
);
467 void __hwasan_load1_noabort(uptr p
) {
468 CheckAddress
<ErrorAction::Recover
, AccessType::Load
, 0>(p
);
470 void __hwasan_load2_noabort(uptr p
) {
471 CheckAddress
<ErrorAction::Recover
, AccessType::Load
, 1>(p
);
473 void __hwasan_load4_noabort(uptr p
) {
474 CheckAddress
<ErrorAction::Recover
, AccessType::Load
, 2>(p
);
476 void __hwasan_load8_noabort(uptr p
) {
477 CheckAddress
<ErrorAction::Recover
, AccessType::Load
, 3>(p
);
479 void __hwasan_load16_noabort(uptr p
) {
480 CheckAddress
<ErrorAction::Recover
, AccessType::Load
, 4>(p
);
483 void __hwasan_storeN(uptr p
, uptr sz
) {
484 CheckAddressSized
<ErrorAction::Abort
, AccessType::Store
>(p
, sz
);
486 void __hwasan_store1(uptr p
) {
487 CheckAddress
<ErrorAction::Abort
, AccessType::Store
, 0>(p
);
489 void __hwasan_store2(uptr p
) {
490 CheckAddress
<ErrorAction::Abort
, AccessType::Store
, 1>(p
);
492 void __hwasan_store4(uptr p
) {
493 CheckAddress
<ErrorAction::Abort
, AccessType::Store
, 2>(p
);
495 void __hwasan_store8(uptr p
) {
496 CheckAddress
<ErrorAction::Abort
, AccessType::Store
, 3>(p
);
498 void __hwasan_store16(uptr p
) {
499 CheckAddress
<ErrorAction::Abort
, AccessType::Store
, 4>(p
);
502 void __hwasan_storeN_noabort(uptr p
, uptr sz
) {
503 CheckAddressSized
<ErrorAction::Recover
, AccessType::Store
>(p
, sz
);
505 void __hwasan_store1_noabort(uptr p
) {
506 CheckAddress
<ErrorAction::Recover
, AccessType::Store
, 0>(p
);
508 void __hwasan_store2_noabort(uptr p
) {
509 CheckAddress
<ErrorAction::Recover
, AccessType::Store
, 1>(p
);
511 void __hwasan_store4_noabort(uptr p
) {
512 CheckAddress
<ErrorAction::Recover
, AccessType::Store
, 2>(p
);
514 void __hwasan_store8_noabort(uptr p
) {
515 CheckAddress
<ErrorAction::Recover
, AccessType::Store
, 3>(p
);
517 void __hwasan_store16_noabort(uptr p
) {
518 CheckAddress
<ErrorAction::Recover
, AccessType::Store
, 4>(p
);
521 void __hwasan_tag_memory(uptr p
, u8 tag
, uptr sz
) {
522 TagMemoryAligned(p
, sz
, tag
);
525 uptr
__hwasan_tag_pointer(uptr p
, u8 tag
) {
526 return AddTagToPointer(p
, tag
);
529 void __hwasan_handle_longjmp(const void *sp_dst
) {
530 uptr dst
= (uptr
)sp_dst
;
531 // HWASan does not support tagged SP.
532 CHECK(GetTagFromPointer(dst
) == 0);
534 uptr sp
= (uptr
)__builtin_frame_address(0);
535 static const uptr kMaxExpectedCleanupSize
= 64 << 20; // 64M
536 if (dst
< sp
|| dst
- sp
> kMaxExpectedCleanupSize
) {
538 "WARNING: HWASan is ignoring requested __hwasan_handle_longjmp: "
539 "stack top: %p; target %p; distance: %p (%zd)\n"
540 "False positive error reports may follow\n",
541 (void *)sp
, (void *)dst
, dst
- sp
);
544 TagMemory(sp
, dst
- sp
, 0);
547 void __hwasan_handle_vfork(const void *sp_dst
) {
548 uptr sp
= (uptr
)sp_dst
;
549 Thread
*t
= GetCurrentThread();
551 uptr top
= t
->stack_top();
552 uptr bottom
= t
->stack_bottom();
553 if (top
== 0 || bottom
== 0 || sp
< bottom
|| sp
>= top
) {
555 "WARNING: HWASan is ignoring requested __hwasan_handle_vfork: "
556 "stack top: %zx; current %zx; bottom: %zx \n"
557 "False positive error reports may follow\n",
561 TagMemory(bottom
, sp
- bottom
, 0);
564 extern "C" void *__hwasan_extra_spill_area() {
565 Thread
*t
= GetCurrentThread();
566 return &t
->vfork_spill();
569 void __hwasan_print_memory_usage() {
570 InternalScopedString s
;
571 HwasanFormatMemoryUsage(s
);
572 Printf("%s\n", s
.data());
575 static const u8 kFallbackTag
= 0xBB & kTagMask
;
577 u8
__hwasan_generate_tag() {
578 Thread
*t
= GetCurrentThread();
579 if (!t
) return kFallbackTag
;
580 return t
->GenerateRandomTag();
583 void __hwasan_add_frame_record(u64 frame_record_info
) {
584 Thread
*t
= GetCurrentThread();
586 t
->stack_allocations()->push(frame_record_info
);
589 #if !SANITIZER_SUPPORTS_WEAK_HOOKS
591 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
592 const char* __hwasan_default_options() { return ""; }
597 SANITIZER_INTERFACE_ATTRIBUTE
598 void __sanitizer_print_stack_trace() {
599 GET_FATAL_STACK_TRACE_PC_BP(StackTrace::GetCurrentPc(), GET_CURRENT_FRAME());
603 // Entry point for interoperability between __hwasan_tag_mismatch (ASM) and the
604 // rest of the mismatch handling code (C++).
605 void __hwasan_tag_mismatch4(uptr addr
, uptr access_info
, uptr
*registers_frame
,
607 __hwasan::HwasanTagMismatch(addr
, (uptr
)__builtin_return_address(0),
608 (uptr
)__builtin_frame_address(0), access_info
,
609 registers_frame
, outsize
);