1 //===-- hwasan_thread_list.h ------------------------------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is a part of HWAddressSanitizer.
11 //===----------------------------------------------------------------------===//
13 // HwasanThreadList is a registry for live threads, as well as an allocator for
14 // HwasanThread objects and their stack history ring buffers. There are
15 // constraints on memory layout of the shadow region and CompactRingBuffer that
16 // are part of the ABI contract between compiler-rt and llvm.
18 // * Start of the shadow memory region is aligned to 2**kShadowBaseAlignment.
19 // * All stack ring buffers are located within (2**kShadowBaseAlignment)
20 // sized region below and adjacent to the shadow region.
21 // * Each ring buffer has a size of (2**N)*4096 where N is in [0, 8), and is
22 // aligned to twice its size. The value of N can be different for each buffer.
24 // These constrains guarantee that, given an address A of any element of the
26 // A_next = (A + sizeof(uptr)) & ~((1 << (N + 13)) - 1)
27 // is the address of the next element of that ring buffer (with wrap-around).
28 // And, with K = kShadowBaseAlignment,
29 // S = (A | ((1 << K) - 1)) + 1
30 // (align up to kShadowBaseAlignment) is the start of the shadow region.
32 // These calculations are used in compiler instrumentation to update the ring
33 // buffer and obtain the base address of shadow using only two inputs: address
34 // of the current element of the ring buffer, and N (i.e. size of the ring
35 // buffer). Since the value of N is very limited, we pack both inputs into a
36 // single thread-local word as
37 // (1 << (N + 56)) | A
38 // See the implementation of class CompactRingBuffer, which is what is stored in
39 // said thread-local word.
41 // Note the unusual way of aligning up the address of the shadow:
42 // (A | ((1 << K) - 1)) + 1
43 // It is only correct if A is not already equal to the shadow base address, but
44 // it saves 2 instructions on AArch64.
47 #include "hwasan_allocator.h"
48 #include "hwasan_flags.h"
49 #include "hwasan_thread.h"
51 #include "sanitizer_common/sanitizer_placement_new.h"
55 static uptr
RingBufferSize() {
56 uptr desired_bytes
= flags()->stack_history_size
* sizeof(uptr
);
57 // FIXME: increase the limit to 8 once this bug is fixed:
58 // https://bugs.llvm.org/show_bug.cgi?id=39030
59 for (int shift
= 1; shift
< 7; ++shift
) {
60 uptr size
= 4096 * (1ULL << shift
);
61 if (size
>= desired_bytes
)
64 Printf("stack history size too large: %d\n", flags()->stack_history_size
);
71 uptr total_stack_size
;
74 class HwasanThreadList
{
76 HwasanThreadList(uptr storage
, uptr size
)
77 : free_space_(storage
), free_space_end_(storage
+ size
) {
78 // [storage, storage + size) is used as a vector of
79 // thread_alloc_size_-sized, ring_buffer_size_*2-aligned elements.
80 // Each element contains
81 // * a ring buffer at offset 0,
82 // * a Thread object at offset ring_buffer_size_.
83 ring_buffer_size_
= RingBufferSize();
85 RoundUpTo(ring_buffer_size_
+ sizeof(Thread
), ring_buffer_size_
* 2);
88 Thread
*CreateCurrentThread(const Thread::InitState
*state
= nullptr) {
91 SpinMutexLock
l(&free_list_mutex_
);
92 if (!free_list_
.empty()) {
93 t
= free_list_
.back();
94 free_list_
.pop_back();
98 uptr start
= (uptr
)t
- ring_buffer_size_
;
99 internal_memset((void *)start
, 0, ring_buffer_size_
+ sizeof(Thread
));
104 SpinMutexLock
l(&live_list_mutex_
);
105 live_list_
.push_back(t
);
107 t
->Init((uptr
)t
- ring_buffer_size_
, ring_buffer_size_
, state
);
112 void DontNeedThread(Thread
*t
) {
113 uptr start
= (uptr
)t
- ring_buffer_size_
;
114 ReleaseMemoryPagesToOS(start
, start
+ thread_alloc_size_
);
117 void RemoveThreadFromLiveList(Thread
*t
) {
118 SpinMutexLock
l(&live_list_mutex_
);
119 for (Thread
*&t2
: live_list_
)
121 // To remove t2, copy the last element of the list in t2's position, and
122 // pop_back(). This works even if t2 is itself the last element.
123 t2
= live_list_
.back();
124 live_list_
.pop_back();
127 CHECK(0 && "thread not found in live list");
130 void ReleaseThread(Thread
*t
) {
131 RemoveThreadStats(t
);
134 RemoveThreadFromLiveList(t
);
135 SpinMutexLock
l(&free_list_mutex_
);
136 free_list_
.push_back(t
);
139 Thread
*GetThreadByBufferAddress(uptr p
) {
140 return (Thread
*)(RoundDownTo(p
, ring_buffer_size_
* 2) +
144 uptr
MemoryUsedPerThread() {
145 uptr res
= sizeof(Thread
) + ring_buffer_size_
;
146 if (auto sz
= flags()->heap_history_size
)
147 res
+= HeapAllocationsRingBuffer::SizeInBytes(sz
);
152 void VisitAllLiveThreads(CB cb
) {
153 SpinMutexLock
l(&live_list_mutex_
);
154 for (Thread
*t
: live_list_
) cb(t
);
157 void AddThreadStats(Thread
*t
) {
158 SpinMutexLock
l(&stats_mutex_
);
159 stats_
.n_live_threads
++;
160 stats_
.total_stack_size
+= t
->stack_size();
163 void RemoveThreadStats(Thread
*t
) {
164 SpinMutexLock
l(&stats_mutex_
);
165 stats_
.n_live_threads
--;
166 stats_
.total_stack_size
-= t
->stack_size();
169 ThreadStats
GetThreadStats() {
170 SpinMutexLock
l(&stats_mutex_
);
174 uptr
GetRingBufferSize() const { return ring_buffer_size_
; }
177 Thread
*AllocThread() {
178 SpinMutexLock
l(&free_space_mutex_
);
179 uptr align
= ring_buffer_size_
* 2;
180 CHECK(IsAligned(free_space_
, align
));
181 Thread
*t
= (Thread
*)(free_space_
+ ring_buffer_size_
);
182 free_space_
+= thread_alloc_size_
;
183 CHECK(free_space_
<= free_space_end_
&& "out of thread memory");
187 SpinMutex free_space_mutex_
;
189 uptr free_space_end_
;
190 uptr ring_buffer_size_
;
191 uptr thread_alloc_size_
;
193 SpinMutex free_list_mutex_
;
194 InternalMmapVector
<Thread
*> free_list_
;
195 SpinMutex live_list_mutex_
;
196 InternalMmapVector
<Thread
*> live_list_
;
199 SpinMutex stats_mutex_
;
202 void InitThreadList(uptr storage
, uptr size
);
203 HwasanThreadList
&hwasanThreadList();
205 } // namespace __hwasan