1 //===-- sanitizer_allocator.cpp -------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is shared between AddressSanitizer and ThreadSanitizer
10 // run-time libraries.
11 // This allocator is used inside run-times.
12 //===----------------------------------------------------------------------===//
14 #include "sanitizer_allocator.h"
16 #include "sanitizer_allocator_checks.h"
17 #include "sanitizer_allocator_internal.h"
18 #include "sanitizer_atomic.h"
19 #include "sanitizer_common.h"
21 namespace __sanitizer
{
23 // Default allocator names.
24 const char *PrimaryAllocatorName
= "SizeClassAllocator";
25 const char *SecondaryAllocatorName
= "LargeMmapAllocator";
27 // ThreadSanitizer for Go uses libc malloc/free.
28 #if defined(SANITIZER_USE_MALLOC)
29 # if SANITIZER_LINUX && !SANITIZER_ANDROID
30 extern "C" void *__libc_malloc(uptr size
);
32 extern "C" void *__libc_memalign(uptr alignment
, uptr size
);
34 extern "C" void *__libc_realloc(void *ptr
, uptr size
);
35 extern "C" void __libc_free(void *ptr
);
38 # define __libc_malloc malloc
40 static void *__libc_memalign(uptr alignment
, uptr size
) {
42 uptr error
= posix_memalign(&p
, alignment
, size
);
43 if (error
) return nullptr;
47 # define __libc_realloc realloc
48 # define __libc_free free
51 static void *RawInternalAlloc(uptr size
, InternalAllocatorCache
*cache
,
56 return __libc_malloc(size
);
58 return __libc_memalign(alignment
, size
);
60 // Windows does not provide __libc_memalign/posix_memalign. It provides
61 // __aligned_malloc, but the allocated blocks can't be passed to free,
62 // they need to be passed to __aligned_free. InternalAlloc interface does
63 // not account for such requirement. Alignemnt does not seem to be used
64 // anywhere in runtime, so just call __libc_malloc for now.
65 DCHECK_EQ(alignment
, 0);
66 return __libc_malloc(size
);
70 static void *RawInternalRealloc(void *ptr
, uptr size
,
71 InternalAllocatorCache
*cache
) {
73 return __libc_realloc(ptr
, size
);
76 static void RawInternalFree(void *ptr
, InternalAllocatorCache
*cache
) {
81 InternalAllocator
*internal_allocator() {
85 #else // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
87 static ALIGNED(64) char internal_alloc_placeholder
[sizeof(InternalAllocator
)];
88 static atomic_uint8_t internal_allocator_initialized
;
89 static StaticSpinMutex internal_alloc_init_mu
;
91 static InternalAllocatorCache internal_allocator_cache
;
92 static StaticSpinMutex internal_allocator_cache_mu
;
94 InternalAllocator
*internal_allocator() {
95 InternalAllocator
*internal_allocator_instance
=
96 reinterpret_cast<InternalAllocator
*>(&internal_alloc_placeholder
);
97 if (atomic_load(&internal_allocator_initialized
, memory_order_acquire
) == 0) {
98 SpinMutexLock
l(&internal_alloc_init_mu
);
99 if (atomic_load(&internal_allocator_initialized
, memory_order_relaxed
) ==
101 internal_allocator_instance
->Init(kReleaseToOSIntervalNever
);
102 atomic_store(&internal_allocator_initialized
, 1, memory_order_release
);
105 return internal_allocator_instance
;
108 static void *RawInternalAlloc(uptr size
, InternalAllocatorCache
*cache
,
110 if (alignment
== 0) alignment
= 8;
112 SpinMutexLock
l(&internal_allocator_cache_mu
);
113 return internal_allocator()->Allocate(&internal_allocator_cache
, size
,
116 return internal_allocator()->Allocate(cache
, size
, alignment
);
119 static void *RawInternalRealloc(void *ptr
, uptr size
,
120 InternalAllocatorCache
*cache
) {
123 SpinMutexLock
l(&internal_allocator_cache_mu
);
124 return internal_allocator()->Reallocate(&internal_allocator_cache
, ptr
,
127 return internal_allocator()->Reallocate(cache
, ptr
, size
, alignment
);
130 static void RawInternalFree(void *ptr
, InternalAllocatorCache
*cache
) {
132 SpinMutexLock
l(&internal_allocator_cache_mu
);
133 return internal_allocator()->Deallocate(&internal_allocator_cache
, ptr
);
135 internal_allocator()->Deallocate(cache
, ptr
);
138 #endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
141 const u64 kBlockMagic
= 0x6A6CB03ABCEBC041ull
;
148 static void NORETURN
ReportInternalAllocatorOutOfMemory(uptr requested_size
) {
149 SetAllocatorOutOfMemory();
150 Report("FATAL: %s: internal allocator is out of memory trying to allocate "
151 "0x%zx bytes\n", SanitizerToolName
, requested_size
);
155 void *InternalAlloc(uptr size
, InternalAllocatorCache
*cache
, uptr alignment
) {
156 uptr s
= size
+ sizeof(BlockHeader
);
159 BlockHeader
*p
= (BlockHeader
*)RawInternalAlloc(s
, cache
, alignment
);
161 ReportInternalAllocatorOutOfMemory(s
);
162 p
->magic
= kBlockMagic
;
166 void *InternalRealloc(void *addr
, uptr size
, InternalAllocatorCache
*cache
) {
168 return InternalAlloc(size
, cache
);
169 uptr s
= size
+ sizeof(BlockHeader
);
172 BlockHeader
*p
= (BlockHeader
*)addr
- 1;
173 CHECK_EQ(kBlockMagic
, p
->magic
);
174 p
= (BlockHeader
*)RawInternalRealloc(p
, s
, cache
);
176 ReportInternalAllocatorOutOfMemory(s
);
180 void *InternalReallocArray(void *addr
, uptr count
, uptr size
,
181 InternalAllocatorCache
*cache
) {
182 if (UNLIKELY(CheckForCallocOverflow(count
, size
))) {
184 "FATAL: %s: reallocarray parameters overflow: count * size (%zd * %zd) "
185 "cannot be represented in type size_t\n",
186 SanitizerToolName
, count
, size
);
189 return InternalRealloc(addr
, count
* size
, cache
);
192 void *InternalCalloc(uptr count
, uptr size
, InternalAllocatorCache
*cache
) {
193 if (UNLIKELY(CheckForCallocOverflow(count
, size
))) {
194 Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) "
195 "cannot be represented in type size_t\n", SanitizerToolName
, count
,
199 void *p
= InternalAlloc(count
* size
, cache
);
201 internal_memset(p
, 0, count
* size
);
205 void InternalFree(void *addr
, InternalAllocatorCache
*cache
) {
208 BlockHeader
*p
= (BlockHeader
*)addr
- 1;
209 CHECK_EQ(kBlockMagic
, p
->magic
);
211 RawInternalFree(p
, cache
);
215 constexpr uptr kLowLevelAllocatorDefaultAlignment
= 8;
216 static uptr low_level_alloc_min_alignment
= kLowLevelAllocatorDefaultAlignment
;
217 static LowLevelAllocateCallback low_level_alloc_callback
;
219 void *LowLevelAllocator::Allocate(uptr size
) {
220 // Align allocation size.
221 size
= RoundUpTo(size
, low_level_alloc_min_alignment
);
222 if (allocated_end_
- allocated_current_
< (sptr
)size
) {
223 uptr size_to_allocate
= RoundUpTo(size
, GetPageSizeCached());
225 (char*)MmapOrDie(size_to_allocate
, __func__
);
226 allocated_end_
= allocated_current_
+ size_to_allocate
;
227 if (low_level_alloc_callback
) {
228 low_level_alloc_callback((uptr
)allocated_current_
,
232 CHECK(allocated_end_
- allocated_current_
>= (sptr
)size
);
233 void *res
= allocated_current_
;
234 allocated_current_
+= size
;
238 void SetLowLevelAllocateMinAlignment(uptr alignment
) {
239 CHECK(IsPowerOfTwo(alignment
));
240 low_level_alloc_min_alignment
= Max(alignment
, low_level_alloc_min_alignment
);
243 void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback
) {
244 low_level_alloc_callback
= callback
;
247 // Allocator's OOM and other errors handling support.
249 static atomic_uint8_t allocator_out_of_memory
= {0};
250 static atomic_uint8_t allocator_may_return_null
= {0};
252 bool IsAllocatorOutOfMemory() {
253 return atomic_load_relaxed(&allocator_out_of_memory
);
256 void SetAllocatorOutOfMemory() {
257 atomic_store_relaxed(&allocator_out_of_memory
, 1);
260 bool AllocatorMayReturnNull() {
261 return atomic_load(&allocator_may_return_null
, memory_order_relaxed
);
264 void SetAllocatorMayReturnNull(bool may_return_null
) {
265 atomic_store(&allocator_may_return_null
, may_return_null
,
266 memory_order_relaxed
);
269 void PrintHintAllocatorCannotReturnNull() {
270 Report("HINT: if you don't care about these errors you may set "
271 "allocator_may_return_null=1\n");
274 } // namespace __sanitizer