1 //===-- sanitizer_allocator.cpp -------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is shared between AddressSanitizer and ThreadSanitizer
10 // run-time libraries.
11 // This allocator is used inside run-times.
12 //===----------------------------------------------------------------------===//
14 #include "sanitizer_allocator.h"
16 #include "sanitizer_allocator_checks.h"
17 #include "sanitizer_allocator_internal.h"
18 #include "sanitizer_atomic.h"
19 #include "sanitizer_common.h"
21 namespace __sanitizer
{
23 // Default allocator names.
24 const char *PrimaryAllocatorName
= "SizeClassAllocator";
25 const char *SecondaryAllocatorName
= "LargeMmapAllocator";
27 // ThreadSanitizer for Go uses libc malloc/free.
28 #if defined(SANITIZER_USE_MALLOC)
29 # if SANITIZER_LINUX && !SANITIZER_ANDROID
30 extern "C" void *__libc_malloc(uptr size
);
32 extern "C" void *__libc_memalign(uptr alignment
, uptr size
);
34 extern "C" void *__libc_realloc(void *ptr
, uptr size
);
35 extern "C" void __libc_free(void *ptr
);
38 # define __libc_malloc malloc
40 static void *__libc_memalign(uptr alignment
, uptr size
) {
42 uptr error
= posix_memalign(&p
, alignment
, size
);
43 if (error
) return nullptr;
47 # define __libc_realloc realloc
48 # define __libc_free free
51 static void *RawInternalAlloc(uptr size
, InternalAllocatorCache
*cache
,
56 return __libc_malloc(size
);
58 return __libc_memalign(alignment
, size
);
60 // Windows does not provide __libc_memalign/posix_memalign. It provides
61 // __aligned_malloc, but the allocated blocks can't be passed to free,
62 // they need to be passed to __aligned_free. InternalAlloc interface does
63 // not account for such requirement. Alignemnt does not seem to be used
64 // anywhere in runtime, so just call __libc_malloc for now.
65 DCHECK_EQ(alignment
, 0);
66 return __libc_malloc(size
);
70 static void *RawInternalRealloc(void *ptr
, uptr size
,
71 InternalAllocatorCache
*cache
) {
73 return __libc_realloc(ptr
, size
);
76 static void RawInternalFree(void *ptr
, InternalAllocatorCache
*cache
) {
81 InternalAllocator
*internal_allocator() {
85 #else // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
87 static ALIGNED(64) char internal_alloc_placeholder
[sizeof(InternalAllocator
)];
88 static atomic_uint8_t internal_allocator_initialized
;
89 static StaticSpinMutex internal_alloc_init_mu
;
91 static InternalAllocatorCache internal_allocator_cache
;
92 static StaticSpinMutex internal_allocator_cache_mu
;
94 InternalAllocator
*internal_allocator() {
95 InternalAllocator
*internal_allocator_instance
=
96 reinterpret_cast<InternalAllocator
*>(&internal_alloc_placeholder
);
97 if (atomic_load(&internal_allocator_initialized
, memory_order_acquire
) == 0) {
98 SpinMutexLock
l(&internal_alloc_init_mu
);
99 if (atomic_load(&internal_allocator_initialized
, memory_order_relaxed
) ==
101 internal_allocator_instance
->Init(kReleaseToOSIntervalNever
);
102 atomic_store(&internal_allocator_initialized
, 1, memory_order_release
);
105 return internal_allocator_instance
;
108 static void *RawInternalAlloc(uptr size
, InternalAllocatorCache
*cache
,
110 if (alignment
== 0) alignment
= 8;
112 SpinMutexLock
l(&internal_allocator_cache_mu
);
113 return internal_allocator()->Allocate(&internal_allocator_cache
, size
,
116 return internal_allocator()->Allocate(cache
, size
, alignment
);
119 static void *RawInternalRealloc(void *ptr
, uptr size
,
120 InternalAllocatorCache
*cache
) {
123 SpinMutexLock
l(&internal_allocator_cache_mu
);
124 return internal_allocator()->Reallocate(&internal_allocator_cache
, ptr
,
127 return internal_allocator()->Reallocate(cache
, ptr
, size
, alignment
);
130 static void RawInternalFree(void *ptr
, InternalAllocatorCache
*cache
) {
132 SpinMutexLock
l(&internal_allocator_cache_mu
);
133 return internal_allocator()->Deallocate(&internal_allocator_cache
, ptr
);
135 internal_allocator()->Deallocate(cache
, ptr
);
138 #endif // SANITIZER_GO || defined(SANITIZER_USE_MALLOC)
140 static void NORETURN
ReportInternalAllocatorOutOfMemory(uptr requested_size
) {
141 SetAllocatorOutOfMemory();
142 Report("FATAL: %s: internal allocator is out of memory trying to allocate "
143 "0x%zx bytes\n", SanitizerToolName
, requested_size
);
147 void *InternalAlloc(uptr size
, InternalAllocatorCache
*cache
, uptr alignment
) {
148 void *p
= RawInternalAlloc(size
, cache
, alignment
);
150 ReportInternalAllocatorOutOfMemory(size
);
154 void *InternalRealloc(void *addr
, uptr size
, InternalAllocatorCache
*cache
) {
155 void *p
= RawInternalRealloc(addr
, size
, cache
);
157 ReportInternalAllocatorOutOfMemory(size
);
161 void *InternalReallocArray(void *addr
, uptr count
, uptr size
,
162 InternalAllocatorCache
*cache
) {
163 if (UNLIKELY(CheckForCallocOverflow(count
, size
))) {
165 "FATAL: %s: reallocarray parameters overflow: count * size (%zd * %zd) "
166 "cannot be represented in type size_t\n",
167 SanitizerToolName
, count
, size
);
170 return InternalRealloc(addr
, count
* size
, cache
);
173 void *InternalCalloc(uptr count
, uptr size
, InternalAllocatorCache
*cache
) {
174 if (UNLIKELY(CheckForCallocOverflow(count
, size
))) {
175 Report("FATAL: %s: calloc parameters overflow: count * size (%zd * %zd) "
176 "cannot be represented in type size_t\n", SanitizerToolName
, count
,
180 void *p
= InternalAlloc(count
* size
, cache
);
182 internal_memset(p
, 0, count
* size
);
186 void InternalFree(void *addr
, InternalAllocatorCache
*cache
) {
187 RawInternalFree(addr
, cache
);
191 constexpr uptr kLowLevelAllocatorDefaultAlignment
= 8;
192 static uptr low_level_alloc_min_alignment
= kLowLevelAllocatorDefaultAlignment
;
193 static LowLevelAllocateCallback low_level_alloc_callback
;
195 void *LowLevelAllocator::Allocate(uptr size
) {
196 // Align allocation size.
197 size
= RoundUpTo(size
, low_level_alloc_min_alignment
);
198 if (allocated_end_
- allocated_current_
< (sptr
)size
) {
199 uptr size_to_allocate
= RoundUpTo(size
, GetPageSizeCached());
201 (char*)MmapOrDie(size_to_allocate
, __func__
);
202 allocated_end_
= allocated_current_
+ size_to_allocate
;
203 if (low_level_alloc_callback
) {
204 low_level_alloc_callback((uptr
)allocated_current_
,
208 CHECK(allocated_end_
- allocated_current_
>= (sptr
)size
);
209 void *res
= allocated_current_
;
210 allocated_current_
+= size
;
214 void SetLowLevelAllocateMinAlignment(uptr alignment
) {
215 CHECK(IsPowerOfTwo(alignment
));
216 low_level_alloc_min_alignment
= Max(alignment
, low_level_alloc_min_alignment
);
219 void SetLowLevelAllocateCallback(LowLevelAllocateCallback callback
) {
220 low_level_alloc_callback
= callback
;
223 // Allocator's OOM and other errors handling support.
225 static atomic_uint8_t allocator_out_of_memory
= {0};
226 static atomic_uint8_t allocator_may_return_null
= {0};
228 bool IsAllocatorOutOfMemory() {
229 return atomic_load_relaxed(&allocator_out_of_memory
);
232 void SetAllocatorOutOfMemory() {
233 atomic_store_relaxed(&allocator_out_of_memory
, 1);
236 bool AllocatorMayReturnNull() {
237 return atomic_load(&allocator_may_return_null
, memory_order_relaxed
);
240 void SetAllocatorMayReturnNull(bool may_return_null
) {
241 atomic_store(&allocator_may_return_null
, may_return_null
,
242 memory_order_relaxed
);
245 void PrintHintAllocatorCannotReturnNull() {
246 Report("HINT: if you don't care about these errors you may set "
247 "allocator_may_return_null=1\n");
250 } // namespace __sanitizer