1 //===-- asan_allocator.cc -------------------------------------------------===//
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
6 //===----------------------------------------------------------------------===//
8 // This file is a part of AddressSanitizer, an address sanity checker.
10 // Implementation of ASan's memory allocator, 2-nd version.
11 // This variant uses the allocator from sanitizer_common, i.e. the one shared
12 // with ThreadSanitizer and MemorySanitizer.
14 //===----------------------------------------------------------------------===//
16 #include "asan_allocator.h"
17 #include "asan_mapping.h"
18 #include "asan_poisoning.h"
19 #include "asan_report.h"
20 #include "asan_stack.h"
21 #include "asan_thread.h"
22 #include "sanitizer_common/sanitizer_allocator_interface.h"
23 #include "sanitizer_common/sanitizer_flags.h"
24 #include "sanitizer_common/sanitizer_internal_defs.h"
25 #include "sanitizer_common/sanitizer_list.h"
26 #include "sanitizer_common/sanitizer_stackdepot.h"
27 #include "sanitizer_common/sanitizer_quarantine.h"
28 #include "lsan/lsan_common.h"
32 // Valid redzone sizes are 16, 32, 64, ... 2048, so we encode them in 3 bits.
33 // We use adaptive redzones: for larger allocation larger redzones are used.
34 static u32
RZLog2Size(u32 rz_log
) {
39 static u32
RZSize2Log(u32 rz_size
) {
40 CHECK_GE(rz_size
, 16);
41 CHECK_LE(rz_size
, 2048);
42 CHECK(IsPowerOfTwo(rz_size
));
43 u32 res
= Log2(rz_size
) - 4;
44 CHECK_EQ(rz_size
, RZLog2Size(res
));
48 static AsanAllocator
&get_allocator();
50 // The memory chunk allocated from the underlying allocator looks like this:
51 // L L L L L L H H U U U U U U R R
52 // L -- left redzone words (0 or more bytes)
53 // H -- ChunkHeader (16 bytes), which is also a part of the left redzone.
55 // R -- right redzone (0 or more bytes)
56 // ChunkBase consists of ChunkHeader and other bytes that overlap with user
59 // If the left redzone is greater than the ChunkHeader size we store a magic
60 // value in the first uptr word of the memory block and store the address of
61 // ChunkBase in the next uptr.
62 // M B L L L L L L L L L H H U U U U U U
64 // ---------------------|
65 // M -- magic value kAllocBegMagic
66 // B -- address of ChunkHeader pointing to the first 'H'
67 static const uptr kAllocBegMagic
= 0xCC6E96B9;
71 u32 chunk_state
: 8; // Must be first.
75 u32 from_memalign
: 1;
80 // This field is used for small sizes. For large sizes it is equal to
81 // SizeClassMap::kMaxSize and the actual size is stored in the
82 // SecondaryAllocator's metadata.
83 u32 user_requested_size
;
87 struct ChunkBase
: ChunkHeader
{
88 // Header2, intersects with user memory.
92 static const uptr kChunkHeaderSize
= sizeof(ChunkHeader
);
93 static const uptr kChunkHeader2Size
= sizeof(ChunkBase
) - kChunkHeaderSize
;
94 COMPILER_CHECK(kChunkHeaderSize
== 16);
95 COMPILER_CHECK(kChunkHeader2Size
<= 16);
97 // Every chunk of memory allocated by this allocator can be in one of 3 states:
98 // CHUNK_AVAILABLE: the chunk is in the free list and ready to be allocated.
99 // CHUNK_ALLOCATED: the chunk is allocated and not yet freed.
100 // CHUNK_QUARANTINE: the chunk was freed and put into quarantine zone.
102 CHUNK_AVAILABLE
= 0, // 0 is the default value even if we didn't set it.
107 struct AsanChunk
: ChunkBase
{
108 uptr
Beg() { return reinterpret_cast<uptr
>(this) + kChunkHeaderSize
; }
109 uptr
UsedSize(bool locked_version
= false) {
110 if (user_requested_size
!= SizeClassMap::kMaxSize
)
111 return user_requested_size
;
112 return *reinterpret_cast<uptr
*>(
113 get_allocator().GetMetaData(AllocBeg(locked_version
)));
115 void *AllocBeg(bool locked_version
= false) {
118 return get_allocator().GetBlockBeginFastLocked(
119 reinterpret_cast<void *>(this));
120 return get_allocator().GetBlockBegin(reinterpret_cast<void *>(this));
122 return reinterpret_cast<void*>(Beg() - RZLog2Size(rz_log
));
124 bool AddrIsInside(uptr addr
, bool locked_version
= false) {
125 return (addr
>= Beg()) && (addr
< Beg() + UsedSize(locked_version
));
129 struct QuarantineCallback
{
130 explicit QuarantineCallback(AllocatorCache
*cache
)
134 void Recycle(AsanChunk
*m
) {
135 CHECK_EQ(m
->chunk_state
, CHUNK_QUARANTINE
);
136 atomic_store((atomic_uint8_t
*)m
, CHUNK_AVAILABLE
, memory_order_relaxed
);
137 CHECK_NE(m
->alloc_tid
, kInvalidTid
);
138 CHECK_NE(m
->free_tid
, kInvalidTid
);
139 PoisonShadow(m
->Beg(),
140 RoundUpTo(m
->UsedSize(), SHADOW_GRANULARITY
),
141 kAsanHeapLeftRedzoneMagic
);
142 void *p
= reinterpret_cast<void *>(m
->AllocBeg());
144 uptr
*alloc_magic
= reinterpret_cast<uptr
*>(p
);
145 CHECK_EQ(alloc_magic
[0], kAllocBegMagic
);
146 // Clear the magic value, as allocator internals may overwrite the
147 // contents of deallocated chunk, confusing GetAsanChunk lookup.
149 CHECK_EQ(alloc_magic
[1], reinterpret_cast<uptr
>(m
));
153 AsanStats
&thread_stats
= GetCurrentThreadStats();
154 thread_stats
.real_frees
++;
155 thread_stats
.really_freed
+= m
->UsedSize();
157 get_allocator().Deallocate(cache_
, p
);
160 void *Allocate(uptr size
) {
161 return get_allocator().Allocate(cache_
, size
, 1, false);
164 void Deallocate(void *p
) {
165 get_allocator().Deallocate(cache_
, p
);
168 AllocatorCache
*cache_
;
171 typedef Quarantine
<QuarantineCallback
, AsanChunk
> AsanQuarantine
;
172 typedef AsanQuarantine::Cache QuarantineCache
;
174 void AsanMapUnmapCallback::OnMap(uptr p
, uptr size
) const {
175 PoisonShadow(p
, size
, kAsanHeapLeftRedzoneMagic
);
177 AsanStats
&thread_stats
= GetCurrentThreadStats();
178 thread_stats
.mmaps
++;
179 thread_stats
.mmaped
+= size
;
181 void AsanMapUnmapCallback::OnUnmap(uptr p
, uptr size
) const {
182 PoisonShadow(p
, size
, 0);
183 // We are about to unmap a chunk of user memory.
184 // Mark the corresponding shadow memory as not needed.
185 FlushUnneededASanShadowMemory(p
, size
);
187 AsanStats
&thread_stats
= GetCurrentThreadStats();
188 thread_stats
.munmaps
++;
189 thread_stats
.munmaped
+= size
;
192 // We can not use THREADLOCAL because it is not supported on some of the
193 // platforms we care about (OSX 10.6, Android).
194 // static THREADLOCAL AllocatorCache cache;
195 AllocatorCache
*GetAllocatorCache(AsanThreadLocalMallocStorage
*ms
) {
197 return &ms
->allocator_cache
;
200 QuarantineCache
*GetQuarantineCache(AsanThreadLocalMallocStorage
*ms
) {
202 CHECK_LE(sizeof(QuarantineCache
), sizeof(ms
->quarantine_cache
));
203 return reinterpret_cast<QuarantineCache
*>(ms
->quarantine_cache
);
206 void AllocatorOptions::SetFrom(const Flags
*f
, const CommonFlags
*cf
) {
207 quarantine_size_mb
= f
->quarantine_size_mb
;
208 min_redzone
= f
->redzone
;
209 max_redzone
= f
->max_redzone
;
210 may_return_null
= cf
->allocator_may_return_null
;
211 alloc_dealloc_mismatch
= f
->alloc_dealloc_mismatch
;
214 void AllocatorOptions::CopyTo(Flags
*f
, CommonFlags
*cf
) {
215 f
->quarantine_size_mb
= quarantine_size_mb
;
216 f
->redzone
= min_redzone
;
217 f
->max_redzone
= max_redzone
;
218 cf
->allocator_may_return_null
= may_return_null
;
219 f
->alloc_dealloc_mismatch
= alloc_dealloc_mismatch
;
223 static const uptr kMaxAllowedMallocSize
=
224 FIRST_32_SECOND_64(3UL << 30, 1ULL << 40);
225 static const uptr kMaxThreadLocalQuarantine
=
226 FIRST_32_SECOND_64(1 << 18, 1 << 20);
228 AsanAllocator allocator
;
229 AsanQuarantine quarantine
;
230 StaticSpinMutex fallback_mutex
;
231 AllocatorCache fallback_allocator_cache
;
232 QuarantineCache fallback_quarantine_cache
;
234 // ------------------- Options --------------------------
235 atomic_uint16_t min_redzone
;
236 atomic_uint16_t max_redzone
;
237 atomic_uint8_t alloc_dealloc_mismatch
;
239 // ------------------- Initialization ------------------------
240 explicit Allocator(LinkerInitialized
)
241 : quarantine(LINKER_INITIALIZED
),
242 fallback_quarantine_cache(LINKER_INITIALIZED
) {}
244 void CheckOptions(const AllocatorOptions
&options
) const {
245 CHECK_GE(options
.min_redzone
, 16);
246 CHECK_GE(options
.max_redzone
, options
.min_redzone
);
247 CHECK_LE(options
.max_redzone
, 2048);
248 CHECK(IsPowerOfTwo(options
.min_redzone
));
249 CHECK(IsPowerOfTwo(options
.max_redzone
));
252 void SharedInitCode(const AllocatorOptions
&options
) {
253 CheckOptions(options
);
254 quarantine
.Init((uptr
)options
.quarantine_size_mb
<< 20,
255 kMaxThreadLocalQuarantine
);
256 atomic_store(&alloc_dealloc_mismatch
, options
.alloc_dealloc_mismatch
,
257 memory_order_release
);
258 atomic_store(&min_redzone
, options
.min_redzone
, memory_order_release
);
259 atomic_store(&max_redzone
, options
.max_redzone
, memory_order_release
);
262 void Initialize(const AllocatorOptions
&options
) {
263 allocator
.Init(options
.may_return_null
);
264 SharedInitCode(options
);
267 void RePoisonChunk(uptr chunk
) {
268 // This could a user-facing chunk (with redzones), or some internal
269 // housekeeping chunk, like TransferBatch. Start by assuming the former.
270 AsanChunk
*ac
= GetAsanChunk((void *)chunk
);
271 uptr allocated_size
= allocator
.GetActuallyAllocatedSize((void *)ac
);
272 uptr beg
= ac
->Beg();
273 uptr end
= ac
->Beg() + ac
->UsedSize(true);
274 uptr chunk_end
= chunk
+ allocated_size
;
275 if (chunk
< beg
&& beg
< end
&& end
<= chunk_end
) {
276 // Looks like a valid AsanChunk. Or maybe not. Be conservative and only
277 // poison the redzones.
278 PoisonShadow(chunk
, beg
- chunk
, kAsanHeapLeftRedzoneMagic
);
279 uptr end_aligned_down
= RoundDownTo(end
, SHADOW_GRANULARITY
);
280 FastPoisonShadowPartialRightRedzone(
281 end_aligned_down
, end
- end_aligned_down
,
282 chunk_end
- end_aligned_down
, kAsanHeapLeftRedzoneMagic
);
284 // This can not be an AsanChunk. Poison everything. It may be reused as
286 PoisonShadow(chunk
, allocated_size
, kAsanHeapLeftRedzoneMagic
);
290 void ReInitialize(const AllocatorOptions
&options
) {
291 allocator
.SetMayReturnNull(options
.may_return_null
);
292 SharedInitCode(options
);
294 // Poison all existing allocation's redzones.
295 if (CanPoisonMemory()) {
296 allocator
.ForceLock();
297 allocator
.ForEachChunk(
298 [](uptr chunk
, void *alloc
) {
299 ((Allocator
*)alloc
)->RePoisonChunk(chunk
);
302 allocator
.ForceUnlock();
306 void GetOptions(AllocatorOptions
*options
) const {
307 options
->quarantine_size_mb
= quarantine
.GetSize() >> 20;
308 options
->min_redzone
= atomic_load(&min_redzone
, memory_order_acquire
);
309 options
->max_redzone
= atomic_load(&max_redzone
, memory_order_acquire
);
310 options
->may_return_null
= allocator
.MayReturnNull();
311 options
->alloc_dealloc_mismatch
=
312 atomic_load(&alloc_dealloc_mismatch
, memory_order_acquire
);
315 // -------------------- Helper methods. -------------------------
316 uptr
ComputeRZLog(uptr user_requested_size
) {
318 user_requested_size
<= 64 - 16 ? 0 :
319 user_requested_size
<= 128 - 32 ? 1 :
320 user_requested_size
<= 512 - 64 ? 2 :
321 user_requested_size
<= 4096 - 128 ? 3 :
322 user_requested_size
<= (1 << 14) - 256 ? 4 :
323 user_requested_size
<= (1 << 15) - 512 ? 5 :
324 user_requested_size
<= (1 << 16) - 1024 ? 6 : 7;
325 u32 min_rz
= atomic_load(&min_redzone
, memory_order_acquire
);
326 u32 max_rz
= atomic_load(&max_redzone
, memory_order_acquire
);
327 return Min(Max(rz_log
, RZSize2Log(min_rz
)), RZSize2Log(max_rz
));
330 // We have an address between two chunks, and we want to report just one.
331 AsanChunk
*ChooseChunk(uptr addr
, AsanChunk
*left_chunk
,
332 AsanChunk
*right_chunk
) {
333 // Prefer an allocated chunk over freed chunk and freed chunk
334 // over available chunk.
335 if (left_chunk
->chunk_state
!= right_chunk
->chunk_state
) {
336 if (left_chunk
->chunk_state
== CHUNK_ALLOCATED
)
338 if (right_chunk
->chunk_state
== CHUNK_ALLOCATED
)
340 if (left_chunk
->chunk_state
== CHUNK_QUARANTINE
)
342 if (right_chunk
->chunk_state
== CHUNK_QUARANTINE
)
345 // Same chunk_state: choose based on offset.
346 sptr l_offset
= 0, r_offset
= 0;
347 CHECK(AsanChunkView(left_chunk
).AddrIsAtRight(addr
, 1, &l_offset
));
348 CHECK(AsanChunkView(right_chunk
).AddrIsAtLeft(addr
, 1, &r_offset
));
349 if (l_offset
< r_offset
)
354 // -------------------- Allocation/Deallocation routines ---------------
355 void *Allocate(uptr size
, uptr alignment
, BufferedStackTrace
*stack
,
356 AllocType alloc_type
, bool can_fill
) {
357 if (UNLIKELY(!asan_inited
))
359 Flags
&fl
= *flags();
361 const uptr min_alignment
= SHADOW_GRANULARITY
;
362 if (alignment
< min_alignment
)
363 alignment
= min_alignment
;
365 // We'd be happy to avoid allocating memory for zero-size requests, but
366 // some programs/tests depend on this behavior and assume that malloc
367 // would not return NULL even for zero-size allocations. Moreover, it
368 // looks like operator new should never return NULL, and results of
369 // consecutive "new" calls must be different even if the allocated size
373 CHECK(IsPowerOfTwo(alignment
));
374 uptr rz_log
= ComputeRZLog(size
);
375 uptr rz_size
= RZLog2Size(rz_log
);
376 uptr rounded_size
= RoundUpTo(Max(size
, kChunkHeader2Size
), alignment
);
377 uptr needed_size
= rounded_size
+ rz_size
;
378 if (alignment
> min_alignment
)
379 needed_size
+= alignment
;
380 bool using_primary_allocator
= true;
381 // If we are allocating from the secondary allocator, there will be no
382 // automatic right redzone, so add the right redzone manually.
383 if (!PrimaryAllocator::CanAllocate(needed_size
, alignment
)) {
384 needed_size
+= rz_size
;
385 using_primary_allocator
= false;
387 CHECK(IsAligned(needed_size
, min_alignment
));
388 if (size
> kMaxAllowedMallocSize
|| needed_size
> kMaxAllowedMallocSize
) {
389 Report("WARNING: AddressSanitizer failed to allocate 0x%zx bytes\n",
391 return allocator
.ReturnNullOrDieOnBadRequest();
394 AsanThread
*t
= GetCurrentThread();
396 bool check_rss_limit
= true;
398 AllocatorCache
*cache
= GetAllocatorCache(&t
->malloc_storage());
400 allocator
.Allocate(cache
, needed_size
, 8, false, check_rss_limit
);
402 SpinMutexLock
l(&fallback_mutex
);
403 AllocatorCache
*cache
= &fallback_allocator_cache
;
405 allocator
.Allocate(cache
, needed_size
, 8, false, check_rss_limit
);
408 if (!allocated
) return allocator
.ReturnNullOrDieOnOOM();
410 if (*(u8
*)MEM_TO_SHADOW((uptr
)allocated
) == 0 && CanPoisonMemory()) {
411 // Heap poisoning is enabled, but the allocator provides an unpoisoned
412 // chunk. This is possible if CanPoisonMemory() was false for some
413 // time, for example, due to flags()->start_disabled.
414 // Anyway, poison the block before using it for anything else.
415 uptr allocated_size
= allocator
.GetActuallyAllocatedSize(allocated
);
416 PoisonShadow((uptr
)allocated
, allocated_size
, kAsanHeapLeftRedzoneMagic
);
419 uptr alloc_beg
= reinterpret_cast<uptr
>(allocated
);
420 uptr alloc_end
= alloc_beg
+ needed_size
;
421 uptr beg_plus_redzone
= alloc_beg
+ rz_size
;
422 uptr user_beg
= beg_plus_redzone
;
423 if (!IsAligned(user_beg
, alignment
))
424 user_beg
= RoundUpTo(user_beg
, alignment
);
425 uptr user_end
= user_beg
+ size
;
426 CHECK_LE(user_end
, alloc_end
);
427 uptr chunk_beg
= user_beg
- kChunkHeaderSize
;
428 AsanChunk
*m
= reinterpret_cast<AsanChunk
*>(chunk_beg
);
429 m
->alloc_type
= alloc_type
;
431 u32 alloc_tid
= t
? t
->tid() : 0;
432 m
->alloc_tid
= alloc_tid
;
433 CHECK_EQ(alloc_tid
, m
->alloc_tid
); // Does alloc_tid fit into the bitfield?
434 m
->free_tid
= kInvalidTid
;
435 m
->from_memalign
= user_beg
!= beg_plus_redzone
;
436 if (alloc_beg
!= chunk_beg
) {
437 CHECK_LE(alloc_beg
+ 2 * sizeof(uptr
), chunk_beg
);
438 reinterpret_cast<uptr
*>(alloc_beg
)[0] = kAllocBegMagic
;
439 reinterpret_cast<uptr
*>(alloc_beg
)[1] = chunk_beg
;
441 if (using_primary_allocator
) {
443 m
->user_requested_size
= size
;
444 CHECK(allocator
.FromPrimary(allocated
));
446 CHECK(!allocator
.FromPrimary(allocated
));
447 m
->user_requested_size
= SizeClassMap::kMaxSize
;
448 uptr
*meta
= reinterpret_cast<uptr
*>(allocator
.GetMetaData(allocated
));
453 m
->alloc_context_id
= StackDepotPut(*stack
);
455 uptr size_rounded_down_to_granularity
=
456 RoundDownTo(size
, SHADOW_GRANULARITY
);
457 // Unpoison the bulk of the memory region.
458 if (size_rounded_down_to_granularity
)
459 PoisonShadow(user_beg
, size_rounded_down_to_granularity
, 0);
460 // Deal with the end of the region if size is not aligned to granularity.
461 if (size
!= size_rounded_down_to_granularity
&& CanPoisonMemory()) {
463 (u8
*)MemToShadow(user_beg
+ size_rounded_down_to_granularity
);
464 *shadow
= fl
.poison_partial
? (size
& (SHADOW_GRANULARITY
- 1)) : 0;
467 AsanStats
&thread_stats
= GetCurrentThreadStats();
468 thread_stats
.mallocs
++;
469 thread_stats
.malloced
+= size
;
470 thread_stats
.malloced_redzones
+= needed_size
- size
;
471 if (needed_size
> SizeClassMap::kMaxSize
)
472 thread_stats
.malloc_large
++;
474 thread_stats
.malloced_by_size
[SizeClassMap::ClassID(needed_size
)]++;
476 void *res
= reinterpret_cast<void *>(user_beg
);
477 if (can_fill
&& fl
.max_malloc_fill_size
) {
478 uptr fill_size
= Min(size
, (uptr
)fl
.max_malloc_fill_size
);
479 REAL(memset
)(res
, fl
.malloc_fill_byte
, fill_size
);
481 #if CAN_SANITIZE_LEAKS
482 m
->lsan_tag
= __lsan::DisabledInThisThread() ? __lsan::kIgnored
483 : __lsan::kDirectlyLeaked
;
485 // Must be the last mutation of metadata in this function.
486 atomic_store((atomic_uint8_t
*)m
, CHUNK_ALLOCATED
, memory_order_release
);
487 ASAN_MALLOC_HOOK(res
, size
);
491 // Set quarantine flag if chunk is allocated, issue ASan error report on
492 // available and quarantined chunks. Return true on success, false otherwise.
493 bool AtomicallySetQuarantineFlagIfAllocated(AsanChunk
*m
, void *ptr
,
494 BufferedStackTrace
*stack
) {
495 u8 old_chunk_state
= CHUNK_ALLOCATED
;
496 // Flip the chunk_state atomically to avoid race on double-free.
497 if (!atomic_compare_exchange_strong((atomic_uint8_t
*)m
, &old_chunk_state
,
499 memory_order_acquire
)) {
500 ReportInvalidFree(ptr
, old_chunk_state
, stack
);
501 // It's not safe to push a chunk in quarantine on invalid free.
504 CHECK_EQ(CHUNK_ALLOCATED
, old_chunk_state
);
508 // Expects the chunk to already be marked as quarantined by using
509 // AtomicallySetQuarantineFlagIfAllocated.
510 void QuarantineChunk(AsanChunk
*m
, void *ptr
, BufferedStackTrace
*stack
,
511 AllocType alloc_type
) {
512 CHECK_EQ(m
->chunk_state
, CHUNK_QUARANTINE
);
513 CHECK_GE(m
->alloc_tid
, 0);
514 if (SANITIZER_WORDSIZE
== 64) // On 32-bits this resides in user area.
515 CHECK_EQ(m
->free_tid
, kInvalidTid
);
516 AsanThread
*t
= GetCurrentThread();
517 m
->free_tid
= t
? t
->tid() : 0;
518 m
->free_context_id
= StackDepotPut(*stack
);
519 // Poison the region.
520 PoisonShadow(m
->Beg(),
521 RoundUpTo(m
->UsedSize(), SHADOW_GRANULARITY
),
524 AsanStats
&thread_stats
= GetCurrentThreadStats();
525 thread_stats
.frees
++;
526 thread_stats
.freed
+= m
->UsedSize();
528 // Push into quarantine.
530 AsanThreadLocalMallocStorage
*ms
= &t
->malloc_storage();
531 AllocatorCache
*ac
= GetAllocatorCache(ms
);
532 quarantine
.Put(GetQuarantineCache(ms
), QuarantineCallback(ac
), m
,
535 SpinMutexLock
l(&fallback_mutex
);
536 AllocatorCache
*ac
= &fallback_allocator_cache
;
537 quarantine
.Put(&fallback_quarantine_cache
, QuarantineCallback(ac
), m
,
542 void Deallocate(void *ptr
, uptr delete_size
, BufferedStackTrace
*stack
,
543 AllocType alloc_type
) {
544 uptr p
= reinterpret_cast<uptr
>(ptr
);
547 uptr chunk_beg
= p
- kChunkHeaderSize
;
548 AsanChunk
*m
= reinterpret_cast<AsanChunk
*>(chunk_beg
);
551 // Must mark the chunk as quarantined before any changes to its metadata.
552 // Do not quarantine given chunk if we failed to set CHUNK_QUARANTINE flag.
553 if (!AtomicallySetQuarantineFlagIfAllocated(m
, ptr
, stack
)) return;
555 if (m
->alloc_type
!= alloc_type
) {
556 if (atomic_load(&alloc_dealloc_mismatch
, memory_order_acquire
)) {
557 ReportAllocTypeMismatch((uptr
)ptr
, stack
, (AllocType
)m
->alloc_type
,
558 (AllocType
)alloc_type
);
562 if (delete_size
&& flags()->new_delete_type_mismatch
&&
563 delete_size
!= m
->UsedSize()) {
564 ReportNewDeleteSizeMismatch(p
, delete_size
, stack
);
567 QuarantineChunk(m
, ptr
, stack
, alloc_type
);
570 void *Reallocate(void *old_ptr
, uptr new_size
, BufferedStackTrace
*stack
) {
571 CHECK(old_ptr
&& new_size
);
572 uptr p
= reinterpret_cast<uptr
>(old_ptr
);
573 uptr chunk_beg
= p
- kChunkHeaderSize
;
574 AsanChunk
*m
= reinterpret_cast<AsanChunk
*>(chunk_beg
);
576 AsanStats
&thread_stats
= GetCurrentThreadStats();
577 thread_stats
.reallocs
++;
578 thread_stats
.realloced
+= new_size
;
580 void *new_ptr
= Allocate(new_size
, 8, stack
, FROM_MALLOC
, true);
582 u8 chunk_state
= m
->chunk_state
;
583 if (chunk_state
!= CHUNK_ALLOCATED
)
584 ReportInvalidFree(old_ptr
, chunk_state
, stack
);
585 CHECK_NE(REAL(memcpy
), nullptr);
586 uptr memcpy_size
= Min(new_size
, m
->UsedSize());
587 // If realloc() races with free(), we may start copying freed memory.
588 // However, we will report racy double-free later anyway.
589 REAL(memcpy
)(new_ptr
, old_ptr
, memcpy_size
);
590 Deallocate(old_ptr
, 0, stack
, FROM_MALLOC
);
595 void *Calloc(uptr nmemb
, uptr size
, BufferedStackTrace
*stack
) {
596 if (CallocShouldReturnNullDueToOverflow(size
, nmemb
))
597 return allocator
.ReturnNullOrDieOnBadRequest();
598 void *ptr
= Allocate(nmemb
* size
, 8, stack
, FROM_MALLOC
, false);
599 // If the memory comes from the secondary allocator no need to clear it
600 // as it comes directly from mmap.
601 if (ptr
&& allocator
.FromPrimary(ptr
))
602 REAL(memset
)(ptr
, 0, nmemb
* size
);
606 void ReportInvalidFree(void *ptr
, u8 chunk_state
, BufferedStackTrace
*stack
) {
607 if (chunk_state
== CHUNK_QUARANTINE
)
608 ReportDoubleFree((uptr
)ptr
, stack
);
610 ReportFreeNotMalloced((uptr
)ptr
, stack
);
613 void CommitBack(AsanThreadLocalMallocStorage
*ms
) {
614 AllocatorCache
*ac
= GetAllocatorCache(ms
);
615 quarantine
.Drain(GetQuarantineCache(ms
), QuarantineCallback(ac
));
616 allocator
.SwallowCache(ac
);
619 // -------------------------- Chunk lookup ----------------------
621 // Assumes alloc_beg == allocator.GetBlockBegin(alloc_beg).
622 AsanChunk
*GetAsanChunk(void *alloc_beg
) {
623 if (!alloc_beg
) return nullptr;
624 if (!allocator
.FromPrimary(alloc_beg
)) {
625 uptr
*meta
= reinterpret_cast<uptr
*>(allocator
.GetMetaData(alloc_beg
));
626 AsanChunk
*m
= reinterpret_cast<AsanChunk
*>(meta
[1]);
629 uptr
*alloc_magic
= reinterpret_cast<uptr
*>(alloc_beg
);
630 if (alloc_magic
[0] == kAllocBegMagic
)
631 return reinterpret_cast<AsanChunk
*>(alloc_magic
[1]);
632 return reinterpret_cast<AsanChunk
*>(alloc_beg
);
635 AsanChunk
*GetAsanChunkByAddr(uptr p
) {
636 void *alloc_beg
= allocator
.GetBlockBegin(reinterpret_cast<void *>(p
));
637 return GetAsanChunk(alloc_beg
);
640 // Allocator must be locked when this function is called.
641 AsanChunk
*GetAsanChunkByAddrFastLocked(uptr p
) {
643 allocator
.GetBlockBeginFastLocked(reinterpret_cast<void *>(p
));
644 return GetAsanChunk(alloc_beg
);
647 uptr
AllocationSize(uptr p
) {
648 AsanChunk
*m
= GetAsanChunkByAddr(p
);
650 if (m
->chunk_state
!= CHUNK_ALLOCATED
) return 0;
651 if (m
->Beg() != p
) return 0;
652 return m
->UsedSize();
655 AsanChunkView
FindHeapChunkByAddress(uptr addr
) {
656 AsanChunk
*m1
= GetAsanChunkByAddr(addr
);
657 if (!m1
) return AsanChunkView(m1
);
659 if (AsanChunkView(m1
).AddrIsAtLeft(addr
, 1, &offset
)) {
660 // The address is in the chunk's left redzone, so maybe it is actually
661 // a right buffer overflow from the other chunk to the left.
662 // Search a bit to the left to see if there is another chunk.
663 AsanChunk
*m2
= nullptr;
664 for (uptr l
= 1; l
< GetPageSizeCached(); l
++) {
665 m2
= GetAsanChunkByAddr(addr
- l
);
666 if (m2
== m1
) continue; // Still the same chunk.
669 if (m2
&& AsanChunkView(m2
).AddrIsAtRight(addr
, 1, &offset
))
670 m1
= ChooseChunk(addr
, m2
, m1
);
672 return AsanChunkView(m1
);
676 allocator
.PrintStats();
680 allocator
.ForceLock();
681 fallback_mutex
.Lock();
685 fallback_mutex
.Unlock();
686 allocator
.ForceUnlock();
689 void ReleaseToOS() { allocator
.ReleaseToOS(); }
692 static Allocator
instance(LINKER_INITIALIZED
);
694 static AsanAllocator
&get_allocator() {
695 return instance
.allocator
;
698 bool AsanChunkView::IsValid() {
699 return chunk_
&& chunk_
->chunk_state
!= CHUNK_AVAILABLE
;
701 bool AsanChunkView::IsAllocated() {
702 return chunk_
&& chunk_
->chunk_state
== CHUNK_ALLOCATED
;
704 uptr
AsanChunkView::Beg() { return chunk_
->Beg(); }
705 uptr
AsanChunkView::End() { return Beg() + UsedSize(); }
706 uptr
AsanChunkView::UsedSize() { return chunk_
->UsedSize(); }
707 uptr
AsanChunkView::AllocTid() { return chunk_
->alloc_tid
; }
708 uptr
AsanChunkView::FreeTid() { return chunk_
->free_tid
; }
709 AllocType
AsanChunkView::GetAllocType() {
710 return (AllocType
)chunk_
->alloc_type
;
713 static StackTrace
GetStackTraceFromId(u32 id
) {
715 StackTrace res
= StackDepotGet(id
);
720 u32
AsanChunkView::GetAllocStackId() { return chunk_
->alloc_context_id
; }
721 u32
AsanChunkView::GetFreeStackId() { return chunk_
->free_context_id
; }
723 StackTrace
AsanChunkView::GetAllocStack() {
724 return GetStackTraceFromId(GetAllocStackId());
727 StackTrace
AsanChunkView::GetFreeStack() {
728 return GetStackTraceFromId(GetFreeStackId());
731 void ReleaseToOS() { instance
.ReleaseToOS(); }
733 void InitializeAllocator(const AllocatorOptions
&options
) {
734 instance
.Initialize(options
);
735 SetAllocatorReleaseToOSCallback(ReleaseToOS
);
738 void ReInitializeAllocator(const AllocatorOptions
&options
) {
739 instance
.ReInitialize(options
);
742 void GetAllocatorOptions(AllocatorOptions
*options
) {
743 instance
.GetOptions(options
);
746 AsanChunkView
FindHeapChunkByAddress(uptr addr
) {
747 return instance
.FindHeapChunkByAddress(addr
);
749 AsanChunkView
FindHeapChunkByAllocBeg(uptr addr
) {
750 return AsanChunkView(instance
.GetAsanChunk(reinterpret_cast<void*>(addr
)));
753 void AsanThreadLocalMallocStorage::CommitBack() {
754 instance
.CommitBack(this);
757 void PrintInternalAllocatorStats() {
758 instance
.PrintStats();
761 void *asan_memalign(uptr alignment
, uptr size
, BufferedStackTrace
*stack
,
762 AllocType alloc_type
) {
763 return instance
.Allocate(size
, alignment
, stack
, alloc_type
, true);
766 void asan_free(void *ptr
, BufferedStackTrace
*stack
, AllocType alloc_type
) {
767 instance
.Deallocate(ptr
, 0, stack
, alloc_type
);
770 void asan_sized_free(void *ptr
, uptr size
, BufferedStackTrace
*stack
,
771 AllocType alloc_type
) {
772 instance
.Deallocate(ptr
, size
, stack
, alloc_type
);
775 void *asan_malloc(uptr size
, BufferedStackTrace
*stack
) {
776 return instance
.Allocate(size
, 8, stack
, FROM_MALLOC
, true);
779 void *asan_calloc(uptr nmemb
, uptr size
, BufferedStackTrace
*stack
) {
780 return instance
.Calloc(nmemb
, size
, stack
);
783 void *asan_realloc(void *p
, uptr size
, BufferedStackTrace
*stack
) {
785 return instance
.Allocate(size
, 8, stack
, FROM_MALLOC
, true);
787 instance
.Deallocate(p
, 0, stack
, FROM_MALLOC
);
790 return instance
.Reallocate(p
, size
, stack
);
793 void *asan_valloc(uptr size
, BufferedStackTrace
*stack
) {
794 return instance
.Allocate(size
, GetPageSizeCached(), stack
, FROM_MALLOC
, true);
797 void *asan_pvalloc(uptr size
, BufferedStackTrace
*stack
) {
798 uptr PageSize
= GetPageSizeCached();
799 size
= RoundUpTo(size
, PageSize
);
801 // pvalloc(0) should allocate one page.
804 return instance
.Allocate(size
, PageSize
, stack
, FROM_MALLOC
, true);
807 int asan_posix_memalign(void **memptr
, uptr alignment
, uptr size
,
808 BufferedStackTrace
*stack
) {
809 void *ptr
= instance
.Allocate(size
, alignment
, stack
, FROM_MALLOC
, true);
810 CHECK(IsAligned((uptr
)ptr
, alignment
));
815 uptr
asan_malloc_usable_size(const void *ptr
, uptr pc
, uptr bp
) {
817 uptr usable_size
= instance
.AllocationSize(reinterpret_cast<uptr
>(ptr
));
818 if (flags()->check_malloc_usable_size
&& (usable_size
== 0)) {
819 GET_STACK_TRACE_FATAL(pc
, bp
);
820 ReportMallocUsableSizeNotOwned((uptr
)ptr
, &stack
);
825 uptr
asan_mz_size(const void *ptr
) {
826 return instance
.AllocationSize(reinterpret_cast<uptr
>(ptr
));
829 void asan_mz_force_lock() {
830 instance
.ForceLock();
833 void asan_mz_force_unlock() {
834 instance
.ForceUnlock();
837 void AsanSoftRssLimitExceededCallback(bool exceeded
) {
838 instance
.allocator
.SetRssLimitIsExceeded(exceeded
);
841 } // namespace __asan
843 // --- Implementation of LSan-specific functions --- {{{1
845 void LockAllocator() {
846 __asan::get_allocator().ForceLock();
849 void UnlockAllocator() {
850 __asan::get_allocator().ForceUnlock();
853 void GetAllocatorGlobalRange(uptr
*begin
, uptr
*end
) {
854 *begin
= (uptr
)&__asan::get_allocator();
855 *end
= *begin
+ sizeof(__asan::get_allocator());
858 uptr
PointsIntoChunk(void* p
) {
859 uptr addr
= reinterpret_cast<uptr
>(p
);
860 __asan::AsanChunk
*m
= __asan::instance
.GetAsanChunkByAddrFastLocked(addr
);
862 uptr chunk
= m
->Beg();
863 if (m
->chunk_state
!= __asan::CHUNK_ALLOCATED
)
865 if (m
->AddrIsInside(addr
, /*locked_version=*/true))
867 if (IsSpecialCaseOfOperatorNew0(chunk
, m
->UsedSize(/*locked_version*/ true),
873 uptr
GetUserBegin(uptr chunk
) {
874 __asan::AsanChunk
*m
= __asan::instance
.GetAsanChunkByAddrFastLocked(chunk
);
879 LsanMetadata::LsanMetadata(uptr chunk
) {
880 metadata_
= reinterpret_cast<void *>(chunk
- __asan::kChunkHeaderSize
);
883 bool LsanMetadata::allocated() const {
884 __asan::AsanChunk
*m
= reinterpret_cast<__asan::AsanChunk
*>(metadata_
);
885 return m
->chunk_state
== __asan::CHUNK_ALLOCATED
;
888 ChunkTag
LsanMetadata::tag() const {
889 __asan::AsanChunk
*m
= reinterpret_cast<__asan::AsanChunk
*>(metadata_
);
890 return static_cast<ChunkTag
>(m
->lsan_tag
);
893 void LsanMetadata::set_tag(ChunkTag value
) {
894 __asan::AsanChunk
*m
= reinterpret_cast<__asan::AsanChunk
*>(metadata_
);
898 uptr
LsanMetadata::requested_size() const {
899 __asan::AsanChunk
*m
= reinterpret_cast<__asan::AsanChunk
*>(metadata_
);
900 return m
->UsedSize(/*locked_version=*/true);
903 u32
LsanMetadata::stack_trace_id() const {
904 __asan::AsanChunk
*m
= reinterpret_cast<__asan::AsanChunk
*>(metadata_
);
905 return m
->alloc_context_id
;
908 void ForEachChunk(ForEachChunkCallback callback
, void *arg
) {
909 __asan::get_allocator().ForEachChunk(callback
, arg
);
912 IgnoreObjectResult
IgnoreObjectLocked(const void *p
) {
913 uptr addr
= reinterpret_cast<uptr
>(p
);
914 __asan::AsanChunk
*m
= __asan::instance
.GetAsanChunkByAddr(addr
);
915 if (!m
) return kIgnoreObjectInvalid
;
916 if ((m
->chunk_state
== __asan::CHUNK_ALLOCATED
) && m
->AddrIsInside(addr
)) {
917 if (m
->lsan_tag
== kIgnored
)
918 return kIgnoreObjectAlreadyIgnored
;
919 m
->lsan_tag
= __lsan::kIgnored
;
920 return kIgnoreObjectSuccess
;
922 return kIgnoreObjectInvalid
;
925 } // namespace __lsan
927 // ---------------------- Interface ---------------- {{{1
928 using namespace __asan
; // NOLINT
930 // ASan allocator doesn't reserve extra bytes, so normally we would
931 // just return "size". We don't want to expose our redzone sizes, etc here.
932 uptr
__sanitizer_get_estimated_allocated_size(uptr size
) {
936 int __sanitizer_get_ownership(const void *p
) {
937 uptr ptr
= reinterpret_cast<uptr
>(p
);
938 return instance
.AllocationSize(ptr
) > 0;
941 uptr
__sanitizer_get_allocated_size(const void *p
) {
943 uptr ptr
= reinterpret_cast<uptr
>(p
);
944 uptr allocated_size
= instance
.AllocationSize(ptr
);
945 // Die if p is not malloced or if it is already freed.
946 if (allocated_size
== 0) {
947 GET_STACK_TRACE_FATAL_HERE
;
948 ReportSanitizerGetAllocatedSizeNotOwned(ptr
, &stack
);
950 return allocated_size
;
953 #if !SANITIZER_SUPPORTS_WEAK_HOOKS
954 // Provide default (no-op) implementation of malloc hooks.
956 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
957 void __sanitizer_malloc_hook(void *ptr
, uptr size
) {
961 SANITIZER_INTERFACE_ATTRIBUTE SANITIZER_WEAK_ATTRIBUTE
962 void __sanitizer_free_hook(void *ptr
) {