1 //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
6 //===----------------------------------------------------------------------===//
8 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 // Main internal TSan header file.
13 // - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
14 // function-scope locals)
15 // - All functions/classes/etc reside in namespace __tsan, except for those
16 // declared in tsan_interface.h.
17 // - Platform-specific files should be used instead of ifdefs (*).
18 // - No system headers included in header files (*).
19 // - Platform specific headres included only into platform-specific files (*).
21 // (*) Except when inlining is critical for performance.
22 //===----------------------------------------------------------------------===//
27 #include "sanitizer_common/sanitizer_allocator.h"
28 #include "sanitizer_common/sanitizer_allocator_internal.h"
29 #include "sanitizer_common/sanitizer_asm.h"
30 #include "sanitizer_common/sanitizer_common.h"
31 #include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
32 #include "sanitizer_common/sanitizer_libignore.h"
33 #include "sanitizer_common/sanitizer_suppressions.h"
34 #include "sanitizer_common/sanitizer_thread_registry.h"
35 #include "tsan_clock.h"
36 #include "tsan_defs.h"
37 #include "tsan_flags.h"
38 #include "tsan_sync.h"
39 #include "tsan_trace.h"
40 #include "tsan_vector.h"
41 #include "tsan_report.h"
42 #include "tsan_platform.h"
43 #include "tsan_mutexset.h"
44 #include "tsan_ignoreset.h"
45 #include "tsan_stack_trace.h"
47 #if SANITIZER_WORDSIZE != 64
48 # error "ThreadSanitizer is supported only on 64-bit platforms"
54 struct MapUnmapCallback
;
55 #if defined(__mips64) || defined(__aarch64__)
56 static const uptr kAllocatorSpace
= 0;
57 static const uptr kAllocatorSize
= SANITIZER_MMAP_RANGE_SIZE
;
58 static const uptr kAllocatorRegionSizeLog
= 20;
59 static const uptr kAllocatorNumRegions
=
60 kAllocatorSize
>> kAllocatorRegionSizeLog
;
61 typedef TwoLevelByteMap
<(kAllocatorNumRegions
>> 12), 1 << 12,
62 MapUnmapCallback
> ByteMap
;
63 typedef SizeClassAllocator32
<kAllocatorSpace
, kAllocatorSize
, 0,
64 CompactSizeClassMap
, kAllocatorRegionSizeLog
, ByteMap
,
65 MapUnmapCallback
> PrimaryAllocator
;
67 typedef SizeClassAllocator64
<kHeapMemBeg
, kHeapMemEnd
- kHeapMemBeg
, 0,
68 DefaultSizeClassMap
, MapUnmapCallback
> PrimaryAllocator
;
70 typedef SizeClassAllocatorLocalCache
<PrimaryAllocator
> AllocatorCache
;
71 typedef LargeMmapAllocator
<MapUnmapCallback
> SecondaryAllocator
;
72 typedef CombinedAllocator
<PrimaryAllocator
, AllocatorCache
,
73 SecondaryAllocator
> Allocator
;
74 Allocator
*allocator();
77 void TsanCheckFailed(const char *file
, int line
, const char *cond
,
80 const u64 kShadowRodata
= (u64
)-1; // .rodata shadow marker
82 // FastState (from most significant bit):
90 FastState(u64 tid
, u64 epoch
) {
91 x_
= tid
<< kTidShift
;
93 DCHECK_EQ(tid
, this->tid());
94 DCHECK_EQ(epoch
, this->epoch());
95 DCHECK_EQ(GetIgnoreBit(), false);
98 explicit FastState(u64 x
)
107 u64 res
= (x_
& ~kIgnoreBit
) >> kTidShift
;
111 u64
TidWithIgnore() const {
112 u64 res
= x_
>> kTidShift
;
117 u64 res
= x_
& ((1ull << kClkBits
) - 1);
121 void IncrementEpoch() {
122 u64 old_epoch
= epoch();
124 DCHECK_EQ(old_epoch
+ 1, epoch());
128 void SetIgnoreBit() { x_
|= kIgnoreBit
; }
129 void ClearIgnoreBit() { x_
&= ~kIgnoreBit
; }
130 bool GetIgnoreBit() const { return (s64
)x_
< 0; }
132 void SetHistorySize(int hs
) {
135 x_
= (x_
& ~(kHistoryMask
<< kHistoryShift
)) | (u64(hs
) << kHistoryShift
);
139 int GetHistorySize() const {
140 return (int)((x_
>> kHistoryShift
) & kHistoryMask
);
143 void ClearHistorySize() {
148 u64
GetTracePos() const {
149 const int hs
= GetHistorySize();
150 // When hs == 0, the trace consists of 2 parts.
151 const u64 mask
= (1ull << (kTracePartSizeBits
+ hs
+ 1)) - 1;
152 return epoch() & mask
;
157 static const int kTidShift
= 64 - kTidBits
- 1;
158 static const u64 kIgnoreBit
= 1ull << 63;
159 static const u64 kFreedBit
= 1ull << 63;
160 static const u64 kHistoryShift
= kClkBits
;
161 static const u64 kHistoryMask
= 7;
165 // Shadow (from most significant bit):
173 class Shadow
: public FastState
{
175 explicit Shadow(u64 x
)
179 explicit Shadow(const FastState
&s
)
184 void SetAddr0AndSizeLog(u64 addr0
, unsigned kAccessSizeLog
) {
185 DCHECK_EQ((x_
>> kClkBits
) & 31, 0);
187 DCHECK_LE(kAccessSizeLog
, 3);
188 x_
|= ((kAccessSizeLog
<< 3) | addr0
) << kClkBits
;
189 DCHECK_EQ(kAccessSizeLog
, size_log());
190 DCHECK_EQ(addr0
, this->addr0());
193 void SetWrite(unsigned kAccessIsWrite
) {
194 DCHECK_EQ(x_
& kReadBit
, 0);
197 DCHECK_EQ(kAccessIsWrite
, IsWrite());
200 void SetAtomic(bool kIsAtomic
) {
204 DCHECK_EQ(IsAtomic(), kIsAtomic
);
207 bool IsAtomic() const {
208 return x_
& kAtomicBit
;
211 bool IsZero() const {
215 static inline bool TidsAreEqual(const Shadow s1
, const Shadow s2
) {
216 u64 shifted_xor
= (s1
.x_
^ s2
.x_
) >> kTidShift
;
217 DCHECK_EQ(shifted_xor
== 0, s1
.TidWithIgnore() == s2
.TidWithIgnore());
218 return shifted_xor
== 0;
222 bool Addr0AndSizeAreEqual(const Shadow s1
, const Shadow s2
) {
223 u64 masked_xor
= ((s1
.x_
^ s2
.x_
) >> kClkBits
) & 31;
224 return masked_xor
== 0;
227 static ALWAYS_INLINE
bool TwoRangesIntersect(Shadow s1
, Shadow s2
,
228 unsigned kS2AccessSize
) {
230 u64 diff
= s1
.addr0() - s2
.addr0();
231 if ((s64
)diff
< 0) { // s1.addr0 < s2.addr0 // NOLINT
232 // if (s1.addr0() + size1) > s2.addr0()) return true;
233 if (s1
.size() > -diff
)
236 // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
237 if (kS2AccessSize
> diff
)
240 DCHECK_EQ(res
, TwoRangesIntersectSlow(s1
, s2
));
241 DCHECK_EQ(res
, TwoRangesIntersectSlow(s2
, s1
));
245 u64 ALWAYS_INLINE
addr0() const { return (x_
>> kClkBits
) & 7; }
246 u64 ALWAYS_INLINE
size() const { return 1ull << size_log(); }
247 bool ALWAYS_INLINE
IsWrite() const { return !IsRead(); }
248 bool ALWAYS_INLINE
IsRead() const { return x_
& kReadBit
; }
250 // The idea behind the freed bit is as follows.
251 // When the memory is freed (or otherwise unaccessible) we write to the shadow
252 // values with tid/epoch related to the free and the freed bit set.
253 // During memory accesses processing the freed bit is considered
254 // as msb of tid. So any access races with shadow with freed bit set
255 // (it is as if write from a thread with which we never synchronized before).
256 // This allows us to detect accesses to freed memory w/o additional
257 // overheads in memory access processing and at the same time restore
258 // tid/epoch of free.
263 bool IsFreed() const {
264 return x_
& kFreedBit
;
267 bool GetFreedAndReset() {
268 bool res
= x_
& kFreedBit
;
273 bool ALWAYS_INLINE
IsBothReadsOrAtomic(bool kIsWrite
, bool kIsAtomic
) const {
274 bool v
= x_
& ((u64(kIsWrite
^ 1) << kReadShift
)
275 | (u64(kIsAtomic
) << kAtomicShift
));
276 DCHECK_EQ(v
, (!IsWrite() && !kIsWrite
) || (IsAtomic() && kIsAtomic
));
280 bool ALWAYS_INLINE
IsRWNotWeaker(bool kIsWrite
, bool kIsAtomic
) const {
281 bool v
= ((x_
>> kReadShift
) & 3)
282 <= u64((kIsWrite
^ 1) | (kIsAtomic
<< 1));
283 DCHECK_EQ(v
, (IsAtomic() < kIsAtomic
) ||
284 (IsAtomic() == kIsAtomic
&& !IsWrite() <= !kIsWrite
));
288 bool ALWAYS_INLINE
IsRWWeakerOrEqual(bool kIsWrite
, bool kIsAtomic
) const {
289 bool v
= ((x_
>> kReadShift
) & 3)
290 >= u64((kIsWrite
^ 1) | (kIsAtomic
<< 1));
291 DCHECK_EQ(v
, (IsAtomic() > kIsAtomic
) ||
292 (IsAtomic() == kIsAtomic
&& !IsWrite() >= !kIsWrite
));
297 static const u64 kReadShift
= 5 + kClkBits
;
298 static const u64 kReadBit
= 1ull << kReadShift
;
299 static const u64 kAtomicShift
= 6 + kClkBits
;
300 static const u64 kAtomicBit
= 1ull << kAtomicShift
;
302 u64
size_log() const { return (x_
>> (3 + kClkBits
)) & 3; }
304 static bool TwoRangesIntersectSlow(const Shadow s1
, const Shadow s2
) {
305 if (s1
.addr0() == s2
.addr0()) return true;
306 if (s1
.addr0() < s2
.addr0() && s1
.addr0() + s1
.size() > s2
.addr0())
308 if (s2
.addr0() < s1
.addr0() && s2
.addr0() + s2
.size() > s1
.addr0())
314 struct ThreadSignalContext
;
320 bool in_blocking_func
;
321 uptr in_signal_handler
;
322 uptr
*shadow_stack_pos
;
325 // This struct is stored in TLS.
327 FastState fast_state
;
328 // Synch epoch represents the threads's epoch before the last synchronization
329 // action. It allows to reduce number of shadow state updates.
330 // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
331 // if we are processing write to X from the same thread at epoch=200,
332 // we do nothing, because both writes happen in the same 'synch epoch'.
333 // That is, if another memory access does not race with the former write,
334 // it does not race with the latter as well.
335 // QUESTION: can we can squeeze this into ThreadState::Fast?
336 // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
337 // taken by epoch between synchs.
338 // This way we can save one load from tls.
339 u64 fast_synch_epoch
;
340 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
341 // We do not distinguish beteween ignoring reads and writes
342 // for better performance.
343 int ignore_reads_and_writes
;
345 // Go does not support ignores.
347 IgnoreSet mop_ignore_set
;
348 IgnoreSet sync_ignore_set
;
350 // C/C++ uses fixed size shadow stack embed into Trace.
351 // Go uses malloc-allocated shadow stack with dynamic size.
353 uptr
*shadow_stack_end
;
354 uptr
*shadow_stack_pos
;
355 u64
*racy_shadow_addr
;
360 AllocatorCache alloc_cache
;
361 InternalAllocatorCache internal_alloc_cache
;
362 Vector
<JmpBuf
> jmp_bufs
;
363 int ignore_interceptors
;
365 #if TSAN_COLLECT_STATS
382 #if SANITIZER_DEBUG && !SANITIZER_GO
383 InternalDeadlockDetector internal_deadlock_detector
;
385 DDPhysicalThread
*dd_pt
;
386 DDLogicalThread
*dd_lt
;
388 atomic_uintptr_t in_signal_handler
;
389 ThreadSignalContext
*signal_ctx
;
391 DenseSlabAllocCache block_cache
;
392 DenseSlabAllocCache sync_cache
;
393 DenseSlabAllocCache clock_cache
;
396 u32 last_sleep_stack_id
;
397 ThreadClock last_sleep_clock
;
400 // Set in regions of runtime that must be signal-safe and fork-safe.
401 // If set, malloc must not be called.
404 explicit ThreadState(Context
*ctx
, int tid
, int unique_id
, u64 epoch
,
405 unsigned reuse_count
,
406 uptr stk_addr
, uptr stk_size
,
407 uptr tls_addr
, uptr tls_size
);
412 ThreadState
*cur_thread();
413 void cur_thread_finalize();
415 __attribute__((tls_model("initial-exec")))
416 extern THREADLOCAL
char cur_thread_placeholder
[];
417 INLINE ThreadState
*cur_thread() {
418 return reinterpret_cast<ThreadState
*>(&cur_thread_placeholder
);
420 INLINE
void cur_thread_finalize() { }
421 #endif // SANITIZER_MAC
422 #endif // SANITIZER_GO
424 class ThreadContext
: public ThreadContextBase
{
426 explicit ThreadContext(int tid
);
429 u32 creation_stack_id
;
431 // Epoch at which the thread had started.
432 // If we see an event from the thread stamped by an older epoch,
433 // the event is from a dead thread that shared tid with this thread.
437 // Override superclass callbacks.
438 void OnDead() override
;
439 void OnJoined(void *arg
) override
;
440 void OnFinished() override
;
441 void OnStarted(void *arg
) override
;
442 void OnCreated(void *arg
) override
;
443 void OnReset() override
;
444 void OnDetached(void *arg
) override
;
449 bool operator==(const RacyStacks
&other
) const {
450 if (hash
[0] == other
.hash
[0] && hash
[1] == other
.hash
[1])
452 if (hash
[0] == other
.hash
[1] && hash
[1] == other
.hash
[0])
463 struct FiredSuppression
{
473 bool after_multithreaded_fork
;
479 int nmissed_expected
;
480 atomic_uint64_t last_symbolize_time_ns
;
482 void *background_thread
;
483 atomic_uint32_t stop_background_thread
;
485 ThreadRegistry
*thread_registry
;
488 Vector
<RacyStacks
> racy_stacks
;
489 Vector
<RacyAddress
> racy_addresses
;
490 // Number of fired suppressions may be large enough.
491 Mutex fired_suppressions_mtx
;
492 InternalMmapVector
<FiredSuppression
> fired_suppressions
;
495 ClockAlloc clock_alloc
;
500 u64 int_alloc_cnt
[MBlockTypeCount
];
501 u64 int_alloc_siz
[MBlockTypeCount
];
504 extern Context
*ctx
; // The one and the only global runtime context.
506 struct ScopedIgnoreInterceptors
{
507 ScopedIgnoreInterceptors() {
509 cur_thread()->ignore_interceptors
++;
513 ~ScopedIgnoreInterceptors() {
515 cur_thread()->ignore_interceptors
--;
522 explicit ScopedReport(ReportType typ
);
525 void AddMemoryAccess(uptr addr
, Shadow s
, StackTrace stack
,
526 const MutexSet
*mset
);
527 void AddStack(StackTrace stack
, bool suppressable
= false);
528 void AddThread(const ThreadContext
*tctx
, bool suppressable
= false);
529 void AddThread(int unique_tid
, bool suppressable
= false);
530 void AddUniqueTid(int unique_tid
);
531 void AddMutex(const SyncVar
*s
);
532 u64
AddMutex(u64 id
);
533 void AddLocation(uptr addr
, uptr size
);
534 void AddSleep(u32 stack_id
);
535 void SetCount(int count
);
537 const ReportDesc
*GetReport() const;
541 // Symbolizer makes lots of intercepted calls. If we try to process them,
542 // at best it will cause deadlocks on internal mutexes.
543 ScopedIgnoreInterceptors ignore_interceptors_
;
545 void AddDeadMutex(u64 id
);
547 ScopedReport(const ScopedReport
&);
548 void operator = (const ScopedReport
&);
551 void RestoreStack(int tid
, const u64 epoch
, VarSizeStackTrace
*stk
,
554 template<typename StackTraceTy
>
555 void ObtainCurrentStack(ThreadState
*thr
, uptr toppc
, StackTraceTy
*stack
) {
556 uptr size
= thr
->shadow_stack_pos
- thr
->shadow_stack
;
558 if (size
+ !!toppc
> kStackTraceMax
) {
559 start
= size
+ !!toppc
- kStackTraceMax
;
560 size
= kStackTraceMax
- !!toppc
;
562 stack
->Init(&thr
->shadow_stack
[start
], size
, toppc
);
566 #if TSAN_COLLECT_STATS
567 void StatAggregate(u64
*dst
, u64
*src
);
568 void StatOutput(u64
*stat
);
571 void ALWAYS_INLINE
StatInc(ThreadState
*thr
, StatType typ
, u64 n
= 1) {
572 #if TSAN_COLLECT_STATS
576 void ALWAYS_INLINE
StatSet(ThreadState
*thr
, StatType typ
, u64 n
) {
577 #if TSAN_COLLECT_STATS
582 void MapShadow(uptr addr
, uptr size
);
583 void MapThreadTrace(uptr addr
, uptr size
, const char *name
);
584 void DontNeedShadowFor(uptr addr
, uptr size
);
585 void InitializeShadowMemory();
586 void InitializeInterceptors();
587 void InitializeLibIgnore();
588 void InitializeDynamicAnnotations();
590 void ForkBefore(ThreadState
*thr
, uptr pc
);
591 void ForkParentAfter(ThreadState
*thr
, uptr pc
);
592 void ForkChildAfter(ThreadState
*thr
, uptr pc
);
594 void ReportRace(ThreadState
*thr
);
595 bool OutputReport(ThreadState
*thr
, const ScopedReport
&srep
);
596 bool IsFiredSuppression(Context
*ctx
, ReportType type
, StackTrace trace
);
597 bool IsExpectedReport(uptr addr
, uptr size
);
598 void PrintMatchedBenignRaces();
600 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
601 # define DPrintf Printf
603 # define DPrintf(...)
606 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
607 # define DPrintf2 Printf
609 # define DPrintf2(...)
612 u32
CurrentStackId(ThreadState
*thr
, uptr pc
);
613 ReportStack
*SymbolizeStackId(u32 stack_id
);
614 void PrintCurrentStack(ThreadState
*thr
, uptr pc
);
615 void PrintCurrentStackSlow(uptr pc
); // uses libunwind
617 void Initialize(ThreadState
*thr
);
618 int Finalize(ThreadState
*thr
);
620 void OnUserAlloc(ThreadState
*thr
, uptr pc
, uptr p
, uptr sz
, bool write
);
621 void OnUserFree(ThreadState
*thr
, uptr pc
, uptr p
, bool write
);
623 void MemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
624 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
);
625 void MemoryAccessImpl(ThreadState
*thr
, uptr addr
,
626 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
,
627 u64
*shadow_mem
, Shadow cur
);
628 void MemoryAccessRange(ThreadState
*thr
, uptr pc
, uptr addr
,
629 uptr size
, bool is_write
);
630 void MemoryAccessRangeStep(ThreadState
*thr
, uptr pc
, uptr addr
,
631 uptr size
, uptr step
, bool is_write
);
632 void UnalignedMemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
633 int size
, bool kAccessIsWrite
, bool kIsAtomic
);
635 const int kSizeLog1
= 0;
636 const int kSizeLog2
= 1;
637 const int kSizeLog4
= 2;
638 const int kSizeLog8
= 3;
640 void ALWAYS_INLINE
MemoryRead(ThreadState
*thr
, uptr pc
,
641 uptr addr
, int kAccessSizeLog
) {
642 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, false, false);
645 void ALWAYS_INLINE
MemoryWrite(ThreadState
*thr
, uptr pc
,
646 uptr addr
, int kAccessSizeLog
) {
647 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, true, false);
650 void ALWAYS_INLINE
MemoryReadAtomic(ThreadState
*thr
, uptr pc
,
651 uptr addr
, int kAccessSizeLog
) {
652 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, false, true);
655 void ALWAYS_INLINE
MemoryWriteAtomic(ThreadState
*thr
, uptr pc
,
656 uptr addr
, int kAccessSizeLog
) {
657 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, true, true);
660 void MemoryResetRange(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
661 void MemoryRangeFreed(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
662 void MemoryRangeImitateWrite(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
664 void ThreadIgnoreBegin(ThreadState
*thr
, uptr pc
);
665 void ThreadIgnoreEnd(ThreadState
*thr
, uptr pc
);
666 void ThreadIgnoreSyncBegin(ThreadState
*thr
, uptr pc
);
667 void ThreadIgnoreSyncEnd(ThreadState
*thr
, uptr pc
);
669 void FuncEntry(ThreadState
*thr
, uptr pc
);
670 void FuncExit(ThreadState
*thr
);
672 int ThreadCreate(ThreadState
*thr
, uptr pc
, uptr uid
, bool detached
);
673 void ThreadStart(ThreadState
*thr
, int tid
, uptr os_id
);
674 void ThreadFinish(ThreadState
*thr
);
675 int ThreadTid(ThreadState
*thr
, uptr pc
, uptr uid
);
676 void ThreadJoin(ThreadState
*thr
, uptr pc
, int tid
);
677 void ThreadDetach(ThreadState
*thr
, uptr pc
, int tid
);
678 void ThreadFinalize(ThreadState
*thr
);
679 void ThreadSetName(ThreadState
*thr
, const char *name
);
680 int ThreadCount(ThreadState
*thr
);
681 void ProcessPendingSignals(ThreadState
*thr
);
683 void MutexCreate(ThreadState
*thr
, uptr pc
, uptr addr
,
684 bool rw
, bool recursive
, bool linker_init
);
685 void MutexDestroy(ThreadState
*thr
, uptr pc
, uptr addr
);
686 void MutexLock(ThreadState
*thr
, uptr pc
, uptr addr
, int rec
= 1,
687 bool try_lock
= false);
688 int MutexUnlock(ThreadState
*thr
, uptr pc
, uptr addr
, bool all
= false);
689 void MutexReadLock(ThreadState
*thr
, uptr pc
, uptr addr
, bool try_lock
= false);
690 void MutexReadUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
691 void MutexReadOrWriteUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
692 void MutexRepair(ThreadState
*thr
, uptr pc
, uptr addr
); // call on EOWNERDEAD
694 void Acquire(ThreadState
*thr
, uptr pc
, uptr addr
);
695 // AcquireGlobal synchronizes the current thread with all other threads.
696 // In terms of happens-before relation, it draws a HB edge from all threads
697 // (where they happen to execute right now) to the current thread. We use it to
698 // handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
699 // right before executing finalizers. This provides a coarse, but simple
700 // approximation of the actual required synchronization.
701 void AcquireGlobal(ThreadState
*thr
, uptr pc
);
702 void Release(ThreadState
*thr
, uptr pc
, uptr addr
);
703 void ReleaseStore(ThreadState
*thr
, uptr pc
, uptr addr
);
704 void AfterSleep(ThreadState
*thr
, uptr pc
);
705 void AcquireImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
706 void ReleaseImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
707 void ReleaseStoreImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
708 void AcquireReleaseImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
710 // The hacky call uses custom calling convention and an assembly thunk.
711 // It is considerably faster that a normal call for the caller
712 // if it is not executed (it is intended for slow paths from hot functions).
713 // The trick is that the call preserves all registers and the compiler
714 // does not treat it as a call.
715 // If it does not work for you, use normal call.
716 #if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC
717 // The caller may not create the stack frame for itself at all,
718 // so we create a reserve stack frame for it (1024b must be enough).
719 #define HACKY_CALL(f) \
720 __asm__ __volatile__("sub $1024, %%rsp;" \
721 CFI_INL_ADJUST_CFA_OFFSET(1024) \
722 ".hidden " #f "_thunk;" \
723 "call " #f "_thunk;" \
724 "add $1024, %%rsp;" \
725 CFI_INL_ADJUST_CFA_OFFSET(-1024) \
728 #define HACKY_CALL(f) f()
731 void TraceSwitch(ThreadState
*thr
);
732 uptr
TraceTopPC(ThreadState
*thr
);
735 Trace
*ThreadTrace(int tid
);
737 extern "C" void __tsan_trace_switch();
738 void ALWAYS_INLINE
TraceAddEvent(ThreadState
*thr
, FastState fs
,
739 EventType typ
, u64 addr
) {
740 if (!kCollectHistory
)
742 DCHECK_GE((int)typ
, 0);
743 DCHECK_LE((int)typ
, 7);
744 DCHECK_EQ(GetLsb(addr
, 61), addr
);
745 StatInc(thr
, StatEvents
);
746 u64 pos
= fs
.GetTracePos();
747 if (UNLIKELY((pos
% kTracePartSize
) == 0)) {
749 HACKY_CALL(__tsan_trace_switch
);
754 Event
*trace
= (Event
*)GetThreadTrace(fs
.tid());
755 Event
*evp
= &trace
[pos
];
756 Event ev
= (u64
)addr
| ((u64
)typ
<< 61);
761 uptr ALWAYS_INLINE
HeapEnd() {
762 return kHeapMemEnd
+ PrimaryAllocator::AdditionalSize();
766 } // namespace __tsan