1 //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
6 //===----------------------------------------------------------------------===//
8 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 // Main internal TSan header file.
13 // - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
14 // function-scope locals)
15 // - All functions/classes/etc reside in namespace __tsan, except for those
16 // declared in tsan_interface.h.
17 // - Platform-specific files should be used instead of ifdefs (*).
18 // - No system headers included in header files (*).
19 // - Platform specific headres included only into platform-specific files (*).
21 // (*) Except when inlining is critical for performance.
22 //===----------------------------------------------------------------------===//
27 #include "sanitizer_common/sanitizer_allocator.h"
28 #include "sanitizer_common/sanitizer_allocator_internal.h"
29 #include "sanitizer_common/sanitizer_asm.h"
30 #include "sanitizer_common/sanitizer_common.h"
31 #include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
32 #include "sanitizer_common/sanitizer_libignore.h"
33 #include "sanitizer_common/sanitizer_suppressions.h"
34 #include "sanitizer_common/sanitizer_thread_registry.h"
35 #include "tsan_clock.h"
36 #include "tsan_defs.h"
37 #include "tsan_flags.h"
38 #include "tsan_sync.h"
39 #include "tsan_trace.h"
40 #include "tsan_vector.h"
41 #include "tsan_report.h"
42 #include "tsan_platform.h"
43 #include "tsan_mutexset.h"
44 #include "tsan_ignoreset.h"
45 #include "tsan_stack_trace.h"
47 #if SANITIZER_WORDSIZE != 64
48 # error "ThreadSanitizer is supported only on 64-bit platforms"
54 struct MapUnmapCallback
;
55 #if defined(__mips64) || defined(__aarch64__)
56 static const uptr kAllocatorSpace
= 0;
57 static const uptr kAllocatorSize
= SANITIZER_MMAP_RANGE_SIZE
;
58 static const uptr kAllocatorRegionSizeLog
= 20;
59 static const uptr kAllocatorNumRegions
=
60 kAllocatorSize
>> kAllocatorRegionSizeLog
;
61 typedef TwoLevelByteMap
<(kAllocatorNumRegions
>> 12), 1 << 12,
62 MapUnmapCallback
> ByteMap
;
63 typedef SizeClassAllocator32
<kAllocatorSpace
, kAllocatorSize
, 0,
64 CompactSizeClassMap
, kAllocatorRegionSizeLog
, ByteMap
,
65 MapUnmapCallback
> PrimaryAllocator
;
67 typedef SizeClassAllocator64
<kHeapMemBeg
, kHeapMemEnd
- kHeapMemBeg
, 0,
68 DefaultSizeClassMap
, MapUnmapCallback
> PrimaryAllocator
;
70 typedef SizeClassAllocatorLocalCache
<PrimaryAllocator
> AllocatorCache
;
71 typedef LargeMmapAllocator
<MapUnmapCallback
> SecondaryAllocator
;
72 typedef CombinedAllocator
<PrimaryAllocator
, AllocatorCache
,
73 SecondaryAllocator
> Allocator
;
74 Allocator
*allocator();
77 void TsanCheckFailed(const char *file
, int line
, const char *cond
,
80 const u64 kShadowRodata
= (u64
)-1; // .rodata shadow marker
82 // FastState (from most significant bit):
90 FastState(u64 tid
, u64 epoch
) {
91 x_
= tid
<< kTidShift
;
93 DCHECK_EQ(tid
, this->tid());
94 DCHECK_EQ(epoch
, this->epoch());
95 DCHECK_EQ(GetIgnoreBit(), false);
98 explicit FastState(u64 x
)
107 u64 res
= (x_
& ~kIgnoreBit
) >> kTidShift
;
111 u64
TidWithIgnore() const {
112 u64 res
= x_
>> kTidShift
;
117 u64 res
= x_
& ((1ull << kClkBits
) - 1);
121 void IncrementEpoch() {
122 u64 old_epoch
= epoch();
124 DCHECK_EQ(old_epoch
+ 1, epoch());
128 void SetIgnoreBit() { x_
|= kIgnoreBit
; }
129 void ClearIgnoreBit() { x_
&= ~kIgnoreBit
; }
130 bool GetIgnoreBit() const { return (s64
)x_
< 0; }
132 void SetHistorySize(int hs
) {
135 x_
= (x_
& ~(kHistoryMask
<< kHistoryShift
)) | (u64(hs
) << kHistoryShift
);
139 int GetHistorySize() const {
140 return (int)((x_
>> kHistoryShift
) & kHistoryMask
);
143 void ClearHistorySize() {
148 u64
GetTracePos() const {
149 const int hs
= GetHistorySize();
150 // When hs == 0, the trace consists of 2 parts.
151 const u64 mask
= (1ull << (kTracePartSizeBits
+ hs
+ 1)) - 1;
152 return epoch() & mask
;
157 static const int kTidShift
= 64 - kTidBits
- 1;
158 static const u64 kIgnoreBit
= 1ull << 63;
159 static const u64 kFreedBit
= 1ull << 63;
160 static const u64 kHistoryShift
= kClkBits
;
161 static const u64 kHistoryMask
= 7;
165 // Shadow (from most significant bit):
173 class Shadow
: public FastState
{
175 explicit Shadow(u64 x
)
179 explicit Shadow(const FastState
&s
)
184 void SetAddr0AndSizeLog(u64 addr0
, unsigned kAccessSizeLog
) {
185 DCHECK_EQ((x_
>> kClkBits
) & 31, 0);
187 DCHECK_LE(kAccessSizeLog
, 3);
188 x_
|= ((kAccessSizeLog
<< 3) | addr0
) << kClkBits
;
189 DCHECK_EQ(kAccessSizeLog
, size_log());
190 DCHECK_EQ(addr0
, this->addr0());
193 void SetWrite(unsigned kAccessIsWrite
) {
194 DCHECK_EQ(x_
& kReadBit
, 0);
197 DCHECK_EQ(kAccessIsWrite
, IsWrite());
200 void SetAtomic(bool kIsAtomic
) {
204 DCHECK_EQ(IsAtomic(), kIsAtomic
);
207 bool IsAtomic() const {
208 return x_
& kAtomicBit
;
211 bool IsZero() const {
215 static inline bool TidsAreEqual(const Shadow s1
, const Shadow s2
) {
216 u64 shifted_xor
= (s1
.x_
^ s2
.x_
) >> kTidShift
;
217 DCHECK_EQ(shifted_xor
== 0, s1
.TidWithIgnore() == s2
.TidWithIgnore());
218 return shifted_xor
== 0;
222 bool Addr0AndSizeAreEqual(const Shadow s1
, const Shadow s2
) {
223 u64 masked_xor
= ((s1
.x_
^ s2
.x_
) >> kClkBits
) & 31;
224 return masked_xor
== 0;
227 static ALWAYS_INLINE
bool TwoRangesIntersect(Shadow s1
, Shadow s2
,
228 unsigned kS2AccessSize
) {
230 u64 diff
= s1
.addr0() - s2
.addr0();
231 if ((s64
)diff
< 0) { // s1.addr0 < s2.addr0 // NOLINT
232 // if (s1.addr0() + size1) > s2.addr0()) return true;
233 if (s1
.size() > -diff
)
236 // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
237 if (kS2AccessSize
> diff
)
240 DCHECK_EQ(res
, TwoRangesIntersectSlow(s1
, s2
));
241 DCHECK_EQ(res
, TwoRangesIntersectSlow(s2
, s1
));
245 u64 ALWAYS_INLINE
addr0() const { return (x_
>> kClkBits
) & 7; }
246 u64 ALWAYS_INLINE
size() const { return 1ull << size_log(); }
247 bool ALWAYS_INLINE
IsWrite() const { return !IsRead(); }
248 bool ALWAYS_INLINE
IsRead() const { return x_
& kReadBit
; }
250 // The idea behind the freed bit is as follows.
251 // When the memory is freed (or otherwise unaccessible) we write to the shadow
252 // values with tid/epoch related to the free and the freed bit set.
253 // During memory accesses processing the freed bit is considered
254 // as msb of tid. So any access races with shadow with freed bit set
255 // (it is as if write from a thread with which we never synchronized before).
256 // This allows us to detect accesses to freed memory w/o additional
257 // overheads in memory access processing and at the same time restore
258 // tid/epoch of free.
263 bool IsFreed() const {
264 return x_
& kFreedBit
;
267 bool GetFreedAndReset() {
268 bool res
= x_
& kFreedBit
;
273 bool ALWAYS_INLINE
IsBothReadsOrAtomic(bool kIsWrite
, bool kIsAtomic
) const {
274 bool v
= x_
& ((u64(kIsWrite
^ 1) << kReadShift
)
275 | (u64(kIsAtomic
) << kAtomicShift
));
276 DCHECK_EQ(v
, (!IsWrite() && !kIsWrite
) || (IsAtomic() && kIsAtomic
));
280 bool ALWAYS_INLINE
IsRWNotWeaker(bool kIsWrite
, bool kIsAtomic
) const {
281 bool v
= ((x_
>> kReadShift
) & 3)
282 <= u64((kIsWrite
^ 1) | (kIsAtomic
<< 1));
283 DCHECK_EQ(v
, (IsAtomic() < kIsAtomic
) ||
284 (IsAtomic() == kIsAtomic
&& !IsWrite() <= !kIsWrite
));
288 bool ALWAYS_INLINE
IsRWWeakerOrEqual(bool kIsWrite
, bool kIsAtomic
) const {
289 bool v
= ((x_
>> kReadShift
) & 3)
290 >= u64((kIsWrite
^ 1) | (kIsAtomic
<< 1));
291 DCHECK_EQ(v
, (IsAtomic() > kIsAtomic
) ||
292 (IsAtomic() == kIsAtomic
&& !IsWrite() >= !kIsWrite
));
297 static const u64 kReadShift
= 5 + kClkBits
;
298 static const u64 kReadBit
= 1ull << kReadShift
;
299 static const u64 kAtomicShift
= 6 + kClkBits
;
300 static const u64 kAtomicBit
= 1ull << kAtomicShift
;
302 u64
size_log() const { return (x_
>> (3 + kClkBits
)) & 3; }
304 static bool TwoRangesIntersectSlow(const Shadow s1
, const Shadow s2
) {
305 if (s1
.addr0() == s2
.addr0()) return true;
306 if (s1
.addr0() < s2
.addr0() && s1
.addr0() + s1
.size() > s2
.addr0())
308 if (s2
.addr0() < s1
.addr0() && s2
.addr0() + s2
.size() > s1
.addr0())
314 struct ThreadSignalContext
;
320 bool in_blocking_func
;
321 uptr in_signal_handler
;
322 uptr
*shadow_stack_pos
;
325 // This struct is stored in TLS.
327 FastState fast_state
;
328 // Synch epoch represents the threads's epoch before the last synchronization
329 // action. It allows to reduce number of shadow state updates.
330 // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
331 // if we are processing write to X from the same thread at epoch=200,
332 // we do nothing, because both writes happen in the same 'synch epoch'.
333 // That is, if another memory access does not race with the former write,
334 // it does not race with the latter as well.
335 // QUESTION: can we can squeeze this into ThreadState::Fast?
336 // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
337 // taken by epoch between synchs.
338 // This way we can save one load from tls.
339 u64 fast_synch_epoch
;
340 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
341 // We do not distinguish beteween ignoring reads and writes
342 // for better performance.
343 int ignore_reads_and_writes
;
345 // Go does not support ignores.
347 IgnoreSet mop_ignore_set
;
348 IgnoreSet sync_ignore_set
;
350 // C/C++ uses fixed size shadow stack embed into Trace.
351 // Go uses malloc-allocated shadow stack with dynamic size.
353 uptr
*shadow_stack_end
;
354 uptr
*shadow_stack_pos
;
355 u64
*racy_shadow_addr
;
360 AllocatorCache alloc_cache
;
361 InternalAllocatorCache internal_alloc_cache
;
362 Vector
<JmpBuf
> jmp_bufs
;
363 int ignore_interceptors
;
365 #if TSAN_COLLECT_STATS
382 #if SANITIZER_DEBUG && !SANITIZER_GO
383 InternalDeadlockDetector internal_deadlock_detector
;
385 DDPhysicalThread
*dd_pt
;
386 DDLogicalThread
*dd_lt
;
388 atomic_uintptr_t in_signal_handler
;
389 ThreadSignalContext
*signal_ctx
;
391 DenseSlabAllocCache block_cache
;
392 DenseSlabAllocCache sync_cache
;
393 DenseSlabAllocCache clock_cache
;
396 u32 last_sleep_stack_id
;
397 ThreadClock last_sleep_clock
;
400 // Set in regions of runtime that must be signal-safe and fork-safe.
401 // If set, malloc must not be called.
404 explicit ThreadState(Context
*ctx
, int tid
, int unique_id
, u64 epoch
,
405 unsigned reuse_count
,
406 uptr stk_addr
, uptr stk_size
,
407 uptr tls_addr
, uptr tls_size
);
411 __attribute__((tls_model("initial-exec")))
412 extern THREADLOCAL
char cur_thread_placeholder
[];
413 INLINE ThreadState
*cur_thread() {
414 return reinterpret_cast<ThreadState
*>(&cur_thread_placeholder
);
418 class ThreadContext
: public ThreadContextBase
{
420 explicit ThreadContext(int tid
);
423 u32 creation_stack_id
;
425 // Epoch at which the thread had started.
426 // If we see an event from the thread stamped by an older epoch,
427 // the event is from a dead thread that shared tid with this thread.
431 // Override superclass callbacks.
432 void OnDead() override
;
433 void OnJoined(void *arg
) override
;
434 void OnFinished() override
;
435 void OnStarted(void *arg
) override
;
436 void OnCreated(void *arg
) override
;
437 void OnReset() override
;
438 void OnDetached(void *arg
) override
;
443 bool operator==(const RacyStacks
&other
) const {
444 if (hash
[0] == other
.hash
[0] && hash
[1] == other
.hash
[1])
446 if (hash
[0] == other
.hash
[1] && hash
[1] == other
.hash
[0])
457 struct FiredSuppression
{
467 bool after_multithreaded_fork
;
473 int nmissed_expected
;
474 atomic_uint64_t last_symbolize_time_ns
;
476 void *background_thread
;
477 atomic_uint32_t stop_background_thread
;
479 ThreadRegistry
*thread_registry
;
482 Vector
<RacyStacks
> racy_stacks
;
483 Vector
<RacyAddress
> racy_addresses
;
484 // Number of fired suppressions may be large enough.
485 Mutex fired_suppressions_mtx
;
486 InternalMmapVector
<FiredSuppression
> fired_suppressions
;
489 ClockAlloc clock_alloc
;
494 u64 int_alloc_cnt
[MBlockTypeCount
];
495 u64 int_alloc_siz
[MBlockTypeCount
];
498 extern Context
*ctx
; // The one and the only global runtime context.
500 struct ScopedIgnoreInterceptors
{
501 ScopedIgnoreInterceptors() {
503 cur_thread()->ignore_interceptors
++;
507 ~ScopedIgnoreInterceptors() {
509 cur_thread()->ignore_interceptors
--;
516 explicit ScopedReport(ReportType typ
);
519 void AddMemoryAccess(uptr addr
, Shadow s
, StackTrace stack
,
520 const MutexSet
*mset
);
521 void AddStack(StackTrace stack
, bool suppressable
= false);
522 void AddThread(const ThreadContext
*tctx
, bool suppressable
= false);
523 void AddThread(int unique_tid
, bool suppressable
= false);
524 void AddUniqueTid(int unique_tid
);
525 void AddMutex(const SyncVar
*s
);
526 u64
AddMutex(u64 id
);
527 void AddLocation(uptr addr
, uptr size
);
528 void AddSleep(u32 stack_id
);
529 void SetCount(int count
);
531 const ReportDesc
*GetReport() const;
535 // Symbolizer makes lots of intercepted calls. If we try to process them,
536 // at best it will cause deadlocks on internal mutexes.
537 ScopedIgnoreInterceptors ignore_interceptors_
;
539 void AddDeadMutex(u64 id
);
541 ScopedReport(const ScopedReport
&);
542 void operator = (const ScopedReport
&);
545 void RestoreStack(int tid
, const u64 epoch
, VarSizeStackTrace
*stk
,
548 template<typename StackTraceTy
>
549 void ObtainCurrentStack(ThreadState
*thr
, uptr toppc
, StackTraceTy
*stack
) {
550 uptr size
= thr
->shadow_stack_pos
- thr
->shadow_stack
;
552 if (size
+ !!toppc
> kStackTraceMax
) {
553 start
= size
+ !!toppc
- kStackTraceMax
;
554 size
= kStackTraceMax
- !!toppc
;
556 stack
->Init(&thr
->shadow_stack
[start
], size
, toppc
);
560 #if TSAN_COLLECT_STATS
561 void StatAggregate(u64
*dst
, u64
*src
);
562 void StatOutput(u64
*stat
);
565 void ALWAYS_INLINE
StatInc(ThreadState
*thr
, StatType typ
, u64 n
= 1) {
566 #if TSAN_COLLECT_STATS
570 void ALWAYS_INLINE
StatSet(ThreadState
*thr
, StatType typ
, u64 n
) {
571 #if TSAN_COLLECT_STATS
576 void MapShadow(uptr addr
, uptr size
);
577 void MapThreadTrace(uptr addr
, uptr size
, const char *name
);
578 void DontNeedShadowFor(uptr addr
, uptr size
);
579 void InitializeShadowMemory();
580 void InitializeInterceptors();
581 void InitializeLibIgnore();
582 void InitializeDynamicAnnotations();
584 void ForkBefore(ThreadState
*thr
, uptr pc
);
585 void ForkParentAfter(ThreadState
*thr
, uptr pc
);
586 void ForkChildAfter(ThreadState
*thr
, uptr pc
);
588 void ReportRace(ThreadState
*thr
);
589 bool OutputReport(ThreadState
*thr
, const ScopedReport
&srep
);
590 bool IsFiredSuppression(Context
*ctx
, ReportType type
, StackTrace trace
);
591 bool IsExpectedReport(uptr addr
, uptr size
);
592 void PrintMatchedBenignRaces();
594 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
595 # define DPrintf Printf
597 # define DPrintf(...)
600 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
601 # define DPrintf2 Printf
603 # define DPrintf2(...)
606 u32
CurrentStackId(ThreadState
*thr
, uptr pc
);
607 ReportStack
*SymbolizeStackId(u32 stack_id
);
608 void PrintCurrentStack(ThreadState
*thr
, uptr pc
);
609 void PrintCurrentStackSlow(uptr pc
); // uses libunwind
611 void Initialize(ThreadState
*thr
);
612 int Finalize(ThreadState
*thr
);
614 void OnUserAlloc(ThreadState
*thr
, uptr pc
, uptr p
, uptr sz
, bool write
);
615 void OnUserFree(ThreadState
*thr
, uptr pc
, uptr p
, bool write
);
617 void MemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
618 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
);
619 void MemoryAccessImpl(ThreadState
*thr
, uptr addr
,
620 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
,
621 u64
*shadow_mem
, Shadow cur
);
622 void MemoryAccessRange(ThreadState
*thr
, uptr pc
, uptr addr
,
623 uptr size
, bool is_write
);
624 void MemoryAccessRangeStep(ThreadState
*thr
, uptr pc
, uptr addr
,
625 uptr size
, uptr step
, bool is_write
);
626 void UnalignedMemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
627 int size
, bool kAccessIsWrite
, bool kIsAtomic
);
629 const int kSizeLog1
= 0;
630 const int kSizeLog2
= 1;
631 const int kSizeLog4
= 2;
632 const int kSizeLog8
= 3;
634 void ALWAYS_INLINE
MemoryRead(ThreadState
*thr
, uptr pc
,
635 uptr addr
, int kAccessSizeLog
) {
636 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, false, false);
639 void ALWAYS_INLINE
MemoryWrite(ThreadState
*thr
, uptr pc
,
640 uptr addr
, int kAccessSizeLog
) {
641 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, true, false);
644 void ALWAYS_INLINE
MemoryReadAtomic(ThreadState
*thr
, uptr pc
,
645 uptr addr
, int kAccessSizeLog
) {
646 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, false, true);
649 void ALWAYS_INLINE
MemoryWriteAtomic(ThreadState
*thr
, uptr pc
,
650 uptr addr
, int kAccessSizeLog
) {
651 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, true, true);
654 void MemoryResetRange(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
655 void MemoryRangeFreed(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
656 void MemoryRangeImitateWrite(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
658 void ThreadIgnoreBegin(ThreadState
*thr
, uptr pc
);
659 void ThreadIgnoreEnd(ThreadState
*thr
, uptr pc
);
660 void ThreadIgnoreSyncBegin(ThreadState
*thr
, uptr pc
);
661 void ThreadIgnoreSyncEnd(ThreadState
*thr
, uptr pc
);
663 void FuncEntry(ThreadState
*thr
, uptr pc
);
664 void FuncExit(ThreadState
*thr
);
666 int ThreadCreate(ThreadState
*thr
, uptr pc
, uptr uid
, bool detached
);
667 void ThreadStart(ThreadState
*thr
, int tid
, uptr os_id
);
668 void ThreadFinish(ThreadState
*thr
);
669 int ThreadTid(ThreadState
*thr
, uptr pc
, uptr uid
);
670 void ThreadJoin(ThreadState
*thr
, uptr pc
, int tid
);
671 void ThreadDetach(ThreadState
*thr
, uptr pc
, int tid
);
672 void ThreadFinalize(ThreadState
*thr
);
673 void ThreadSetName(ThreadState
*thr
, const char *name
);
674 int ThreadCount(ThreadState
*thr
);
675 void ProcessPendingSignals(ThreadState
*thr
);
677 void MutexCreate(ThreadState
*thr
, uptr pc
, uptr addr
,
678 bool rw
, bool recursive
, bool linker_init
);
679 void MutexDestroy(ThreadState
*thr
, uptr pc
, uptr addr
);
680 void MutexLock(ThreadState
*thr
, uptr pc
, uptr addr
, int rec
= 1,
681 bool try_lock
= false);
682 int MutexUnlock(ThreadState
*thr
, uptr pc
, uptr addr
, bool all
= false);
683 void MutexReadLock(ThreadState
*thr
, uptr pc
, uptr addr
, bool try_lock
= false);
684 void MutexReadUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
685 void MutexReadOrWriteUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
686 void MutexRepair(ThreadState
*thr
, uptr pc
, uptr addr
); // call on EOWNERDEAD
688 void Acquire(ThreadState
*thr
, uptr pc
, uptr addr
);
689 // AcquireGlobal synchronizes the current thread with all other threads.
690 // In terms of happens-before relation, it draws a HB edge from all threads
691 // (where they happen to execute right now) to the current thread. We use it to
692 // handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
693 // right before executing finalizers. This provides a coarse, but simple
694 // approximation of the actual required synchronization.
695 void AcquireGlobal(ThreadState
*thr
, uptr pc
);
696 void Release(ThreadState
*thr
, uptr pc
, uptr addr
);
697 void ReleaseStore(ThreadState
*thr
, uptr pc
, uptr addr
);
698 void AfterSleep(ThreadState
*thr
, uptr pc
);
699 void AcquireImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
700 void ReleaseImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
701 void ReleaseStoreImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
702 void AcquireReleaseImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
704 // The hacky call uses custom calling convention and an assembly thunk.
705 // It is considerably faster that a normal call for the caller
706 // if it is not executed (it is intended for slow paths from hot functions).
707 // The trick is that the call preserves all registers and the compiler
708 // does not treat it as a call.
709 // If it does not work for you, use normal call.
710 #if !SANITIZER_DEBUG && defined(__x86_64__)
711 // The caller may not create the stack frame for itself at all,
712 // so we create a reserve stack frame for it (1024b must be enough).
713 #define HACKY_CALL(f) \
714 __asm__ __volatile__("sub $1024, %%rsp;" \
715 CFI_INL_ADJUST_CFA_OFFSET(1024) \
716 ".hidden " #f "_thunk;" \
717 "call " #f "_thunk;" \
718 "add $1024, %%rsp;" \
719 CFI_INL_ADJUST_CFA_OFFSET(-1024) \
722 #define HACKY_CALL(f) f()
725 void TraceSwitch(ThreadState
*thr
);
726 uptr
TraceTopPC(ThreadState
*thr
);
729 Trace
*ThreadTrace(int tid
);
731 extern "C" void __tsan_trace_switch();
732 void ALWAYS_INLINE
TraceAddEvent(ThreadState
*thr
, FastState fs
,
733 EventType typ
, u64 addr
) {
734 if (!kCollectHistory
)
736 DCHECK_GE((int)typ
, 0);
737 DCHECK_LE((int)typ
, 7);
738 DCHECK_EQ(GetLsb(addr
, 61), addr
);
739 StatInc(thr
, StatEvents
);
740 u64 pos
= fs
.GetTracePos();
741 if (UNLIKELY((pos
% kTracePartSize
) == 0)) {
743 HACKY_CALL(__tsan_trace_switch
);
748 Event
*trace
= (Event
*)GetThreadTrace(fs
.tid());
749 Event
*evp
= &trace
[pos
];
750 Event ev
= (u64
)addr
| ((u64
)typ
<< 61);
755 uptr ALWAYS_INLINE
HeapEnd() {
756 return kHeapMemEnd
+ PrimaryAllocator::AdditionalSize();
760 } // namespace __tsan