1 //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
6 //===----------------------------------------------------------------------===//
8 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 // Main internal TSan header file.
13 // - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
14 // function-scope locals)
15 // - All functions/classes/etc reside in namespace __tsan, except for those
16 // declared in tsan_interface.h.
17 // - Platform-specific files should be used instead of ifdefs (*).
18 // - No system headers included in header files (*).
19 // - Platform specific headres included only into platform-specific files (*).
21 // (*) Except when inlining is critical for performance.
22 //===----------------------------------------------------------------------===//
27 #include "sanitizer_common/sanitizer_allocator.h"
28 #include "sanitizer_common/sanitizer_allocator_internal.h"
29 #include "sanitizer_common/sanitizer_asm.h"
30 #include "sanitizer_common/sanitizer_common.h"
31 #include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
32 #include "sanitizer_common/sanitizer_libignore.h"
33 #include "sanitizer_common/sanitizer_suppressions.h"
34 #include "sanitizer_common/sanitizer_thread_registry.h"
35 #include "tsan_clock.h"
36 #include "tsan_defs.h"
37 #include "tsan_flags.h"
38 #include "tsan_sync.h"
39 #include "tsan_trace.h"
40 #include "tsan_vector.h"
41 #include "tsan_report.h"
42 #include "tsan_platform.h"
43 #include "tsan_mutexset.h"
44 #include "tsan_ignoreset.h"
46 #if SANITIZER_WORDSIZE != 64
47 # error "ThreadSanitizer is supported only on 64-bit platforms"
52 // Descriptor of user's memory block.
55 u64 mtx : 1; // must be first
57 u64 stk : 31; // on word boundary
59 u64 siz : 128 - 1 - 31 - 44 - kTidBits; // 39
63 void Init(uptr siz
, u32 tid
, u32 stk
) {
65 raw
[1] |= (u64
)siz
<< ((1 + 44 + 31 + kTidBits
) % 64);
66 raw
[1] |= (u64
)tid
<< ((1 + 44 + 31) % 64);
67 raw
[0] |= (u64
)stk
<< (1 + 44);
68 raw
[1] |= (u64
)stk
>> (64 - 44 - 1);
69 DCHECK_EQ(Size(), siz
);
70 DCHECK_EQ(Tid(), tid
);
71 DCHECK_EQ(StackId(), stk
);
75 return GetLsb(raw
[1] >> ((1 + 44 + 31) % 64), kTidBits
);
79 return raw
[1] >> ((1 + 31 + 44 + kTidBits
) % 64);
83 return (raw
[0] >> (1 + 44)) | GetLsb(raw
[1] << (64 - 44 - 1), 31);
86 SyncVar
*ListHead() const {
87 return (SyncVar
*)(GetLsb(raw
[0] >> 1, 44) << 3);
90 void ListPush(SyncVar
*v
) {
91 SyncVar
*lst
= ListHead();
93 u64 x
= (u64
)v
^ (u64
)lst
;
96 DCHECK_EQ(ListHead(), v
);
100 SyncVar
*lst
= ListHead();
101 SyncVar
*nxt
= lst
->next
;
103 u64 x
= (u64
)lst
^ (u64
)nxt
;
106 DCHECK_EQ(ListHead(), nxt
);
111 SyncVar
*lst
= ListHead();
115 DCHECK_EQ(ListHead(), 0);
120 typedef GenericScopedLock
<MBlock
> ScopedLock
;
124 #if defined(TSAN_COMPAT_SHADOW) && TSAN_COMPAT_SHADOW
125 const uptr kAllocatorSpace
= 0x7d0000000000ULL
;
127 const uptr kAllocatorSpace
= 0x7d0000000000ULL
;
129 const uptr kAllocatorSize
= 0x10000000000ULL
; // 1T.
131 struct MapUnmapCallback
;
132 typedef SizeClassAllocator64
<kAllocatorSpace
, kAllocatorSize
, sizeof(MBlock
),
133 DefaultSizeClassMap
, MapUnmapCallback
> PrimaryAllocator
;
134 typedef SizeClassAllocatorLocalCache
<PrimaryAllocator
> AllocatorCache
;
135 typedef LargeMmapAllocator
<MapUnmapCallback
> SecondaryAllocator
;
136 typedef CombinedAllocator
<PrimaryAllocator
, AllocatorCache
,
137 SecondaryAllocator
> Allocator
;
138 Allocator
*allocator();
141 void TsanCheckFailed(const char *file
, int line
, const char *cond
,
144 const u64 kShadowRodata
= (u64
)-1; // .rodata shadow marker
146 // FastState (from most significant bit):
154 FastState(u64 tid
, u64 epoch
) {
155 x_
= tid
<< kTidShift
;
156 x_
|= epoch
<< kClkShift
;
157 DCHECK_EQ(tid
, this->tid());
158 DCHECK_EQ(epoch
, this->epoch());
159 DCHECK_EQ(GetIgnoreBit(), false);
162 explicit FastState(u64 x
)
171 u64 res
= (x_
& ~kIgnoreBit
) >> kTidShift
;
175 u64
TidWithIgnore() const {
176 u64 res
= x_
>> kTidShift
;
181 u64 res
= (x_
<< (kTidBits
+ 1)) >> (64 - kClkBits
);
185 void IncrementEpoch() {
186 u64 old_epoch
= epoch();
187 x_
+= 1 << kClkShift
;
188 DCHECK_EQ(old_epoch
+ 1, epoch());
192 void SetIgnoreBit() { x_
|= kIgnoreBit
; }
193 void ClearIgnoreBit() { x_
&= ~kIgnoreBit
; }
194 bool GetIgnoreBit() const { return (s64
)x_
< 0; }
196 void SetHistorySize(int hs
) {
202 int GetHistorySize() const {
203 return (int)(x_
& 7);
206 void ClearHistorySize() {
210 u64
GetTracePos() const {
211 const int hs
= GetHistorySize();
212 // When hs == 0, the trace consists of 2 parts.
213 const u64 mask
= (1ull << (kTracePartSizeBits
+ hs
+ 1)) - 1;
214 return epoch() & mask
;
219 static const int kTidShift
= 64 - kTidBits
- 1;
220 static const int kClkShift
= kTidShift
- kClkBits
;
221 static const u64 kIgnoreBit
= 1ull << 63;
222 static const u64 kFreedBit
= 1ull << 63;
226 // Shadow (from most significant bit):
234 class Shadow
: public FastState
{
236 explicit Shadow(u64 x
)
240 explicit Shadow(const FastState
&s
)
245 void SetAddr0AndSizeLog(u64 addr0
, unsigned kAccessSizeLog
) {
246 DCHECK_EQ(x_
& 31, 0);
248 DCHECK_LE(kAccessSizeLog
, 3);
249 x_
|= (kAccessSizeLog
<< 3) | addr0
;
250 DCHECK_EQ(kAccessSizeLog
, size_log());
251 DCHECK_EQ(addr0
, this->addr0());
254 void SetWrite(unsigned kAccessIsWrite
) {
255 DCHECK_EQ(x_
& kReadBit
, 0);
258 DCHECK_EQ(kAccessIsWrite
, IsWrite());
261 void SetAtomic(bool kIsAtomic
) {
265 DCHECK_EQ(IsAtomic(), kIsAtomic
);
268 bool IsAtomic() const {
269 return x_
& kAtomicBit
;
272 bool IsZero() const {
276 static inline bool TidsAreEqual(const Shadow s1
, const Shadow s2
) {
277 u64 shifted_xor
= (s1
.x_
^ s2
.x_
) >> kTidShift
;
278 DCHECK_EQ(shifted_xor
== 0, s1
.TidWithIgnore() == s2
.TidWithIgnore());
279 return shifted_xor
== 0;
282 static inline bool Addr0AndSizeAreEqual(const Shadow s1
, const Shadow s2
) {
283 u64 masked_xor
= (s1
.x_
^ s2
.x_
) & 31;
284 return masked_xor
== 0;
287 static inline bool TwoRangesIntersect(Shadow s1
, Shadow s2
,
288 unsigned kS2AccessSize
) {
290 u64 diff
= s1
.addr0() - s2
.addr0();
291 if ((s64
)diff
< 0) { // s1.addr0 < s2.addr0 // NOLINT
292 // if (s1.addr0() + size1) > s2.addr0()) return true;
293 if (s1
.size() > -diff
) res
= true;
295 // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
296 if (kS2AccessSize
> diff
) res
= true;
298 DCHECK_EQ(res
, TwoRangesIntersectSLOW(s1
, s2
));
299 DCHECK_EQ(res
, TwoRangesIntersectSLOW(s2
, s1
));
303 // The idea behind the offset is as follows.
304 // Consider that we have 8 bool's contained within a single 8-byte block
305 // (mapped to a single shadow "cell"). Now consider that we write to the bools
306 // from a single thread (which we consider the common case).
307 // W/o offsetting each access will have to scan 4 shadow values at average
308 // to find the corresponding shadow value for the bool.
309 // With offsetting we start scanning shadow with the offset so that
310 // each access hits necessary shadow straight off (at least in an expected
312 // This logic works seamlessly for any layout of user data. For example,
313 // if user data is {int, short, char, char}, then accesses to the int are
314 // offsetted to 0, short - 4, 1st char - 6, 2nd char - 7. Hopefully, accesses
315 // from a single thread won't need to scan all 8 shadow values.
316 unsigned ComputeSearchOffset() {
319 u64
addr0() const { return x_
& 7; }
320 u64
size() const { return 1ull << size_log(); }
321 bool IsWrite() const { return !IsRead(); }
322 bool IsRead() const { return x_
& kReadBit
; }
324 // The idea behind the freed bit is as follows.
325 // When the memory is freed (or otherwise unaccessible) we write to the shadow
326 // values with tid/epoch related to the free and the freed bit set.
327 // During memory accesses processing the freed bit is considered
328 // as msb of tid. So any access races with shadow with freed bit set
329 // (it is as if write from a thread with which we never synchronized before).
330 // This allows us to detect accesses to freed memory w/o additional
331 // overheads in memory access processing and at the same time restore
332 // tid/epoch of free.
337 bool IsFreed() const {
338 return x_
& kFreedBit
;
341 bool GetFreedAndReset() {
342 bool res
= x_
& kFreedBit
;
347 bool IsBothReadsOrAtomic(bool kIsWrite
, bool kIsAtomic
) const {
348 // analyzes 5-th bit (is_read) and 6-th bit (is_atomic)
349 bool v
= x_
& u64(((kIsWrite
^ 1) << kReadShift
)
350 | (kIsAtomic
<< kAtomicShift
));
351 DCHECK_EQ(v
, (!IsWrite() && !kIsWrite
) || (IsAtomic() && kIsAtomic
));
355 bool IsRWNotWeaker(bool kIsWrite
, bool kIsAtomic
) const {
356 bool v
= ((x_
>> kReadShift
) & 3)
357 <= u64((kIsWrite
^ 1) | (kIsAtomic
<< 1));
358 DCHECK_EQ(v
, (IsAtomic() < kIsAtomic
) ||
359 (IsAtomic() == kIsAtomic
&& !IsWrite() <= !kIsWrite
));
363 bool IsRWWeakerOrEqual(bool kIsWrite
, bool kIsAtomic
) const {
364 bool v
= ((x_
>> kReadShift
) & 3)
365 >= u64((kIsWrite
^ 1) | (kIsAtomic
<< 1));
366 DCHECK_EQ(v
, (IsAtomic() > kIsAtomic
) ||
367 (IsAtomic() == kIsAtomic
&& !IsWrite() >= !kIsWrite
));
372 static const u64 kReadShift
= 5;
373 static const u64 kReadBit
= 1ull << kReadShift
;
374 static const u64 kAtomicShift
= 6;
375 static const u64 kAtomicBit
= 1ull << kAtomicShift
;
377 u64
size_log() const { return (x_
>> 3) & 3; }
379 static bool TwoRangesIntersectSLOW(const Shadow s1
, const Shadow s2
) {
380 if (s1
.addr0() == s2
.addr0()) return true;
381 if (s1
.addr0() < s2
.addr0() && s1
.addr0() + s1
.size() > s2
.addr0())
383 if (s2
.addr0() < s1
.addr0() && s2
.addr0() + s2
.size() > s1
.addr0())
389 struct SignalContext
;
394 uptr
*shadow_stack_pos
;
397 // This struct is stored in TLS.
399 FastState fast_state
;
400 // Synch epoch represents the threads's epoch before the last synchronization
401 // action. It allows to reduce number of shadow state updates.
402 // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
403 // if we are processing write to X from the same thread at epoch=200,
404 // we do nothing, because both writes happen in the same 'synch epoch'.
405 // That is, if another memory access does not race with the former write,
406 // it does not race with the latter as well.
407 // QUESTION: can we can squeeze this into ThreadState::Fast?
408 // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
409 // taken by epoch between synchs.
410 // This way we can save one load from tls.
411 u64 fast_synch_epoch
;
412 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
413 // We do not distinguish beteween ignoring reads and writes
414 // for better performance.
415 int ignore_reads_and_writes
;
417 // Go does not support ignores.
419 IgnoreSet mop_ignore_set
;
420 IgnoreSet sync_ignore_set
;
422 // C/C++ uses fixed size shadow stack embed into Trace.
423 // Go uses malloc-allocated shadow stack with dynamic size.
425 uptr
*shadow_stack_end
;
426 uptr
*shadow_stack_pos
;
427 u64
*racy_shadow_addr
;
432 AllocatorCache alloc_cache
;
433 InternalAllocatorCache internal_alloc_cache
;
434 Vector
<JmpBuf
> jmp_bufs
;
435 int ignore_interceptors
;
451 InternalDeadlockDetector internal_deadlock_detector
;
452 DDPhysicalThread
*dd_pt
;
453 DDLogicalThread
*dd_lt
;
455 bool in_signal_handler
;
456 SignalContext
*signal_ctx
;
459 u32 last_sleep_stack_id
;
460 ThreadClock last_sleep_clock
;
463 // Set in regions of runtime that must be signal-safe and fork-safe.
464 // If set, malloc must not be called.
467 explicit ThreadState(Context
*ctx
, int tid
, int unique_id
, u64 epoch
,
468 unsigned reuse_count
,
469 uptr stk_addr
, uptr stk_size
,
470 uptr tls_addr
, uptr tls_size
);
474 __attribute__((tls_model("initial-exec")))
475 extern THREADLOCAL
char cur_thread_placeholder
[];
476 INLINE ThreadState
*cur_thread() {
477 return reinterpret_cast<ThreadState
*>(&cur_thread_placeholder
);
481 class ThreadContext
: public ThreadContextBase
{
483 explicit ThreadContext(int tid
);
486 u32 creation_stack_id
;
488 // Epoch at which the thread had started.
489 // If we see an event from the thread stamped by an older epoch,
490 // the event is from a dead thread that shared tid with this thread.
494 // Override superclass callbacks.
496 void OnJoined(void *arg
);
498 void OnStarted(void *arg
);
499 void OnCreated(void *arg
);
505 bool operator==(const RacyStacks
&other
) const {
506 if (hash
[0] == other
.hash
[0] && hash
[1] == other
.hash
[1])
508 if (hash
[0] == other
.hash
[1] && hash
[1] == other
.hash
[0])
519 struct FiredSuppression
{
529 bool after_multithreaded_fork
;
535 int nmissed_expected
;
536 atomic_uint64_t last_symbolize_time_ns
;
538 void *background_thread
;
539 atomic_uint32_t stop_background_thread
;
541 ThreadRegistry
*thread_registry
;
543 Vector
<RacyStacks
> racy_stacks
;
544 Vector
<RacyAddress
> racy_addresses
;
545 // Number of fired suppressions may be large enough.
546 InternalMmapVector
<FiredSuppression
> fired_suppressions
;
552 u64 int_alloc_cnt
[MBlockTypeCount
];
553 u64 int_alloc_siz
[MBlockTypeCount
];
556 extern Context
*ctx
; // The one and the only global runtime context.
558 struct ScopedIgnoreInterceptors
{
559 ScopedIgnoreInterceptors() {
561 cur_thread()->ignore_interceptors
++;
565 ~ScopedIgnoreInterceptors() {
567 cur_thread()->ignore_interceptors
--;
574 explicit ScopedReport(ReportType typ
);
577 void AddStack(const StackTrace
*stack
);
578 void AddMemoryAccess(uptr addr
, Shadow s
, const StackTrace
*stack
,
579 const MutexSet
*mset
);
580 void AddThread(const ThreadContext
*tctx
);
581 void AddThread(int unique_tid
);
582 void AddUniqueTid(int unique_tid
);
583 void AddMutex(const SyncVar
*s
);
584 u64
AddMutex(u64 id
);
585 void AddLocation(uptr addr
, uptr size
);
586 void AddSleep(u32 stack_id
);
587 void SetCount(int count
);
589 const ReportDesc
*GetReport() const;
593 // Symbolizer makes lots of intercepted calls. If we try to process them,
594 // at best it will cause deadlocks on internal mutexes.
595 ScopedIgnoreInterceptors ignore_interceptors_
;
597 void AddDeadMutex(u64 id
);
599 ScopedReport(const ScopedReport
&);
600 void operator = (const ScopedReport
&);
603 void RestoreStack(int tid
, const u64 epoch
, StackTrace
*stk
, MutexSet
*mset
);
605 void StatAggregate(u64
*dst
, u64
*src
);
606 void StatOutput(u64
*stat
);
607 void ALWAYS_INLINE
StatInc(ThreadState
*thr
, StatType typ
, u64 n
= 1) {
611 void ALWAYS_INLINE
StatSet(ThreadState
*thr
, StatType typ
, u64 n
) {
616 void MapShadow(uptr addr
, uptr size
);
617 void MapThreadTrace(uptr addr
, uptr size
);
618 void DontNeedShadowFor(uptr addr
, uptr size
);
619 void InitializeShadowMemory();
620 void InitializeInterceptors();
621 void InitializeLibIgnore();
622 void InitializeDynamicAnnotations();
624 void ForkBefore(ThreadState
*thr
, uptr pc
);
625 void ForkParentAfter(ThreadState
*thr
, uptr pc
);
626 void ForkChildAfter(ThreadState
*thr
, uptr pc
);
628 void ReportRace(ThreadState
*thr
);
629 bool OutputReport(Context
*ctx
,
630 const ScopedReport
&srep
,
631 const ReportStack
*suppress_stack1
,
632 const ReportStack
*suppress_stack2
= 0,
633 const ReportLocation
*suppress_loc
= 0);
634 bool IsFiredSuppression(Context
*ctx
,
635 const ScopedReport
&srep
,
636 const StackTrace
&trace
);
637 bool IsExpectedReport(uptr addr
, uptr size
);
638 void PrintMatchedBenignRaces();
639 bool FrameIsInternal(const ReportStack
*frame
);
640 ReportStack
*SkipTsanInternalFrames(ReportStack
*ent
);
642 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
643 # define DPrintf Printf
645 # define DPrintf(...)
648 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
649 # define DPrintf2 Printf
651 # define DPrintf2(...)
654 u32
CurrentStackId(ThreadState
*thr
, uptr pc
);
655 ReportStack
*SymbolizeStackId(u32 stack_id
);
656 void PrintCurrentStack(ThreadState
*thr
, uptr pc
);
657 void PrintCurrentStackSlow(); // uses libunwind
659 void Initialize(ThreadState
*thr
);
660 int Finalize(ThreadState
*thr
);
662 SyncVar
* GetJavaSync(ThreadState
*thr
, uptr pc
, uptr addr
,
663 bool write_lock
, bool create
);
664 SyncVar
* GetAndRemoveJavaSync(ThreadState
*thr
, uptr pc
, uptr addr
);
666 void MemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
667 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
);
668 void MemoryAccessImpl(ThreadState
*thr
, uptr addr
,
669 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
,
670 u64
*shadow_mem
, Shadow cur
);
671 void MemoryAccessRange(ThreadState
*thr
, uptr pc
, uptr addr
,
672 uptr size
, bool is_write
);
673 void MemoryAccessRangeStep(ThreadState
*thr
, uptr pc
, uptr addr
,
674 uptr size
, uptr step
, bool is_write
);
675 void UnalignedMemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
676 int size
, bool kAccessIsWrite
, bool kIsAtomic
);
678 const int kSizeLog1
= 0;
679 const int kSizeLog2
= 1;
680 const int kSizeLog4
= 2;
681 const int kSizeLog8
= 3;
683 void ALWAYS_INLINE
MemoryRead(ThreadState
*thr
, uptr pc
,
684 uptr addr
, int kAccessSizeLog
) {
685 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, false, false);
688 void ALWAYS_INLINE
MemoryWrite(ThreadState
*thr
, uptr pc
,
689 uptr addr
, int kAccessSizeLog
) {
690 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, true, false);
693 void ALWAYS_INLINE
MemoryReadAtomic(ThreadState
*thr
, uptr pc
,
694 uptr addr
, int kAccessSizeLog
) {
695 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, false, true);
698 void ALWAYS_INLINE
MemoryWriteAtomic(ThreadState
*thr
, uptr pc
,
699 uptr addr
, int kAccessSizeLog
) {
700 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, true, true);
703 void MemoryResetRange(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
704 void MemoryRangeFreed(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
705 void MemoryRangeImitateWrite(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
707 void ThreadIgnoreBegin(ThreadState
*thr
, uptr pc
);
708 void ThreadIgnoreEnd(ThreadState
*thr
, uptr pc
);
709 void ThreadIgnoreSyncBegin(ThreadState
*thr
, uptr pc
);
710 void ThreadIgnoreSyncEnd(ThreadState
*thr
, uptr pc
);
712 void FuncEntry(ThreadState
*thr
, uptr pc
);
713 void FuncExit(ThreadState
*thr
);
715 int ThreadCreate(ThreadState
*thr
, uptr pc
, uptr uid
, bool detached
);
716 void ThreadStart(ThreadState
*thr
, int tid
, uptr os_id
);
717 void ThreadFinish(ThreadState
*thr
);
718 int ThreadTid(ThreadState
*thr
, uptr pc
, uptr uid
);
719 void ThreadJoin(ThreadState
*thr
, uptr pc
, int tid
);
720 void ThreadDetach(ThreadState
*thr
, uptr pc
, int tid
);
721 void ThreadFinalize(ThreadState
*thr
);
722 void ThreadSetName(ThreadState
*thr
, const char *name
);
723 int ThreadCount(ThreadState
*thr
);
724 void ProcessPendingSignals(ThreadState
*thr
);
726 void MutexCreate(ThreadState
*thr
, uptr pc
, uptr addr
,
727 bool rw
, bool recursive
, bool linker_init
);
728 void MutexDestroy(ThreadState
*thr
, uptr pc
, uptr addr
);
729 void MutexLock(ThreadState
*thr
, uptr pc
, uptr addr
, int rec
= 1,
730 bool try_lock
= false);
731 int MutexUnlock(ThreadState
*thr
, uptr pc
, uptr addr
, bool all
= false);
732 void MutexReadLock(ThreadState
*thr
, uptr pc
, uptr addr
, bool try_lock
= false);
733 void MutexReadUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
734 void MutexReadOrWriteUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
735 void MutexRepair(ThreadState
*thr
, uptr pc
, uptr addr
); // call on EOWNERDEAD
737 void Acquire(ThreadState
*thr
, uptr pc
, uptr addr
);
738 void AcquireGlobal(ThreadState
*thr
, uptr pc
);
739 void Release(ThreadState
*thr
, uptr pc
, uptr addr
);
740 void ReleaseStore(ThreadState
*thr
, uptr pc
, uptr addr
);
741 void AfterSleep(ThreadState
*thr
, uptr pc
);
742 void AcquireImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
743 void ReleaseImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
744 void ReleaseStoreImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
745 void AcquireReleaseImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
747 // The hacky call uses custom calling convention and an assembly thunk.
748 // It is considerably faster that a normal call for the caller
749 // if it is not executed (it is intended for slow paths from hot functions).
750 // The trick is that the call preserves all registers and the compiler
751 // does not treat it as a call.
752 // If it does not work for you, use normal call.
754 // The caller may not create the stack frame for itself at all,
755 // so we create a reserve stack frame for it (1024b must be enough).
756 #define HACKY_CALL(f) \
757 __asm__ __volatile__("sub $1024, %%rsp;" \
758 CFI_INL_ADJUST_CFA_OFFSET(1024) \
759 ".hidden " #f "_thunk;" \
760 "call " #f "_thunk;" \
761 "add $1024, %%rsp;" \
762 CFI_INL_ADJUST_CFA_OFFSET(-1024) \
765 #define HACKY_CALL(f) f()
768 void TraceSwitch(ThreadState
*thr
);
769 uptr
TraceTopPC(ThreadState
*thr
);
772 Trace
*ThreadTrace(int tid
);
774 extern "C" void __tsan_trace_switch();
775 void ALWAYS_INLINE
TraceAddEvent(ThreadState
*thr
, FastState fs
,
776 EventType typ
, u64 addr
) {
777 if (!kCollectHistory
)
779 DCHECK_GE((int)typ
, 0);
780 DCHECK_LE((int)typ
, 7);
781 DCHECK_EQ(GetLsb(addr
, 61), addr
);
782 StatInc(thr
, StatEvents
);
783 u64 pos
= fs
.GetTracePos();
784 if (UNLIKELY((pos
% kTracePartSize
) == 0)) {
786 HACKY_CALL(__tsan_trace_switch
);
791 Event
*trace
= (Event
*)GetThreadTrace(fs
.tid());
792 Event
*evp
= &trace
[pos
];
793 Event ev
= (u64
)addr
| ((u64
)typ
<< 61);
797 } // namespace __tsan