1 //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is a part of ThreadSanitizer (TSan), a race detector.
11 // Main internal TSan header file.
14 // - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
15 // function-scope locals)
16 // - All functions/classes/etc reside in namespace __tsan, except for those
17 // declared in tsan_interface.h.
18 // - Platform-specific files should be used instead of ifdefs (*).
19 // - No system headers included in header files (*).
20 // - Platform specific headres included only into platform-specific files (*).
22 // (*) Except when inlining is critical for performance.
23 //===----------------------------------------------------------------------===//
28 #include "sanitizer_common/sanitizer_allocator.h"
29 #include "sanitizer_common/sanitizer_allocator_internal.h"
30 #include "sanitizer_common/sanitizer_asm.h"
31 #include "sanitizer_common/sanitizer_common.h"
32 #include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
33 #include "sanitizer_common/sanitizer_libignore.h"
34 #include "sanitizer_common/sanitizer_suppressions.h"
35 #include "sanitizer_common/sanitizer_thread_registry.h"
36 #include "sanitizer_common/sanitizer_vector.h"
37 #include "tsan_clock.h"
38 #include "tsan_defs.h"
39 #include "tsan_flags.h"
40 #include "tsan_mman.h"
41 #include "tsan_sync.h"
42 #include "tsan_trace.h"
43 #include "tsan_report.h"
44 #include "tsan_platform.h"
45 #include "tsan_mutexset.h"
46 #include "tsan_ignoreset.h"
47 #include "tsan_stack_trace.h"
49 #if SANITIZER_WORDSIZE != 64
50 # error "ThreadSanitizer is supported only on 64-bit platforms"
56 struct MapUnmapCallback
;
57 #if defined(__mips64) || defined(__aarch64__) || defined(__powerpc__)
60 static const uptr kSpaceBeg
= 0;
61 static const u64 kSpaceSize
= SANITIZER_MMAP_RANGE_SIZE
;
62 static const uptr kMetadataSize
= 0;
63 typedef __sanitizer::CompactSizeClassMap SizeClassMap
;
64 static const uptr kRegionSizeLog
= 20;
65 using AddressSpaceView
= LocalAddressSpaceView
;
66 typedef __tsan::MapUnmapCallback MapUnmapCallback
;
67 static const uptr kFlags
= 0;
69 typedef SizeClassAllocator32
<AP32
> PrimaryAllocator
;
71 struct AP64
{ // Allocator64 parameters. Deliberately using a short name.
72 static const uptr kSpaceBeg
= Mapping::kHeapMemBeg
;
73 static const uptr kSpaceSize
= Mapping::kHeapMemEnd
- Mapping::kHeapMemBeg
;
74 static const uptr kMetadataSize
= 0;
75 typedef DefaultSizeClassMap SizeClassMap
;
76 typedef __tsan::MapUnmapCallback MapUnmapCallback
;
77 static const uptr kFlags
= 0;
78 using AddressSpaceView
= LocalAddressSpaceView
;
80 typedef SizeClassAllocator64
<AP64
> PrimaryAllocator
;
82 typedef CombinedAllocator
<PrimaryAllocator
> Allocator
;
83 typedef Allocator::AllocatorCache AllocatorCache
;
84 Allocator
*allocator();
87 const u64 kShadowRodata
= (u64
)-1; // .rodata shadow marker
89 // FastState (from most significant bit):
97 FastState(u64 tid
, u64 epoch
) {
98 x_
= tid
<< kTidShift
;
100 DCHECK_EQ(tid
, this->tid());
101 DCHECK_EQ(epoch
, this->epoch());
102 DCHECK_EQ(GetIgnoreBit(), false);
105 explicit FastState(u64 x
)
114 u64 res
= (x_
& ~kIgnoreBit
) >> kTidShift
;
118 u64
TidWithIgnore() const {
119 u64 res
= x_
>> kTidShift
;
124 u64 res
= x_
& ((1ull << kClkBits
) - 1);
128 void IncrementEpoch() {
129 u64 old_epoch
= epoch();
131 DCHECK_EQ(old_epoch
+ 1, epoch());
135 void SetIgnoreBit() { x_
|= kIgnoreBit
; }
136 void ClearIgnoreBit() { x_
&= ~kIgnoreBit
; }
137 bool GetIgnoreBit() const { return (s64
)x_
< 0; }
139 void SetHistorySize(int hs
) {
142 x_
= (x_
& ~(kHistoryMask
<< kHistoryShift
)) | (u64(hs
) << kHistoryShift
);
146 int GetHistorySize() const {
147 return (int)((x_
>> kHistoryShift
) & kHistoryMask
);
150 void ClearHistorySize() {
155 u64
GetTracePos() const {
156 const int hs
= GetHistorySize();
157 // When hs == 0, the trace consists of 2 parts.
158 const u64 mask
= (1ull << (kTracePartSizeBits
+ hs
+ 1)) - 1;
159 return epoch() & mask
;
164 static const int kTidShift
= 64 - kTidBits
- 1;
165 static const u64 kIgnoreBit
= 1ull << 63;
166 static const u64 kFreedBit
= 1ull << 63;
167 static const u64 kHistoryShift
= kClkBits
;
168 static const u64 kHistoryMask
= 7;
172 // Shadow (from most significant bit):
180 class Shadow
: public FastState
{
182 explicit Shadow(u64 x
)
186 explicit Shadow(const FastState
&s
)
191 void SetAddr0AndSizeLog(u64 addr0
, unsigned kAccessSizeLog
) {
192 DCHECK_EQ((x_
>> kClkBits
) & 31, 0);
194 DCHECK_LE(kAccessSizeLog
, 3);
195 x_
|= ((kAccessSizeLog
<< 3) | addr0
) << kClkBits
;
196 DCHECK_EQ(kAccessSizeLog
, size_log());
197 DCHECK_EQ(addr0
, this->addr0());
200 void SetWrite(unsigned kAccessIsWrite
) {
201 DCHECK_EQ(x_
& kReadBit
, 0);
204 DCHECK_EQ(kAccessIsWrite
, IsWrite());
207 void SetAtomic(bool kIsAtomic
) {
211 DCHECK_EQ(IsAtomic(), kIsAtomic
);
214 bool IsAtomic() const {
215 return x_
& kAtomicBit
;
218 bool IsZero() const {
222 static inline bool TidsAreEqual(const Shadow s1
, const Shadow s2
) {
223 u64 shifted_xor
= (s1
.x_
^ s2
.x_
) >> kTidShift
;
224 DCHECK_EQ(shifted_xor
== 0, s1
.TidWithIgnore() == s2
.TidWithIgnore());
225 return shifted_xor
== 0;
229 bool Addr0AndSizeAreEqual(const Shadow s1
, const Shadow s2
) {
230 u64 masked_xor
= ((s1
.x_
^ s2
.x_
) >> kClkBits
) & 31;
231 return masked_xor
== 0;
234 static ALWAYS_INLINE
bool TwoRangesIntersect(Shadow s1
, Shadow s2
,
235 unsigned kS2AccessSize
) {
237 u64 diff
= s1
.addr0() - s2
.addr0();
238 if ((s64
)diff
< 0) { // s1.addr0 < s2.addr0
239 // if (s1.addr0() + size1) > s2.addr0()) return true;
240 if (s1
.size() > -diff
)
243 // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
244 if (kS2AccessSize
> diff
)
247 DCHECK_EQ(res
, TwoRangesIntersectSlow(s1
, s2
));
248 DCHECK_EQ(res
, TwoRangesIntersectSlow(s2
, s1
));
252 u64 ALWAYS_INLINE
addr0() const { return (x_
>> kClkBits
) & 7; }
253 u64 ALWAYS_INLINE
size() const { return 1ull << size_log(); }
254 bool ALWAYS_INLINE
IsWrite() const { return !IsRead(); }
255 bool ALWAYS_INLINE
IsRead() const { return x_
& kReadBit
; }
257 // The idea behind the freed bit is as follows.
258 // When the memory is freed (or otherwise unaccessible) we write to the shadow
259 // values with tid/epoch related to the free and the freed bit set.
260 // During memory accesses processing the freed bit is considered
261 // as msb of tid. So any access races with shadow with freed bit set
262 // (it is as if write from a thread with which we never synchronized before).
263 // This allows us to detect accesses to freed memory w/o additional
264 // overheads in memory access processing and at the same time restore
265 // tid/epoch of free.
270 bool IsFreed() const {
271 return x_
& kFreedBit
;
274 bool GetFreedAndReset() {
275 bool res
= x_
& kFreedBit
;
280 bool ALWAYS_INLINE
IsBothReadsOrAtomic(bool kIsWrite
, bool kIsAtomic
) const {
281 bool v
= x_
& ((u64(kIsWrite
^ 1) << kReadShift
)
282 | (u64(kIsAtomic
) << kAtomicShift
));
283 DCHECK_EQ(v
, (!IsWrite() && !kIsWrite
) || (IsAtomic() && kIsAtomic
));
287 bool ALWAYS_INLINE
IsRWNotWeaker(bool kIsWrite
, bool kIsAtomic
) const {
288 bool v
= ((x_
>> kReadShift
) & 3)
289 <= u64((kIsWrite
^ 1) | (kIsAtomic
<< 1));
290 DCHECK_EQ(v
, (IsAtomic() < kIsAtomic
) ||
291 (IsAtomic() == kIsAtomic
&& !IsWrite() <= !kIsWrite
));
295 bool ALWAYS_INLINE
IsRWWeakerOrEqual(bool kIsWrite
, bool kIsAtomic
) const {
296 bool v
= ((x_
>> kReadShift
) & 3)
297 >= u64((kIsWrite
^ 1) | (kIsAtomic
<< 1));
298 DCHECK_EQ(v
, (IsAtomic() > kIsAtomic
) ||
299 (IsAtomic() == kIsAtomic
&& !IsWrite() >= !kIsWrite
));
304 static const u64 kReadShift
= 5 + kClkBits
;
305 static const u64 kReadBit
= 1ull << kReadShift
;
306 static const u64 kAtomicShift
= 6 + kClkBits
;
307 static const u64 kAtomicBit
= 1ull << kAtomicShift
;
309 u64
size_log() const { return (x_
>> (3 + kClkBits
)) & 3; }
311 static bool TwoRangesIntersectSlow(const Shadow s1
, const Shadow s2
) {
312 if (s1
.addr0() == s2
.addr0()) return true;
313 if (s1
.addr0() < s2
.addr0() && s1
.addr0() + s1
.size() > s2
.addr0())
315 if (s2
.addr0() < s1
.addr0() && s2
.addr0() + s2
.size() > s1
.addr0())
321 struct ThreadSignalContext
;
326 bool in_blocking_func
;
327 uptr in_signal_handler
;
328 uptr
*shadow_stack_pos
;
331 // A Processor represents a physical thread, or a P for Go.
332 // It is used to store internal resources like allocate cache, and does not
333 // participate in race-detection logic (invisible to end user).
334 // In C++ it is tied to an OS thread just like ThreadState, however ideally
335 // it should be tied to a CPU (this way we will have fewer allocator caches).
336 // In Go it is tied to a P, so there are significantly fewer Processor's than
337 // ThreadState's (which are tied to Gs).
338 // A ThreadState must be wired with a Processor to handle events.
340 ThreadState
*thr
; // currently wired thread, or nullptr
342 AllocatorCache alloc_cache
;
343 InternalAllocatorCache internal_alloc_cache
;
345 DenseSlabAllocCache block_cache
;
346 DenseSlabAllocCache sync_cache
;
347 DenseSlabAllocCache clock_cache
;
348 DDPhysicalThread
*dd_pt
;
352 // ScopedGlobalProcessor temporary setups a global processor for the current
353 // thread, if it does not have one. Intended for interceptors that can run
354 // at the very thread end, when we already destroyed the thread processor.
355 struct ScopedGlobalProcessor
{
356 ScopedGlobalProcessor();
357 ~ScopedGlobalProcessor();
361 // This struct is stored in TLS.
363 FastState fast_state
;
364 // Synch epoch represents the threads's epoch before the last synchronization
365 // action. It allows to reduce number of shadow state updates.
366 // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
367 // if we are processing write to X from the same thread at epoch=200,
368 // we do nothing, because both writes happen in the same 'synch epoch'.
369 // That is, if another memory access does not race with the former write,
370 // it does not race with the latter as well.
371 // QUESTION: can we can squeeze this into ThreadState::Fast?
372 // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
373 // taken by epoch between synchs.
374 // This way we can save one load from tls.
375 u64 fast_synch_epoch
;
376 // Technically `current` should be a separate THREADLOCAL variable;
377 // but it is placed here in order to share cache line with previous fields.
378 ThreadState
* current
;
379 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
380 // We do not distinguish beteween ignoring reads and writes
381 // for better performance.
382 int ignore_reads_and_writes
;
384 int suppress_reports
;
385 // Go does not support ignores.
387 IgnoreSet mop_ignore_set
;
388 IgnoreSet sync_ignore_set
;
390 // C/C++ uses fixed size shadow stack embed into Trace.
391 // Go uses malloc-allocated shadow stack with dynamic size.
393 uptr
*shadow_stack_end
;
394 uptr
*shadow_stack_pos
;
395 u64
*racy_shadow_addr
;
400 Vector
<JmpBuf
> jmp_bufs
;
401 int ignore_interceptors
;
403 #if TSAN_COLLECT_STATS
420 #if SANITIZER_DEBUG && !SANITIZER_GO
421 InternalDeadlockDetector internal_deadlock_detector
;
423 DDLogicalThread
*dd_lt
;
425 // Current wired Processor, or nullptr. Required to handle any events.
428 Processor
*proc() { return proc1
; }
433 atomic_uintptr_t in_signal_handler
;
434 ThreadSignalContext
*signal_ctx
;
437 u32 last_sleep_stack_id
;
438 ThreadClock last_sleep_clock
;
441 // Set in regions of runtime that must be signal-safe and fork-safe.
442 // If set, malloc must not be called.
445 const ReportDesc
*current_report
;
447 explicit ThreadState(Context
*ctx
, u32 tid
, int unique_id
, u64 epoch
,
448 unsigned reuse_count
, uptr stk_addr
, uptr stk_size
,
449 uptr tls_addr
, uptr tls_size
);
453 #if SANITIZER_MAC || SANITIZER_ANDROID
454 ThreadState
*cur_thread();
455 void set_cur_thread(ThreadState
*thr
);
456 void cur_thread_finalize();
457 inline void cur_thread_init() { }
459 __attribute__((tls_model("initial-exec")))
460 extern THREADLOCAL
char cur_thread_placeholder
[];
461 inline ThreadState
*cur_thread() {
462 return reinterpret_cast<ThreadState
*>(cur_thread_placeholder
)->current
;
464 inline void cur_thread_init() {
465 ThreadState
*thr
= reinterpret_cast<ThreadState
*>(cur_thread_placeholder
);
466 if (UNLIKELY(!thr
->current
))
469 inline void set_cur_thread(ThreadState
*thr
) {
470 reinterpret_cast<ThreadState
*>(cur_thread_placeholder
)->current
= thr
;
472 inline void cur_thread_finalize() { }
473 #endif // SANITIZER_MAC || SANITIZER_ANDROID
474 #endif // SANITIZER_GO
476 class ThreadContext final
: public ThreadContextBase
{
478 explicit ThreadContext(int tid
);
481 u32 creation_stack_id
;
483 // Epoch at which the thread had started.
484 // If we see an event from the thread stamped by an older epoch,
485 // the event is from a dead thread that shared tid with this thread.
489 // Override superclass callbacks.
490 void OnDead() override
;
491 void OnJoined(void *arg
) override
;
492 void OnFinished() override
;
493 void OnStarted(void *arg
) override
;
494 void OnCreated(void *arg
) override
;
495 void OnReset() override
;
496 void OnDetached(void *arg
) override
;
501 bool operator==(const RacyStacks
&other
) const {
502 if (hash
[0] == other
.hash
[0] && hash
[1] == other
.hash
[1])
504 if (hash
[0] == other
.hash
[1] && hash
[1] == other
.hash
[0])
515 struct FiredSuppression
{
526 bool after_multithreaded_fork
;
533 int nmissed_expected
;
534 atomic_uint64_t last_symbolize_time_ns
;
536 void *background_thread
;
537 atomic_uint32_t stop_background_thread
;
539 ThreadRegistry
*thread_registry
;
542 Vector
<RacyStacks
> racy_stacks
;
543 Vector
<RacyAddress
> racy_addresses
;
544 // Number of fired suppressions may be large enough.
545 Mutex fired_suppressions_mtx
;
546 InternalMmapVector
<FiredSuppression
> fired_suppressions
;
549 ClockAlloc clock_alloc
;
554 u64 int_alloc_cnt
[MBlockTypeCount
];
555 u64 int_alloc_siz
[MBlockTypeCount
];
558 extern Context
*ctx
; // The one and the only global runtime context.
560 ALWAYS_INLINE Flags
*flags() {
564 struct ScopedIgnoreInterceptors
{
565 ScopedIgnoreInterceptors() {
567 cur_thread()->ignore_interceptors
++;
571 ~ScopedIgnoreInterceptors() {
573 cur_thread()->ignore_interceptors
--;
578 const char *GetObjectTypeFromTag(uptr tag
);
579 const char *GetReportHeaderFromTag(uptr tag
);
580 uptr
TagFromShadowStackFrame(uptr pc
);
582 class ScopedReportBase
{
584 void AddMemoryAccess(uptr addr
, uptr external_tag
, Shadow s
, StackTrace stack
,
585 const MutexSet
*mset
);
586 void AddStack(StackTrace stack
, bool suppressable
= false);
587 void AddThread(const ThreadContext
*tctx
, bool suppressable
= false);
588 void AddThread(int unique_tid
, bool suppressable
= false);
589 void AddUniqueTid(int unique_tid
);
590 void AddMutex(const SyncVar
*s
);
591 u64
AddMutex(u64 id
);
592 void AddLocation(uptr addr
, uptr size
);
593 void AddSleep(u32 stack_id
);
594 void SetCount(int count
);
596 const ReportDesc
*GetReport() const;
599 ScopedReportBase(ReportType typ
, uptr tag
);
604 // Symbolizer makes lots of intercepted calls. If we try to process them,
605 // at best it will cause deadlocks on internal mutexes.
606 ScopedIgnoreInterceptors ignore_interceptors_
;
608 void AddDeadMutex(u64 id
);
610 ScopedReportBase(const ScopedReportBase
&) = delete;
611 void operator=(const ScopedReportBase
&) = delete;
614 class ScopedReport
: public ScopedReportBase
{
616 explicit ScopedReport(ReportType typ
, uptr tag
= kExternalTagNone
);
620 ScopedErrorReportLock lock_
;
623 bool ShouldReport(ThreadState
*thr
, ReportType typ
);
624 ThreadContext
*IsThreadStackOrTls(uptr addr
, bool *is_stack
);
625 void RestoreStack(int tid
, const u64 epoch
, VarSizeStackTrace
*stk
,
626 MutexSet
*mset
, uptr
*tag
= nullptr);
628 // The stack could look like:
629 // <start> | <main> | <foo> | tag | <bar>
630 // This will extract the tag and keep:
631 // <start> | <main> | <foo> | <bar>
632 template<typename StackTraceTy
>
633 void ExtractTagFromStack(StackTraceTy
*stack
, uptr
*tag
= nullptr) {
634 if (stack
->size
< 2) return;
635 uptr possible_tag_pc
= stack
->trace
[stack
->size
- 2];
636 uptr possible_tag
= TagFromShadowStackFrame(possible_tag_pc
);
637 if (possible_tag
== kExternalTagNone
) return;
638 stack
->trace_buffer
[stack
->size
- 2] = stack
->trace_buffer
[stack
->size
- 1];
640 if (tag
) *tag
= possible_tag
;
643 template<typename StackTraceTy
>
644 void ObtainCurrentStack(ThreadState
*thr
, uptr toppc
, StackTraceTy
*stack
,
645 uptr
*tag
= nullptr) {
646 uptr size
= thr
->shadow_stack_pos
- thr
->shadow_stack
;
648 if (size
+ !!toppc
> kStackTraceMax
) {
649 start
= size
+ !!toppc
- kStackTraceMax
;
650 size
= kStackTraceMax
- !!toppc
;
652 stack
->Init(&thr
->shadow_stack
[start
], size
, toppc
);
653 ExtractTagFromStack(stack
, tag
);
656 #define GET_STACK_TRACE_FATAL(thr, pc) \
657 VarSizeStackTrace stack; \
658 ObtainCurrentStack(thr, pc, &stack); \
659 stack.ReverseOrder();
661 #if TSAN_COLLECT_STATS
662 void StatAggregate(u64
*dst
, u64
*src
);
663 void StatOutput(u64
*stat
);
666 void ALWAYS_INLINE
StatInc(ThreadState
*thr
, StatType typ
, u64 n
= 1) {
667 #if TSAN_COLLECT_STATS
671 void ALWAYS_INLINE
StatSet(ThreadState
*thr
, StatType typ
, u64 n
) {
672 #if TSAN_COLLECT_STATS
677 void MapShadow(uptr addr
, uptr size
);
678 void MapThreadTrace(uptr addr
, uptr size
, const char *name
);
679 void DontNeedShadowFor(uptr addr
, uptr size
);
680 void UnmapShadow(ThreadState
*thr
, uptr addr
, uptr size
);
681 void InitializeShadowMemory();
682 void InitializeInterceptors();
683 void InitializeLibIgnore();
684 void InitializeDynamicAnnotations();
686 void ForkBefore(ThreadState
*thr
, uptr pc
);
687 void ForkParentAfter(ThreadState
*thr
, uptr pc
);
688 void ForkChildAfter(ThreadState
*thr
, uptr pc
);
690 void ReportRace(ThreadState
*thr
);
691 bool OutputReport(ThreadState
*thr
, const ScopedReport
&srep
);
692 bool IsFiredSuppression(Context
*ctx
, ReportType type
, StackTrace trace
);
693 bool IsExpectedReport(uptr addr
, uptr size
);
694 void PrintMatchedBenignRaces();
696 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
697 # define DPrintf Printf
699 # define DPrintf(...)
702 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
703 # define DPrintf2 Printf
705 # define DPrintf2(...)
708 u32
CurrentStackId(ThreadState
*thr
, uptr pc
);
709 ReportStack
*SymbolizeStackId(u32 stack_id
);
710 void PrintCurrentStack(ThreadState
*thr
, uptr pc
);
711 void PrintCurrentStackSlow(uptr pc
); // uses libunwind
713 void Initialize(ThreadState
*thr
);
714 void MaybeSpawnBackgroundThread();
715 int Finalize(ThreadState
*thr
);
717 void OnUserAlloc(ThreadState
*thr
, uptr pc
, uptr p
, uptr sz
, bool write
);
718 void OnUserFree(ThreadState
*thr
, uptr pc
, uptr p
, bool write
);
720 void MemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
721 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
);
722 void MemoryAccessImpl(ThreadState
*thr
, uptr addr
,
723 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
,
724 u64
*shadow_mem
, Shadow cur
);
725 void MemoryAccessRange(ThreadState
*thr
, uptr pc
, uptr addr
,
726 uptr size
, bool is_write
);
727 void MemoryAccessRangeStep(ThreadState
*thr
, uptr pc
, uptr addr
,
728 uptr size
, uptr step
, bool is_write
);
729 void UnalignedMemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
730 int size
, bool kAccessIsWrite
, bool kIsAtomic
);
732 const int kSizeLog1
= 0;
733 const int kSizeLog2
= 1;
734 const int kSizeLog4
= 2;
735 const int kSizeLog8
= 3;
737 void ALWAYS_INLINE
MemoryRead(ThreadState
*thr
, uptr pc
,
738 uptr addr
, int kAccessSizeLog
) {
739 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, false, false);
742 void ALWAYS_INLINE
MemoryWrite(ThreadState
*thr
, uptr pc
,
743 uptr addr
, int kAccessSizeLog
) {
744 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, true, false);
747 void ALWAYS_INLINE
MemoryReadAtomic(ThreadState
*thr
, uptr pc
,
748 uptr addr
, int kAccessSizeLog
) {
749 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, false, true);
752 void ALWAYS_INLINE
MemoryWriteAtomic(ThreadState
*thr
, uptr pc
,
753 uptr addr
, int kAccessSizeLog
) {
754 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, true, true);
757 void MemoryResetRange(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
758 void MemoryRangeFreed(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
759 void MemoryRangeImitateWrite(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
760 void MemoryRangeImitateWriteOrResetRange(ThreadState
*thr
, uptr pc
, uptr addr
,
763 void ThreadIgnoreBegin(ThreadState
*thr
, uptr pc
, bool save_stack
= true);
764 void ThreadIgnoreEnd(ThreadState
*thr
, uptr pc
);
765 void ThreadIgnoreSyncBegin(ThreadState
*thr
, uptr pc
, bool save_stack
= true);
766 void ThreadIgnoreSyncEnd(ThreadState
*thr
, uptr pc
);
768 void FuncEntry(ThreadState
*thr
, uptr pc
);
769 void FuncExit(ThreadState
*thr
);
771 int ThreadCreate(ThreadState
*thr
, uptr pc
, uptr uid
, bool detached
);
772 void ThreadStart(ThreadState
*thr
, int tid
, tid_t os_id
,
773 ThreadType thread_type
);
774 void ThreadFinish(ThreadState
*thr
);
775 int ThreadConsumeTid(ThreadState
*thr
, uptr pc
, uptr uid
);
776 void ThreadJoin(ThreadState
*thr
, uptr pc
, int tid
);
777 void ThreadDetach(ThreadState
*thr
, uptr pc
, int tid
);
778 void ThreadFinalize(ThreadState
*thr
);
779 void ThreadSetName(ThreadState
*thr
, const char *name
);
780 int ThreadCount(ThreadState
*thr
);
781 void ProcessPendingSignals(ThreadState
*thr
);
782 void ThreadNotJoined(ThreadState
*thr
, uptr pc
, int tid
, uptr uid
);
784 Processor
*ProcCreate();
785 void ProcDestroy(Processor
*proc
);
786 void ProcWire(Processor
*proc
, ThreadState
*thr
);
787 void ProcUnwire(Processor
*proc
, ThreadState
*thr
);
789 // Note: the parameter is called flagz, because flags is already taken
790 // by the global function that returns flags.
791 void MutexCreate(ThreadState
*thr
, uptr pc
, uptr addr
, u32 flagz
= 0);
792 void MutexDestroy(ThreadState
*thr
, uptr pc
, uptr addr
, u32 flagz
= 0);
793 void MutexPreLock(ThreadState
*thr
, uptr pc
, uptr addr
, u32 flagz
= 0);
794 void MutexPostLock(ThreadState
*thr
, uptr pc
, uptr addr
, u32 flagz
= 0,
796 int MutexUnlock(ThreadState
*thr
, uptr pc
, uptr addr
, u32 flagz
= 0);
797 void MutexPreReadLock(ThreadState
*thr
, uptr pc
, uptr addr
, u32 flagz
= 0);
798 void MutexPostReadLock(ThreadState
*thr
, uptr pc
, uptr addr
, u32 flagz
= 0);
799 void MutexReadUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
800 void MutexReadOrWriteUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
801 void MutexRepair(ThreadState
*thr
, uptr pc
, uptr addr
); // call on EOWNERDEAD
802 void MutexInvalidAccess(ThreadState
*thr
, uptr pc
, uptr addr
);
804 void Acquire(ThreadState
*thr
, uptr pc
, uptr addr
);
805 // AcquireGlobal synchronizes the current thread with all other threads.
806 // In terms of happens-before relation, it draws a HB edge from all threads
807 // (where they happen to execute right now) to the current thread. We use it to
808 // handle Go finalizers. Namely, finalizer goroutine executes AcquireGlobal
809 // right before executing finalizers. This provides a coarse, but simple
810 // approximation of the actual required synchronization.
811 void AcquireGlobal(ThreadState
*thr
, uptr pc
);
812 void Release(ThreadState
*thr
, uptr pc
, uptr addr
);
813 void ReleaseStoreAcquire(ThreadState
*thr
, uptr pc
, uptr addr
);
814 void ReleaseStore(ThreadState
*thr
, uptr pc
, uptr addr
);
815 void AfterSleep(ThreadState
*thr
, uptr pc
);
816 void AcquireImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
817 void ReleaseImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
818 void ReleaseStoreAcquireImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
819 void ReleaseStoreImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
820 void AcquireReleaseImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
822 // The hacky call uses custom calling convention and an assembly thunk.
823 // It is considerably faster that a normal call for the caller
824 // if it is not executed (it is intended for slow paths from hot functions).
825 // The trick is that the call preserves all registers and the compiler
826 // does not treat it as a call.
827 // If it does not work for you, use normal call.
828 #if !SANITIZER_DEBUG && defined(__x86_64__) && !SANITIZER_MAC
829 // The caller may not create the stack frame for itself at all,
830 // so we create a reserve stack frame for it (1024b must be enough).
831 #define HACKY_CALL(f) \
832 __asm__ __volatile__("sub $1024, %%rsp;" \
833 CFI_INL_ADJUST_CFA_OFFSET(1024) \
834 ".hidden " #f "_thunk;" \
835 "call " #f "_thunk;" \
836 "add $1024, %%rsp;" \
837 CFI_INL_ADJUST_CFA_OFFSET(-1024) \
840 #define HACKY_CALL(f) f()
843 void TraceSwitch(ThreadState
*thr
);
844 uptr
TraceTopPC(ThreadState
*thr
);
847 Trace
*ThreadTrace(int tid
);
849 extern "C" void __tsan_trace_switch();
850 void ALWAYS_INLINE
TraceAddEvent(ThreadState
*thr
, FastState fs
,
851 EventType typ
, u64 addr
) {
852 if (!kCollectHistory
)
854 DCHECK_GE((int)typ
, 0);
855 DCHECK_LE((int)typ
, 7);
856 DCHECK_EQ(GetLsb(addr
, kEventPCBits
), addr
);
857 StatInc(thr
, StatEvents
);
858 u64 pos
= fs
.GetTracePos();
859 if (UNLIKELY((pos
% kTracePartSize
) == 0)) {
861 HACKY_CALL(__tsan_trace_switch
);
866 Event
*trace
= (Event
*)GetThreadTrace(fs
.tid());
867 Event
*evp
= &trace
[pos
];
868 Event ev
= (u64
)addr
| ((u64
)typ
<< kEventPCBits
);
873 uptr ALWAYS_INLINE
HeapEnd() {
874 return HeapMemEnd() + PrimaryAllocator::AdditionalSize();
878 ThreadState
*FiberCreate(ThreadState
*thr
, uptr pc
, unsigned flags
);
879 void FiberDestroy(ThreadState
*thr
, uptr pc
, ThreadState
*fiber
);
880 void FiberSwitch(ThreadState
*thr
, uptr pc
, ThreadState
*fiber
, unsigned flags
);
882 // These need to match __tsan_switch_to_fiber_* flags defined in
883 // tsan_interface.h. See documentation there as well.
884 enum FiberSwitchFlags
{
885 FiberSwitchFlagNoSync
= 1 << 0, // __tsan_switch_to_fiber_no_sync
888 } // namespace __tsan