1 //===-- tsan_rtl.h ----------------------------------------------*- C++ -*-===//
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
6 //===----------------------------------------------------------------------===//
8 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 // Main internal TSan header file.
13 // - C++ run-time should not be used (static CTORs, RTTI, exceptions, static
14 // function-scope locals)
15 // - All functions/classes/etc reside in namespace __tsan, except for those
16 // declared in tsan_interface.h.
17 // - Platform-specific files should be used instead of ifdefs (*).
18 // - No system headers included in header files (*).
19 // - Platform specific headres included only into platform-specific files (*).
21 // (*) Except when inlining is critical for performance.
22 //===----------------------------------------------------------------------===//
27 #include "sanitizer_common/sanitizer_allocator.h"
28 #include "sanitizer_common/sanitizer_allocator_internal.h"
29 #include "sanitizer_common/sanitizer_asm.h"
30 #include "sanitizer_common/sanitizer_common.h"
31 #include "sanitizer_common/sanitizer_deadlock_detector_interface.h"
32 #include "sanitizer_common/sanitizer_libignore.h"
33 #include "sanitizer_common/sanitizer_suppressions.h"
34 #include "sanitizer_common/sanitizer_thread_registry.h"
35 #include "tsan_clock.h"
36 #include "tsan_defs.h"
37 #include "tsan_flags.h"
38 #include "tsan_sync.h"
39 #include "tsan_trace.h"
40 #include "tsan_vector.h"
41 #include "tsan_report.h"
42 #include "tsan_platform.h"
43 #include "tsan_mutexset.h"
44 #include "tsan_ignoreset.h"
45 #include "tsan_stack_trace.h"
47 #if SANITIZER_WORDSIZE != 64
48 # error "ThreadSanitizer is supported only on 64-bit platforms"
54 const uptr kAllocatorSpace
= 0x7d0000000000ULL
;
55 const uptr kAllocatorSize
= 0x10000000000ULL
; // 1T.
57 struct MapUnmapCallback
;
58 typedef SizeClassAllocator64
<kAllocatorSpace
, kAllocatorSize
, 0,
59 DefaultSizeClassMap
, MapUnmapCallback
> PrimaryAllocator
;
60 typedef SizeClassAllocatorLocalCache
<PrimaryAllocator
> AllocatorCache
;
61 typedef LargeMmapAllocator
<MapUnmapCallback
> SecondaryAllocator
;
62 typedef CombinedAllocator
<PrimaryAllocator
, AllocatorCache
,
63 SecondaryAllocator
> Allocator
;
64 Allocator
*allocator();
67 void TsanCheckFailed(const char *file
, int line
, const char *cond
,
70 const u64 kShadowRodata
= (u64
)-1; // .rodata shadow marker
72 // FastState (from most significant bit):
80 FastState(u64 tid
, u64 epoch
) {
81 x_
= tid
<< kTidShift
;
83 DCHECK_EQ(tid
, this->tid());
84 DCHECK_EQ(epoch
, this->epoch());
85 DCHECK_EQ(GetIgnoreBit(), false);
88 explicit FastState(u64 x
)
97 u64 res
= (x_
& ~kIgnoreBit
) >> kTidShift
;
101 u64
TidWithIgnore() const {
102 u64 res
= x_
>> kTidShift
;
107 u64 res
= x_
& ((1ull << kClkBits
) - 1);
111 void IncrementEpoch() {
112 u64 old_epoch
= epoch();
114 DCHECK_EQ(old_epoch
+ 1, epoch());
118 void SetIgnoreBit() { x_
|= kIgnoreBit
; }
119 void ClearIgnoreBit() { x_
&= ~kIgnoreBit
; }
120 bool GetIgnoreBit() const { return (s64
)x_
< 0; }
122 void SetHistorySize(int hs
) {
125 x_
= (x_
& ~(kHistoryMask
<< kHistoryShift
)) | (u64(hs
) << kHistoryShift
);
129 int GetHistorySize() const {
130 return (int)((x_
>> kHistoryShift
) & kHistoryMask
);
133 void ClearHistorySize() {
138 u64
GetTracePos() const {
139 const int hs
= GetHistorySize();
140 // When hs == 0, the trace consists of 2 parts.
141 const u64 mask
= (1ull << (kTracePartSizeBits
+ hs
+ 1)) - 1;
142 return epoch() & mask
;
147 static const int kTidShift
= 64 - kTidBits
- 1;
148 static const u64 kIgnoreBit
= 1ull << 63;
149 static const u64 kFreedBit
= 1ull << 63;
150 static const u64 kHistoryShift
= kClkBits
;
151 static const u64 kHistoryMask
= 7;
155 // Shadow (from most significant bit):
163 class Shadow
: public FastState
{
165 explicit Shadow(u64 x
)
169 explicit Shadow(const FastState
&s
)
174 void SetAddr0AndSizeLog(u64 addr0
, unsigned kAccessSizeLog
) {
175 DCHECK_EQ((x_
>> kClkBits
) & 31, 0);
177 DCHECK_LE(kAccessSizeLog
, 3);
178 x_
|= ((kAccessSizeLog
<< 3) | addr0
) << kClkBits
;
179 DCHECK_EQ(kAccessSizeLog
, size_log());
180 DCHECK_EQ(addr0
, this->addr0());
183 void SetWrite(unsigned kAccessIsWrite
) {
184 DCHECK_EQ(x_
& kReadBit
, 0);
187 DCHECK_EQ(kAccessIsWrite
, IsWrite());
190 void SetAtomic(bool kIsAtomic
) {
194 DCHECK_EQ(IsAtomic(), kIsAtomic
);
197 bool IsAtomic() const {
198 return x_
& kAtomicBit
;
201 bool IsZero() const {
205 static inline bool TidsAreEqual(const Shadow s1
, const Shadow s2
) {
206 u64 shifted_xor
= (s1
.x_
^ s2
.x_
) >> kTidShift
;
207 DCHECK_EQ(shifted_xor
== 0, s1
.TidWithIgnore() == s2
.TidWithIgnore());
208 return shifted_xor
== 0;
212 bool Addr0AndSizeAreEqual(const Shadow s1
, const Shadow s2
) {
213 u64 masked_xor
= ((s1
.x_
^ s2
.x_
) >> kClkBits
) & 31;
214 return masked_xor
== 0;
217 static ALWAYS_INLINE
bool TwoRangesIntersect(Shadow s1
, Shadow s2
,
218 unsigned kS2AccessSize
) {
220 u64 diff
= s1
.addr0() - s2
.addr0();
221 if ((s64
)diff
< 0) { // s1.addr0 < s2.addr0 // NOLINT
222 // if (s1.addr0() + size1) > s2.addr0()) return true;
223 if (s1
.size() > -diff
)
226 // if (s2.addr0() + kS2AccessSize > s1.addr0()) return true;
227 if (kS2AccessSize
> diff
)
230 DCHECK_EQ(res
, TwoRangesIntersectSlow(s1
, s2
));
231 DCHECK_EQ(res
, TwoRangesIntersectSlow(s2
, s1
));
235 u64 ALWAYS_INLINE
addr0() const { return (x_
>> kClkBits
) & 7; }
236 u64 ALWAYS_INLINE
size() const { return 1ull << size_log(); }
237 bool ALWAYS_INLINE
IsWrite() const { return !IsRead(); }
238 bool ALWAYS_INLINE
IsRead() const { return x_
& kReadBit
; }
240 // The idea behind the freed bit is as follows.
241 // When the memory is freed (or otherwise unaccessible) we write to the shadow
242 // values with tid/epoch related to the free and the freed bit set.
243 // During memory accesses processing the freed bit is considered
244 // as msb of tid. So any access races with shadow with freed bit set
245 // (it is as if write from a thread with which we never synchronized before).
246 // This allows us to detect accesses to freed memory w/o additional
247 // overheads in memory access processing and at the same time restore
248 // tid/epoch of free.
253 bool IsFreed() const {
254 return x_
& kFreedBit
;
257 bool GetFreedAndReset() {
258 bool res
= x_
& kFreedBit
;
263 bool ALWAYS_INLINE
IsBothReadsOrAtomic(bool kIsWrite
, bool kIsAtomic
) const {
264 bool v
= x_
& ((u64(kIsWrite
^ 1) << kReadShift
)
265 | (u64(kIsAtomic
) << kAtomicShift
));
266 DCHECK_EQ(v
, (!IsWrite() && !kIsWrite
) || (IsAtomic() && kIsAtomic
));
270 bool ALWAYS_INLINE
IsRWNotWeaker(bool kIsWrite
, bool kIsAtomic
) const {
271 bool v
= ((x_
>> kReadShift
) & 3)
272 <= u64((kIsWrite
^ 1) | (kIsAtomic
<< 1));
273 DCHECK_EQ(v
, (IsAtomic() < kIsAtomic
) ||
274 (IsAtomic() == kIsAtomic
&& !IsWrite() <= !kIsWrite
));
278 bool ALWAYS_INLINE
IsRWWeakerOrEqual(bool kIsWrite
, bool kIsAtomic
) const {
279 bool v
= ((x_
>> kReadShift
) & 3)
280 >= u64((kIsWrite
^ 1) | (kIsAtomic
<< 1));
281 DCHECK_EQ(v
, (IsAtomic() > kIsAtomic
) ||
282 (IsAtomic() == kIsAtomic
&& !IsWrite() >= !kIsWrite
));
287 static const u64 kReadShift
= 5 + kClkBits
;
288 static const u64 kReadBit
= 1ull << kReadShift
;
289 static const u64 kAtomicShift
= 6 + kClkBits
;
290 static const u64 kAtomicBit
= 1ull << kAtomicShift
;
292 u64
size_log() const { return (x_
>> (3 + kClkBits
)) & 3; }
294 static bool TwoRangesIntersectSlow(const Shadow s1
, const Shadow s2
) {
295 if (s1
.addr0() == s2
.addr0()) return true;
296 if (s1
.addr0() < s2
.addr0() && s1
.addr0() + s1
.size() > s2
.addr0())
298 if (s2
.addr0() < s1
.addr0() && s2
.addr0() + s2
.size() > s1
.addr0())
304 struct SignalContext
;
310 bool in_blocking_func
;
311 uptr in_signal_handler
;
312 uptr
*shadow_stack_pos
;
315 // This struct is stored in TLS.
317 FastState fast_state
;
318 // Synch epoch represents the threads's epoch before the last synchronization
319 // action. It allows to reduce number of shadow state updates.
320 // For example, fast_synch_epoch=100, last write to addr X was at epoch=150,
321 // if we are processing write to X from the same thread at epoch=200,
322 // we do nothing, because both writes happen in the same 'synch epoch'.
323 // That is, if another memory access does not race with the former write,
324 // it does not race with the latter as well.
325 // QUESTION: can we can squeeze this into ThreadState::Fast?
326 // E.g. ThreadState::Fast is a 44-bit, 32 are taken by synch_epoch and 12 are
327 // taken by epoch between synchs.
328 // This way we can save one load from tls.
329 u64 fast_synch_epoch
;
330 // This is a slow path flag. On fast path, fast_state.GetIgnoreBit() is read.
331 // We do not distinguish beteween ignoring reads and writes
332 // for better performance.
333 int ignore_reads_and_writes
;
335 // Go does not support ignores.
337 IgnoreSet mop_ignore_set
;
338 IgnoreSet sync_ignore_set
;
340 // C/C++ uses fixed size shadow stack embed into Trace.
341 // Go uses malloc-allocated shadow stack with dynamic size.
343 uptr
*shadow_stack_end
;
344 uptr
*shadow_stack_pos
;
345 u64
*racy_shadow_addr
;
350 AllocatorCache alloc_cache
;
351 InternalAllocatorCache internal_alloc_cache
;
352 Vector
<JmpBuf
> jmp_bufs
;
353 int ignore_interceptors
;
369 InternalDeadlockDetector internal_deadlock_detector
;
370 DDPhysicalThread
*dd_pt
;
371 DDLogicalThread
*dd_lt
;
373 atomic_uintptr_t in_signal_handler
;
374 SignalContext
*signal_ctx
;
376 DenseSlabAllocCache block_cache
;
377 DenseSlabAllocCache sync_cache
;
378 DenseSlabAllocCache clock_cache
;
381 u32 last_sleep_stack_id
;
382 ThreadClock last_sleep_clock
;
385 // Set in regions of runtime that must be signal-safe and fork-safe.
386 // If set, malloc must not be called.
389 explicit ThreadState(Context
*ctx
, int tid
, int unique_id
, u64 epoch
,
390 unsigned reuse_count
,
391 uptr stk_addr
, uptr stk_size
,
392 uptr tls_addr
, uptr tls_size
);
396 __attribute__((tls_model("initial-exec")))
397 extern THREADLOCAL
char cur_thread_placeholder
[];
398 INLINE ThreadState
*cur_thread() {
399 return reinterpret_cast<ThreadState
*>(&cur_thread_placeholder
);
403 class ThreadContext
: public ThreadContextBase
{
405 explicit ThreadContext(int tid
);
408 u32 creation_stack_id
;
410 // Epoch at which the thread had started.
411 // If we see an event from the thread stamped by an older epoch,
412 // the event is from a dead thread that shared tid with this thread.
416 // Override superclass callbacks.
418 void OnJoined(void *arg
);
420 void OnStarted(void *arg
);
421 void OnCreated(void *arg
);
423 void OnDetached(void *arg
);
428 bool operator==(const RacyStacks
&other
) const {
429 if (hash
[0] == other
.hash
[0] && hash
[1] == other
.hash
[1])
431 if (hash
[0] == other
.hash
[1] && hash
[1] == other
.hash
[0])
442 struct FiredSuppression
{
452 bool after_multithreaded_fork
;
458 int nmissed_expected
;
459 atomic_uint64_t last_symbolize_time_ns
;
461 void *background_thread
;
462 atomic_uint32_t stop_background_thread
;
464 ThreadRegistry
*thread_registry
;
466 Vector
<RacyStacks
> racy_stacks
;
467 Vector
<RacyAddress
> racy_addresses
;
468 // Number of fired suppressions may be large enough.
469 InternalMmapVector
<FiredSuppression
> fired_suppressions
;
472 ClockAlloc clock_alloc
;
477 u64 int_alloc_cnt
[MBlockTypeCount
];
478 u64 int_alloc_siz
[MBlockTypeCount
];
481 extern Context
*ctx
; // The one and the only global runtime context.
483 struct ScopedIgnoreInterceptors
{
484 ScopedIgnoreInterceptors() {
486 cur_thread()->ignore_interceptors
++;
490 ~ScopedIgnoreInterceptors() {
492 cur_thread()->ignore_interceptors
--;
499 explicit ScopedReport(ReportType typ
);
502 void AddMemoryAccess(uptr addr
, Shadow s
, const StackTrace
*stack
,
503 const MutexSet
*mset
);
504 void AddStack(const StackTrace
*stack
, bool suppressable
= false);
505 void AddThread(const ThreadContext
*tctx
, bool suppressable
= false);
506 void AddThread(int unique_tid
, bool suppressable
= false);
507 void AddUniqueTid(int unique_tid
);
508 void AddMutex(const SyncVar
*s
);
509 u64
AddMutex(u64 id
);
510 void AddLocation(uptr addr
, uptr size
);
511 void AddSleep(u32 stack_id
);
512 void SetCount(int count
);
514 const ReportDesc
*GetReport() const;
518 // Symbolizer makes lots of intercepted calls. If we try to process them,
519 // at best it will cause deadlocks on internal mutexes.
520 ScopedIgnoreInterceptors ignore_interceptors_
;
522 void AddDeadMutex(u64 id
);
524 ScopedReport(const ScopedReport
&);
525 void operator = (const ScopedReport
&);
528 void RestoreStack(int tid
, const u64 epoch
, StackTrace
*stk
, MutexSet
*mset
);
530 void StatAggregate(u64
*dst
, u64
*src
);
531 void StatOutput(u64
*stat
);
532 void ALWAYS_INLINE
StatInc(ThreadState
*thr
, StatType typ
, u64 n
= 1) {
536 void ALWAYS_INLINE
StatSet(ThreadState
*thr
, StatType typ
, u64 n
) {
541 void MapShadow(uptr addr
, uptr size
);
542 void MapThreadTrace(uptr addr
, uptr size
);
543 void DontNeedShadowFor(uptr addr
, uptr size
);
544 void InitializeShadowMemory();
545 void InitializeInterceptors();
546 void InitializeLibIgnore();
547 void InitializeDynamicAnnotations();
549 void ForkBefore(ThreadState
*thr
, uptr pc
);
550 void ForkParentAfter(ThreadState
*thr
, uptr pc
);
551 void ForkChildAfter(ThreadState
*thr
, uptr pc
);
553 void ReportRace(ThreadState
*thr
);
554 bool OutputReport(ThreadState
*thr
, const ScopedReport
&srep
);
555 bool IsFiredSuppression(Context
*ctx
,
556 const ScopedReport
&srep
,
557 const StackTrace
&trace
);
558 bool IsExpectedReport(uptr addr
, uptr size
);
559 void PrintMatchedBenignRaces();
560 bool FrameIsInternal(const ReportStack
*frame
);
561 ReportStack
*SkipTsanInternalFrames(ReportStack
*ent
);
563 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 1
564 # define DPrintf Printf
566 # define DPrintf(...)
569 #if defined(TSAN_DEBUG_OUTPUT) && TSAN_DEBUG_OUTPUT >= 2
570 # define DPrintf2 Printf
572 # define DPrintf2(...)
575 u32
CurrentStackId(ThreadState
*thr
, uptr pc
);
576 ReportStack
*SymbolizeStackId(u32 stack_id
);
577 void PrintCurrentStack(ThreadState
*thr
, uptr pc
);
578 void PrintCurrentStackSlow(); // uses libunwind
580 void Initialize(ThreadState
*thr
);
581 int Finalize(ThreadState
*thr
);
583 void OnUserAlloc(ThreadState
*thr
, uptr pc
, uptr p
, uptr sz
, bool write
);
584 void OnUserFree(ThreadState
*thr
, uptr pc
, uptr p
, bool write
);
586 void MemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
587 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
);
588 void MemoryAccessImpl(ThreadState
*thr
, uptr addr
,
589 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
,
590 u64
*shadow_mem
, Shadow cur
);
591 void MemoryAccessRange(ThreadState
*thr
, uptr pc
, uptr addr
,
592 uptr size
, bool is_write
);
593 void MemoryAccessRangeStep(ThreadState
*thr
, uptr pc
, uptr addr
,
594 uptr size
, uptr step
, bool is_write
);
595 void UnalignedMemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
596 int size
, bool kAccessIsWrite
, bool kIsAtomic
);
598 const int kSizeLog1
= 0;
599 const int kSizeLog2
= 1;
600 const int kSizeLog4
= 2;
601 const int kSizeLog8
= 3;
603 void ALWAYS_INLINE
MemoryRead(ThreadState
*thr
, uptr pc
,
604 uptr addr
, int kAccessSizeLog
) {
605 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, false, false);
608 void ALWAYS_INLINE
MemoryWrite(ThreadState
*thr
, uptr pc
,
609 uptr addr
, int kAccessSizeLog
) {
610 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, true, false);
613 void ALWAYS_INLINE
MemoryReadAtomic(ThreadState
*thr
, uptr pc
,
614 uptr addr
, int kAccessSizeLog
) {
615 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, false, true);
618 void ALWAYS_INLINE
MemoryWriteAtomic(ThreadState
*thr
, uptr pc
,
619 uptr addr
, int kAccessSizeLog
) {
620 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, true, true);
623 void MemoryResetRange(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
624 void MemoryRangeFreed(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
625 void MemoryRangeImitateWrite(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
);
627 void ThreadIgnoreBegin(ThreadState
*thr
, uptr pc
);
628 void ThreadIgnoreEnd(ThreadState
*thr
, uptr pc
);
629 void ThreadIgnoreSyncBegin(ThreadState
*thr
, uptr pc
);
630 void ThreadIgnoreSyncEnd(ThreadState
*thr
, uptr pc
);
632 void FuncEntry(ThreadState
*thr
, uptr pc
);
633 void FuncExit(ThreadState
*thr
);
635 int ThreadCreate(ThreadState
*thr
, uptr pc
, uptr uid
, bool detached
);
636 void ThreadStart(ThreadState
*thr
, int tid
, uptr os_id
);
637 void ThreadFinish(ThreadState
*thr
);
638 int ThreadTid(ThreadState
*thr
, uptr pc
, uptr uid
);
639 void ThreadJoin(ThreadState
*thr
, uptr pc
, int tid
);
640 void ThreadDetach(ThreadState
*thr
, uptr pc
, int tid
);
641 void ThreadFinalize(ThreadState
*thr
);
642 void ThreadSetName(ThreadState
*thr
, const char *name
);
643 int ThreadCount(ThreadState
*thr
);
644 void ProcessPendingSignals(ThreadState
*thr
);
646 void MutexCreate(ThreadState
*thr
, uptr pc
, uptr addr
,
647 bool rw
, bool recursive
, bool linker_init
);
648 void MutexDestroy(ThreadState
*thr
, uptr pc
, uptr addr
);
649 void MutexLock(ThreadState
*thr
, uptr pc
, uptr addr
, int rec
= 1,
650 bool try_lock
= false);
651 int MutexUnlock(ThreadState
*thr
, uptr pc
, uptr addr
, bool all
= false);
652 void MutexReadLock(ThreadState
*thr
, uptr pc
, uptr addr
, bool try_lock
= false);
653 void MutexReadUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
654 void MutexReadOrWriteUnlock(ThreadState
*thr
, uptr pc
, uptr addr
);
655 void MutexRepair(ThreadState
*thr
, uptr pc
, uptr addr
); // call on EOWNERDEAD
657 void Acquire(ThreadState
*thr
, uptr pc
, uptr addr
);
658 void AcquireGlobal(ThreadState
*thr
, uptr pc
);
659 void Release(ThreadState
*thr
, uptr pc
, uptr addr
);
660 void ReleaseStore(ThreadState
*thr
, uptr pc
, uptr addr
);
661 void AfterSleep(ThreadState
*thr
, uptr pc
);
662 void AcquireImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
663 void ReleaseImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
664 void ReleaseStoreImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
665 void AcquireReleaseImpl(ThreadState
*thr
, uptr pc
, SyncClock
*c
);
667 // The hacky call uses custom calling convention and an assembly thunk.
668 // It is considerably faster that a normal call for the caller
669 // if it is not executed (it is intended for slow paths from hot functions).
670 // The trick is that the call preserves all registers and the compiler
671 // does not treat it as a call.
672 // If it does not work for you, use normal call.
674 // The caller may not create the stack frame for itself at all,
675 // so we create a reserve stack frame for it (1024b must be enough).
676 #define HACKY_CALL(f) \
677 __asm__ __volatile__("sub $1024, %%rsp;" \
678 CFI_INL_ADJUST_CFA_OFFSET(1024) \
679 ".hidden " #f "_thunk;" \
680 "call " #f "_thunk;" \
681 "add $1024, %%rsp;" \
682 CFI_INL_ADJUST_CFA_OFFSET(-1024) \
685 #define HACKY_CALL(f) f()
688 void TraceSwitch(ThreadState
*thr
);
689 uptr
TraceTopPC(ThreadState
*thr
);
692 Trace
*ThreadTrace(int tid
);
694 extern "C" void __tsan_trace_switch();
695 void ALWAYS_INLINE
TraceAddEvent(ThreadState
*thr
, FastState fs
,
696 EventType typ
, u64 addr
) {
697 if (!kCollectHistory
)
699 DCHECK_GE((int)typ
, 0);
700 DCHECK_LE((int)typ
, 7);
701 DCHECK_EQ(GetLsb(addr
, 61), addr
);
702 StatInc(thr
, StatEvents
);
703 u64 pos
= fs
.GetTracePos();
704 if (UNLIKELY((pos
% kTracePartSize
) == 0)) {
706 HACKY_CALL(__tsan_trace_switch
);
711 Event
*trace
= (Event
*)GetThreadTrace(fs
.tid());
712 Event
*evp
= &trace
[pos
];
713 Event ev
= (u64
)addr
| ((u64
)typ
<< 61);
717 } // namespace __tsan