1 //===-- tsan_rtl.cc -------------------------------------------------------===//
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
6 //===----------------------------------------------------------------------===//
8 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 // Main file (entry points) for the TSan run-time.
11 //===----------------------------------------------------------------------===//
13 #include "sanitizer_common/sanitizer_atomic.h"
14 #include "sanitizer_common/sanitizer_common.h"
15 #include "sanitizer_common/sanitizer_libc.h"
16 #include "sanitizer_common/sanitizer_stackdepot.h"
17 #include "sanitizer_common/sanitizer_placement_new.h"
18 #include "sanitizer_common/sanitizer_symbolizer.h"
19 #include "tsan_defs.h"
20 #include "tsan_platform.h"
22 #include "tsan_mman.h"
23 #include "tsan_suppressions.h"
24 #include "tsan_symbolize.h"
27 // <emmintrin.h> transitively includes <stdlib.h>,
28 // and it's prohibited to include std headers into tsan runtime.
29 // So we do this dirty trick.
30 #define _MM_MALLOC_H_INCLUDED
32 #include <emmintrin.h>
36 volatile int __tsan_resumed
= 0;
38 extern "C" void __tsan_resume() {
45 THREADLOCAL
char cur_thread_placeholder
[sizeof(ThreadState
)] ALIGNED(64);
47 static char ctx_placeholder
[sizeof(Context
)] ALIGNED(64);
50 // Can be overriden by a front-end.
51 #ifdef TSAN_EXTERNAL_HOOKS
52 bool OnFinalize(bool failed
);
55 SANITIZER_INTERFACE_ATTRIBUTE
56 bool WEAK
OnFinalize(bool failed
) {
59 SANITIZER_INTERFACE_ATTRIBUTE
60 void WEAK
OnInitialize() {}
63 static char thread_registry_placeholder
[sizeof(ThreadRegistry
)];
65 static ThreadContextBase
*CreateThreadContext(u32 tid
) {
66 // Map thread trace when context is created.
67 MapThreadTrace(GetThreadTrace(tid
), TraceSize() * sizeof(Event
));
68 MapThreadTrace(GetThreadTraceHeader(tid
), sizeof(Trace
));
69 new(ThreadTrace(tid
)) Trace();
70 void *mem
= internal_alloc(MBlockThreadContex
, sizeof(ThreadContext
));
71 return new(mem
) ThreadContext(tid
);
75 static const u32 kThreadQuarantineSize
= 16;
77 static const u32 kThreadQuarantineSize
= 64;
82 , report_mtx(MutexTypeReport
, StatMtxReport
)
85 , thread_registry(new(thread_registry_placeholder
) ThreadRegistry(
86 CreateThreadContext
, kMaxTid
, kThreadQuarantineSize
, kMaxTidReuse
))
87 , racy_stacks(MBlockRacyStacks
)
88 , racy_addresses(MBlockRacyAddresses
)
89 , fired_suppressions(8) {
92 // The objects are allocated in TLS, so one may rely on zero-initialization.
93 ThreadState::ThreadState(Context
*ctx
, int tid
, int unique_id
, u64 epoch
,
95 uptr stk_addr
, uptr stk_size
,
96 uptr tls_addr
, uptr tls_size
)
97 : fast_state(tid
, epoch
)
98 // Do not touch these, rely on zero initialization,
99 // they may be accessed before the ctor.
100 // , ignore_reads_and_writes()
101 // , ignore_interceptors()
102 , clock(tid
, reuse_count
)
104 , jmp_bufs(MBlockJmpBuf
)
107 , unique_id(unique_id
)
113 , last_sleep_clock(tid
)
118 static void MemoryProfiler(Context
*ctx
, fd_t fd
, int i
) {
120 uptr n_running_threads
;
121 ctx
->thread_registry
->GetNumberOfThreads(&n_threads
, &n_running_threads
);
122 InternalScopedBuffer
<char> buf(4096);
123 WriteMemoryProfile(buf
.data(), buf
.size(), n_threads
, n_running_threads
);
124 internal_write(fd
, buf
.data(), internal_strlen(buf
.data()));
127 static void BackgroundThread(void *arg
) {
129 // This is a non-initialized non-user thread, nothing to see here.
130 // We don't use ScopedIgnoreInterceptors, because we want ignores to be
131 // enabled even when the thread function exits (e.g. during pthread thread
133 cur_thread()->ignore_interceptors
++;
135 const u64 kMs2Ns
= 1000 * 1000;
137 fd_t mprof_fd
= kInvalidFd
;
138 if (flags()->profile_memory
&& flags()->profile_memory
[0]) {
139 if (internal_strcmp(flags()->profile_memory
, "stdout") == 0) {
141 } else if (internal_strcmp(flags()->profile_memory
, "stderr") == 0) {
144 InternalScopedBuffer
<char> filename(4096);
145 internal_snprintf(filename
.data(), filename
.size(), "%s.%d",
146 flags()->profile_memory
, (int)internal_getpid());
147 uptr openrv
= OpenFile(filename
.data(), true);
148 if (internal_iserror(openrv
)) {
149 Printf("ThreadSanitizer: failed to open memory profile file '%s'\n",
157 u64 last_flush
= NanoTime();
160 atomic_load(&ctx
->stop_background_thread
, memory_order_relaxed
) == 0;
163 u64 now
= NanoTime();
165 // Flush memory if requested.
166 if (flags()->flush_memory_ms
> 0) {
167 if (last_flush
+ flags()->flush_memory_ms
* kMs2Ns
< now
) {
168 VPrintf(1, "ThreadSanitizer: periodic memory flush\n");
170 last_flush
= NanoTime();
173 // GetRSS can be expensive on huge programs, so don't do it every 100ms.
174 if (flags()->memory_limit_mb
> 0) {
176 uptr limit
= uptr(flags()->memory_limit_mb
) << 20;
177 VPrintf(1, "ThreadSanitizer: memory flush check"
178 " RSS=%llu LAST=%llu LIMIT=%llu\n",
179 (u64
)rss
>> 20, (u64
)last_rss
>> 20, (u64
)limit
>> 20);
180 if (2 * rss
> limit
+ last_rss
) {
181 VPrintf(1, "ThreadSanitizer: flushing memory due to RSS\n");
184 VPrintf(1, "ThreadSanitizer: memory flushed RSS=%llu\n", (u64
)rss
>>20);
189 // Write memory profile if requested.
190 if (mprof_fd
!= kInvalidFd
)
191 MemoryProfiler(ctx
, mprof_fd
, i
);
194 // Flush symbolizer cache if requested.
195 if (flags()->flush_symbolizer_ms
> 0) {
196 u64 last
= atomic_load(&ctx
->last_symbolize_time_ns
,
197 memory_order_relaxed
);
198 if (last
!= 0 && last
+ flags()->flush_symbolizer_ms
* kMs2Ns
< now
) {
199 Lock
l(&ctx
->report_mtx
);
200 SpinMutexLock
l2(&CommonSanitizerReportMutex
);
202 atomic_store(&ctx
->last_symbolize_time_ns
, 0, memory_order_relaxed
);
209 static void StartBackgroundThread() {
210 ctx
->background_thread
= internal_start_thread(&BackgroundThread
, 0);
214 static void StopBackgroundThread() {
215 atomic_store(&ctx
->stop_background_thread
, 1, memory_order_relaxed
);
216 internal_join_thread(ctx
->background_thread
);
217 ctx
->background_thread
= 0;
221 void DontNeedShadowFor(uptr addr
, uptr size
) {
222 uptr shadow_beg
= MemToShadow(addr
);
223 uptr shadow_end
= MemToShadow(addr
+ size
);
224 FlushUnneededShadowMemory(shadow_beg
, shadow_end
- shadow_beg
);
227 void MapShadow(uptr addr
, uptr size
) {
228 // Global data is not 64K aligned, but there are no adjacent mappings,
229 // so we can get away with unaligned mapping.
230 // CHECK_EQ(addr, addr & ~((64 << 10) - 1)); // windows wants 64K alignment
231 MmapFixedNoReserve(MemToShadow(addr
), size
* kShadowMultiplier
);
233 // Meta shadow is 2:1, so tread carefully.
234 static bool data_mapped
= false;
235 static uptr mapped_meta_end
= 0;
236 uptr meta_begin
= (uptr
)MemToMeta(addr
);
237 uptr meta_end
= (uptr
)MemToMeta(addr
+ size
);
238 meta_begin
= RoundDownTo(meta_begin
, 64 << 10);
239 meta_end
= RoundUpTo(meta_end
, 64 << 10);
241 // First call maps data+bss.
243 MmapFixedNoReserve(meta_begin
, meta_end
- meta_begin
);
245 // Mapping continous heap.
246 // Windows wants 64K alignment.
247 meta_begin
= RoundDownTo(meta_begin
, 64 << 10);
248 meta_end
= RoundUpTo(meta_end
, 64 << 10);
249 if (meta_end
<= mapped_meta_end
)
251 if (meta_begin
< mapped_meta_end
)
252 meta_begin
= mapped_meta_end
;
253 MmapFixedNoReserve(meta_begin
, meta_end
- meta_begin
);
254 mapped_meta_end
= meta_end
;
256 VPrintf(2, "mapped meta shadow for (%p-%p) at (%p-%p)\n",
257 addr
, addr
+size
, meta_begin
, meta_end
);
260 void MapThreadTrace(uptr addr
, uptr size
) {
261 DPrintf("#0: Mapping trace at %p-%p(0x%zx)\n", addr
, addr
+ size
, size
);
262 CHECK_GE(addr
, kTraceMemBeg
);
263 CHECK_LE(addr
+ size
, kTraceMemEnd
);
264 CHECK_EQ(addr
, addr
& ~((64 << 10) - 1)); // windows wants 64K alignment
265 uptr addr1
= (uptr
)MmapFixedNoReserve(addr
, size
);
267 Printf("FATAL: ThreadSanitizer can not mmap thread trace (%p/%p->%p)\n",
273 static void CheckShadowMapping() {
274 for (uptr i
= 0; i
< ARRAY_SIZE(UserRegions
); i
+= 2) {
275 const uptr beg
= UserRegions
[i
];
276 const uptr end
= UserRegions
[i
+ 1];
277 VPrintf(3, "checking shadow region %p-%p\n", beg
, end
);
278 for (uptr p0
= beg
; p0
<= end
; p0
+= (end
- beg
) / 4) {
279 for (int x
= -1; x
<= 1; x
++) {
280 const uptr p
= p0
+ x
;
281 if (p
< beg
|| p
>= end
)
283 const uptr s
= MemToShadow(p
);
284 VPrintf(3, " checking pointer %p -> %p\n", p
, s
);
286 CHECK(IsShadowMem(s
));
287 CHECK_EQ(p
& ~(kShadowCell
- 1), ShadowToMem(s
));
288 const uptr m
= (uptr
)MemToMeta(p
);
295 void Initialize(ThreadState
*thr
) {
296 // Thread safe because done before all threads exist.
297 static bool is_initialized
= false;
300 is_initialized
= true;
301 // We are not ready to handle interceptors yet.
302 ScopedIgnoreInterceptors ignore
;
303 SanitizerToolName
= "ThreadSanitizer";
304 // Install tool-specific callbacks in sanitizer_common.
305 SetCheckFailedCallback(TsanCheckFailed
);
307 ctx
= new(ctx_placeholder
) Context
;
308 const char *options
= GetEnv(kTsanOptionsEnv
);
309 InitializeFlags(&ctx
->flags
, options
);
311 InitializeAllocator();
313 InitializeInterceptors();
314 CheckShadowMapping();
315 InitializePlatform();
317 InitializeDynamicAnnotations();
319 InitializeShadowMemory();
321 // Setup correct file descriptor for error reports.
322 __sanitizer_set_report_path(common_flags()->log_path
);
323 InitializeSuppressions();
325 InitializeLibIgnore();
326 Symbolizer::GetOrInit()->AddHooks(EnterSymbolizer
, ExitSymbolizer
);
328 StartBackgroundThread();
330 SetSandboxingCallback(StopBackgroundThread
);
332 if (common_flags()->detect_deadlocks
)
333 ctx
->dd
= DDetector::Create(flags());
335 VPrintf(1, "***** Running under ThreadSanitizer v2 (pid %d) *****\n",
336 (int)internal_getpid());
338 // Initialize thread 0.
339 int tid
= ThreadCreate(thr
, 0, 0, true);
341 ThreadStart(thr
, tid
, internal_getpid());
342 ctx
->initialized
= true;
344 if (flags()->stop_on_start
) {
345 Printf("ThreadSanitizer is suspended at startup (pid %d)."
346 " Call __tsan_resume().\n",
347 (int)internal_getpid());
348 while (__tsan_resumed
== 0) {}
354 int Finalize(ThreadState
*thr
) {
357 if (flags()->atexit_sleep_ms
> 0 && ThreadCount(thr
) > 1)
358 SleepForMillis(flags()->atexit_sleep_ms
);
360 // Wait for pending reports.
361 ctx
->report_mtx
.Lock();
362 CommonSanitizerReportMutex
.Lock();
363 CommonSanitizerReportMutex
.Unlock();
364 ctx
->report_mtx
.Unlock();
367 if (common_flags()->verbosity
)
368 AllocatorPrintStats();
373 if (ctx
->nreported
) {
376 Printf("ThreadSanitizer: reported %d warnings\n", ctx
->nreported
);
378 Printf("Found %d data race(s)\n", ctx
->nreported
);
382 if (ctx
->nmissed_expected
) {
384 Printf("ThreadSanitizer: missed %d expected races\n",
385 ctx
->nmissed_expected
);
388 if (common_flags()->print_suppressions
)
389 PrintMatchedSuppressions();
391 if (flags()->print_benign
)
392 PrintMatchedBenignRaces();
395 failed
= OnFinalize(failed
);
397 StatAggregate(ctx
->stat
, thr
->stat
);
398 StatOutput(ctx
->stat
);
399 return failed
? flags()->exitcode
: 0;
403 void ForkBefore(ThreadState
*thr
, uptr pc
) {
404 ctx
->thread_registry
->Lock();
405 ctx
->report_mtx
.Lock();
408 void ForkParentAfter(ThreadState
*thr
, uptr pc
) {
409 ctx
->report_mtx
.Unlock();
410 ctx
->thread_registry
->Unlock();
413 void ForkChildAfter(ThreadState
*thr
, uptr pc
) {
414 ctx
->report_mtx
.Unlock();
415 ctx
->thread_registry
->Unlock();
418 ctx
->thread_registry
->GetNumberOfThreads(0, 0, &nthread
/* alive threads */);
419 VPrintf(1, "ThreadSanitizer: forked new process with pid %d,"
420 " parent had %d threads\n", (int)internal_getpid(), (int)nthread
);
422 internal_start_thread(&BackgroundThread
, 0);
424 // We've just forked a multi-threaded process. We cannot reasonably function
425 // after that (some mutexes may be locked before fork). So just enable
426 // ignores for everything in the hope that we will exec soon.
427 ctx
->after_multithreaded_fork
= true;
428 thr
->ignore_interceptors
++;
429 ThreadIgnoreBegin(thr
, pc
);
430 ThreadIgnoreSyncBegin(thr
, pc
);
437 void GrowShadowStack(ThreadState
*thr
) {
438 const int sz
= thr
->shadow_stack_end
- thr
->shadow_stack
;
439 const int newsz
= 2 * sz
;
440 uptr
*newstack
= (uptr
*)internal_alloc(MBlockShadowStack
,
441 newsz
* sizeof(uptr
));
442 internal_memcpy(newstack
, thr
->shadow_stack
, sz
* sizeof(uptr
));
443 internal_free(thr
->shadow_stack
);
444 thr
->shadow_stack
= newstack
;
445 thr
->shadow_stack_pos
= newstack
+ sz
;
446 thr
->shadow_stack_end
= newstack
+ newsz
;
450 u32
CurrentStackId(ThreadState
*thr
, uptr pc
) {
451 if (thr
->shadow_stack_pos
== 0) // May happen during bootstrap.
455 DCHECK_LT(thr
->shadow_stack_pos
, thr
->shadow_stack_end
);
457 if (thr
->shadow_stack_pos
== thr
->shadow_stack_end
)
458 GrowShadowStack(thr
);
460 thr
->shadow_stack_pos
[0] = pc
;
461 thr
->shadow_stack_pos
++;
463 u32 id
= StackDepotPut(
464 StackTrace(thr
->shadow_stack
, thr
->shadow_stack_pos
- thr
->shadow_stack
));
466 thr
->shadow_stack_pos
--;
470 void TraceSwitch(ThreadState
*thr
) {
472 Trace
*thr_trace
= ThreadTrace(thr
->tid
);
473 Lock
l(&thr_trace
->mtx
);
474 unsigned trace
= (thr
->fast_state
.epoch() / kTracePartSize
) % TraceParts();
475 TraceHeader
*hdr
= &thr_trace
->headers
[trace
];
476 hdr
->epoch0
= thr
->fast_state
.epoch();
477 ObtainCurrentStack(thr
, 0, &hdr
->stack0
);
478 hdr
->mset0
= thr
->mset
;
482 Trace
*ThreadTrace(int tid
) {
483 return (Trace
*)GetThreadTraceHeader(tid
);
486 uptr
TraceTopPC(ThreadState
*thr
) {
487 Event
*events
= (Event
*)GetThreadTrace(thr
->tid
);
488 uptr pc
= events
[thr
->fast_state
.GetTracePos()];
493 return (uptr
)(1ull << (kTracePartSizeBits
+ flags()->history_size
+ 1));
497 return TraceSize() / kTracePartSize
;
501 extern "C" void __tsan_trace_switch() {
502 TraceSwitch(cur_thread());
505 extern "C" void __tsan_report_race() {
506 ReportRace(cur_thread());
511 Shadow
LoadShadow(u64
*p
) {
512 u64 raw
= atomic_load((atomic_uint64_t
*)p
, memory_order_relaxed
);
517 void StoreShadow(u64
*sp
, u64 s
) {
518 atomic_store((atomic_uint64_t
*)sp
, s
, memory_order_relaxed
);
522 void StoreIfNotYetStored(u64
*sp
, u64
*s
) {
528 void HandleRace(ThreadState
*thr
, u64
*shadow_mem
,
529 Shadow cur
, Shadow old
) {
530 thr
->racy_state
[0] = cur
.raw();
531 thr
->racy_state
[1] = old
.raw();
532 thr
->racy_shadow_addr
= shadow_mem
;
534 HACKY_CALL(__tsan_report_race
);
540 static inline bool HappensBefore(Shadow old
, ThreadState
*thr
) {
541 return thr
->clock
.get(old
.TidWithIgnore()) >= old
.epoch();
545 void MemoryAccessImpl1(ThreadState
*thr
, uptr addr
,
546 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
,
547 u64
*shadow_mem
, Shadow cur
) {
548 StatInc(thr
, StatMop
);
549 StatInc(thr
, kAccessIsWrite
? StatMopWrite
: StatMopRead
);
550 StatInc(thr
, (StatType
)(StatMop1
+ kAccessSizeLog
));
552 // This potentially can live in an MMX/SSE scratch register.
553 // The required intrinsics are:
554 // __m128i _mm_move_epi64(__m128i*);
555 // _mm_storel_epi64(u64*, __m128i);
556 u64 store_word
= cur
.raw();
558 // scan all the shadow values and dispatch to 4 categories:
559 // same, replace, candidate and race (see comments below).
560 // we consider only 3 cases regarding access sizes:
561 // equal, intersect and not intersect. initially I considered
562 // larger and smaller as well, it allowed to replace some
563 // 'candidates' with 'same' or 'replace', but I think
564 // it's just not worth it (performance- and complexity-wise).
567 if (kShadowCnt
== 1) {
569 #include "tsan_update_shadow_word_inl.h"
570 } else if (kShadowCnt
== 2) {
572 #include "tsan_update_shadow_word_inl.h"
574 #include "tsan_update_shadow_word_inl.h"
575 } else if (kShadowCnt
== 4) {
577 #include "tsan_update_shadow_word_inl.h"
579 #include "tsan_update_shadow_word_inl.h"
581 #include "tsan_update_shadow_word_inl.h"
583 #include "tsan_update_shadow_word_inl.h"
584 } else if (kShadowCnt
== 8) {
586 #include "tsan_update_shadow_word_inl.h"
588 #include "tsan_update_shadow_word_inl.h"
590 #include "tsan_update_shadow_word_inl.h"
592 #include "tsan_update_shadow_word_inl.h"
594 #include "tsan_update_shadow_word_inl.h"
596 #include "tsan_update_shadow_word_inl.h"
598 #include "tsan_update_shadow_word_inl.h"
600 #include "tsan_update_shadow_word_inl.h"
605 // we did not find any races and had already stored
606 // the current access info, so we are done
607 if (LIKELY(store_word
== 0))
609 // choose a random candidate slot and replace it
610 StoreShadow(shadow_mem
+ (cur
.epoch() % kShadowCnt
), store_word
);
611 StatInc(thr
, StatShadowReplace
);
614 HandleRace(thr
, shadow_mem
, cur
, old
);
618 void UnalignedMemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
619 int size
, bool kAccessIsWrite
, bool kIsAtomic
) {
622 int kAccessSizeLog
= kSizeLog1
;
623 if (size
>= 8 && (addr
& ~7) == ((addr
+ 7) & ~7)) {
625 kAccessSizeLog
= kSizeLog8
;
626 } else if (size
>= 4 && (addr
& ~7) == ((addr
+ 3) & ~7)) {
628 kAccessSizeLog
= kSizeLog4
;
629 } else if (size
>= 2 && (addr
& ~7) == ((addr
+ 1) & ~7)) {
631 kAccessSizeLog
= kSizeLog2
;
633 MemoryAccess(thr
, pc
, addr
, kAccessSizeLog
, kAccessIsWrite
, kIsAtomic
);
640 bool ContainsSameAccessSlow(u64
*s
, u64 a
, u64 sync_epoch
, bool is_write
) {
642 for (uptr i
= 0; i
< kShadowCnt
; i
++) {
643 Shadow
old(LoadShadow(&s
[i
]));
644 if (Shadow::Addr0AndSizeAreEqual(cur
, old
) &&
645 old
.TidWithIgnore() == cur
.TidWithIgnore() &&
646 old
.epoch() > sync_epoch
&&
647 old
.IsAtomic() == cur
.IsAtomic() &&
648 old
.IsRead() <= cur
.IsRead())
654 #if defined(__SSE3__) && TSAN_SHADOW_COUNT == 4
655 #define SHUF(v0, v1, i0, i1, i2, i3) _mm_castps_si128(_mm_shuffle_ps( \
656 _mm_castsi128_ps(v0), _mm_castsi128_ps(v1), \
657 (i0)*1 + (i1)*4 + (i2)*16 + (i3)*64))
659 bool ContainsSameAccessFast(u64
*s
, u64 a
, u64 sync_epoch
, bool is_write
) {
660 // This is an optimized version of ContainsSameAccessSlow.
661 // load current access into access[0:63]
662 const m128 access
= _mm_cvtsi64_si128(a
);
663 // duplicate high part of access in addr0:
664 // addr0[0:31] = access[32:63]
665 // addr0[32:63] = access[32:63]
666 // addr0[64:95] = access[32:63]
667 // addr0[96:127] = access[32:63]
668 const m128 addr0
= SHUF(access
, access
, 1, 1, 1, 1);
669 // load 4 shadow slots
670 const m128 shadow0
= _mm_load_si128((__m128i
*)s
);
671 const m128 shadow1
= _mm_load_si128((__m128i
*)s
+ 1);
672 // load high parts of 4 shadow slots into addr_vect:
673 // addr_vect[0:31] = shadow0[32:63]
674 // addr_vect[32:63] = shadow0[96:127]
675 // addr_vect[64:95] = shadow1[32:63]
676 // addr_vect[96:127] = shadow1[96:127]
677 m128 addr_vect
= SHUF(shadow0
, shadow1
, 1, 3, 1, 3);
679 // set IsRead bit in addr_vect
680 const m128 rw_mask1
= _mm_cvtsi64_si128(1<<15);
681 const m128 rw_mask
= SHUF(rw_mask1
, rw_mask1
, 0, 0, 0, 0);
682 addr_vect
= _mm_or_si128(addr_vect
, rw_mask
);
684 // addr0 == addr_vect?
685 const m128 addr_res
= _mm_cmpeq_epi32(addr0
, addr_vect
);
686 // epoch1[0:63] = sync_epoch
687 const m128 epoch1
= _mm_cvtsi64_si128(sync_epoch
);
688 // epoch[0:31] = sync_epoch[0:31]
689 // epoch[32:63] = sync_epoch[0:31]
690 // epoch[64:95] = sync_epoch[0:31]
691 // epoch[96:127] = sync_epoch[0:31]
692 const m128 epoch
= SHUF(epoch1
, epoch1
, 0, 0, 0, 0);
693 // load low parts of shadow cell epochs into epoch_vect:
694 // epoch_vect[0:31] = shadow0[0:31]
695 // epoch_vect[32:63] = shadow0[64:95]
696 // epoch_vect[64:95] = shadow1[0:31]
697 // epoch_vect[96:127] = shadow1[64:95]
698 const m128 epoch_vect
= SHUF(shadow0
, shadow1
, 0, 2, 0, 2);
699 // epoch_vect >= sync_epoch?
700 const m128 epoch_res
= _mm_cmpgt_epi32(epoch_vect
, epoch
);
701 // addr_res & epoch_res
702 const m128 res
= _mm_and_si128(addr_res
, epoch_res
);
706 // mask[15] = res[127]
707 const int mask
= _mm_movemask_epi8(res
);
713 bool ContainsSameAccess(u64
*s
, u64 a
, u64 sync_epoch
, bool is_write
) {
714 #if defined(__SSE3__) && TSAN_SHADOW_COUNT == 4
715 bool res
= ContainsSameAccessFast(s
, a
, sync_epoch
, is_write
);
716 // NOTE: this check can fail if the shadow is concurrently mutated
718 DCHECK_EQ(res
, ContainsSameAccessSlow(s
, a
, sync_epoch
, is_write
));
721 return ContainsSameAccessSlow(s
, a
, sync_epoch
, is_write
);
726 void MemoryAccess(ThreadState
*thr
, uptr pc
, uptr addr
,
727 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
) {
728 u64
*shadow_mem
= (u64
*)MemToShadow(addr
);
729 DPrintf2("#%d: MemoryAccess: @%p %p size=%d"
730 " is_write=%d shadow_mem=%p {%zx, %zx, %zx, %zx}\n",
731 (int)thr
->fast_state
.tid(), (void*)pc
, (void*)addr
,
732 (int)(1 << kAccessSizeLog
), kAccessIsWrite
, shadow_mem
,
733 (uptr
)shadow_mem
[0], (uptr
)shadow_mem
[1],
734 (uptr
)shadow_mem
[2], (uptr
)shadow_mem
[3]);
736 if (!IsAppMem(addr
)) {
737 Printf("Access to non app mem %zx\n", addr
);
738 DCHECK(IsAppMem(addr
));
740 if (!IsShadowMem((uptr
)shadow_mem
)) {
741 Printf("Bad shadow addr %p (%zx)\n", shadow_mem
, addr
);
742 DCHECK(IsShadowMem((uptr
)shadow_mem
));
746 if (kCppMode
&& *shadow_mem
== kShadowRodata
) {
747 // Access to .rodata section, no races here.
748 // Measurements show that it can be 10-20% of all memory accesses.
749 StatInc(thr
, StatMop
);
750 StatInc(thr
, kAccessIsWrite
? StatMopWrite
: StatMopRead
);
751 StatInc(thr
, (StatType
)(StatMop1
+ kAccessSizeLog
));
752 StatInc(thr
, StatMopRodata
);
756 FastState fast_state
= thr
->fast_state
;
757 if (fast_state
.GetIgnoreBit()) {
758 StatInc(thr
, StatMop
);
759 StatInc(thr
, kAccessIsWrite
? StatMopWrite
: StatMopRead
);
760 StatInc(thr
, (StatType
)(StatMop1
+ kAccessSizeLog
));
761 StatInc(thr
, StatMopIgnored
);
765 Shadow
cur(fast_state
);
766 cur
.SetAddr0AndSizeLog(addr
& 7, kAccessSizeLog
);
767 cur
.SetWrite(kAccessIsWrite
);
768 cur
.SetAtomic(kIsAtomic
);
770 if (LIKELY(ContainsSameAccess(shadow_mem
, cur
.raw(),
771 thr
->fast_synch_epoch
, kAccessIsWrite
))) {
772 StatInc(thr
, StatMop
);
773 StatInc(thr
, kAccessIsWrite
? StatMopWrite
: StatMopRead
);
774 StatInc(thr
, (StatType
)(StatMop1
+ kAccessSizeLog
));
775 StatInc(thr
, StatMopSame
);
779 if (kCollectHistory
) {
780 fast_state
.IncrementEpoch();
781 thr
->fast_state
= fast_state
;
782 TraceAddEvent(thr
, fast_state
, EventTypeMop
, pc
);
783 cur
.IncrementEpoch();
786 MemoryAccessImpl1(thr
, addr
, kAccessSizeLog
, kAccessIsWrite
, kIsAtomic
,
790 // Called by MemoryAccessRange in tsan_rtl_thread.cc
792 void MemoryAccessImpl(ThreadState
*thr
, uptr addr
,
793 int kAccessSizeLog
, bool kAccessIsWrite
, bool kIsAtomic
,
794 u64
*shadow_mem
, Shadow cur
) {
795 if (LIKELY(ContainsSameAccess(shadow_mem
, cur
.raw(),
796 thr
->fast_synch_epoch
, kAccessIsWrite
))) {
797 StatInc(thr
, StatMop
);
798 StatInc(thr
, kAccessIsWrite
? StatMopWrite
: StatMopRead
);
799 StatInc(thr
, (StatType
)(StatMop1
+ kAccessSizeLog
));
800 StatInc(thr
, StatMopSame
);
804 MemoryAccessImpl1(thr
, addr
, kAccessSizeLog
, kAccessIsWrite
, kIsAtomic
,
808 static void MemoryRangeSet(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
,
815 uptr offset
= addr
% kShadowCell
;
817 offset
= kShadowCell
- offset
;
823 DCHECK_EQ(addr
% 8, 0);
824 // If a user passes some insane arguments (memset(0)),
825 // let it just crash as usual.
826 if (!IsAppMem(addr
) || !IsAppMem(addr
+ size
- 1))
828 // Don't want to touch lots of shadow memory.
829 // If a program maps 10MB stack, there is no need reset the whole range.
830 size
= (size
+ (kShadowCell
- 1)) & ~(kShadowCell
- 1);
831 // UnmapOrDie/MmapFixedNoReserve does not work on Windows,
832 // so we do it only for C/C++.
833 if (kGoMode
|| size
< common_flags()->clear_shadow_mmap_threshold
) {
834 u64
*p
= (u64
*)MemToShadow(addr
);
835 CHECK(IsShadowMem((uptr
)p
));
836 CHECK(IsShadowMem((uptr
)(p
+ size
* kShadowCnt
/ kShadowCell
- 1)));
837 // FIXME: may overwrite a part outside the region
838 for (uptr i
= 0; i
< size
/ kShadowCell
* kShadowCnt
;) {
840 for (uptr j
= 1; j
< kShadowCnt
; j
++)
844 // The region is big, reset only beginning and end.
845 const uptr kPageSize
= 4096;
846 u64
*begin
= (u64
*)MemToShadow(addr
);
847 u64
*end
= begin
+ size
/ kShadowCell
* kShadowCnt
;
849 // Set at least first kPageSize/2 to page boundary.
850 while ((p
< begin
+ kPageSize
/ kShadowSize
/ 2) || ((uptr
)p
% kPageSize
)) {
852 for (uptr j
= 1; j
< kShadowCnt
; j
++)
855 // Reset middle part.
857 p
= RoundDown(end
, kPageSize
);
858 UnmapOrDie((void*)p1
, (uptr
)p
- (uptr
)p1
);
859 MmapFixedNoReserve((uptr
)p1
, (uptr
)p
- (uptr
)p1
);
863 for (uptr j
= 1; j
< kShadowCnt
; j
++)
869 void MemoryResetRange(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
) {
870 MemoryRangeSet(thr
, pc
, addr
, size
, 0);
873 void MemoryRangeFreed(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
) {
874 // Processing more than 1k (4k of shadow) is expensive,
875 // can cause excessive memory consumption (user does not necessary touch
876 // the whole range) and most likely unnecessary.
879 CHECK_EQ(thr
->is_freeing
, false);
880 thr
->is_freeing
= true;
881 MemoryAccessRange(thr
, pc
, addr
, size
, true);
882 thr
->is_freeing
= false;
883 if (kCollectHistory
) {
884 thr
->fast_state
.IncrementEpoch();
885 TraceAddEvent(thr
, thr
->fast_state
, EventTypeMop
, pc
);
887 Shadow
s(thr
->fast_state
);
891 s
.SetAddr0AndSizeLog(0, 3);
892 MemoryRangeSet(thr
, pc
, addr
, size
, s
.raw());
895 void MemoryRangeImitateWrite(ThreadState
*thr
, uptr pc
, uptr addr
, uptr size
) {
896 if (kCollectHistory
) {
897 thr
->fast_state
.IncrementEpoch();
898 TraceAddEvent(thr
, thr
->fast_state
, EventTypeMop
, pc
);
900 Shadow
s(thr
->fast_state
);
903 s
.SetAddr0AndSizeLog(0, 3);
904 MemoryRangeSet(thr
, pc
, addr
, size
, s
.raw());
908 void FuncEntry(ThreadState
*thr
, uptr pc
) {
909 StatInc(thr
, StatFuncEnter
);
910 DPrintf2("#%d: FuncEntry %p\n", (int)thr
->fast_state
.tid(), (void*)pc
);
911 if (kCollectHistory
) {
912 thr
->fast_state
.IncrementEpoch();
913 TraceAddEvent(thr
, thr
->fast_state
, EventTypeFuncEnter
, pc
);
916 // Shadow stack maintenance can be replaced with
917 // stack unwinding during trace switch (which presumably must be faster).
918 DCHECK_GE(thr
->shadow_stack_pos
, thr
->shadow_stack
);
920 DCHECK_LT(thr
->shadow_stack_pos
, thr
->shadow_stack_end
);
922 if (thr
->shadow_stack_pos
== thr
->shadow_stack_end
)
923 GrowShadowStack(thr
);
925 thr
->shadow_stack_pos
[0] = pc
;
926 thr
->shadow_stack_pos
++;
930 void FuncExit(ThreadState
*thr
) {
931 StatInc(thr
, StatFuncExit
);
932 DPrintf2("#%d: FuncExit\n", (int)thr
->fast_state
.tid());
933 if (kCollectHistory
) {
934 thr
->fast_state
.IncrementEpoch();
935 TraceAddEvent(thr
, thr
->fast_state
, EventTypeFuncExit
, 0);
938 DCHECK_GT(thr
->shadow_stack_pos
, thr
->shadow_stack
);
940 DCHECK_LT(thr
->shadow_stack_pos
, thr
->shadow_stack_end
);
942 thr
->shadow_stack_pos
--;
945 void ThreadIgnoreBegin(ThreadState
*thr
, uptr pc
) {
946 DPrintf("#%d: ThreadIgnoreBegin\n", thr
->tid
);
947 thr
->ignore_reads_and_writes
++;
948 CHECK_GT(thr
->ignore_reads_and_writes
, 0);
949 thr
->fast_state
.SetIgnoreBit();
951 if (!ctx
->after_multithreaded_fork
)
952 thr
->mop_ignore_set
.Add(CurrentStackId(thr
, pc
));
956 void ThreadIgnoreEnd(ThreadState
*thr
, uptr pc
) {
957 DPrintf("#%d: ThreadIgnoreEnd\n", thr
->tid
);
958 thr
->ignore_reads_and_writes
--;
959 CHECK_GE(thr
->ignore_reads_and_writes
, 0);
960 if (thr
->ignore_reads_and_writes
== 0) {
961 thr
->fast_state
.ClearIgnoreBit();
963 thr
->mop_ignore_set
.Reset();
968 void ThreadIgnoreSyncBegin(ThreadState
*thr
, uptr pc
) {
969 DPrintf("#%d: ThreadIgnoreSyncBegin\n", thr
->tid
);
971 CHECK_GT(thr
->ignore_sync
, 0);
973 if (!ctx
->after_multithreaded_fork
)
974 thr
->sync_ignore_set
.Add(CurrentStackId(thr
, pc
));
978 void ThreadIgnoreSyncEnd(ThreadState
*thr
, uptr pc
) {
979 DPrintf("#%d: ThreadIgnoreSyncEnd\n", thr
->tid
);
981 CHECK_GE(thr
->ignore_sync
, 0);
983 if (thr
->ignore_sync
== 0)
984 thr
->sync_ignore_set
.Reset();
988 bool MD5Hash::operator==(const MD5Hash
&other
) const {
989 return hash
[0] == other
.hash
[0] && hash
[1] == other
.hash
[1];
993 void build_consistency_debug() {}
995 void build_consistency_release() {}
998 #if TSAN_COLLECT_STATS
999 void build_consistency_stats() {}
1001 void build_consistency_nostats() {}
1004 #if TSAN_SHADOW_COUNT == 1
1005 void build_consistency_shadow1() {}
1006 #elif TSAN_SHADOW_COUNT == 2
1007 void build_consistency_shadow2() {}
1008 #elif TSAN_SHADOW_COUNT == 4
1009 void build_consistency_shadow4() {}
1011 void build_consistency_shadow8() {}
1014 } // namespace __tsan
1017 // Must be included in this file to make sure everything is inlined.
1018 #include "tsan_interface_inl.h"