1 //===-- tsan_interceptors_posix.cpp ---------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is a part of ThreadSanitizer (TSan), a race detector.
11 // FIXME: move as many interceptors as possible into
12 // sanitizer_common/sanitizer_common_interceptors.inc
13 //===----------------------------------------------------------------------===//
15 #include "sanitizer_common/sanitizer_atomic.h"
16 #include "sanitizer_common/sanitizer_errno.h"
17 #include "sanitizer_common/sanitizer_libc.h"
18 #include "sanitizer_common/sanitizer_linux.h"
19 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
20 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
21 #include "sanitizer_common/sanitizer_placement_new.h"
22 #include "sanitizer_common/sanitizer_posix.h"
23 #include "sanitizer_common/sanitizer_stacktrace.h"
24 #include "sanitizer_common/sanitizer_tls_get_addr.h"
25 #include "interception/interception.h"
26 #include "tsan_interceptors.h"
27 #include "tsan_interface.h"
28 #include "tsan_platform.h"
29 #include "tsan_suppressions.h"
31 #include "tsan_mman.h"
36 using namespace __tsan
;
38 #if SANITIZER_FREEBSD || SANITIZER_MAC
39 #define stdout __stdoutp
40 #define stderr __stderrp
44 #define dirfd(dirp) (*(int *)(dirp))
45 #define fileno_unlocked(fp) \
46 (((__sanitizer_FILE *)fp)->_file == -1 \
48 : (int)(unsigned short)(((__sanitizer_FILE *)fp)->_file))
50 #define stdout ((__sanitizer_FILE*)&__sF[1])
51 #define stderr ((__sanitizer_FILE*)&__sF[2])
53 #define nanosleep __nanosleep50
54 #define vfork __vfork14
58 const int kSigCount
= 129;
60 const int kSigCount
= 65;
65 u64 opaque
[768 / sizeof(u64
) + 1];
69 // The size is determined by looking at sizeof of real ucontext_t on linux.
70 u64 opaque
[936 / sizeof(u64
) + 1];
74 #if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1 || \
76 #define PTHREAD_ABI_BASE "GLIBC_2.3.2"
77 #elif defined(__aarch64__) || SANITIZER_PPC64V2
78 #define PTHREAD_ABI_BASE "GLIBC_2.17"
81 extern "C" int pthread_attr_init(void *attr
);
82 extern "C" int pthread_attr_destroy(void *attr
);
83 DECLARE_REAL(int, pthread_attr_getdetachstate
, void *, void *)
84 extern "C" int pthread_attr_setstacksize(void *attr
, uptr stacksize
);
85 extern "C" int pthread_atfork(void (*prepare
)(void), void (*parent
)(void),
87 extern "C" int pthread_key_create(unsigned *key
, void (*destructor
)(void* v
));
88 extern "C" int pthread_setspecific(unsigned key
, const void *v
);
89 DECLARE_REAL(int, pthread_mutexattr_gettype
, void *, void *)
90 DECLARE_REAL(int, fflush
, __sanitizer_FILE
*fp
)
91 DECLARE_REAL_AND_INTERCEPTOR(void *, malloc
, uptr size
)
92 DECLARE_REAL_AND_INTERCEPTOR(void, free
, void *ptr
)
93 extern "C" void *pthread_self();
94 extern "C" void _exit(int status
);
96 extern "C" int fileno_unlocked(void *stream
);
97 extern "C" int dirfd(void *dirp
);
100 extern __sanitizer_FILE __sF
[];
102 extern __sanitizer_FILE
*stdout
, *stderr
;
104 #if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
105 const int PTHREAD_MUTEX_RECURSIVE
= 1;
106 const int PTHREAD_MUTEX_RECURSIVE_NP
= 1;
108 const int PTHREAD_MUTEX_RECURSIVE
= 2;
109 const int PTHREAD_MUTEX_RECURSIVE_NP
= 2;
111 #if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
112 const int EPOLL_CTL_ADD
= 1;
114 const int SIGILL
= 4;
115 const int SIGTRAP
= 5;
116 const int SIGABRT
= 6;
117 const int SIGFPE
= 8;
118 const int SIGSEGV
= 11;
119 const int SIGPIPE
= 13;
120 const int SIGTERM
= 15;
121 #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
122 const int SIGBUS
= 10;
123 const int SIGSYS
= 12;
125 const int SIGBUS
= 7;
126 const int SIGSYS
= 31;
128 void *const MAP_FAILED
= (void*)-1;
130 const int PTHREAD_BARRIER_SERIAL_THREAD
= 1234567;
132 const int PTHREAD_BARRIER_SERIAL_THREAD
= -1;
134 const int MAP_FIXED
= 0x10;
136 typedef __sanitizer::u16 mode_t
;
138 // From /usr/include/unistd.h
139 # define F_ULOCK 0 /* Unlock a previously locked region. */
140 # define F_LOCK 1 /* Lock a region for exclusive use. */
141 # define F_TLOCK 2 /* Test and lock a region for exclusive use. */
142 # define F_TEST 3 /* Test a region for other processes locks. */
144 #if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
145 const int SA_SIGINFO
= 0x40;
146 const int SIG_SETMASK
= 3;
147 #elif defined(__mips__)
148 const int SA_SIGINFO
= 8;
149 const int SIG_SETMASK
= 3;
151 const int SA_SIGINFO
= 4;
152 const int SIG_SETMASK
= 2;
155 #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
156 (!cur_thread_init()->is_inited)
161 __sanitizer_siginfo siginfo
;
165 struct ThreadSignalContext
{
167 atomic_uintptr_t in_blocking_func
;
168 SignalDesc pending_signals
[kSigCount
];
169 // emptyset and oldset are too big for stack.
170 __sanitizer_sigset_t emptyset
;
171 __sanitizer_sigset_t oldset
;
174 // The sole reason tsan wraps atexit callbacks is to establish synchronization
175 // between callback setup and callback execution.
181 // InterceptorContext holds all global data required for interceptors.
182 // It's explicitly constructed in InitializeInterceptors with placement new
183 // and is never destroyed. This allows usage of members with non-trivial
184 // constructors and destructors.
185 struct InterceptorContext
{
186 // The object is 64-byte aligned, because we want hot data to be located
187 // in a single cache line if possible (it's accessed in every interceptor).
188 ALIGNED(64) LibIgnore libignore
;
189 __sanitizer_sigaction sigactions
[kSigCount
];
190 #if !SANITIZER_MAC && !SANITIZER_NETBSD
191 unsigned finalize_key
;
195 Vector
<struct AtExitCtx
*> AtExitStack
;
197 InterceptorContext() : libignore(LINKER_INITIALIZED
), atexit_mu(MutexTypeAtExit
), AtExitStack() {}
200 static ALIGNED(64) char interceptor_placeholder
[sizeof(InterceptorContext
)];
201 InterceptorContext
*interceptor_ctx() {
202 return reinterpret_cast<InterceptorContext
*>(&interceptor_placeholder
[0]);
205 LibIgnore
*libignore() {
206 return &interceptor_ctx()->libignore
;
209 void InitializeLibIgnore() {
210 const SuppressionContext
&supp
= *Suppressions();
211 const uptr n
= supp
.SuppressionCount();
212 for (uptr i
= 0; i
< n
; i
++) {
213 const Suppression
*s
= supp
.SuppressionAt(i
);
214 if (0 == internal_strcmp(s
->type
, kSuppressionLib
))
215 libignore()->AddIgnoredLibrary(s
->templ
);
217 if (flags()->ignore_noninstrumented_modules
)
218 libignore()->IgnoreNoninstrumentedModules(true);
219 libignore()->OnLibraryLoaded(0);
222 // The following two hooks can be used by for cooperative scheduling when
224 #ifdef TSAN_EXTERNAL_HOOKS
225 void OnPotentiallyBlockingRegionBegin();
226 void OnPotentiallyBlockingRegionEnd();
228 SANITIZER_WEAK_CXX_DEFAULT_IMPL
void OnPotentiallyBlockingRegionBegin() {}
229 SANITIZER_WEAK_CXX_DEFAULT_IMPL
void OnPotentiallyBlockingRegionEnd() {}
232 } // namespace __tsan
234 static ThreadSignalContext
*SigCtx(ThreadState
*thr
) {
235 ThreadSignalContext
*ctx
= (ThreadSignalContext
*)thr
->signal_ctx
;
236 if (ctx
== 0 && !thr
->is_dead
) {
237 ctx
= (ThreadSignalContext
*)MmapOrDie(sizeof(*ctx
), "ThreadSignalContext");
238 MemoryResetRange(thr
, (uptr
)&SigCtx
, (uptr
)ctx
, sizeof(*ctx
));
239 thr
->signal_ctx
= ctx
;
244 ScopedInterceptor::ScopedInterceptor(ThreadState
*thr
, const char *fname
,
246 : thr_(thr
), in_ignored_lib_(false), ignoring_(false) {
248 if (!thr_
->is_inited
) return;
249 if (!thr_
->ignore_interceptors
) FuncEntry(thr
, pc
);
250 DPrintf("#%d: intercept %s()\n", thr_
->tid
, fname
);
252 !thr_
->in_ignored_lib
&& (flags()->ignore_interceptors_accesses
||
253 libignore()->IsIgnored(pc
, &in_ignored_lib_
));
257 ScopedInterceptor::~ScopedInterceptor() {
258 if (!thr_
->is_inited
) return;
260 if (!thr_
->ignore_interceptors
) {
261 ProcessPendingSignals(thr_
);
263 CheckedMutex::CheckNoLocks();
268 void ScopedInterceptor::EnableIgnoresImpl() {
269 ThreadIgnoreBegin(thr_
, 0);
270 if (flags()->ignore_noninstrumented_modules
)
271 thr_
->suppress_reports
++;
272 if (in_ignored_lib_
) {
273 DCHECK(!thr_
->in_ignored_lib
);
274 thr_
->in_ignored_lib
= true;
279 void ScopedInterceptor::DisableIgnoresImpl() {
280 ThreadIgnoreEnd(thr_
);
281 if (flags()->ignore_noninstrumented_modules
)
282 thr_
->suppress_reports
--;
283 if (in_ignored_lib_
) {
284 DCHECK(thr_
->in_ignored_lib
);
285 thr_
->in_ignored_lib
= false;
289 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
290 #if SANITIZER_FREEBSD
291 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
292 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
293 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
294 #elif SANITIZER_NETBSD
295 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
296 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \
297 INTERCEPT_FUNCTION(__libc_##func)
298 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \
299 INTERCEPT_FUNCTION(__libc_thr_##func)
301 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
302 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
303 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
306 #define READ_STRING_OF_LEN(thr, pc, s, len, n) \
307 MemoryAccessRange((thr), (pc), (uptr)(s), \
308 common_flags()->strict_string_checks ? (len) + 1 : (n), false)
310 #define READ_STRING(thr, pc, s, n) \
311 READ_STRING_OF_LEN((thr), (pc), (s), internal_strlen(s), (n))
313 #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name))
315 struct BlockingCall
{
316 explicit BlockingCall(ThreadState
*thr
)
320 atomic_store(&ctx
->in_blocking_func
, 1, memory_order_relaxed
);
321 if (atomic_load(&thr
->pending_signals
, memory_order_relaxed
) == 0)
323 atomic_store(&ctx
->in_blocking_func
, 0, memory_order_relaxed
);
324 ProcessPendingSignals(thr
);
326 // When we are in a "blocking call", we process signals asynchronously
327 // (right when they arrive). In this context we do not expect to be
328 // executing any user/runtime code. The known interceptor sequence when
329 // this is not true is: pthread_join -> munmap(stack). It's fine
330 // to ignore munmap in this case -- we handle stack shadow separately.
331 thr
->ignore_interceptors
++;
335 thr
->ignore_interceptors
--;
336 atomic_store(&ctx
->in_blocking_func
, 0, memory_order_relaxed
);
340 ThreadSignalContext
*ctx
;
343 TSAN_INTERCEPTOR(unsigned, sleep
, unsigned sec
) {
344 SCOPED_TSAN_INTERCEPTOR(sleep
, sec
);
345 unsigned res
= BLOCK_REAL(sleep
)(sec
);
350 TSAN_INTERCEPTOR(int, usleep
, long_t usec
) {
351 SCOPED_TSAN_INTERCEPTOR(usleep
, usec
);
352 int res
= BLOCK_REAL(usleep
)(usec
);
357 TSAN_INTERCEPTOR(int, nanosleep
, void *req
, void *rem
) {
358 SCOPED_TSAN_INTERCEPTOR(nanosleep
, req
, rem
);
359 int res
= BLOCK_REAL(nanosleep
)(req
, rem
);
364 TSAN_INTERCEPTOR(int, pause
, int fake
) {
365 SCOPED_TSAN_INTERCEPTOR(pause
, fake
);
366 return BLOCK_REAL(pause
)(fake
);
369 static void at_exit_wrapper() {
372 // Ensure thread-safety.
373 Lock
l(&interceptor_ctx()->atexit_mu
);
375 // Pop AtExitCtx from the top of the stack of callback functions
376 uptr element
= interceptor_ctx()->AtExitStack
.Size() - 1;
377 ctx
= interceptor_ctx()->AtExitStack
[element
];
378 interceptor_ctx()->AtExitStack
.PopBack();
381 Acquire(cur_thread(), (uptr
)0, (uptr
)ctx
);
382 ((void(*)())ctx
->f
)();
386 static void cxa_at_exit_wrapper(void *arg
) {
387 Acquire(cur_thread(), 0, (uptr
)arg
);
388 AtExitCtx
*ctx
= (AtExitCtx
*)arg
;
389 ((void(*)(void *arg
))ctx
->f
)(ctx
->arg
);
393 static int setup_at_exit_wrapper(ThreadState
*thr
, uptr pc
, void(*f
)(),
394 void *arg
, void *dso
);
396 #if !SANITIZER_ANDROID
397 TSAN_INTERCEPTOR(int, atexit
, void (*f
)()) {
400 // We want to setup the atexit callback even if we are in ignored lib
402 SCOPED_INTERCEPTOR_RAW(atexit
, f
);
403 return setup_at_exit_wrapper(thr
, pc
, (void(*)())f
, 0, 0);
407 TSAN_INTERCEPTOR(int, __cxa_atexit
, void (*f
)(void *a
), void *arg
, void *dso
) {
410 SCOPED_TSAN_INTERCEPTOR(__cxa_atexit
, f
, arg
, dso
);
411 return setup_at_exit_wrapper(thr
, pc
, (void(*)())f
, arg
, dso
);
414 static int setup_at_exit_wrapper(ThreadState
*thr
, uptr pc
, void(*f
)(),
415 void *arg
, void *dso
) {
416 auto *ctx
= New
<AtExitCtx
>();
419 Release(thr
, pc
, (uptr
)ctx
);
420 // Memory allocation in __cxa_atexit will race with free during exit,
421 // because we do not see synchronization around atexit callback list.
422 ThreadIgnoreBegin(thr
, pc
);
425 // NetBSD does not preserve the 2nd argument if dso is equal to 0
426 // Store ctx in a local stack-like structure
428 // Ensure thread-safety.
429 Lock
l(&interceptor_ctx()->atexit_mu
);
430 // __cxa_atexit calls calloc. If we don't ignore interceptors, we will fail
431 // due to atexit_mu held on exit from the calloc interceptor.
432 ScopedIgnoreInterceptors ignore
;
434 res
= REAL(__cxa_atexit
)((void (*)(void *a
))at_exit_wrapper
, 0, 0);
435 // Push AtExitCtx on the top of the stack of callback functions
437 interceptor_ctx()->AtExitStack
.PushBack(ctx
);
440 res
= REAL(__cxa_atexit
)(cxa_at_exit_wrapper
, ctx
, dso
);
442 ThreadIgnoreEnd(thr
);
446 #if !SANITIZER_MAC && !SANITIZER_NETBSD
447 static void on_exit_wrapper(int status
, void *arg
) {
448 ThreadState
*thr
= cur_thread();
450 Acquire(thr
, pc
, (uptr
)arg
);
451 AtExitCtx
*ctx
= (AtExitCtx
*)arg
;
452 ((void(*)(int status
, void *arg
))ctx
->f
)(status
, ctx
->arg
);
456 TSAN_INTERCEPTOR(int, on_exit
, void(*f
)(int, void*), void *arg
) {
459 SCOPED_TSAN_INTERCEPTOR(on_exit
, f
, arg
);
460 auto *ctx
= New
<AtExitCtx
>();
461 ctx
->f
= (void(*)())f
;
463 Release(thr
, pc
, (uptr
)ctx
);
464 // Memory allocation in __cxa_atexit will race with free during exit,
465 // because we do not see synchronization around atexit callback list.
466 ThreadIgnoreBegin(thr
, pc
);
467 int res
= REAL(on_exit
)(on_exit_wrapper
, ctx
);
468 ThreadIgnoreEnd(thr
);
471 #define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit)
473 #define TSAN_MAYBE_INTERCEPT_ON_EXIT
477 static void JmpBufGarbageCollect(ThreadState
*thr
, uptr sp
) {
478 for (uptr i
= 0; i
< thr
->jmp_bufs
.Size(); i
++) {
479 JmpBuf
*buf
= &thr
->jmp_bufs
[i
];
481 uptr sz
= thr
->jmp_bufs
.Size();
482 internal_memcpy(buf
, &thr
->jmp_bufs
[sz
- 1], sizeof(*buf
));
483 thr
->jmp_bufs
.PopBack();
489 static void SetJmp(ThreadState
*thr
, uptr sp
) {
490 if (!thr
->is_inited
) // called from libc guts during bootstrap
493 JmpBufGarbageCollect(thr
, sp
);
495 JmpBuf
*buf
= thr
->jmp_bufs
.PushBack();
497 buf
->shadow_stack_pos
= thr
->shadow_stack_pos
;
498 ThreadSignalContext
*sctx
= SigCtx(thr
);
499 buf
->int_signal_send
= sctx
? sctx
->int_signal_send
: 0;
500 buf
->in_blocking_func
= sctx
?
501 atomic_load(&sctx
->in_blocking_func
, memory_order_relaxed
) :
503 buf
->in_signal_handler
= atomic_load(&thr
->in_signal_handler
,
504 memory_order_relaxed
);
507 static void LongJmp(ThreadState
*thr
, uptr
*env
) {
508 uptr sp
= ExtractLongJmpSp(env
);
509 // Find the saved buf with matching sp.
510 for (uptr i
= 0; i
< thr
->jmp_bufs
.Size(); i
++) {
511 JmpBuf
*buf
= &thr
->jmp_bufs
[i
];
513 CHECK_GE(thr
->shadow_stack_pos
, buf
->shadow_stack_pos
);
515 while (thr
->shadow_stack_pos
> buf
->shadow_stack_pos
)
517 ThreadSignalContext
*sctx
= SigCtx(thr
);
519 sctx
->int_signal_send
= buf
->int_signal_send
;
520 atomic_store(&sctx
->in_blocking_func
, buf
->in_blocking_func
,
521 memory_order_relaxed
);
523 atomic_store(&thr
->in_signal_handler
, buf
->in_signal_handler
,
524 memory_order_relaxed
);
525 JmpBufGarbageCollect(thr
, buf
->sp
- 1); // do not collect buf->sp
529 Printf("ThreadSanitizer: can't find longjmp buf\n");
533 // FIXME: put everything below into a common extern "C" block?
534 extern "C" void __tsan_setjmp(uptr sp
) { SetJmp(cur_thread_init(), sp
); }
537 TSAN_INTERCEPTOR(int, setjmp
, void *env
);
538 TSAN_INTERCEPTOR(int, _setjmp
, void *env
);
539 TSAN_INTERCEPTOR(int, sigsetjmp
, void *env
);
540 #else // SANITIZER_MAC
543 #define setjmp_symname __setjmp14
544 #define sigsetjmp_symname __sigsetjmp14
546 #define setjmp_symname setjmp
547 #define sigsetjmp_symname sigsetjmp
550 #define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x
551 #define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x)
552 #define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname)
553 #define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname)
555 #define TSAN_STRING_SETJMP SANITIZER_STRINGIFY(setjmp_symname)
556 #define TSAN_STRING_SIGSETJMP SANITIZER_STRINGIFY(sigsetjmp_symname)
558 // Not called. Merely to satisfy TSAN_INTERCEPT().
559 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
560 int TSAN_INTERCEPTOR_SETJMP(void *env
);
561 extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env
) {
566 // FIXME: any reason to have a separate declaration?
567 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
568 int __interceptor__setjmp(void *env
);
569 extern "C" int __interceptor__setjmp(void *env
) {
574 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
575 int TSAN_INTERCEPTOR_SIGSETJMP(void *env
);
576 extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env
) {
581 #if !SANITIZER_NETBSD
582 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
583 int __interceptor___sigsetjmp(void *env
);
584 extern "C" int __interceptor___sigsetjmp(void *env
) {
590 extern "C" int setjmp_symname(void *env
);
591 extern "C" int _setjmp(void *env
);
592 extern "C" int sigsetjmp_symname(void *env
);
593 #if !SANITIZER_NETBSD
594 extern "C" int __sigsetjmp(void *env
);
596 DEFINE_REAL(int, setjmp_symname
, void *env
)
597 DEFINE_REAL(int, _setjmp
, void *env
)
598 DEFINE_REAL(int, sigsetjmp_symname
, void *env
)
599 #if !SANITIZER_NETBSD
600 DEFINE_REAL(int, __sigsetjmp
, void *env
)
602 #endif // SANITIZER_MAC
605 #define longjmp_symname __longjmp14
606 #define siglongjmp_symname __siglongjmp14
608 #define longjmp_symname longjmp
609 #define siglongjmp_symname siglongjmp
612 TSAN_INTERCEPTOR(void, longjmp_symname
, uptr
*env
, int val
) {
613 // Note: if we call REAL(longjmp) in the context of ScopedInterceptor,
614 // bad things will happen. We will jump over ScopedInterceptor dtor and can
615 // leave thr->in_ignored_lib set.
617 SCOPED_INTERCEPTOR_RAW(longjmp_symname
, env
, val
);
619 LongJmp(cur_thread(), env
);
620 REAL(longjmp_symname
)(env
, val
);
623 TSAN_INTERCEPTOR(void, siglongjmp_symname
, uptr
*env
, int val
) {
625 SCOPED_INTERCEPTOR_RAW(siglongjmp_symname
, env
, val
);
627 LongJmp(cur_thread(), env
);
628 REAL(siglongjmp_symname
)(env
, val
);
632 TSAN_INTERCEPTOR(void, _longjmp
, uptr
*env
, int val
) {
634 SCOPED_INTERCEPTOR_RAW(_longjmp
, env
, val
);
636 LongJmp(cur_thread(), env
);
637 REAL(_longjmp
)(env
, val
);
642 TSAN_INTERCEPTOR(void*, malloc
, uptr size
) {
644 return InternalAlloc(size
);
647 SCOPED_INTERCEPTOR_RAW(malloc
, size
);
648 p
= user_alloc(thr
, pc
, size
);
650 invoke_malloc_hook(p
, size
);
654 // In glibc<2.25, dynamic TLS blocks are allocated by __libc_memalign. Intercept
655 // __libc_memalign so that (1) we can detect races (2) free will not be called
656 // on libc internally allocated blocks.
657 TSAN_INTERCEPTOR(void*, __libc_memalign
, uptr align
, uptr sz
) {
658 SCOPED_INTERCEPTOR_RAW(__libc_memalign
, align
, sz
);
659 return user_memalign(thr
, pc
, align
, sz
);
662 TSAN_INTERCEPTOR(void*, calloc
, uptr size
, uptr n
) {
664 return InternalCalloc(size
, n
);
667 SCOPED_INTERCEPTOR_RAW(calloc
, size
, n
);
668 p
= user_calloc(thr
, pc
, size
, n
);
670 invoke_malloc_hook(p
, n
* size
);
674 TSAN_INTERCEPTOR(void*, realloc
, void *p
, uptr size
) {
676 return InternalRealloc(p
, size
);
680 SCOPED_INTERCEPTOR_RAW(realloc
, p
, size
);
681 p
= user_realloc(thr
, pc
, p
, size
);
683 invoke_malloc_hook(p
, size
);
687 TSAN_INTERCEPTOR(void*, reallocarray
, void *p
, uptr size
, uptr n
) {
689 return InternalReallocArray(p
, size
, n
);
693 SCOPED_INTERCEPTOR_RAW(reallocarray
, p
, size
, n
);
694 p
= user_reallocarray(thr
, pc
, p
, size
, n
);
696 invoke_malloc_hook(p
, size
);
700 TSAN_INTERCEPTOR(void, free
, void *p
) {
704 return InternalFree(p
);
706 SCOPED_INTERCEPTOR_RAW(free
, p
);
707 user_free(thr
, pc
, p
);
710 TSAN_INTERCEPTOR(void, cfree
, void *p
) {
714 return InternalFree(p
);
716 SCOPED_INTERCEPTOR_RAW(cfree
, p
);
717 user_free(thr
, pc
, p
);
720 TSAN_INTERCEPTOR(uptr
, malloc_usable_size
, void *p
) {
721 SCOPED_INTERCEPTOR_RAW(malloc_usable_size
, p
);
722 return user_alloc_usable_size(p
);
726 TSAN_INTERCEPTOR(char *, strcpy
, char *dst
, const char *src
) {
727 SCOPED_TSAN_INTERCEPTOR(strcpy
, dst
, src
);
728 uptr srclen
= internal_strlen(src
);
729 MemoryAccessRange(thr
, pc
, (uptr
)dst
, srclen
+ 1, true);
730 MemoryAccessRange(thr
, pc
, (uptr
)src
, srclen
+ 1, false);
731 return REAL(strcpy
)(dst
, src
);
734 TSAN_INTERCEPTOR(char*, strncpy
, char *dst
, char *src
, uptr n
) {
735 SCOPED_TSAN_INTERCEPTOR(strncpy
, dst
, src
, n
);
736 uptr srclen
= internal_strnlen(src
, n
);
737 MemoryAccessRange(thr
, pc
, (uptr
)dst
, n
, true);
738 MemoryAccessRange(thr
, pc
, (uptr
)src
, min(srclen
+ 1, n
), false);
739 return REAL(strncpy
)(dst
, src
, n
);
742 TSAN_INTERCEPTOR(char*, strdup
, const char *str
) {
743 SCOPED_TSAN_INTERCEPTOR(strdup
, str
);
744 // strdup will call malloc, so no instrumentation is required here.
745 return REAL(strdup
)(str
);
748 // Zero out addr if it points into shadow memory and was provided as a hint
749 // only, i.e., MAP_FIXED is not set.
750 static bool fix_mmap_addr(void **addr
, long_t sz
, int flags
) {
752 if (!IsAppMem((uptr
)*addr
) || !IsAppMem((uptr
)*addr
+ sz
- 1)) {
753 if (flags
& MAP_FIXED
) {
754 errno
= errno_EINVAL
;
764 template <class Mmap
>
765 static void *mmap_interceptor(ThreadState
*thr
, uptr pc
, Mmap real_mmap
,
766 void *addr
, SIZE_T sz
, int prot
, int flags
,
767 int fd
, OFF64_T off
) {
768 if (!fix_mmap_addr(&addr
, sz
, flags
)) return MAP_FAILED
;
769 void *res
= real_mmap(addr
, sz
, prot
, flags
, fd
, off
);
770 if (res
!= MAP_FAILED
) {
771 if (!IsAppMem((uptr
)res
) || !IsAppMem((uptr
)res
+ sz
- 1)) {
772 Report("ThreadSanitizer: mmap at bad address: addr=%p size=%p res=%p\n",
773 addr
, (void*)sz
, res
);
776 if (fd
> 0) FdAccess(thr
, pc
, fd
);
777 MemoryRangeImitateWriteOrResetRange(thr
, pc
, (uptr
)res
, sz
);
782 TSAN_INTERCEPTOR(int, munmap
, void *addr
, long_t sz
) {
783 SCOPED_TSAN_INTERCEPTOR(munmap
, addr
, sz
);
784 UnmapShadow(thr
, (uptr
)addr
, sz
);
785 int res
= REAL(munmap
)(addr
, sz
);
790 TSAN_INTERCEPTOR(void*, memalign
, uptr align
, uptr sz
) {
791 SCOPED_INTERCEPTOR_RAW(memalign
, align
, sz
);
792 return user_memalign(thr
, pc
, align
, sz
);
794 #define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign)
796 #define TSAN_MAYBE_INTERCEPT_MEMALIGN
800 TSAN_INTERCEPTOR(void*, aligned_alloc
, uptr align
, uptr sz
) {
802 return InternalAlloc(sz
, nullptr, align
);
803 SCOPED_INTERCEPTOR_RAW(aligned_alloc
, align
, sz
);
804 return user_aligned_alloc(thr
, pc
, align
, sz
);
807 TSAN_INTERCEPTOR(void*, valloc
, uptr sz
) {
809 return InternalAlloc(sz
, nullptr, GetPageSizeCached());
810 SCOPED_INTERCEPTOR_RAW(valloc
, sz
);
811 return user_valloc(thr
, pc
, sz
);
816 TSAN_INTERCEPTOR(void*, pvalloc
, uptr sz
) {
817 if (in_symbolizer()) {
818 uptr PageSize
= GetPageSizeCached();
819 sz
= sz
? RoundUpTo(sz
, PageSize
) : PageSize
;
820 return InternalAlloc(sz
, nullptr, PageSize
);
822 SCOPED_INTERCEPTOR_RAW(pvalloc
, sz
);
823 return user_pvalloc(thr
, pc
, sz
);
825 #define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc)
827 #define TSAN_MAYBE_INTERCEPT_PVALLOC
831 TSAN_INTERCEPTOR(int, posix_memalign
, void **memptr
, uptr align
, uptr sz
) {
832 if (in_symbolizer()) {
833 void *p
= InternalAlloc(sz
, nullptr, align
);
839 SCOPED_INTERCEPTOR_RAW(posix_memalign
, memptr
, align
, sz
);
840 return user_posix_memalign(thr
, pc
, memptr
, align
, sz
);
844 // Both __cxa_guard_acquire and pthread_once 0-initialize
845 // the object initially. pthread_once does not have any
846 // other ABI requirements. __cxa_guard_acquire assumes
847 // that any non-0 value in the first byte means that
848 // initialization is completed. Contents of the remaining
849 // bytes are up to us.
850 constexpr u32 kGuardInit
= 0;
851 constexpr u32 kGuardDone
= 1;
852 constexpr u32 kGuardRunning
= 1 << 16;
853 constexpr u32 kGuardWaiter
= 1 << 17;
855 static int guard_acquire(ThreadState
*thr
, uptr pc
, atomic_uint32_t
*g
,
856 bool blocking_hooks
= true) {
858 OnPotentiallyBlockingRegionBegin();
859 auto on_exit
= at_scope_exit([blocking_hooks
] {
861 OnPotentiallyBlockingRegionEnd();
865 u32 cmp
= atomic_load(g
, memory_order_acquire
);
866 if (cmp
== kGuardInit
) {
867 if (atomic_compare_exchange_strong(g
, &cmp
, kGuardRunning
,
868 memory_order_relaxed
))
870 } else if (cmp
== kGuardDone
) {
871 if (!thr
->in_ignored_lib
)
872 Acquire(thr
, pc
, (uptr
)g
);
875 if ((cmp
& kGuardWaiter
) ||
876 atomic_compare_exchange_strong(g
, &cmp
, cmp
| kGuardWaiter
,
877 memory_order_relaxed
))
878 FutexWait(g
, cmp
| kGuardWaiter
);
883 static void guard_release(ThreadState
*thr
, uptr pc
, atomic_uint32_t
*g
) {
884 if (!thr
->in_ignored_lib
)
885 Release(thr
, pc
, (uptr
)g
);
886 u32 old
= atomic_exchange(g
, kGuardDone
, memory_order_release
);
887 if (old
& kGuardWaiter
)
888 FutexWake(g
, 1 << 30);
891 // __cxa_guard_acquire and friends need to be intercepted in a special way -
892 // regular interceptors will break statically-linked libstdc++. Linux
893 // interceptors are especially defined as weak functions (so that they don't
894 // cause link errors when user defines them as well). So they silently
895 // auto-disable themselves when such symbol is already present in the binary. If
896 // we link libstdc++ statically, it will bring own __cxa_guard_acquire which
897 // will silently replace our interceptor. That's why on Linux we simply export
898 // these interceptors with INTERFACE_ATTRIBUTE.
899 // On OS X, we don't support statically linking, so we just use a regular
902 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
904 #define STDCXX_INTERCEPTOR(rettype, name, ...) \
905 extern "C" rettype INTERFACE_ATTRIBUTE name(__VA_ARGS__)
908 // Used in thread-safe function static initialization.
909 STDCXX_INTERCEPTOR(int, __cxa_guard_acquire
, atomic_uint32_t
*g
) {
910 SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire
, g
);
911 return guard_acquire(thr
, pc
, g
);
914 STDCXX_INTERCEPTOR(void, __cxa_guard_release
, atomic_uint32_t
*g
) {
915 SCOPED_INTERCEPTOR_RAW(__cxa_guard_release
, g
);
916 guard_release(thr
, pc
, g
);
919 STDCXX_INTERCEPTOR(void, __cxa_guard_abort
, atomic_uint32_t
*g
) {
920 SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort
, g
);
921 atomic_store(g
, kGuardInit
, memory_order_relaxed
);
925 void DestroyThreadState() {
926 ThreadState
*thr
= cur_thread();
927 Processor
*proc
= thr
->proc();
929 ProcUnwire(proc
, thr
);
932 cur_thread_finalize();
935 void PlatformCleanUpThreadState(ThreadState
*thr
) {
936 ThreadSignalContext
*sctx
= thr
->signal_ctx
;
939 UnmapOrDie(sctx
, sizeof(*sctx
));
942 } // namespace __tsan
944 #if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
945 static void thread_finalize(void *v
) {
948 if (pthread_setspecific(interceptor_ctx()->finalize_key
,
949 (void*)(iter
- 1))) {
950 Printf("ThreadSanitizer: failed to set thread key\n");
955 DestroyThreadState();
961 void* (*callback
)(void *arg
);
968 extern "C" void *__tsan_thread_start_func(void *arg
) {
969 ThreadParam
*p
= (ThreadParam
*)arg
;
970 void* (*callback
)(void *arg
) = p
->callback
;
971 void *param
= p
->param
;
973 ThreadState
*thr
= cur_thread_init();
974 // Thread-local state is not initialized yet.
975 ScopedIgnoreInterceptors ignore
;
976 #if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
977 ThreadIgnoreBegin(thr
, 0);
978 if (pthread_setspecific(interceptor_ctx()->finalize_key
,
979 (void *)GetPthreadDestructorIterations())) {
980 Printf("ThreadSanitizer: failed to set thread key\n");
983 ThreadIgnoreEnd(thr
);
986 Processor
*proc
= ProcCreate();
988 ThreadStart(thr
, p
->tid
, GetTid(), ThreadType::Regular
);
991 void *res
= callback(param
);
992 // Prevent the callback from being tail called,
993 // it mixes up stack traces.
994 volatile int foo
= 42;
999 TSAN_INTERCEPTOR(int, pthread_create
,
1000 void *th
, void *attr
, void *(*callback
)(void*), void * param
) {
1001 SCOPED_INTERCEPTOR_RAW(pthread_create
, th
, attr
, callback
, param
);
1003 MaybeSpawnBackgroundThread();
1005 if (ctx
->after_multithreaded_fork
) {
1006 if (flags()->die_after_fork
) {
1007 Report("ThreadSanitizer: starting new threads after multi-threaded "
1008 "fork is not supported. Dying (set die_after_fork=0 to override)\n");
1012 "ThreadSanitizer: starting new threads after multi-threaded "
1013 "fork is not supported (pid %lu). Continuing because of "
1014 "die_after_fork=0, but you are on your own\n",
1018 __sanitizer_pthread_attr_t myattr
;
1020 pthread_attr_init(&myattr
);
1024 REAL(pthread_attr_getdetachstate
)(attr
, &detached
);
1025 AdjustStackSize(attr
);
1028 p
.callback
= callback
;
1033 // Otherwise we see false positives in pthread stack manipulation.
1034 ScopedIgnoreInterceptors ignore
;
1035 ThreadIgnoreBegin(thr
, pc
);
1036 res
= REAL(pthread_create
)(th
, attr
, __tsan_thread_start_func
, &p
);
1037 ThreadIgnoreEnd(thr
);
1040 p
.tid
= ThreadCreate(thr
, pc
, *(uptr
*)th
, IsStateDetached(detached
));
1041 CHECK_NE(p
.tid
, kMainTid
);
1042 // Synchronization on p.tid serves two purposes:
1043 // 1. ThreadCreate must finish before the new thread starts.
1044 // Otherwise the new thread can call pthread_detach, but the pthread_t
1045 // identifier is not yet registered in ThreadRegistry by ThreadCreate.
1046 // 2. ThreadStart must finish before this thread continues.
1047 // Otherwise, this thread can call pthread_detach and reset thr->sync
1048 // before the new thread got a chance to acquire from it in ThreadStart.
1052 if (attr
== &myattr
)
1053 pthread_attr_destroy(&myattr
);
1057 TSAN_INTERCEPTOR(int, pthread_join
, void *th
, void **ret
) {
1058 SCOPED_INTERCEPTOR_RAW(pthread_join
, th
, ret
);
1059 Tid tid
= ThreadConsumeTid(thr
, pc
, (uptr
)th
);
1060 ThreadIgnoreBegin(thr
, pc
);
1061 int res
= BLOCK_REAL(pthread_join
)(th
, ret
);
1062 ThreadIgnoreEnd(thr
);
1064 ThreadJoin(thr
, pc
, tid
);
1069 DEFINE_REAL_PTHREAD_FUNCTIONS
1071 TSAN_INTERCEPTOR(int, pthread_detach
, void *th
) {
1072 SCOPED_INTERCEPTOR_RAW(pthread_detach
, th
);
1073 Tid tid
= ThreadConsumeTid(thr
, pc
, (uptr
)th
);
1074 int res
= REAL(pthread_detach
)(th
);
1076 ThreadDetach(thr
, pc
, tid
);
1081 TSAN_INTERCEPTOR(void, pthread_exit
, void *retval
) {
1083 SCOPED_INTERCEPTOR_RAW(pthread_exit
, retval
);
1084 #if !SANITIZER_MAC && !SANITIZER_ANDROID
1085 CHECK_EQ(thr
, &cur_thread_placeholder
);
1088 REAL(pthread_exit
)(retval
);
1092 TSAN_INTERCEPTOR(int, pthread_tryjoin_np
, void *th
, void **ret
) {
1093 SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np
, th
, ret
);
1094 Tid tid
= ThreadConsumeTid(thr
, pc
, (uptr
)th
);
1095 ThreadIgnoreBegin(thr
, pc
);
1096 int res
= REAL(pthread_tryjoin_np
)(th
, ret
);
1097 ThreadIgnoreEnd(thr
);
1099 ThreadJoin(thr
, pc
, tid
);
1101 ThreadNotJoined(thr
, pc
, tid
, (uptr
)th
);
1105 TSAN_INTERCEPTOR(int, pthread_timedjoin_np
, void *th
, void **ret
,
1106 const struct timespec
*abstime
) {
1107 SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np
, th
, ret
, abstime
);
1108 Tid tid
= ThreadConsumeTid(thr
, pc
, (uptr
)th
);
1109 ThreadIgnoreBegin(thr
, pc
);
1110 int res
= BLOCK_REAL(pthread_timedjoin_np
)(th
, ret
, abstime
);
1111 ThreadIgnoreEnd(thr
);
1113 ThreadJoin(thr
, pc
, tid
);
1115 ThreadNotJoined(thr
, pc
, tid
, (uptr
)th
);
1121 // NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2).
1122 // pthread_cond_t has different size in the different versions.
1123 // If call new REAL functions for old pthread_cond_t, they will corrupt memory
1124 // after pthread_cond_t (old cond is smaller).
1125 // If we call old REAL functions for new pthread_cond_t, we will lose some
1126 // functionality (e.g. old functions do not support waiting against
1128 // Proper handling would require to have 2 versions of interceptors as well.
1129 // But this is messy, in particular requires linker scripts when sanitizer
1130 // runtime is linked into a shared library.
1131 // Instead we assume we don't have dynamic libraries built against old
1132 // pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag
1133 // that allows to work with old libraries (but this mode does not support
1134 // some features, e.g. pthread_condattr_getpshared).
1135 static void *init_cond(void *c
, bool force
= false) {
1136 // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions.
1137 // So we allocate additional memory on the side large enough to hold
1138 // any pthread_cond_t object. Always call new REAL functions, but pass
1139 // the aux object to them.
1140 // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes
1141 // first word of pthread_cond_t to zero.
1142 // It's all relevant only for linux.
1143 if (!common_flags()->legacy_pthread_cond
)
1145 atomic_uintptr_t
*p
= (atomic_uintptr_t
*)c
;
1146 uptr cond
= atomic_load(p
, memory_order_acquire
);
1147 if (!force
&& cond
!= 0)
1149 void *newcond
= WRAP(malloc
)(pthread_cond_t_sz
);
1150 internal_memset(newcond
, 0, pthread_cond_t_sz
);
1151 if (atomic_compare_exchange_strong(p
, &cond
, (uptr
)newcond
,
1152 memory_order_acq_rel
))
1154 WRAP(free
)(newcond
);
1161 struct CondMutexUnlockCtx
{
1162 ScopedInterceptor
*si
;
1169 int Cancel() const { return fn(); }
1170 void Unlock() const;
1174 void CondMutexUnlockCtx
<Fn
>::Unlock() const {
1175 // pthread_cond_wait interceptor has enabled async signal delivery
1176 // (see BlockingCall below). Disable async signals since we are running
1177 // tsan code. Also ScopedInterceptor and BlockingCall destructors won't run
1178 // since the thread is cancelled, so we have to manually execute them
1179 // (the thread still can run some user code due to pthread_cleanup_push).
1180 ThreadSignalContext
*ctx
= SigCtx(thr
);
1181 CHECK_EQ(atomic_load(&ctx
->in_blocking_func
, memory_order_relaxed
), 1);
1182 atomic_store(&ctx
->in_blocking_func
, 0, memory_order_relaxed
);
1183 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagDoPreLockOnPostLock
);
1184 // Undo BlockingCall ctor effects.
1185 thr
->ignore_interceptors
--;
1186 si
->~ScopedInterceptor();
1190 INTERCEPTOR(int, pthread_cond_init
, void *c
, void *a
) {
1191 void *cond
= init_cond(c
, true);
1192 SCOPED_TSAN_INTERCEPTOR(pthread_cond_init
, cond
, a
);
1193 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), true);
1194 return REAL(pthread_cond_init
)(cond
, a
);
1198 int cond_wait(ThreadState
*thr
, uptr pc
, ScopedInterceptor
*si
, const Fn
&fn
,
1200 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), false);
1201 MutexUnlock(thr
, pc
, (uptr
)m
);
1203 // This ensures that we handle mutex lock even in case of pthread_cancel.
1204 // See test/tsan/cond_cancel.cpp.
1206 // Enable signal delivery while the thread is blocked.
1207 BlockingCall
bc(thr
);
1208 CondMutexUnlockCtx
<Fn
> arg
= {si
, thr
, pc
, m
, c
, fn
};
1209 res
= call_pthread_cancel_with_cleanup(
1210 [](void *arg
) -> int {
1211 return ((const CondMutexUnlockCtx
<Fn
> *)arg
)->Cancel();
1213 [](void *arg
) { ((const CondMutexUnlockCtx
<Fn
> *)arg
)->Unlock(); },
1216 if (res
== errno_EOWNERDEAD
) MutexRepair(thr
, pc
, (uptr
)m
);
1217 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagDoPreLockOnPostLock
);
1221 INTERCEPTOR(int, pthread_cond_wait
, void *c
, void *m
) {
1222 void *cond
= init_cond(c
);
1223 SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait
, cond
, m
);
1225 thr
, pc
, &si
, [=]() { return REAL(pthread_cond_wait
)(cond
, m
); }, cond
,
1229 INTERCEPTOR(int, pthread_cond_timedwait
, void *c
, void *m
, void *abstime
) {
1230 void *cond
= init_cond(c
);
1231 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait
, cond
, m
, abstime
);
1234 [=]() { return REAL(pthread_cond_timedwait
)(cond
, m
, abstime
); }, cond
,
1239 INTERCEPTOR(int, pthread_cond_clockwait
, void *c
, void *m
,
1240 __sanitizer_clockid_t clock
, void *abstime
) {
1241 void *cond
= init_cond(c
);
1242 SCOPED_TSAN_INTERCEPTOR(pthread_cond_clockwait
, cond
, m
, clock
, abstime
);
1245 [=]() { return REAL(pthread_cond_clockwait
)(cond
, m
, clock
, abstime
); },
1248 #define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT TSAN_INTERCEPT(pthread_cond_clockwait)
1250 #define TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT
1254 INTERCEPTOR(int, pthread_cond_timedwait_relative_np
, void *c
, void *m
,
1256 void *cond
= init_cond(c
);
1257 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np
, cond
, m
, reltime
);
1261 return REAL(pthread_cond_timedwait_relative_np
)(cond
, m
, reltime
);
1267 INTERCEPTOR(int, pthread_cond_signal
, void *c
) {
1268 void *cond
= init_cond(c
);
1269 SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal
, cond
);
1270 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), false);
1271 return REAL(pthread_cond_signal
)(cond
);
1274 INTERCEPTOR(int, pthread_cond_broadcast
, void *c
) {
1275 void *cond
= init_cond(c
);
1276 SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast
, cond
);
1277 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), false);
1278 return REAL(pthread_cond_broadcast
)(cond
);
1281 INTERCEPTOR(int, pthread_cond_destroy
, void *c
) {
1282 void *cond
= init_cond(c
);
1283 SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy
, cond
);
1284 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), true);
1285 int res
= REAL(pthread_cond_destroy
)(cond
);
1286 if (common_flags()->legacy_pthread_cond
) {
1287 // Free our aux cond and zero the pointer to not leave dangling pointers.
1289 atomic_store((atomic_uintptr_t
*)c
, 0, memory_order_relaxed
);
1294 TSAN_INTERCEPTOR(int, pthread_mutex_init
, void *m
, void *a
) {
1295 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init
, m
, a
);
1296 int res
= REAL(pthread_mutex_init
)(m
, a
);
1301 if (REAL(pthread_mutexattr_gettype
)(a
, &type
) == 0)
1302 if (type
== PTHREAD_MUTEX_RECURSIVE
||
1303 type
== PTHREAD_MUTEX_RECURSIVE_NP
)
1304 flagz
|= MutexFlagWriteReentrant
;
1306 MutexCreate(thr
, pc
, (uptr
)m
, flagz
);
1311 TSAN_INTERCEPTOR(int, pthread_mutex_destroy
, void *m
) {
1312 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy
, m
);
1313 int res
= REAL(pthread_mutex_destroy
)(m
);
1314 if (res
== 0 || res
== errno_EBUSY
) {
1315 MutexDestroy(thr
, pc
, (uptr
)m
);
1320 TSAN_INTERCEPTOR(int, pthread_mutex_trylock
, void *m
) {
1321 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock
, m
);
1322 int res
= REAL(pthread_mutex_trylock
)(m
);
1323 if (res
== errno_EOWNERDEAD
)
1324 MutexRepair(thr
, pc
, (uptr
)m
);
1325 if (res
== 0 || res
== errno_EOWNERDEAD
)
1326 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagTryLock
);
1331 TSAN_INTERCEPTOR(int, pthread_mutex_timedlock
, void *m
, void *abstime
) {
1332 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock
, m
, abstime
);
1333 int res
= REAL(pthread_mutex_timedlock
)(m
, abstime
);
1335 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagTryLock
);
1342 TSAN_INTERCEPTOR(int, pthread_spin_init
, void *m
, int pshared
) {
1343 SCOPED_TSAN_INTERCEPTOR(pthread_spin_init
, m
, pshared
);
1344 int res
= REAL(pthread_spin_init
)(m
, pshared
);
1346 MutexCreate(thr
, pc
, (uptr
)m
);
1351 TSAN_INTERCEPTOR(int, pthread_spin_destroy
, void *m
) {
1352 SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy
, m
);
1353 int res
= REAL(pthread_spin_destroy
)(m
);
1355 MutexDestroy(thr
, pc
, (uptr
)m
);
1360 TSAN_INTERCEPTOR(int, pthread_spin_lock
, void *m
) {
1361 SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock
, m
);
1362 MutexPreLock(thr
, pc
, (uptr
)m
);
1363 int res
= REAL(pthread_spin_lock
)(m
);
1365 MutexPostLock(thr
, pc
, (uptr
)m
);
1370 TSAN_INTERCEPTOR(int, pthread_spin_trylock
, void *m
) {
1371 SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock
, m
);
1372 int res
= REAL(pthread_spin_trylock
)(m
);
1374 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagTryLock
);
1379 TSAN_INTERCEPTOR(int, pthread_spin_unlock
, void *m
) {
1380 SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock
, m
);
1381 MutexUnlock(thr
, pc
, (uptr
)m
);
1382 int res
= REAL(pthread_spin_unlock
)(m
);
1387 TSAN_INTERCEPTOR(int, pthread_rwlock_init
, void *m
, void *a
) {
1388 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init
, m
, a
);
1389 int res
= REAL(pthread_rwlock_init
)(m
, a
);
1391 MutexCreate(thr
, pc
, (uptr
)m
);
1396 TSAN_INTERCEPTOR(int, pthread_rwlock_destroy
, void *m
) {
1397 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy
, m
);
1398 int res
= REAL(pthread_rwlock_destroy
)(m
);
1400 MutexDestroy(thr
, pc
, (uptr
)m
);
1405 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock
, void *m
) {
1406 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock
, m
);
1407 MutexPreReadLock(thr
, pc
, (uptr
)m
);
1408 int res
= REAL(pthread_rwlock_rdlock
)(m
);
1410 MutexPostReadLock(thr
, pc
, (uptr
)m
);
1415 TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock
, void *m
) {
1416 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock
, m
);
1417 int res
= REAL(pthread_rwlock_tryrdlock
)(m
);
1419 MutexPostReadLock(thr
, pc
, (uptr
)m
, MutexFlagTryLock
);
1425 TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock
, void *m
, void *abstime
) {
1426 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock
, m
, abstime
);
1427 int res
= REAL(pthread_rwlock_timedrdlock
)(m
, abstime
);
1429 MutexPostReadLock(thr
, pc
, (uptr
)m
);
1435 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock
, void *m
) {
1436 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock
, m
);
1437 MutexPreLock(thr
, pc
, (uptr
)m
);
1438 int res
= REAL(pthread_rwlock_wrlock
)(m
);
1440 MutexPostLock(thr
, pc
, (uptr
)m
);
1445 TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock
, void *m
) {
1446 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock
, m
);
1447 int res
= REAL(pthread_rwlock_trywrlock
)(m
);
1449 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagTryLock
);
1455 TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock
, void *m
, void *abstime
) {
1456 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock
, m
, abstime
);
1457 int res
= REAL(pthread_rwlock_timedwrlock
)(m
, abstime
);
1459 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagTryLock
);
1465 TSAN_INTERCEPTOR(int, pthread_rwlock_unlock
, void *m
) {
1466 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock
, m
);
1467 MutexReadOrWriteUnlock(thr
, pc
, (uptr
)m
);
1468 int res
= REAL(pthread_rwlock_unlock
)(m
);
1473 TSAN_INTERCEPTOR(int, pthread_barrier_init
, void *b
, void *a
, unsigned count
) {
1474 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init
, b
, a
, count
);
1475 MemoryAccess(thr
, pc
, (uptr
)b
, 1, kAccessWrite
);
1476 int res
= REAL(pthread_barrier_init
)(b
, a
, count
);
1480 TSAN_INTERCEPTOR(int, pthread_barrier_destroy
, void *b
) {
1481 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy
, b
);
1482 MemoryAccess(thr
, pc
, (uptr
)b
, 1, kAccessWrite
);
1483 int res
= REAL(pthread_barrier_destroy
)(b
);
1487 TSAN_INTERCEPTOR(int, pthread_barrier_wait
, void *b
) {
1488 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait
, b
);
1489 Release(thr
, pc
, (uptr
)b
);
1490 MemoryAccess(thr
, pc
, (uptr
)b
, 1, kAccessRead
);
1491 int res
= REAL(pthread_barrier_wait
)(b
);
1492 MemoryAccess(thr
, pc
, (uptr
)b
, 1, kAccessRead
);
1493 if (res
== 0 || res
== PTHREAD_BARRIER_SERIAL_THREAD
) {
1494 Acquire(thr
, pc
, (uptr
)b
);
1500 TSAN_INTERCEPTOR(int, pthread_once
, void *o
, void (*f
)()) {
1501 SCOPED_INTERCEPTOR_RAW(pthread_once
, o
, f
);
1502 if (o
== 0 || f
== 0)
1503 return errno_EINVAL
;
1507 a
= static_cast<atomic_uint32_t
*>((void *)((char *)o
+ sizeof(long_t
)));
1508 else if (SANITIZER_NETBSD
)
1509 a
= static_cast<atomic_uint32_t
*>
1510 ((void *)((char *)o
+ __sanitizer::pthread_mutex_t_sz
));
1512 a
= static_cast<atomic_uint32_t
*>(o
);
1514 // Mac OS X appears to use pthread_once() where calling BlockingRegion hooks
1515 // result in crashes due to too little stack space.
1516 if (guard_acquire(thr
, pc
, a
, !SANITIZER_MAC
)) {
1518 guard_release(thr
, pc
, a
);
1523 #if SANITIZER_LINUX && !SANITIZER_ANDROID
1524 TSAN_INTERCEPTOR(int, __fxstat
, int version
, int fd
, void *buf
) {
1525 SCOPED_TSAN_INTERCEPTOR(__fxstat
, version
, fd
, buf
);
1527 FdAccess(thr
, pc
, fd
);
1528 return REAL(__fxstat
)(version
, fd
, buf
);
1530 #define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat)
1532 #define TSAN_MAYBE_INTERCEPT___FXSTAT
1535 TSAN_INTERCEPTOR(int, fstat
, int fd
, void *buf
) {
1536 #if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD
1537 SCOPED_TSAN_INTERCEPTOR(fstat
, fd
, buf
);
1539 FdAccess(thr
, pc
, fd
);
1540 return REAL(fstat
)(fd
, buf
);
1542 SCOPED_TSAN_INTERCEPTOR(__fxstat
, 0, fd
, buf
);
1544 FdAccess(thr
, pc
, fd
);
1545 return REAL(__fxstat
)(0, fd
, buf
);
1549 #if SANITIZER_LINUX && !SANITIZER_ANDROID
1550 TSAN_INTERCEPTOR(int, __fxstat64
, int version
, int fd
, void *buf
) {
1551 SCOPED_TSAN_INTERCEPTOR(__fxstat64
, version
, fd
, buf
);
1553 FdAccess(thr
, pc
, fd
);
1554 return REAL(__fxstat64
)(version
, fd
, buf
);
1556 #define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64)
1558 #define TSAN_MAYBE_INTERCEPT___FXSTAT64
1561 #if SANITIZER_LINUX && !SANITIZER_ANDROID
1562 TSAN_INTERCEPTOR(int, fstat64
, int fd
, void *buf
) {
1563 SCOPED_TSAN_INTERCEPTOR(__fxstat64
, 0, fd
, buf
);
1565 FdAccess(thr
, pc
, fd
);
1566 return REAL(__fxstat64
)(0, fd
, buf
);
1568 #define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64)
1570 #define TSAN_MAYBE_INTERCEPT_FSTAT64
1573 TSAN_INTERCEPTOR(int, open
, const char *name
, int oflag
, ...) {
1575 va_start(ap
, oflag
);
1576 mode_t mode
= va_arg(ap
, int);
1578 SCOPED_TSAN_INTERCEPTOR(open
, name
, oflag
, mode
);
1579 READ_STRING(thr
, pc
, name
, 0);
1580 int fd
= REAL(open
)(name
, oflag
, mode
);
1582 FdFileCreate(thr
, pc
, fd
);
1587 TSAN_INTERCEPTOR(int, open64
, const char *name
, int oflag
, ...) {
1589 va_start(ap
, oflag
);
1590 mode_t mode
= va_arg(ap
, int);
1592 SCOPED_TSAN_INTERCEPTOR(open64
, name
, oflag
, mode
);
1593 READ_STRING(thr
, pc
, name
, 0);
1594 int fd
= REAL(open64
)(name
, oflag
, mode
);
1596 FdFileCreate(thr
, pc
, fd
);
1599 #define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64)
1601 #define TSAN_MAYBE_INTERCEPT_OPEN64
1604 TSAN_INTERCEPTOR(int, creat
, const char *name
, int mode
) {
1605 SCOPED_TSAN_INTERCEPTOR(creat
, name
, mode
);
1606 READ_STRING(thr
, pc
, name
, 0);
1607 int fd
= REAL(creat
)(name
, mode
);
1609 FdFileCreate(thr
, pc
, fd
);
1614 TSAN_INTERCEPTOR(int, creat64
, const char *name
, int mode
) {
1615 SCOPED_TSAN_INTERCEPTOR(creat64
, name
, mode
);
1616 READ_STRING(thr
, pc
, name
, 0);
1617 int fd
= REAL(creat64
)(name
, mode
);
1619 FdFileCreate(thr
, pc
, fd
);
1622 #define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64)
1624 #define TSAN_MAYBE_INTERCEPT_CREAT64
1627 TSAN_INTERCEPTOR(int, dup
, int oldfd
) {
1628 SCOPED_TSAN_INTERCEPTOR(dup
, oldfd
);
1629 int newfd
= REAL(dup
)(oldfd
);
1630 if (oldfd
>= 0 && newfd
>= 0 && newfd
!= oldfd
)
1631 FdDup(thr
, pc
, oldfd
, newfd
, true);
1635 TSAN_INTERCEPTOR(int, dup2
, int oldfd
, int newfd
) {
1636 SCOPED_TSAN_INTERCEPTOR(dup2
, oldfd
, newfd
);
1637 int newfd2
= REAL(dup2
)(oldfd
, newfd
);
1638 if (oldfd
>= 0 && newfd2
>= 0 && newfd2
!= oldfd
)
1639 FdDup(thr
, pc
, oldfd
, newfd2
, false);
1644 TSAN_INTERCEPTOR(int, dup3
, int oldfd
, int newfd
, int flags
) {
1645 SCOPED_TSAN_INTERCEPTOR(dup3
, oldfd
, newfd
, flags
);
1646 int newfd2
= REAL(dup3
)(oldfd
, newfd
, flags
);
1647 if (oldfd
>= 0 && newfd2
>= 0 && newfd2
!= oldfd
)
1648 FdDup(thr
, pc
, oldfd
, newfd2
, false);
1654 TSAN_INTERCEPTOR(int, eventfd
, unsigned initval
, int flags
) {
1655 SCOPED_TSAN_INTERCEPTOR(eventfd
, initval
, flags
);
1656 int fd
= REAL(eventfd
)(initval
, flags
);
1658 FdEventCreate(thr
, pc
, fd
);
1661 #define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd)
1663 #define TSAN_MAYBE_INTERCEPT_EVENTFD
1667 TSAN_INTERCEPTOR(int, signalfd
, int fd
, void *mask
, int flags
) {
1668 SCOPED_TSAN_INTERCEPTOR(signalfd
, fd
, mask
, flags
);
1670 FdClose(thr
, pc
, fd
);
1671 fd
= REAL(signalfd
)(fd
, mask
, flags
);
1673 FdSignalCreate(thr
, pc
, fd
);
1676 #define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd)
1678 #define TSAN_MAYBE_INTERCEPT_SIGNALFD
1682 TSAN_INTERCEPTOR(int, inotify_init
, int fake
) {
1683 SCOPED_TSAN_INTERCEPTOR(inotify_init
, fake
);
1684 int fd
= REAL(inotify_init
)(fake
);
1686 FdInotifyCreate(thr
, pc
, fd
);
1689 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init)
1691 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT
1695 TSAN_INTERCEPTOR(int, inotify_init1
, int flags
) {
1696 SCOPED_TSAN_INTERCEPTOR(inotify_init1
, flags
);
1697 int fd
= REAL(inotify_init1
)(flags
);
1699 FdInotifyCreate(thr
, pc
, fd
);
1702 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1)
1704 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1
1707 TSAN_INTERCEPTOR(int, socket
, int domain
, int type
, int protocol
) {
1708 SCOPED_TSAN_INTERCEPTOR(socket
, domain
, type
, protocol
);
1709 int fd
= REAL(socket
)(domain
, type
, protocol
);
1711 FdSocketCreate(thr
, pc
, fd
);
1715 TSAN_INTERCEPTOR(int, socketpair
, int domain
, int type
, int protocol
, int *fd
) {
1716 SCOPED_TSAN_INTERCEPTOR(socketpair
, domain
, type
, protocol
, fd
);
1717 int res
= REAL(socketpair
)(domain
, type
, protocol
, fd
);
1718 if (res
== 0 && fd
[0] >= 0 && fd
[1] >= 0)
1719 FdPipeCreate(thr
, pc
, fd
[0], fd
[1]);
1723 TSAN_INTERCEPTOR(int, connect
, int fd
, void *addr
, unsigned addrlen
) {
1724 SCOPED_TSAN_INTERCEPTOR(connect
, fd
, addr
, addrlen
);
1725 FdSocketConnecting(thr
, pc
, fd
);
1726 int res
= REAL(connect
)(fd
, addr
, addrlen
);
1727 if (res
== 0 && fd
>= 0)
1728 FdSocketConnect(thr
, pc
, fd
);
1732 TSAN_INTERCEPTOR(int, bind
, int fd
, void *addr
, unsigned addrlen
) {
1733 SCOPED_TSAN_INTERCEPTOR(bind
, fd
, addr
, addrlen
);
1734 int res
= REAL(bind
)(fd
, addr
, addrlen
);
1735 if (fd
> 0 && res
== 0)
1736 FdAccess(thr
, pc
, fd
);
1740 TSAN_INTERCEPTOR(int, listen
, int fd
, int backlog
) {
1741 SCOPED_TSAN_INTERCEPTOR(listen
, fd
, backlog
);
1742 int res
= REAL(listen
)(fd
, backlog
);
1743 if (fd
> 0 && res
== 0)
1744 FdAccess(thr
, pc
, fd
);
1748 TSAN_INTERCEPTOR(int, close
, int fd
) {
1749 SCOPED_TSAN_INTERCEPTOR(close
, fd
);
1751 FdClose(thr
, pc
, fd
);
1752 return REAL(close
)(fd
);
1756 TSAN_INTERCEPTOR(int, __close
, int fd
) {
1757 SCOPED_TSAN_INTERCEPTOR(__close
, fd
);
1759 FdClose(thr
, pc
, fd
);
1760 return REAL(__close
)(fd
);
1762 #define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close)
1764 #define TSAN_MAYBE_INTERCEPT___CLOSE
1768 #if SANITIZER_LINUX && !SANITIZER_ANDROID
1769 TSAN_INTERCEPTOR(void, __res_iclose
, void *state
, bool free_addr
) {
1770 SCOPED_TSAN_INTERCEPTOR(__res_iclose
, state
, free_addr
);
1772 int cnt
= ExtractResolvFDs(state
, fds
, ARRAY_SIZE(fds
));
1773 for (int i
= 0; i
< cnt
; i
++) {
1775 FdClose(thr
, pc
, fds
[i
]);
1777 REAL(__res_iclose
)(state
, free_addr
);
1779 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose)
1781 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE
1784 TSAN_INTERCEPTOR(int, pipe
, int *pipefd
) {
1785 SCOPED_TSAN_INTERCEPTOR(pipe
, pipefd
);
1786 int res
= REAL(pipe
)(pipefd
);
1787 if (res
== 0 && pipefd
[0] >= 0 && pipefd
[1] >= 0)
1788 FdPipeCreate(thr
, pc
, pipefd
[0], pipefd
[1]);
1793 TSAN_INTERCEPTOR(int, pipe2
, int *pipefd
, int flags
) {
1794 SCOPED_TSAN_INTERCEPTOR(pipe2
, pipefd
, flags
);
1795 int res
= REAL(pipe2
)(pipefd
, flags
);
1796 if (res
== 0 && pipefd
[0] >= 0 && pipefd
[1] >= 0)
1797 FdPipeCreate(thr
, pc
, pipefd
[0], pipefd
[1]);
1802 TSAN_INTERCEPTOR(int, unlink
, char *path
) {
1803 SCOPED_TSAN_INTERCEPTOR(unlink
, path
);
1804 Release(thr
, pc
, File2addr(path
));
1805 int res
= REAL(unlink
)(path
);
1809 TSAN_INTERCEPTOR(void*, tmpfile
, int fake
) {
1810 SCOPED_TSAN_INTERCEPTOR(tmpfile
, fake
);
1811 void *res
= REAL(tmpfile
)(fake
);
1813 int fd
= fileno_unlocked(res
);
1815 FdFileCreate(thr
, pc
, fd
);
1821 TSAN_INTERCEPTOR(void*, tmpfile64
, int fake
) {
1822 SCOPED_TSAN_INTERCEPTOR(tmpfile64
, fake
);
1823 void *res
= REAL(tmpfile64
)(fake
);
1825 int fd
= fileno_unlocked(res
);
1827 FdFileCreate(thr
, pc
, fd
);
1831 #define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64)
1833 #define TSAN_MAYBE_INTERCEPT_TMPFILE64
1836 static void FlushStreams() {
1837 // Flushing all the streams here may freeze the process if a child thread is
1838 // performing file stream operations at the same time.
1839 REAL(fflush
)(stdout
);
1840 REAL(fflush
)(stderr
);
1843 TSAN_INTERCEPTOR(void, abort
, int fake
) {
1844 SCOPED_TSAN_INTERCEPTOR(abort
, fake
);
1849 TSAN_INTERCEPTOR(int, rmdir
, char *path
) {
1850 SCOPED_TSAN_INTERCEPTOR(rmdir
, path
);
1851 Release(thr
, pc
, Dir2addr(path
));
1852 int res
= REAL(rmdir
)(path
);
1856 TSAN_INTERCEPTOR(int, closedir
, void *dirp
) {
1857 SCOPED_TSAN_INTERCEPTOR(closedir
, dirp
);
1859 int fd
= dirfd(dirp
);
1860 FdClose(thr
, pc
, fd
);
1862 return REAL(closedir
)(dirp
);
1866 TSAN_INTERCEPTOR(int, epoll_create
, int size
) {
1867 SCOPED_TSAN_INTERCEPTOR(epoll_create
, size
);
1868 int fd
= REAL(epoll_create
)(size
);
1870 FdPollCreate(thr
, pc
, fd
);
1874 TSAN_INTERCEPTOR(int, epoll_create1
, int flags
) {
1875 SCOPED_TSAN_INTERCEPTOR(epoll_create1
, flags
);
1876 int fd
= REAL(epoll_create1
)(flags
);
1878 FdPollCreate(thr
, pc
, fd
);
1882 TSAN_INTERCEPTOR(int, epoll_ctl
, int epfd
, int op
, int fd
, void *ev
) {
1883 SCOPED_TSAN_INTERCEPTOR(epoll_ctl
, epfd
, op
, fd
, ev
);
1885 FdAccess(thr
, pc
, epfd
);
1886 if (epfd
>= 0 && fd
>= 0)
1887 FdAccess(thr
, pc
, fd
);
1888 if (op
== EPOLL_CTL_ADD
&& epfd
>= 0)
1889 FdRelease(thr
, pc
, epfd
);
1890 int res
= REAL(epoll_ctl
)(epfd
, op
, fd
, ev
);
1894 TSAN_INTERCEPTOR(int, epoll_wait
, int epfd
, void *ev
, int cnt
, int timeout
) {
1895 SCOPED_TSAN_INTERCEPTOR(epoll_wait
, epfd
, ev
, cnt
, timeout
);
1897 FdAccess(thr
, pc
, epfd
);
1898 int res
= BLOCK_REAL(epoll_wait
)(epfd
, ev
, cnt
, timeout
);
1899 if (res
> 0 && epfd
>= 0)
1900 FdAcquire(thr
, pc
, epfd
);
1904 TSAN_INTERCEPTOR(int, epoll_pwait
, int epfd
, void *ev
, int cnt
, int timeout
,
1906 SCOPED_TSAN_INTERCEPTOR(epoll_pwait
, epfd
, ev
, cnt
, timeout
, sigmask
);
1908 FdAccess(thr
, pc
, epfd
);
1909 int res
= BLOCK_REAL(epoll_pwait
)(epfd
, ev
, cnt
, timeout
, sigmask
);
1910 if (res
> 0 && epfd
>= 0)
1911 FdAcquire(thr
, pc
, epfd
);
1915 #define TSAN_MAYBE_INTERCEPT_EPOLL \
1916 TSAN_INTERCEPT(epoll_create); \
1917 TSAN_INTERCEPT(epoll_create1); \
1918 TSAN_INTERCEPT(epoll_ctl); \
1919 TSAN_INTERCEPT(epoll_wait); \
1920 TSAN_INTERCEPT(epoll_pwait)
1922 #define TSAN_MAYBE_INTERCEPT_EPOLL
1925 // The following functions are intercepted merely to process pending signals.
1926 // If program blocks signal X, we must deliver the signal before the function
1927 // returns. Similarly, if program unblocks a signal (or returns from sigsuspend)
1928 // it's better to deliver the signal straight away.
1929 TSAN_INTERCEPTOR(int, sigsuspend
, const __sanitizer_sigset_t
*mask
) {
1930 SCOPED_TSAN_INTERCEPTOR(sigsuspend
, mask
);
1931 return REAL(sigsuspend
)(mask
);
1934 TSAN_INTERCEPTOR(int, sigblock
, int mask
) {
1935 SCOPED_TSAN_INTERCEPTOR(sigblock
, mask
);
1936 return REAL(sigblock
)(mask
);
1939 TSAN_INTERCEPTOR(int, sigsetmask
, int mask
) {
1940 SCOPED_TSAN_INTERCEPTOR(sigsetmask
, mask
);
1941 return REAL(sigsetmask
)(mask
);
1944 TSAN_INTERCEPTOR(int, pthread_sigmask
, int how
, const __sanitizer_sigset_t
*set
,
1945 __sanitizer_sigset_t
*oldset
) {
1946 SCOPED_TSAN_INTERCEPTOR(pthread_sigmask
, how
, set
, oldset
);
1947 return REAL(pthread_sigmask
)(how
, set
, oldset
);
1952 static void ReportErrnoSpoiling(ThreadState
*thr
, uptr pc
) {
1953 VarSizeStackTrace stack
;
1954 // StackTrace::GetNestInstructionPc(pc) is used because return address is
1955 // expected, OutputReport() will undo this.
1956 ObtainCurrentStack(thr
, StackTrace::GetNextInstructionPc(pc
), &stack
);
1957 ThreadRegistryLock
l(&ctx
->thread_registry
);
1958 ScopedReport
rep(ReportTypeErrnoInSignal
);
1959 if (!IsFiredSuppression(ctx
, ReportTypeErrnoInSignal
, stack
)) {
1960 rep
.AddStack(stack
, true);
1961 OutputReport(thr
, rep
);
1965 static void CallUserSignalHandler(ThreadState
*thr
, bool sync
, bool acquire
,
1966 int sig
, __sanitizer_siginfo
*info
,
1968 __sanitizer_sigaction
*sigactions
= interceptor_ctx()->sigactions
;
1970 Acquire(thr
, 0, (uptr
)&sigactions
[sig
]);
1971 // Signals are generally asynchronous, so if we receive a signals when
1972 // ignores are enabled we should disable ignores. This is critical for sync
1973 // and interceptors, because otherwise we can miss synchronization and report
1975 int ignore_reads_and_writes
= thr
->ignore_reads_and_writes
;
1976 int ignore_interceptors
= thr
->ignore_interceptors
;
1977 int ignore_sync
= thr
->ignore_sync
;
1978 // For symbolizer we only process SIGSEGVs synchronously
1979 // (bug in symbolizer or in tsan). But we want to reset
1980 // in_symbolizer to fail gracefully. Symbolizer and user code
1981 // use different memory allocators, so if we don't reset
1982 // in_symbolizer we can get memory allocated with one being
1983 // feed with another, which can cause more crashes.
1984 int in_symbolizer
= thr
->in_symbolizer
;
1985 if (!ctx
->after_multithreaded_fork
) {
1986 thr
->ignore_reads_and_writes
= 0;
1987 thr
->fast_state
.ClearIgnoreBit();
1988 thr
->ignore_interceptors
= 0;
1989 thr
->ignore_sync
= 0;
1990 thr
->in_symbolizer
= 0;
1992 // Ensure that the handler does not spoil errno.
1993 const int saved_errno
= errno
;
1995 // This code races with sigaction. Be careful to not read sa_sigaction twice.
1996 // Also need to remember pc for reporting before the call,
1997 // because the handler can reset it.
1998 volatile uptr pc
= (sigactions
[sig
].sa_flags
& SA_SIGINFO
)
1999 ? (uptr
)sigactions
[sig
].sigaction
2000 : (uptr
)sigactions
[sig
].handler
;
2001 if (pc
!= sig_dfl
&& pc
!= sig_ign
) {
2002 // The callback can be either sa_handler or sa_sigaction.
2003 // They have different signatures, but we assume that passing
2004 // additional arguments to sa_handler works and is harmless.
2005 ((__sanitizer_sigactionhandler_ptr
)pc
)(sig
, info
, uctx
);
2007 if (!ctx
->after_multithreaded_fork
) {
2008 thr
->ignore_reads_and_writes
= ignore_reads_and_writes
;
2009 if (ignore_reads_and_writes
)
2010 thr
->fast_state
.SetIgnoreBit();
2011 thr
->ignore_interceptors
= ignore_interceptors
;
2012 thr
->ignore_sync
= ignore_sync
;
2013 thr
->in_symbolizer
= in_symbolizer
;
2015 // We do not detect errno spoiling for SIGTERM,
2016 // because some SIGTERM handlers do spoil errno but reraise SIGTERM,
2017 // tsan reports false positive in such case.
2018 // It's difficult to properly detect this situation (reraise),
2019 // because in async signal processing case (when handler is called directly
2020 // from rtl_generic_sighandler) we have not yet received the reraised
2021 // signal; and it looks too fragile to intercept all ways to reraise a signal.
2022 if (ShouldReport(thr
, ReportTypeErrnoInSignal
) && !sync
&& sig
!= SIGTERM
&&
2024 ReportErrnoSpoiling(thr
, pc
);
2025 errno
= saved_errno
;
2028 void ProcessPendingSignalsImpl(ThreadState
*thr
) {
2029 atomic_store(&thr
->pending_signals
, 0, memory_order_relaxed
);
2030 ThreadSignalContext
*sctx
= SigCtx(thr
);
2033 atomic_fetch_add(&thr
->in_signal_handler
, 1, memory_order_relaxed
);
2034 internal_sigfillset(&sctx
->emptyset
);
2035 int res
= REAL(pthread_sigmask
)(SIG_SETMASK
, &sctx
->emptyset
, &sctx
->oldset
);
2037 for (int sig
= 0; sig
< kSigCount
; sig
++) {
2038 SignalDesc
*signal
= &sctx
->pending_signals
[sig
];
2039 if (signal
->armed
) {
2040 signal
->armed
= false;
2041 CallUserSignalHandler(thr
, false, true, sig
, &signal
->siginfo
,
2045 res
= REAL(pthread_sigmask
)(SIG_SETMASK
, &sctx
->oldset
, 0);
2047 atomic_fetch_add(&thr
->in_signal_handler
, -1, memory_order_relaxed
);
2050 } // namespace __tsan
2052 static bool is_sync_signal(ThreadSignalContext
*sctx
, int sig
) {
2053 return sig
== SIGSEGV
|| sig
== SIGBUS
|| sig
== SIGILL
|| sig
== SIGTRAP
||
2054 sig
== SIGABRT
|| sig
== SIGFPE
|| sig
== SIGPIPE
|| sig
== SIGSYS
||
2055 // If we are sending signal to ourselves, we must process it now.
2056 (sctx
&& sig
== sctx
->int_signal_send
);
2059 void sighandler(int sig
, __sanitizer_siginfo
*info
, void *ctx
) {
2060 ThreadState
*thr
= cur_thread_init();
2061 ThreadSignalContext
*sctx
= SigCtx(thr
);
2062 if (sig
< 0 || sig
>= kSigCount
) {
2063 VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig
);
2066 // Don't mess with synchronous signals.
2067 const bool sync
= is_sync_signal(sctx
, sig
);
2069 // If we are in blocking function, we can safely process it now
2070 // (but check if we are in a recursive interceptor,
2071 // i.e. pthread_join()->munmap()).
2072 (sctx
&& atomic_load(&sctx
->in_blocking_func
, memory_order_relaxed
))) {
2073 atomic_fetch_add(&thr
->in_signal_handler
, 1, memory_order_relaxed
);
2074 if (sctx
&& atomic_load(&sctx
->in_blocking_func
, memory_order_relaxed
)) {
2075 atomic_store(&sctx
->in_blocking_func
, 0, memory_order_relaxed
);
2076 CallUserSignalHandler(thr
, sync
, true, sig
, info
, ctx
);
2077 atomic_store(&sctx
->in_blocking_func
, 1, memory_order_relaxed
);
2079 // Be very conservative with when we do acquire in this case.
2080 // It's unsafe to do acquire in async handlers, because ThreadState
2081 // can be in inconsistent state.
2082 // SIGSYS looks relatively safe -- it's synchronous and can actually
2083 // need some global state.
2084 bool acq
= (sig
== SIGSYS
);
2085 CallUserSignalHandler(thr
, sync
, acq
, sig
, info
, ctx
);
2087 atomic_fetch_add(&thr
->in_signal_handler
, -1, memory_order_relaxed
);
2093 SignalDesc
*signal
= &sctx
->pending_signals
[sig
];
2094 if (signal
->armed
== false) {
2095 signal
->armed
= true;
2096 internal_memcpy(&signal
->siginfo
, info
, sizeof(*info
));
2097 internal_memcpy(&signal
->ctx
, ctx
, sizeof(signal
->ctx
));
2098 atomic_store(&thr
->pending_signals
, 1, memory_order_relaxed
);
2102 TSAN_INTERCEPTOR(int, raise
, int sig
) {
2103 SCOPED_TSAN_INTERCEPTOR(raise
, sig
);
2104 ThreadSignalContext
*sctx
= SigCtx(thr
);
2106 int prev
= sctx
->int_signal_send
;
2107 sctx
->int_signal_send
= sig
;
2108 int res
= REAL(raise
)(sig
);
2109 CHECK_EQ(sctx
->int_signal_send
, sig
);
2110 sctx
->int_signal_send
= prev
;
2114 TSAN_INTERCEPTOR(int, kill
, int pid
, int sig
) {
2115 SCOPED_TSAN_INTERCEPTOR(kill
, pid
, sig
);
2116 ThreadSignalContext
*sctx
= SigCtx(thr
);
2118 int prev
= sctx
->int_signal_send
;
2119 if (pid
== (int)internal_getpid()) {
2120 sctx
->int_signal_send
= sig
;
2122 int res
= REAL(kill
)(pid
, sig
);
2123 if (pid
== (int)internal_getpid()) {
2124 CHECK_EQ(sctx
->int_signal_send
, sig
);
2125 sctx
->int_signal_send
= prev
;
2130 TSAN_INTERCEPTOR(int, pthread_kill
, void *tid
, int sig
) {
2131 SCOPED_TSAN_INTERCEPTOR(pthread_kill
, tid
, sig
);
2132 ThreadSignalContext
*sctx
= SigCtx(thr
);
2134 int prev
= sctx
->int_signal_send
;
2135 if (tid
== pthread_self()) {
2136 sctx
->int_signal_send
= sig
;
2138 int res
= REAL(pthread_kill
)(tid
, sig
);
2139 if (tid
== pthread_self()) {
2140 CHECK_EQ(sctx
->int_signal_send
, sig
);
2141 sctx
->int_signal_send
= prev
;
2146 TSAN_INTERCEPTOR(int, gettimeofday
, void *tv
, void *tz
) {
2147 SCOPED_TSAN_INTERCEPTOR(gettimeofday
, tv
, tz
);
2148 // It's intercepted merely to process pending signals.
2149 return REAL(gettimeofday
)(tv
, tz
);
2152 TSAN_INTERCEPTOR(int, getaddrinfo
, void *node
, void *service
,
2153 void *hints
, void *rv
) {
2154 SCOPED_TSAN_INTERCEPTOR(getaddrinfo
, node
, service
, hints
, rv
);
2155 // We miss atomic synchronization in getaddrinfo,
2156 // and can report false race between malloc and free
2157 // inside of getaddrinfo. So ignore memory accesses.
2158 ThreadIgnoreBegin(thr
, pc
);
2159 int res
= REAL(getaddrinfo
)(node
, service
, hints
, rv
);
2160 ThreadIgnoreEnd(thr
);
2164 TSAN_INTERCEPTOR(int, fork
, int fake
) {
2165 if (in_symbolizer())
2166 return REAL(fork
)(fake
);
2167 SCOPED_INTERCEPTOR_RAW(fork
, fake
);
2168 return REAL(fork
)(fake
);
2171 void atfork_prepare() {
2172 if (in_symbolizer())
2174 ThreadState
*thr
= cur_thread();
2175 const uptr pc
= StackTrace::GetCurrentPc();
2176 ForkBefore(thr
, pc
);
2179 void atfork_parent() {
2180 if (in_symbolizer())
2182 ThreadState
*thr
= cur_thread();
2183 const uptr pc
= StackTrace::GetCurrentPc();
2184 ForkParentAfter(thr
, pc
);
2187 void atfork_child() {
2188 if (in_symbolizer())
2190 ThreadState
*thr
= cur_thread();
2191 const uptr pc
= StackTrace::GetCurrentPc();
2192 ForkChildAfter(thr
, pc
);
2196 TSAN_INTERCEPTOR(int, vfork
, int fake
) {
2197 // Some programs (e.g. openjdk) call close for all file descriptors
2198 // in the child process. Under tsan it leads to false positives, because
2199 // address space is shared, so the parent process also thinks that
2200 // the descriptors are closed (while they are actually not).
2201 // This leads to false positives due to missed synchronization.
2202 // Strictly saying this is undefined behavior, because vfork child is not
2203 // allowed to call any functions other than exec/exit. But this is what
2204 // openjdk does, so we want to handle it.
2205 // We could disable interceptors in the child process. But it's not possible
2206 // to simply intercept and wrap vfork, because vfork child is not allowed
2207 // to return from the function that calls vfork, and that's exactly what
2208 // we would do. So this would require some assembly trickery as well.
2209 // Instead we simply turn vfork into fork.
2210 return WRAP(fork
)(fake
);
2213 #if !SANITIZER_MAC && !SANITIZER_ANDROID
2214 typedef int (*dl_iterate_phdr_cb_t
)(__sanitizer_dl_phdr_info
*info
, SIZE_T size
,
2216 struct dl_iterate_phdr_data
{
2219 dl_iterate_phdr_cb_t cb
;
2223 static bool IsAppNotRodata(uptr addr
) {
2224 return IsAppMem(addr
) && *MemToShadow(addr
) != kShadowRodata
;
2227 static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info
*info
, SIZE_T size
,
2229 dl_iterate_phdr_data
*cbdata
= (dl_iterate_phdr_data
*)data
;
2230 // dlopen/dlclose allocate/free dynamic-linker-internal memory, which is later
2231 // accessible in dl_iterate_phdr callback. But we don't see synchronization
2232 // inside of dynamic linker, so we "unpoison" it here in order to not
2233 // produce false reports. Ignoring malloc/free in dlopen/dlclose is not enough
2234 // because some libc functions call __libc_dlopen.
2235 if (info
&& IsAppNotRodata((uptr
)info
->dlpi_name
))
2236 MemoryResetRange(cbdata
->thr
, cbdata
->pc
, (uptr
)info
->dlpi_name
,
2237 internal_strlen(info
->dlpi_name
));
2238 int res
= cbdata
->cb(info
, size
, cbdata
->data
);
2239 // Perform the check one more time in case info->dlpi_name was overwritten
2240 // by user callback.
2241 if (info
&& IsAppNotRodata((uptr
)info
->dlpi_name
))
2242 MemoryResetRange(cbdata
->thr
, cbdata
->pc
, (uptr
)info
->dlpi_name
,
2243 internal_strlen(info
->dlpi_name
));
2247 TSAN_INTERCEPTOR(int, dl_iterate_phdr
, dl_iterate_phdr_cb_t cb
, void *data
) {
2248 SCOPED_TSAN_INTERCEPTOR(dl_iterate_phdr
, cb
, data
);
2249 dl_iterate_phdr_data cbdata
;
2254 int res
= REAL(dl_iterate_phdr
)(dl_iterate_phdr_cb
, &cbdata
);
2259 static int OnExit(ThreadState
*thr
) {
2260 int status
= Finalize(thr
);
2265 struct TsanInterceptorContext
{
2271 static void HandleRecvmsg(ThreadState
*thr
, uptr pc
,
2272 __sanitizer_msghdr
*msg
) {
2274 int cnt
= ExtractRecvmsgFDs(msg
, fds
, ARRAY_SIZE(fds
));
2275 for (int i
= 0; i
< cnt
; i
++)
2276 FdEventCreate(thr
, pc
, fds
[i
]);
2280 #include "sanitizer_common/sanitizer_platform_interceptors.h"
2281 // Causes interceptor recursion (getaddrinfo() and fopen())
2282 #undef SANITIZER_INTERCEPT_GETADDRINFO
2283 // We define our own.
2284 #if SANITIZER_INTERCEPT_TLS_GET_ADDR
2285 #define NEED_TLS_GET_ADDR
2287 #undef SANITIZER_INTERCEPT_TLS_GET_ADDR
2288 #define SANITIZER_INTERCEPT_TLS_GET_OFFSET 1
2289 #undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK
2291 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
2292 #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
2293 INTERCEPT_FUNCTION_VER(name, ver)
2294 #define COMMON_INTERCEPT_FUNCTION_VER_UNVERSIONED_FALLBACK(name, ver) \
2295 (INTERCEPT_FUNCTION_VER(name, ver) || INTERCEPT_FUNCTION(name))
2297 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
2298 MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \
2299 ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
2302 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
2303 MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \
2304 ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
2307 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
2308 SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \
2309 TsanInterceptorContext _ctx = {thr, pc}; \
2310 ctx = (void *)&_ctx; \
2313 #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
2314 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
2315 TsanInterceptorContext _ctx = {thr, pc}; \
2316 ctx = (void *)&_ctx; \
2319 #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
2321 Acquire(thr, pc, File2addr(path)); \
2323 int fd = fileno_unlocked(file); \
2324 if (fd >= 0) FdFileCreate(thr, pc, fd); \
2327 #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \
2329 int fd = fileno_unlocked(file); \
2330 if (fd >= 0) FdClose(thr, pc, fd); \
2333 #define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \
2334 libignore()->OnLibraryLoaded(filename)
2336 #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
2337 libignore()->OnLibraryUnloaded()
2339 #define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \
2340 Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u)
2342 #define COMMON_INTERCEPTOR_RELEASE(ctx, u) \
2343 Release(((TsanInterceptorContext *) ctx)->thr, pc, u)
2345 #define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
2346 Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path))
2348 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
2349 FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2351 #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
2352 FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2354 #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \
2355 FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2357 #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
2358 FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd)
2360 #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
2361 ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
2363 #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
2364 __tsan::ctx->thread_registry.SetThreadNameByUserId(thread, name)
2366 #define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
2368 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
2369 OnExit(((TsanInterceptorContext *) ctx)->thr)
2371 #define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \
2372 MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \
2373 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2375 #define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \
2376 MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \
2377 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2379 #define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
2380 MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
2381 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2383 #define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
2384 MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
2385 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2387 #define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \
2388 MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \
2389 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2391 #define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \
2394 return mmap_interceptor(thr, pc, REAL(mmap), addr, sz, prot, flags, fd, \
2399 #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \
2400 HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \
2401 ((TsanInterceptorContext *)ctx)->pc, msg)
2404 #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \
2405 if (TsanThread *t = GetCurrentThread()) { \
2406 *begin = t->tls_begin(); \
2407 *end = t->tls_end(); \
2409 *begin = *end = 0; \
2412 #define COMMON_INTERCEPTOR_USER_CALLBACK_START() \
2413 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START()
2415 #define COMMON_INTERCEPTOR_USER_CALLBACK_END() \
2416 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END()
2418 #include "sanitizer_common/sanitizer_common_interceptors.inc"
2420 static int sigaction_impl(int sig
, const __sanitizer_sigaction
*act
,
2421 __sanitizer_sigaction
*old
);
2422 static __sanitizer_sighandler_ptr
signal_impl(int sig
,
2423 __sanitizer_sighandler_ptr h
);
2425 #define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \
2426 { return sigaction_impl(signo, act, oldact); }
2428 #define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \
2429 { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); }
2431 #include "sanitizer_common/sanitizer_signal_interceptors.inc"
2433 int sigaction_impl(int sig
, const __sanitizer_sigaction
*act
,
2434 __sanitizer_sigaction
*old
) {
2435 // Note: if we call REAL(sigaction) directly for any reason without proxying
2436 // the signal handler through sighandler, very bad things will happen.
2437 // The handler will run synchronously and corrupt tsan per-thread state.
2438 SCOPED_INTERCEPTOR_RAW(sigaction
, sig
, act
, old
);
2439 if (sig
<= 0 || sig
>= kSigCount
) {
2440 errno
= errno_EINVAL
;
2443 __sanitizer_sigaction
*sigactions
= interceptor_ctx()->sigactions
;
2444 __sanitizer_sigaction old_stored
;
2445 if (old
) internal_memcpy(&old_stored
, &sigactions
[sig
], sizeof(old_stored
));
2446 __sanitizer_sigaction newact
;
2448 // Copy act into sigactions[sig].
2449 // Can't use struct copy, because compiler can emit call to memcpy.
2450 // Can't use internal_memcpy, because it copies byte-by-byte,
2451 // and signal handler reads the handler concurrently. It it can read
2452 // some bytes from old value and some bytes from new value.
2453 // Use volatile to prevent insertion of memcpy.
2454 sigactions
[sig
].handler
=
2455 *(volatile __sanitizer_sighandler_ptr
const *)&act
->handler
;
2456 sigactions
[sig
].sa_flags
= *(volatile int const *)&act
->sa_flags
;
2457 internal_memcpy(&sigactions
[sig
].sa_mask
, &act
->sa_mask
,
2458 sizeof(sigactions
[sig
].sa_mask
));
2459 #if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
2460 sigactions
[sig
].sa_restorer
= act
->sa_restorer
;
2462 internal_memcpy(&newact
, act
, sizeof(newact
));
2463 internal_sigfillset(&newact
.sa_mask
);
2464 if ((act
->sa_flags
& SA_SIGINFO
) ||
2465 ((uptr
)act
->handler
!= sig_ign
&& (uptr
)act
->handler
!= sig_dfl
)) {
2466 newact
.sa_flags
|= SA_SIGINFO
;
2467 newact
.sigaction
= sighandler
;
2469 ReleaseStore(thr
, pc
, (uptr
)&sigactions
[sig
]);
2472 int res
= REAL(sigaction
)(sig
, act
, old
);
2473 if (res
== 0 && old
&& old
->sigaction
== sighandler
)
2474 internal_memcpy(old
, &old_stored
, sizeof(*old
));
2478 static __sanitizer_sighandler_ptr
signal_impl(int sig
,
2479 __sanitizer_sighandler_ptr h
) {
2480 __sanitizer_sigaction act
;
2482 internal_memset(&act
.sa_mask
, -1, sizeof(act
.sa_mask
));
2484 __sanitizer_sigaction old
;
2485 int res
= sigaction_symname(sig
, &act
, &old
);
2486 if (res
) return (__sanitizer_sighandler_ptr
)sig_err
;
2490 #define TSAN_SYSCALL() \
2491 ThreadState *thr = cur_thread(); \
2492 if (thr->ignore_interceptors) \
2494 ScopedSyscall scoped_syscall(thr)
2496 struct ScopedSyscall
{
2499 explicit ScopedSyscall(ThreadState
*thr
) : thr(thr
) { LazyInitialize(thr
); }
2502 ProcessPendingSignals(thr
);
2506 #if !SANITIZER_FREEBSD && !SANITIZER_MAC
2507 static void syscall_access_range(uptr pc
, uptr p
, uptr s
, bool write
) {
2509 MemoryAccessRange(thr
, pc
, p
, s
, write
);
2512 static USED
void syscall_acquire(uptr pc
, uptr addr
) {
2514 Acquire(thr
, pc
, addr
);
2515 DPrintf("syscall_acquire(0x%zx))\n", addr
);
2518 static USED
void syscall_release(uptr pc
, uptr addr
) {
2520 DPrintf("syscall_release(0x%zx)\n", addr
);
2521 Release(thr
, pc
, addr
);
2524 static void syscall_fd_close(uptr pc
, int fd
) {
2526 FdClose(thr
, pc
, fd
);
2529 static USED
void syscall_fd_acquire(uptr pc
, int fd
) {
2531 FdAcquire(thr
, pc
, fd
);
2532 DPrintf("syscall_fd_acquire(%d)\n", fd
);
2535 static USED
void syscall_fd_release(uptr pc
, int fd
) {
2537 DPrintf("syscall_fd_release(%d)\n", fd
);
2538 FdRelease(thr
, pc
, fd
);
2541 static void syscall_pre_fork(uptr pc
) { ForkBefore(cur_thread(), pc
); }
2543 static void syscall_post_fork(uptr pc
, int pid
) {
2544 ThreadState
*thr
= cur_thread();
2547 ForkChildAfter(thr
, pc
);
2549 } else if (pid
> 0) {
2551 ForkParentAfter(thr
, pc
);
2554 ForkParentAfter(thr
, pc
);
2559 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \
2560 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false)
2562 #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
2563 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true)
2565 #define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
2571 #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
2577 #define COMMON_SYSCALL_ACQUIRE(addr) \
2578 syscall_acquire(GET_CALLER_PC(), (uptr)(addr))
2580 #define COMMON_SYSCALL_RELEASE(addr) \
2581 syscall_release(GET_CALLER_PC(), (uptr)(addr))
2583 #define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd)
2585 #define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd)
2587 #define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd)
2589 #define COMMON_SYSCALL_PRE_FORK() \
2590 syscall_pre_fork(GET_CALLER_PC())
2592 #define COMMON_SYSCALL_POST_FORK(res) \
2593 syscall_post_fork(GET_CALLER_PC(), res)
2595 #include "sanitizer_common/sanitizer_common_syscalls.inc"
2596 #include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
2598 #ifdef NEED_TLS_GET_ADDR
2600 static void handle_tls_addr(void *arg
, void *res
) {
2601 ThreadState
*thr
= cur_thread();
2604 DTLS::DTV
*dtv
= DTLS_on_tls_get_addr(arg
, res
, thr
->tls_addr
,
2605 thr
->tls_addr
+ thr
->tls_size
);
2608 // New DTLS block has been allocated.
2609 MemoryResetRange(thr
, 0, dtv
->beg
, dtv
->size
);
2613 // Define own interceptor instead of sanitizer_common's for three reasons:
2614 // 1. It must not process pending signals.
2615 // Signal handlers may contain MOVDQA instruction (see below).
2616 // 2. It must be as simple as possible to not contain MOVDQA.
2617 // 3. Sanitizer_common version uses COMMON_INTERCEPTOR_INITIALIZE_RANGE which
2618 // is empty for tsan (meant only for msan).
2619 // Note: __tls_get_addr can be called with mis-aligned stack due to:
2620 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066
2621 // So the interceptor must work with mis-aligned stack, in particular, does not
2622 // execute MOVDQA with stack addresses.
2623 TSAN_INTERCEPTOR(void *, __tls_get_addr
, void *arg
) {
2624 void *res
= REAL(__tls_get_addr
)(arg
);
2625 handle_tls_addr(arg
, res
);
2628 #else // SANITIZER_S390
2629 TSAN_INTERCEPTOR(uptr
, __tls_get_addr_internal
, void *arg
) {
2630 uptr res
= __tls_get_offset_wrapper(arg
, REAL(__tls_get_offset
));
2631 char *tp
= static_cast<char *>(__builtin_thread_pointer());
2632 handle_tls_addr(arg
, res
+ tp
);
2638 #if SANITIZER_NETBSD
2639 TSAN_INTERCEPTOR(void, _lwp_exit
) {
2640 SCOPED_TSAN_INTERCEPTOR(_lwp_exit
);
2641 DestroyThreadState();
2644 #define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit)
2646 #define TSAN_MAYBE_INTERCEPT__LWP_EXIT
2649 #if SANITIZER_FREEBSD
2650 TSAN_INTERCEPTOR(void, thr_exit
, tid_t
*state
) {
2651 SCOPED_TSAN_INTERCEPTOR(thr_exit
, state
);
2652 DestroyThreadState();
2653 REAL(thr_exit(state
));
2655 #define TSAN_MAYBE_INTERCEPT_THR_EXIT TSAN_INTERCEPT(thr_exit)
2657 #define TSAN_MAYBE_INTERCEPT_THR_EXIT
2660 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init
, void *c
, void *a
)
2661 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal
, void *c
)
2662 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast
, void *c
)
2663 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait
, void *c
, void *m
)
2664 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy
, void *c
)
2665 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init
, void *m
, void *a
)
2666 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy
, void *m
)
2667 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock
, void *m
)
2668 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init
, void *m
, void *a
)
2669 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy
, void *m
)
2670 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock
, void *m
)
2671 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_tryrdlock
, void *m
)
2672 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock
, void *m
)
2673 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock
, void *m
)
2674 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock
, void *m
)
2675 TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once
, void *o
, void (*f
)())
2676 TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(int, sigsetmask
, sigmask
, int a
, void *b
,
2681 static void finalize(void *arg
) {
2682 ThreadState
*thr
= cur_thread();
2683 int status
= Finalize(thr
);
2684 // Make sure the output is not lost.
2690 #if !SANITIZER_MAC && !SANITIZER_ANDROID
2691 static void unreachable() {
2692 Report("FATAL: ThreadSanitizer: unreachable called\n");
2697 // Define default implementation since interception of libdispatch is optional.
2698 SANITIZER_WEAK_ATTRIBUTE
void InitializeLibdispatchInterceptors() {}
2700 void InitializeInterceptors() {
2702 // We need to setup it early, because functions like dlsym() can call it.
2703 REAL(memset
) = internal_memset
;
2704 REAL(memcpy
) = internal_memcpy
;
2707 new(interceptor_ctx()) InterceptorContext();
2709 InitializeCommonInterceptors();
2710 InitializeSignalInterceptors();
2711 InitializeLibdispatchInterceptors();
2714 // We can not use TSAN_INTERCEPT to get setjmp addr,
2715 // because it does &setjmp and setjmp is not present in some versions of libc.
2716 using __interception::InterceptFunction
;
2717 InterceptFunction(TSAN_STRING_SETJMP
, (uptr
*)&REAL(setjmp_symname
), 0, 0);
2718 InterceptFunction("_setjmp", (uptr
*)&REAL(_setjmp
), 0, 0);
2719 InterceptFunction(TSAN_STRING_SIGSETJMP
, (uptr
*)&REAL(sigsetjmp_symname
), 0,
2721 #if !SANITIZER_NETBSD
2722 InterceptFunction("__sigsetjmp", (uptr
*)&REAL(__sigsetjmp
), 0, 0);
2726 TSAN_INTERCEPT(longjmp_symname
);
2727 TSAN_INTERCEPT(siglongjmp_symname
);
2728 #if SANITIZER_NETBSD
2729 TSAN_INTERCEPT(_longjmp
);
2732 TSAN_INTERCEPT(malloc
);
2733 TSAN_INTERCEPT(__libc_memalign
);
2734 TSAN_INTERCEPT(calloc
);
2735 TSAN_INTERCEPT(realloc
);
2736 TSAN_INTERCEPT(reallocarray
);
2737 TSAN_INTERCEPT(free
);
2738 TSAN_INTERCEPT(cfree
);
2739 TSAN_INTERCEPT(munmap
);
2740 TSAN_MAYBE_INTERCEPT_MEMALIGN
;
2741 TSAN_INTERCEPT(valloc
);
2742 TSAN_MAYBE_INTERCEPT_PVALLOC
;
2743 TSAN_INTERCEPT(posix_memalign
);
2745 TSAN_INTERCEPT(strcpy
);
2746 TSAN_INTERCEPT(strncpy
);
2747 TSAN_INTERCEPT(strdup
);
2749 TSAN_INTERCEPT(pthread_create
);
2750 TSAN_INTERCEPT(pthread_join
);
2751 TSAN_INTERCEPT(pthread_detach
);
2752 TSAN_INTERCEPT(pthread_exit
);
2754 TSAN_INTERCEPT(pthread_tryjoin_np
);
2755 TSAN_INTERCEPT(pthread_timedjoin_np
);
2758 TSAN_INTERCEPT_VER(pthread_cond_init
, PTHREAD_ABI_BASE
);
2759 TSAN_INTERCEPT_VER(pthread_cond_signal
, PTHREAD_ABI_BASE
);
2760 TSAN_INTERCEPT_VER(pthread_cond_broadcast
, PTHREAD_ABI_BASE
);
2761 TSAN_INTERCEPT_VER(pthread_cond_wait
, PTHREAD_ABI_BASE
);
2762 TSAN_INTERCEPT_VER(pthread_cond_timedwait
, PTHREAD_ABI_BASE
);
2763 TSAN_INTERCEPT_VER(pthread_cond_destroy
, PTHREAD_ABI_BASE
);
2765 TSAN_MAYBE_PTHREAD_COND_CLOCKWAIT
;
2767 TSAN_INTERCEPT(pthread_mutex_init
);
2768 TSAN_INTERCEPT(pthread_mutex_destroy
);
2769 TSAN_INTERCEPT(pthread_mutex_trylock
);
2770 TSAN_INTERCEPT(pthread_mutex_timedlock
);
2772 TSAN_INTERCEPT(pthread_spin_init
);
2773 TSAN_INTERCEPT(pthread_spin_destroy
);
2774 TSAN_INTERCEPT(pthread_spin_lock
);
2775 TSAN_INTERCEPT(pthread_spin_trylock
);
2776 TSAN_INTERCEPT(pthread_spin_unlock
);
2778 TSAN_INTERCEPT(pthread_rwlock_init
);
2779 TSAN_INTERCEPT(pthread_rwlock_destroy
);
2780 TSAN_INTERCEPT(pthread_rwlock_rdlock
);
2781 TSAN_INTERCEPT(pthread_rwlock_tryrdlock
);
2782 TSAN_INTERCEPT(pthread_rwlock_timedrdlock
);
2783 TSAN_INTERCEPT(pthread_rwlock_wrlock
);
2784 TSAN_INTERCEPT(pthread_rwlock_trywrlock
);
2785 TSAN_INTERCEPT(pthread_rwlock_timedwrlock
);
2786 TSAN_INTERCEPT(pthread_rwlock_unlock
);
2788 TSAN_INTERCEPT(pthread_barrier_init
);
2789 TSAN_INTERCEPT(pthread_barrier_destroy
);
2790 TSAN_INTERCEPT(pthread_barrier_wait
);
2792 TSAN_INTERCEPT(pthread_once
);
2794 TSAN_INTERCEPT(fstat
);
2795 TSAN_MAYBE_INTERCEPT___FXSTAT
;
2796 TSAN_MAYBE_INTERCEPT_FSTAT64
;
2797 TSAN_MAYBE_INTERCEPT___FXSTAT64
;
2798 TSAN_INTERCEPT(open
);
2799 TSAN_MAYBE_INTERCEPT_OPEN64
;
2800 TSAN_INTERCEPT(creat
);
2801 TSAN_MAYBE_INTERCEPT_CREAT64
;
2802 TSAN_INTERCEPT(dup
);
2803 TSAN_INTERCEPT(dup2
);
2804 TSAN_INTERCEPT(dup3
);
2805 TSAN_MAYBE_INTERCEPT_EVENTFD
;
2806 TSAN_MAYBE_INTERCEPT_SIGNALFD
;
2807 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT
;
2808 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1
;
2809 TSAN_INTERCEPT(socket
);
2810 TSAN_INTERCEPT(socketpair
);
2811 TSAN_INTERCEPT(connect
);
2812 TSAN_INTERCEPT(bind
);
2813 TSAN_INTERCEPT(listen
);
2814 TSAN_MAYBE_INTERCEPT_EPOLL
;
2815 TSAN_INTERCEPT(close
);
2816 TSAN_MAYBE_INTERCEPT___CLOSE
;
2817 TSAN_MAYBE_INTERCEPT___RES_ICLOSE
;
2818 TSAN_INTERCEPT(pipe
);
2819 TSAN_INTERCEPT(pipe2
);
2821 TSAN_INTERCEPT(unlink
);
2822 TSAN_INTERCEPT(tmpfile
);
2823 TSAN_MAYBE_INTERCEPT_TMPFILE64
;
2824 TSAN_INTERCEPT(abort
);
2825 TSAN_INTERCEPT(rmdir
);
2826 TSAN_INTERCEPT(closedir
);
2828 TSAN_INTERCEPT(sigsuspend
);
2829 TSAN_INTERCEPT(sigblock
);
2830 TSAN_INTERCEPT(sigsetmask
);
2831 TSAN_INTERCEPT(pthread_sigmask
);
2832 TSAN_INTERCEPT(raise
);
2833 TSAN_INTERCEPT(kill
);
2834 TSAN_INTERCEPT(pthread_kill
);
2835 TSAN_INTERCEPT(sleep
);
2836 TSAN_INTERCEPT(usleep
);
2837 TSAN_INTERCEPT(nanosleep
);
2838 TSAN_INTERCEPT(pause
);
2839 TSAN_INTERCEPT(gettimeofday
);
2840 TSAN_INTERCEPT(getaddrinfo
);
2842 TSAN_INTERCEPT(fork
);
2843 TSAN_INTERCEPT(vfork
);
2844 #if !SANITIZER_ANDROID
2845 TSAN_INTERCEPT(dl_iterate_phdr
);
2847 TSAN_MAYBE_INTERCEPT_ON_EXIT
;
2848 TSAN_INTERCEPT(__cxa_atexit
);
2849 TSAN_INTERCEPT(_exit
);
2851 #ifdef NEED_TLS_GET_ADDR
2853 TSAN_INTERCEPT(__tls_get_addr
);
2855 TSAN_INTERCEPT(__tls_get_addr_internal
);
2856 TSAN_INTERCEPT(__tls_get_offset
);
2860 TSAN_MAYBE_INTERCEPT__LWP_EXIT
;
2861 TSAN_MAYBE_INTERCEPT_THR_EXIT
;
2863 #if !SANITIZER_MAC && !SANITIZER_ANDROID
2864 // Need to setup it, because interceptors check that the function is resolved.
2865 // But atexit is emitted directly into the module, so can't be resolved.
2866 REAL(atexit
) = (int(*)(void(*)()))unreachable
;
2869 if (REAL(__cxa_atexit
)(&finalize
, 0, 0)) {
2870 Printf("ThreadSanitizer: failed to setup atexit callback\n");
2873 if (pthread_atfork(atfork_prepare
, atfork_parent
, atfork_child
)) {
2874 Printf("ThreadSanitizer: failed to setup atfork callbacks\n");
2878 #if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
2879 if (pthread_key_create(&interceptor_ctx()->finalize_key
, &thread_finalize
)) {
2880 Printf("ThreadSanitizer: failed to create thread key\n");
2885 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init
);
2886 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal
);
2887 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast
);
2888 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_wait
);
2889 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy
);
2890 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init
);
2891 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy
);
2892 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock
);
2893 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init
);
2894 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy
);
2895 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock
);
2896 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_tryrdlock
);
2897 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_wrlock
);
2898 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock
);
2899 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock
);
2900 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once
);
2901 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(sigsetmask
);
2906 } // namespace __tsan
2908 // Invisible barrier for tests.
2909 // There were several unsuccessful iterations for this functionality:
2910 // 1. Initially it was implemented in user code using
2911 // REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on
2912 // MacOS. Futexes are linux-specific for this matter.
2913 // 2. Then we switched to atomics+usleep(10). But usleep produced parasitic
2914 // "as-if synchronized via sleep" messages in reports which failed some
2916 // 3. Then we switched to atomics+sched_yield. But this produced tons of tsan-
2917 // visible events, which lead to "failed to restore stack trace" failures.
2918 // Note that no_sanitize_thread attribute does not turn off atomic interception
2919 // so attaching it to the function defined in user code does not help.
2920 // That's why we now have what we have.
2921 constexpr u32 kBarrierThreadBits
= 10;
2922 constexpr u32 kBarrierThreads
= 1 << kBarrierThreadBits
;
2924 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_testonly_barrier_init(
2925 atomic_uint32_t
*barrier
, u32 num_threads
) {
2926 if (num_threads
>= kBarrierThreads
) {
2927 Printf("barrier_init: count is too large (%d)\n", num_threads
);
2930 // kBarrierThreadBits lsb is thread count,
2931 // the remaining are count of entered threads.
2932 atomic_store(barrier
, num_threads
, memory_order_relaxed
);
2935 static u32
barrier_epoch(u32 value
) {
2936 return (value
>> kBarrierThreadBits
) / (value
& (kBarrierThreads
- 1));
2939 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
void __tsan_testonly_barrier_wait(
2940 atomic_uint32_t
*barrier
) {
2941 u32 old
= atomic_fetch_add(barrier
, kBarrierThreads
, memory_order_relaxed
);
2942 u32 old_epoch
= barrier_epoch(old
);
2943 if (barrier_epoch(old
+ kBarrierThreads
) != old_epoch
) {
2944 FutexWake(barrier
, (1 << 30));
2948 u32 cur
= atomic_load(barrier
, memory_order_relaxed
);
2949 if (barrier_epoch(cur
) != old_epoch
)
2951 FutexWait(barrier
, cur
);