1 //===-- tsan_interceptors.cc ----------------------------------------------===//
3 // This file is distributed under the University of Illinois Open Source
4 // License. See LICENSE.TXT for details.
6 //===----------------------------------------------------------------------===//
8 // This file is a part of ThreadSanitizer (TSan), a race detector.
10 // FIXME: move as many interceptors as possible into
11 // sanitizer_common/sanitizer_common_interceptors.inc
12 //===----------------------------------------------------------------------===//
14 #include "sanitizer_common/sanitizer_atomic.h"
15 #include "sanitizer_common/sanitizer_libc.h"
16 #include "sanitizer_common/sanitizer_linux.h"
17 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
18 #include "sanitizer_common/sanitizer_placement_new.h"
19 #include "sanitizer_common/sanitizer_stacktrace.h"
20 #include "interception/interception.h"
21 #include "tsan_interface.h"
22 #include "tsan_platform.h"
23 #include "tsan_suppressions.h"
25 #include "tsan_mman.h"
28 using namespace __tsan
; // NOLINT
31 #define __errno_location __error
32 #define __libc_malloc __malloc
33 #define __libc_realloc __realloc
34 #define __libc_calloc __calloc
35 #define __libc_free __free
36 #define stdout __stdoutp
37 #define stderr __stderrp
40 const int kSigCount
= 65;
43 // The size is determined by looking at sizeof of real siginfo_t on linux.
44 u64 opaque
[128 / sizeof(u64
)];
48 // The size is determined by looking at sizeof of real ucontext_t on linux.
49 u64 opaque
[936 / sizeof(u64
) + 1];
52 extern "C" int pthread_attr_init(void *attr
);
53 extern "C" int pthread_attr_destroy(void *attr
);
54 DECLARE_REAL(int, pthread_attr_getdetachstate
, void *, void *)
55 extern "C" int pthread_attr_setstacksize(void *attr
, uptr stacksize
);
56 extern "C" int pthread_key_create(unsigned *key
, void (*destructor
)(void* v
));
57 extern "C" int pthread_setspecific(unsigned key
, const void *v
);
58 DECLARE_REAL(int, pthread_mutexattr_gettype
, void *, void *)
59 extern "C" int pthread_yield();
60 extern "C" int pthread_sigmask(int how
, const __sanitizer_sigset_t
*set
,
61 __sanitizer_sigset_t
*oldset
);
62 // REAL(sigfillset) defined in common interceptors.
63 DECLARE_REAL(int, sigfillset
, __sanitizer_sigset_t
*set
)
64 DECLARE_REAL(int, fflush
, __sanitizer_FILE
*fp
)
65 extern "C" void *pthread_self();
66 extern "C" void _exit(int status
);
67 extern "C" int *__errno_location();
68 extern "C" int fileno_unlocked(void *stream
);
69 extern "C" void *__libc_malloc(uptr size
);
70 extern "C" void *__libc_calloc(uptr size
, uptr n
);
71 extern "C" void *__libc_realloc(void *ptr
, uptr size
);
72 extern "C" void __libc_free(void *ptr
);
73 #if !SANITIZER_FREEBSD
74 extern "C" int mallopt(int param
, int value
);
76 extern __sanitizer_FILE
*stdout
, *stderr
;
77 const int PTHREAD_MUTEX_RECURSIVE
= 1;
78 const int PTHREAD_MUTEX_RECURSIVE_NP
= 1;
79 const int EINVAL
= 22;
81 const int EOWNERDEAD
= 130;
82 const int EPOLL_CTL_ADD
= 1;
84 const int SIGABRT
= 6;
86 const int SIGSEGV
= 11;
87 const int SIGPIPE
= 13;
88 const int SIGTERM
= 15;
90 const int SIGSYS
= 31;
91 void *const MAP_FAILED
= (void*)-1;
92 const int PTHREAD_BARRIER_SERIAL_THREAD
= -1;
93 const int MAP_FIXED
= 0x10;
94 typedef long long_t
; // NOLINT
96 // From /usr/include/unistd.h
97 # define F_ULOCK 0 /* Unlock a previously locked region. */
98 # define F_LOCK 1 /* Lock a region for exclusive use. */
99 # define F_TLOCK 2 /* Test and lock a region for exclusive use. */
100 # define F_TEST 3 /* Test a region for other processes locks. */
102 typedef void (*sighandler_t
)(int sig
);
104 #define errno (*__errno_location())
108 sighandler_t sa_handler
;
109 void (*sa_sigaction
)(int sig
, my_siginfo_t
*siginfo
, void *uctx
);
111 #if SANITIZER_FREEBSD
113 __sanitizer_sigset_t sa_mask
;
115 __sanitizer_sigset_t sa_mask
;
117 void (*sa_restorer
)();
121 const sighandler_t SIG_DFL
= (sighandler_t
)0;
122 const sighandler_t SIG_IGN
= (sighandler_t
)1;
123 const sighandler_t SIG_ERR
= (sighandler_t
)-1;
124 const int SA_SIGINFO
= 4;
125 const int SIG_SETMASK
= 2;
131 static sigaction_t sigactions
[kSigCount
];
137 my_siginfo_t siginfo
;
141 struct SignalContext
{
143 atomic_uintptr_t in_blocking_func
;
144 atomic_uintptr_t have_pending_signals
;
145 SignalDesc pending_signals
[kSigCount
];
148 // The object is 64-byte aligned, because we want hot data to be located in
149 // a single cache line if possible (it's accessed in every interceptor).
150 static ALIGNED(64) char libignore_placeholder
[sizeof(LibIgnore
)];
151 static LibIgnore
*libignore() {
152 return reinterpret_cast<LibIgnore
*>(&libignore_placeholder
[0]);
155 void InitializeLibIgnore() {
156 libignore()->Init(*SuppressionContext::Get());
157 libignore()->OnLibraryLoaded(0);
160 } // namespace __tsan
162 static SignalContext
*SigCtx(ThreadState
*thr
) {
163 SignalContext
*ctx
= (SignalContext
*)thr
->signal_ctx
;
164 if (ctx
== 0 && !thr
->is_dead
) {
165 ctx
= (SignalContext
*)MmapOrDie(sizeof(*ctx
), "SignalContext");
166 MemoryResetRange(thr
, (uptr
)&SigCtx
, (uptr
)ctx
, sizeof(*ctx
));
167 thr
->signal_ctx
= ctx
;
172 static unsigned g_thread_finalize_key
;
174 class ScopedInterceptor
{
176 ScopedInterceptor(ThreadState
*thr
, const char *fname
, uptr pc
);
177 ~ScopedInterceptor();
179 ThreadState
*const thr_
;
181 bool in_ignored_lib_
;
184 ScopedInterceptor::ScopedInterceptor(ThreadState
*thr
, const char *fname
,
188 , in_ignored_lib_(false) {
189 if (!thr_
->ignore_interceptors
) {
193 DPrintf("#%d: intercept %s()\n", thr_
->tid
, fname
);
194 if (!thr_
->in_ignored_lib
&& libignore()->IsIgnored(pc
)) {
195 in_ignored_lib_
= true;
196 thr_
->in_ignored_lib
= true;
197 ThreadIgnoreBegin(thr_
, pc_
);
201 ScopedInterceptor::~ScopedInterceptor() {
202 if (in_ignored_lib_
) {
203 thr_
->in_ignored_lib
= false;
204 ThreadIgnoreEnd(thr_
, pc_
);
206 if (!thr_
->ignore_interceptors
) {
207 ProcessPendingSignals(thr_
);
213 #define SCOPED_INTERCEPTOR_RAW(func, ...) \
214 ThreadState *thr = cur_thread(); \
215 const uptr caller_pc = GET_CALLER_PC(); \
216 ScopedInterceptor si(thr, #func, caller_pc); \
217 const uptr pc = StackTrace::GetCurrentPc(); \
221 #define SCOPED_TSAN_INTERCEPTOR(func, ...) \
222 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
223 if (REAL(func) == 0) { \
224 Report("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
227 if (thr->ignore_interceptors || thr->in_ignored_lib) \
228 return REAL(func)(__VA_ARGS__); \
231 #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
232 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
233 #if SANITIZER_FREEBSD
234 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
236 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
239 #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name))
241 struct BlockingCall
{
242 explicit BlockingCall(ThreadState
*thr
)
246 atomic_store(&ctx
->in_blocking_func
, 1, memory_order_relaxed
);
247 if (atomic_load(&ctx
->have_pending_signals
, memory_order_relaxed
) == 0)
249 atomic_store(&ctx
->in_blocking_func
, 0, memory_order_relaxed
);
250 ProcessPendingSignals(thr
);
252 // When we are in a "blocking call", we process signals asynchronously
253 // (right when they arrive). In this context we do not expect to be
254 // executing any user/runtime code. The known interceptor sequence when
255 // this is not true is: pthread_join -> munmap(stack). It's fine
256 // to ignore munmap in this case -- we handle stack shadow separately.
257 thr
->ignore_interceptors
++;
261 thr
->ignore_interceptors
--;
262 atomic_store(&ctx
->in_blocking_func
, 0, memory_order_relaxed
);
269 TSAN_INTERCEPTOR(unsigned, sleep
, unsigned sec
) {
270 SCOPED_TSAN_INTERCEPTOR(sleep
, sec
);
271 unsigned res
= BLOCK_REAL(sleep
)(sec
);
276 TSAN_INTERCEPTOR(int, usleep
, long_t usec
) {
277 SCOPED_TSAN_INTERCEPTOR(usleep
, usec
);
278 int res
= BLOCK_REAL(usleep
)(usec
);
283 TSAN_INTERCEPTOR(int, nanosleep
, void *req
, void *rem
) {
284 SCOPED_TSAN_INTERCEPTOR(nanosleep
, req
, rem
);
285 int res
= BLOCK_REAL(nanosleep
)(req
, rem
);
290 // The sole reason tsan wraps atexit callbacks is to establish synchronization
291 // between callback setup and callback execution.
297 static void at_exit_wrapper(void *arg
) {
298 ThreadState
*thr
= cur_thread();
300 Acquire(thr
, pc
, (uptr
)arg
);
301 AtExitCtx
*ctx
= (AtExitCtx
*)arg
;
302 ((void(*)(void *arg
))ctx
->f
)(ctx
->arg
);
306 static int setup_at_exit_wrapper(ThreadState
*thr
, uptr pc
, void(*f
)(),
307 void *arg
, void *dso
);
309 TSAN_INTERCEPTOR(int, atexit
, void (*f
)()) {
310 if (cur_thread()->in_symbolizer
)
312 // We want to setup the atexit callback even if we are in ignored lib
314 SCOPED_INTERCEPTOR_RAW(atexit
, f
);
315 return setup_at_exit_wrapper(thr
, pc
, (void(*)())f
, 0, 0);
318 TSAN_INTERCEPTOR(int, __cxa_atexit
, void (*f
)(void *a
), void *arg
, void *dso
) {
319 if (cur_thread()->in_symbolizer
)
321 SCOPED_TSAN_INTERCEPTOR(__cxa_atexit
, f
, arg
, dso
);
322 return setup_at_exit_wrapper(thr
, pc
, (void(*)())f
, arg
, dso
);
325 static int setup_at_exit_wrapper(ThreadState
*thr
, uptr pc
, void(*f
)(),
326 void *arg
, void *dso
) {
327 AtExitCtx
*ctx
= (AtExitCtx
*)__libc_malloc(sizeof(AtExitCtx
));
330 Release(thr
, pc
, (uptr
)ctx
);
331 // Memory allocation in __cxa_atexit will race with free during exit,
332 // because we do not see synchronization around atexit callback list.
333 ThreadIgnoreBegin(thr
, pc
);
334 int res
= REAL(__cxa_atexit
)(at_exit_wrapper
, ctx
, dso
);
335 ThreadIgnoreEnd(thr
, pc
);
339 static void on_exit_wrapper(int status
, void *arg
) {
340 ThreadState
*thr
= cur_thread();
342 Acquire(thr
, pc
, (uptr
)arg
);
343 AtExitCtx
*ctx
= (AtExitCtx
*)arg
;
344 ((void(*)(int status
, void *arg
))ctx
->f
)(status
, ctx
->arg
);
348 TSAN_INTERCEPTOR(int, on_exit
, void(*f
)(int, void*), void *arg
) {
349 if (cur_thread()->in_symbolizer
)
351 SCOPED_TSAN_INTERCEPTOR(on_exit
, f
, arg
);
352 AtExitCtx
*ctx
= (AtExitCtx
*)__libc_malloc(sizeof(AtExitCtx
));
353 ctx
->f
= (void(*)())f
;
355 Release(thr
, pc
, (uptr
)ctx
);
356 // Memory allocation in __cxa_atexit will race with free during exit,
357 // because we do not see synchronization around atexit callback list.
358 ThreadIgnoreBegin(thr
, pc
);
359 int res
= REAL(on_exit
)(on_exit_wrapper
, ctx
);
360 ThreadIgnoreEnd(thr
, pc
);
365 static void JmpBufGarbageCollect(ThreadState
*thr
, uptr sp
) {
366 for (uptr i
= 0; i
< thr
->jmp_bufs
.Size(); i
++) {
367 JmpBuf
*buf
= &thr
->jmp_bufs
[i
];
369 uptr sz
= thr
->jmp_bufs
.Size();
370 thr
->jmp_bufs
[i
] = thr
->jmp_bufs
[sz
- 1];
371 thr
->jmp_bufs
.PopBack();
377 static void SetJmp(ThreadState
*thr
, uptr sp
, uptr mangled_sp
) {
378 if (thr
->shadow_stack_pos
== 0) // called from libc guts during bootstrap
381 JmpBufGarbageCollect(thr
, sp
);
383 JmpBuf
*buf
= thr
->jmp_bufs
.PushBack();
385 buf
->mangled_sp
= mangled_sp
;
386 buf
->shadow_stack_pos
= thr
->shadow_stack_pos
;
387 SignalContext
*sctx
= SigCtx(thr
);
388 buf
->int_signal_send
= sctx
? sctx
->int_signal_send
: 0;
389 buf
->in_blocking_func
= sctx
?
390 atomic_load(&sctx
->in_blocking_func
, memory_order_relaxed
) :
392 buf
->in_signal_handler
= atomic_load(&thr
->in_signal_handler
,
393 memory_order_relaxed
);
396 static void LongJmp(ThreadState
*thr
, uptr
*env
) {
397 #if SANITIZER_FREEBSD
398 uptr mangled_sp
= env
[2];
400 uptr mangled_sp
= env
[6];
401 #endif // SANITIZER_FREEBSD
402 // Find the saved buf by mangled_sp.
403 for (uptr i
= 0; i
< thr
->jmp_bufs
.Size(); i
++) {
404 JmpBuf
*buf
= &thr
->jmp_bufs
[i
];
405 if (buf
->mangled_sp
== mangled_sp
) {
406 CHECK_GE(thr
->shadow_stack_pos
, buf
->shadow_stack_pos
);
408 while (thr
->shadow_stack_pos
> buf
->shadow_stack_pos
)
410 SignalContext
*sctx
= SigCtx(thr
);
412 sctx
->int_signal_send
= buf
->int_signal_send
;
413 atomic_store(&sctx
->in_blocking_func
, buf
->in_blocking_func
,
414 memory_order_relaxed
);
416 atomic_store(&thr
->in_signal_handler
, buf
->in_signal_handler
,
417 memory_order_relaxed
);
418 JmpBufGarbageCollect(thr
, buf
->sp
- 1); // do not collect buf->sp
422 Printf("ThreadSanitizer: can't find longjmp buf\n");
426 // FIXME: put everything below into a common extern "C" block?
427 extern "C" void __tsan_setjmp(uptr sp
, uptr mangled_sp
) {
428 SetJmp(cur_thread(), sp
, mangled_sp
);
431 // Not called. Merely to satisfy TSAN_INTERCEPT().
432 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
433 int __interceptor_setjmp(void *env
);
434 extern "C" int __interceptor_setjmp(void *env
) {
439 // FIXME: any reason to have a separate declaration?
440 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
441 int __interceptor__setjmp(void *env
);
442 extern "C" int __interceptor__setjmp(void *env
) {
447 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
448 int __interceptor_sigsetjmp(void *env
);
449 extern "C" int __interceptor_sigsetjmp(void *env
) {
454 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
455 int __interceptor___sigsetjmp(void *env
);
456 extern "C" int __interceptor___sigsetjmp(void *env
) {
461 extern "C" int setjmp(void *env
);
462 extern "C" int _setjmp(void *env
);
463 extern "C" int sigsetjmp(void *env
);
464 extern "C" int __sigsetjmp(void *env
);
465 DEFINE_REAL(int, setjmp
, void *env
)
466 DEFINE_REAL(int, _setjmp
, void *env
)
467 DEFINE_REAL(int, sigsetjmp
, void *env
)
468 DEFINE_REAL(int, __sigsetjmp
, void *env
)
470 TSAN_INTERCEPTOR(void, longjmp
, uptr
*env
, int val
) {
472 SCOPED_TSAN_INTERCEPTOR(longjmp
, env
, val
);
474 LongJmp(cur_thread(), env
);
475 REAL(longjmp
)(env
, val
);
478 TSAN_INTERCEPTOR(void, siglongjmp
, uptr
*env
, int val
) {
480 SCOPED_TSAN_INTERCEPTOR(siglongjmp
, env
, val
);
482 LongJmp(cur_thread(), env
);
483 REAL(siglongjmp
)(env
, val
);
486 TSAN_INTERCEPTOR(void*, malloc
, uptr size
) {
487 if (cur_thread()->in_symbolizer
)
488 return __libc_malloc(size
);
491 SCOPED_INTERCEPTOR_RAW(malloc
, size
);
492 p
= user_alloc(thr
, pc
, size
);
494 invoke_malloc_hook(p
, size
);
498 TSAN_INTERCEPTOR(void*, __libc_memalign
, uptr align
, uptr sz
) {
499 SCOPED_TSAN_INTERCEPTOR(__libc_memalign
, align
, sz
);
500 return user_alloc(thr
, pc
, sz
, align
);
503 TSAN_INTERCEPTOR(void*, calloc
, uptr size
, uptr n
) {
504 if (cur_thread()->in_symbolizer
)
505 return __libc_calloc(size
, n
);
506 if (__sanitizer::CallocShouldReturnNullDueToOverflow(size
, n
))
507 return AllocatorReturnNull();
510 SCOPED_INTERCEPTOR_RAW(calloc
, size
, n
);
511 p
= user_alloc(thr
, pc
, n
* size
);
513 internal_memset(p
, 0, n
* size
);
515 invoke_malloc_hook(p
, n
* size
);
519 TSAN_INTERCEPTOR(void*, realloc
, void *p
, uptr size
) {
520 if (cur_thread()->in_symbolizer
)
521 return __libc_realloc(p
, size
);
525 SCOPED_INTERCEPTOR_RAW(realloc
, p
, size
);
526 p
= user_realloc(thr
, pc
, p
, size
);
528 invoke_malloc_hook(p
, size
);
532 TSAN_INTERCEPTOR(void, free
, void *p
) {
535 if (cur_thread()->in_symbolizer
)
536 return __libc_free(p
);
538 SCOPED_INTERCEPTOR_RAW(free
, p
);
539 user_free(thr
, pc
, p
);
542 TSAN_INTERCEPTOR(void, cfree
, void *p
) {
545 if (cur_thread()->in_symbolizer
)
546 return __libc_free(p
);
548 SCOPED_INTERCEPTOR_RAW(cfree
, p
);
549 user_free(thr
, pc
, p
);
552 TSAN_INTERCEPTOR(uptr
, malloc_usable_size
, void *p
) {
553 SCOPED_INTERCEPTOR_RAW(malloc_usable_size
, p
);
554 return user_alloc_usable_size(p
);
557 #define OPERATOR_NEW_BODY(mangled_name) \
558 if (cur_thread()->in_symbolizer) \
559 return __libc_malloc(size); \
562 SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
563 p = user_alloc(thr, pc, size); \
565 invoke_malloc_hook(p, size); \
568 SANITIZER_INTERFACE_ATTRIBUTE
569 void *operator new(__sanitizer::uptr size
);
570 void *operator new(__sanitizer::uptr size
) {
571 OPERATOR_NEW_BODY(_Znwm
);
574 SANITIZER_INTERFACE_ATTRIBUTE
575 void *operator new[](__sanitizer::uptr size
);
576 void *operator new[](__sanitizer::uptr size
) {
577 OPERATOR_NEW_BODY(_Znam
);
580 SANITIZER_INTERFACE_ATTRIBUTE
581 void *operator new(__sanitizer::uptr size
, std::nothrow_t
const&);
582 void *operator new(__sanitizer::uptr size
, std::nothrow_t
const&) {
583 OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t
);
586 SANITIZER_INTERFACE_ATTRIBUTE
587 void *operator new[](__sanitizer::uptr size
, std::nothrow_t
const&);
588 void *operator new[](__sanitizer::uptr size
, std::nothrow_t
const&) {
589 OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t
);
592 #define OPERATOR_DELETE_BODY(mangled_name) \
593 if (ptr == 0) return; \
594 if (cur_thread()->in_symbolizer) \
595 return __libc_free(ptr); \
596 invoke_free_hook(ptr); \
597 SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \
598 user_free(thr, pc, ptr);
600 SANITIZER_INTERFACE_ATTRIBUTE
601 void operator delete(void *ptr
) throw();
602 void operator delete(void *ptr
) throw() {
603 OPERATOR_DELETE_BODY(_ZdlPv
);
606 SANITIZER_INTERFACE_ATTRIBUTE
607 void operator delete[](void *ptr
) throw();
608 void operator delete[](void *ptr
) throw() {
609 OPERATOR_DELETE_BODY(_ZdaPv
);
612 SANITIZER_INTERFACE_ATTRIBUTE
613 void operator delete(void *ptr
, std::nothrow_t
const&);
614 void operator delete(void *ptr
, std::nothrow_t
const&) {
615 OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t
);
618 SANITIZER_INTERFACE_ATTRIBUTE
619 void operator delete[](void *ptr
, std::nothrow_t
const&);
620 void operator delete[](void *ptr
, std::nothrow_t
const&) {
621 OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t
);
624 TSAN_INTERCEPTOR(uptr
, strlen
, const char *s
) {
625 SCOPED_TSAN_INTERCEPTOR(strlen
, s
);
626 uptr len
= internal_strlen(s
);
627 MemoryAccessRange(thr
, pc
, (uptr
)s
, len
+ 1, false);
631 TSAN_INTERCEPTOR(void*, memset
, void *dst
, int v
, uptr size
) {
632 SCOPED_TSAN_INTERCEPTOR(memset
, dst
, v
, size
);
633 MemoryAccessRange(thr
, pc
, (uptr
)dst
, size
, true);
634 return internal_memset(dst
, v
, size
);
637 TSAN_INTERCEPTOR(void*, memcpy
, void *dst
, const void *src
, uptr size
) {
638 SCOPED_TSAN_INTERCEPTOR(memcpy
, dst
, src
, size
);
639 MemoryAccessRange(thr
, pc
, (uptr
)dst
, size
, true);
640 MemoryAccessRange(thr
, pc
, (uptr
)src
, size
, false);
641 return internal_memcpy(dst
, src
, size
);
644 TSAN_INTERCEPTOR(int, memcmp
, const void *s1
, const void *s2
, uptr n
) {
645 SCOPED_TSAN_INTERCEPTOR(memcmp
, s1
, s2
, n
);
648 for (; len
< n
; len
++) {
649 if ((res
= ((unsigned char*)s1
)[len
] - ((unsigned char*)s2
)[len
]))
652 MemoryAccessRange(thr
, pc
, (uptr
)s1
, len
< n
? len
+ 1 : n
, false);
653 MemoryAccessRange(thr
, pc
, (uptr
)s2
, len
< n
? len
+ 1 : n
, false);
657 TSAN_INTERCEPTOR(void*, memmove
, void *dst
, void *src
, uptr n
) {
658 SCOPED_TSAN_INTERCEPTOR(memmove
, dst
, src
, n
);
659 MemoryAccessRange(thr
, pc
, (uptr
)dst
, n
, true);
660 MemoryAccessRange(thr
, pc
, (uptr
)src
, n
, false);
661 return REAL(memmove
)(dst
, src
, n
);
664 TSAN_INTERCEPTOR(char*, strchr
, char *s
, int c
) {
665 SCOPED_TSAN_INTERCEPTOR(strchr
, s
, c
);
666 char *res
= REAL(strchr
)(s
, c
);
667 uptr len
= res
? (char*)res
- (char*)s
+ 1 : internal_strlen(s
) + 1;
668 MemoryAccessRange(thr
, pc
, (uptr
)s
, len
, false);
672 TSAN_INTERCEPTOR(char*, strchrnul
, char *s
, int c
) {
673 SCOPED_TSAN_INTERCEPTOR(strchrnul
, s
, c
);
674 char *res
= REAL(strchrnul
)(s
, c
);
675 uptr len
= (char*)res
- (char*)s
+ 1;
676 MemoryAccessRange(thr
, pc
, (uptr
)s
, len
, false);
680 TSAN_INTERCEPTOR(char*, strrchr
, char *s
, int c
) {
681 SCOPED_TSAN_INTERCEPTOR(strrchr
, s
, c
);
682 MemoryAccessRange(thr
, pc
, (uptr
)s
, internal_strlen(s
) + 1, false);
683 return REAL(strrchr
)(s
, c
);
686 TSAN_INTERCEPTOR(char*, strcpy
, char *dst
, const char *src
) { // NOLINT
687 SCOPED_TSAN_INTERCEPTOR(strcpy
, dst
, src
); // NOLINT
688 uptr srclen
= internal_strlen(src
);
689 MemoryAccessRange(thr
, pc
, (uptr
)dst
, srclen
+ 1, true);
690 MemoryAccessRange(thr
, pc
, (uptr
)src
, srclen
+ 1, false);
691 return REAL(strcpy
)(dst
, src
); // NOLINT
694 TSAN_INTERCEPTOR(char*, strncpy
, char *dst
, char *src
, uptr n
) {
695 SCOPED_TSAN_INTERCEPTOR(strncpy
, dst
, src
, n
);
696 uptr srclen
= internal_strnlen(src
, n
);
697 MemoryAccessRange(thr
, pc
, (uptr
)dst
, n
, true);
698 MemoryAccessRange(thr
, pc
, (uptr
)src
, min(srclen
+ 1, n
), false);
699 return REAL(strncpy
)(dst
, src
, n
);
702 TSAN_INTERCEPTOR(const char*, strstr
, const char *s1
, const char *s2
) {
703 SCOPED_TSAN_INTERCEPTOR(strstr
, s1
, s2
);
704 const char *res
= REAL(strstr
)(s1
, s2
);
705 uptr len1
= internal_strlen(s1
);
706 uptr len2
= internal_strlen(s2
);
707 MemoryAccessRange(thr
, pc
, (uptr
)s1
, len1
+ 1, false);
708 MemoryAccessRange(thr
, pc
, (uptr
)s2
, len2
+ 1, false);
712 TSAN_INTERCEPTOR(char*, strdup
, const char *str
) {
713 SCOPED_TSAN_INTERCEPTOR(strdup
, str
);
714 // strdup will call malloc, so no instrumentation is required here.
715 return REAL(strdup
)(str
);
718 static bool fix_mmap_addr(void **addr
, long_t sz
, int flags
) {
720 if (!IsAppMem((uptr
)*addr
) || !IsAppMem((uptr
)*addr
+ sz
- 1)) {
721 if (flags
& MAP_FIXED
) {
732 TSAN_INTERCEPTOR(void*, mmap
, void *addr
, long_t sz
, int prot
,
733 int flags
, int fd
, unsigned off
) {
734 SCOPED_TSAN_INTERCEPTOR(mmap
, addr
, sz
, prot
, flags
, fd
, off
);
735 if (!fix_mmap_addr(&addr
, sz
, flags
))
737 void *res
= REAL(mmap
)(addr
, sz
, prot
, flags
, fd
, off
);
738 if (res
!= MAP_FAILED
) {
740 FdAccess(thr
, pc
, fd
);
741 MemoryRangeImitateWrite(thr
, pc
, (uptr
)res
, sz
);
746 #if !SANITIZER_FREEBSD
747 TSAN_INTERCEPTOR(void*, mmap64
, void *addr
, long_t sz
, int prot
,
748 int flags
, int fd
, u64 off
) {
749 SCOPED_TSAN_INTERCEPTOR(mmap64
, addr
, sz
, prot
, flags
, fd
, off
);
750 if (!fix_mmap_addr(&addr
, sz
, flags
))
752 void *res
= REAL(mmap64
)(addr
, sz
, prot
, flags
, fd
, off
);
753 if (res
!= MAP_FAILED
) {
755 FdAccess(thr
, pc
, fd
);
756 MemoryRangeImitateWrite(thr
, pc
, (uptr
)res
, sz
);
760 #define TSAN_MAYBE_INTERCEPT_MMAP64 TSAN_INTERCEPT(mmap64)
762 #define TSAN_MAYBE_INTERCEPT_MMAP64
765 TSAN_INTERCEPTOR(int, munmap
, void *addr
, long_t sz
) {
766 SCOPED_TSAN_INTERCEPTOR(munmap
, addr
, sz
);
767 DontNeedShadowFor((uptr
)addr
, sz
);
768 int res
= REAL(munmap
)(addr
, sz
);
772 #if !SANITIZER_FREEBSD
773 TSAN_INTERCEPTOR(void*, memalign
, uptr align
, uptr sz
) {
774 SCOPED_INTERCEPTOR_RAW(memalign
, align
, sz
);
775 return user_alloc(thr
, pc
, sz
, align
);
777 #define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign)
779 #define TSAN_MAYBE_INTERCEPT_MEMALIGN
782 TSAN_INTERCEPTOR(void*, aligned_alloc
, uptr align
, uptr sz
) {
783 SCOPED_INTERCEPTOR_RAW(memalign
, align
, sz
);
784 return user_alloc(thr
, pc
, sz
, align
);
787 TSAN_INTERCEPTOR(void*, valloc
, uptr sz
) {
788 SCOPED_INTERCEPTOR_RAW(valloc
, sz
);
789 return user_alloc(thr
, pc
, sz
, GetPageSizeCached());
792 #if !SANITIZER_FREEBSD
793 TSAN_INTERCEPTOR(void*, pvalloc
, uptr sz
) {
794 SCOPED_INTERCEPTOR_RAW(pvalloc
, sz
);
795 sz
= RoundUp(sz
, GetPageSizeCached());
796 return user_alloc(thr
, pc
, sz
, GetPageSizeCached());
798 #define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc)
800 #define TSAN_MAYBE_INTERCEPT_PVALLOC
803 TSAN_INTERCEPTOR(int, posix_memalign
, void **memptr
, uptr align
, uptr sz
) {
804 SCOPED_INTERCEPTOR_RAW(posix_memalign
, memptr
, align
, sz
);
805 *memptr
= user_alloc(thr
, pc
, sz
, align
);
809 // Used in thread-safe function static initialization.
810 extern "C" int INTERFACE_ATTRIBUTE
__cxa_guard_acquire(atomic_uint32_t
*g
) {
811 SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire
, g
);
813 u32 cmp
= atomic_load(g
, memory_order_acquire
);
815 if (atomic_compare_exchange_strong(g
, &cmp
, 1<<16, memory_order_relaxed
))
817 } else if (cmp
== 1) {
818 Acquire(thr
, pc
, (uptr
)g
);
821 internal_sched_yield();
826 extern "C" void INTERFACE_ATTRIBUTE
__cxa_guard_release(atomic_uint32_t
*g
) {
827 SCOPED_INTERCEPTOR_RAW(__cxa_guard_release
, g
);
828 Release(thr
, pc
, (uptr
)g
);
829 atomic_store(g
, 1, memory_order_release
);
832 extern "C" void INTERFACE_ATTRIBUTE
__cxa_guard_abort(atomic_uint32_t
*g
) {
833 SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort
, g
);
834 atomic_store(g
, 0, memory_order_relaxed
);
837 static void thread_finalize(void *v
) {
840 if (pthread_setspecific(g_thread_finalize_key
, (void*)(iter
- 1))) {
841 Printf("ThreadSanitizer: failed to set thread key\n");
847 ThreadState
*thr
= cur_thread();
849 SignalContext
*sctx
= thr
->signal_ctx
;
852 UnmapOrDie(sctx
, sizeof(*sctx
));
859 void* (*callback
)(void *arg
);
861 atomic_uintptr_t tid
;
864 extern "C" void *__tsan_thread_start_func(void *arg
) {
865 ThreadParam
*p
= (ThreadParam
*)arg
;
866 void* (*callback
)(void *arg
) = p
->callback
;
867 void *param
= p
->param
;
870 ThreadState
*thr
= cur_thread();
871 // Thread-local state is not initialized yet.
872 ScopedIgnoreInterceptors ignore
;
873 ThreadIgnoreBegin(thr
, 0);
874 if (pthread_setspecific(g_thread_finalize_key
,
875 (void *)kPthreadDestructorIterations
)) {
876 Printf("ThreadSanitizer: failed to set thread key\n");
879 ThreadIgnoreEnd(thr
, 0);
880 while ((tid
= atomic_load(&p
->tid
, memory_order_acquire
)) == 0)
882 atomic_store(&p
->tid
, 0, memory_order_release
);
883 ThreadStart(thr
, tid
, GetTid());
885 void *res
= callback(param
);
886 // Prevent the callback from being tail called,
887 // it mixes up stack traces.
888 volatile int foo
= 42;
893 TSAN_INTERCEPTOR(int, pthread_create
,
894 void *th
, void *attr
, void *(*callback
)(void*), void * param
) {
895 SCOPED_INTERCEPTOR_RAW(pthread_create
, th
, attr
, callback
, param
);
896 if (ctx
->after_multithreaded_fork
) {
897 if (flags()->die_after_fork
) {
898 Report("ThreadSanitizer: starting new threads after multi-threaded "
899 "fork is not supported. Dying (set die_after_fork=0 to override)\n");
902 VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded "
903 "fork is not supported (pid %d). Continuing because of "
904 "die_after_fork=0, but you are on your own\n", internal_getpid());
907 __sanitizer_pthread_attr_t myattr
;
909 pthread_attr_init(&myattr
);
913 REAL(pthread_attr_getdetachstate
)(attr
, &detached
);
914 AdjustStackSize(attr
);
917 p
.callback
= callback
;
919 atomic_store(&p
.tid
, 0, memory_order_relaxed
);
922 // Otherwise we see false positives in pthread stack manipulation.
923 ScopedIgnoreInterceptors ignore
;
924 ThreadIgnoreBegin(thr
, pc
);
925 res
= REAL(pthread_create
)(th
, attr
, __tsan_thread_start_func
, &p
);
926 ThreadIgnoreEnd(thr
, pc
);
929 int tid
= ThreadCreate(thr
, pc
, *(uptr
*)th
, detached
);
931 atomic_store(&p
.tid
, tid
, memory_order_release
);
932 while (atomic_load(&p
.tid
, memory_order_acquire
) != 0)
936 pthread_attr_destroy(&myattr
);
940 TSAN_INTERCEPTOR(int, pthread_join
, void *th
, void **ret
) {
941 SCOPED_INTERCEPTOR_RAW(pthread_join
, th
, ret
);
942 int tid
= ThreadTid(thr
, pc
, (uptr
)th
);
943 ThreadIgnoreBegin(thr
, pc
);
944 int res
= BLOCK_REAL(pthread_join
)(th
, ret
);
945 ThreadIgnoreEnd(thr
, pc
);
947 ThreadJoin(thr
, pc
, tid
);
952 TSAN_INTERCEPTOR(int, pthread_detach
, void *th
) {
953 SCOPED_TSAN_INTERCEPTOR(pthread_detach
, th
);
954 int tid
= ThreadTid(thr
, pc
, (uptr
)th
);
955 int res
= REAL(pthread_detach
)(th
);
957 ThreadDetach(thr
, pc
, tid
);
963 // NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2).
964 // pthread_cond_t has different size in the different versions.
965 // If call new REAL functions for old pthread_cond_t, they will corrupt memory
966 // after pthread_cond_t (old cond is smaller).
967 // If we call old REAL functions for new pthread_cond_t, we will lose some
968 // functionality (e.g. old functions do not support waiting against
970 // Proper handling would require to have 2 versions of interceptors as well.
971 // But this is messy, in particular requires linker scripts when sanitizer
972 // runtime is linked into a shared library.
973 // Instead we assume we don't have dynamic libraries built against old
974 // pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag
975 // that allows to work with old libraries (but this mode does not support
976 // some features, e.g. pthread_condattr_getpshared).
977 static void *init_cond(void *c
, bool force
= false) {
978 // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions.
979 // So we allocate additional memory on the side large enough to hold
980 // any pthread_cond_t object. Always call new REAL functions, but pass
981 // the aux object to them.
982 // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes
983 // first word of pthread_cond_t to zero.
984 // It's all relevant only for linux.
985 if (!common_flags()->legacy_pthread_cond
)
987 atomic_uintptr_t
*p
= (atomic_uintptr_t
*)c
;
988 uptr cond
= atomic_load(p
, memory_order_acquire
);
989 if (!force
&& cond
!= 0)
991 void *newcond
= WRAP(malloc
)(pthread_cond_t_sz
);
992 internal_memset(newcond
, 0, pthread_cond_t_sz
);
993 if (atomic_compare_exchange_strong(p
, &cond
, (uptr
)newcond
,
994 memory_order_acq_rel
))
1000 struct CondMutexUnlockCtx
{
1006 static void cond_mutex_unlock(CondMutexUnlockCtx
*arg
) {
1007 MutexLock(arg
->thr
, arg
->pc
, (uptr
)arg
->m
);
1010 INTERCEPTOR(int, pthread_cond_init
, void *c
, void *a
) {
1011 void *cond
= init_cond(c
, true);
1012 SCOPED_TSAN_INTERCEPTOR(pthread_cond_init
, cond
, a
);
1013 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), true);
1014 return REAL(pthread_cond_init
)(cond
, a
);
1017 INTERCEPTOR(int, pthread_cond_wait
, void *c
, void *m
) {
1018 void *cond
= init_cond(c
);
1019 SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait
, cond
, m
);
1020 MutexUnlock(thr
, pc
, (uptr
)m
);
1021 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), false);
1022 CondMutexUnlockCtx arg
= {thr
, pc
, m
};
1023 // This ensures that we handle mutex lock even in case of pthread_cancel.
1024 // See test/tsan/cond_cancel.cc.
1025 int res
= call_pthread_cancel_with_cleanup(
1026 (int(*)(void *c
, void *m
, void *abstime
))REAL(pthread_cond_wait
),
1027 cond
, m
, 0, (void(*)(void *arg
))cond_mutex_unlock
, &arg
);
1028 if (res
== errno_EOWNERDEAD
)
1029 MutexRepair(thr
, pc
, (uptr
)m
);
1030 MutexLock(thr
, pc
, (uptr
)m
);
1034 INTERCEPTOR(int, pthread_cond_timedwait
, void *c
, void *m
, void *abstime
) {
1035 void *cond
= init_cond(c
);
1036 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait
, cond
, m
, abstime
);
1037 MutexUnlock(thr
, pc
, (uptr
)m
);
1038 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), false);
1039 CondMutexUnlockCtx arg
= {thr
, pc
, m
};
1040 // This ensures that we handle mutex lock even in case of pthread_cancel.
1041 // See test/tsan/cond_cancel.cc.
1042 int res
= call_pthread_cancel_with_cleanup(
1043 REAL(pthread_cond_timedwait
), cond
, m
, abstime
,
1044 (void(*)(void *arg
))cond_mutex_unlock
, &arg
);
1045 if (res
== errno_EOWNERDEAD
)
1046 MutexRepair(thr
, pc
, (uptr
)m
);
1047 MutexLock(thr
, pc
, (uptr
)m
);
1051 INTERCEPTOR(int, pthread_cond_signal
, void *c
) {
1052 void *cond
= init_cond(c
);
1053 SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal
, cond
);
1054 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), false);
1055 return REAL(pthread_cond_signal
)(cond
);
1058 INTERCEPTOR(int, pthread_cond_broadcast
, void *c
) {
1059 void *cond
= init_cond(c
);
1060 SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast
, cond
);
1061 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), false);
1062 return REAL(pthread_cond_broadcast
)(cond
);
1065 INTERCEPTOR(int, pthread_cond_destroy
, void *c
) {
1066 void *cond
= init_cond(c
);
1067 SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy
, cond
);
1068 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), true);
1069 int res
= REAL(pthread_cond_destroy
)(cond
);
1070 if (common_flags()->legacy_pthread_cond
) {
1071 // Free our aux cond and zero the pointer to not leave dangling pointers.
1073 atomic_store((atomic_uintptr_t
*)c
, 0, memory_order_relaxed
);
1078 TSAN_INTERCEPTOR(int, pthread_mutex_init
, void *m
, void *a
) {
1079 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init
, m
, a
);
1080 int res
= REAL(pthread_mutex_init
)(m
, a
);
1082 bool recursive
= false;
1085 if (REAL(pthread_mutexattr_gettype
)(a
, &type
) == 0)
1086 recursive
= (type
== PTHREAD_MUTEX_RECURSIVE
1087 || type
== PTHREAD_MUTEX_RECURSIVE_NP
);
1089 MutexCreate(thr
, pc
, (uptr
)m
, false, recursive
, false);
1094 TSAN_INTERCEPTOR(int, pthread_mutex_destroy
, void *m
) {
1095 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy
, m
);
1096 int res
= REAL(pthread_mutex_destroy
)(m
);
1097 if (res
== 0 || res
== EBUSY
) {
1098 MutexDestroy(thr
, pc
, (uptr
)m
);
1103 TSAN_INTERCEPTOR(int, pthread_mutex_trylock
, void *m
) {
1104 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock
, m
);
1105 int res
= REAL(pthread_mutex_trylock
)(m
);
1106 if (res
== EOWNERDEAD
)
1107 MutexRepair(thr
, pc
, (uptr
)m
);
1108 if (res
== 0 || res
== EOWNERDEAD
)
1109 MutexLock(thr
, pc
, (uptr
)m
, /*rec=*/1, /*try_lock=*/true);
1113 TSAN_INTERCEPTOR(int, pthread_mutex_timedlock
, void *m
, void *abstime
) {
1114 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock
, m
, abstime
);
1115 int res
= REAL(pthread_mutex_timedlock
)(m
, abstime
);
1117 MutexLock(thr
, pc
, (uptr
)m
);
1122 TSAN_INTERCEPTOR(int, pthread_spin_init
, void *m
, int pshared
) {
1123 SCOPED_TSAN_INTERCEPTOR(pthread_spin_init
, m
, pshared
);
1124 int res
= REAL(pthread_spin_init
)(m
, pshared
);
1126 MutexCreate(thr
, pc
, (uptr
)m
, false, false, false);
1131 TSAN_INTERCEPTOR(int, pthread_spin_destroy
, void *m
) {
1132 SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy
, m
);
1133 int res
= REAL(pthread_spin_destroy
)(m
);
1135 MutexDestroy(thr
, pc
, (uptr
)m
);
1140 TSAN_INTERCEPTOR(int, pthread_spin_lock
, void *m
) {
1141 SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock
, m
);
1142 int res
= REAL(pthread_spin_lock
)(m
);
1144 MutexLock(thr
, pc
, (uptr
)m
);
1149 TSAN_INTERCEPTOR(int, pthread_spin_trylock
, void *m
) {
1150 SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock
, m
);
1151 int res
= REAL(pthread_spin_trylock
)(m
);
1153 MutexLock(thr
, pc
, (uptr
)m
, /*rec=*/1, /*try_lock=*/true);
1158 TSAN_INTERCEPTOR(int, pthread_spin_unlock
, void *m
) {
1159 SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock
, m
);
1160 MutexUnlock(thr
, pc
, (uptr
)m
);
1161 int res
= REAL(pthread_spin_unlock
)(m
);
1165 TSAN_INTERCEPTOR(int, pthread_rwlock_init
, void *m
, void *a
) {
1166 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init
, m
, a
);
1167 int res
= REAL(pthread_rwlock_init
)(m
, a
);
1169 MutexCreate(thr
, pc
, (uptr
)m
, true, false, false);
1174 TSAN_INTERCEPTOR(int, pthread_rwlock_destroy
, void *m
) {
1175 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy
, m
);
1176 int res
= REAL(pthread_rwlock_destroy
)(m
);
1178 MutexDestroy(thr
, pc
, (uptr
)m
);
1183 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock
, void *m
) {
1184 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock
, m
);
1185 int res
= REAL(pthread_rwlock_rdlock
)(m
);
1187 MutexReadLock(thr
, pc
, (uptr
)m
);
1192 TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock
, void *m
) {
1193 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock
, m
);
1194 int res
= REAL(pthread_rwlock_tryrdlock
)(m
);
1196 MutexReadLock(thr
, pc
, (uptr
)m
, /*try_lock=*/true);
1201 TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock
, void *m
, void *abstime
) {
1202 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock
, m
, abstime
);
1203 int res
= REAL(pthread_rwlock_timedrdlock
)(m
, abstime
);
1205 MutexReadLock(thr
, pc
, (uptr
)m
);
1210 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock
, void *m
) {
1211 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock
, m
);
1212 int res
= REAL(pthread_rwlock_wrlock
)(m
);
1214 MutexLock(thr
, pc
, (uptr
)m
);
1219 TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock
, void *m
) {
1220 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock
, m
);
1221 int res
= REAL(pthread_rwlock_trywrlock
)(m
);
1223 MutexLock(thr
, pc
, (uptr
)m
, /*rec=*/1, /*try_lock=*/true);
1228 TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock
, void *m
, void *abstime
) {
1229 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock
, m
, abstime
);
1230 int res
= REAL(pthread_rwlock_timedwrlock
)(m
, abstime
);
1232 MutexLock(thr
, pc
, (uptr
)m
);
1237 TSAN_INTERCEPTOR(int, pthread_rwlock_unlock
, void *m
) {
1238 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock
, m
);
1239 MutexReadOrWriteUnlock(thr
, pc
, (uptr
)m
);
1240 int res
= REAL(pthread_rwlock_unlock
)(m
);
1244 TSAN_INTERCEPTOR(int, pthread_barrier_init
, void *b
, void *a
, unsigned count
) {
1245 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init
, b
, a
, count
);
1246 MemoryWrite(thr
, pc
, (uptr
)b
, kSizeLog1
);
1247 int res
= REAL(pthread_barrier_init
)(b
, a
, count
);
1251 TSAN_INTERCEPTOR(int, pthread_barrier_destroy
, void *b
) {
1252 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy
, b
);
1253 MemoryWrite(thr
, pc
, (uptr
)b
, kSizeLog1
);
1254 int res
= REAL(pthread_barrier_destroy
)(b
);
1258 TSAN_INTERCEPTOR(int, pthread_barrier_wait
, void *b
) {
1259 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait
, b
);
1260 Release(thr
, pc
, (uptr
)b
);
1261 MemoryRead(thr
, pc
, (uptr
)b
, kSizeLog1
);
1262 int res
= REAL(pthread_barrier_wait
)(b
);
1263 MemoryRead(thr
, pc
, (uptr
)b
, kSizeLog1
);
1264 if (res
== 0 || res
== PTHREAD_BARRIER_SERIAL_THREAD
) {
1265 Acquire(thr
, pc
, (uptr
)b
);
1270 TSAN_INTERCEPTOR(int, pthread_once
, void *o
, void (*f
)()) {
1271 SCOPED_INTERCEPTOR_RAW(pthread_once
, o
, f
);
1272 if (o
== 0 || f
== 0)
1274 atomic_uint32_t
*a
= static_cast<atomic_uint32_t
*>(o
);
1275 u32 v
= atomic_load(a
, memory_order_acquire
);
1276 if (v
== 0 && atomic_compare_exchange_strong(a
, &v
, 1,
1277 memory_order_relaxed
)) {
1279 if (!thr
->in_ignored_lib
)
1280 Release(thr
, pc
, (uptr
)o
);
1281 atomic_store(a
, 2, memory_order_release
);
1285 v
= atomic_load(a
, memory_order_acquire
);
1287 if (!thr
->in_ignored_lib
)
1288 Acquire(thr
, pc
, (uptr
)o
);
1293 TSAN_INTERCEPTOR(int, sem_init
, void *s
, int pshared
, unsigned value
) {
1294 SCOPED_TSAN_INTERCEPTOR(sem_init
, s
, pshared
, value
);
1295 int res
= REAL(sem_init
)(s
, pshared
, value
);
1299 TSAN_INTERCEPTOR(int, sem_destroy
, void *s
) {
1300 SCOPED_TSAN_INTERCEPTOR(sem_destroy
, s
);
1301 int res
= REAL(sem_destroy
)(s
);
1305 TSAN_INTERCEPTOR(int, sem_wait
, void *s
) {
1306 SCOPED_TSAN_INTERCEPTOR(sem_wait
, s
);
1307 int res
= BLOCK_REAL(sem_wait
)(s
);
1309 Acquire(thr
, pc
, (uptr
)s
);
1314 TSAN_INTERCEPTOR(int, sem_trywait
, void *s
) {
1315 SCOPED_TSAN_INTERCEPTOR(sem_trywait
, s
);
1316 int res
= BLOCK_REAL(sem_trywait
)(s
);
1318 Acquire(thr
, pc
, (uptr
)s
);
1323 TSAN_INTERCEPTOR(int, sem_timedwait
, void *s
, void *abstime
) {
1324 SCOPED_TSAN_INTERCEPTOR(sem_timedwait
, s
, abstime
);
1325 int res
= BLOCK_REAL(sem_timedwait
)(s
, abstime
);
1327 Acquire(thr
, pc
, (uptr
)s
);
1332 TSAN_INTERCEPTOR(int, sem_post
, void *s
) {
1333 SCOPED_TSAN_INTERCEPTOR(sem_post
, s
);
1334 Release(thr
, pc
, (uptr
)s
);
1335 int res
= REAL(sem_post
)(s
);
1339 TSAN_INTERCEPTOR(int, sem_getvalue
, void *s
, int *sval
) {
1340 SCOPED_TSAN_INTERCEPTOR(sem_getvalue
, s
, sval
);
1341 int res
= REAL(sem_getvalue
)(s
, sval
);
1343 Acquire(thr
, pc
, (uptr
)s
);
1348 #if !SANITIZER_FREEBSD
1349 TSAN_INTERCEPTOR(int, __xstat
, int version
, const char *path
, void *buf
) {
1350 SCOPED_TSAN_INTERCEPTOR(__xstat
, version
, path
, buf
);
1351 return REAL(__xstat
)(version
, path
, buf
);
1353 #define TSAN_MAYBE_INTERCEPT___XSTAT TSAN_INTERCEPT(__xstat)
1355 #define TSAN_MAYBE_INTERCEPT___XSTAT
1358 TSAN_INTERCEPTOR(int, stat
, const char *path
, void *buf
) {
1359 #if SANITIZER_FREEBSD
1360 SCOPED_TSAN_INTERCEPTOR(stat
, path
, buf
);
1361 return REAL(stat
)(path
, buf
);
1363 SCOPED_TSAN_INTERCEPTOR(__xstat
, 0, path
, buf
);
1364 return REAL(__xstat
)(0, path
, buf
);
1368 #if !SANITIZER_FREEBSD
1369 TSAN_INTERCEPTOR(int, __xstat64
, int version
, const char *path
, void *buf
) {
1370 SCOPED_TSAN_INTERCEPTOR(__xstat64
, version
, path
, buf
);
1371 return REAL(__xstat64
)(version
, path
, buf
);
1373 #define TSAN_MAYBE_INTERCEPT___XSTAT64 TSAN_INTERCEPT(__xstat64)
1375 #define TSAN_MAYBE_INTERCEPT___XSTAT64
1378 #if !SANITIZER_FREEBSD
1379 TSAN_INTERCEPTOR(int, stat64
, const char *path
, void *buf
) {
1380 SCOPED_TSAN_INTERCEPTOR(__xstat64
, 0, path
, buf
);
1381 return REAL(__xstat64
)(0, path
, buf
);
1383 #define TSAN_MAYBE_INTERCEPT_STAT64 TSAN_INTERCEPT(stat64)
1385 #define TSAN_MAYBE_INTERCEPT_STAT64
1388 #if !SANITIZER_FREEBSD
1389 TSAN_INTERCEPTOR(int, __lxstat
, int version
, const char *path
, void *buf
) {
1390 SCOPED_TSAN_INTERCEPTOR(__lxstat
, version
, path
, buf
);
1391 return REAL(__lxstat
)(version
, path
, buf
);
1393 #define TSAN_MAYBE_INTERCEPT___LXSTAT TSAN_INTERCEPT(__lxstat)
1395 #define TSAN_MAYBE_INTERCEPT___LXSTAT
1398 TSAN_INTERCEPTOR(int, lstat
, const char *path
, void *buf
) {
1399 #if SANITIZER_FREEBSD
1400 SCOPED_TSAN_INTERCEPTOR(lstat
, path
, buf
);
1401 return REAL(lstat
)(path
, buf
);
1403 SCOPED_TSAN_INTERCEPTOR(__lxstat
, 0, path
, buf
);
1404 return REAL(__lxstat
)(0, path
, buf
);
1408 #if !SANITIZER_FREEBSD
1409 TSAN_INTERCEPTOR(int, __lxstat64
, int version
, const char *path
, void *buf
) {
1410 SCOPED_TSAN_INTERCEPTOR(__lxstat64
, version
, path
, buf
);
1411 return REAL(__lxstat64
)(version
, path
, buf
);
1413 #define TSAN_MAYBE_INTERCEPT___LXSTAT64 TSAN_INTERCEPT(__lxstat64)
1415 #define TSAN_MAYBE_INTERCEPT___LXSTAT64
1418 #if !SANITIZER_FREEBSD
1419 TSAN_INTERCEPTOR(int, lstat64
, const char *path
, void *buf
) {
1420 SCOPED_TSAN_INTERCEPTOR(__lxstat64
, 0, path
, buf
);
1421 return REAL(__lxstat64
)(0, path
, buf
);
1423 #define TSAN_MAYBE_INTERCEPT_LSTAT64 TSAN_INTERCEPT(lstat64)
1425 #define TSAN_MAYBE_INTERCEPT_LSTAT64
1428 #if !SANITIZER_FREEBSD
1429 TSAN_INTERCEPTOR(int, __fxstat
, int version
, int fd
, void *buf
) {
1430 SCOPED_TSAN_INTERCEPTOR(__fxstat
, version
, fd
, buf
);
1432 FdAccess(thr
, pc
, fd
);
1433 return REAL(__fxstat
)(version
, fd
, buf
);
1435 #define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat)
1437 #define TSAN_MAYBE_INTERCEPT___FXSTAT
1440 TSAN_INTERCEPTOR(int, fstat
, int fd
, void *buf
) {
1441 #if SANITIZER_FREEBSD
1442 SCOPED_TSAN_INTERCEPTOR(fstat
, fd
, buf
);
1444 FdAccess(thr
, pc
, fd
);
1445 return REAL(fstat
)(fd
, buf
);
1447 SCOPED_TSAN_INTERCEPTOR(__fxstat
, 0, fd
, buf
);
1449 FdAccess(thr
, pc
, fd
);
1450 return REAL(__fxstat
)(0, fd
, buf
);
1454 #if !SANITIZER_FREEBSD
1455 TSAN_INTERCEPTOR(int, __fxstat64
, int version
, int fd
, void *buf
) {
1456 SCOPED_TSAN_INTERCEPTOR(__fxstat64
, version
, fd
, buf
);
1458 FdAccess(thr
, pc
, fd
);
1459 return REAL(__fxstat64
)(version
, fd
, buf
);
1461 #define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64)
1463 #define TSAN_MAYBE_INTERCEPT___FXSTAT64
1466 #if !SANITIZER_FREEBSD
1467 TSAN_INTERCEPTOR(int, fstat64
, int fd
, void *buf
) {
1468 SCOPED_TSAN_INTERCEPTOR(__fxstat64
, 0, fd
, buf
);
1470 FdAccess(thr
, pc
, fd
);
1471 return REAL(__fxstat64
)(0, fd
, buf
);
1473 #define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64)
1475 #define TSAN_MAYBE_INTERCEPT_FSTAT64
1478 TSAN_INTERCEPTOR(int, open
, const char *name
, int flags
, int mode
) {
1479 SCOPED_TSAN_INTERCEPTOR(open
, name
, flags
, mode
);
1480 int fd
= REAL(open
)(name
, flags
, mode
);
1482 FdFileCreate(thr
, pc
, fd
);
1486 #if !SANITIZER_FREEBSD
1487 TSAN_INTERCEPTOR(int, open64
, const char *name
, int flags
, int mode
) {
1488 SCOPED_TSAN_INTERCEPTOR(open64
, name
, flags
, mode
);
1489 int fd
= REAL(open64
)(name
, flags
, mode
);
1491 FdFileCreate(thr
, pc
, fd
);
1494 #define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64)
1496 #define TSAN_MAYBE_INTERCEPT_OPEN64
1499 TSAN_INTERCEPTOR(int, creat
, const char *name
, int mode
) {
1500 SCOPED_TSAN_INTERCEPTOR(creat
, name
, mode
);
1501 int fd
= REAL(creat
)(name
, mode
);
1503 FdFileCreate(thr
, pc
, fd
);
1507 #if !SANITIZER_FREEBSD
1508 TSAN_INTERCEPTOR(int, creat64
, const char *name
, int mode
) {
1509 SCOPED_TSAN_INTERCEPTOR(creat64
, name
, mode
);
1510 int fd
= REAL(creat64
)(name
, mode
);
1512 FdFileCreate(thr
, pc
, fd
);
1515 #define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64)
1517 #define TSAN_MAYBE_INTERCEPT_CREAT64
1520 TSAN_INTERCEPTOR(int, dup
, int oldfd
) {
1521 SCOPED_TSAN_INTERCEPTOR(dup
, oldfd
);
1522 int newfd
= REAL(dup
)(oldfd
);
1523 if (oldfd
>= 0 && newfd
>= 0 && newfd
!= oldfd
)
1524 FdDup(thr
, pc
, oldfd
, newfd
);
1528 TSAN_INTERCEPTOR(int, dup2
, int oldfd
, int newfd
) {
1529 SCOPED_TSAN_INTERCEPTOR(dup2
, oldfd
, newfd
);
1530 int newfd2
= REAL(dup2
)(oldfd
, newfd
);
1531 if (oldfd
>= 0 && newfd2
>= 0 && newfd2
!= oldfd
)
1532 FdDup(thr
, pc
, oldfd
, newfd2
);
1536 TSAN_INTERCEPTOR(int, dup3
, int oldfd
, int newfd
, int flags
) {
1537 SCOPED_TSAN_INTERCEPTOR(dup3
, oldfd
, newfd
, flags
);
1538 int newfd2
= REAL(dup3
)(oldfd
, newfd
, flags
);
1539 if (oldfd
>= 0 && newfd2
>= 0 && newfd2
!= oldfd
)
1540 FdDup(thr
, pc
, oldfd
, newfd2
);
1544 #if !SANITIZER_FREEBSD
1545 TSAN_INTERCEPTOR(int, eventfd
, unsigned initval
, int flags
) {
1546 SCOPED_TSAN_INTERCEPTOR(eventfd
, initval
, flags
);
1547 int fd
= REAL(eventfd
)(initval
, flags
);
1549 FdEventCreate(thr
, pc
, fd
);
1552 #define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd)
1554 #define TSAN_MAYBE_INTERCEPT_EVENTFD
1557 #if !SANITIZER_FREEBSD
1558 TSAN_INTERCEPTOR(int, signalfd
, int fd
, void *mask
, int flags
) {
1559 SCOPED_TSAN_INTERCEPTOR(signalfd
, fd
, mask
, flags
);
1561 FdClose(thr
, pc
, fd
);
1562 fd
= REAL(signalfd
)(fd
, mask
, flags
);
1564 FdSignalCreate(thr
, pc
, fd
);
1567 #define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd)
1569 #define TSAN_MAYBE_INTERCEPT_SIGNALFD
1572 #if !SANITIZER_FREEBSD
1573 TSAN_INTERCEPTOR(int, inotify_init
, int fake
) {
1574 SCOPED_TSAN_INTERCEPTOR(inotify_init
, fake
);
1575 int fd
= REAL(inotify_init
)(fake
);
1577 FdInotifyCreate(thr
, pc
, fd
);
1580 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init)
1582 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT
1585 #if !SANITIZER_FREEBSD
1586 TSAN_INTERCEPTOR(int, inotify_init1
, int flags
) {
1587 SCOPED_TSAN_INTERCEPTOR(inotify_init1
, flags
);
1588 int fd
= REAL(inotify_init1
)(flags
);
1590 FdInotifyCreate(thr
, pc
, fd
);
1593 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1)
1595 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1
1598 TSAN_INTERCEPTOR(int, socket
, int domain
, int type
, int protocol
) {
1599 SCOPED_TSAN_INTERCEPTOR(socket
, domain
, type
, protocol
);
1600 int fd
= REAL(socket
)(domain
, type
, protocol
);
1602 FdSocketCreate(thr
, pc
, fd
);
1606 TSAN_INTERCEPTOR(int, socketpair
, int domain
, int type
, int protocol
, int *fd
) {
1607 SCOPED_TSAN_INTERCEPTOR(socketpair
, domain
, type
, protocol
, fd
);
1608 int res
= REAL(socketpair
)(domain
, type
, protocol
, fd
);
1609 if (res
== 0 && fd
[0] >= 0 && fd
[1] >= 0)
1610 FdPipeCreate(thr
, pc
, fd
[0], fd
[1]);
1614 TSAN_INTERCEPTOR(int, connect
, int fd
, void *addr
, unsigned addrlen
) {
1615 SCOPED_TSAN_INTERCEPTOR(connect
, fd
, addr
, addrlen
);
1616 FdSocketConnecting(thr
, pc
, fd
);
1617 int res
= REAL(connect
)(fd
, addr
, addrlen
);
1618 if (res
== 0 && fd
>= 0)
1619 FdSocketConnect(thr
, pc
, fd
);
1623 TSAN_INTERCEPTOR(int, bind
, int fd
, void *addr
, unsigned addrlen
) {
1624 SCOPED_TSAN_INTERCEPTOR(bind
, fd
, addr
, addrlen
);
1625 int res
= REAL(bind
)(fd
, addr
, addrlen
);
1626 if (fd
> 0 && res
== 0)
1627 FdAccess(thr
, pc
, fd
);
1631 TSAN_INTERCEPTOR(int, listen
, int fd
, int backlog
) {
1632 SCOPED_TSAN_INTERCEPTOR(listen
, fd
, backlog
);
1633 int res
= REAL(listen
)(fd
, backlog
);
1634 if (fd
> 0 && res
== 0)
1635 FdAccess(thr
, pc
, fd
);
1639 #if !SANITIZER_FREEBSD
1640 TSAN_INTERCEPTOR(int, epoll_create
, int size
) {
1641 SCOPED_TSAN_INTERCEPTOR(epoll_create
, size
);
1642 int fd
= REAL(epoll_create
)(size
);
1644 FdPollCreate(thr
, pc
, fd
);
1647 #define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE TSAN_INTERCEPT(epoll_create)
1649 #define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE
1652 #if !SANITIZER_FREEBSD
1653 TSAN_INTERCEPTOR(int, epoll_create1
, int flags
) {
1654 SCOPED_TSAN_INTERCEPTOR(epoll_create1
, flags
);
1655 int fd
= REAL(epoll_create1
)(flags
);
1657 FdPollCreate(thr
, pc
, fd
);
1660 #define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE1 TSAN_INTERCEPT(epoll_create1)
1662 #define TSAN_MAYBE_INTERCEPT_EPOLL_CREATE1
1665 TSAN_INTERCEPTOR(int, close
, int fd
) {
1666 SCOPED_TSAN_INTERCEPTOR(close
, fd
);
1668 FdClose(thr
, pc
, fd
);
1669 return REAL(close
)(fd
);
1672 #if !SANITIZER_FREEBSD
1673 TSAN_INTERCEPTOR(int, __close
, int fd
) {
1674 SCOPED_TSAN_INTERCEPTOR(__close
, fd
);
1676 FdClose(thr
, pc
, fd
);
1677 return REAL(__close
)(fd
);
1679 #define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close)
1681 #define TSAN_MAYBE_INTERCEPT___CLOSE
1685 #if !SANITIZER_FREEBSD
1686 TSAN_INTERCEPTOR(void, __res_iclose
, void *state
, bool free_addr
) {
1687 SCOPED_TSAN_INTERCEPTOR(__res_iclose
, state
, free_addr
);
1689 int cnt
= ExtractResolvFDs(state
, fds
, ARRAY_SIZE(fds
));
1690 for (int i
= 0; i
< cnt
; i
++) {
1692 FdClose(thr
, pc
, fds
[i
]);
1694 REAL(__res_iclose
)(state
, free_addr
);
1696 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose)
1698 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE
1701 TSAN_INTERCEPTOR(int, pipe
, int *pipefd
) {
1702 SCOPED_TSAN_INTERCEPTOR(pipe
, pipefd
);
1703 int res
= REAL(pipe
)(pipefd
);
1704 if (res
== 0 && pipefd
[0] >= 0 && pipefd
[1] >= 0)
1705 FdPipeCreate(thr
, pc
, pipefd
[0], pipefd
[1]);
1709 TSAN_INTERCEPTOR(int, pipe2
, int *pipefd
, int flags
) {
1710 SCOPED_TSAN_INTERCEPTOR(pipe2
, pipefd
, flags
);
1711 int res
= REAL(pipe2
)(pipefd
, flags
);
1712 if (res
== 0 && pipefd
[0] >= 0 && pipefd
[1] >= 0)
1713 FdPipeCreate(thr
, pc
, pipefd
[0], pipefd
[1]);
1717 TSAN_INTERCEPTOR(long_t
, send
, int fd
, void *buf
, long_t len
, int flags
) {
1718 SCOPED_TSAN_INTERCEPTOR(send
, fd
, buf
, len
, flags
);
1720 FdAccess(thr
, pc
, fd
);
1721 FdRelease(thr
, pc
, fd
);
1723 int res
= REAL(send
)(fd
, buf
, len
, flags
);
1727 TSAN_INTERCEPTOR(long_t
, sendmsg
, int fd
, void *msg
, int flags
) {
1728 SCOPED_TSAN_INTERCEPTOR(sendmsg
, fd
, msg
, flags
);
1730 FdAccess(thr
, pc
, fd
);
1731 FdRelease(thr
, pc
, fd
);
1733 int res
= REAL(sendmsg
)(fd
, msg
, flags
);
1737 TSAN_INTERCEPTOR(long_t
, recv
, int fd
, void *buf
, long_t len
, int flags
) {
1738 SCOPED_TSAN_INTERCEPTOR(recv
, fd
, buf
, len
, flags
);
1740 FdAccess(thr
, pc
, fd
);
1741 int res
= REAL(recv
)(fd
, buf
, len
, flags
);
1742 if (res
>= 0 && fd
>= 0) {
1743 FdAcquire(thr
, pc
, fd
);
1748 TSAN_INTERCEPTOR(int, unlink
, char *path
) {
1749 SCOPED_TSAN_INTERCEPTOR(unlink
, path
);
1750 Release(thr
, pc
, File2addr(path
));
1751 int res
= REAL(unlink
)(path
);
1755 TSAN_INTERCEPTOR(void*, tmpfile
, int fake
) {
1756 SCOPED_TSAN_INTERCEPTOR(tmpfile
, fake
);
1757 void *res
= REAL(tmpfile
)(fake
);
1759 int fd
= fileno_unlocked(res
);
1761 FdFileCreate(thr
, pc
, fd
);
1766 #if !SANITIZER_FREEBSD
1767 TSAN_INTERCEPTOR(void*, tmpfile64
, int fake
) {
1768 SCOPED_TSAN_INTERCEPTOR(tmpfile64
, fake
);
1769 void *res
= REAL(tmpfile64
)(fake
);
1771 int fd
= fileno_unlocked(res
);
1773 FdFileCreate(thr
, pc
, fd
);
1777 #define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64)
1779 #define TSAN_MAYBE_INTERCEPT_TMPFILE64
1782 TSAN_INTERCEPTOR(uptr
, fread
, void *ptr
, uptr size
, uptr nmemb
, void *f
) {
1783 // libc file streams can call user-supplied functions, see fopencookie.
1785 SCOPED_TSAN_INTERCEPTOR(fread
, ptr
, size
, nmemb
, f
);
1786 MemoryAccessRange(thr
, pc
, (uptr
)ptr
, size
* nmemb
, true);
1788 return REAL(fread
)(ptr
, size
, nmemb
, f
);
1791 TSAN_INTERCEPTOR(uptr
, fwrite
, const void *p
, uptr size
, uptr nmemb
, void *f
) {
1792 // libc file streams can call user-supplied functions, see fopencookie.
1794 SCOPED_TSAN_INTERCEPTOR(fwrite
, p
, size
, nmemb
, f
);
1795 MemoryAccessRange(thr
, pc
, (uptr
)p
, size
* nmemb
, false);
1797 return REAL(fwrite
)(p
, size
, nmemb
, f
);
1800 TSAN_INTERCEPTOR(void, abort
, int fake
) {
1801 SCOPED_TSAN_INTERCEPTOR(abort
, fake
);
1806 TSAN_INTERCEPTOR(int, puts
, const char *s
) {
1807 SCOPED_TSAN_INTERCEPTOR(puts
, s
);
1808 MemoryAccessRange(thr
, pc
, (uptr
)s
, internal_strlen(s
), false);
1809 return REAL(puts
)(s
);
1812 TSAN_INTERCEPTOR(int, rmdir
, char *path
) {
1813 SCOPED_TSAN_INTERCEPTOR(rmdir
, path
);
1814 Release(thr
, pc
, Dir2addr(path
));
1815 int res
= REAL(rmdir
)(path
);
1819 TSAN_INTERCEPTOR(void*, opendir
, char *path
) {
1820 SCOPED_TSAN_INTERCEPTOR(opendir
, path
);
1821 void *res
= REAL(opendir
)(path
);
1823 Acquire(thr
, pc
, Dir2addr(path
));
1827 #if !SANITIZER_FREEBSD
1828 TSAN_INTERCEPTOR(int, epoll_ctl
, int epfd
, int op
, int fd
, void *ev
) {
1829 SCOPED_TSAN_INTERCEPTOR(epoll_ctl
, epfd
, op
, fd
, ev
);
1831 FdAccess(thr
, pc
, epfd
);
1832 if (epfd
>= 0 && fd
>= 0)
1833 FdAccess(thr
, pc
, fd
);
1834 if (op
== EPOLL_CTL_ADD
&& epfd
>= 0)
1835 FdRelease(thr
, pc
, epfd
);
1836 int res
= REAL(epoll_ctl
)(epfd
, op
, fd
, ev
);
1839 #define TSAN_MAYBE_INTERCEPT_EPOLL_CTL TSAN_INTERCEPT(epoll_ctl)
1841 #define TSAN_MAYBE_INTERCEPT_EPOLL_CTL
1844 #if !SANITIZER_FREEBSD
1845 TSAN_INTERCEPTOR(int, epoll_wait
, int epfd
, void *ev
, int cnt
, int timeout
) {
1846 SCOPED_TSAN_INTERCEPTOR(epoll_wait
, epfd
, ev
, cnt
, timeout
);
1848 FdAccess(thr
, pc
, epfd
);
1849 int res
= BLOCK_REAL(epoll_wait
)(epfd
, ev
, cnt
, timeout
);
1850 if (res
> 0 && epfd
>= 0)
1851 FdAcquire(thr
, pc
, epfd
);
1854 #define TSAN_MAYBE_INTERCEPT_EPOLL_WAIT TSAN_INTERCEPT(epoll_wait)
1856 #define TSAN_MAYBE_INTERCEPT_EPOLL_WAIT
1861 static void CallUserSignalHandler(ThreadState
*thr
, bool sync
, bool acquire
,
1862 bool sigact
, int sig
, my_siginfo_t
*info
, void *uctx
) {
1864 Acquire(thr
, 0, (uptr
)&sigactions
[sig
]);
1865 // Ensure that the handler does not spoil errno.
1866 const int saved_errno
= errno
;
1868 // Need to remember pc before the call, because the handler can reset it.
1870 (uptr
)sigactions
[sig
].sa_sigaction
:
1871 (uptr
)sigactions
[sig
].sa_handler
;
1872 pc
+= 1; // return address is expected, OutputReport() will undo this
1874 sigactions
[sig
].sa_sigaction(sig
, info
, uctx
);
1876 sigactions
[sig
].sa_handler(sig
);
1877 // We do not detect errno spoiling for SIGTERM,
1878 // because some SIGTERM handlers do spoil errno but reraise SIGTERM,
1879 // tsan reports false positive in such case.
1880 // It's difficult to properly detect this situation (reraise),
1881 // because in async signal processing case (when handler is called directly
1882 // from rtl_generic_sighandler) we have not yet received the reraised
1883 // signal; and it looks too fragile to intercept all ways to reraise a signal.
1884 if (flags()->report_bugs
&& !sync
&& sig
!= SIGTERM
&& errno
!= 99) {
1885 VarSizeStackTrace stack
;
1886 ObtainCurrentStack(thr
, pc
, &stack
);
1887 ThreadRegistryLock
l(ctx
->thread_registry
);
1888 ScopedReport
rep(ReportTypeErrnoInSignal
);
1889 if (!IsFiredSuppression(ctx
, rep
, stack
)) {
1890 rep
.AddStack(stack
, true);
1891 OutputReport(thr
, rep
);
1894 errno
= saved_errno
;
1897 void ProcessPendingSignals(ThreadState
*thr
) {
1898 SignalContext
*sctx
= SigCtx(thr
);
1900 atomic_load(&sctx
->have_pending_signals
, memory_order_relaxed
) == 0)
1902 atomic_store(&sctx
->have_pending_signals
, 0, memory_order_relaxed
);
1903 atomic_fetch_add(&thr
->in_signal_handler
, 1, memory_order_relaxed
);
1904 // These are too big for stack.
1905 static THREADLOCAL __sanitizer_sigset_t emptyset
, oldset
;
1906 REAL(sigfillset
)(&emptyset
);
1907 pthread_sigmask(SIG_SETMASK
, &emptyset
, &oldset
);
1908 for (int sig
= 0; sig
< kSigCount
; sig
++) {
1909 SignalDesc
*signal
= &sctx
->pending_signals
[sig
];
1910 if (signal
->armed
) {
1911 signal
->armed
= false;
1912 if (sigactions
[sig
].sa_handler
!= SIG_DFL
1913 && sigactions
[sig
].sa_handler
!= SIG_IGN
) {
1914 CallUserSignalHandler(thr
, false, true, signal
->sigaction
,
1915 sig
, &signal
->siginfo
, &signal
->ctx
);
1919 pthread_sigmask(SIG_SETMASK
, &oldset
, 0);
1920 atomic_fetch_add(&thr
->in_signal_handler
, -1, memory_order_relaxed
);
1923 } // namespace __tsan
1925 static bool is_sync_signal(SignalContext
*sctx
, int sig
) {
1926 return sig
== SIGSEGV
|| sig
== SIGBUS
|| sig
== SIGILL
||
1927 sig
== SIGABRT
|| sig
== SIGFPE
|| sig
== SIGPIPE
|| sig
== SIGSYS
||
1928 // If we are sending signal to ourselves, we must process it now.
1929 (sctx
&& sig
== sctx
->int_signal_send
);
1932 void ALWAYS_INLINE
rtl_generic_sighandler(bool sigact
, int sig
,
1933 my_siginfo_t
*info
, void *ctx
) {
1934 ThreadState
*thr
= cur_thread();
1935 SignalContext
*sctx
= SigCtx(thr
);
1936 if (sig
< 0 || sig
>= kSigCount
) {
1937 VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig
);
1940 // Don't mess with synchronous signals.
1941 const bool sync
= is_sync_signal(sctx
, sig
);
1943 // If we are in blocking function, we can safely process it now
1944 // (but check if we are in a recursive interceptor,
1945 // i.e. pthread_join()->munmap()).
1946 (sctx
&& atomic_load(&sctx
->in_blocking_func
, memory_order_relaxed
))) {
1947 atomic_fetch_add(&thr
->in_signal_handler
, 1, memory_order_relaxed
);
1948 if (sctx
&& atomic_load(&sctx
->in_blocking_func
, memory_order_relaxed
)) {
1949 // We ignore interceptors in blocking functions,
1950 // temporary enbled them again while we are calling user function.
1951 int const i
= thr
->ignore_interceptors
;
1952 thr
->ignore_interceptors
= 0;
1953 atomic_store(&sctx
->in_blocking_func
, 0, memory_order_relaxed
);
1954 CallUserSignalHandler(thr
, sync
, true, sigact
, sig
, info
, ctx
);
1955 thr
->ignore_interceptors
= i
;
1956 atomic_store(&sctx
->in_blocking_func
, 1, memory_order_relaxed
);
1958 // Be very conservative with when we do acquire in this case.
1959 // It's unsafe to do acquire in async handlers, because ThreadState
1960 // can be in inconsistent state.
1961 // SIGSYS looks relatively safe -- it's synchronous and can actually
1962 // need some global state.
1963 bool acq
= (sig
== SIGSYS
);
1964 CallUserSignalHandler(thr
, sync
, acq
, sigact
, sig
, info
, ctx
);
1966 atomic_fetch_add(&thr
->in_signal_handler
, -1, memory_order_relaxed
);
1972 SignalDesc
*signal
= &sctx
->pending_signals
[sig
];
1973 if (signal
->armed
== false) {
1974 signal
->armed
= true;
1975 signal
->sigaction
= sigact
;
1977 internal_memcpy(&signal
->siginfo
, info
, sizeof(*info
));
1979 internal_memcpy(&signal
->ctx
, ctx
, sizeof(signal
->ctx
));
1980 atomic_store(&sctx
->have_pending_signals
, 1, memory_order_relaxed
);
1984 static void rtl_sighandler(int sig
) {
1985 rtl_generic_sighandler(false, sig
, 0, 0);
1988 static void rtl_sigaction(int sig
, my_siginfo_t
*info
, void *ctx
) {
1989 rtl_generic_sighandler(true, sig
, info
, ctx
);
1992 TSAN_INTERCEPTOR(int, sigaction
, int sig
, sigaction_t
*act
, sigaction_t
*old
) {
1993 SCOPED_TSAN_INTERCEPTOR(sigaction
, sig
, act
, old
);
1995 internal_memcpy(old
, &sigactions
[sig
], sizeof(*old
));
1998 internal_memcpy(&sigactions
[sig
], act
, sizeof(*act
));
2000 internal_memcpy(&newact
, act
, sizeof(newact
));
2001 REAL(sigfillset
)(&newact
.sa_mask
);
2002 if (act
->sa_handler
!= SIG_IGN
&& act
->sa_handler
!= SIG_DFL
) {
2003 if (newact
.sa_flags
& SA_SIGINFO
)
2004 newact
.sa_sigaction
= rtl_sigaction
;
2006 newact
.sa_handler
= rtl_sighandler
;
2008 ReleaseStore(thr
, pc
, (uptr
)&sigactions
[sig
]);
2009 int res
= REAL(sigaction
)(sig
, &newact
, 0);
2013 TSAN_INTERCEPTOR(sighandler_t
, signal
, int sig
, sighandler_t h
) {
2016 REAL(memset
)(&act
.sa_mask
, -1, sizeof(act
.sa_mask
));
2019 int res
= sigaction(sig
, &act
, &old
);
2022 return old
.sa_handler
;
2025 TSAN_INTERCEPTOR(int, sigsuspend
, const __sanitizer_sigset_t
*mask
) {
2026 SCOPED_TSAN_INTERCEPTOR(sigsuspend
, mask
);
2027 return REAL(sigsuspend
)(mask
);
2030 TSAN_INTERCEPTOR(int, raise
, int sig
) {
2031 SCOPED_TSAN_INTERCEPTOR(raise
, sig
);
2032 SignalContext
*sctx
= SigCtx(thr
);
2034 int prev
= sctx
->int_signal_send
;
2035 sctx
->int_signal_send
= sig
;
2036 int res
= REAL(raise
)(sig
);
2037 CHECK_EQ(sctx
->int_signal_send
, sig
);
2038 sctx
->int_signal_send
= prev
;
2042 TSAN_INTERCEPTOR(int, kill
, int pid
, int sig
) {
2043 SCOPED_TSAN_INTERCEPTOR(kill
, pid
, sig
);
2044 SignalContext
*sctx
= SigCtx(thr
);
2046 int prev
= sctx
->int_signal_send
;
2047 if (pid
== (int)internal_getpid()) {
2048 sctx
->int_signal_send
= sig
;
2050 int res
= REAL(kill
)(pid
, sig
);
2051 if (pid
== (int)internal_getpid()) {
2052 CHECK_EQ(sctx
->int_signal_send
, sig
);
2053 sctx
->int_signal_send
= prev
;
2058 TSAN_INTERCEPTOR(int, pthread_kill
, void *tid
, int sig
) {
2059 SCOPED_TSAN_INTERCEPTOR(pthread_kill
, tid
, sig
);
2060 SignalContext
*sctx
= SigCtx(thr
);
2062 int prev
= sctx
->int_signal_send
;
2063 if (tid
== pthread_self()) {
2064 sctx
->int_signal_send
= sig
;
2066 int res
= REAL(pthread_kill
)(tid
, sig
);
2067 if (tid
== pthread_self()) {
2068 CHECK_EQ(sctx
->int_signal_send
, sig
);
2069 sctx
->int_signal_send
= prev
;
2074 TSAN_INTERCEPTOR(int, gettimeofday
, void *tv
, void *tz
) {
2075 SCOPED_TSAN_INTERCEPTOR(gettimeofday
, tv
, tz
);
2076 // It's intercepted merely to process pending signals.
2077 return REAL(gettimeofday
)(tv
, tz
);
2080 TSAN_INTERCEPTOR(int, getaddrinfo
, void *node
, void *service
,
2081 void *hints
, void *rv
) {
2082 SCOPED_TSAN_INTERCEPTOR(getaddrinfo
, node
, service
, hints
, rv
);
2083 // We miss atomic synchronization in getaddrinfo,
2084 // and can report false race between malloc and free
2085 // inside of getaddrinfo. So ignore memory accesses.
2086 ThreadIgnoreBegin(thr
, pc
);
2087 int res
= REAL(getaddrinfo
)(node
, service
, hints
, rv
);
2088 ThreadIgnoreEnd(thr
, pc
);
2092 TSAN_INTERCEPTOR(int, fork
, int fake
) {
2093 if (cur_thread()->in_symbolizer
)
2094 return REAL(fork
)(fake
);
2095 SCOPED_INTERCEPTOR_RAW(fork
, fake
);
2096 ForkBefore(thr
, pc
);
2097 int pid
= REAL(fork
)(fake
);
2100 ForkChildAfter(thr
, pc
);
2102 } else if (pid
> 0) {
2104 ForkParentAfter(thr
, pc
);
2107 ForkParentAfter(thr
, pc
);
2112 TSAN_INTERCEPTOR(int, vfork
, int fake
) {
2113 // Some programs (e.g. openjdk) call close for all file descriptors
2114 // in the child process. Under tsan it leads to false positives, because
2115 // address space is shared, so the parent process also thinks that
2116 // the descriptors are closed (while they are actually not).
2117 // This leads to false positives due to missed synchronization.
2118 // Strictly saying this is undefined behavior, because vfork child is not
2119 // allowed to call any functions other than exec/exit. But this is what
2120 // openjdk does, so we want to handle it.
2121 // We could disable interceptors in the child process. But it's not possible
2122 // to simply intercept and wrap vfork, because vfork child is not allowed
2123 // to return from the function that calls vfork, and that's exactly what
2124 // we would do. So this would require some assembly trickery as well.
2125 // Instead we simply turn vfork into fork.
2126 return WRAP(fork
)(fake
);
2129 static int OnExit(ThreadState
*thr
) {
2130 int status
= Finalize(thr
);
2135 struct TsanInterceptorContext
{
2137 const uptr caller_pc
;
2141 static void HandleRecvmsg(ThreadState
*thr
, uptr pc
,
2142 __sanitizer_msghdr
*msg
) {
2144 int cnt
= ExtractRecvmsgFDs(msg
, fds
, ARRAY_SIZE(fds
));
2145 for (int i
= 0; i
< cnt
; i
++)
2146 FdEventCreate(thr
, pc
, fds
[i
]);
2149 #include "sanitizer_common/sanitizer_platform_interceptors.h"
2150 // Causes interceptor recursion (getaddrinfo() and fopen())
2151 #undef SANITIZER_INTERCEPT_GETADDRINFO
2152 // There interceptors do not seem to be strictly necessary for tsan.
2153 // But we see cases where the interceptors consume 70% of execution time.
2154 // Memory blocks passed to fgetgrent_r are "written to" by tsan several times.
2155 // First, there is some recursion (getgrnam_r calls fgetgrent_r), and each
2156 // function "writes to" the buffer. Then, the same memory is "written to"
2157 // twice, first as buf and then as pwbufp (both of them refer to the same
2159 #undef SANITIZER_INTERCEPT_GETPWENT
2160 #undef SANITIZER_INTERCEPT_GETPWENT_R
2161 #undef SANITIZER_INTERCEPT_FGETPWENT
2162 #undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS
2163 #undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS
2165 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
2167 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
2168 MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \
2169 ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
2172 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
2173 MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \
2174 ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
2177 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
2178 SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \
2179 TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
2180 ctx = (void *)&_ctx; \
2183 #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
2184 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
2185 TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
2186 ctx = (void *)&_ctx; \
2189 #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
2190 Acquire(thr, pc, File2addr(path)); \
2192 int fd = fileno_unlocked(file); \
2193 if (fd >= 0) FdFileCreate(thr, pc, fd); \
2196 #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \
2198 int fd = fileno_unlocked(file); \
2199 if (fd >= 0) FdClose(thr, pc, fd); \
2202 #define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, res) \
2203 libignore()->OnLibraryLoaded(filename)
2205 #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
2206 libignore()->OnLibraryUnloaded()
2208 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
2209 FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2211 #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
2212 FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2214 #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \
2215 FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2217 #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
2218 FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd)
2220 #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
2221 ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
2223 #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
2224 __tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name)
2226 #define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
2228 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
2229 OnExit(((TsanInterceptorContext *) ctx)->thr)
2231 #define COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m) \
2232 MutexLock(((TsanInterceptorContext *)ctx)->thr, \
2233 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2235 #define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
2236 MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
2237 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2239 #define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
2240 MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
2241 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2243 #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \
2244 HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \
2245 ((TsanInterceptorContext *)ctx)->pc, msg)
2247 #include "sanitizer_common/sanitizer_common_interceptors.inc"
2249 #define TSAN_SYSCALL() \
2250 ThreadState *thr = cur_thread(); \
2251 if (thr->ignore_interceptors) \
2253 ScopedSyscall scoped_syscall(thr) \
2256 struct ScopedSyscall
{
2259 explicit ScopedSyscall(ThreadState
*thr
)
2265 ProcessPendingSignals(thr
);
2269 static void syscall_access_range(uptr pc
, uptr p
, uptr s
, bool write
) {
2271 MemoryAccessRange(thr
, pc
, p
, s
, write
);
2274 static void syscall_acquire(uptr pc
, uptr addr
) {
2276 Acquire(thr
, pc
, addr
);
2277 DPrintf("syscall_acquire(%p)\n", addr
);
2280 static void syscall_release(uptr pc
, uptr addr
) {
2282 DPrintf("syscall_release(%p)\n", addr
);
2283 Release(thr
, pc
, addr
);
2286 static void syscall_fd_close(uptr pc
, int fd
) {
2288 FdClose(thr
, pc
, fd
);
2291 static USED
void syscall_fd_acquire(uptr pc
, int fd
) {
2293 FdAcquire(thr
, pc
, fd
);
2294 DPrintf("syscall_fd_acquire(%p)\n", fd
);
2297 static USED
void syscall_fd_release(uptr pc
, int fd
) {
2299 DPrintf("syscall_fd_release(%p)\n", fd
);
2300 FdRelease(thr
, pc
, fd
);
2303 static void syscall_pre_fork(uptr pc
) {
2305 ForkBefore(thr
, pc
);
2308 static void syscall_post_fork(uptr pc
, int pid
) {
2312 ForkChildAfter(thr
, pc
);
2314 } else if (pid
> 0) {
2316 ForkParentAfter(thr
, pc
);
2319 ForkParentAfter(thr
, pc
);
2323 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \
2324 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false)
2326 #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
2327 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true)
2329 #define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
2335 #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
2341 #define COMMON_SYSCALL_ACQUIRE(addr) \
2342 syscall_acquire(GET_CALLER_PC(), (uptr)(addr))
2344 #define COMMON_SYSCALL_RELEASE(addr) \
2345 syscall_release(GET_CALLER_PC(), (uptr)(addr))
2347 #define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd)
2349 #define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd)
2351 #define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd)
2353 #define COMMON_SYSCALL_PRE_FORK() \
2354 syscall_pre_fork(GET_CALLER_PC())
2356 #define COMMON_SYSCALL_POST_FORK(res) \
2357 syscall_post_fork(GET_CALLER_PC(), res)
2359 #include "sanitizer_common/sanitizer_common_syscalls.inc"
2363 static void finalize(void *arg
) {
2364 ThreadState
*thr
= cur_thread();
2365 int status
= Finalize(thr
);
2366 // Make sure the output is not lost.
2367 // Flushing all the streams here may freeze the process if a child thread is
2368 // performing file stream operations at the same time.
2369 REAL(fflush
)(stdout
);
2370 REAL(fflush
)(stderr
);
2372 REAL(_exit
)(status
);
2375 static void unreachable() {
2376 Report("FATAL: ThreadSanitizer: unreachable called\n");
2380 void InitializeInterceptors() {
2381 // We need to setup it early, because functions like dlsym() can call it.
2382 REAL(memset
) = internal_memset
;
2383 REAL(memcpy
) = internal_memcpy
;
2384 REAL(memcmp
) = internal_memcmp
;
2386 // Instruct libc malloc to consume less memory.
2387 #if !SANITIZER_FREEBSD
2388 mallopt(1, 0); // M_MXFAST
2389 mallopt(-3, 32*1024); // M_MMAP_THRESHOLD
2392 InitializeCommonInterceptors();
2394 // We can not use TSAN_INTERCEPT to get setjmp addr,
2395 // because it does &setjmp and setjmp is not present in some versions of libc.
2396 using __interception::GetRealFunctionAddress
;
2397 GetRealFunctionAddress("setjmp", (uptr
*)&REAL(setjmp
), 0, 0);
2398 GetRealFunctionAddress("_setjmp", (uptr
*)&REAL(_setjmp
), 0, 0);
2399 GetRealFunctionAddress("sigsetjmp", (uptr
*)&REAL(sigsetjmp
), 0, 0);
2400 GetRealFunctionAddress("__sigsetjmp", (uptr
*)&REAL(__sigsetjmp
), 0, 0);
2402 TSAN_INTERCEPT(longjmp
);
2403 TSAN_INTERCEPT(siglongjmp
);
2405 TSAN_INTERCEPT(malloc
);
2406 TSAN_INTERCEPT(__libc_memalign
);
2407 TSAN_INTERCEPT(calloc
);
2408 TSAN_INTERCEPT(realloc
);
2409 TSAN_INTERCEPT(free
);
2410 TSAN_INTERCEPT(cfree
);
2411 TSAN_INTERCEPT(mmap
);
2412 TSAN_MAYBE_INTERCEPT_MMAP64
;
2413 TSAN_INTERCEPT(munmap
);
2414 TSAN_MAYBE_INTERCEPT_MEMALIGN
;
2415 TSAN_INTERCEPT(valloc
);
2416 TSAN_MAYBE_INTERCEPT_PVALLOC
;
2417 TSAN_INTERCEPT(posix_memalign
);
2419 TSAN_INTERCEPT(strlen
);
2420 TSAN_INTERCEPT(memset
);
2421 TSAN_INTERCEPT(memcpy
);
2422 TSAN_INTERCEPT(memmove
);
2423 TSAN_INTERCEPT(memcmp
);
2424 TSAN_INTERCEPT(strchr
);
2425 TSAN_INTERCEPT(strchrnul
);
2426 TSAN_INTERCEPT(strrchr
);
2427 TSAN_INTERCEPT(strcpy
); // NOLINT
2428 TSAN_INTERCEPT(strncpy
);
2429 TSAN_INTERCEPT(strstr
);
2430 TSAN_INTERCEPT(strdup
);
2432 TSAN_INTERCEPT(pthread_create
);
2433 TSAN_INTERCEPT(pthread_join
);
2434 TSAN_INTERCEPT(pthread_detach
);
2436 TSAN_INTERCEPT_VER(pthread_cond_init
, "GLIBC_2.3.2");
2437 TSAN_INTERCEPT_VER(pthread_cond_signal
, "GLIBC_2.3.2");
2438 TSAN_INTERCEPT_VER(pthread_cond_broadcast
, "GLIBC_2.3.2");
2439 TSAN_INTERCEPT_VER(pthread_cond_wait
, "GLIBC_2.3.2");
2440 TSAN_INTERCEPT_VER(pthread_cond_timedwait
, "GLIBC_2.3.2");
2441 TSAN_INTERCEPT_VER(pthread_cond_destroy
, "GLIBC_2.3.2");
2443 TSAN_INTERCEPT(pthread_mutex_init
);
2444 TSAN_INTERCEPT(pthread_mutex_destroy
);
2445 TSAN_INTERCEPT(pthread_mutex_trylock
);
2446 TSAN_INTERCEPT(pthread_mutex_timedlock
);
2448 TSAN_INTERCEPT(pthread_spin_init
);
2449 TSAN_INTERCEPT(pthread_spin_destroy
);
2450 TSAN_INTERCEPT(pthread_spin_lock
);
2451 TSAN_INTERCEPT(pthread_spin_trylock
);
2452 TSAN_INTERCEPT(pthread_spin_unlock
);
2454 TSAN_INTERCEPT(pthread_rwlock_init
);
2455 TSAN_INTERCEPT(pthread_rwlock_destroy
);
2456 TSAN_INTERCEPT(pthread_rwlock_rdlock
);
2457 TSAN_INTERCEPT(pthread_rwlock_tryrdlock
);
2458 TSAN_INTERCEPT(pthread_rwlock_timedrdlock
);
2459 TSAN_INTERCEPT(pthread_rwlock_wrlock
);
2460 TSAN_INTERCEPT(pthread_rwlock_trywrlock
);
2461 TSAN_INTERCEPT(pthread_rwlock_timedwrlock
);
2462 TSAN_INTERCEPT(pthread_rwlock_unlock
);
2464 TSAN_INTERCEPT(pthread_barrier_init
);
2465 TSAN_INTERCEPT(pthread_barrier_destroy
);
2466 TSAN_INTERCEPT(pthread_barrier_wait
);
2468 TSAN_INTERCEPT(pthread_once
);
2470 TSAN_INTERCEPT(sem_init
);
2471 TSAN_INTERCEPT(sem_destroy
);
2472 TSAN_INTERCEPT(sem_wait
);
2473 TSAN_INTERCEPT(sem_trywait
);
2474 TSAN_INTERCEPT(sem_timedwait
);
2475 TSAN_INTERCEPT(sem_post
);
2476 TSAN_INTERCEPT(sem_getvalue
);
2478 TSAN_INTERCEPT(stat
);
2479 TSAN_MAYBE_INTERCEPT___XSTAT
;
2480 TSAN_MAYBE_INTERCEPT_STAT64
;
2481 TSAN_MAYBE_INTERCEPT___XSTAT64
;
2482 TSAN_INTERCEPT(lstat
);
2483 TSAN_MAYBE_INTERCEPT___LXSTAT
;
2484 TSAN_MAYBE_INTERCEPT_LSTAT64
;
2485 TSAN_MAYBE_INTERCEPT___LXSTAT64
;
2486 TSAN_INTERCEPT(fstat
);
2487 TSAN_MAYBE_INTERCEPT___FXSTAT
;
2488 TSAN_MAYBE_INTERCEPT_FSTAT64
;
2489 TSAN_MAYBE_INTERCEPT___FXSTAT64
;
2490 TSAN_INTERCEPT(open
);
2491 TSAN_MAYBE_INTERCEPT_OPEN64
;
2492 TSAN_INTERCEPT(creat
);
2493 TSAN_MAYBE_INTERCEPT_CREAT64
;
2494 TSAN_INTERCEPT(dup
);
2495 TSAN_INTERCEPT(dup2
);
2496 TSAN_INTERCEPT(dup3
);
2497 TSAN_MAYBE_INTERCEPT_EVENTFD
;
2498 TSAN_MAYBE_INTERCEPT_SIGNALFD
;
2499 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT
;
2500 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1
;
2501 TSAN_INTERCEPT(socket
);
2502 TSAN_INTERCEPT(socketpair
);
2503 TSAN_INTERCEPT(connect
);
2504 TSAN_INTERCEPT(bind
);
2505 TSAN_INTERCEPT(listen
);
2506 TSAN_MAYBE_INTERCEPT_EPOLL_CREATE
;
2507 TSAN_MAYBE_INTERCEPT_EPOLL_CREATE1
;
2508 TSAN_INTERCEPT(close
);
2509 TSAN_MAYBE_INTERCEPT___CLOSE
;
2510 TSAN_MAYBE_INTERCEPT___RES_ICLOSE
;
2511 TSAN_INTERCEPT(pipe
);
2512 TSAN_INTERCEPT(pipe2
);
2514 TSAN_INTERCEPT(send
);
2515 TSAN_INTERCEPT(sendmsg
);
2516 TSAN_INTERCEPT(recv
);
2518 TSAN_INTERCEPT(unlink
);
2519 TSAN_INTERCEPT(tmpfile
);
2520 TSAN_MAYBE_INTERCEPT_TMPFILE64
;
2521 TSAN_INTERCEPT(fread
);
2522 TSAN_INTERCEPT(fwrite
);
2523 TSAN_INTERCEPT(abort
);
2524 TSAN_INTERCEPT(puts
);
2525 TSAN_INTERCEPT(rmdir
);
2526 TSAN_INTERCEPT(opendir
);
2528 TSAN_MAYBE_INTERCEPT_EPOLL_CTL
;
2529 TSAN_MAYBE_INTERCEPT_EPOLL_WAIT
;
2531 TSAN_INTERCEPT(sigaction
);
2532 TSAN_INTERCEPT(signal
);
2533 TSAN_INTERCEPT(sigsuspend
);
2534 TSAN_INTERCEPT(raise
);
2535 TSAN_INTERCEPT(kill
);
2536 TSAN_INTERCEPT(pthread_kill
);
2537 TSAN_INTERCEPT(sleep
);
2538 TSAN_INTERCEPT(usleep
);
2539 TSAN_INTERCEPT(nanosleep
);
2540 TSAN_INTERCEPT(gettimeofday
);
2541 TSAN_INTERCEPT(getaddrinfo
);
2543 TSAN_INTERCEPT(fork
);
2544 TSAN_INTERCEPT(vfork
);
2545 TSAN_INTERCEPT(on_exit
);
2546 TSAN_INTERCEPT(__cxa_atexit
);
2547 TSAN_INTERCEPT(_exit
);
2549 // Need to setup it, because interceptors check that the function is resolved.
2550 // But atexit is emitted directly into the module, so can't be resolved.
2551 REAL(atexit
) = (int(*)(void(*)()))unreachable
;
2552 if (REAL(__cxa_atexit
)(&finalize
, 0, 0)) {
2553 Printf("ThreadSanitizer: failed to setup atexit callback\n");
2557 if (pthread_key_create(&g_thread_finalize_key
, &thread_finalize
)) {
2558 Printf("ThreadSanitizer: failed to create thread key\n");
2565 void *internal_start_thread(void(*func
)(void *arg
), void *arg
) {
2566 // Start the thread with signals blocked, otherwise it can steal user signals.
2567 __sanitizer_sigset_t set
, old
;
2568 internal_sigfillset(&set
);
2569 internal_sigprocmask(SIG_SETMASK
, &set
, &old
);
2571 REAL(pthread_create
)(&th
, 0, (void*(*)(void *arg
))func
, arg
);
2572 internal_sigprocmask(SIG_SETMASK
, &old
, 0);
2576 void internal_join_thread(void *th
) {
2577 REAL(pthread_join
)(th
, 0);
2580 } // namespace __tsan