1 //===-- tsan_interceptors.cc ----------------------------------------------===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file is a part of ThreadSanitizer (TSan), a race detector.
12 //===----------------------------------------------------------------------===//
14 #include "sanitizer_common/sanitizer_atomic.h"
15 #include "sanitizer_common/sanitizer_libc.h"
16 #include "sanitizer_common/sanitizer_placement_new.h"
17 #include "sanitizer_common/sanitizer_stacktrace.h"
18 #include "interception/interception.h"
19 #include "tsan_interface.h"
20 #include "tsan_platform.h"
22 #include "tsan_mman.h"
24 using namespace __tsan
; // NOLINT
26 const int kSigCount
= 128;
33 u64 val
[1024 / 8 / sizeof(u64
)];
40 extern "C" int pthread_attr_init(void *attr
);
41 extern "C" int pthread_attr_destroy(void *attr
);
42 extern "C" int pthread_attr_getdetachstate(void *attr
, int *v
);
43 extern "C" int pthread_attr_setstacksize(void *attr
, uptr stacksize
);
44 extern "C" int pthread_attr_getstacksize(void *attr
, uptr
*stacksize
);
45 extern "C" int pthread_key_create(unsigned *key
, void (*destructor
)(void* v
));
46 extern "C" int pthread_setspecific(unsigned key
, const void *v
);
47 extern "C" int pthread_mutexattr_gettype(void *a
, int *type
);
48 extern "C" int pthread_yield();
49 extern "C" int pthread_sigmask(int how
, const sigset_t
*set
, sigset_t
*oldset
);
50 extern "C" int sigfillset(sigset_t
*set
);
51 extern "C" void *pthread_self();
52 extern "C" void _exit(int status
);
53 extern "C" int __cxa_atexit(void (*func
)(void *arg
), void *arg
, void *dso
);
54 extern "C" int *__errno_location();
55 const int PTHREAD_MUTEX_RECURSIVE
= 1;
56 const int PTHREAD_MUTEX_RECURSIVE_NP
= 1;
57 const int kPthreadAttrSize
= 56;
58 const int EINVAL
= 22;
60 const int EPOLL_CTL_ADD
= 1;
62 const int SIGABRT
= 6;
64 const int SIGSEGV
= 11;
65 const int SIGPIPE
= 13;
67 void *const MAP_FAILED
= (void*)-1;
68 const int PTHREAD_BARRIER_SERIAL_THREAD
= -1;
69 const int MAP_FIXED
= 0x10;
70 typedef long long_t
; // NOLINT
72 // From /usr/include/unistd.h
73 # define F_ULOCK 0 /* Unlock a previously locked region. */
74 # define F_LOCK 1 /* Lock a region for exclusive use. */
75 # define F_TLOCK 2 /* Test and lock a region for exclusive use. */
76 # define F_TEST 3 /* Test a region for other processes locks. */
78 typedef void (*sighandler_t
)(int sig
);
80 #define errno (*__errno_location())
82 union pthread_attr_t
{
83 char size
[kPthreadAttrSize
];
89 sighandler_t sa_handler
;
90 void (*sa_sigaction
)(int sig
, my_siginfo_t
*siginfo
, void *uctx
);
94 void (*sa_restorer
)();
97 const sighandler_t SIG_DFL
= (sighandler_t
)0;
98 const sighandler_t SIG_IGN
= (sighandler_t
)1;
99 const sighandler_t SIG_ERR
= (sighandler_t
)-1;
100 const int SA_SIGINFO
= 4;
101 const int SIG_SETMASK
= 2;
107 static sigaction_t sigactions
[kSigCount
];
113 my_siginfo_t siginfo
;
117 struct SignalContext
{
118 int in_blocking_func
;
120 int pending_signal_count
;
121 SignalDesc pending_signals
[kSigCount
];
125 static SignalContext
*SigCtx(ThreadState
*thr
) {
126 SignalContext
*ctx
= (SignalContext
*)thr
->signal_ctx
;
127 if (ctx
== 0 && thr
->is_alive
) {
129 ctx
= (SignalContext
*)internal_alloc(
130 MBlockSignal
, sizeof(*ctx
));
131 MemoryResetRange(thr
, 0, (uptr
)ctx
, sizeof(*ctx
));
132 internal_memset(ctx
, 0, sizeof(*ctx
));
133 thr
->signal_ctx
= ctx
;
138 static unsigned g_thread_finalize_key
;
140 class ScopedInterceptor
{
142 ScopedInterceptor(ThreadState
*thr
, const char *fname
, uptr pc
);
143 ~ScopedInterceptor();
145 ThreadState
*const thr_
;
149 ScopedInterceptor::ScopedInterceptor(ThreadState
*thr
, const char *fname
,
152 , in_rtl_(thr
->in_rtl
) {
153 if (thr_
->in_rtl
== 0) {
157 DPrintf("#%d: intercept %s()\n", thr_
->tid
, fname
);
163 ScopedInterceptor::~ScopedInterceptor() {
165 if (thr_
->in_rtl
== 0) {
167 ProcessPendingSignals(thr_
);
169 CHECK_EQ(in_rtl_
, thr_
->in_rtl
);
172 #define SCOPED_INTERCEPTOR_RAW(func, ...) \
173 ThreadState *thr = cur_thread(); \
174 StatInc(thr, StatInterceptor); \
175 StatInc(thr, StatInt_##func); \
176 const uptr caller_pc = GET_CALLER_PC(); \
177 ScopedInterceptor si(thr, #func, caller_pc); \
178 /* Subtract one from pc as we need current instruction address */ \
179 const uptr pc = __sanitizer::StackTrace::GetCurrentPc() - 1; \
183 #define SCOPED_TSAN_INTERCEPTOR(func, ...) \
184 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
185 if (REAL(func) == 0) { \
186 Printf("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
189 if (thr->in_rtl > 1) \
190 return REAL(func)(__VA_ARGS__); \
193 #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
194 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
196 #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name))
198 struct BlockingCall
{
199 explicit BlockingCall(ThreadState
*thr
)
201 ctx
->in_blocking_func
++;
205 ctx
->in_blocking_func
--;
211 TSAN_INTERCEPTOR(unsigned, sleep
, unsigned sec
) {
212 SCOPED_TSAN_INTERCEPTOR(sleep
, sec
);
213 unsigned res
= BLOCK_REAL(sleep
)(sec
);
218 TSAN_INTERCEPTOR(int, usleep
, long_t usec
) {
219 SCOPED_TSAN_INTERCEPTOR(usleep
, usec
);
220 int res
= BLOCK_REAL(usleep
)(usec
);
225 TSAN_INTERCEPTOR(int, nanosleep
, void *req
, void *rem
) {
226 SCOPED_TSAN_INTERCEPTOR(nanosleep
, req
, rem
);
227 int res
= BLOCK_REAL(nanosleep
)(req
, rem
);
232 class AtExitContext
{
235 : mtx_(MutexTypeAtExit
, StatMtxAtExit
)
239 typedef void(*atexit_t
)();
241 int atexit(ThreadState
*thr
, uptr pc
, atexit_t f
) {
243 if (pos_
== kMaxAtExit
)
245 Release(thr
, pc
, (uptr
)this);
251 void exit(ThreadState
*thr
, uptr pc
) {
252 CHECK_EQ(thr
->in_rtl
, 0);
261 Acquire(thr
, pc
, (uptr
)this);
266 DPrintf("#%d: executing atexit func %p\n", thr
->tid
, f
);
267 CHECK_EQ(thr
->in_rtl
, 0);
273 static const int kMaxAtExit
= 128;
275 atexit_t stack_
[kMaxAtExit
];
279 static AtExitContext
*atexit_ctx
;
281 static void finalize(void *arg
) {
282 ThreadState
* thr
= cur_thread();
284 atexit_ctx
->exit(thr
, pc
);
287 DestroyAndFree(atexit_ctx
);
289 int status
= Finalize(cur_thread());
294 TSAN_INTERCEPTOR(int, atexit
, void (*f
)()) {
295 SCOPED_TSAN_INTERCEPTOR(atexit
, f
);
296 return atexit_ctx
->atexit(thr
, pc
, f
);
299 TSAN_INTERCEPTOR(void, longjmp
, void *env
, int val
) {
300 SCOPED_TSAN_INTERCEPTOR(longjmp
, env
, val
);
301 Printf("ThreadSanitizer: longjmp() is not supported\n");
305 TSAN_INTERCEPTOR(void, siglongjmp
, void *env
, int val
) {
306 SCOPED_TSAN_INTERCEPTOR(siglongjmp
, env
, val
);
307 Printf("ThreadSanitizer: siglongjmp() is not supported\n");
311 static uptr
fd2addr(int fd
) {
317 static uptr
epollfd2addr(int fd
) {
323 static uptr
file2addr(char *path
) {
329 static uptr
dir2addr(char *path
) {
335 TSAN_INTERCEPTOR(void*, malloc
, uptr size
) {
338 SCOPED_INTERCEPTOR_RAW(malloc
, size
);
339 p
= user_alloc(thr
, pc
, size
);
341 invoke_malloc_hook(p
, size
);
345 TSAN_INTERCEPTOR(void*, __libc_memalign
, uptr align
, uptr sz
) {
346 SCOPED_TSAN_INTERCEPTOR(__libc_memalign
, align
, sz
);
347 return user_alloc(thr
, pc
, sz
, align
);
350 TSAN_INTERCEPTOR(void*, calloc
, uptr size
, uptr n
) {
353 SCOPED_INTERCEPTOR_RAW(calloc
, size
, n
);
354 p
= user_alloc(thr
, pc
, n
* size
);
355 if (p
) internal_memset(p
, 0, n
* size
);
357 invoke_malloc_hook(p
, n
* size
);
361 TSAN_INTERCEPTOR(void*, realloc
, void *p
, uptr size
) {
365 SCOPED_INTERCEPTOR_RAW(realloc
, p
, size
);
366 p
= user_realloc(thr
, pc
, p
, size
);
368 invoke_malloc_hook(p
, size
);
372 TSAN_INTERCEPTOR(void, free
, void *p
) {
376 SCOPED_INTERCEPTOR_RAW(free
, p
);
377 user_free(thr
, pc
, p
);
380 TSAN_INTERCEPTOR(void, cfree
, void *p
) {
384 SCOPED_INTERCEPTOR_RAW(cfree
, p
);
385 user_free(thr
, pc
, p
);
388 #define OPERATOR_NEW_BODY(mangled_name) \
391 SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
392 p = user_alloc(thr, pc, size); \
394 invoke_malloc_hook(p, size); \
397 void *operator new(__sanitizer::uptr size
) {
398 OPERATOR_NEW_BODY(_Znwm
);
400 void *operator new[](__sanitizer::uptr size
) {
401 OPERATOR_NEW_BODY(_Znam
);
403 void *operator new(__sanitizer::uptr size
, std::nothrow_t
const&) {
404 OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t
);
406 void *operator new[](__sanitizer::uptr size
, std::nothrow_t
const&) {
407 OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t
);
410 #define OPERATOR_DELETE_BODY(mangled_name) \
411 if (ptr == 0) return; \
412 invoke_free_hook(ptr); \
413 SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \
414 user_free(thr, pc, ptr);
416 void operator delete(void *ptr
) {
417 OPERATOR_DELETE_BODY(_ZdlPv
);
419 void operator delete[](void *ptr
) {
420 OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t
);
422 void operator delete(void *ptr
, std::nothrow_t
const&) {
423 OPERATOR_DELETE_BODY(_ZdaPv
);
425 void operator delete[](void *ptr
, std::nothrow_t
const&) {
426 OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t
);
429 TSAN_INTERCEPTOR(uptr
, strlen
, const char *s
) {
430 SCOPED_TSAN_INTERCEPTOR(strlen
, s
);
431 uptr len
= internal_strlen(s
);
432 MemoryAccessRange(thr
, pc
, (uptr
)s
, len
+ 1, false);
436 TSAN_INTERCEPTOR(void*, memset
, void *dst
, int v
, uptr size
) {
437 SCOPED_TSAN_INTERCEPTOR(memset
, dst
, v
, size
);
438 MemoryAccessRange(thr
, pc
, (uptr
)dst
, size
, true);
439 return internal_memset(dst
, v
, size
);
442 TSAN_INTERCEPTOR(void*, memcpy
, void *dst
, const void *src
, uptr size
) {
443 SCOPED_TSAN_INTERCEPTOR(memcpy
, dst
, src
, size
);
444 MemoryAccessRange(thr
, pc
, (uptr
)dst
, size
, true);
445 MemoryAccessRange(thr
, pc
, (uptr
)src
, size
, false);
446 return internal_memcpy(dst
, src
, size
);
449 TSAN_INTERCEPTOR(int, memcmp
, const void *s1
, const void *s2
, uptr n
) {
450 SCOPED_TSAN_INTERCEPTOR(memcmp
, s1
, s2
, n
);
453 for (; len
< n
; len
++) {
454 if ((res
= ((unsigned char*)s1
)[len
] - ((unsigned char*)s2
)[len
]))
457 MemoryAccessRange(thr
, pc
, (uptr
)s1
, len
< n
? len
+ 1 : n
, false);
458 MemoryAccessRange(thr
, pc
, (uptr
)s2
, len
< n
? len
+ 1 : n
, false);
462 TSAN_INTERCEPTOR(int, strcmp
, const char *s1
, const char *s2
) {
463 SCOPED_TSAN_INTERCEPTOR(strcmp
, s1
, s2
);
465 for (; s1
[len
] && s2
[len
]; len
++) {
466 if (s1
[len
] != s2
[len
])
469 MemoryAccessRange(thr
, pc
, (uptr
)s1
, len
+ 1, false);
470 MemoryAccessRange(thr
, pc
, (uptr
)s2
, len
+ 1, false);
471 return s1
[len
] - s2
[len
];
474 TSAN_INTERCEPTOR(int, strncmp
, const char *s1
, const char *s2
, uptr n
) {
475 SCOPED_TSAN_INTERCEPTOR(strncmp
, s1
, s2
, n
);
477 for (; len
< n
&& s1
[len
] && s2
[len
]; len
++) {
478 if (s1
[len
] != s2
[len
])
481 MemoryAccessRange(thr
, pc
, (uptr
)s1
, len
< n
? len
+ 1 : n
, false);
482 MemoryAccessRange(thr
, pc
, (uptr
)s2
, len
< n
? len
+ 1 : n
, false);
483 return len
== n
? 0 : s1
[len
] - s2
[len
];
486 TSAN_INTERCEPTOR(void*, memchr
, void *s
, int c
, uptr n
) {
487 SCOPED_TSAN_INTERCEPTOR(memchr
, s
, c
, n
);
488 void *res
= REAL(memchr
)(s
, c
, n
);
489 uptr len
= res
? (char*)res
- (char*)s
+ 1 : n
;
490 MemoryAccessRange(thr
, pc
, (uptr
)s
, len
, false);
494 TSAN_INTERCEPTOR(void*, memrchr
, char *s
, int c
, uptr n
) {
495 SCOPED_TSAN_INTERCEPTOR(memrchr
, s
, c
, n
);
496 MemoryAccessRange(thr
, pc
, (uptr
)s
, n
, false);
497 return REAL(memrchr
)(s
, c
, n
);
500 TSAN_INTERCEPTOR(void*, memmove
, void *dst
, void *src
, uptr n
) {
501 SCOPED_TSAN_INTERCEPTOR(memmove
, dst
, src
, n
);
502 MemoryAccessRange(thr
, pc
, (uptr
)dst
, n
, true);
503 MemoryAccessRange(thr
, pc
, (uptr
)src
, n
, false);
504 return REAL(memmove
)(dst
, src
, n
);
507 TSAN_INTERCEPTOR(char*, strchr
, char *s
, int c
) {
508 SCOPED_TSAN_INTERCEPTOR(strchr
, s
, c
);
509 char *res
= REAL(strchr
)(s
, c
);
510 uptr len
= res
? (char*)res
- (char*)s
+ 1 : internal_strlen(s
) + 1;
511 MemoryAccessRange(thr
, pc
, (uptr
)s
, len
, false);
515 TSAN_INTERCEPTOR(char*, strchrnul
, char *s
, int c
) {
516 SCOPED_TSAN_INTERCEPTOR(strchrnul
, s
, c
);
517 char *res
= REAL(strchrnul
)(s
, c
);
518 uptr len
= (char*)res
- (char*)s
+ 1;
519 MemoryAccessRange(thr
, pc
, (uptr
)s
, len
, false);
523 TSAN_INTERCEPTOR(char*, strrchr
, char *s
, int c
) {
524 SCOPED_TSAN_INTERCEPTOR(strrchr
, s
, c
);
525 MemoryAccessRange(thr
, pc
, (uptr
)s
, internal_strlen(s
) + 1, false);
526 return REAL(strrchr
)(s
, c
);
529 TSAN_INTERCEPTOR(char*, strcpy
, char *dst
, const char *src
) { // NOLINT
530 SCOPED_TSAN_INTERCEPTOR(strcpy
, dst
, src
); // NOLINT
531 uptr srclen
= internal_strlen(src
);
532 MemoryAccessRange(thr
, pc
, (uptr
)dst
, srclen
+ 1, true);
533 MemoryAccessRange(thr
, pc
, (uptr
)src
, srclen
+ 1, false);
534 return REAL(strcpy
)(dst
, src
); // NOLINT
537 TSAN_INTERCEPTOR(char*, strncpy
, char *dst
, char *src
, uptr n
) {
538 SCOPED_TSAN_INTERCEPTOR(strncpy
, dst
, src
, n
);
539 uptr srclen
= internal_strnlen(src
, n
);
540 MemoryAccessRange(thr
, pc
, (uptr
)dst
, n
, true);
541 MemoryAccessRange(thr
, pc
, (uptr
)src
, min(srclen
+ 1, n
), false);
542 return REAL(strncpy
)(dst
, src
, n
);
545 TSAN_INTERCEPTOR(const char*, strstr
, const char *s1
, const char *s2
) {
546 SCOPED_TSAN_INTERCEPTOR(strstr
, s1
, s2
);
547 const char *res
= REAL(strstr
)(s1
, s2
);
548 uptr len1
= internal_strlen(s1
);
549 uptr len2
= internal_strlen(s2
);
550 MemoryAccessRange(thr
, pc
, (uptr
)s1
, len1
+ 1, false);
551 MemoryAccessRange(thr
, pc
, (uptr
)s2
, len2
+ 1, false);
555 static bool fix_mmap_addr(void **addr
, long_t sz
, int flags
) {
557 if (!IsAppMem((uptr
)*addr
) || !IsAppMem((uptr
)*addr
+ sz
- 1)) {
558 if (flags
& MAP_FIXED
) {
569 TSAN_INTERCEPTOR(void*, mmap
, void *addr
, long_t sz
, int prot
,
570 int flags
, int fd
, unsigned off
) {
571 SCOPED_TSAN_INTERCEPTOR(mmap
, addr
, sz
, prot
, flags
, fd
, off
);
572 if (!fix_mmap_addr(&addr
, sz
, flags
))
574 void *res
= REAL(mmap
)(addr
, sz
, prot
, flags
, fd
, off
);
575 if (res
!= MAP_FAILED
) {
576 MemoryResetRange(thr
, pc
, (uptr
)res
, sz
);
581 TSAN_INTERCEPTOR(void*, mmap64
, void *addr
, long_t sz
, int prot
,
582 int flags
, int fd
, u64 off
) {
583 SCOPED_TSAN_INTERCEPTOR(mmap64
, addr
, sz
, prot
, flags
, fd
, off
);
584 if (!fix_mmap_addr(&addr
, sz
, flags
))
586 void *res
= REAL(mmap64
)(addr
, sz
, prot
, flags
, fd
, off
);
587 if (res
!= MAP_FAILED
) {
588 MemoryResetRange(thr
, pc
, (uptr
)res
, sz
);
593 TSAN_INTERCEPTOR(int, munmap
, void *addr
, long_t sz
) {
594 SCOPED_TSAN_INTERCEPTOR(munmap
, addr
, sz
);
595 int res
= REAL(munmap
)(addr
, sz
);
599 TSAN_INTERCEPTOR(void*, memalign
, uptr align
, uptr sz
) {
600 SCOPED_TSAN_INTERCEPTOR(memalign
, align
, sz
);
601 return user_alloc(thr
, pc
, sz
, align
);
604 TSAN_INTERCEPTOR(void*, valloc
, uptr sz
) {
605 SCOPED_TSAN_INTERCEPTOR(valloc
, sz
);
606 return user_alloc(thr
, pc
, sz
, GetPageSizeCached());
609 TSAN_INTERCEPTOR(void*, pvalloc
, uptr sz
) {
610 SCOPED_TSAN_INTERCEPTOR(pvalloc
, sz
);
611 sz
= RoundUp(sz
, GetPageSizeCached());
612 return user_alloc(thr
, pc
, sz
, GetPageSizeCached());
615 TSAN_INTERCEPTOR(int, posix_memalign
, void **memptr
, uptr align
, uptr sz
) {
616 SCOPED_TSAN_INTERCEPTOR(posix_memalign
, memptr
, align
, sz
);
617 *memptr
= user_alloc(thr
, pc
, sz
, align
);
621 // Used in thread-safe function static initialization.
622 extern "C" int INTERFACE_ATTRIBUTE
__cxa_guard_acquire(atomic_uint32_t
*g
) {
623 SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire
, g
);
625 u32 cmp
= atomic_load(g
, memory_order_acquire
);
627 if (atomic_compare_exchange_strong(g
, &cmp
, 1<<16, memory_order_relaxed
))
629 } else if (cmp
== 1) {
630 Acquire(thr
, pc
, (uptr
)g
);
633 internal_sched_yield();
638 extern "C" void INTERFACE_ATTRIBUTE
__cxa_guard_release(atomic_uint32_t
*g
) {
639 SCOPED_INTERCEPTOR_RAW(__cxa_guard_release
, g
);
640 Release(thr
, pc
, (uptr
)g
);
641 atomic_store(g
, 1, memory_order_release
);
644 extern "C" void INTERFACE_ATTRIBUTE
__cxa_guard_abort(atomic_uint32_t
*g
) {
645 SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort
, g
);
646 atomic_store(g
, 0, memory_order_relaxed
);
649 static void thread_finalize(void *v
) {
652 if (pthread_setspecific(g_thread_finalize_key
, (void*)(iter
- 1))) {
653 Printf("ThreadSanitizer: failed to set thread key\n");
660 ThreadState
*thr
= cur_thread();
662 SignalContext
*sctx
= thr
->signal_ctx
;
672 void* (*callback
)(void *arg
);
674 atomic_uintptr_t tid
;
677 extern "C" void *__tsan_thread_start_func(void *arg
) {
678 ThreadParam
*p
= (ThreadParam
*)arg
;
679 void* (*callback
)(void *arg
) = p
->callback
;
680 void *param
= p
->param
;
683 ThreadState
*thr
= cur_thread();
685 if (pthread_setspecific(g_thread_finalize_key
, (void*)4)) {
686 Printf("ThreadSanitizer: failed to set thread key\n");
689 while ((tid
= atomic_load(&p
->tid
, memory_order_acquire
)) == 0)
691 atomic_store(&p
->tid
, 0, memory_order_release
);
692 ThreadStart(thr
, tid
, GetTid());
693 CHECK_EQ(thr
->in_rtl
, 1);
695 void *res
= callback(param
);
696 // Prevent the callback from being tail called,
697 // it mixes up stack traces.
698 volatile int foo
= 42;
703 TSAN_INTERCEPTOR(int, pthread_create
,
704 void *th
, void *attr
, void *(*callback
)(void*), void * param
) {
705 SCOPED_TSAN_INTERCEPTOR(pthread_create
, th
, attr
, callback
, param
);
706 pthread_attr_t myattr
;
708 pthread_attr_init(&myattr
);
712 pthread_attr_getdetachstate(attr
, &detached
);
714 pthread_attr_getstacksize(attr
, &stacksize
);
715 // We place the huge ThreadState object into TLS, account for that.
716 const uptr minstacksize
= GetTlsSize() + 128*1024;
717 if (stacksize
< minstacksize
) {
718 DPrintf("ThreadSanitizer: stacksize %zu->%zu\n", stacksize
, minstacksize
);
719 pthread_attr_setstacksize(attr
, minstacksize
);
722 p
.callback
= callback
;
724 atomic_store(&p
.tid
, 0, memory_order_relaxed
);
725 int res
= REAL(pthread_create
)(th
, attr
, __tsan_thread_start_func
, &p
);
727 int tid
= ThreadCreate(thr
, pc
, *(uptr
*)th
, detached
);
729 atomic_store(&p
.tid
, tid
, memory_order_release
);
730 while (atomic_load(&p
.tid
, memory_order_acquire
) != 0)
734 pthread_attr_destroy(&myattr
);
738 TSAN_INTERCEPTOR(int, pthread_join
, void *th
, void **ret
) {
739 SCOPED_TSAN_INTERCEPTOR(pthread_join
, th
, ret
);
740 int tid
= ThreadTid(thr
, pc
, (uptr
)th
);
741 int res
= BLOCK_REAL(pthread_join
)(th
, ret
);
743 ThreadJoin(thr
, pc
, tid
);
748 TSAN_INTERCEPTOR(int, pthread_detach
, void *th
) {
749 SCOPED_TSAN_INTERCEPTOR(pthread_detach
, th
);
750 int tid
= ThreadTid(thr
, pc
, (uptr
)th
);
751 int res
= REAL(pthread_detach
)(th
);
753 ThreadDetach(thr
, pc
, tid
);
758 TSAN_INTERCEPTOR(int, pthread_mutex_init
, void *m
, void *a
) {
759 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init
, m
, a
);
760 int res
= REAL(pthread_mutex_init
)(m
, a
);
762 bool recursive
= false;
765 if (pthread_mutexattr_gettype(a
, &type
) == 0)
766 recursive
= (type
== PTHREAD_MUTEX_RECURSIVE
767 || type
== PTHREAD_MUTEX_RECURSIVE_NP
);
769 MutexCreate(thr
, pc
, (uptr
)m
, false, recursive
, false);
774 TSAN_INTERCEPTOR(int, pthread_mutex_destroy
, void *m
) {
775 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy
, m
);
776 int res
= REAL(pthread_mutex_destroy
)(m
);
777 if (res
== 0 || res
== EBUSY
) {
778 MutexDestroy(thr
, pc
, (uptr
)m
);
783 TSAN_INTERCEPTOR(int, pthread_mutex_lock
, void *m
) {
784 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock
, m
);
785 int res
= REAL(pthread_mutex_lock
)(m
);
787 MutexLock(thr
, pc
, (uptr
)m
);
792 TSAN_INTERCEPTOR(int, pthread_mutex_trylock
, void *m
) {
793 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock
, m
);
794 int res
= REAL(pthread_mutex_trylock
)(m
);
796 MutexLock(thr
, pc
, (uptr
)m
);
801 TSAN_INTERCEPTOR(int, pthread_mutex_timedlock
, void *m
, void *abstime
) {
802 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock
, m
, abstime
);
803 int res
= REAL(pthread_mutex_timedlock
)(m
, abstime
);
805 MutexLock(thr
, pc
, (uptr
)m
);
810 TSAN_INTERCEPTOR(int, pthread_mutex_unlock
, void *m
) {
811 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_unlock
, m
);
812 MutexUnlock(thr
, pc
, (uptr
)m
);
813 int res
= REAL(pthread_mutex_unlock
)(m
);
817 TSAN_INTERCEPTOR(int, pthread_spin_init
, void *m
, int pshared
) {
818 SCOPED_TSAN_INTERCEPTOR(pthread_spin_init
, m
, pshared
);
819 int res
= REAL(pthread_spin_init
)(m
, pshared
);
821 MutexCreate(thr
, pc
, (uptr
)m
, false, false, false);
826 TSAN_INTERCEPTOR(int, pthread_spin_destroy
, void *m
) {
827 SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy
, m
);
828 int res
= REAL(pthread_spin_destroy
)(m
);
830 MutexDestroy(thr
, pc
, (uptr
)m
);
835 TSAN_INTERCEPTOR(int, pthread_spin_lock
, void *m
) {
836 SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock
, m
);
837 int res
= REAL(pthread_spin_lock
)(m
);
839 MutexLock(thr
, pc
, (uptr
)m
);
844 TSAN_INTERCEPTOR(int, pthread_spin_trylock
, void *m
) {
845 SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock
, m
);
846 int res
= REAL(pthread_spin_trylock
)(m
);
848 MutexLock(thr
, pc
, (uptr
)m
);
853 TSAN_INTERCEPTOR(int, pthread_spin_unlock
, void *m
) {
854 SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock
, m
);
855 MutexUnlock(thr
, pc
, (uptr
)m
);
856 int res
= REAL(pthread_spin_unlock
)(m
);
860 TSAN_INTERCEPTOR(int, pthread_rwlock_init
, void *m
, void *a
) {
861 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init
, m
, a
);
862 int res
= REAL(pthread_rwlock_init
)(m
, a
);
864 MutexCreate(thr
, pc
, (uptr
)m
, true, false, false);
869 TSAN_INTERCEPTOR(int, pthread_rwlock_destroy
, void *m
) {
870 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy
, m
);
871 int res
= REAL(pthread_rwlock_destroy
)(m
);
873 MutexDestroy(thr
, pc
, (uptr
)m
);
878 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock
, void *m
) {
879 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock
, m
);
880 int res
= REAL(pthread_rwlock_rdlock
)(m
);
882 MutexReadLock(thr
, pc
, (uptr
)m
);
887 TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock
, void *m
) {
888 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock
, m
);
889 int res
= REAL(pthread_rwlock_tryrdlock
)(m
);
891 MutexReadLock(thr
, pc
, (uptr
)m
);
896 TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock
, void *m
, void *abstime
) {
897 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock
, m
, abstime
);
898 int res
= REAL(pthread_rwlock_timedrdlock
)(m
, abstime
);
900 MutexReadLock(thr
, pc
, (uptr
)m
);
905 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock
, void *m
) {
906 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock
, m
);
907 int res
= REAL(pthread_rwlock_wrlock
)(m
);
909 MutexLock(thr
, pc
, (uptr
)m
);
914 TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock
, void *m
) {
915 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock
, m
);
916 int res
= REAL(pthread_rwlock_trywrlock
)(m
);
918 MutexLock(thr
, pc
, (uptr
)m
);
923 TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock
, void *m
, void *abstime
) {
924 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock
, m
, abstime
);
925 int res
= REAL(pthread_rwlock_timedwrlock
)(m
, abstime
);
927 MutexLock(thr
, pc
, (uptr
)m
);
932 TSAN_INTERCEPTOR(int, pthread_rwlock_unlock
, void *m
) {
933 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock
, m
);
934 MutexReadOrWriteUnlock(thr
, pc
, (uptr
)m
);
935 int res
= REAL(pthread_rwlock_unlock
)(m
);
939 TSAN_INTERCEPTOR(int, pthread_cond_init
, void *c
, void *a
) {
940 SCOPED_TSAN_INTERCEPTOR(pthread_cond_init
, c
, a
);
941 int res
= REAL(pthread_cond_init
)(c
, a
);
945 TSAN_INTERCEPTOR(int, pthread_cond_destroy
, void *c
) {
946 SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy
, c
);
947 int res
= REAL(pthread_cond_destroy
)(c
);
951 TSAN_INTERCEPTOR(int, pthread_cond_signal
, void *c
) {
952 SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal
, c
);
953 int res
= REAL(pthread_cond_signal
)(c
);
957 TSAN_INTERCEPTOR(int, pthread_cond_broadcast
, void *c
) {
958 SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast
, c
);
959 int res
= REAL(pthread_cond_broadcast
)(c
);
963 TSAN_INTERCEPTOR(int, pthread_cond_wait
, void *c
, void *m
) {
964 SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait
, c
, m
);
965 MutexUnlock(thr
, pc
, (uptr
)m
);
966 int res
= REAL(pthread_cond_wait
)(c
, m
);
967 MutexLock(thr
, pc
, (uptr
)m
);
971 TSAN_INTERCEPTOR(int, pthread_cond_timedwait
, void *c
, void *m
, void *abstime
) {
972 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait
, c
, m
, abstime
);
973 MutexUnlock(thr
, pc
, (uptr
)m
);
974 int res
= REAL(pthread_cond_timedwait
)(c
, m
, abstime
);
975 MutexLock(thr
, pc
, (uptr
)m
);
979 TSAN_INTERCEPTOR(int, pthread_barrier_init
, void *b
, void *a
, unsigned count
) {
980 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init
, b
, a
, count
);
981 MemoryWrite1Byte(thr
, pc
, (uptr
)b
);
982 int res
= REAL(pthread_barrier_init
)(b
, a
, count
);
986 TSAN_INTERCEPTOR(int, pthread_barrier_destroy
, void *b
) {
987 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy
, b
);
988 MemoryWrite1Byte(thr
, pc
, (uptr
)b
);
989 int res
= REAL(pthread_barrier_destroy
)(b
);
993 TSAN_INTERCEPTOR(int, pthread_barrier_wait
, void *b
) {
994 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait
, b
);
995 Release(thr
, pc
, (uptr
)b
);
996 MemoryRead1Byte(thr
, pc
, (uptr
)b
);
997 int res
= REAL(pthread_barrier_wait
)(b
);
998 MemoryRead1Byte(thr
, pc
, (uptr
)b
);
999 if (res
== 0 || res
== PTHREAD_BARRIER_SERIAL_THREAD
) {
1000 Acquire(thr
, pc
, (uptr
)b
);
1005 TSAN_INTERCEPTOR(int, pthread_once
, void *o
, void (*f
)()) {
1006 SCOPED_TSAN_INTERCEPTOR(pthread_once
, o
, f
);
1007 if (o
== 0 || f
== 0)
1009 atomic_uint32_t
*a
= static_cast<atomic_uint32_t
*>(o
);
1010 u32 v
= atomic_load(a
, memory_order_acquire
);
1011 if (v
== 0 && atomic_compare_exchange_strong(a
, &v
, 1,
1012 memory_order_relaxed
)) {
1013 const int old_in_rtl
= thr
->in_rtl
;
1016 CHECK_EQ(thr
->in_rtl
, 0);
1017 thr
->in_rtl
= old_in_rtl
;
1018 Release(thr
, pc
, (uptr
)o
);
1019 atomic_store(a
, 2, memory_order_release
);
1023 v
= atomic_load(a
, memory_order_acquire
);
1025 Acquire(thr
, pc
, (uptr
)o
);
1030 TSAN_INTERCEPTOR(int, sem_init
, void *s
, int pshared
, unsigned value
) {
1031 SCOPED_TSAN_INTERCEPTOR(sem_init
, s
, pshared
, value
);
1032 int res
= REAL(sem_init
)(s
, pshared
, value
);
1036 TSAN_INTERCEPTOR(int, sem_destroy
, void *s
) {
1037 SCOPED_TSAN_INTERCEPTOR(sem_destroy
, s
);
1038 int res
= REAL(sem_destroy
)(s
);
1042 TSAN_INTERCEPTOR(int, sem_wait
, void *s
) {
1043 SCOPED_TSAN_INTERCEPTOR(sem_wait
, s
);
1044 int res
= BLOCK_REAL(sem_wait
)(s
);
1046 Acquire(thr
, pc
, (uptr
)s
);
1051 TSAN_INTERCEPTOR(int, sem_trywait
, void *s
) {
1052 SCOPED_TSAN_INTERCEPTOR(sem_trywait
, s
);
1053 int res
= BLOCK_REAL(sem_trywait
)(s
);
1055 Acquire(thr
, pc
, (uptr
)s
);
1060 TSAN_INTERCEPTOR(int, sem_timedwait
, void *s
, void *abstime
) {
1061 SCOPED_TSAN_INTERCEPTOR(sem_timedwait
, s
, abstime
);
1062 int res
= BLOCK_REAL(sem_timedwait
)(s
, abstime
);
1064 Acquire(thr
, pc
, (uptr
)s
);
1069 TSAN_INTERCEPTOR(int, sem_post
, void *s
) {
1070 SCOPED_TSAN_INTERCEPTOR(sem_post
, s
);
1071 Release(thr
, pc
, (uptr
)s
);
1072 int res
= REAL(sem_post
)(s
);
1076 TSAN_INTERCEPTOR(int, sem_getvalue
, void *s
, int *sval
) {
1077 SCOPED_TSAN_INTERCEPTOR(sem_getvalue
, s
, sval
);
1078 int res
= REAL(sem_getvalue
)(s
, sval
);
1080 Acquire(thr
, pc
, (uptr
)s
);
1085 TSAN_INTERCEPTOR(long_t
, read
, int fd
, void *buf
, long_t sz
) {
1086 SCOPED_TSAN_INTERCEPTOR(read
, fd
, buf
, sz
);
1087 int res
= REAL(read
)(fd
, buf
, sz
);
1089 Acquire(thr
, pc
, fd2addr(fd
));
1094 TSAN_INTERCEPTOR(long_t
, pread
, int fd
, void *buf
, long_t sz
, unsigned off
) {
1095 SCOPED_TSAN_INTERCEPTOR(pread
, fd
, buf
, sz
, off
);
1096 int res
= REAL(pread
)(fd
, buf
, sz
, off
);
1098 Acquire(thr
, pc
, fd2addr(fd
));
1103 TSAN_INTERCEPTOR(long_t
, pread64
, int fd
, void *buf
, long_t sz
, u64 off
) {
1104 SCOPED_TSAN_INTERCEPTOR(pread64
, fd
, buf
, sz
, off
);
1105 int res
= REAL(pread64
)(fd
, buf
, sz
, off
);
1107 Acquire(thr
, pc
, fd2addr(fd
));
1112 TSAN_INTERCEPTOR(long_t
, readv
, int fd
, void *vec
, int cnt
) {
1113 SCOPED_TSAN_INTERCEPTOR(readv
, fd
, vec
, cnt
);
1114 int res
= REAL(readv
)(fd
, vec
, cnt
);
1116 Acquire(thr
, pc
, fd2addr(fd
));
1121 TSAN_INTERCEPTOR(long_t
, preadv64
, int fd
, void *vec
, int cnt
, u64 off
) {
1122 SCOPED_TSAN_INTERCEPTOR(preadv64
, fd
, vec
, cnt
, off
);
1123 int res
= REAL(preadv64
)(fd
, vec
, cnt
, off
);
1125 Acquire(thr
, pc
, fd2addr(fd
));
1130 TSAN_INTERCEPTOR(long_t
, write
, int fd
, void *buf
, long_t sz
) {
1131 SCOPED_TSAN_INTERCEPTOR(write
, fd
, buf
, sz
);
1132 Release(thr
, pc
, fd2addr(fd
));
1133 int res
= REAL(write
)(fd
, buf
, sz
);
1137 TSAN_INTERCEPTOR(long_t
, pwrite
, int fd
, void *buf
, long_t sz
, unsigned off
) {
1138 SCOPED_TSAN_INTERCEPTOR(pwrite
, fd
, buf
, sz
, off
);
1139 Release(thr
, pc
, fd2addr(fd
));
1140 int res
= REAL(pwrite
)(fd
, buf
, sz
, off
);
1144 TSAN_INTERCEPTOR(long_t
, pwrite64
, int fd
, void *buf
, long_t sz
, u64 off
) {
1145 SCOPED_TSAN_INTERCEPTOR(pwrite64
, fd
, buf
, sz
, off
);
1146 Release(thr
, pc
, fd2addr(fd
));
1147 int res
= REAL(pwrite64
)(fd
, buf
, sz
, off
);
1151 TSAN_INTERCEPTOR(long_t
, writev
, int fd
, void *vec
, int cnt
) {
1152 SCOPED_TSAN_INTERCEPTOR(writev
, fd
, vec
, cnt
);
1153 Release(thr
, pc
, fd2addr(fd
));
1154 int res
= REAL(writev
)(fd
, vec
, cnt
);
1158 TSAN_INTERCEPTOR(long_t
, pwritev64
, int fd
, void *vec
, int cnt
, u64 off
) {
1159 SCOPED_TSAN_INTERCEPTOR(pwritev64
, fd
, vec
, cnt
, off
);
1160 Release(thr
, pc
, fd2addr(fd
));
1161 int res
= REAL(pwritev64
)(fd
, vec
, cnt
, off
);
1165 TSAN_INTERCEPTOR(long_t
, send
, int fd
, void *buf
, long_t len
, int flags
) {
1166 SCOPED_TSAN_INTERCEPTOR(send
, fd
, buf
, len
, flags
);
1167 Release(thr
, pc
, fd2addr(fd
));
1168 int res
= REAL(send
)(fd
, buf
, len
, flags
);
1172 TSAN_INTERCEPTOR(long_t
, sendmsg
, int fd
, void *msg
, int flags
) {
1173 SCOPED_TSAN_INTERCEPTOR(sendmsg
, fd
, msg
, flags
);
1174 Release(thr
, pc
, fd2addr(fd
));
1175 int res
= REAL(sendmsg
)(fd
, msg
, flags
);
1179 TSAN_INTERCEPTOR(long_t
, recv
, int fd
, void *buf
, long_t len
, int flags
) {
1180 SCOPED_TSAN_INTERCEPTOR(recv
, fd
, buf
, len
, flags
);
1181 int res
= REAL(recv
)(fd
, buf
, len
, flags
);
1183 Acquire(thr
, pc
, fd2addr(fd
));
1188 TSAN_INTERCEPTOR(long_t
, recvmsg
, int fd
, void *msg
, int flags
) {
1189 SCOPED_TSAN_INTERCEPTOR(recvmsg
, fd
, msg
, flags
);
1190 int res
= REAL(recvmsg
)(fd
, msg
, flags
);
1192 Acquire(thr
, pc
, fd2addr(fd
));
1197 TSAN_INTERCEPTOR(int, unlink
, char *path
) {
1198 SCOPED_TSAN_INTERCEPTOR(unlink
, path
);
1199 Release(thr
, pc
, file2addr(path
));
1200 int res
= REAL(unlink
)(path
);
1204 TSAN_INTERCEPTOR(void*, fopen
, char *path
, char *mode
) {
1205 SCOPED_TSAN_INTERCEPTOR(fopen
, path
, mode
);
1206 void *res
= REAL(fopen
)(path
, mode
);
1207 Acquire(thr
, pc
, file2addr(path
));
1211 TSAN_INTERCEPTOR(uptr
, fread
, void *ptr
, uptr size
, uptr nmemb
, void *f
) {
1212 SCOPED_TSAN_INTERCEPTOR(fread
, ptr
, size
, nmemb
, f
);
1213 MemoryAccessRange(thr
, pc
, (uptr
)ptr
, size
* nmemb
, true);
1214 return REAL(fread
)(ptr
, size
, nmemb
, f
);
1217 TSAN_INTERCEPTOR(uptr
, fwrite
, const void *p
, uptr size
, uptr nmemb
, void *f
) {
1218 SCOPED_TSAN_INTERCEPTOR(fwrite
, p
, size
, nmemb
, f
);
1219 MemoryAccessRange(thr
, pc
, (uptr
)p
, size
* nmemb
, false);
1220 return REAL(fwrite
)(p
, size
, nmemb
, f
);
1223 TSAN_INTERCEPTOR(int, puts
, const char *s
) {
1224 SCOPED_TSAN_INTERCEPTOR(puts
, s
);
1225 MemoryAccessRange(thr
, pc
, (uptr
)s
, internal_strlen(s
), false);
1226 return REAL(puts
)(s
);
1229 TSAN_INTERCEPTOR(int, rmdir
, char *path
) {
1230 SCOPED_TSAN_INTERCEPTOR(rmdir
, path
);
1231 Release(thr
, pc
, dir2addr(path
));
1232 int res
= REAL(rmdir
)(path
);
1236 TSAN_INTERCEPTOR(void*, opendir
, char *path
) {
1237 SCOPED_TSAN_INTERCEPTOR(opendir
, path
);
1238 void *res
= REAL(opendir
)(path
);
1239 Acquire(thr
, pc
, dir2addr(path
));
1243 TSAN_INTERCEPTOR(int, epoll_ctl
, int epfd
, int op
, int fd
, void *ev
) {
1244 SCOPED_TSAN_INTERCEPTOR(epoll_ctl
, epfd
, op
, fd
, ev
);
1245 if (op
== EPOLL_CTL_ADD
) {
1246 Release(thr
, pc
, epollfd2addr(epfd
));
1248 int res
= REAL(epoll_ctl
)(epfd
, op
, fd
, ev
);
1252 TSAN_INTERCEPTOR(int, epoll_wait
, int epfd
, void *ev
, int cnt
, int timeout
) {
1253 SCOPED_TSAN_INTERCEPTOR(epoll_wait
, epfd
, ev
, cnt
, timeout
);
1254 int res
= BLOCK_REAL(epoll_wait
)(epfd
, ev
, cnt
, timeout
);
1256 Acquire(thr
, pc
, epollfd2addr(epfd
));
1261 TSAN_INTERCEPTOR(int, poll
, void *fds
, long_t nfds
, int timeout
) {
1262 SCOPED_TSAN_INTERCEPTOR(poll
, fds
, nfds
, timeout
);
1263 int res
= BLOCK_REAL(poll
)(fds
, nfds
, timeout
);
1267 static void ALWAYS_INLINE
rtl_generic_sighandler(bool sigact
, int sig
,
1268 my_siginfo_t
*info
, void *ctx
) {
1269 ThreadState
*thr
= cur_thread();
1270 SignalContext
*sctx
= SigCtx(thr
);
1271 // Don't mess with synchronous signals.
1272 if (sig
== SIGSEGV
|| sig
== SIGBUS
|| sig
== SIGILL
||
1273 sig
== SIGABRT
|| sig
== SIGFPE
|| sig
== SIGPIPE
||
1274 // If we are sending signal to ourselves, we must process it now.
1275 (sctx
&& sig
== sctx
->int_signal_send
) ||
1276 // If we are in blocking function, we can safely process it now
1277 // (but check if we are in a recursive interceptor,
1278 // i.e. pthread_join()->munmap()).
1279 (sctx
&& sctx
->in_blocking_func
== 1 && thr
->in_rtl
== 1)) {
1280 CHECK(thr
->in_rtl
== 0 || thr
->in_rtl
== 1);
1281 int in_rtl
= thr
->in_rtl
;
1283 CHECK_EQ(thr
->in_signal_handler
, false);
1284 thr
->in_signal_handler
= true;
1286 sigactions
[sig
].sa_sigaction(sig
, info
, ctx
);
1288 sigactions
[sig
].sa_handler(sig
);
1289 CHECK_EQ(thr
->in_signal_handler
, true);
1290 thr
->in_signal_handler
= false;
1291 thr
->in_rtl
= in_rtl
;
1297 SignalDesc
*signal
= &sctx
->pending_signals
[sig
];
1298 if (signal
->armed
== false) {
1299 signal
->armed
= true;
1300 signal
->sigaction
= sigact
;
1302 internal_memcpy(&signal
->siginfo
, info
, sizeof(*info
));
1304 internal_memcpy(&signal
->ctx
, ctx
, sizeof(signal
->ctx
));
1305 sctx
->pending_signal_count
++;
1309 static void rtl_sighandler(int sig
) {
1310 rtl_generic_sighandler(false, sig
, 0, 0);
1313 static void rtl_sigaction(int sig
, my_siginfo_t
*info
, void *ctx
) {
1314 rtl_generic_sighandler(true, sig
, info
, ctx
);
1317 TSAN_INTERCEPTOR(int, sigaction
, int sig
, sigaction_t
*act
, sigaction_t
*old
) {
1318 SCOPED_TSAN_INTERCEPTOR(sigaction
, sig
, act
, old
);
1320 internal_memcpy(old
, &sigactions
[sig
], sizeof(*old
));
1323 internal_memcpy(&sigactions
[sig
], act
, sizeof(*act
));
1325 internal_memcpy(&newact
, act
, sizeof(newact
));
1326 sigfillset(&newact
.sa_mask
);
1327 if (act
->sa_handler
!= SIG_IGN
&& act
->sa_handler
!= SIG_DFL
) {
1328 if (newact
.sa_flags
& SA_SIGINFO
)
1329 newact
.sa_sigaction
= rtl_sigaction
;
1331 newact
.sa_handler
= rtl_sighandler
;
1333 int res
= REAL(sigaction
)(sig
, &newact
, 0);
1337 TSAN_INTERCEPTOR(sighandler_t
, signal
, int sig
, sighandler_t h
) {
1340 REAL(memset
)(&act
.sa_mask
, -1, sizeof(act
.sa_mask
));
1343 int res
= sigaction(sig
, &act
, &old
);
1346 return old
.sa_handler
;
1349 TSAN_INTERCEPTOR(int, raise
, int sig
) {
1350 SCOPED_TSAN_INTERCEPTOR(raise
, sig
);
1351 SignalContext
*sctx
= SigCtx(thr
);
1353 int prev
= sctx
->int_signal_send
;
1354 sctx
->int_signal_send
= sig
;
1355 int res
= REAL(raise
)(sig
);
1356 CHECK_EQ(sctx
->int_signal_send
, sig
);
1357 sctx
->int_signal_send
= prev
;
1361 TSAN_INTERCEPTOR(int, kill
, int pid
, int sig
) {
1362 SCOPED_TSAN_INTERCEPTOR(kill
, pid
, sig
);
1363 SignalContext
*sctx
= SigCtx(thr
);
1365 int prev
= sctx
->int_signal_send
;
1366 if (pid
== GetPid()) {
1367 sctx
->int_signal_send
= sig
;
1369 int res
= REAL(kill
)(pid
, sig
);
1370 if (pid
== GetPid()) {
1371 CHECK_EQ(sctx
->int_signal_send
, sig
);
1372 sctx
->int_signal_send
= prev
;
1377 TSAN_INTERCEPTOR(int, pthread_kill
, void *tid
, int sig
) {
1378 SCOPED_TSAN_INTERCEPTOR(pthread_kill
, tid
, sig
);
1379 SignalContext
*sctx
= SigCtx(thr
);
1381 int prev
= sctx
->int_signal_send
;
1382 if (tid
== pthread_self()) {
1383 sctx
->int_signal_send
= sig
;
1385 int res
= REAL(pthread_kill
)(tid
, sig
);
1386 if (tid
== pthread_self()) {
1387 CHECK_EQ(sctx
->int_signal_send
, sig
);
1388 sctx
->int_signal_send
= prev
;
1393 TSAN_INTERCEPTOR(int, gettimeofday
, void *tv
, void *tz
) {
1394 SCOPED_TSAN_INTERCEPTOR(gettimeofday
, tv
, tz
);
1395 // It's intercepted merely to process pending signals.
1396 return REAL(gettimeofday
)(tv
, tz
);
1399 // Linux kernel has a bug that leads to kernel deadlock if a process
1400 // maps TBs of memory and then calls mlock().
1401 static void MlockIsUnsupported() {
1402 static atomic_uint8_t printed
;
1403 if (atomic_exchange(&printed
, 1, memory_order_relaxed
))
1405 Printf("INFO: ThreadSanitizer ignores mlock/mlockall/munlock/munlockall\n");
1408 TSAN_INTERCEPTOR(int, mlock
, const void *addr
, uptr len
) {
1409 MlockIsUnsupported();
1413 TSAN_INTERCEPTOR(int, munlock
, const void *addr
, uptr len
) {
1414 MlockIsUnsupported();
1418 TSAN_INTERCEPTOR(int, mlockall
, int flags
) {
1419 MlockIsUnsupported();
1423 TSAN_INTERCEPTOR(int, munlockall
, void) {
1424 MlockIsUnsupported();
1430 void ProcessPendingSignals(ThreadState
*thr
) {
1431 CHECK_EQ(thr
->in_rtl
, 0);
1432 SignalContext
*sctx
= SigCtx(thr
);
1433 if (sctx
== 0 || sctx
->pending_signal_count
== 0 || thr
->in_signal_handler
)
1435 Context
*ctx
= CTX();
1436 thr
->in_signal_handler
= true;
1437 sctx
->pending_signal_count
= 0;
1438 // These are too big for stack.
1439 static THREADLOCAL sigset_t emptyset
, oldset
;
1440 sigfillset(&emptyset
);
1441 pthread_sigmask(SIG_SETMASK
, &emptyset
, &oldset
);
1442 for (int sig
= 0; sig
< kSigCount
; sig
++) {
1443 SignalDesc
*signal
= &sctx
->pending_signals
[sig
];
1444 if (signal
->armed
) {
1445 signal
->armed
= false;
1446 if (sigactions
[sig
].sa_handler
!= SIG_DFL
1447 && sigactions
[sig
].sa_handler
!= SIG_IGN
) {
1448 // Insure that the handler does not spoil errno.
1449 const int saved_errno
= errno
;
1451 if (signal
->sigaction
)
1452 sigactions
[sig
].sa_sigaction(sig
, &signal
->siginfo
, &signal
->ctx
);
1454 sigactions
[sig
].sa_handler(sig
);
1455 if (flags()->report_bugs
&& errno
!= 0) {
1457 __tsan::StackTrace stack
;
1458 uptr pc
= signal
->sigaction
?
1459 (uptr
)sigactions
[sig
].sa_sigaction
:
1460 (uptr
)sigactions
[sig
].sa_handler
;
1462 ScopedReport
rep(ReportTypeErrnoInSignal
);
1463 if (!IsFiredSuppression(ctx
, rep
, stack
)) {
1464 rep
.AddStack(&stack
);
1465 OutputReport(ctx
, rep
, rep
.GetReport()->stacks
[0]);
1468 errno
= saved_errno
;
1472 pthread_sigmask(SIG_SETMASK
, &oldset
, 0);
1473 CHECK_EQ(thr
->in_signal_handler
, true);
1474 thr
->in_signal_handler
= false;
1477 static void unreachable() {
1478 Printf("FATAL: ThreadSanitizer: unreachable called\n");
1482 void InitializeInterceptors() {
1483 CHECK_GT(cur_thread()->in_rtl
, 0);
1485 // We need to setup it early, because functions like dlsym() can call it.
1486 REAL(memset
) = internal_memset
;
1487 REAL(memcpy
) = internal_memcpy
;
1488 REAL(memcmp
) = internal_memcmp
;
1490 TSAN_INTERCEPT(longjmp
);
1491 TSAN_INTERCEPT(siglongjmp
);
1493 TSAN_INTERCEPT(malloc
);
1494 TSAN_INTERCEPT(__libc_memalign
);
1495 TSAN_INTERCEPT(calloc
);
1496 TSAN_INTERCEPT(realloc
);
1497 TSAN_INTERCEPT(free
);
1498 TSAN_INTERCEPT(cfree
);
1499 TSAN_INTERCEPT(mmap
);
1500 TSAN_INTERCEPT(mmap64
);
1501 TSAN_INTERCEPT(munmap
);
1502 TSAN_INTERCEPT(memalign
);
1503 TSAN_INTERCEPT(valloc
);
1504 TSAN_INTERCEPT(pvalloc
);
1505 TSAN_INTERCEPT(posix_memalign
);
1507 TSAN_INTERCEPT(strlen
);
1508 TSAN_INTERCEPT(memset
);
1509 TSAN_INTERCEPT(memcpy
);
1510 TSAN_INTERCEPT(strcmp
);
1511 TSAN_INTERCEPT(memchr
);
1512 TSAN_INTERCEPT(memrchr
);
1513 TSAN_INTERCEPT(memmove
);
1514 TSAN_INTERCEPT(memcmp
);
1515 TSAN_INTERCEPT(strchr
);
1516 TSAN_INTERCEPT(strchrnul
);
1517 TSAN_INTERCEPT(strrchr
);
1518 TSAN_INTERCEPT(strncmp
);
1519 TSAN_INTERCEPT(strcpy
); // NOLINT
1520 TSAN_INTERCEPT(strncpy
);
1521 TSAN_INTERCEPT(strstr
);
1523 TSAN_INTERCEPT(pthread_create
);
1524 TSAN_INTERCEPT(pthread_join
);
1525 TSAN_INTERCEPT(pthread_detach
);
1527 TSAN_INTERCEPT(pthread_mutex_init
);
1528 TSAN_INTERCEPT(pthread_mutex_destroy
);
1529 TSAN_INTERCEPT(pthread_mutex_lock
);
1530 TSAN_INTERCEPT(pthread_mutex_trylock
);
1531 TSAN_INTERCEPT(pthread_mutex_timedlock
);
1532 TSAN_INTERCEPT(pthread_mutex_unlock
);
1534 TSAN_INTERCEPT(pthread_spin_init
);
1535 TSAN_INTERCEPT(pthread_spin_destroy
);
1536 TSAN_INTERCEPT(pthread_spin_lock
);
1537 TSAN_INTERCEPT(pthread_spin_trylock
);
1538 TSAN_INTERCEPT(pthread_spin_unlock
);
1540 TSAN_INTERCEPT(pthread_rwlock_init
);
1541 TSAN_INTERCEPT(pthread_rwlock_destroy
);
1542 TSAN_INTERCEPT(pthread_rwlock_rdlock
);
1543 TSAN_INTERCEPT(pthread_rwlock_tryrdlock
);
1544 TSAN_INTERCEPT(pthread_rwlock_timedrdlock
);
1545 TSAN_INTERCEPT(pthread_rwlock_wrlock
);
1546 TSAN_INTERCEPT(pthread_rwlock_trywrlock
);
1547 TSAN_INTERCEPT(pthread_rwlock_timedwrlock
);
1548 TSAN_INTERCEPT(pthread_rwlock_unlock
);
1550 TSAN_INTERCEPT(pthread_cond_init
);
1551 TSAN_INTERCEPT(pthread_cond_destroy
);
1552 TSAN_INTERCEPT(pthread_cond_signal
);
1553 TSAN_INTERCEPT(pthread_cond_broadcast
);
1554 TSAN_INTERCEPT(pthread_cond_wait
);
1555 TSAN_INTERCEPT(pthread_cond_timedwait
);
1557 TSAN_INTERCEPT(pthread_barrier_init
);
1558 TSAN_INTERCEPT(pthread_barrier_destroy
);
1559 TSAN_INTERCEPT(pthread_barrier_wait
);
1561 TSAN_INTERCEPT(pthread_once
);
1563 TSAN_INTERCEPT(sem_init
);
1564 TSAN_INTERCEPT(sem_destroy
);
1565 TSAN_INTERCEPT(sem_wait
);
1566 TSAN_INTERCEPT(sem_trywait
);
1567 TSAN_INTERCEPT(sem_timedwait
);
1568 TSAN_INTERCEPT(sem_post
);
1569 TSAN_INTERCEPT(sem_getvalue
);
1571 TSAN_INTERCEPT(read
);
1572 TSAN_INTERCEPT(pread
);
1573 TSAN_INTERCEPT(pread64
);
1574 TSAN_INTERCEPT(readv
);
1575 TSAN_INTERCEPT(preadv64
);
1576 TSAN_INTERCEPT(write
);
1577 TSAN_INTERCEPT(pwrite
);
1578 TSAN_INTERCEPT(pwrite64
);
1579 TSAN_INTERCEPT(writev
);
1580 TSAN_INTERCEPT(pwritev64
);
1581 TSAN_INTERCEPT(send
);
1582 TSAN_INTERCEPT(sendmsg
);
1583 TSAN_INTERCEPT(recv
);
1584 TSAN_INTERCEPT(recvmsg
);
1586 TSAN_INTERCEPT(unlink
);
1587 TSAN_INTERCEPT(fopen
);
1588 TSAN_INTERCEPT(fread
);
1589 TSAN_INTERCEPT(fwrite
);
1590 TSAN_INTERCEPT(puts
);
1591 TSAN_INTERCEPT(rmdir
);
1592 TSAN_INTERCEPT(opendir
);
1594 TSAN_INTERCEPT(epoll_ctl
);
1595 TSAN_INTERCEPT(epoll_wait
);
1596 TSAN_INTERCEPT(poll
);
1598 TSAN_INTERCEPT(sigaction
);
1599 TSAN_INTERCEPT(signal
);
1600 TSAN_INTERCEPT(raise
);
1601 TSAN_INTERCEPT(kill
);
1602 TSAN_INTERCEPT(pthread_kill
);
1603 TSAN_INTERCEPT(sleep
);
1604 TSAN_INTERCEPT(usleep
);
1605 TSAN_INTERCEPT(nanosleep
);
1606 TSAN_INTERCEPT(gettimeofday
);
1608 TSAN_INTERCEPT(mlock
);
1609 TSAN_INTERCEPT(munlock
);
1610 TSAN_INTERCEPT(mlockall
);
1611 TSAN_INTERCEPT(munlockall
);
1613 // Need to setup it, because interceptors check that the function is resolved.
1614 // But atexit is emitted directly into the module, so can't be resolved.
1615 REAL(atexit
) = (int(*)(void(*)()))unreachable
;
1616 atexit_ctx
= new(internal_alloc(MBlockAtExit
, sizeof(AtExitContext
)))
1619 if (__cxa_atexit(&finalize
, 0, 0)) {
1620 Printf("ThreadSanitizer: failed to setup atexit callback\n");
1624 if (pthread_key_create(&g_thread_finalize_key
, &thread_finalize
)) {
1625 Printf("ThreadSanitizer: failed to create thread key\n");
1630 void internal_start_thread(void(*func
)(void *arg
), void *arg
) {
1632 REAL(pthread_create
)(&th
, 0, (void*(*)(void *arg
))func
, arg
);
1633 REAL(pthread_detach
)(th
);
1636 } // namespace __tsan