tsan: detect races on fd passed to epoll_ctl
[blocksruntime.git] / lib / tsan / rtl / tsan_interceptors.cc
blob143eec643460da57f55783f3cc53119bc2897450
1 //===-- tsan_interceptors.cc ----------------------------------------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file is a part of ThreadSanitizer (TSan), a race detector.
12 // FIXME: move as many interceptors as possible into
13 // sanitizer_common/sanitizer_common_interceptors.h
14 //===----------------------------------------------------------------------===//
16 #include "sanitizer_common/sanitizer_atomic.h"
17 #include "sanitizer_common/sanitizer_libc.h"
18 #include "sanitizer_common/sanitizer_placement_new.h"
19 #include "sanitizer_common/sanitizer_stacktrace.h"
20 #include "interception/interception.h"
21 #include "tsan_interface.h"
22 #include "tsan_platform.h"
23 #include "tsan_rtl.h"
24 #include "tsan_mman.h"
25 #include "tsan_fd.h"
27 using namespace __tsan; // NOLINT
29 const int kSigCount = 128;
31 struct my_siginfo_t {
32 int opaque[128];
35 struct sigset_t {
36 u64 val[1024 / 8 / sizeof(u64)];
39 struct ucontext_t {
40 uptr opaque[117];
43 extern "C" int pthread_attr_init(void *attr);
44 extern "C" int pthread_attr_destroy(void *attr);
45 extern "C" int pthread_attr_getdetachstate(void *attr, int *v);
46 extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize);
47 extern "C" int pthread_attr_getstacksize(void *attr, uptr *stacksize);
48 extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v));
49 extern "C" int pthread_setspecific(unsigned key, const void *v);
50 extern "C" int pthread_mutexattr_gettype(void *a, int *type);
51 extern "C" int pthread_yield();
52 extern "C" int pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset);
53 extern "C" int sigfillset(sigset_t *set);
54 extern "C" void *pthread_self();
55 extern "C" void _exit(int status);
56 extern "C" int __cxa_atexit(void (*func)(void *arg), void *arg, void *dso);
57 extern "C" int *__errno_location();
58 extern "C" int fileno_unlocked(void *stream);
59 const int PTHREAD_MUTEX_RECURSIVE = 1;
60 const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
61 const int kPthreadAttrSize = 56;
62 const int EINVAL = 22;
63 const int EBUSY = 16;
64 const int EPOLL_CTL_ADD = 1;
65 const int SIGILL = 4;
66 const int SIGABRT = 6;
67 const int SIGFPE = 8;
68 const int SIGSEGV = 11;
69 const int SIGPIPE = 13;
70 const int SIGBUS = 7;
71 void *const MAP_FAILED = (void*)-1;
72 const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
73 const int MAP_FIXED = 0x10;
74 typedef long long_t; // NOLINT
76 // From /usr/include/unistd.h
77 # define F_ULOCK 0 /* Unlock a previously locked region. */
78 # define F_LOCK 1 /* Lock a region for exclusive use. */
79 # define F_TLOCK 2 /* Test and lock a region for exclusive use. */
80 # define F_TEST 3 /* Test a region for other processes locks. */
82 typedef void (*sighandler_t)(int sig);
84 #define errno (*__errno_location())
86 union pthread_attr_t {
87 char size[kPthreadAttrSize];
88 void *align;
91 struct sigaction_t {
92 union {
93 sighandler_t sa_handler;
94 void (*sa_sigaction)(int sig, my_siginfo_t *siginfo, void *uctx);
96 sigset_t sa_mask;
97 int sa_flags;
98 void (*sa_restorer)();
101 const sighandler_t SIG_DFL = (sighandler_t)0;
102 const sighandler_t SIG_IGN = (sighandler_t)1;
103 const sighandler_t SIG_ERR = (sighandler_t)-1;
104 const int SA_SIGINFO = 4;
105 const int SIG_SETMASK = 2;
107 namespace std {
108 struct nothrow_t {};
109 } // namespace std
111 static sigaction_t sigactions[kSigCount];
113 namespace __tsan {
114 struct SignalDesc {
115 bool armed;
116 bool sigaction;
117 my_siginfo_t siginfo;
118 ucontext_t ctx;
121 struct SignalContext {
122 int in_blocking_func;
123 int int_signal_send;
124 int pending_signal_count;
125 SignalDesc pending_signals[kSigCount];
129 static SignalContext *SigCtx(ThreadState *thr) {
130 SignalContext *ctx = (SignalContext*)thr->signal_ctx;
131 if (ctx == 0 && thr->is_alive) {
132 ScopedInRtl in_rtl;
133 ctx = (SignalContext*)MmapOrDie(sizeof(*ctx), "SignalContext");
134 MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx));
135 thr->signal_ctx = ctx;
137 return ctx;
140 static unsigned g_thread_finalize_key;
142 class ScopedInterceptor {
143 public:
144 ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc);
145 ~ScopedInterceptor();
146 private:
147 ThreadState *const thr_;
148 const int in_rtl_;
151 ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
152 uptr pc)
153 : thr_(thr)
154 , in_rtl_(thr->in_rtl) {
155 if (thr_->in_rtl == 0) {
156 Initialize(thr);
157 FuncEntry(thr, pc);
158 thr_->in_rtl++;
159 DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
160 } else {
161 thr_->in_rtl++;
165 ScopedInterceptor::~ScopedInterceptor() {
166 thr_->in_rtl--;
167 if (thr_->in_rtl == 0) {
168 FuncExit(thr_);
169 ProcessPendingSignals(thr_);
171 CHECK_EQ(in_rtl_, thr_->in_rtl);
174 #define SCOPED_INTERCEPTOR_RAW(func, ...) \
175 ThreadState *thr = cur_thread(); \
176 StatInc(thr, StatInterceptor); \
177 StatInc(thr, StatInt_##func); \
178 const uptr caller_pc = GET_CALLER_PC(); \
179 ScopedInterceptor si(thr, #func, caller_pc); \
180 const uptr pc = __sanitizer::StackTrace::GetPreviousInstructionPc( \
181 __sanitizer::StackTrace::GetCurrentPc()); \
182 (void)pc; \
183 /**/
185 #define SCOPED_TSAN_INTERCEPTOR(func, ...) \
186 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
187 if (REAL(func) == 0) { \
188 Printf("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
189 Die(); \
191 if (thr->in_rtl > 1) \
192 return REAL(func)(__VA_ARGS__); \
193 /**/
195 #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
196 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
198 #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name))
200 struct BlockingCall {
201 explicit BlockingCall(ThreadState *thr)
202 : ctx(SigCtx(thr)) {
203 ctx->in_blocking_func++;
206 ~BlockingCall() {
207 ctx->in_blocking_func--;
210 SignalContext *ctx;
213 TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) {
214 SCOPED_TSAN_INTERCEPTOR(sleep, sec);
215 unsigned res = BLOCK_REAL(sleep)(sec);
216 AfterSleep(thr, pc);
217 return res;
220 TSAN_INTERCEPTOR(int, usleep, long_t usec) {
221 SCOPED_TSAN_INTERCEPTOR(usleep, usec);
222 int res = BLOCK_REAL(usleep)(usec);
223 AfterSleep(thr, pc);
224 return res;
227 TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) {
228 SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem);
229 int res = BLOCK_REAL(nanosleep)(req, rem);
230 AfterSleep(thr, pc);
231 return res;
234 class AtExitContext {
235 public:
236 AtExitContext()
237 : mtx_(MutexTypeAtExit, StatMtxAtExit)
238 , pos_() {
241 typedef void(*atexit_t)();
243 int atexit(ThreadState *thr, uptr pc, atexit_t f) {
244 Lock l(&mtx_);
245 if (pos_ == kMaxAtExit)
246 return 1;
247 Release(thr, pc, (uptr)this);
248 stack_[pos_] = f;
249 pos_++;
250 return 0;
253 void exit(ThreadState *thr, uptr pc) {
254 CHECK_EQ(thr->in_rtl, 0);
255 for (;;) {
256 atexit_t f = 0;
258 Lock l(&mtx_);
259 if (pos_) {
260 pos_--;
261 f = stack_[pos_];
262 ScopedInRtl in_rtl;
263 Acquire(thr, pc, (uptr)this);
266 if (f == 0)
267 break;
268 DPrintf("#%d: executing atexit func %p\n", thr->tid, f);
269 CHECK_EQ(thr->in_rtl, 0);
270 f();
274 private:
275 static const int kMaxAtExit = 128;
276 Mutex mtx_;
277 atexit_t stack_[kMaxAtExit];
278 int pos_;
281 static AtExitContext *atexit_ctx;
283 static void finalize(void *arg) {
284 ThreadState * thr = cur_thread();
285 uptr pc = 0;
286 atexit_ctx->exit(thr, pc);
288 ScopedInRtl in_rtl;
289 DestroyAndFree(atexit_ctx);
291 int status = Finalize(cur_thread());
292 if (status)
293 _exit(status);
296 TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
297 SCOPED_TSAN_INTERCEPTOR(atexit, f);
298 return atexit_ctx->atexit(thr, pc, f);
301 TSAN_INTERCEPTOR(void, longjmp, void *env, int val) {
302 SCOPED_TSAN_INTERCEPTOR(longjmp, env, val);
303 Printf("ThreadSanitizer: longjmp() is not supported\n");
304 Die();
307 TSAN_INTERCEPTOR(void, siglongjmp, void *env, int val) {
308 SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val);
309 Printf("ThreadSanitizer: siglongjmp() is not supported\n");
310 Die();
313 TSAN_INTERCEPTOR(void*, malloc, uptr size) {
314 void *p = 0;
316 SCOPED_INTERCEPTOR_RAW(malloc, size);
317 p = user_alloc(thr, pc, size);
319 invoke_malloc_hook(p, size);
320 return p;
323 TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
324 SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz);
325 return user_alloc(thr, pc, sz, align);
328 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
329 void *p = 0;
331 SCOPED_INTERCEPTOR_RAW(calloc, size, n);
332 p = user_alloc(thr, pc, n * size);
333 if (p) internal_memset(p, 0, n * size);
335 invoke_malloc_hook(p, n * size);
336 return p;
339 TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
340 if (p)
341 invoke_free_hook(p);
343 SCOPED_INTERCEPTOR_RAW(realloc, p, size);
344 p = user_realloc(thr, pc, p, size);
346 invoke_malloc_hook(p, size);
347 return p;
350 TSAN_INTERCEPTOR(void, free, void *p) {
351 if (p == 0)
352 return;
353 invoke_free_hook(p);
354 SCOPED_INTERCEPTOR_RAW(free, p);
355 user_free(thr, pc, p);
358 TSAN_INTERCEPTOR(void, cfree, void *p) {
359 if (p == 0)
360 return;
361 invoke_free_hook(p);
362 SCOPED_INTERCEPTOR_RAW(cfree, p);
363 user_free(thr, pc, p);
366 #define OPERATOR_NEW_BODY(mangled_name) \
367 void *p = 0; \
369 SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
370 p = user_alloc(thr, pc, size); \
372 invoke_malloc_hook(p, size); \
373 return p;
375 void *operator new(__sanitizer::uptr size) {
376 OPERATOR_NEW_BODY(_Znwm);
378 void *operator new[](__sanitizer::uptr size) {
379 OPERATOR_NEW_BODY(_Znam);
381 void *operator new(__sanitizer::uptr size, std::nothrow_t const&) {
382 OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t);
384 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) {
385 OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t);
388 #define OPERATOR_DELETE_BODY(mangled_name) \
389 if (ptr == 0) return; \
390 invoke_free_hook(ptr); \
391 SCOPED_INTERCEPTOR_RAW(mangled_name, ptr); \
392 user_free(thr, pc, ptr);
394 void operator delete(void *ptr) {
395 OPERATOR_DELETE_BODY(_ZdlPv);
397 void operator delete[](void *ptr) {
398 OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t);
400 void operator delete(void *ptr, std::nothrow_t const&) {
401 OPERATOR_DELETE_BODY(_ZdaPv);
403 void operator delete[](void *ptr, std::nothrow_t const&) {
404 OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t);
407 TSAN_INTERCEPTOR(uptr, strlen, const char *s) {
408 SCOPED_TSAN_INTERCEPTOR(strlen, s);
409 uptr len = internal_strlen(s);
410 MemoryAccessRange(thr, pc, (uptr)s, len + 1, false);
411 return len;
414 TSAN_INTERCEPTOR(void*, memset, void *dst, int v, uptr size) {
415 SCOPED_TSAN_INTERCEPTOR(memset, dst, v, size);
416 MemoryAccessRange(thr, pc, (uptr)dst, size, true);
417 return internal_memset(dst, v, size);
420 TSAN_INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) {
421 SCOPED_TSAN_INTERCEPTOR(memcpy, dst, src, size);
422 MemoryAccessRange(thr, pc, (uptr)dst, size, true);
423 MemoryAccessRange(thr, pc, (uptr)src, size, false);
424 return internal_memcpy(dst, src, size);
427 TSAN_INTERCEPTOR(int, memcmp, const void *s1, const void *s2, uptr n) {
428 SCOPED_TSAN_INTERCEPTOR(memcmp, s1, s2, n);
429 int res = 0;
430 uptr len = 0;
431 for (; len < n; len++) {
432 if ((res = ((unsigned char*)s1)[len] - ((unsigned char*)s2)[len]))
433 break;
435 MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false);
436 MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false);
437 return res;
440 TSAN_INTERCEPTOR(int, strcmp, const char *s1, const char *s2) {
441 SCOPED_TSAN_INTERCEPTOR(strcmp, s1, s2);
442 uptr len = 0;
443 for (; s1[len] && s2[len]; len++) {
444 if (s1[len] != s2[len])
445 break;
447 MemoryAccessRange(thr, pc, (uptr)s1, len + 1, false);
448 MemoryAccessRange(thr, pc, (uptr)s2, len + 1, false);
449 return s1[len] - s2[len];
452 TSAN_INTERCEPTOR(int, strncmp, const char *s1, const char *s2, uptr n) {
453 SCOPED_TSAN_INTERCEPTOR(strncmp, s1, s2, n);
454 uptr len = 0;
455 for (; len < n && s1[len] && s2[len]; len++) {
456 if (s1[len] != s2[len])
457 break;
459 MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false);
460 MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false);
461 return len == n ? 0 : s1[len] - s2[len];
464 TSAN_INTERCEPTOR(void*, memchr, void *s, int c, uptr n) {
465 SCOPED_TSAN_INTERCEPTOR(memchr, s, c, n);
466 void *res = REAL(memchr)(s, c, n);
467 uptr len = res ? (char*)res - (char*)s + 1 : n;
468 MemoryAccessRange(thr, pc, (uptr)s, len, false);
469 return res;
472 TSAN_INTERCEPTOR(void*, memrchr, char *s, int c, uptr n) {
473 SCOPED_TSAN_INTERCEPTOR(memrchr, s, c, n);
474 MemoryAccessRange(thr, pc, (uptr)s, n, false);
475 return REAL(memrchr)(s, c, n);
478 TSAN_INTERCEPTOR(void*, memmove, void *dst, void *src, uptr n) {
479 SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n);
480 MemoryAccessRange(thr, pc, (uptr)dst, n, true);
481 MemoryAccessRange(thr, pc, (uptr)src, n, false);
482 return REAL(memmove)(dst, src, n);
485 TSAN_INTERCEPTOR(char*, strchr, char *s, int c) {
486 SCOPED_TSAN_INTERCEPTOR(strchr, s, c);
487 char *res = REAL(strchr)(s, c);
488 uptr len = res ? (char*)res - (char*)s + 1 : internal_strlen(s) + 1;
489 MemoryAccessRange(thr, pc, (uptr)s, len, false);
490 return res;
493 TSAN_INTERCEPTOR(char*, strchrnul, char *s, int c) {
494 SCOPED_TSAN_INTERCEPTOR(strchrnul, s, c);
495 char *res = REAL(strchrnul)(s, c);
496 uptr len = (char*)res - (char*)s + 1;
497 MemoryAccessRange(thr, pc, (uptr)s, len, false);
498 return res;
501 TSAN_INTERCEPTOR(char*, strrchr, char *s, int c) {
502 SCOPED_TSAN_INTERCEPTOR(strrchr, s, c);
503 MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s) + 1, false);
504 return REAL(strrchr)(s, c);
507 TSAN_INTERCEPTOR(char*, strcpy, char *dst, const char *src) { // NOLINT
508 SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src); // NOLINT
509 uptr srclen = internal_strlen(src);
510 MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true);
511 MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false);
512 return REAL(strcpy)(dst, src); // NOLINT
515 TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) {
516 SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n);
517 uptr srclen = internal_strnlen(src, n);
518 MemoryAccessRange(thr, pc, (uptr)dst, n, true);
519 MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false);
520 return REAL(strncpy)(dst, src, n);
523 TSAN_INTERCEPTOR(const char*, strstr, const char *s1, const char *s2) {
524 SCOPED_TSAN_INTERCEPTOR(strstr, s1, s2);
525 const char *res = REAL(strstr)(s1, s2);
526 uptr len1 = internal_strlen(s1);
527 uptr len2 = internal_strlen(s2);
528 MemoryAccessRange(thr, pc, (uptr)s1, len1 + 1, false);
529 MemoryAccessRange(thr, pc, (uptr)s2, len2 + 1, false);
530 return res;
533 static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
534 if (*addr) {
535 if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
536 if (flags & MAP_FIXED) {
537 errno = EINVAL;
538 return false;
539 } else {
540 *addr = 0;
544 return true;
547 TSAN_INTERCEPTOR(void*, mmap, void *addr, long_t sz, int prot,
548 int flags, int fd, unsigned off) {
549 SCOPED_TSAN_INTERCEPTOR(mmap, addr, sz, prot, flags, fd, off);
550 if (!fix_mmap_addr(&addr, sz, flags))
551 return MAP_FAILED;
552 void *res = REAL(mmap)(addr, sz, prot, flags, fd, off);
553 if (res != MAP_FAILED) {
554 MemoryResetRange(thr, pc, (uptr)res, sz);
556 return res;
559 TSAN_INTERCEPTOR(void*, mmap64, void *addr, long_t sz, int prot,
560 int flags, int fd, u64 off) {
561 SCOPED_TSAN_INTERCEPTOR(mmap64, addr, sz, prot, flags, fd, off);
562 if (!fix_mmap_addr(&addr, sz, flags))
563 return MAP_FAILED;
564 void *res = REAL(mmap64)(addr, sz, prot, flags, fd, off);
565 if (res != MAP_FAILED) {
566 MemoryResetRange(thr, pc, (uptr)res, sz);
568 return res;
571 TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
572 SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz);
573 int res = REAL(munmap)(addr, sz);
574 return res;
577 TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
578 SCOPED_TSAN_INTERCEPTOR(memalign, align, sz);
579 return user_alloc(thr, pc, sz, align);
582 TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
583 SCOPED_TSAN_INTERCEPTOR(valloc, sz);
584 return user_alloc(thr, pc, sz, GetPageSizeCached());
587 TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
588 SCOPED_TSAN_INTERCEPTOR(pvalloc, sz);
589 sz = RoundUp(sz, GetPageSizeCached());
590 return user_alloc(thr, pc, sz, GetPageSizeCached());
593 TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
594 SCOPED_TSAN_INTERCEPTOR(posix_memalign, memptr, align, sz);
595 *memptr = user_alloc(thr, pc, sz, align);
596 return 0;
599 // Used in thread-safe function static initialization.
600 extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) {
601 SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g);
602 for (;;) {
603 u32 cmp = atomic_load(g, memory_order_acquire);
604 if (cmp == 0) {
605 if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed))
606 return 1;
607 } else if (cmp == 1) {
608 Acquire(thr, pc, (uptr)g);
609 return 0;
610 } else {
611 internal_sched_yield();
616 extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_release(atomic_uint32_t *g) {
617 SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);
618 Release(thr, pc, (uptr)g);
619 atomic_store(g, 1, memory_order_release);
622 extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_abort(atomic_uint32_t *g) {
623 SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
624 atomic_store(g, 0, memory_order_relaxed);
627 static void thread_finalize(void *v) {
628 uptr iter = (uptr)v;
629 if (iter > 1) {
630 if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) {
631 Printf("ThreadSanitizer: failed to set thread key\n");
632 Die();
634 return;
637 ScopedInRtl in_rtl;
638 ThreadState *thr = cur_thread();
639 ThreadFinish(thr);
640 SignalContext *sctx = thr->signal_ctx;
641 if (sctx) {
642 thr->signal_ctx = 0;
643 UnmapOrDie(sctx, sizeof(*sctx));
649 struct ThreadParam {
650 void* (*callback)(void *arg);
651 void *param;
652 atomic_uintptr_t tid;
655 extern "C" void *__tsan_thread_start_func(void *arg) {
656 ThreadParam *p = (ThreadParam*)arg;
657 void* (*callback)(void *arg) = p->callback;
658 void *param = p->param;
659 int tid = 0;
661 ThreadState *thr = cur_thread();
662 ScopedInRtl in_rtl;
663 if (pthread_setspecific(g_thread_finalize_key, (void*)4)) {
664 Printf("ThreadSanitizer: failed to set thread key\n");
665 Die();
667 while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)
668 pthread_yield();
669 atomic_store(&p->tid, 0, memory_order_release);
670 ThreadStart(thr, tid, GetTid());
671 CHECK_EQ(thr->in_rtl, 1);
673 void *res = callback(param);
674 // Prevent the callback from being tail called,
675 // it mixes up stack traces.
676 volatile int foo = 42;
677 foo++;
678 return res;
681 TSAN_INTERCEPTOR(int, pthread_create,
682 void *th, void *attr, void *(*callback)(void*), void * param) {
683 SCOPED_TSAN_INTERCEPTOR(pthread_create, th, attr, callback, param);
684 pthread_attr_t myattr;
685 if (attr == 0) {
686 pthread_attr_init(&myattr);
687 attr = &myattr;
689 int detached = 0;
690 pthread_attr_getdetachstate(attr, &detached);
691 uptr stacksize = 0;
692 pthread_attr_getstacksize(attr, &stacksize);
693 // We place the huge ThreadState object into TLS, account for that.
694 const uptr minstacksize = GetTlsSize() + 128*1024;
695 if (stacksize < minstacksize) {
696 DPrintf("ThreadSanitizer: stacksize %zu->%zu\n", stacksize, minstacksize);
697 pthread_attr_setstacksize(attr, minstacksize);
699 ThreadParam p;
700 p.callback = callback;
701 p.param = param;
702 atomic_store(&p.tid, 0, memory_order_relaxed);
703 int res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p);
704 if (res == 0) {
705 int tid = ThreadCreate(thr, pc, *(uptr*)th, detached);
706 CHECK_NE(tid, 0);
707 atomic_store(&p.tid, tid, memory_order_release);
708 while (atomic_load(&p.tid, memory_order_acquire) != 0)
709 pthread_yield();
711 if (attr == &myattr)
712 pthread_attr_destroy(&myattr);
713 return res;
716 TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) {
717 SCOPED_TSAN_INTERCEPTOR(pthread_join, th, ret);
718 int tid = ThreadTid(thr, pc, (uptr)th);
719 int res = BLOCK_REAL(pthread_join)(th, ret);
720 if (res == 0) {
721 ThreadJoin(thr, pc, tid);
723 return res;
726 TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
727 SCOPED_TSAN_INTERCEPTOR(pthread_detach, th);
728 int tid = ThreadTid(thr, pc, (uptr)th);
729 int res = REAL(pthread_detach)(th);
730 if (res == 0) {
731 ThreadDetach(thr, pc, tid);
733 return res;
736 TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
737 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a);
738 int res = REAL(pthread_mutex_init)(m, a);
739 if (res == 0) {
740 bool recursive = false;
741 if (a) {
742 int type = 0;
743 if (pthread_mutexattr_gettype(a, &type) == 0)
744 recursive = (type == PTHREAD_MUTEX_RECURSIVE
745 || type == PTHREAD_MUTEX_RECURSIVE_NP);
747 MutexCreate(thr, pc, (uptr)m, false, recursive, false);
749 return res;
752 TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
753 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m);
754 int res = REAL(pthread_mutex_destroy)(m);
755 if (res == 0 || res == EBUSY) {
756 MutexDestroy(thr, pc, (uptr)m);
758 return res;
761 TSAN_INTERCEPTOR(int, pthread_mutex_lock, void *m) {
762 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_lock, m);
763 int res = REAL(pthread_mutex_lock)(m);
764 if (res == 0) {
765 MutexLock(thr, pc, (uptr)m);
767 return res;
770 TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
771 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
772 int res = REAL(pthread_mutex_trylock)(m);
773 if (res == 0) {
774 MutexLock(thr, pc, (uptr)m);
776 return res;
779 TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) {
780 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
781 int res = REAL(pthread_mutex_timedlock)(m, abstime);
782 if (res == 0) {
783 MutexLock(thr, pc, (uptr)m);
785 return res;
788 TSAN_INTERCEPTOR(int, pthread_mutex_unlock, void *m) {
789 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_unlock, m);
790 MutexUnlock(thr, pc, (uptr)m);
791 int res = REAL(pthread_mutex_unlock)(m);
792 return res;
795 TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {
796 SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);
797 int res = REAL(pthread_spin_init)(m, pshared);
798 if (res == 0) {
799 MutexCreate(thr, pc, (uptr)m, false, false, false);
801 return res;
804 TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
805 SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m);
806 int res = REAL(pthread_spin_destroy)(m);
807 if (res == 0) {
808 MutexDestroy(thr, pc, (uptr)m);
810 return res;
813 TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
814 SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
815 int res = REAL(pthread_spin_lock)(m);
816 if (res == 0) {
817 MutexLock(thr, pc, (uptr)m);
819 return res;
822 TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) {
823 SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m);
824 int res = REAL(pthread_spin_trylock)(m);
825 if (res == 0) {
826 MutexLock(thr, pc, (uptr)m);
828 return res;
831 TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) {
832 SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m);
833 MutexUnlock(thr, pc, (uptr)m);
834 int res = REAL(pthread_spin_unlock)(m);
835 return res;
838 TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) {
839 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a);
840 int res = REAL(pthread_rwlock_init)(m, a);
841 if (res == 0) {
842 MutexCreate(thr, pc, (uptr)m, true, false, false);
844 return res;
847 TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) {
848 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m);
849 int res = REAL(pthread_rwlock_destroy)(m);
850 if (res == 0) {
851 MutexDestroy(thr, pc, (uptr)m);
853 return res;
856 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) {
857 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m);
858 int res = REAL(pthread_rwlock_rdlock)(m);
859 if (res == 0) {
860 MutexReadLock(thr, pc, (uptr)m);
862 return res;
865 TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
866 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m);
867 int res = REAL(pthread_rwlock_tryrdlock)(m);
868 if (res == 0) {
869 MutexReadLock(thr, pc, (uptr)m);
871 return res;
874 TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) {
875 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
876 int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
877 if (res == 0) {
878 MutexReadLock(thr, pc, (uptr)m);
880 return res;
883 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
884 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
885 int res = REAL(pthread_rwlock_wrlock)(m);
886 if (res == 0) {
887 MutexLock(thr, pc, (uptr)m);
889 return res;
892 TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
893 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m);
894 int res = REAL(pthread_rwlock_trywrlock)(m);
895 if (res == 0) {
896 MutexLock(thr, pc, (uptr)m);
898 return res;
901 TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) {
902 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
903 int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
904 if (res == 0) {
905 MutexLock(thr, pc, (uptr)m);
907 return res;
910 TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) {
911 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m);
912 MutexReadOrWriteUnlock(thr, pc, (uptr)m);
913 int res = REAL(pthread_rwlock_unlock)(m);
914 return res;
917 // libpthread.so contains several versions of pthread_cond_init symbol.
918 // When we just dlsym() it, we get the wrong (old) version.
920 TSAN_INTERCEPTOR(int, pthread_cond_init, void *c, void *a) {
921 SCOPED_TSAN_INTERCEPTOR(pthread_cond_init, c, a);
922 int res = REAL(pthread_cond_init)(c, a);
923 return res;
927 TSAN_INTERCEPTOR(int, pthread_cond_destroy, void *c) {
928 SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, c);
929 int res = REAL(pthread_cond_destroy)(c);
930 return res;
933 TSAN_INTERCEPTOR(int, pthread_cond_signal, void *c) {
934 SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal, c);
935 int res = REAL(pthread_cond_signal)(c);
936 return res;
939 TSAN_INTERCEPTOR(int, pthread_cond_broadcast, void *c) {
940 SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast, c);
941 int res = REAL(pthread_cond_broadcast)(c);
942 return res;
945 TSAN_INTERCEPTOR(int, pthread_cond_wait, void *c, void *m) {
946 SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait, c, m);
947 MutexUnlock(thr, pc, (uptr)m);
948 int res = REAL(pthread_cond_wait)(c, m);
949 MutexLock(thr, pc, (uptr)m);
950 return res;
953 TSAN_INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m, void *abstime) {
954 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, c, m, abstime);
955 MutexUnlock(thr, pc, (uptr)m);
956 int res = REAL(pthread_cond_timedwait)(c, m, abstime);
957 MutexLock(thr, pc, (uptr)m);
958 return res;
961 TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) {
962 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count);
963 MemoryWrite1Byte(thr, pc, (uptr)b);
964 int res = REAL(pthread_barrier_init)(b, a, count);
965 return res;
968 TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
969 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b);
970 MemoryWrite1Byte(thr, pc, (uptr)b);
971 int res = REAL(pthread_barrier_destroy)(b);
972 return res;
975 TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
976 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b);
977 Release(thr, pc, (uptr)b);
978 MemoryRead1Byte(thr, pc, (uptr)b);
979 int res = REAL(pthread_barrier_wait)(b);
980 MemoryRead1Byte(thr, pc, (uptr)b);
981 if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) {
982 Acquire(thr, pc, (uptr)b);
984 return res;
987 TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
988 SCOPED_TSAN_INTERCEPTOR(pthread_once, o, f);
989 if (o == 0 || f == 0)
990 return EINVAL;
991 atomic_uint32_t *a = static_cast<atomic_uint32_t*>(o);
992 u32 v = atomic_load(a, memory_order_acquire);
993 if (v == 0 && atomic_compare_exchange_strong(a, &v, 1,
994 memory_order_relaxed)) {
995 const int old_in_rtl = thr->in_rtl;
996 thr->in_rtl = 0;
997 (*f)();
998 CHECK_EQ(thr->in_rtl, 0);
999 thr->in_rtl = old_in_rtl;
1000 Release(thr, pc, (uptr)o);
1001 atomic_store(a, 2, memory_order_release);
1002 } else {
1003 while (v != 2) {
1004 pthread_yield();
1005 v = atomic_load(a, memory_order_acquire);
1007 Acquire(thr, pc, (uptr)o);
1009 return 0;
1012 TSAN_INTERCEPTOR(int, sem_init, void *s, int pshared, unsigned value) {
1013 SCOPED_TSAN_INTERCEPTOR(sem_init, s, pshared, value);
1014 int res = REAL(sem_init)(s, pshared, value);
1015 return res;
1018 TSAN_INTERCEPTOR(int, sem_destroy, void *s) {
1019 SCOPED_TSAN_INTERCEPTOR(sem_destroy, s);
1020 int res = REAL(sem_destroy)(s);
1021 return res;
1024 TSAN_INTERCEPTOR(int, sem_wait, void *s) {
1025 SCOPED_TSAN_INTERCEPTOR(sem_wait, s);
1026 int res = BLOCK_REAL(sem_wait)(s);
1027 if (res == 0) {
1028 Acquire(thr, pc, (uptr)s);
1030 return res;
1033 TSAN_INTERCEPTOR(int, sem_trywait, void *s) {
1034 SCOPED_TSAN_INTERCEPTOR(sem_trywait, s);
1035 int res = BLOCK_REAL(sem_trywait)(s);
1036 if (res == 0) {
1037 Acquire(thr, pc, (uptr)s);
1039 return res;
1042 TSAN_INTERCEPTOR(int, sem_timedwait, void *s, void *abstime) {
1043 SCOPED_TSAN_INTERCEPTOR(sem_timedwait, s, abstime);
1044 int res = BLOCK_REAL(sem_timedwait)(s, abstime);
1045 if (res == 0) {
1046 Acquire(thr, pc, (uptr)s);
1048 return res;
1051 TSAN_INTERCEPTOR(int, sem_post, void *s) {
1052 SCOPED_TSAN_INTERCEPTOR(sem_post, s);
1053 Release(thr, pc, (uptr)s);
1054 int res = REAL(sem_post)(s);
1055 return res;
1058 TSAN_INTERCEPTOR(int, sem_getvalue, void *s, int *sval) {
1059 SCOPED_TSAN_INTERCEPTOR(sem_getvalue, s, sval);
1060 int res = REAL(sem_getvalue)(s, sval);
1061 if (res == 0) {
1062 Acquire(thr, pc, (uptr)s);
1064 return res;
1067 TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) {
1068 SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode);
1069 int fd = REAL(open)(name, flags, mode);
1070 if (fd >= 0)
1071 FdFileCreate(thr, pc, fd);
1072 return fd;
1075 TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) {
1076 SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode);
1077 int fd = REAL(open64)(name, flags, mode);
1078 if (fd >= 0)
1079 FdFileCreate(thr, pc, fd);
1080 return fd;
1083 TSAN_INTERCEPTOR(int, creat, const char *name, int mode) {
1084 SCOPED_TSAN_INTERCEPTOR(creat, name, mode);
1085 int fd = REAL(creat)(name, mode);
1086 if (fd >= 0)
1087 FdFileCreate(thr, pc, fd);
1088 return fd;
1091 TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) {
1092 SCOPED_TSAN_INTERCEPTOR(creat64, name, mode);
1093 int fd = REAL(creat64)(name, mode);
1094 if (fd >= 0)
1095 FdFileCreate(thr, pc, fd);
1096 return fd;
1099 TSAN_INTERCEPTOR(int, dup, int oldfd) {
1100 SCOPED_TSAN_INTERCEPTOR(dup, oldfd);
1101 int newfd = REAL(dup)(oldfd);
1102 if (oldfd >= 0 && newfd >= 0 && newfd != oldfd)
1103 FdDup(thr, pc, oldfd, newfd);
1104 return newfd;
1107 TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) {
1108 SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd);
1109 int newfd2 = REAL(dup2)(oldfd, newfd);
1110 if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
1111 FdDup(thr, pc, oldfd, newfd2);
1112 return newfd2;
1115 TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) {
1116 SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags);
1117 int newfd2 = REAL(dup3)(oldfd, newfd, flags);
1118 if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
1119 FdDup(thr, pc, oldfd, newfd2);
1120 return newfd2;
1123 TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) {
1124 SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags);
1125 int fd = REAL(eventfd)(initval, flags);
1126 if (fd >= 0)
1127 FdEventCreate(thr, pc, fd);
1128 return fd;
1131 TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {
1132 SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags);
1133 if (fd >= 0)
1134 FdClose(thr, pc, fd);
1135 fd = REAL(signalfd)(fd, mask, flags);
1136 if (fd >= 0)
1137 FdSignalCreate(thr, pc, fd);
1138 return fd;
1141 TSAN_INTERCEPTOR(int, inotify_init, int fake) {
1142 SCOPED_TSAN_INTERCEPTOR(inotify_init, fake);
1143 int fd = REAL(inotify_init)(fake);
1144 if (fd >= 0)
1145 FdInotifyCreate(thr, pc, fd);
1146 return fd;
1149 TSAN_INTERCEPTOR(int, inotify_init1, int flags) {
1150 SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags);
1151 int fd = REAL(inotify_init1)(flags);
1152 if (fd >= 0)
1153 FdInotifyCreate(thr, pc, fd);
1154 return fd;
1157 TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) {
1158 SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol);
1159 int fd = REAL(socket)(domain, type, protocol);
1160 if (fd >= 0)
1161 FdSocketCreate(thr, pc, fd);
1162 return fd;
1165 TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) {
1166 SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd);
1167 int res = REAL(socketpair)(domain, type, protocol, fd);
1168 if (res == 0 && fd[0] >= 0 && fd[1] >= 0)
1169 FdPipeCreate(thr, pc, fd[0], fd[1]);
1170 return res;
1173 TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) {
1174 SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen);
1175 FdSocketConnecting(thr, pc, fd);
1176 int res = REAL(connect)(fd, addr, addrlen);
1177 if (res == 0 && fd >= 0)
1178 FdSocketConnect(thr, pc, fd);
1179 return res;
1182 TSAN_INTERCEPTOR(int, accept, int fd, void *addr, unsigned *addrlen) {
1183 SCOPED_TSAN_INTERCEPTOR(accept, fd, addr, addrlen);
1184 int fd2 = REAL(accept)(fd, addr, addrlen);
1185 if (fd >= 0 && fd2 >= 0)
1186 FdSocketAccept(thr, pc, fd, fd2);
1187 return fd2;
1190 TSAN_INTERCEPTOR(int, accept4, int fd, void *addr, unsigned *addrlen, int f) {
1191 SCOPED_TSAN_INTERCEPTOR(accept4, fd, addr, addrlen, f);
1192 int fd2 = REAL(accept4)(fd, addr, addrlen, f);
1193 if (fd >= 0 && fd2 >= 0)
1194 FdSocketAccept(thr, pc, fd, fd2);
1195 return fd2;
1198 TSAN_INTERCEPTOR(int, epoll_create, int size) {
1199 SCOPED_TSAN_INTERCEPTOR(epoll_create, size);
1200 int fd = REAL(epoll_create)(size);
1201 if (fd >= 0)
1202 FdPollCreate(thr, pc, fd);
1203 return fd;
1206 TSAN_INTERCEPTOR(int, epoll_create1, int flags) {
1207 SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags);
1208 int fd = REAL(epoll_create1)(flags);
1209 if (fd >= 0)
1210 FdPollCreate(thr, pc, fd);
1211 return fd;
1214 TSAN_INTERCEPTOR(int, close, int fd) {
1215 SCOPED_TSAN_INTERCEPTOR(close, fd);
1216 if (fd >= 0)
1217 FdClose(thr, pc, fd);
1218 return REAL(close)(fd);
1221 TSAN_INTERCEPTOR(int, __close, int fd) {
1222 SCOPED_TSAN_INTERCEPTOR(__close, fd);
1223 if (fd >= 0)
1224 FdClose(thr, pc, fd);
1225 return REAL(__close)(fd);
1228 TSAN_INTERCEPTOR(int, pipe, int *pipefd) {
1229 SCOPED_TSAN_INTERCEPTOR(pipe, pipefd);
1230 int res = REAL(pipe)(pipefd);
1231 if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
1232 FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
1233 return res;
1236 TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) {
1237 SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags);
1238 int res = REAL(pipe2)(pipefd, flags);
1239 if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
1240 FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
1241 return res;
1244 TSAN_INTERCEPTOR(long_t, read, int fd, void *buf, long_t sz) {
1245 SCOPED_TSAN_INTERCEPTOR(read, fd, buf, sz);
1246 int res = REAL(read)(fd, buf, sz);
1247 if (res >= 0 && fd >= 0) {
1248 FdAcquire(thr, pc, fd);
1250 return res;
1253 TSAN_INTERCEPTOR(long_t, pread, int fd, void *buf, long_t sz, unsigned off) {
1254 SCOPED_TSAN_INTERCEPTOR(pread, fd, buf, sz, off);
1255 int res = REAL(pread)(fd, buf, sz, off);
1256 if (res >= 0 && fd >= 0) {
1257 FdAcquire(thr, pc, fd);
1259 return res;
1262 TSAN_INTERCEPTOR(long_t, pread64, int fd, void *buf, long_t sz, u64 off) {
1263 SCOPED_TSAN_INTERCEPTOR(pread64, fd, buf, sz, off);
1264 int res = REAL(pread64)(fd, buf, sz, off);
1265 if (res >= 0 && fd >= 0) {
1266 FdAcquire(thr, pc, fd);
1268 return res;
1271 TSAN_INTERCEPTOR(long_t, readv, int fd, void *vec, int cnt) {
1272 SCOPED_TSAN_INTERCEPTOR(readv, fd, vec, cnt);
1273 int res = REAL(readv)(fd, vec, cnt);
1274 if (res >= 0 && fd >= 0) {
1275 FdAcquire(thr, pc, fd);
1277 return res;
1280 TSAN_INTERCEPTOR(long_t, preadv64, int fd, void *vec, int cnt, u64 off) {
1281 SCOPED_TSAN_INTERCEPTOR(preadv64, fd, vec, cnt, off);
1282 int res = REAL(preadv64)(fd, vec, cnt, off);
1283 if (res >= 0 && fd >= 0) {
1284 FdAcquire(thr, pc, fd);
1286 return res;
1289 TSAN_INTERCEPTOR(long_t, write, int fd, void *buf, long_t sz) {
1290 SCOPED_TSAN_INTERCEPTOR(write, fd, buf, sz);
1291 if (fd >= 0)
1292 FdRelease(thr, pc, fd);
1293 int res = REAL(write)(fd, buf, sz);
1294 return res;
1297 TSAN_INTERCEPTOR(long_t, pwrite, int fd, void *buf, long_t sz, unsigned off) {
1298 SCOPED_TSAN_INTERCEPTOR(pwrite, fd, buf, sz, off);
1299 if (fd >= 0)
1300 FdRelease(thr, pc, fd);
1301 int res = REAL(pwrite)(fd, buf, sz, off);
1302 return res;
1305 TSAN_INTERCEPTOR(long_t, pwrite64, int fd, void *buf, long_t sz, u64 off) {
1306 SCOPED_TSAN_INTERCEPTOR(pwrite64, fd, buf, sz, off);
1307 if (fd >= 0)
1308 FdRelease(thr, pc, fd);
1309 int res = REAL(pwrite64)(fd, buf, sz, off);
1310 return res;
1313 TSAN_INTERCEPTOR(long_t, writev, int fd, void *vec, int cnt) {
1314 SCOPED_TSAN_INTERCEPTOR(writev, fd, vec, cnt);
1315 if (fd >= 0)
1316 FdRelease(thr, pc, fd);
1317 int res = REAL(writev)(fd, vec, cnt);
1318 return res;
1321 TSAN_INTERCEPTOR(long_t, pwritev64, int fd, void *vec, int cnt, u64 off) {
1322 SCOPED_TSAN_INTERCEPTOR(pwritev64, fd, vec, cnt, off);
1323 if (fd >= 0)
1324 FdRelease(thr, pc, fd);
1325 int res = REAL(pwritev64)(fd, vec, cnt, off);
1326 return res;
1329 TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) {
1330 SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags);
1331 if (fd >= 0)
1332 FdRelease(thr, pc, fd);
1333 int res = REAL(send)(fd, buf, len, flags);
1334 return res;
1337 TSAN_INTERCEPTOR(long_t, sendmsg, int fd, void *msg, int flags) {
1338 SCOPED_TSAN_INTERCEPTOR(sendmsg, fd, msg, flags);
1339 if (fd >= 0)
1340 FdRelease(thr, pc, fd);
1341 int res = REAL(sendmsg)(fd, msg, flags);
1342 return res;
1345 TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) {
1346 SCOPED_TSAN_INTERCEPTOR(recv, fd, buf, len, flags);
1347 int res = REAL(recv)(fd, buf, len, flags);
1348 if (res >= 0 && fd >= 0) {
1349 FdAcquire(thr, pc, fd);
1351 return res;
1354 TSAN_INTERCEPTOR(long_t, recvmsg, int fd, void *msg, int flags) {
1355 SCOPED_TSAN_INTERCEPTOR(recvmsg, fd, msg, flags);
1356 int res = REAL(recvmsg)(fd, msg, flags);
1357 if (res >= 0 && fd >= 0) {
1358 FdAcquire(thr, pc, fd);
1360 return res;
1363 TSAN_INTERCEPTOR(int, unlink, char *path) {
1364 SCOPED_TSAN_INTERCEPTOR(unlink, path);
1365 Release(thr, pc, File2addr(path));
1366 int res = REAL(unlink)(path);
1367 return res;
1370 TSAN_INTERCEPTOR(void*, fopen, char *path, char *mode) {
1371 SCOPED_TSAN_INTERCEPTOR(fopen, path, mode);
1372 void *res = REAL(fopen)(path, mode);
1373 Acquire(thr, pc, File2addr(path));
1374 if (res) {
1375 int fd = fileno_unlocked(res);
1376 if (fd >= 0)
1377 FdFileCreate(thr, pc, fd);
1379 return res;
1382 TSAN_INTERCEPTOR(void*, freopen, char *path, char *mode, void *stream) {
1383 SCOPED_TSAN_INTERCEPTOR(freopen, path, mode, stream);
1384 if (stream) {
1385 int fd = fileno_unlocked(stream);
1386 if (fd >= 0)
1387 FdClose(thr, pc, fd);
1389 void *res = REAL(freopen)(path, mode, stream);
1390 Acquire(thr, pc, File2addr(path));
1391 if (res) {
1392 int fd = fileno_unlocked(res);
1393 if (fd >= 0)
1394 FdFileCreate(thr, pc, fd);
1396 return res;
1399 TSAN_INTERCEPTOR(int, fclose, void *stream) {
1401 SCOPED_TSAN_INTERCEPTOR(fclose, stream);
1402 if (stream) {
1403 int fd = fileno_unlocked(stream);
1404 if (fd >= 0)
1405 FdClose(thr, pc, fd);
1408 return REAL(fclose)(stream);
1411 TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) {
1413 SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f);
1414 MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true);
1416 return REAL(fread)(ptr, size, nmemb, f);
1419 TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) {
1421 SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f);
1422 MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false);
1424 return REAL(fwrite)(p, size, nmemb, f);
1427 TSAN_INTERCEPTOR(int, puts, const char *s) {
1428 SCOPED_TSAN_INTERCEPTOR(puts, s);
1429 MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s), false);
1430 return REAL(puts)(s);
1433 TSAN_INTERCEPTOR(int, rmdir, char *path) {
1434 SCOPED_TSAN_INTERCEPTOR(rmdir, path);
1435 Release(thr, pc, Dir2addr(path));
1436 int res = REAL(rmdir)(path);
1437 return res;
1440 TSAN_INTERCEPTOR(void*, opendir, char *path) {
1441 SCOPED_TSAN_INTERCEPTOR(opendir, path);
1442 void *res = REAL(opendir)(path);
1443 if (res != 0)
1444 Acquire(thr, pc, Dir2addr(path));
1445 return res;
1448 TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) {
1449 SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev);
1450 if (op == EPOLL_CTL_ADD && epfd >= 0) {
1451 FdRelease(thr, pc, epfd);
1453 int res = REAL(epoll_ctl)(epfd, op, fd, ev);
1454 FdAccess(thr, pc, fd);
1455 return res;
1458 TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) {
1459 SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout);
1460 int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout);
1461 if (res > 0 && epfd >= 0) {
1462 FdAcquire(thr, pc, epfd);
1464 return res;
1467 TSAN_INTERCEPTOR(int, poll, void *fds, long_t nfds, int timeout) {
1468 SCOPED_TSAN_INTERCEPTOR(poll, fds, nfds, timeout);
1469 int res = BLOCK_REAL(poll)(fds, nfds, timeout);
1470 return res;
1473 static void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
1474 my_siginfo_t *info, void *ctx) {
1475 ThreadState *thr = cur_thread();
1476 SignalContext *sctx = SigCtx(thr);
1477 // Don't mess with synchronous signals.
1478 if (sig == SIGSEGV || sig == SIGBUS || sig == SIGILL ||
1479 sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE ||
1480 // If we are sending signal to ourselves, we must process it now.
1481 (sctx && sig == sctx->int_signal_send) ||
1482 // If we are in blocking function, we can safely process it now
1483 // (but check if we are in a recursive interceptor,
1484 // i.e. pthread_join()->munmap()).
1485 (sctx && sctx->in_blocking_func == 1 && thr->in_rtl == 1)) {
1486 CHECK(thr->in_rtl == 0 || thr->in_rtl == 1);
1487 int in_rtl = thr->in_rtl;
1488 thr->in_rtl = 0;
1489 CHECK_EQ(thr->in_signal_handler, false);
1490 thr->in_signal_handler = true;
1491 if (sigact)
1492 sigactions[sig].sa_sigaction(sig, info, ctx);
1493 else
1494 sigactions[sig].sa_handler(sig);
1495 CHECK_EQ(thr->in_signal_handler, true);
1496 thr->in_signal_handler = false;
1497 thr->in_rtl = in_rtl;
1498 return;
1501 if (sctx == 0)
1502 return;
1503 SignalDesc *signal = &sctx->pending_signals[sig];
1504 if (signal->armed == false) {
1505 signal->armed = true;
1506 signal->sigaction = sigact;
1507 if (info)
1508 internal_memcpy(&signal->siginfo, info, sizeof(*info));
1509 if (ctx)
1510 internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
1511 sctx->pending_signal_count++;
1515 static void rtl_sighandler(int sig) {
1516 rtl_generic_sighandler(false, sig, 0, 0);
1519 static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) {
1520 rtl_generic_sighandler(true, sig, info, ctx);
1523 TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) {
1524 SCOPED_TSAN_INTERCEPTOR(sigaction, sig, act, old);
1525 if (old)
1526 internal_memcpy(old, &sigactions[sig], sizeof(*old));
1527 if (act == 0)
1528 return 0;
1529 internal_memcpy(&sigactions[sig], act, sizeof(*act));
1530 sigaction_t newact;
1531 internal_memcpy(&newact, act, sizeof(newact));
1532 sigfillset(&newact.sa_mask);
1533 if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) {
1534 if (newact.sa_flags & SA_SIGINFO)
1535 newact.sa_sigaction = rtl_sigaction;
1536 else
1537 newact.sa_handler = rtl_sighandler;
1539 int res = REAL(sigaction)(sig, &newact, 0);
1540 return res;
1543 TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) {
1544 sigaction_t act;
1545 act.sa_handler = h;
1546 REAL(memset)(&act.sa_mask, -1, sizeof(act.sa_mask));
1547 act.sa_flags = 0;
1548 sigaction_t old;
1549 int res = sigaction(sig, &act, &old);
1550 if (res)
1551 return SIG_ERR;
1552 return old.sa_handler;
1555 TSAN_INTERCEPTOR(int, raise, int sig) {
1556 SCOPED_TSAN_INTERCEPTOR(raise, sig);
1557 SignalContext *sctx = SigCtx(thr);
1558 CHECK_NE(sctx, 0);
1559 int prev = sctx->int_signal_send;
1560 sctx->int_signal_send = sig;
1561 int res = REAL(raise)(sig);
1562 CHECK_EQ(sctx->int_signal_send, sig);
1563 sctx->int_signal_send = prev;
1564 return res;
1567 TSAN_INTERCEPTOR(int, kill, int pid, int sig) {
1568 SCOPED_TSAN_INTERCEPTOR(kill, pid, sig);
1569 SignalContext *sctx = SigCtx(thr);
1570 CHECK_NE(sctx, 0);
1571 int prev = sctx->int_signal_send;
1572 if (pid == GetPid()) {
1573 sctx->int_signal_send = sig;
1575 int res = REAL(kill)(pid, sig);
1576 if (pid == GetPid()) {
1577 CHECK_EQ(sctx->int_signal_send, sig);
1578 sctx->int_signal_send = prev;
1580 return res;
1583 TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) {
1584 SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig);
1585 SignalContext *sctx = SigCtx(thr);
1586 CHECK_NE(sctx, 0);
1587 int prev = sctx->int_signal_send;
1588 if (tid == pthread_self()) {
1589 sctx->int_signal_send = sig;
1591 int res = REAL(pthread_kill)(tid, sig);
1592 if (tid == pthread_self()) {
1593 CHECK_EQ(sctx->int_signal_send, sig);
1594 sctx->int_signal_send = prev;
1596 return res;
1599 TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) {
1600 SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz);
1601 // It's intercepted merely to process pending signals.
1602 return REAL(gettimeofday)(tv, tz);
1605 // Linux kernel has a bug that leads to kernel deadlock if a process
1606 // maps TBs of memory and then calls mlock().
1607 static void MlockIsUnsupported() {
1608 static atomic_uint8_t printed;
1609 if (atomic_exchange(&printed, 1, memory_order_relaxed))
1610 return;
1611 Printf("INFO: ThreadSanitizer ignores mlock/mlockall/munlock/munlockall\n");
1614 TSAN_INTERCEPTOR(int, mlock, const void *addr, uptr len) {
1615 MlockIsUnsupported();
1616 return 0;
1619 TSAN_INTERCEPTOR(int, munlock, const void *addr, uptr len) {
1620 MlockIsUnsupported();
1621 return 0;
1624 TSAN_INTERCEPTOR(int, mlockall, int flags) {
1625 MlockIsUnsupported();
1626 return 0;
1629 TSAN_INTERCEPTOR(int, munlockall, void) {
1630 MlockIsUnsupported();
1631 return 0;
1634 TSAN_INTERCEPTOR(int, fork, int fake) {
1635 SCOPED_TSAN_INTERCEPTOR(fork, fake);
1636 // It's intercepted merely to process pending signals.
1637 int pid = REAL(fork)(fake);
1638 if (pid == 0) {
1639 // child
1640 FdOnFork(thr, pc);
1641 } else if (pid > 0) {
1642 // parent
1644 return pid;
1647 namespace __tsan {
1649 void ProcessPendingSignals(ThreadState *thr) {
1650 CHECK_EQ(thr->in_rtl, 0);
1651 SignalContext *sctx = SigCtx(thr);
1652 if (sctx == 0 || sctx->pending_signal_count == 0 || thr->in_signal_handler)
1653 return;
1654 Context *ctx = CTX();
1655 thr->in_signal_handler = true;
1656 sctx->pending_signal_count = 0;
1657 // These are too big for stack.
1658 static THREADLOCAL sigset_t emptyset, oldset;
1659 sigfillset(&emptyset);
1660 pthread_sigmask(SIG_SETMASK, &emptyset, &oldset);
1661 for (int sig = 0; sig < kSigCount; sig++) {
1662 SignalDesc *signal = &sctx->pending_signals[sig];
1663 if (signal->armed) {
1664 signal->armed = false;
1665 if (sigactions[sig].sa_handler != SIG_DFL
1666 && sigactions[sig].sa_handler != SIG_IGN) {
1667 // Insure that the handler does not spoil errno.
1668 const int saved_errno = errno;
1669 errno = 0;
1670 if (signal->sigaction)
1671 sigactions[sig].sa_sigaction(sig, &signal->siginfo, &signal->ctx);
1672 else
1673 sigactions[sig].sa_handler(sig);
1674 if (flags()->report_bugs && errno != 0) {
1675 ScopedInRtl in_rtl;
1676 __tsan::StackTrace stack;
1677 uptr pc = signal->sigaction ?
1678 (uptr)sigactions[sig].sa_sigaction :
1679 (uptr)sigactions[sig].sa_handler;
1680 stack.Init(&pc, 1);
1681 ScopedReport rep(ReportTypeErrnoInSignal);
1682 if (!IsFiredSuppression(ctx, rep, stack)) {
1683 rep.AddStack(&stack);
1684 OutputReport(ctx, rep, rep.GetReport()->stacks[0]);
1687 errno = saved_errno;
1691 pthread_sigmask(SIG_SETMASK, &oldset, 0);
1692 CHECK_EQ(thr->in_signal_handler, true);
1693 thr->in_signal_handler = false;
1696 static void unreachable() {
1697 Printf("FATAL: ThreadSanitizer: unreachable called\n");
1698 Die();
1701 void InitializeInterceptors() {
1702 CHECK_GT(cur_thread()->in_rtl, 0);
1704 // We need to setup it early, because functions like dlsym() can call it.
1705 REAL(memset) = internal_memset;
1706 REAL(memcpy) = internal_memcpy;
1707 REAL(memcmp) = internal_memcmp;
1709 TSAN_INTERCEPT(longjmp);
1710 TSAN_INTERCEPT(siglongjmp);
1712 TSAN_INTERCEPT(malloc);
1713 TSAN_INTERCEPT(__libc_memalign);
1714 TSAN_INTERCEPT(calloc);
1715 TSAN_INTERCEPT(realloc);
1716 TSAN_INTERCEPT(free);
1717 TSAN_INTERCEPT(cfree);
1718 TSAN_INTERCEPT(mmap);
1719 TSAN_INTERCEPT(mmap64);
1720 TSAN_INTERCEPT(munmap);
1721 TSAN_INTERCEPT(memalign);
1722 TSAN_INTERCEPT(valloc);
1723 TSAN_INTERCEPT(pvalloc);
1724 TSAN_INTERCEPT(posix_memalign);
1726 TSAN_INTERCEPT(strlen);
1727 TSAN_INTERCEPT(memset);
1728 TSAN_INTERCEPT(memcpy);
1729 TSAN_INTERCEPT(strcmp);
1730 TSAN_INTERCEPT(memchr);
1731 TSAN_INTERCEPT(memrchr);
1732 TSAN_INTERCEPT(memmove);
1733 TSAN_INTERCEPT(memcmp);
1734 TSAN_INTERCEPT(strchr);
1735 TSAN_INTERCEPT(strchrnul);
1736 TSAN_INTERCEPT(strrchr);
1737 TSAN_INTERCEPT(strncmp);
1738 TSAN_INTERCEPT(strcpy); // NOLINT
1739 TSAN_INTERCEPT(strncpy);
1740 TSAN_INTERCEPT(strstr);
1742 TSAN_INTERCEPT(pthread_create);
1743 TSAN_INTERCEPT(pthread_join);
1744 TSAN_INTERCEPT(pthread_detach);
1746 TSAN_INTERCEPT(pthread_mutex_init);
1747 TSAN_INTERCEPT(pthread_mutex_destroy);
1748 TSAN_INTERCEPT(pthread_mutex_lock);
1749 TSAN_INTERCEPT(pthread_mutex_trylock);
1750 TSAN_INTERCEPT(pthread_mutex_timedlock);
1751 TSAN_INTERCEPT(pthread_mutex_unlock);
1753 TSAN_INTERCEPT(pthread_spin_init);
1754 TSAN_INTERCEPT(pthread_spin_destroy);
1755 TSAN_INTERCEPT(pthread_spin_lock);
1756 TSAN_INTERCEPT(pthread_spin_trylock);
1757 TSAN_INTERCEPT(pthread_spin_unlock);
1759 TSAN_INTERCEPT(pthread_rwlock_init);
1760 TSAN_INTERCEPT(pthread_rwlock_destroy);
1761 TSAN_INTERCEPT(pthread_rwlock_rdlock);
1762 TSAN_INTERCEPT(pthread_rwlock_tryrdlock);
1763 TSAN_INTERCEPT(pthread_rwlock_timedrdlock);
1764 TSAN_INTERCEPT(pthread_rwlock_wrlock);
1765 TSAN_INTERCEPT(pthread_rwlock_trywrlock);
1766 TSAN_INTERCEPT(pthread_rwlock_timedwrlock);
1767 TSAN_INTERCEPT(pthread_rwlock_unlock);
1769 // TSAN_INTERCEPT(pthread_cond_init);
1770 TSAN_INTERCEPT(pthread_cond_destroy);
1771 TSAN_INTERCEPT(pthread_cond_signal);
1772 TSAN_INTERCEPT(pthread_cond_broadcast);
1773 TSAN_INTERCEPT(pthread_cond_wait);
1774 TSAN_INTERCEPT(pthread_cond_timedwait);
1776 TSAN_INTERCEPT(pthread_barrier_init);
1777 TSAN_INTERCEPT(pthread_barrier_destroy);
1778 TSAN_INTERCEPT(pthread_barrier_wait);
1780 TSAN_INTERCEPT(pthread_once);
1782 TSAN_INTERCEPT(sem_init);
1783 TSAN_INTERCEPT(sem_destroy);
1784 TSAN_INTERCEPT(sem_wait);
1785 TSAN_INTERCEPT(sem_trywait);
1786 TSAN_INTERCEPT(sem_timedwait);
1787 TSAN_INTERCEPT(sem_post);
1788 TSAN_INTERCEPT(sem_getvalue);
1790 TSAN_INTERCEPT(open);
1791 TSAN_INTERCEPT(open64);
1792 TSAN_INTERCEPT(creat);
1793 TSAN_INTERCEPT(creat64);
1794 TSAN_INTERCEPT(dup);
1795 TSAN_INTERCEPT(dup2);
1796 TSAN_INTERCEPT(dup3);
1797 TSAN_INTERCEPT(eventfd);
1798 TSAN_INTERCEPT(signalfd);
1799 TSAN_INTERCEPT(inotify_init);
1800 TSAN_INTERCEPT(inotify_init1);
1801 TSAN_INTERCEPT(socket);
1802 TSAN_INTERCEPT(socketpair);
1803 TSAN_INTERCEPT(connect);
1804 TSAN_INTERCEPT(accept);
1805 TSAN_INTERCEPT(accept4);
1806 TSAN_INTERCEPT(epoll_create);
1807 TSAN_INTERCEPT(epoll_create1);
1808 TSAN_INTERCEPT(close);
1809 TSAN_INTERCEPT(pipe);
1810 TSAN_INTERCEPT(pipe2);
1812 TSAN_INTERCEPT(read);
1813 TSAN_INTERCEPT(pread);
1814 TSAN_INTERCEPT(pread64);
1815 TSAN_INTERCEPT(readv);
1816 TSAN_INTERCEPT(preadv64);
1817 TSAN_INTERCEPT(write);
1818 TSAN_INTERCEPT(pwrite);
1819 TSAN_INTERCEPT(pwrite64);
1820 TSAN_INTERCEPT(writev);
1821 TSAN_INTERCEPT(pwritev64);
1822 TSAN_INTERCEPT(send);
1823 TSAN_INTERCEPT(sendmsg);
1824 TSAN_INTERCEPT(recv);
1825 TSAN_INTERCEPT(recvmsg);
1827 TSAN_INTERCEPT(unlink);
1828 TSAN_INTERCEPT(fopen);
1829 TSAN_INTERCEPT(freopen);
1830 TSAN_INTERCEPT(fclose);
1831 TSAN_INTERCEPT(fread);
1832 TSAN_INTERCEPT(fwrite);
1833 TSAN_INTERCEPT(puts);
1834 TSAN_INTERCEPT(rmdir);
1835 TSAN_INTERCEPT(opendir);
1837 TSAN_INTERCEPT(epoll_ctl);
1838 TSAN_INTERCEPT(epoll_wait);
1839 TSAN_INTERCEPT(poll);
1841 TSAN_INTERCEPT(sigaction);
1842 TSAN_INTERCEPT(signal);
1843 TSAN_INTERCEPT(raise);
1844 TSAN_INTERCEPT(kill);
1845 TSAN_INTERCEPT(pthread_kill);
1846 TSAN_INTERCEPT(sleep);
1847 TSAN_INTERCEPT(usleep);
1848 TSAN_INTERCEPT(nanosleep);
1849 TSAN_INTERCEPT(gettimeofday);
1851 TSAN_INTERCEPT(mlock);
1852 TSAN_INTERCEPT(munlock);
1853 TSAN_INTERCEPT(mlockall);
1854 TSAN_INTERCEPT(munlockall);
1856 TSAN_INTERCEPT(fork);
1858 // Need to setup it, because interceptors check that the function is resolved.
1859 // But atexit is emitted directly into the module, so can't be resolved.
1860 REAL(atexit) = (int(*)(void(*)()))unreachable;
1861 atexit_ctx = new(internal_alloc(MBlockAtExit, sizeof(AtExitContext)))
1862 AtExitContext();
1864 if (__cxa_atexit(&finalize, 0, 0)) {
1865 Printf("ThreadSanitizer: failed to setup atexit callback\n");
1866 Die();
1869 if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) {
1870 Printf("ThreadSanitizer: failed to create thread key\n");
1871 Die();
1874 FdInit();
1877 void internal_start_thread(void(*func)(void *arg), void *arg) {
1878 void *th;
1879 REAL(pthread_create)(&th, 0, (void*(*)(void *arg))func, arg);
1880 REAL(pthread_detach)(th);
1883 } // namespace __tsan