lib/tsan/rtl/tsan_interceptors.cc

   1 //===-- tsan_interceptors.cc ----------------------------------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file is a part of ThreadSanitizer (TSan), a race detector.
  11 //
  12 // FIXME: move as many interceptors as possible into
  13 // sanitizer_common/sanitizer_common_interceptors.inc
  14 //===----------------------------------------------------------------------===//
  15
  16 #include "sanitizer_common/sanitizer_atomic.h"
  17 #include "sanitizer_common/sanitizer_libc.h"
  18 #include "sanitizer_common/sanitizer_linux.h"
  19 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
  20 #include "sanitizer_common/sanitizer_placement_new.h"
  21 #include "sanitizer_common/sanitizer_stacktrace.h"
  22 #include "interception/interception.h"
  23 #include "tsan_interface.h"
  24 #include "tsan_platform.h"
  25 #include "tsan_suppressions.h"
  26 #include "tsan_rtl.h"
  27 #include "tsan_mman.h"
  28 #include "tsan_fd.h"
  29
  30 using namespace __tsan;  // NOLINT
  31
  32 const int kSigCount = 64;
  33
  34 struct my_siginfo_t {
  35   // The size is determined by looking at sizeof of real siginfo_t on linux.
  36   u64 opaque[128 / sizeof(u64)];
  37 };
  38
  39 struct ucontext_t {
  40   // The size is determined by looking at sizeof of real ucontext_t on linux.
  41   u64 opaque[936 / sizeof(u64) + 1];
  42 };
  43
  44 extern "C" int pthread_attr_init(void *attr);
  45 extern "C" int pthread_attr_destroy(void *attr);
  46 DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *)
  47 extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize);
  48 extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v));
  49 extern "C" int pthread_setspecific(unsigned key, const void *v);
  50 extern "C" int pthread_mutexattr_gettype(void *a, int *type);
  51 extern "C" int pthread_yield();
  52 extern "C" int pthread_sigmask(int how, const __sanitizer_sigset_t *set,
  53                                __sanitizer_sigset_t *oldset);
  54 // REAL(sigfillset) defined in common interceptors.
  55 DECLARE_REAL(int, sigfillset, __sanitizer_sigset_t *set)
  56 extern "C" void *pthread_self();
  57 extern "C" void _exit(int status);
  58 extern "C" int *__errno_location();
  59 extern "C" int fileno_unlocked(void *stream);
  60 extern "C" void *__libc_malloc(uptr size);
  61 extern "C" void *__libc_calloc(uptr size, uptr n);
  62 extern "C" void *__libc_realloc(void *ptr, uptr size);
  63 extern "C" void __libc_free(void *ptr);
  64 extern "C" int mallopt(int param, int value);
  65 const int PTHREAD_MUTEX_RECURSIVE = 1;
  66 const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
  67 const int EINVAL = 22;
  68 const int EBUSY = 16;
  69 const int EOWNERDEAD = 130;
  70 const int EPOLL_CTL_ADD = 1;
  71 const int SIGILL = 4;
  72 const int SIGABRT = 6;
  73 const int SIGFPE = 8;
  74 const int SIGSEGV = 11;
  75 const int SIGPIPE = 13;
  76 const int SIGBUS = 7;
  77 const int SIGSYS = 31;
  78 void *const MAP_FAILED = (void*)-1;
  79 const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
  80 const int MAP_FIXED = 0x10;
  81 typedef long long_t;  // NOLINT
  82
  83 // From /usr/include/unistd.h
  84 # define F_ULOCK 0      /* Unlock a previously locked region.  */
  85 # define F_LOCK  1      /* Lock a region for exclusive use.  */
  86 # define F_TLOCK 2      /* Test and lock a region for exclusive use.  */
  87 # define F_TEST  3      /* Test a region for other processes locks.  */
  88
  89 typedef void (*sighandler_t)(int sig);
  90
  91 #define errno (*__errno_location())
  92
  93 struct sigaction_t {
  94   union {
  95     sighandler_t sa_handler;
  96     void (*sa_sigaction)(int sig, my_siginfo_t *siginfo, void *uctx);
  97   };
  98   __sanitizer_sigset_t sa_mask;
  99   int sa_flags;
 100   void (*sa_restorer)();
 101 };
 102
 103 const sighandler_t SIG_DFL = (sighandler_t)0;
 104 const sighandler_t SIG_IGN = (sighandler_t)1;
 105 const sighandler_t SIG_ERR = (sighandler_t)-1;
 106 const int SA_SIGINFO = 4;
 107 const int SIG_SETMASK = 2;
 108
 109 namespace std {
 110 struct nothrow_t {};
 111 }  // namespace std
 112
 113 static sigaction_t sigactions[kSigCount];
 114
 115 namespace __tsan {
 116 struct SignalDesc {
 117   bool armed;
 118   bool sigaction;
 119   my_siginfo_t siginfo;
 120   ucontext_t ctx;
 121 };
 122
 123 struct SignalContext {
 124   int in_blocking_func;
 125   int int_signal_send;
 126   int pending_signal_count;
 127   SignalDesc pending_signals[kSigCount];
 128 };
 129
 130 // The object is 64-byte aligned, because we want hot data to be located in
 131 // a single cache line if possible (it's accessed in every interceptor).
 132 static ALIGNED(64) char libignore_placeholder[sizeof(LibIgnore)];
 133 static LibIgnore *libignore() {
 134   return reinterpret_cast<LibIgnore*>(&libignore_placeholder[0]);
 135 }
 136
 137 void InitializeLibIgnore() {
 138   libignore()->Init(*GetSuppressionContext());
 139   libignore()->OnLibraryLoaded(0);
 140 }
 141
 142 }  // namespace __tsan
 143
 144 static SignalContext *SigCtx(ThreadState *thr) {
 145   SignalContext *ctx = (SignalContext*)thr->signal_ctx;
 146   if (ctx == 0 && thr->is_alive) {
 147     ScopedInRtl in_rtl;
 148     ctx = (SignalContext*)MmapOrDie(sizeof(*ctx), "SignalContext");
 149     MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx));
 150     thr->signal_ctx = ctx;
 151   }
 152   return ctx;
 153 }
 154
 155 static unsigned g_thread_finalize_key;
 156
 157 class ScopedInterceptor {
 158  public:
 159   ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc);
 160   ~ScopedInterceptor();
 161  private:
 162   ThreadState *const thr_;
 163   const int in_rtl_;
 164   bool in_ignored_lib_;
 165 };
 166
 167 ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
 168                                      uptr pc)
 169     : thr_(thr)
 170     , in_rtl_(thr->in_rtl)
 171     , in_ignored_lib_(false) {
 172   if (thr_->in_rtl == 0) {
 173     Initialize(thr);
 174     FuncEntry(thr, pc);
 175     thr_->in_rtl++;
 176     DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
 177   } else {
 178     thr_->in_rtl++;
 179   }
 180   if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) {
 181     in_ignored_lib_ = true;
 182     thr_->in_ignored_lib = true;
 183     ThreadIgnoreBegin(thr_);
 184   }
 185 }
 186
 187 ScopedInterceptor::~ScopedInterceptor() {
 188   if (in_ignored_lib_) {
 189     thr_->in_ignored_lib = false;
 190     ThreadIgnoreEnd(thr_);
 191   }
 192   thr_->in_rtl--;
 193   if (thr_->in_rtl == 0) {
 194     FuncExit(thr_);
 195     ProcessPendingSignals(thr_);
 196   }
 197   CHECK_EQ(in_rtl_, thr_->in_rtl);
 198 }
 199
 200 #define SCOPED_INTERCEPTOR_RAW(func, ...) \
 201     ThreadState *thr = cur_thread(); \
 202     StatInc(thr, StatInterceptor); \
 203     StatInc(thr, StatInt_##func); \
 204     const uptr caller_pc = GET_CALLER_PC(); \
 205     ScopedInterceptor si(thr, #func, caller_pc); \
 206     const uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \
 207     (void)pc; \
 208 /**/
 209
 210 #define SCOPED_TSAN_INTERCEPTOR(func, ...) \
 211     SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
 212     if (REAL(func) == 0) { \
 213       Printf("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
 214       Die(); \
 215     } \
 216     if (thr->in_rtl > 1 || thr->in_ignored_lib) \
 217       return REAL(func)(__VA_ARGS__); \
 218 /**/
 219
 220 #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
 221 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
 222
 223 #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name))
 224
 225 struct BlockingCall {
 226   explicit BlockingCall(ThreadState *thr)
 227       : ctx(SigCtx(thr)) {
 228     ctx->in_blocking_func++;
 229   }
 230
 231   ~BlockingCall() {
 232     ctx->in_blocking_func--;
 233   }
 234
 235   SignalContext *ctx;
 236 };
 237
 238 TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) {
 239   SCOPED_TSAN_INTERCEPTOR(sleep, sec);
 240   unsigned res = BLOCK_REAL(sleep)(sec);
 241   AfterSleep(thr, pc);
 242   return res;
 243 }
 244
 245 TSAN_INTERCEPTOR(int, usleep, long_t usec) {
 246   SCOPED_TSAN_INTERCEPTOR(usleep, usec);
 247   int res = BLOCK_REAL(usleep)(usec);
 248   AfterSleep(thr, pc);
 249   return res;
 250 }
 251
 252 TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) {
 253   SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem);
 254   int res = BLOCK_REAL(nanosleep)(req, rem);
 255   AfterSleep(thr, pc);
 256   return res;
 257 }
 258
 259 TSAN_INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
 260   SCOPED_INTERCEPTOR_RAW(dlopen, filename, flag);
 261   // dlopen will execute global constructors, so it must be not in rtl.
 262   CHECK_EQ(thr->in_rtl, 1);
 263   thr->in_rtl = 0;
 264   void *res = REAL(dlopen)(filename, flag);
 265   thr->in_rtl = 1;
 266   libignore()->OnLibraryLoaded(filename);
 267   return res;
 268 }
 269
 270 TSAN_INTERCEPTOR(int, dlclose, void *handle) {
 271   SCOPED_INTERCEPTOR_RAW(dlclose, handle);
 272   // dlclose will execute global destructors, so it must be not in rtl.
 273   CHECK_EQ(thr->in_rtl, 1);
 274   thr->in_rtl = 0;
 275   int res = REAL(dlclose)(handle);
 276   thr->in_rtl = 1;
 277   libignore()->OnLibraryUnloaded();
 278   return res;
 279 }
 280
 281 class AtExitContext {
 282  public:
 283   AtExitContext()
 284     : mtx_(MutexTypeAtExit, StatMtxAtExit)
 285     , pos_() {
 286   }
 287
 288   typedef void(*atexit_t)();
 289
 290   int atexit(ThreadState *thr, uptr pc, bool is_on_exit,
 291              atexit_t f, void *arg) {
 292     Lock l(&mtx_);
 293     if (pos_ == kMaxAtExit)
 294       return 1;
 295     Release(thr, pc, (uptr)this);
 296     stack_[pos_] = f;
 297     args_[pos_] = arg;
 298     is_on_exits_[pos_] = is_on_exit;
 299     pos_++;
 300     return 0;
 301   }
 302
 303   void exit(ThreadState *thr, uptr pc) {
 304     CHECK_EQ(thr->in_rtl, 0);
 305     for (;;) {
 306       atexit_t f = 0;
 307       void *arg = 0;
 308       bool is_on_exit = false;
 309       {
 310         Lock l(&mtx_);
 311         if (pos_) {
 312           pos_--;
 313           f = stack_[pos_];
 314           arg = args_[pos_];
 315           is_on_exit = is_on_exits_[pos_];
 316           ScopedInRtl in_rtl;
 317           Acquire(thr, pc, (uptr)this);
 318         }
 319       }
 320       if (f == 0)
 321         break;
 322       DPrintf("#%d: executing atexit func %p\n", thr->tid, f);
 323       CHECK_EQ(thr->in_rtl, 0);
 324       if (is_on_exit)
 325         ((void(*)(int status, void *arg))f)(0, arg);
 326       else
 327         ((void(*)(void *arg, void *dso))f)(arg, 0);
 328     }
 329   }
 330
 331  private:
 332   static const int kMaxAtExit = 128;
 333   Mutex mtx_;
 334   atexit_t stack_[kMaxAtExit];
 335   void *args_[kMaxAtExit];
 336   bool is_on_exits_[kMaxAtExit];
 337   int pos_;
 338 };
 339
 340 static AtExitContext *atexit_ctx;
 341
 342 TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
 343   if (cur_thread()->in_symbolizer)
 344     return 0;
 345   SCOPED_TSAN_INTERCEPTOR(atexit, f);
 346   return atexit_ctx->atexit(thr, pc, false, (void(*)())f, 0);
 347 }
 348
 349 TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) {
 350   if (cur_thread()->in_symbolizer)
 351     return 0;
 352   SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
 353   return atexit_ctx->atexit(thr, pc, true, (void(*)())f, arg);
 354 }
 355
 356 TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) {
 357   if (cur_thread()->in_symbolizer)
 358     return 0;
 359   SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
 360   if (dso) {
 361     // Memory allocation in __cxa_atexit will race with free during exit,
 362     // because we do not see synchronization around atexit callback list.
 363     ThreadIgnoreBegin(thr);
 364     int res = REAL(__cxa_atexit)(f, arg, dso);
 365     ThreadIgnoreEnd(thr);
 366     return res;
 367   }
 368   return atexit_ctx->atexit(thr, pc, false, (void(*)())f, arg);
 369 }
 370
 371 // Cleanup old bufs.
 372 static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) {
 373   for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
 374     JmpBuf *buf = &thr->jmp_bufs[i];
 375     if (buf->sp <= sp) {
 376       uptr sz = thr->jmp_bufs.Size();
 377       thr->jmp_bufs[i] = thr->jmp_bufs[sz - 1];
 378       thr->jmp_bufs.PopBack();
 379       i--;
 380     }
 381   }
 382 }
 383
 384 static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) {
 385   if (thr->shadow_stack_pos == 0)  // called from libc guts during bootstrap
 386     return;
 387   // Cleanup old bufs.
 388   JmpBufGarbageCollect(thr, sp);
 389   // Remember the buf.
 390   JmpBuf *buf = thr->jmp_bufs.PushBack();
 391   buf->sp = sp;
 392   buf->mangled_sp = mangled_sp;
 393   buf->shadow_stack_pos = thr->shadow_stack_pos;
 394 }
 395
 396 static void LongJmp(ThreadState *thr, uptr *env) {
 397   uptr mangled_sp = env[6];
 398   // Find the saved buf by mangled_sp.
 399   for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
 400     JmpBuf *buf = &thr->jmp_bufs[i];
 401     if (buf->mangled_sp == mangled_sp) {
 402       CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos);
 403       // Unwind the stack.
 404       while (thr->shadow_stack_pos > buf->shadow_stack_pos)
 405         FuncExit(thr);
 406       JmpBufGarbageCollect(thr, buf->sp - 1);  // do not collect buf->sp
 407       return;
 408     }
 409   }
 410   Printf("ThreadSanitizer: can't find longjmp buf\n");
 411   CHECK(0);
 412 }
 413
 414 // FIXME: put everything below into a common extern "C" block?
 415 extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) {
 416   ScopedInRtl in_rtl;
 417   SetJmp(cur_thread(), sp, mangled_sp);
 418 }
 419
 420 // Not called.  Merely to satisfy TSAN_INTERCEPT().
 421 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 422 int __interceptor_setjmp(void *env);
 423 extern "C" int __interceptor_setjmp(void *env) {
 424   CHECK(0);
 425   return 0;
 426 }
 427
 428 // FIXME: any reason to have a separate declaration?
 429 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 430 int __interceptor__setjmp(void *env);
 431 extern "C" int __interceptor__setjmp(void *env) {
 432   CHECK(0);
 433   return 0;
 434 }
 435
 436 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 437 int __interceptor_sigsetjmp(void *env);
 438 extern "C" int __interceptor_sigsetjmp(void *env) {
 439   CHECK(0);
 440   return 0;
 441 }
 442
 443 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 444 int __interceptor___sigsetjmp(void *env);
 445 extern "C" int __interceptor___sigsetjmp(void *env) {
 446   CHECK(0);
 447   return 0;
 448 }
 449
 450 extern "C" int setjmp(void *env);
 451 extern "C" int _setjmp(void *env);
 452 extern "C" int sigsetjmp(void *env);
 453 extern "C" int __sigsetjmp(void *env);
 454 DEFINE_REAL(int, setjmp, void *env)
 455 DEFINE_REAL(int, _setjmp, void *env)
 456 DEFINE_REAL(int, sigsetjmp, void *env)
 457 DEFINE_REAL(int, __sigsetjmp, void *env)
 458
 459 TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) {
 460   {
 461     SCOPED_TSAN_INTERCEPTOR(longjmp, env, val);
 462   }
 463   LongJmp(cur_thread(), env);
 464   REAL(longjmp)(env, val);
 465 }
 466
 467 TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) {
 468   {
 469     SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val);
 470   }
 471   LongJmp(cur_thread(), env);
 472   REAL(siglongjmp)(env, val);
 473 }
 474
 475 TSAN_INTERCEPTOR(void*, malloc, uptr size) {
 476   if (cur_thread()->in_symbolizer)
 477     return __libc_malloc(size);
 478   void *p = 0;
 479   {
 480     SCOPED_INTERCEPTOR_RAW(malloc, size);
 481     p = user_alloc(thr, pc, size);
 482   }
 483   invoke_malloc_hook(p, size);
 484   return p;
 485 }
 486
 487 TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
 488   SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz);
 489   return user_alloc(thr, pc, sz, align);
 490 }
 491
 492 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
 493   if (cur_thread()->in_symbolizer)
 494     return __libc_calloc(size, n);
 495   if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n))
 496     return AllocatorReturnNull();
 497   void *p = 0;
 498   {
 499     SCOPED_INTERCEPTOR_RAW(calloc, size, n);
 500     p = user_alloc(thr, pc, n * size);
 501     if (p)
 502       internal_memset(p, 0, n * size);
 503   }
 504   invoke_malloc_hook(p, n * size);
 505   return p;
 506 }
 507
 508 TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
 509   if (cur_thread()->in_symbolizer)
 510     return __libc_realloc(p, size);
 511   if (p)
 512     invoke_free_hook(p);
 513   {
 514     SCOPED_INTERCEPTOR_RAW(realloc, p, size);
 515     p = user_realloc(thr, pc, p, size);
 516   }
 517   invoke_malloc_hook(p, size);
 518   return p;
 519 }
 520
 521 TSAN_INTERCEPTOR(void, free, void *p) {
 522   if (p == 0)
 523     return;
 524   if (cur_thread()->in_symbolizer)
 525     return __libc_free(p);
 526   invoke_free_hook(p);
 527   SCOPED_INTERCEPTOR_RAW(free, p);
 528   user_free(thr, pc, p);
 529 }
 530
 531 TSAN_INTERCEPTOR(void, cfree, void *p) {
 532   if (p == 0)
 533     return;
 534   if (cur_thread()->in_symbolizer)
 535     return __libc_free(p);
 536   invoke_free_hook(p);
 537   SCOPED_INTERCEPTOR_RAW(cfree, p);
 538   user_free(thr, pc, p);
 539 }
 540
 541 TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) {
 542   SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p);
 543   return user_alloc_usable_size(thr, pc, p);
 544 }
 545
 546 #define OPERATOR_NEW_BODY(mangled_name) \
 547   if (cur_thread()->in_symbolizer) \
 548     return __libc_malloc(size); \
 549   void *p = 0; \
 550   {  \
 551     SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
 552     p = user_alloc(thr, pc, size); \
 553   }  \
 554   invoke_malloc_hook(p, size);  \
 555   return p;
 556
 557 SANITIZER_INTERFACE_ATTRIBUTE
 558 void *operator new(__sanitizer::uptr size);
 559 void *operator new(__sanitizer::uptr size) {
 560   OPERATOR_NEW_BODY(_Znwm);
 561 }
 562
 563 SANITIZER_INTERFACE_ATTRIBUTE
 564 void *operator new[](__sanitizer::uptr size);
 565 void *operator new[](__sanitizer::uptr size) {
 566   OPERATOR_NEW_BODY(_Znam);
 567 }
 568
 569 SANITIZER_INTERFACE_ATTRIBUTE
 570 void *operator new(__sanitizer::uptr size, std::nothrow_t const&);
 571 void *operator new(__sanitizer::uptr size, std::nothrow_t const&) {
 572   OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t);
 573 }
 574
 575 SANITIZER_INTERFACE_ATTRIBUTE
 576 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&);
 577 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) {
 578   OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t);
 579 }
 580
 581 #define OPERATOR_DELETE_BODY(mangled_name) \
 582   if (ptr == 0) return;  \
 583   if (cur_thread()->in_symbolizer) \
 584     return __libc_free(ptr); \
 585   invoke_free_hook(ptr);  \
 586   SCOPED_INTERCEPTOR_RAW(mangled_name, ptr);  \
 587   user_free(thr, pc, ptr);
 588
 589 SANITIZER_INTERFACE_ATTRIBUTE
 590 void operator delete(void *ptr);
 591 void operator delete(void *ptr) {
 592   OPERATOR_DELETE_BODY(_ZdlPv);
 593 }
 594
 595 SANITIZER_INTERFACE_ATTRIBUTE
 596 void operator delete[](void *ptr);
 597 void operator delete[](void *ptr) {
 598   OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t);
 599 }
 600
 601 SANITIZER_INTERFACE_ATTRIBUTE
 602 void operator delete(void *ptr, std::nothrow_t const&);
 603 void operator delete(void *ptr, std::nothrow_t const&) {
 604   OPERATOR_DELETE_BODY(_ZdaPv);
 605 }
 606
 607 SANITIZER_INTERFACE_ATTRIBUTE
 608 void operator delete[](void *ptr, std::nothrow_t const&);
 609 void operator delete[](void *ptr, std::nothrow_t const&) {
 610   OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t);
 611 }
 612
 613 TSAN_INTERCEPTOR(uptr, strlen, const char *s) {
 614   SCOPED_TSAN_INTERCEPTOR(strlen, s);
 615   uptr len = internal_strlen(s);
 616   MemoryAccessRange(thr, pc, (uptr)s, len + 1, false);
 617   return len;
 618 }
 619
 620 TSAN_INTERCEPTOR(void*, memset, void *dst, int v, uptr size) {
 621   SCOPED_TSAN_INTERCEPTOR(memset, dst, v, size);
 622   MemoryAccessRange(thr, pc, (uptr)dst, size, true);
 623   return internal_memset(dst, v, size);
 624 }
 625
 626 TSAN_INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) {
 627   SCOPED_TSAN_INTERCEPTOR(memcpy, dst, src, size);
 628   MemoryAccessRange(thr, pc, (uptr)dst, size, true);
 629   MemoryAccessRange(thr, pc, (uptr)src, size, false);
 630   return internal_memcpy(dst, src, size);
 631 }
 632
 633 TSAN_INTERCEPTOR(int, memcmp, const void *s1, const void *s2, uptr n) {
 634   SCOPED_TSAN_INTERCEPTOR(memcmp, s1, s2, n);
 635   int res = 0;
 636   uptr len = 0;
 637   for (; len < n; len++) {
 638     if ((res = ((unsigned char*)s1)[len] - ((unsigned char*)s2)[len]))
 639       break;
 640   }
 641   MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false);
 642   MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false);
 643   return res;
 644 }
 645
 646 TSAN_INTERCEPTOR(void*, memchr, void *s, int c, uptr n) {
 647   SCOPED_TSAN_INTERCEPTOR(memchr, s, c, n);
 648   void *res = REAL(memchr)(s, c, n);
 649   uptr len = res ? (char*)res - (char*)s + 1 : n;
 650   MemoryAccessRange(thr, pc, (uptr)s, len, false);
 651   return res;
 652 }
 653
 654 TSAN_INTERCEPTOR(void*, memrchr, char *s, int c, uptr n) {
 655   SCOPED_TSAN_INTERCEPTOR(memrchr, s, c, n);
 656   MemoryAccessRange(thr, pc, (uptr)s, n, false);
 657   return REAL(memrchr)(s, c, n);
 658 }
 659
 660 TSAN_INTERCEPTOR(void*, memmove, void *dst, void *src, uptr n) {
 661   SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n);
 662   MemoryAccessRange(thr, pc, (uptr)dst, n, true);
 663   MemoryAccessRange(thr, pc, (uptr)src, n, false);
 664   return REAL(memmove)(dst, src, n);
 665 }
 666
 667 TSAN_INTERCEPTOR(char*, strchr, char *s, int c) {
 668   SCOPED_TSAN_INTERCEPTOR(strchr, s, c);
 669   char *res = REAL(strchr)(s, c);
 670   uptr len = res ? (char*)res - (char*)s + 1 : internal_strlen(s) + 1;
 671   MemoryAccessRange(thr, pc, (uptr)s, len, false);
 672   return res;
 673 }
 674
 675 TSAN_INTERCEPTOR(char*, strchrnul, char *s, int c) {
 676   SCOPED_TSAN_INTERCEPTOR(strchrnul, s, c);
 677   char *res = REAL(strchrnul)(s, c);
 678   uptr len = (char*)res - (char*)s + 1;
 679   MemoryAccessRange(thr, pc, (uptr)s, len, false);
 680   return res;
 681 }
 682
 683 TSAN_INTERCEPTOR(char*, strrchr, char *s, int c) {
 684   SCOPED_TSAN_INTERCEPTOR(strrchr, s, c);
 685   MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s) + 1, false);
 686   return REAL(strrchr)(s, c);
 687 }
 688
 689 TSAN_INTERCEPTOR(char*, strcpy, char *dst, const char *src) {  // NOLINT
 690   SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src);  // NOLINT
 691   uptr srclen = internal_strlen(src);
 692   MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true);
 693   MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false);
 694   return REAL(strcpy)(dst, src);  // NOLINT
 695 }
 696
 697 TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) {
 698   SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n);
 699   uptr srclen = internal_strnlen(src, n);
 700   MemoryAccessRange(thr, pc, (uptr)dst, n, true);
 701   MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false);
 702   return REAL(strncpy)(dst, src, n);
 703 }
 704
 705 TSAN_INTERCEPTOR(const char*, strstr, const char *s1, const char *s2) {
 706   SCOPED_TSAN_INTERCEPTOR(strstr, s1, s2);
 707   const char *res = REAL(strstr)(s1, s2);
 708   uptr len1 = internal_strlen(s1);
 709   uptr len2 = internal_strlen(s2);
 710   MemoryAccessRange(thr, pc, (uptr)s1, len1 + 1, false);
 711   MemoryAccessRange(thr, pc, (uptr)s2, len2 + 1, false);
 712   return res;
 713 }
 714
 715 TSAN_INTERCEPTOR(char*, strdup, const char *str) {
 716   SCOPED_TSAN_INTERCEPTOR(strdup, str);
 717   // strdup will call malloc, so no instrumentation is required here.
 718   return REAL(strdup)(str);
 719 }
 720
 721 static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
 722   if (*addr) {
 723     if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
 724       if (flags & MAP_FIXED) {
 725         errno = EINVAL;
 726         return false;
 727       } else {
 728         *addr = 0;
 729       }
 730     }
 731   }
 732   return true;
 733 }
 734
 735 TSAN_INTERCEPTOR(void*, mmap, void *addr, long_t sz, int prot,
 736                          int flags, int fd, unsigned off) {
 737   SCOPED_TSAN_INTERCEPTOR(mmap, addr, sz, prot, flags, fd, off);
 738   if (!fix_mmap_addr(&addr, sz, flags))
 739     return MAP_FAILED;
 740   void *res = REAL(mmap)(addr, sz, prot, flags, fd, off);
 741   if (res != MAP_FAILED) {
 742     if (fd > 0)
 743       FdAccess(thr, pc, fd);
 744     MemoryRangeImitateWrite(thr, pc, (uptr)res, sz);
 745   }
 746   return res;
 747 }
 748
 749 TSAN_INTERCEPTOR(void*, mmap64, void *addr, long_t sz, int prot,
 750                            int flags, int fd, u64 off) {
 751   SCOPED_TSAN_INTERCEPTOR(mmap64, addr, sz, prot, flags, fd, off);
 752   if (!fix_mmap_addr(&addr, sz, flags))
 753     return MAP_FAILED;
 754   void *res = REAL(mmap64)(addr, sz, prot, flags, fd, off);
 755   if (res != MAP_FAILED) {
 756     if (fd > 0)
 757       FdAccess(thr, pc, fd);
 758     MemoryRangeImitateWrite(thr, pc, (uptr)res, sz);
 759   }
 760   return res;
 761 }
 762
 763 TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
 764   SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz);
 765   DontNeedShadowFor((uptr)addr, sz);
 766   int res = REAL(munmap)(addr, sz);
 767   return res;
 768 }
 769
 770 TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
 771   SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
 772   return user_alloc(thr, pc, sz, align);
 773 }
 774
 775 TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
 776   SCOPED_INTERCEPTOR_RAW(valloc, sz);
 777   return user_alloc(thr, pc, sz, GetPageSizeCached());
 778 }
 779
 780 TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
 781   SCOPED_INTERCEPTOR_RAW(pvalloc, sz);
 782   sz = RoundUp(sz, GetPageSizeCached());
 783   return user_alloc(thr, pc, sz, GetPageSizeCached());
 784 }
 785
 786 TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
 787   SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz);
 788   *memptr = user_alloc(thr, pc, sz, align);
 789   return 0;
 790 }
 791
 792 // Used in thread-safe function static initialization.
 793 extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) {
 794   SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g);
 795   for (;;) {
 796     u32 cmp = atomic_load(g, memory_order_acquire);
 797     if (cmp == 0) {
 798       if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed))
 799         return 1;
 800     } else if (cmp == 1) {
 801       Acquire(thr, pc, (uptr)g);
 802       return 0;
 803     } else {
 804       internal_sched_yield();
 805     }
 806   }
 807 }
 808
 809 extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_release(atomic_uint32_t *g) {
 810   SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);
 811   Release(thr, pc, (uptr)g);
 812   atomic_store(g, 1, memory_order_release);
 813 }
 814
 815 extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_abort(atomic_uint32_t *g) {
 816   SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
 817   atomic_store(g, 0, memory_order_relaxed);
 818 }
 819
 820 static void thread_finalize(void *v) {
 821   uptr iter = (uptr)v;
 822   if (iter > 1) {
 823     if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) {
 824       Printf("ThreadSanitizer: failed to set thread key\n");
 825       Die();
 826     }
 827     return;
 828   }
 829   {
 830     ScopedInRtl in_rtl;
 831     ThreadState *thr = cur_thread();
 832     ThreadFinish(thr);
 833     SignalContext *sctx = thr->signal_ctx;
 834     if (sctx) {
 835       thr->signal_ctx = 0;
 836       UnmapOrDie(sctx, sizeof(*sctx));
 837     }
 838   }
 839 }
 840
 841
 842 struct ThreadParam {
 843   void* (*callback)(void *arg);
 844   void *param;
 845   atomic_uintptr_t tid;
 846 };
 847
 848 extern "C" void *__tsan_thread_start_func(void *arg) {
 849   ThreadParam *p = (ThreadParam*)arg;
 850   void* (*callback)(void *arg) = p->callback;
 851   void *param = p->param;
 852   int tid = 0;
 853   {
 854     ThreadState *thr = cur_thread();
 855     ScopedInRtl in_rtl;
 856     if (pthread_setspecific(g_thread_finalize_key,
 857                             (void *)kPthreadDestructorIterations)) {
 858       Printf("ThreadSanitizer: failed to set thread key\n");
 859       Die();
 860     }
 861     while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)
 862       pthread_yield();
 863     atomic_store(&p->tid, 0, memory_order_release);
 864     ThreadStart(thr, tid, GetTid());
 865     CHECK_EQ(thr->in_rtl, 1);
 866   }
 867   void *res = callback(param);
 868   // Prevent the callback from being tail called,
 869   // it mixes up stack traces.
 870   volatile int foo = 42;
 871   foo++;
 872   return res;
 873 }
 874
 875 TSAN_INTERCEPTOR(int, pthread_create,
 876     void *th, void *attr, void *(*callback)(void*), void * param) {
 877   SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param);
 878   __sanitizer_pthread_attr_t myattr;
 879   if (attr == 0) {
 880     pthread_attr_init(&myattr);
 881     attr = &myattr;
 882   }
 883   int detached = 0;
 884   REAL(pthread_attr_getdetachstate)(attr, &detached);
 885   AdjustStackSizeLinux(attr);
 886
 887   ThreadParam p;
 888   p.callback = callback;
 889   p.param = param;
 890   atomic_store(&p.tid, 0, memory_order_relaxed);
 891   int res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p);
 892   if (res == 0) {
 893     int tid = ThreadCreate(thr, pc, *(uptr*)th, detached);
 894     CHECK_NE(tid, 0);
 895     atomic_store(&p.tid, tid, memory_order_release);
 896     while (atomic_load(&p.tid, memory_order_acquire) != 0)
 897       pthread_yield();
 898   }
 899   if (attr == &myattr)
 900     pthread_attr_destroy(&myattr);
 901   return res;
 902 }
 903
 904 TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) {
 905   SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret);
 906   int tid = ThreadTid(thr, pc, (uptr)th);
 907   int res = BLOCK_REAL(pthread_join)(th, ret);
 908   if (res == 0) {
 909     ThreadJoin(thr, pc, tid);
 910   }
 911   return res;
 912 }
 913
 914 TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
 915   SCOPED_TSAN_INTERCEPTOR(pthread_detach, th);
 916   int tid = ThreadTid(thr, pc, (uptr)th);
 917   int res = REAL(pthread_detach)(th);
 918   if (res == 0) {
 919     ThreadDetach(thr, pc, tid);
 920   }
 921   return res;
 922 }
 923
 924 TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
 925   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a);
 926   int res = REAL(pthread_mutex_init)(m, a);
 927   if (res == 0) {
 928     bool recursive = false;
 929     if (a) {
 930       int type = 0;
 931       if (pthread_mutexattr_gettype(a, &type) == 0)
 932         recursive = (type == PTHREAD_MUTEX_RECURSIVE
 933             || type == PTHREAD_MUTEX_RECURSIVE_NP);
 934     }
 935     MutexCreate(thr, pc, (uptr)m, false, recursive, false);
 936   }
 937   return res;
 938 }
 939
 940 TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
 941   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m);
 942   int res = REAL(pthread_mutex_destroy)(m);
 943   if (res == 0 || res == EBUSY) {
 944     MutexDestroy(thr, pc, (uptr)m);
 945   }
 946   return res;
 947 }
 948
 949 TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
 950   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
 951   int res = REAL(pthread_mutex_trylock)(m);
 952   if (res == EOWNERDEAD)
 953     MutexRepair(thr, pc, (uptr)m);
 954   if (res == 0 || res == EOWNERDEAD)
 955     MutexLock(thr, pc, (uptr)m);
 956   return res;
 957 }
 958
 959 TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) {
 960   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
 961   int res = REAL(pthread_mutex_timedlock)(m, abstime);
 962   if (res == 0) {
 963     MutexLock(thr, pc, (uptr)m);
 964   }
 965   return res;
 966 }
 967
 968 TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {
 969   SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);
 970   int res = REAL(pthread_spin_init)(m, pshared);
 971   if (res == 0) {
 972     MutexCreate(thr, pc, (uptr)m, false, false, false);
 973   }
 974   return res;
 975 }
 976
 977 TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
 978   SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m);
 979   int res = REAL(pthread_spin_destroy)(m);
 980   if (res == 0) {
 981     MutexDestroy(thr, pc, (uptr)m);
 982   }
 983   return res;
 984 }
 985
 986 TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
 987   SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
 988   int res = REAL(pthread_spin_lock)(m);
 989   if (res == 0) {
 990     MutexLock(thr, pc, (uptr)m);
 991   }
 992   return res;
 993 }
 994
 995 TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) {
 996   SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m);
 997   int res = REAL(pthread_spin_trylock)(m);
 998   if (res == 0) {
 999     MutexLock(thr, pc, (uptr)m);
1000   }
1001   return res;
1002 }
1003
1004 TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) {
1005   SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m);
1006   MutexUnlock(thr, pc, (uptr)m);
1007   int res = REAL(pthread_spin_unlock)(m);
1008   return res;
1009 }
1010
1011 TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) {
1012   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a);
1013   int res = REAL(pthread_rwlock_init)(m, a);
1014   if (res == 0) {
1015     MutexCreate(thr, pc, (uptr)m, true, false, false);
1016   }
1017   return res;
1018 }
1019
1020 TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) {
1021   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m);
1022   int res = REAL(pthread_rwlock_destroy)(m);
1023   if (res == 0) {
1024     MutexDestroy(thr, pc, (uptr)m);
1025   }
1026   return res;
1027 }
1028
1029 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) {
1030   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m);
1031   int res = REAL(pthread_rwlock_rdlock)(m);
1032   if (res == 0) {
1033     MutexReadLock(thr, pc, (uptr)m);
1034   }
1035   return res;
1036 }
1037
1038 TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
1039   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m);
1040   int res = REAL(pthread_rwlock_tryrdlock)(m);
1041   if (res == 0) {
1042     MutexReadLock(thr, pc, (uptr)m);
1043   }
1044   return res;
1045 }
1046
1047 TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) {
1048   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
1049   int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
1050   if (res == 0) {
1051     MutexReadLock(thr, pc, (uptr)m);
1052   }
1053   return res;
1054 }
1055
1056 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
1057   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
1058   int res = REAL(pthread_rwlock_wrlock)(m);
1059   if (res == 0) {
1060     MutexLock(thr, pc, (uptr)m);
1061   }
1062   return res;
1063 }
1064
1065 TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
1066   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m);
1067   int res = REAL(pthread_rwlock_trywrlock)(m);
1068   if (res == 0) {
1069     MutexLock(thr, pc, (uptr)m);
1070   }
1071   return res;
1072 }
1073
1074 TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) {
1075   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
1076   int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
1077   if (res == 0) {
1078     MutexLock(thr, pc, (uptr)m);
1079   }
1080   return res;
1081 }
1082
1083 TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) {
1084   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m);
1085   MutexReadOrWriteUnlock(thr, pc, (uptr)m);
1086   int res = REAL(pthread_rwlock_unlock)(m);
1087   return res;
1088 }
1089
1090 TSAN_INTERCEPTOR(int, pthread_cond_destroy, void *c) {
1091   SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy, c);
1092   MemoryWrite(thr, pc, (uptr)c, kSizeLog1);
1093   int res = REAL(pthread_cond_destroy)(c);
1094   return res;
1095 }
1096
1097 TSAN_INTERCEPTOR(int, pthread_cond_timedwait, void *c, void *m,
1098     void *abstime) {
1099   SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait, c, m, abstime);
1100   MutexUnlock(thr, pc, (uptr)m);
1101   MemoryRead(thr, pc, (uptr)c, kSizeLog1);
1102   int res = REAL(pthread_cond_timedwait)(c, m, abstime);
1103   MutexLock(thr, pc, (uptr)m);
1104   return res;
1105 }
1106
1107 TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) {
1108   SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count);
1109   MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
1110   int res = REAL(pthread_barrier_init)(b, a, count);
1111   return res;
1112 }
1113
1114 TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
1115   SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b);
1116   MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
1117   int res = REAL(pthread_barrier_destroy)(b);
1118   return res;
1119 }
1120
1121 TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
1122   SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b);
1123   Release(thr, pc, (uptr)b);
1124   MemoryRead(thr, pc, (uptr)b, kSizeLog1);
1125   int res = REAL(pthread_barrier_wait)(b);
1126   MemoryRead(thr, pc, (uptr)b, kSizeLog1);
1127   if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) {
1128     Acquire(thr, pc, (uptr)b);
1129   }
1130   return res;
1131 }
1132
1133 TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
1134   SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
1135   // Using SCOPED_INTERCEPTOR_RAW, because if we are called from an ignored lib,
1136   // the user callback must be executed with thr->in_rtl == 0.
1137   if (o == 0 || f == 0)
1138     return EINVAL;
1139   atomic_uint32_t *a = static_cast<atomic_uint32_t*>(o);
1140   u32 v = atomic_load(a, memory_order_acquire);
1141   if (v == 0 && atomic_compare_exchange_strong(a, &v, 1,
1142                                                memory_order_relaxed)) {
1143     const int old_in_rtl = thr->in_rtl;
1144     thr->in_rtl = 0;
1145     (*f)();
1146     CHECK_EQ(thr->in_rtl, 0);
1147     thr->in_rtl = old_in_rtl;
1148     if (!thr->in_ignored_lib)
1149       Release(thr, pc, (uptr)o);
1150     atomic_store(a, 2, memory_order_release);
1151   } else {
1152     while (v != 2) {
1153       pthread_yield();
1154       v = atomic_load(a, memory_order_acquire);
1155     }
1156     if (!thr->in_ignored_lib)
1157       Acquire(thr, pc, (uptr)o);
1158   }
1159   return 0;
1160 }
1161
1162 TSAN_INTERCEPTOR(int, sem_init, void *s, int pshared, unsigned value) {
1163   SCOPED_TSAN_INTERCEPTOR(sem_init, s, pshared, value);
1164   int res = REAL(sem_init)(s, pshared, value);
1165   return res;
1166 }
1167
1168 TSAN_INTERCEPTOR(int, sem_destroy, void *s) {
1169   SCOPED_TSAN_INTERCEPTOR(sem_destroy, s);
1170   int res = REAL(sem_destroy)(s);
1171   return res;
1172 }
1173
1174 TSAN_INTERCEPTOR(int, sem_wait, void *s) {
1175   SCOPED_TSAN_INTERCEPTOR(sem_wait, s);
1176   int res = BLOCK_REAL(sem_wait)(s);
1177   if (res == 0) {
1178     Acquire(thr, pc, (uptr)s);
1179   }
1180   return res;
1181 }
1182
1183 TSAN_INTERCEPTOR(int, sem_trywait, void *s) {
1184   SCOPED_TSAN_INTERCEPTOR(sem_trywait, s);
1185   int res = BLOCK_REAL(sem_trywait)(s);
1186   if (res == 0) {
1187     Acquire(thr, pc, (uptr)s);
1188   }
1189   return res;
1190 }
1191
1192 TSAN_INTERCEPTOR(int, sem_timedwait, void *s, void *abstime) {
1193   SCOPED_TSAN_INTERCEPTOR(sem_timedwait, s, abstime);
1194   int res = BLOCK_REAL(sem_timedwait)(s, abstime);
1195   if (res == 0) {
1196     Acquire(thr, pc, (uptr)s);
1197   }
1198   return res;
1199 }
1200
1201 TSAN_INTERCEPTOR(int, sem_post, void *s) {
1202   SCOPED_TSAN_INTERCEPTOR(sem_post, s);
1203   Release(thr, pc, (uptr)s);
1204   int res = REAL(sem_post)(s);
1205   return res;
1206 }
1207
1208 TSAN_INTERCEPTOR(int, sem_getvalue, void *s, int *sval) {
1209   SCOPED_TSAN_INTERCEPTOR(sem_getvalue, s, sval);
1210   int res = REAL(sem_getvalue)(s, sval);
1211   if (res == 0) {
1212     Acquire(thr, pc, (uptr)s);
1213   }
1214   return res;
1215 }
1216
1217 TSAN_INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) {
1218   SCOPED_TSAN_INTERCEPTOR(__xstat, version, path, buf);
1219   return REAL(__xstat)(version, path, buf);
1220 }
1221
1222 TSAN_INTERCEPTOR(int, stat, const char *path, void *buf) {
1223   SCOPED_TSAN_INTERCEPTOR(__xstat, 0, path, buf);
1224   return REAL(__xstat)(0, path, buf);
1225 }
1226
1227 TSAN_INTERCEPTOR(int, __xstat64, int version, const char *path, void *buf) {
1228   SCOPED_TSAN_INTERCEPTOR(__xstat64, version, path, buf);
1229   return REAL(__xstat64)(version, path, buf);
1230 }
1231
1232 TSAN_INTERCEPTOR(int, stat64, const char *path, void *buf) {
1233   SCOPED_TSAN_INTERCEPTOR(__xstat64, 0, path, buf);
1234   return REAL(__xstat64)(0, path, buf);
1235 }
1236
1237 TSAN_INTERCEPTOR(int, __lxstat, int version, const char *path, void *buf) {
1238   SCOPED_TSAN_INTERCEPTOR(__lxstat, version, path, buf);
1239   return REAL(__lxstat)(version, path, buf);
1240 }
1241
1242 TSAN_INTERCEPTOR(int, lstat, const char *path, void *buf) {
1243   SCOPED_TSAN_INTERCEPTOR(__lxstat, 0, path, buf);
1244   return REAL(__lxstat)(0, path, buf);
1245 }
1246
1247 TSAN_INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) {
1248   SCOPED_TSAN_INTERCEPTOR(__lxstat64, version, path, buf);
1249   return REAL(__lxstat64)(version, path, buf);
1250 }
1251
1252 TSAN_INTERCEPTOR(int, lstat64, const char *path, void *buf) {
1253   SCOPED_TSAN_INTERCEPTOR(__lxstat64, 0, path, buf);
1254   return REAL(__lxstat64)(0, path, buf);
1255 }
1256
1257 TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
1258   SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf);
1259   if (fd > 0)
1260     FdAccess(thr, pc, fd);
1261   return REAL(__fxstat)(version, fd, buf);
1262 }
1263
1264 TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
1265   SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
1266   if (fd > 0)
1267     FdAccess(thr, pc, fd);
1268   return REAL(__fxstat)(0, fd, buf);
1269 }
1270
1271 TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
1272   SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
1273   if (fd > 0)
1274     FdAccess(thr, pc, fd);
1275   return REAL(__fxstat64)(version, fd, buf);
1276 }
1277
1278 TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
1279   SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);
1280   if (fd > 0)
1281     FdAccess(thr, pc, fd);
1282   return REAL(__fxstat64)(0, fd, buf);
1283 }
1284
1285 TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) {
1286   SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode);
1287   int fd = REAL(open)(name, flags, mode);
1288   if (fd >= 0)
1289     FdFileCreate(thr, pc, fd);
1290   return fd;
1291 }
1292
1293 TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) {
1294   SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode);
1295   int fd = REAL(open64)(name, flags, mode);
1296   if (fd >= 0)
1297     FdFileCreate(thr, pc, fd);
1298   return fd;
1299 }
1300
1301 TSAN_INTERCEPTOR(int, creat, const char *name, int mode) {
1302   SCOPED_TSAN_INTERCEPTOR(creat, name, mode);
1303   int fd = REAL(creat)(name, mode);
1304   if (fd >= 0)
1305     FdFileCreate(thr, pc, fd);
1306   return fd;
1307 }
1308
1309 TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) {
1310   SCOPED_TSAN_INTERCEPTOR(creat64, name, mode);
1311   int fd = REAL(creat64)(name, mode);
1312   if (fd >= 0)
1313     FdFileCreate(thr, pc, fd);
1314   return fd;
1315 }
1316
1317 TSAN_INTERCEPTOR(int, dup, int oldfd) {
1318   SCOPED_TSAN_INTERCEPTOR(dup, oldfd);
1319   int newfd = REAL(dup)(oldfd);
1320   if (oldfd >= 0 && newfd >= 0 && newfd != oldfd)
1321     FdDup(thr, pc, oldfd, newfd);
1322   return newfd;
1323 }
1324
1325 TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) {
1326   SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd);
1327   int newfd2 = REAL(dup2)(oldfd, newfd);
1328   if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
1329     FdDup(thr, pc, oldfd, newfd2);
1330   return newfd2;
1331 }
1332
1333 TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) {
1334   SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags);
1335   int newfd2 = REAL(dup3)(oldfd, newfd, flags);
1336   if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
1337     FdDup(thr, pc, oldfd, newfd2);
1338   return newfd2;
1339 }
1340
1341 TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) {
1342   SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags);
1343   int fd = REAL(eventfd)(initval, flags);
1344   if (fd >= 0)
1345     FdEventCreate(thr, pc, fd);
1346   return fd;
1347 }
1348
1349 TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {
1350   SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags);
1351   if (fd >= 0)
1352     FdClose(thr, pc, fd);
1353   fd = REAL(signalfd)(fd, mask, flags);
1354   if (fd >= 0)
1355     FdSignalCreate(thr, pc, fd);
1356   return fd;
1357 }
1358
1359 TSAN_INTERCEPTOR(int, inotify_init, int fake) {
1360   SCOPED_TSAN_INTERCEPTOR(inotify_init, fake);
1361   int fd = REAL(inotify_init)(fake);
1362   if (fd >= 0)
1363     FdInotifyCreate(thr, pc, fd);
1364   return fd;
1365 }
1366
1367 TSAN_INTERCEPTOR(int, inotify_init1, int flags) {
1368   SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags);
1369   int fd = REAL(inotify_init1)(flags);
1370   if (fd >= 0)
1371     FdInotifyCreate(thr, pc, fd);
1372   return fd;
1373 }
1374
1375 TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) {
1376   SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol);
1377   int fd = REAL(socket)(domain, type, protocol);
1378   if (fd >= 0)
1379     FdSocketCreate(thr, pc, fd);
1380   return fd;
1381 }
1382
1383 TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) {
1384   SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd);
1385   int res = REAL(socketpair)(domain, type, protocol, fd);
1386   if (res == 0 && fd[0] >= 0 && fd[1] >= 0)
1387     FdPipeCreate(thr, pc, fd[0], fd[1]);
1388   return res;
1389 }
1390
1391 TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) {
1392   SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen);
1393   FdSocketConnecting(thr, pc, fd);
1394   int res = REAL(connect)(fd, addr, addrlen);
1395   if (res == 0 && fd >= 0)
1396     FdSocketConnect(thr, pc, fd);
1397   return res;
1398 }
1399
1400 TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) {
1401   SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen);
1402   int res = REAL(bind)(fd, addr, addrlen);
1403   if (fd > 0 && res == 0)
1404     FdAccess(thr, pc, fd);
1405   return res;
1406 }
1407
1408 TSAN_INTERCEPTOR(int, listen, int fd, int backlog) {
1409   SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog);
1410   int res = REAL(listen)(fd, backlog);
1411   if (fd > 0 && res == 0)
1412     FdAccess(thr, pc, fd);
1413   return res;
1414 }
1415
1416 TSAN_INTERCEPTOR(int, epoll_create, int size) {
1417   SCOPED_TSAN_INTERCEPTOR(epoll_create, size);
1418   int fd = REAL(epoll_create)(size);
1419   if (fd >= 0)
1420     FdPollCreate(thr, pc, fd);
1421   return fd;
1422 }
1423
1424 TSAN_INTERCEPTOR(int, epoll_create1, int flags) {
1425   SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags);
1426   int fd = REAL(epoll_create1)(flags);
1427   if (fd >= 0)
1428     FdPollCreate(thr, pc, fd);
1429   return fd;
1430 }
1431
1432 TSAN_INTERCEPTOR(int, close, int fd) {
1433   SCOPED_TSAN_INTERCEPTOR(close, fd);
1434   if (fd >= 0)
1435     FdClose(thr, pc, fd);
1436   return REAL(close)(fd);
1437 }
1438
1439 TSAN_INTERCEPTOR(int, __close, int fd) {
1440   SCOPED_TSAN_INTERCEPTOR(__close, fd);
1441   if (fd >= 0)
1442     FdClose(thr, pc, fd);
1443   return REAL(__close)(fd);
1444 }
1445
1446 // glibc guts
1447 TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) {
1448   SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr);
1449   int fds[64];
1450   int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds));
1451   for (int i = 0; i < cnt; i++) {
1452     if (fds[i] > 0)
1453       FdClose(thr, pc, fds[i]);
1454   }
1455   REAL(__res_iclose)(state, free_addr);
1456 }
1457
1458 TSAN_INTERCEPTOR(int, pipe, int *pipefd) {
1459   SCOPED_TSAN_INTERCEPTOR(pipe, pipefd);
1460   int res = REAL(pipe)(pipefd);
1461   if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
1462     FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
1463   return res;
1464 }
1465
1466 TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) {
1467   SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags);
1468   int res = REAL(pipe2)(pipefd, flags);
1469   if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
1470     FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
1471   return res;
1472 }
1473
1474 TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) {
1475   SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags);
1476   if (fd >= 0) {
1477     FdAccess(thr, pc, fd);
1478     FdRelease(thr, pc, fd);
1479   }
1480   int res = REAL(send)(fd, buf, len, flags);
1481   return res;
1482 }
1483
1484 TSAN_INTERCEPTOR(long_t, sendmsg, int fd, void *msg, int flags) {
1485   SCOPED_TSAN_INTERCEPTOR(sendmsg, fd, msg, flags);
1486   if (fd >= 0) {
1487     FdAccess(thr, pc, fd);
1488     FdRelease(thr, pc, fd);
1489   }
1490   int res = REAL(sendmsg)(fd, msg, flags);
1491   return res;
1492 }
1493
1494 TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) {
1495   SCOPED_TSAN_INTERCEPTOR(recv, fd, buf, len, flags);
1496   if (fd >= 0)
1497     FdAccess(thr, pc, fd);
1498   int res = REAL(recv)(fd, buf, len, flags);
1499   if (res >= 0 && fd >= 0) {
1500     FdAcquire(thr, pc, fd);
1501   }
1502   return res;
1503 }
1504
1505 TSAN_INTERCEPTOR(int, unlink, char *path) {
1506   SCOPED_TSAN_INTERCEPTOR(unlink, path);
1507   Release(thr, pc, File2addr(path));
1508   int res = REAL(unlink)(path);
1509   return res;
1510 }
1511
1512 TSAN_INTERCEPTOR(void*, fopen, char *path, char *mode) {
1513   SCOPED_TSAN_INTERCEPTOR(fopen, path, mode);
1514   void *res = REAL(fopen)(path, mode);
1515   Acquire(thr, pc, File2addr(path));
1516   if (res) {
1517     int fd = fileno_unlocked(res);
1518     if (fd >= 0)
1519       FdFileCreate(thr, pc, fd);
1520   }
1521   return res;
1522 }
1523
1524 TSAN_INTERCEPTOR(void*, freopen, char *path, char *mode, void *stream) {
1525   SCOPED_TSAN_INTERCEPTOR(freopen, path, mode, stream);
1526   if (stream) {
1527     int fd = fileno_unlocked(stream);
1528     if (fd >= 0)
1529       FdClose(thr, pc, fd);
1530   }
1531   void *res = REAL(freopen)(path, mode, stream);
1532   Acquire(thr, pc, File2addr(path));
1533   if (res) {
1534     int fd = fileno_unlocked(res);
1535     if (fd >= 0)
1536       FdFileCreate(thr, pc, fd);
1537   }
1538   return res;
1539 }
1540
1541 TSAN_INTERCEPTOR(int, fclose, void *stream) {
1542   // libc file streams can call user-supplied functions, see fopencookie.
1543   {
1544     SCOPED_TSAN_INTERCEPTOR(fclose, stream);
1545     if (stream) {
1546       int fd = fileno_unlocked(stream);
1547       if (fd >= 0)
1548         FdClose(thr, pc, fd);
1549     }
1550   }
1551   return REAL(fclose)(stream);
1552 }
1553
1554 TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) {
1555   // libc file streams can call user-supplied functions, see fopencookie.
1556   {
1557     SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f);
1558     MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true);
1559   }
1560   return REAL(fread)(ptr, size, nmemb, f);
1561 }
1562
1563 TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) {
1564   // libc file streams can call user-supplied functions, see fopencookie.
1565   {
1566     SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f);
1567     MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false);
1568   }
1569   return REAL(fwrite)(p, size, nmemb, f);
1570 }
1571
1572 TSAN_INTERCEPTOR(int, fflush, void *stream) {
1573   // libc file streams can call user-supplied functions, see fopencookie.
1574   {
1575     SCOPED_TSAN_INTERCEPTOR(fflush, stream);
1576   }
1577   return REAL(fflush)(stream);
1578 }
1579
1580 TSAN_INTERCEPTOR(void, abort, int fake) {
1581   SCOPED_TSAN_INTERCEPTOR(abort, fake);
1582   REAL(fflush)(0);
1583   REAL(abort)(fake);
1584 }
1585
1586 TSAN_INTERCEPTOR(int, puts, const char *s) {
1587   SCOPED_TSAN_INTERCEPTOR(puts, s);
1588   MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s), false);
1589   return REAL(puts)(s);
1590 }
1591
1592 TSAN_INTERCEPTOR(int, rmdir, char *path) {
1593   SCOPED_TSAN_INTERCEPTOR(rmdir, path);
1594   Release(thr, pc, Dir2addr(path));
1595   int res = REAL(rmdir)(path);
1596   return res;
1597 }
1598
1599 TSAN_INTERCEPTOR(void*, opendir, char *path) {
1600   SCOPED_TSAN_INTERCEPTOR(opendir, path);
1601   void *res = REAL(opendir)(path);
1602   if (res != 0)
1603     Acquire(thr, pc, Dir2addr(path));
1604   return res;
1605 }
1606
1607 TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) {
1608   SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev);
1609   if (epfd >= 0)
1610     FdAccess(thr, pc, epfd);
1611   if (epfd >= 0 && fd >= 0)
1612     FdAccess(thr, pc, fd);
1613   if (op == EPOLL_CTL_ADD && epfd >= 0)
1614     FdRelease(thr, pc, epfd);
1615   int res = REAL(epoll_ctl)(epfd, op, fd, ev);
1616   return res;
1617 }
1618
1619 TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) {
1620   SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout);
1621   if (epfd >= 0)
1622     FdAccess(thr, pc, epfd);
1623   int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout);
1624   if (res > 0 && epfd >= 0)
1625     FdAcquire(thr, pc, epfd);
1626   return res;
1627 }
1628
1629 void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
1630     my_siginfo_t *info, void *ctx) {
1631   ThreadState *thr = cur_thread();
1632   SignalContext *sctx = SigCtx(thr);
1633   // Don't mess with synchronous signals.
1634   if (sig == SIGSEGV || sig == SIGBUS || sig == SIGILL ||
1635       sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS ||
1636       // If we are sending signal to ourselves, we must process it now.
1637       (sctx && sig == sctx->int_signal_send) ||
1638       // If we are in blocking function, we can safely process it now
1639       // (but check if we are in a recursive interceptor,
1640       // i.e. pthread_join()->munmap()).
1641       (sctx && sctx->in_blocking_func == 1 && thr->in_rtl == 1)) {
1642     int in_rtl = thr->in_rtl;
1643     thr->in_rtl = 0;
1644     CHECK_EQ(thr->in_signal_handler, false);
1645     thr->in_signal_handler = true;
1646     if (sigact)
1647       sigactions[sig].sa_sigaction(sig, info, ctx);
1648     else
1649       sigactions[sig].sa_handler(sig);
1650     CHECK_EQ(thr->in_signal_handler, true);
1651     thr->in_signal_handler = false;
1652     thr->in_rtl = in_rtl;
1653     return;
1654   }
1655
1656   if (sctx == 0)
1657     return;
1658   SignalDesc *signal = &sctx->pending_signals[sig];
1659   if (signal->armed == false) {
1660     signal->armed = true;
1661     signal->sigaction = sigact;
1662     if (info)
1663       internal_memcpy(&signal->siginfo, info, sizeof(*info));
1664     if (ctx)
1665       internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
1666     sctx->pending_signal_count++;
1667   }
1668 }
1669
1670 static void rtl_sighandler(int sig) {
1671   rtl_generic_sighandler(false, sig, 0, 0);
1672 }
1673
1674 static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) {
1675   rtl_generic_sighandler(true, sig, info, ctx);
1676 }
1677
1678 TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) {
1679   SCOPED_TSAN_INTERCEPTOR(sigaction, sig, act, old);
1680   if (old)
1681     internal_memcpy(old, &sigactions[sig], sizeof(*old));
1682   if (act == 0)
1683     return 0;
1684   internal_memcpy(&sigactions[sig], act, sizeof(*act));
1685   sigaction_t newact;
1686   internal_memcpy(&newact, act, sizeof(newact));
1687   REAL(sigfillset)(&newact.sa_mask);
1688   if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) {
1689     if (newact.sa_flags & SA_SIGINFO)
1690       newact.sa_sigaction = rtl_sigaction;
1691     else
1692       newact.sa_handler = rtl_sighandler;
1693   }
1694   int res = REAL(sigaction)(sig, &newact, 0);
1695   return res;
1696 }
1697
1698 TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) {
1699   sigaction_t act;
1700   act.sa_handler = h;
1701   REAL(memset)(&act.sa_mask, -1, sizeof(act.sa_mask));
1702   act.sa_flags = 0;
1703   sigaction_t old;
1704   int res = sigaction(sig, &act, &old);
1705   if (res)
1706     return SIG_ERR;
1707   return old.sa_handler;
1708 }
1709
1710 TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) {
1711   SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask);
1712   return REAL(sigsuspend)(mask);
1713 }
1714
1715 TSAN_INTERCEPTOR(int, raise, int sig) {
1716   SCOPED_TSAN_INTERCEPTOR(raise, sig);
1717   SignalContext *sctx = SigCtx(thr);
1718   CHECK_NE(sctx, 0);
1719   int prev = sctx->int_signal_send;
1720   sctx->int_signal_send = sig;
1721   int res = REAL(raise)(sig);
1722   CHECK_EQ(sctx->int_signal_send, sig);
1723   sctx->int_signal_send = prev;
1724   return res;
1725 }
1726
1727 TSAN_INTERCEPTOR(int, kill, int pid, int sig) {
1728   SCOPED_TSAN_INTERCEPTOR(kill, pid, sig);
1729   SignalContext *sctx = SigCtx(thr);
1730   CHECK_NE(sctx, 0);
1731   int prev = sctx->int_signal_send;
1732   if (pid == (int)internal_getpid()) {
1733     sctx->int_signal_send = sig;
1734   }
1735   int res = REAL(kill)(pid, sig);
1736   if (pid == (int)internal_getpid()) {
1737     CHECK_EQ(sctx->int_signal_send, sig);
1738     sctx->int_signal_send = prev;
1739   }
1740   return res;
1741 }
1742
1743 TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) {
1744   SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig);
1745   SignalContext *sctx = SigCtx(thr);
1746   CHECK_NE(sctx, 0);
1747   int prev = sctx->int_signal_send;
1748   if (tid == pthread_self()) {
1749     sctx->int_signal_send = sig;
1750   }
1751   int res = REAL(pthread_kill)(tid, sig);
1752   if (tid == pthread_self()) {
1753     CHECK_EQ(sctx->int_signal_send, sig);
1754     sctx->int_signal_send = prev;
1755   }
1756   return res;
1757 }
1758
1759 TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) {
1760   SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz);
1761   // It's intercepted merely to process pending signals.
1762   return REAL(gettimeofday)(tv, tz);
1763 }
1764
1765 TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service,
1766     void *hints, void *rv) {
1767   SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv);
1768   // We miss atomic synchronization in getaddrinfo,
1769   // and can report false race between malloc and free
1770   // inside of getaddrinfo. So ignore memory accesses.
1771   ThreadIgnoreBegin(thr);
1772   // getaddrinfo calls fopen, which can be intercepted by user.
1773   thr->in_rtl--;
1774   CHECK_EQ(thr->in_rtl, 0);
1775   int res = REAL(getaddrinfo)(node, service, hints, rv);
1776   thr->in_rtl++;
1777   ThreadIgnoreEnd(thr);
1778   return res;
1779 }
1780
1781 // Linux kernel has a bug that leads to kernel deadlock if a process
1782 // maps TBs of memory and then calls mlock().
1783 static void MlockIsUnsupported() {
1784   static atomic_uint8_t printed;
1785   if (atomic_exchange(&printed, 1, memory_order_relaxed))
1786     return;
1787   if (flags()->verbosity > 0)
1788     Printf("INFO: ThreadSanitizer ignores mlock/mlockall/munlock/munlockall\n");
1789 }
1790
1791 TSAN_INTERCEPTOR(int, mlock, const void *addr, uptr len) {
1792   MlockIsUnsupported();
1793   return 0;
1794 }
1795
1796 TSAN_INTERCEPTOR(int, munlock, const void *addr, uptr len) {
1797   MlockIsUnsupported();
1798   return 0;
1799 }
1800
1801 TSAN_INTERCEPTOR(int, mlockall, int flags) {
1802   MlockIsUnsupported();
1803   return 0;
1804 }
1805
1806 TSAN_INTERCEPTOR(int, munlockall, void) {
1807   MlockIsUnsupported();
1808   return 0;
1809 }
1810
1811 TSAN_INTERCEPTOR(int, fork, int fake) {
1812   SCOPED_INTERCEPTOR_RAW(fork, fake);
1813   int pid = REAL(fork)(fake);
1814   if (pid == 0) {
1815     // child
1816     FdOnFork(thr, pc);
1817   } else if (pid > 0) {
1818     // parent
1819   }
1820   return pid;
1821 }
1822
1823 static int OnExit(ThreadState *thr) {
1824   int status = Finalize(thr);
1825   REAL(fflush)(0);
1826   return status;
1827 }
1828
1829 struct TsanInterceptorContext {
1830   ThreadState *thr;
1831   const uptr caller_pc;
1832   const uptr pc;
1833 };
1834
1835 #include "sanitizer_common/sanitizer_platform_interceptors.h"
1836 // Causes interceptor recursion (getpwuid_r() calls fopen())
1837 #undef SANITIZER_INTERCEPT_GETPWNAM_AND_FRIENDS
1838 #undef SANITIZER_INTERCEPT_GETPWNAM_R_AND_FRIENDS
1839 // Causes interceptor recursion (getaddrinfo() and fopen())
1840 #undef SANITIZER_INTERCEPT_GETADDRINFO
1841 #undef SANITIZER_INTERCEPT_GETNAMEINFO
1842 // Causes interceptor recursion (glob64() calls lstat64())
1843 #undef SANITIZER_INTERCEPT_GLOB
1844
1845 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
1846 #define COMMON_INTERCEPTOR_UNPOISON_PARAM(ctx, count) \
1847   do {                                                \
1848   } while (false)
1849
1850 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size)                    \
1851   MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr,                 \
1852                     ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
1853                     true)
1854
1855 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size)                       \
1856   MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr,                  \
1857                     ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
1858                     false)
1859
1860 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...)      \
1861   SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__);         \
1862   TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
1863   ctx = (void *)&_ctx;                                \
1864   (void) ctx;
1865
1866 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
1867   FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd)
1868
1869 #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
1870   FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd)
1871
1872 #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \
1873   FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd)
1874
1875 #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
1876   FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd)
1877
1878 #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
1879   ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
1880
1881 #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
1882   CTX()->thread_registry->SetThreadNameByUserId(thread, name)
1883
1884 #define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
1885
1886 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
1887   OnExit(((TsanInterceptorContext *) ctx)->thr)
1888
1889 #define COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m) \
1890   MutexLock(((TsanInterceptorContext *)ctx)->thr, \
1891             ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
1892
1893 #define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
1894   MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
1895             ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
1896
1897 #define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
1898   MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
1899             ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
1900
1901 #include "sanitizer_common/sanitizer_common_interceptors.inc"
1902
1903 #define TSAN_SYSCALL() \
1904   ThreadState *thr = cur_thread(); \
1905   ScopedSyscall scoped_syscall(thr) \
1906 /**/
1907
1908 struct ScopedSyscall {
1909   ThreadState *thr;
1910
1911   explicit ScopedSyscall(ThreadState *thr)
1912       : thr(thr) {
1913     if (thr->in_rtl == 0)
1914       Initialize(thr);
1915     thr->in_rtl++;
1916   }
1917
1918   ~ScopedSyscall() {
1919     thr->in_rtl--;
1920     if (thr->in_rtl == 0)
1921       ProcessPendingSignals(thr);
1922   }
1923 };
1924
1925 static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
1926   TSAN_SYSCALL();
1927   MemoryAccessRange(thr, pc, p, s, write);
1928 }
1929
1930 static void syscall_fd_close(uptr pc, int fd) {
1931   TSAN_SYSCALL();
1932   if (fd >= 0)
1933     FdClose(thr, pc, fd);
1934 }
1935
1936 static void syscall_pre_fork(uptr pc) {
1937   TSAN_SYSCALL();
1938 }
1939
1940 static void syscall_post_fork(uptr pc, int res) {
1941   TSAN_SYSCALL();
1942   if (res == 0) {
1943     // child
1944     FdOnFork(thr, pc);
1945   } else if (res > 0) {
1946     // parent
1947   }
1948 }
1949
1950 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \
1951   syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false)
1952 #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
1953   syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true)
1954 #define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
1955   do {                                       \
1956     (void)(p);                               \
1957     (void)(s);                               \
1958   } while (false)
1959 #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
1960   do {                                        \
1961     (void)(p);                                \
1962     (void)(s);                                \
1963   } while (false)
1964 #define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd)
1965 #define COMMON_SYSCALL_PRE_FORK() \
1966   syscall_pre_fork(GET_CALLER_PC())
1967 #define COMMON_SYSCALL_POST_FORK(res) \
1968   syscall_post_fork(GET_CALLER_PC(), res)
1969 #include "sanitizer_common/sanitizer_common_syscalls.inc"
1970
1971 namespace __tsan {
1972
1973 static void finalize(void *arg) {
1974   ThreadState *thr = cur_thread();
1975   uptr pc = 0;
1976   atexit_ctx->exit(thr, pc);
1977   int status = Finalize(thr);
1978   REAL(fflush)(0);
1979   if (status)
1980     REAL(_exit)(status);
1981 }
1982
1983 void ProcessPendingSignals(ThreadState *thr) {
1984   CHECK_EQ(thr->in_rtl, 0);
1985   SignalContext *sctx = SigCtx(thr);
1986   if (sctx == 0 || sctx->pending_signal_count == 0 || thr->in_signal_handler)
1987     return;
1988   Context *ctx = CTX();
1989   thr->in_signal_handler = true;
1990   sctx->pending_signal_count = 0;
1991   // These are too big for stack.
1992   static THREADLOCAL __sanitizer_sigset_t emptyset, oldset;
1993   REAL(sigfillset)(&emptyset);
1994   pthread_sigmask(SIG_SETMASK, &emptyset, &oldset);
1995   for (int sig = 0; sig < kSigCount; sig++) {
1996     SignalDesc *signal = &sctx->pending_signals[sig];
1997     if (signal->armed) {
1998       signal->armed = false;
1999       if (sigactions[sig].sa_handler != SIG_DFL
2000           && sigactions[sig].sa_handler != SIG_IGN) {
2001         // Insure that the handler does not spoil errno.
2002         const int saved_errno = errno;
2003         errno = 0;
2004         if (signal->sigaction)
2005           sigactions[sig].sa_sigaction(sig, &signal->siginfo, &signal->ctx);
2006         else
2007           sigactions[sig].sa_handler(sig);
2008         if (flags()->report_bugs && errno != 0) {
2009           ScopedInRtl in_rtl;
2010           __tsan::StackTrace stack;
2011           uptr pc = signal->sigaction ?
2012               (uptr)sigactions[sig].sa_sigaction :
2013               (uptr)sigactions[sig].sa_handler;
2014           pc += 1;  // return address is expected, OutputReport() will undo this
2015           stack.Init(&pc, 1);
2016           ThreadRegistryLock l(ctx->thread_registry);
2017           ScopedReport rep(ReportTypeErrnoInSignal);
2018           if (!IsFiredSuppression(ctx, rep, stack)) {
2019             rep.AddStack(&stack);
2020             OutputReport(ctx, rep, rep.GetReport()->stacks[0]);
2021           }
2022         }
2023         errno = saved_errno;
2024       }
2025     }
2026   }
2027   pthread_sigmask(SIG_SETMASK, &oldset, 0);
2028   CHECK_EQ(thr->in_signal_handler, true);
2029   thr->in_signal_handler = false;
2030 }
2031
2032 static void unreachable() {
2033   Printf("FATAL: ThreadSanitizer: unreachable called\n");
2034   Die();
2035 }
2036
2037 void InitializeInterceptors() {
2038   CHECK_GT(cur_thread()->in_rtl, 0);
2039
2040   // We need to setup it early, because functions like dlsym() can call it.
2041   REAL(memset) = internal_memset;
2042   REAL(memcpy) = internal_memcpy;
2043   REAL(memcmp) = internal_memcmp;
2044
2045   // Instruct libc malloc to consume less memory.
2046   mallopt(1, 0);  // M_MXFAST
2047   mallopt(-3, 32*1024);  // M_MMAP_THRESHOLD
2048
2049   SANITIZER_COMMON_INTERCEPTORS_INIT;
2050
2051   TSAN_INTERCEPT(setjmp);
2052   TSAN_INTERCEPT(_setjmp);
2053   TSAN_INTERCEPT(sigsetjmp);
2054   TSAN_INTERCEPT(__sigsetjmp);
2055   TSAN_INTERCEPT(longjmp);
2056   TSAN_INTERCEPT(siglongjmp);
2057
2058   TSAN_INTERCEPT(malloc);
2059   TSAN_INTERCEPT(__libc_memalign);
2060   TSAN_INTERCEPT(calloc);
2061   TSAN_INTERCEPT(realloc);
2062   TSAN_INTERCEPT(free);
2063   TSAN_INTERCEPT(cfree);
2064   TSAN_INTERCEPT(mmap);
2065   TSAN_INTERCEPT(mmap64);
2066   TSAN_INTERCEPT(munmap);
2067   TSAN_INTERCEPT(memalign);
2068   TSAN_INTERCEPT(valloc);
2069   TSAN_INTERCEPT(pvalloc);
2070   TSAN_INTERCEPT(posix_memalign);
2071
2072   TSAN_INTERCEPT(strlen);
2073   TSAN_INTERCEPT(memset);
2074   TSAN_INTERCEPT(memcpy);
2075   TSAN_INTERCEPT(memchr);
2076   TSAN_INTERCEPT(memrchr);
2077   TSAN_INTERCEPT(memmove);
2078   TSAN_INTERCEPT(memcmp);
2079   TSAN_INTERCEPT(strchr);
2080   TSAN_INTERCEPT(strchrnul);
2081   TSAN_INTERCEPT(strrchr);
2082   TSAN_INTERCEPT(strcpy);  // NOLINT
2083   TSAN_INTERCEPT(strncpy);
2084   TSAN_INTERCEPT(strstr);
2085   TSAN_INTERCEPT(strdup);
2086
2087   TSAN_INTERCEPT(pthread_create);
2088   TSAN_INTERCEPT(pthread_join);
2089   TSAN_INTERCEPT(pthread_detach);
2090
2091   TSAN_INTERCEPT(pthread_mutex_init);
2092   TSAN_INTERCEPT(pthread_mutex_destroy);
2093   TSAN_INTERCEPT(pthread_mutex_trylock);
2094   TSAN_INTERCEPT(pthread_mutex_timedlock);
2095
2096   TSAN_INTERCEPT(pthread_spin_init);
2097   TSAN_INTERCEPT(pthread_spin_destroy);
2098   TSAN_INTERCEPT(pthread_spin_lock);
2099   TSAN_INTERCEPT(pthread_spin_trylock);
2100   TSAN_INTERCEPT(pthread_spin_unlock);
2101
2102   TSAN_INTERCEPT(pthread_rwlock_init);
2103   TSAN_INTERCEPT(pthread_rwlock_destroy);
2104   TSAN_INTERCEPT(pthread_rwlock_rdlock);
2105   TSAN_INTERCEPT(pthread_rwlock_tryrdlock);
2106   TSAN_INTERCEPT(pthread_rwlock_timedrdlock);
2107   TSAN_INTERCEPT(pthread_rwlock_wrlock);
2108   TSAN_INTERCEPT(pthread_rwlock_trywrlock);
2109   TSAN_INTERCEPT(pthread_rwlock_timedwrlock);
2110   TSAN_INTERCEPT(pthread_rwlock_unlock);
2111
2112   INTERCEPT_FUNCTION_VER(pthread_cond_destroy, "GLIBC_2.3.2");
2113   INTERCEPT_FUNCTION_VER(pthread_cond_timedwait, "GLIBC_2.3.2");
2114
2115   TSAN_INTERCEPT(pthread_barrier_init);
2116   TSAN_INTERCEPT(pthread_barrier_destroy);
2117   TSAN_INTERCEPT(pthread_barrier_wait);
2118
2119   TSAN_INTERCEPT(pthread_once);
2120
2121   TSAN_INTERCEPT(sem_init);
2122   TSAN_INTERCEPT(sem_destroy);
2123   TSAN_INTERCEPT(sem_wait);
2124   TSAN_INTERCEPT(sem_trywait);
2125   TSAN_INTERCEPT(sem_timedwait);
2126   TSAN_INTERCEPT(sem_post);
2127   TSAN_INTERCEPT(sem_getvalue);
2128
2129   TSAN_INTERCEPT(stat);
2130   TSAN_INTERCEPT(__xstat);
2131   TSAN_INTERCEPT(stat64);
2132   TSAN_INTERCEPT(__xstat64);
2133   TSAN_INTERCEPT(lstat);
2134   TSAN_INTERCEPT(__lxstat);
2135   TSAN_INTERCEPT(lstat64);
2136   TSAN_INTERCEPT(__lxstat64);
2137   TSAN_INTERCEPT(fstat);
2138   TSAN_INTERCEPT(__fxstat);
2139   TSAN_INTERCEPT(fstat64);
2140   TSAN_INTERCEPT(__fxstat64);
2141   TSAN_INTERCEPT(open);
2142   TSAN_INTERCEPT(open64);
2143   TSAN_INTERCEPT(creat);
2144   TSAN_INTERCEPT(creat64);
2145   TSAN_INTERCEPT(dup);
2146   TSAN_INTERCEPT(dup2);
2147   TSAN_INTERCEPT(dup3);
2148   TSAN_INTERCEPT(eventfd);
2149   TSAN_INTERCEPT(signalfd);
2150   TSAN_INTERCEPT(inotify_init);
2151   TSAN_INTERCEPT(inotify_init1);
2152   TSAN_INTERCEPT(socket);
2153   TSAN_INTERCEPT(socketpair);
2154   TSAN_INTERCEPT(connect);
2155   TSAN_INTERCEPT(bind);
2156   TSAN_INTERCEPT(listen);
2157   TSAN_INTERCEPT(epoll_create);
2158   TSAN_INTERCEPT(epoll_create1);
2159   TSAN_INTERCEPT(close);
2160   TSAN_INTERCEPT(__close);
2161   TSAN_INTERCEPT(__res_iclose);
2162   TSAN_INTERCEPT(pipe);
2163   TSAN_INTERCEPT(pipe2);
2164
2165   TSAN_INTERCEPT(send);
2166   TSAN_INTERCEPT(sendmsg);
2167   TSAN_INTERCEPT(recv);
2168
2169   TSAN_INTERCEPT(unlink);
2170   TSAN_INTERCEPT(fopen);
2171   TSAN_INTERCEPT(freopen);
2172   TSAN_INTERCEPT(fclose);
2173   TSAN_INTERCEPT(fread);
2174   TSAN_INTERCEPT(fwrite);
2175   TSAN_INTERCEPT(fflush);
2176   TSAN_INTERCEPT(abort);
2177   TSAN_INTERCEPT(puts);
2178   TSAN_INTERCEPT(rmdir);
2179   TSAN_INTERCEPT(opendir);
2180
2181   TSAN_INTERCEPT(epoll_ctl);
2182   TSAN_INTERCEPT(epoll_wait);
2183
2184   TSAN_INTERCEPT(sigaction);
2185   TSAN_INTERCEPT(signal);
2186   TSAN_INTERCEPT(sigsuspend);
2187   TSAN_INTERCEPT(raise);
2188   TSAN_INTERCEPT(kill);
2189   TSAN_INTERCEPT(pthread_kill);
2190   TSAN_INTERCEPT(sleep);
2191   TSAN_INTERCEPT(usleep);
2192   TSAN_INTERCEPT(nanosleep);
2193   TSAN_INTERCEPT(gettimeofday);
2194   TSAN_INTERCEPT(getaddrinfo);
2195
2196   TSAN_INTERCEPT(mlock);
2197   TSAN_INTERCEPT(munlock);
2198   TSAN_INTERCEPT(mlockall);
2199   TSAN_INTERCEPT(munlockall);
2200
2201   TSAN_INTERCEPT(fork);
2202   TSAN_INTERCEPT(dlopen);
2203   TSAN_INTERCEPT(dlclose);
2204   TSAN_INTERCEPT(on_exit);
2205   TSAN_INTERCEPT(__cxa_atexit);
2206   TSAN_INTERCEPT(_exit);
2207
2208   // Need to setup it, because interceptors check that the function is resolved.
2209   // But atexit is emitted directly into the module, so can't be resolved.
2210   REAL(atexit) = (int(*)(void(*)()))unreachable;
2211   atexit_ctx = new(internal_alloc(MBlockAtExit, sizeof(AtExitContext)))
2212       AtExitContext();
2213
2214   if (REAL(__cxa_atexit)(&finalize, 0, 0)) {
2215     Printf("ThreadSanitizer: failed to setup atexit callback\n");
2216     Die();
2217   }
2218
2219   if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) {
2220     Printf("ThreadSanitizer: failed to create thread key\n");
2221     Die();
2222   }
2223
2224   FdInit();
2225 }
2226
2227 void internal_start_thread(void(*func)(void *arg), void *arg) {
2228   // Start the thread with signals blocked, otherwise it can steal users
2229   // signals.
2230   __sanitizer_kernel_sigset_t set, old;
2231   internal_sigfillset(&set);
2232   internal_sigprocmask(SIG_SETMASK, &set, &old);
2233   void *th;
2234   REAL(pthread_create)(&th, 0, (void*(*)(void *arg))func, arg);
2235   REAL(pthread_detach)(th);
2236   internal_sigprocmask(SIG_SETMASK, &old, 0);
2237 }
2238
2239 }  // namespace __tsan