lib/tsan/rtl/tsan_interceptors.cc

   1 //===-- tsan_interceptors.cc ----------------------------------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file is a part of ThreadSanitizer (TSan), a race detector.
  11 //
  12 // FIXME: move as many interceptors as possible into
  13 // sanitizer_common/sanitizer_common_interceptors.inc
  14 //===----------------------------------------------------------------------===//
  15
  16 #include "sanitizer_common/sanitizer_atomic.h"
  17 #include "sanitizer_common/sanitizer_libc.h"
  18 #include "sanitizer_common/sanitizer_linux.h"
  19 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
  20 #include "sanitizer_common/sanitizer_placement_new.h"
  21 #include "sanitizer_common/sanitizer_stacktrace.h"
  22 #include "interception/interception.h"
  23 #include "tsan_interface.h"
  24 #include "tsan_platform.h"
  25 #include "tsan_suppressions.h"
  26 #include "tsan_rtl.h"
  27 #include "tsan_mman.h"
  28 #include "tsan_fd.h"
  29
  30 using namespace __tsan;  // NOLINT
  31
  32 const int kSigCount = 65;
  33
  34 struct my_siginfo_t {
  35   // The size is determined by looking at sizeof of real siginfo_t on linux.
  36   u64 opaque[128 / sizeof(u64)];
  37 };
  38
  39 struct ucontext_t {
  40   // The size is determined by looking at sizeof of real ucontext_t on linux.
  41   u64 opaque[936 / sizeof(u64) + 1];
  42 };
  43
  44 extern "C" int pthread_attr_init(void *attr);
  45 extern "C" int pthread_attr_destroy(void *attr);
  46 DECLARE_REAL(int, pthread_attr_getdetachstate, void *, void *)
  47 extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize);
  48 extern "C" int pthread_key_create(unsigned *key, void (*destructor)(void* v));
  49 extern "C" int pthread_setspecific(unsigned key, const void *v);
  50 extern "C" int pthread_mutexattr_gettype(void *a, int *type);
  51 extern "C" int pthread_yield();
  52 extern "C" int pthread_sigmask(int how, const __sanitizer_sigset_t *set,
  53                                __sanitizer_sigset_t *oldset);
  54 // REAL(sigfillset) defined in common interceptors.
  55 DECLARE_REAL(int, sigfillset, __sanitizer_sigset_t *set)
  56 extern "C" void *pthread_self();
  57 extern "C" void _exit(int status);
  58 extern "C" int *__errno_location();
  59 extern "C" int fileno_unlocked(void *stream);
  60 extern "C" void *__libc_malloc(uptr size);
  61 extern "C" void *__libc_calloc(uptr size, uptr n);
  62 extern "C" void *__libc_realloc(void *ptr, uptr size);
  63 extern "C" void __libc_free(void *ptr);
  64 extern "C" int mallopt(int param, int value);
  65 extern void *stdout, *stderr;
  66 const int PTHREAD_MUTEX_RECURSIVE = 1;
  67 const int PTHREAD_MUTEX_RECURSIVE_NP = 1;
  68 const int EINVAL = 22;
  69 const int EBUSY = 16;
  70 const int EOWNERDEAD = 130;
  71 const int EPOLL_CTL_ADD = 1;
  72 const int SIGILL = 4;
  73 const int SIGABRT = 6;
  74 const int SIGFPE = 8;
  75 const int SIGSEGV = 11;
  76 const int SIGPIPE = 13;
  77 const int SIGTERM = 15;
  78 const int SIGBUS = 7;
  79 const int SIGSYS = 31;
  80 void *const MAP_FAILED = (void*)-1;
  81 const int PTHREAD_BARRIER_SERIAL_THREAD = -1;
  82 const int MAP_FIXED = 0x10;
  83 typedef long long_t;  // NOLINT
  84
  85 // From /usr/include/unistd.h
  86 # define F_ULOCK 0      /* Unlock a previously locked region.  */
  87 # define F_LOCK  1      /* Lock a region for exclusive use.  */
  88 # define F_TLOCK 2      /* Test and lock a region for exclusive use.  */
  89 # define F_TEST  3      /* Test a region for other processes locks.  */
  90
  91 typedef void (*sighandler_t)(int sig);
  92
  93 #define errno (*__errno_location())
  94
  95 struct sigaction_t {
  96   union {
  97     sighandler_t sa_handler;
  98     void (*sa_sigaction)(int sig, my_siginfo_t *siginfo, void *uctx);
  99   };
 100   __sanitizer_sigset_t sa_mask;
 101   int sa_flags;
 102   void (*sa_restorer)();
 103 };
 104
 105 const sighandler_t SIG_DFL = (sighandler_t)0;
 106 const sighandler_t SIG_IGN = (sighandler_t)1;
 107 const sighandler_t SIG_ERR = (sighandler_t)-1;
 108 const int SA_SIGINFO = 4;
 109 const int SIG_SETMASK = 2;
 110
 111 namespace std {
 112 struct nothrow_t {};
 113 }  // namespace std
 114
 115 static sigaction_t sigactions[kSigCount];
 116
 117 namespace __tsan {
 118 struct SignalDesc {
 119   bool armed;
 120   bool sigaction;
 121   my_siginfo_t siginfo;
 122   ucontext_t ctx;
 123 };
 124
 125 struct SignalContext {
 126   int in_blocking_func;
 127   int int_signal_send;
 128   int pending_signal_count;
 129   SignalDesc pending_signals[kSigCount];
 130 };
 131
 132 // The object is 64-byte aligned, because we want hot data to be located in
 133 // a single cache line if possible (it's accessed in every interceptor).
 134 static ALIGNED(64) char libignore_placeholder[sizeof(LibIgnore)];
 135 static LibIgnore *libignore() {
 136   return reinterpret_cast<LibIgnore*>(&libignore_placeholder[0]);
 137 }
 138
 139 void InitializeLibIgnore() {
 140   libignore()->Init(*GetSuppressionContext());
 141   libignore()->OnLibraryLoaded(0);
 142 }
 143
 144 }  // namespace __tsan
 145
 146 static SignalContext *SigCtx(ThreadState *thr) {
 147   SignalContext *ctx = (SignalContext*)thr->signal_ctx;
 148   if (ctx == 0 && thr->is_alive) {
 149     ctx = (SignalContext*)MmapOrDie(sizeof(*ctx), "SignalContext");
 150     MemoryResetRange(thr, (uptr)&SigCtx, (uptr)ctx, sizeof(*ctx));
 151     thr->signal_ctx = ctx;
 152   }
 153   return ctx;
 154 }
 155
 156 static unsigned g_thread_finalize_key;
 157
 158 class ScopedInterceptor {
 159  public:
 160   ScopedInterceptor(ThreadState *thr, const char *fname, uptr pc);
 161   ~ScopedInterceptor();
 162  private:
 163   ThreadState *const thr_;
 164   const uptr pc_;
 165   bool in_ignored_lib_;
 166 };
 167
 168 ScopedInterceptor::ScopedInterceptor(ThreadState *thr, const char *fname,
 169                                      uptr pc)
 170     : thr_(thr)
 171     , pc_(pc)
 172     , in_ignored_lib_(false) {
 173   if (!thr_->ignore_interceptors) {
 174     Initialize(thr);
 175     FuncEntry(thr, pc);
 176   }
 177   DPrintf("#%d: intercept %s()\n", thr_->tid, fname);
 178   if (!thr_->in_ignored_lib && libignore()->IsIgnored(pc)) {
 179     in_ignored_lib_ = true;
 180     thr_->in_ignored_lib = true;
 181     ThreadIgnoreBegin(thr_, pc_);
 182   }
 183 }
 184
 185 ScopedInterceptor::~ScopedInterceptor() {
 186   if (in_ignored_lib_) {
 187     thr_->in_ignored_lib = false;
 188     ThreadIgnoreEnd(thr_, pc_);
 189   }
 190   if (!thr_->ignore_interceptors) {
 191     ProcessPendingSignals(thr_);
 192     FuncExit(thr_);
 193   }
 194 }
 195
 196 #define SCOPED_INTERCEPTOR_RAW(func, ...) \
 197     ThreadState *thr = cur_thread(); \
 198     const uptr caller_pc = GET_CALLER_PC(); \
 199     ScopedInterceptor si(thr, #func, caller_pc); \
 200     const uptr pc = __sanitizer::StackTrace::GetCurrentPc(); \
 201     (void)pc; \
 202 /**/
 203
 204 #define SCOPED_TSAN_INTERCEPTOR(func, ...) \
 205     SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
 206     if (REAL(func) == 0) { \
 207       Printf("FATAL: ThreadSanitizer: failed to intercept %s\n", #func); \
 208       Die(); \
 209     } \
 210     if (thr->ignore_interceptors || thr->in_ignored_lib) \
 211       return REAL(func)(__VA_ARGS__); \
 212 /**/
 213
 214 #define TSAN_INTERCEPTOR(ret, func, ...) INTERCEPTOR(ret, func, __VA_ARGS__)
 215 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
 216
 217 #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name))
 218
 219 struct BlockingCall {
 220   explicit BlockingCall(ThreadState *thr)
 221       : ctx(SigCtx(thr)) {
 222     ctx->in_blocking_func++;
 223   }
 224
 225   ~BlockingCall() {
 226     ctx->in_blocking_func--;
 227   }
 228
 229   SignalContext *ctx;
 230
 231   // When we are in a "blocking call", we process signals asynchronously
 232   // (right when they arrive). In this context we do not expect to be
 233   // executing any user/runtime code. The known interceptor sequence when
 234   // this is not true is: pthread_join -> munmap(stack). It's fine
 235   // to ignore munmap in this case -- we handle stack shadow separately.
 236   ScopedIgnoreInterceptors ignore_interceptors;
 237 };
 238
 239 TSAN_INTERCEPTOR(unsigned, sleep, unsigned sec) {
 240   SCOPED_TSAN_INTERCEPTOR(sleep, sec);
 241   unsigned res = BLOCK_REAL(sleep)(sec);
 242   AfterSleep(thr, pc);
 243   return res;
 244 }
 245
 246 TSAN_INTERCEPTOR(int, usleep, long_t usec) {
 247   SCOPED_TSAN_INTERCEPTOR(usleep, usec);
 248   int res = BLOCK_REAL(usleep)(usec);
 249   AfterSleep(thr, pc);
 250   return res;
 251 }
 252
 253 TSAN_INTERCEPTOR(int, nanosleep, void *req, void *rem) {
 254   SCOPED_TSAN_INTERCEPTOR(nanosleep, req, rem);
 255   int res = BLOCK_REAL(nanosleep)(req, rem);
 256   AfterSleep(thr, pc);
 257   return res;
 258 }
 259
 260 TSAN_INTERCEPTOR(void*, dlopen, const char *filename, int flag) {
 261   SCOPED_INTERCEPTOR_RAW(dlopen, filename, flag);
 262   void *res = REAL(dlopen)(filename, flag);
 263   libignore()->OnLibraryLoaded(filename);
 264   return res;
 265 }
 266
 267 TSAN_INTERCEPTOR(int, dlclose, void *handle) {
 268   SCOPED_INTERCEPTOR_RAW(dlclose, handle);
 269   int res = REAL(dlclose)(handle);
 270   libignore()->OnLibraryUnloaded();
 271   return res;
 272 }
 273
 274 class AtExitContext {
 275  public:
 276   AtExitContext()
 277     : mtx_(MutexTypeAtExit, StatMtxAtExit)
 278     , pos_() {
 279   }
 280
 281   typedef void(*atexit_t)();
 282
 283   int atexit(ThreadState *thr, uptr pc, bool is_on_exit,
 284              atexit_t f, void *arg) {
 285     Lock l(&mtx_);
 286     if (pos_ == kMaxAtExit)
 287       return 1;
 288     Release(thr, pc, (uptr)this);
 289     stack_[pos_] = f;
 290     args_[pos_] = arg;
 291     is_on_exits_[pos_] = is_on_exit;
 292     pos_++;
 293     return 0;
 294   }
 295
 296   void exit(ThreadState *thr, uptr pc) {
 297     for (;;) {
 298       atexit_t f = 0;
 299       void *arg = 0;
 300       bool is_on_exit = false;
 301       {
 302         Lock l(&mtx_);
 303         if (pos_) {
 304           pos_--;
 305           f = stack_[pos_];
 306           arg = args_[pos_];
 307           is_on_exit = is_on_exits_[pos_];
 308           Acquire(thr, pc, (uptr)this);
 309         }
 310       }
 311       if (f == 0)
 312         break;
 313       DPrintf("#%d: executing atexit func %p\n", thr->tid, f);
 314       if (is_on_exit)
 315         ((void(*)(int status, void *arg))f)(0, arg);
 316       else
 317         ((void(*)(void *arg, void *dso))f)(arg, 0);
 318     }
 319   }
 320
 321  private:
 322   static const int kMaxAtExit = 128;
 323   Mutex mtx_;
 324   atexit_t stack_[kMaxAtExit];
 325   void *args_[kMaxAtExit];
 326   bool is_on_exits_[kMaxAtExit];
 327   int pos_;
 328 };
 329
 330 static AtExitContext *atexit_ctx;
 331
 332 TSAN_INTERCEPTOR(int, atexit, void (*f)()) {
 333   if (cur_thread()->in_symbolizer)
 334     return 0;
 335   SCOPED_TSAN_INTERCEPTOR(atexit, f);
 336   return atexit_ctx->atexit(thr, pc, false, (void(*)())f, 0);
 337 }
 338
 339 TSAN_INTERCEPTOR(int, on_exit, void(*f)(int, void*), void *arg) {
 340   if (cur_thread()->in_symbolizer)
 341     return 0;
 342   SCOPED_TSAN_INTERCEPTOR(on_exit, f, arg);
 343   return atexit_ctx->atexit(thr, pc, true, (void(*)())f, arg);
 344 }
 345
 346 TSAN_INTERCEPTOR(int, __cxa_atexit, void (*f)(void *a), void *arg, void *dso) {
 347   if (cur_thread()->in_symbolizer)
 348     return 0;
 349   SCOPED_TSAN_INTERCEPTOR(__cxa_atexit, f, arg, dso);
 350   if (dso) {
 351     // Memory allocation in __cxa_atexit will race with free during exit,
 352     // because we do not see synchronization around atexit callback list.
 353     ThreadIgnoreBegin(thr, pc);
 354     int res = REAL(__cxa_atexit)(f, arg, dso);
 355     ThreadIgnoreEnd(thr, pc);
 356     return res;
 357   }
 358   return atexit_ctx->atexit(thr, pc, false, (void(*)())f, arg);
 359 }
 360
 361 // Cleanup old bufs.
 362 static void JmpBufGarbageCollect(ThreadState *thr, uptr sp) {
 363   for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
 364     JmpBuf *buf = &thr->jmp_bufs[i];
 365     if (buf->sp <= sp) {
 366       uptr sz = thr->jmp_bufs.Size();
 367       thr->jmp_bufs[i] = thr->jmp_bufs[sz - 1];
 368       thr->jmp_bufs.PopBack();
 369       i--;
 370     }
 371   }
 372 }
 373
 374 static void SetJmp(ThreadState *thr, uptr sp, uptr mangled_sp) {
 375   if (thr->shadow_stack_pos == 0)  // called from libc guts during bootstrap
 376     return;
 377   // Cleanup old bufs.
 378   JmpBufGarbageCollect(thr, sp);
 379   // Remember the buf.
 380   JmpBuf *buf = thr->jmp_bufs.PushBack();
 381   buf->sp = sp;
 382   buf->mangled_sp = mangled_sp;
 383   buf->shadow_stack_pos = thr->shadow_stack_pos;
 384 }
 385
 386 static void LongJmp(ThreadState *thr, uptr *env) {
 387   uptr mangled_sp = env[6];
 388   // Find the saved buf by mangled_sp.
 389   for (uptr i = 0; i < thr->jmp_bufs.Size(); i++) {
 390     JmpBuf *buf = &thr->jmp_bufs[i];
 391     if (buf->mangled_sp == mangled_sp) {
 392       CHECK_GE(thr->shadow_stack_pos, buf->shadow_stack_pos);
 393       // Unwind the stack.
 394       while (thr->shadow_stack_pos > buf->shadow_stack_pos)
 395         FuncExit(thr);
 396       JmpBufGarbageCollect(thr, buf->sp - 1);  // do not collect buf->sp
 397       return;
 398     }
 399   }
 400   Printf("ThreadSanitizer: can't find longjmp buf\n");
 401   CHECK(0);
 402 }
 403
 404 // FIXME: put everything below into a common extern "C" block?
 405 extern "C" void __tsan_setjmp(uptr sp, uptr mangled_sp) {
 406   SetJmp(cur_thread(), sp, mangled_sp);
 407 }
 408
 409 // Not called.  Merely to satisfy TSAN_INTERCEPT().
 410 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 411 int __interceptor_setjmp(void *env);
 412 extern "C" int __interceptor_setjmp(void *env) {
 413   CHECK(0);
 414   return 0;
 415 }
 416
 417 // FIXME: any reason to have a separate declaration?
 418 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 419 int __interceptor__setjmp(void *env);
 420 extern "C" int __interceptor__setjmp(void *env) {
 421   CHECK(0);
 422   return 0;
 423 }
 424
 425 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 426 int __interceptor_sigsetjmp(void *env);
 427 extern "C" int __interceptor_sigsetjmp(void *env) {
 428   CHECK(0);
 429   return 0;
 430 }
 431
 432 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
 433 int __interceptor___sigsetjmp(void *env);
 434 extern "C" int __interceptor___sigsetjmp(void *env) {
 435   CHECK(0);
 436   return 0;
 437 }
 438
 439 extern "C" int setjmp(void *env);
 440 extern "C" int _setjmp(void *env);
 441 extern "C" int sigsetjmp(void *env);
 442 extern "C" int __sigsetjmp(void *env);
 443 DEFINE_REAL(int, setjmp, void *env)
 444 DEFINE_REAL(int, _setjmp, void *env)
 445 DEFINE_REAL(int, sigsetjmp, void *env)
 446 DEFINE_REAL(int, __sigsetjmp, void *env)
 447
 448 TSAN_INTERCEPTOR(void, longjmp, uptr *env, int val) {
 449   {
 450     SCOPED_TSAN_INTERCEPTOR(longjmp, env, val);
 451   }
 452   LongJmp(cur_thread(), env);
 453   REAL(longjmp)(env, val);
 454 }
 455
 456 TSAN_INTERCEPTOR(void, siglongjmp, uptr *env, int val) {
 457   {
 458     SCOPED_TSAN_INTERCEPTOR(siglongjmp, env, val);
 459   }
 460   LongJmp(cur_thread(), env);
 461   REAL(siglongjmp)(env, val);
 462 }
 463
 464 TSAN_INTERCEPTOR(void*, malloc, uptr size) {
 465   if (cur_thread()->in_symbolizer)
 466     return __libc_malloc(size);
 467   void *p = 0;
 468   {
 469     SCOPED_INTERCEPTOR_RAW(malloc, size);
 470     p = user_alloc(thr, pc, size);
 471   }
 472   invoke_malloc_hook(p, size);
 473   return p;
 474 }
 475
 476 TSAN_INTERCEPTOR(void*, __libc_memalign, uptr align, uptr sz) {
 477   SCOPED_TSAN_INTERCEPTOR(__libc_memalign, align, sz);
 478   return user_alloc(thr, pc, sz, align);
 479 }
 480
 481 TSAN_INTERCEPTOR(void*, calloc, uptr size, uptr n) {
 482   if (cur_thread()->in_symbolizer)
 483     return __libc_calloc(size, n);
 484   if (__sanitizer::CallocShouldReturnNullDueToOverflow(size, n))
 485     return AllocatorReturnNull();
 486   void *p = 0;
 487   {
 488     SCOPED_INTERCEPTOR_RAW(calloc, size, n);
 489     p = user_alloc(thr, pc, n * size);
 490     if (p)
 491       internal_memset(p, 0, n * size);
 492   }
 493   invoke_malloc_hook(p, n * size);
 494   return p;
 495 }
 496
 497 TSAN_INTERCEPTOR(void*, realloc, void *p, uptr size) {
 498   if (cur_thread()->in_symbolizer)
 499     return __libc_realloc(p, size);
 500   if (p)
 501     invoke_free_hook(p);
 502   {
 503     SCOPED_INTERCEPTOR_RAW(realloc, p, size);
 504     p = user_realloc(thr, pc, p, size);
 505   }
 506   invoke_malloc_hook(p, size);
 507   return p;
 508 }
 509
 510 TSAN_INTERCEPTOR(void, free, void *p) {
 511   if (p == 0)
 512     return;
 513   if (cur_thread()->in_symbolizer)
 514     return __libc_free(p);
 515   invoke_free_hook(p);
 516   SCOPED_INTERCEPTOR_RAW(free, p);
 517   user_free(thr, pc, p);
 518 }
 519
 520 TSAN_INTERCEPTOR(void, cfree, void *p) {
 521   if (p == 0)
 522     return;
 523   if (cur_thread()->in_symbolizer)
 524     return __libc_free(p);
 525   invoke_free_hook(p);
 526   SCOPED_INTERCEPTOR_RAW(cfree, p);
 527   user_free(thr, pc, p);
 528 }
 529
 530 TSAN_INTERCEPTOR(uptr, malloc_usable_size, void *p) {
 531   SCOPED_INTERCEPTOR_RAW(malloc_usable_size, p);
 532   return user_alloc_usable_size(thr, pc, p);
 533 }
 534
 535 #define OPERATOR_NEW_BODY(mangled_name) \
 536   if (cur_thread()->in_symbolizer) \
 537     return __libc_malloc(size); \
 538   void *p = 0; \
 539   {  \
 540     SCOPED_INTERCEPTOR_RAW(mangled_name, size); \
 541     p = user_alloc(thr, pc, size); \
 542   }  \
 543   invoke_malloc_hook(p, size);  \
 544   return p;
 545
 546 SANITIZER_INTERFACE_ATTRIBUTE
 547 void *operator new(__sanitizer::uptr size);
 548 void *operator new(__sanitizer::uptr size) {
 549   OPERATOR_NEW_BODY(_Znwm);
 550 }
 551
 552 SANITIZER_INTERFACE_ATTRIBUTE
 553 void *operator new[](__sanitizer::uptr size);
 554 void *operator new[](__sanitizer::uptr size) {
 555   OPERATOR_NEW_BODY(_Znam);
 556 }
 557
 558 SANITIZER_INTERFACE_ATTRIBUTE
 559 void *operator new(__sanitizer::uptr size, std::nothrow_t const&);
 560 void *operator new(__sanitizer::uptr size, std::nothrow_t const&) {
 561   OPERATOR_NEW_BODY(_ZnwmRKSt9nothrow_t);
 562 }
 563
 564 SANITIZER_INTERFACE_ATTRIBUTE
 565 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&);
 566 void *operator new[](__sanitizer::uptr size, std::nothrow_t const&) {
 567   OPERATOR_NEW_BODY(_ZnamRKSt9nothrow_t);
 568 }
 569
 570 #define OPERATOR_DELETE_BODY(mangled_name) \
 571   if (ptr == 0) return;  \
 572   if (cur_thread()->in_symbolizer) \
 573     return __libc_free(ptr); \
 574   invoke_free_hook(ptr);  \
 575   SCOPED_INTERCEPTOR_RAW(mangled_name, ptr);  \
 576   user_free(thr, pc, ptr);
 577
 578 SANITIZER_INTERFACE_ATTRIBUTE
 579 void operator delete(void *ptr) throw();
 580 void operator delete(void *ptr) throw() {
 581   OPERATOR_DELETE_BODY(_ZdlPv);
 582 }
 583
 584 SANITIZER_INTERFACE_ATTRIBUTE
 585 void operator delete[](void *ptr) throw();
 586 void operator delete[](void *ptr) throw() {
 587   OPERATOR_DELETE_BODY(_ZdaPv);
 588 }
 589
 590 SANITIZER_INTERFACE_ATTRIBUTE
 591 void operator delete(void *ptr, std::nothrow_t const&);
 592 void operator delete(void *ptr, std::nothrow_t const&) {
 593   OPERATOR_DELETE_BODY(_ZdlPvRKSt9nothrow_t);
 594 }
 595
 596 SANITIZER_INTERFACE_ATTRIBUTE
 597 void operator delete[](void *ptr, std::nothrow_t const&);
 598 void operator delete[](void *ptr, std::nothrow_t const&) {
 599   OPERATOR_DELETE_BODY(_ZdaPvRKSt9nothrow_t);
 600 }
 601
 602 TSAN_INTERCEPTOR(uptr, strlen, const char *s) {
 603   SCOPED_TSAN_INTERCEPTOR(strlen, s);
 604   uptr len = internal_strlen(s);
 605   MemoryAccessRange(thr, pc, (uptr)s, len + 1, false);
 606   return len;
 607 }
 608
 609 TSAN_INTERCEPTOR(void*, memset, void *dst, int v, uptr size) {
 610   SCOPED_TSAN_INTERCEPTOR(memset, dst, v, size);
 611   MemoryAccessRange(thr, pc, (uptr)dst, size, true);
 612   return internal_memset(dst, v, size);
 613 }
 614
 615 TSAN_INTERCEPTOR(void*, memcpy, void *dst, const void *src, uptr size) {
 616   SCOPED_TSAN_INTERCEPTOR(memcpy, dst, src, size);
 617   MemoryAccessRange(thr, pc, (uptr)dst, size, true);
 618   MemoryAccessRange(thr, pc, (uptr)src, size, false);
 619   return internal_memcpy(dst, src, size);
 620 }
 621
 622 TSAN_INTERCEPTOR(int, memcmp, const void *s1, const void *s2, uptr n) {
 623   SCOPED_TSAN_INTERCEPTOR(memcmp, s1, s2, n);
 624   int res = 0;
 625   uptr len = 0;
 626   for (; len < n; len++) {
 627     if ((res = ((unsigned char*)s1)[len] - ((unsigned char*)s2)[len]))
 628       break;
 629   }
 630   MemoryAccessRange(thr, pc, (uptr)s1, len < n ? len + 1 : n, false);
 631   MemoryAccessRange(thr, pc, (uptr)s2, len < n ? len + 1 : n, false);
 632   return res;
 633 }
 634
 635 TSAN_INTERCEPTOR(void*, memchr, void *s, int c, uptr n) {
 636   SCOPED_TSAN_INTERCEPTOR(memchr, s, c, n);
 637   void *res = REAL(memchr)(s, c, n);
 638   uptr len = res ? (char*)res - (char*)s + 1 : n;
 639   MemoryAccessRange(thr, pc, (uptr)s, len, false);
 640   return res;
 641 }
 642
 643 TSAN_INTERCEPTOR(void*, memrchr, char *s, int c, uptr n) {
 644   SCOPED_TSAN_INTERCEPTOR(memrchr, s, c, n);
 645   MemoryAccessRange(thr, pc, (uptr)s, n, false);
 646   return REAL(memrchr)(s, c, n);
 647 }
 648
 649 TSAN_INTERCEPTOR(void*, memmove, void *dst, void *src, uptr n) {
 650   SCOPED_TSAN_INTERCEPTOR(memmove, dst, src, n);
 651   MemoryAccessRange(thr, pc, (uptr)dst, n, true);
 652   MemoryAccessRange(thr, pc, (uptr)src, n, false);
 653   return REAL(memmove)(dst, src, n);
 654 }
 655
 656 TSAN_INTERCEPTOR(char*, strchr, char *s, int c) {
 657   SCOPED_TSAN_INTERCEPTOR(strchr, s, c);
 658   char *res = REAL(strchr)(s, c);
 659   uptr len = res ? (char*)res - (char*)s + 1 : internal_strlen(s) + 1;
 660   MemoryAccessRange(thr, pc, (uptr)s, len, false);
 661   return res;
 662 }
 663
 664 TSAN_INTERCEPTOR(char*, strchrnul, char *s, int c) {
 665   SCOPED_TSAN_INTERCEPTOR(strchrnul, s, c);
 666   char *res = REAL(strchrnul)(s, c);
 667   uptr len = (char*)res - (char*)s + 1;
 668   MemoryAccessRange(thr, pc, (uptr)s, len, false);
 669   return res;
 670 }
 671
 672 TSAN_INTERCEPTOR(char*, strrchr, char *s, int c) {
 673   SCOPED_TSAN_INTERCEPTOR(strrchr, s, c);
 674   MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s) + 1, false);
 675   return REAL(strrchr)(s, c);
 676 }
 677
 678 TSAN_INTERCEPTOR(char*, strcpy, char *dst, const char *src) {  // NOLINT
 679   SCOPED_TSAN_INTERCEPTOR(strcpy, dst, src);  // NOLINT
 680   uptr srclen = internal_strlen(src);
 681   MemoryAccessRange(thr, pc, (uptr)dst, srclen + 1, true);
 682   MemoryAccessRange(thr, pc, (uptr)src, srclen + 1, false);
 683   return REAL(strcpy)(dst, src);  // NOLINT
 684 }
 685
 686 TSAN_INTERCEPTOR(char*, strncpy, char *dst, char *src, uptr n) {
 687   SCOPED_TSAN_INTERCEPTOR(strncpy, dst, src, n);
 688   uptr srclen = internal_strnlen(src, n);
 689   MemoryAccessRange(thr, pc, (uptr)dst, n, true);
 690   MemoryAccessRange(thr, pc, (uptr)src, min(srclen + 1, n), false);
 691   return REAL(strncpy)(dst, src, n);
 692 }
 693
 694 TSAN_INTERCEPTOR(const char*, strstr, const char *s1, const char *s2) {
 695   SCOPED_TSAN_INTERCEPTOR(strstr, s1, s2);
 696   const char *res = REAL(strstr)(s1, s2);
 697   uptr len1 = internal_strlen(s1);
 698   uptr len2 = internal_strlen(s2);
 699   MemoryAccessRange(thr, pc, (uptr)s1, len1 + 1, false);
 700   MemoryAccessRange(thr, pc, (uptr)s2, len2 + 1, false);
 701   return res;
 702 }
 703
 704 TSAN_INTERCEPTOR(char*, strdup, const char *str) {
 705   SCOPED_TSAN_INTERCEPTOR(strdup, str);
 706   // strdup will call malloc, so no instrumentation is required here.
 707   return REAL(strdup)(str);
 708 }
 709
 710 static bool fix_mmap_addr(void **addr, long_t sz, int flags) {
 711   if (*addr) {
 712     if (!IsAppMem((uptr)*addr) || !IsAppMem((uptr)*addr + sz - 1)) {
 713       if (flags & MAP_FIXED) {
 714         errno = EINVAL;
 715         return false;
 716       } else {
 717         *addr = 0;
 718       }
 719     }
 720   }
 721   return true;
 722 }
 723
 724 TSAN_INTERCEPTOR(void*, mmap, void *addr, long_t sz, int prot,
 725                          int flags, int fd, unsigned off) {
 726   SCOPED_TSAN_INTERCEPTOR(mmap, addr, sz, prot, flags, fd, off);
 727   if (!fix_mmap_addr(&addr, sz, flags))
 728     return MAP_FAILED;
 729   void *res = REAL(mmap)(addr, sz, prot, flags, fd, off);
 730   if (res != MAP_FAILED) {
 731     if (fd > 0)
 732       FdAccess(thr, pc, fd);
 733     MemoryRangeImitateWrite(thr, pc, (uptr)res, sz);
 734   }
 735   return res;
 736 }
 737
 738 TSAN_INTERCEPTOR(void*, mmap64, void *addr, long_t sz, int prot,
 739                            int flags, int fd, u64 off) {
 740   SCOPED_TSAN_INTERCEPTOR(mmap64, addr, sz, prot, flags, fd, off);
 741   if (!fix_mmap_addr(&addr, sz, flags))
 742     return MAP_FAILED;
 743   void *res = REAL(mmap64)(addr, sz, prot, flags, fd, off);
 744   if (res != MAP_FAILED) {
 745     if (fd > 0)
 746       FdAccess(thr, pc, fd);
 747     MemoryRangeImitateWrite(thr, pc, (uptr)res, sz);
 748   }
 749   return res;
 750 }
 751
 752 TSAN_INTERCEPTOR(int, munmap, void *addr, long_t sz) {
 753   SCOPED_TSAN_INTERCEPTOR(munmap, addr, sz);
 754   DontNeedShadowFor((uptr)addr, sz);
 755   int res = REAL(munmap)(addr, sz);
 756   return res;
 757 }
 758
 759 TSAN_INTERCEPTOR(void*, memalign, uptr align, uptr sz) {
 760   SCOPED_INTERCEPTOR_RAW(memalign, align, sz);
 761   return user_alloc(thr, pc, sz, align);
 762 }
 763
 764 TSAN_INTERCEPTOR(void*, valloc, uptr sz) {
 765   SCOPED_INTERCEPTOR_RAW(valloc, sz);
 766   return user_alloc(thr, pc, sz, GetPageSizeCached());
 767 }
 768
 769 TSAN_INTERCEPTOR(void*, pvalloc, uptr sz) {
 770   SCOPED_INTERCEPTOR_RAW(pvalloc, sz);
 771   sz = RoundUp(sz, GetPageSizeCached());
 772   return user_alloc(thr, pc, sz, GetPageSizeCached());
 773 }
 774
 775 TSAN_INTERCEPTOR(int, posix_memalign, void **memptr, uptr align, uptr sz) {
 776   SCOPED_INTERCEPTOR_RAW(posix_memalign, memptr, align, sz);
 777   *memptr = user_alloc(thr, pc, sz, align);
 778   return 0;
 779 }
 780
 781 // Used in thread-safe function static initialization.
 782 extern "C" int INTERFACE_ATTRIBUTE __cxa_guard_acquire(atomic_uint32_t *g) {
 783   SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire, g);
 784   for (;;) {
 785     u32 cmp = atomic_load(g, memory_order_acquire);
 786     if (cmp == 0) {
 787       if (atomic_compare_exchange_strong(g, &cmp, 1<<16, memory_order_relaxed))
 788         return 1;
 789     } else if (cmp == 1) {
 790       Acquire(thr, pc, (uptr)g);
 791       return 0;
 792     } else {
 793       internal_sched_yield();
 794     }
 795   }
 796 }
 797
 798 extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_release(atomic_uint32_t *g) {
 799   SCOPED_INTERCEPTOR_RAW(__cxa_guard_release, g);
 800   Release(thr, pc, (uptr)g);
 801   atomic_store(g, 1, memory_order_release);
 802 }
 803
 804 extern "C" void INTERFACE_ATTRIBUTE __cxa_guard_abort(atomic_uint32_t *g) {
 805   SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort, g);
 806   atomic_store(g, 0, memory_order_relaxed);
 807 }
 808
 809 static void thread_finalize(void *v) {
 810   uptr iter = (uptr)v;
 811   if (iter > 1) {
 812     if (pthread_setspecific(g_thread_finalize_key, (void*)(iter - 1))) {
 813       Printf("ThreadSanitizer: failed to set thread key\n");
 814       Die();
 815     }
 816     return;
 817   }
 818   {
 819     ThreadState *thr = cur_thread();
 820     ThreadFinish(thr);
 821     SignalContext *sctx = thr->signal_ctx;
 822     if (sctx) {
 823       thr->signal_ctx = 0;
 824       UnmapOrDie(sctx, sizeof(*sctx));
 825     }
 826   }
 827 }
 828
 829
 830 struct ThreadParam {
 831   void* (*callback)(void *arg);
 832   void *param;
 833   atomic_uintptr_t tid;
 834 };
 835
 836 extern "C" void *__tsan_thread_start_func(void *arg) {
 837   ThreadParam *p = (ThreadParam*)arg;
 838   void* (*callback)(void *arg) = p->callback;
 839   void *param = p->param;
 840   int tid = 0;
 841   {
 842     ThreadState *thr = cur_thread();
 843     // Thread-local state is not initialized yet.
 844     ScopedIgnoreInterceptors ignore;
 845     if (pthread_setspecific(g_thread_finalize_key,
 846                             (void *)kPthreadDestructorIterations)) {
 847       Printf("ThreadSanitizer: failed to set thread key\n");
 848       Die();
 849     }
 850     while ((tid = atomic_load(&p->tid, memory_order_acquire)) == 0)
 851       pthread_yield();
 852     atomic_store(&p->tid, 0, memory_order_release);
 853     ThreadStart(thr, tid, GetTid());
 854   }
 855   void *res = callback(param);
 856   // Prevent the callback from being tail called,
 857   // it mixes up stack traces.
 858   volatile int foo = 42;
 859   foo++;
 860   return res;
 861 }
 862
 863 TSAN_INTERCEPTOR(int, pthread_create,
 864     void *th, void *attr, void *(*callback)(void*), void * param) {
 865   SCOPED_INTERCEPTOR_RAW(pthread_create, th, attr, callback, param);
 866   if (CTX()->after_multithreaded_fork) {
 867     if (flags()->die_after_fork) {
 868       Printf("ThreadSanitizer: starting new threads after muti-threaded"
 869           " fork is not supported. Dying (set die_after_fork=0 to override)\n");
 870       Die();
 871     } else {
 872       VPrintf(1, "ThreadSanitizer: starting new threads after muti-threaded"
 873           " fork is not supported. Continuing because die_after_fork=0,"
 874           " but you are on your own\n");
 875     }
 876   }
 877   __sanitizer_pthread_attr_t myattr;
 878   if (attr == 0) {
 879     pthread_attr_init(&myattr);
 880     attr = &myattr;
 881   }
 882   int detached = 0;
 883   REAL(pthread_attr_getdetachstate)(attr, &detached);
 884   AdjustStackSize(attr);
 885
 886   ThreadParam p;
 887   p.callback = callback;
 888   p.param = param;
 889   atomic_store(&p.tid, 0, memory_order_relaxed);
 890   int res = -1;
 891   {
 892     // Otherwise we see false positives in pthread stack manipulation.
 893     ScopedIgnoreInterceptors ignore;
 894     ThreadIgnoreBegin(thr, pc);
 895     res = REAL(pthread_create)(th, attr, __tsan_thread_start_func, &p);
 896     ThreadIgnoreEnd(thr, pc);
 897   }
 898   if (res == 0) {
 899     int tid = ThreadCreate(thr, pc, *(uptr*)th, detached);
 900     CHECK_NE(tid, 0);
 901     atomic_store(&p.tid, tid, memory_order_release);
 902     while (atomic_load(&p.tid, memory_order_acquire) != 0)
 903       pthread_yield();
 904   }
 905   if (attr == &myattr)
 906     pthread_attr_destroy(&myattr);
 907   return res;
 908 }
 909
 910 TSAN_INTERCEPTOR(int, pthread_join, void *th, void **ret) {
 911   SCOPED_INTERCEPTOR_RAW(pthread_join, th, ret);
 912   int tid = ThreadTid(thr, pc, (uptr)th);
 913   ThreadIgnoreBegin(thr, pc);
 914   int res = BLOCK_REAL(pthread_join)(th, ret);
 915   ThreadIgnoreEnd(thr, pc);
 916   if (res == 0) {
 917     ThreadJoin(thr, pc, tid);
 918   }
 919   return res;
 920 }
 921
 922 TSAN_INTERCEPTOR(int, pthread_detach, void *th) {
 923   SCOPED_TSAN_INTERCEPTOR(pthread_detach, th);
 924   int tid = ThreadTid(thr, pc, (uptr)th);
 925   int res = REAL(pthread_detach)(th);
 926   if (res == 0) {
 927     ThreadDetach(thr, pc, tid);
 928   }
 929   return res;
 930 }
 931
 932 TSAN_INTERCEPTOR(int, pthread_mutex_init, void *m, void *a) {
 933   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init, m, a);
 934   int res = REAL(pthread_mutex_init)(m, a);
 935   if (res == 0) {
 936     bool recursive = false;
 937     if (a) {
 938       int type = 0;
 939       if (pthread_mutexattr_gettype(a, &type) == 0)
 940         recursive = (type == PTHREAD_MUTEX_RECURSIVE
 941             || type == PTHREAD_MUTEX_RECURSIVE_NP);
 942     }
 943     MutexCreate(thr, pc, (uptr)m, false, recursive, false);
 944   }
 945   return res;
 946 }
 947
 948 TSAN_INTERCEPTOR(int, pthread_mutex_destroy, void *m) {
 949   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy, m);
 950   int res = REAL(pthread_mutex_destroy)(m);
 951   if (res == 0 || res == EBUSY) {
 952     MutexDestroy(thr, pc, (uptr)m);
 953   }
 954   return res;
 955 }
 956
 957 TSAN_INTERCEPTOR(int, pthread_mutex_trylock, void *m) {
 958   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock, m);
 959   int res = REAL(pthread_mutex_trylock)(m);
 960   if (res == EOWNERDEAD)
 961     MutexRepair(thr, pc, (uptr)m);
 962   if (res == 0 || res == EOWNERDEAD)
 963     MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
 964   return res;
 965 }
 966
 967 TSAN_INTERCEPTOR(int, pthread_mutex_timedlock, void *m, void *abstime) {
 968   SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock, m, abstime);
 969   int res = REAL(pthread_mutex_timedlock)(m, abstime);
 970   if (res == 0) {
 971     MutexLock(thr, pc, (uptr)m);
 972   }
 973   return res;
 974 }
 975
 976 TSAN_INTERCEPTOR(int, pthread_spin_init, void *m, int pshared) {
 977   SCOPED_TSAN_INTERCEPTOR(pthread_spin_init, m, pshared);
 978   int res = REAL(pthread_spin_init)(m, pshared);
 979   if (res == 0) {
 980     MutexCreate(thr, pc, (uptr)m, false, false, false);
 981   }
 982   return res;
 983 }
 984
 985 TSAN_INTERCEPTOR(int, pthread_spin_destroy, void *m) {
 986   SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy, m);
 987   int res = REAL(pthread_spin_destroy)(m);
 988   if (res == 0) {
 989     MutexDestroy(thr, pc, (uptr)m);
 990   }
 991   return res;
 992 }
 993
 994 TSAN_INTERCEPTOR(int, pthread_spin_lock, void *m) {
 995   SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock, m);
 996   int res = REAL(pthread_spin_lock)(m);
 997   if (res == 0) {
 998     MutexLock(thr, pc, (uptr)m);
 999   }
1000   return res;
1001 }
1002
1003 TSAN_INTERCEPTOR(int, pthread_spin_trylock, void *m) {
1004   SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock, m);
1005   int res = REAL(pthread_spin_trylock)(m);
1006   if (res == 0) {
1007     MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
1008   }
1009   return res;
1010 }
1011
1012 TSAN_INTERCEPTOR(int, pthread_spin_unlock, void *m) {
1013   SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock, m);
1014   MutexUnlock(thr, pc, (uptr)m);
1015   int res = REAL(pthread_spin_unlock)(m);
1016   return res;
1017 }
1018
1019 TSAN_INTERCEPTOR(int, pthread_rwlock_init, void *m, void *a) {
1020   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init, m, a);
1021   int res = REAL(pthread_rwlock_init)(m, a);
1022   if (res == 0) {
1023     MutexCreate(thr, pc, (uptr)m, true, false, false);
1024   }
1025   return res;
1026 }
1027
1028 TSAN_INTERCEPTOR(int, pthread_rwlock_destroy, void *m) {
1029   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy, m);
1030   int res = REAL(pthread_rwlock_destroy)(m);
1031   if (res == 0) {
1032     MutexDestroy(thr, pc, (uptr)m);
1033   }
1034   return res;
1035 }
1036
1037 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock, void *m) {
1038   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock, m);
1039   int res = REAL(pthread_rwlock_rdlock)(m);
1040   if (res == 0) {
1041     MutexReadLock(thr, pc, (uptr)m);
1042   }
1043   return res;
1044 }
1045
1046 TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock, void *m) {
1047   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock, m);
1048   int res = REAL(pthread_rwlock_tryrdlock)(m);
1049   if (res == 0) {
1050     MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
1051   }
1052   return res;
1053 }
1054
1055 TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock, void *m, void *abstime) {
1056   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock, m, abstime);
1057   int res = REAL(pthread_rwlock_timedrdlock)(m, abstime);
1058   if (res == 0) {
1059     MutexReadLock(thr, pc, (uptr)m);
1060   }
1061   return res;
1062 }
1063
1064 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock, void *m) {
1065   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock, m);
1066   int res = REAL(pthread_rwlock_wrlock)(m);
1067   if (res == 0) {
1068     MutexLock(thr, pc, (uptr)m);
1069   }
1070   return res;
1071 }
1072
1073 TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock, void *m) {
1074   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock, m);
1075   int res = REAL(pthread_rwlock_trywrlock)(m);
1076   if (res == 0) {
1077     MutexLock(thr, pc, (uptr)m, /*rec=*/1, /*try_lock=*/true);
1078   }
1079   return res;
1080 }
1081
1082 TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock, void *m, void *abstime) {
1083   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock, m, abstime);
1084   int res = REAL(pthread_rwlock_timedwrlock)(m, abstime);
1085   if (res == 0) {
1086     MutexLock(thr, pc, (uptr)m);
1087   }
1088   return res;
1089 }
1090
1091 TSAN_INTERCEPTOR(int, pthread_rwlock_unlock, void *m) {
1092   SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock, m);
1093   MutexReadOrWriteUnlock(thr, pc, (uptr)m);
1094   int res = REAL(pthread_rwlock_unlock)(m);
1095   return res;
1096 }
1097
1098 TSAN_INTERCEPTOR(int, pthread_barrier_init, void *b, void *a, unsigned count) {
1099   SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init, b, a, count);
1100   MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
1101   int res = REAL(pthread_barrier_init)(b, a, count);
1102   return res;
1103 }
1104
1105 TSAN_INTERCEPTOR(int, pthread_barrier_destroy, void *b) {
1106   SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy, b);
1107   MemoryWrite(thr, pc, (uptr)b, kSizeLog1);
1108   int res = REAL(pthread_barrier_destroy)(b);
1109   return res;
1110 }
1111
1112 TSAN_INTERCEPTOR(int, pthread_barrier_wait, void *b) {
1113   SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait, b);
1114   Release(thr, pc, (uptr)b);
1115   MemoryRead(thr, pc, (uptr)b, kSizeLog1);
1116   int res = REAL(pthread_barrier_wait)(b);
1117   MemoryRead(thr, pc, (uptr)b, kSizeLog1);
1118   if (res == 0 || res == PTHREAD_BARRIER_SERIAL_THREAD) {
1119     Acquire(thr, pc, (uptr)b);
1120   }
1121   return res;
1122 }
1123
1124 TSAN_INTERCEPTOR(int, pthread_once, void *o, void (*f)()) {
1125   SCOPED_INTERCEPTOR_RAW(pthread_once, o, f);
1126   if (o == 0 || f == 0)
1127     return EINVAL;
1128   atomic_uint32_t *a = static_cast<atomic_uint32_t*>(o);
1129   u32 v = atomic_load(a, memory_order_acquire);
1130   if (v == 0 && atomic_compare_exchange_strong(a, &v, 1,
1131                                                memory_order_relaxed)) {
1132     (*f)();
1133     if (!thr->in_ignored_lib)
1134       Release(thr, pc, (uptr)o);
1135     atomic_store(a, 2, memory_order_release);
1136   } else {
1137     while (v != 2) {
1138       pthread_yield();
1139       v = atomic_load(a, memory_order_acquire);
1140     }
1141     if (!thr->in_ignored_lib)
1142       Acquire(thr, pc, (uptr)o);
1143   }
1144   return 0;
1145 }
1146
1147 TSAN_INTERCEPTOR(int, sem_init, void *s, int pshared, unsigned value) {
1148   SCOPED_TSAN_INTERCEPTOR(sem_init, s, pshared, value);
1149   int res = REAL(sem_init)(s, pshared, value);
1150   return res;
1151 }
1152
1153 TSAN_INTERCEPTOR(int, sem_destroy, void *s) {
1154   SCOPED_TSAN_INTERCEPTOR(sem_destroy, s);
1155   int res = REAL(sem_destroy)(s);
1156   return res;
1157 }
1158
1159 TSAN_INTERCEPTOR(int, sem_wait, void *s) {
1160   SCOPED_TSAN_INTERCEPTOR(sem_wait, s);
1161   int res = BLOCK_REAL(sem_wait)(s);
1162   if (res == 0) {
1163     Acquire(thr, pc, (uptr)s);
1164   }
1165   return res;
1166 }
1167
1168 TSAN_INTERCEPTOR(int, sem_trywait, void *s) {
1169   SCOPED_TSAN_INTERCEPTOR(sem_trywait, s);
1170   int res = BLOCK_REAL(sem_trywait)(s);
1171   if (res == 0) {
1172     Acquire(thr, pc, (uptr)s);
1173   }
1174   return res;
1175 }
1176
1177 TSAN_INTERCEPTOR(int, sem_timedwait, void *s, void *abstime) {
1178   SCOPED_TSAN_INTERCEPTOR(sem_timedwait, s, abstime);
1179   int res = BLOCK_REAL(sem_timedwait)(s, abstime);
1180   if (res == 0) {
1181     Acquire(thr, pc, (uptr)s);
1182   }
1183   return res;
1184 }
1185
1186 TSAN_INTERCEPTOR(int, sem_post, void *s) {
1187   SCOPED_TSAN_INTERCEPTOR(sem_post, s);
1188   Release(thr, pc, (uptr)s);
1189   int res = REAL(sem_post)(s);
1190   return res;
1191 }
1192
1193 TSAN_INTERCEPTOR(int, sem_getvalue, void *s, int *sval) {
1194   SCOPED_TSAN_INTERCEPTOR(sem_getvalue, s, sval);
1195   int res = REAL(sem_getvalue)(s, sval);
1196   if (res == 0) {
1197     Acquire(thr, pc, (uptr)s);
1198   }
1199   return res;
1200 }
1201
1202 TSAN_INTERCEPTOR(int, __xstat, int version, const char *path, void *buf) {
1203   SCOPED_TSAN_INTERCEPTOR(__xstat, version, path, buf);
1204   return REAL(__xstat)(version, path, buf);
1205 }
1206
1207 TSAN_INTERCEPTOR(int, stat, const char *path, void *buf) {
1208   SCOPED_TSAN_INTERCEPTOR(__xstat, 0, path, buf);
1209   return REAL(__xstat)(0, path, buf);
1210 }
1211
1212 TSAN_INTERCEPTOR(int, __xstat64, int version, const char *path, void *buf) {
1213   SCOPED_TSAN_INTERCEPTOR(__xstat64, version, path, buf);
1214   return REAL(__xstat64)(version, path, buf);
1215 }
1216
1217 TSAN_INTERCEPTOR(int, stat64, const char *path, void *buf) {
1218   SCOPED_TSAN_INTERCEPTOR(__xstat64, 0, path, buf);
1219   return REAL(__xstat64)(0, path, buf);
1220 }
1221
1222 TSAN_INTERCEPTOR(int, __lxstat, int version, const char *path, void *buf) {
1223   SCOPED_TSAN_INTERCEPTOR(__lxstat, version, path, buf);
1224   return REAL(__lxstat)(version, path, buf);
1225 }
1226
1227 TSAN_INTERCEPTOR(int, lstat, const char *path, void *buf) {
1228   SCOPED_TSAN_INTERCEPTOR(__lxstat, 0, path, buf);
1229   return REAL(__lxstat)(0, path, buf);
1230 }
1231
1232 TSAN_INTERCEPTOR(int, __lxstat64, int version, const char *path, void *buf) {
1233   SCOPED_TSAN_INTERCEPTOR(__lxstat64, version, path, buf);
1234   return REAL(__lxstat64)(version, path, buf);
1235 }
1236
1237 TSAN_INTERCEPTOR(int, lstat64, const char *path, void *buf) {
1238   SCOPED_TSAN_INTERCEPTOR(__lxstat64, 0, path, buf);
1239   return REAL(__lxstat64)(0, path, buf);
1240 }
1241
1242 TSAN_INTERCEPTOR(int, __fxstat, int version, int fd, void *buf) {
1243   SCOPED_TSAN_INTERCEPTOR(__fxstat, version, fd, buf);
1244   if (fd > 0)
1245     FdAccess(thr, pc, fd);
1246   return REAL(__fxstat)(version, fd, buf);
1247 }
1248
1249 TSAN_INTERCEPTOR(int, fstat, int fd, void *buf) {
1250   SCOPED_TSAN_INTERCEPTOR(__fxstat, 0, fd, buf);
1251   if (fd > 0)
1252     FdAccess(thr, pc, fd);
1253   return REAL(__fxstat)(0, fd, buf);
1254 }
1255
1256 TSAN_INTERCEPTOR(int, __fxstat64, int version, int fd, void *buf) {
1257   SCOPED_TSAN_INTERCEPTOR(__fxstat64, version, fd, buf);
1258   if (fd > 0)
1259     FdAccess(thr, pc, fd);
1260   return REAL(__fxstat64)(version, fd, buf);
1261 }
1262
1263 TSAN_INTERCEPTOR(int, fstat64, int fd, void *buf) {
1264   SCOPED_TSAN_INTERCEPTOR(__fxstat64, 0, fd, buf);
1265   if (fd > 0)
1266     FdAccess(thr, pc, fd);
1267   return REAL(__fxstat64)(0, fd, buf);
1268 }
1269
1270 TSAN_INTERCEPTOR(int, open, const char *name, int flags, int mode) {
1271   SCOPED_TSAN_INTERCEPTOR(open, name, flags, mode);
1272   int fd = REAL(open)(name, flags, mode);
1273   if (fd >= 0)
1274     FdFileCreate(thr, pc, fd);
1275   return fd;
1276 }
1277
1278 TSAN_INTERCEPTOR(int, open64, const char *name, int flags, int mode) {
1279   SCOPED_TSAN_INTERCEPTOR(open64, name, flags, mode);
1280   int fd = REAL(open64)(name, flags, mode);
1281   if (fd >= 0)
1282     FdFileCreate(thr, pc, fd);
1283   return fd;
1284 }
1285
1286 TSAN_INTERCEPTOR(int, creat, const char *name, int mode) {
1287   SCOPED_TSAN_INTERCEPTOR(creat, name, mode);
1288   int fd = REAL(creat)(name, mode);
1289   if (fd >= 0)
1290     FdFileCreate(thr, pc, fd);
1291   return fd;
1292 }
1293
1294 TSAN_INTERCEPTOR(int, creat64, const char *name, int mode) {
1295   SCOPED_TSAN_INTERCEPTOR(creat64, name, mode);
1296   int fd = REAL(creat64)(name, mode);
1297   if (fd >= 0)
1298     FdFileCreate(thr, pc, fd);
1299   return fd;
1300 }
1301
1302 TSAN_INTERCEPTOR(int, dup, int oldfd) {
1303   SCOPED_TSAN_INTERCEPTOR(dup, oldfd);
1304   int newfd = REAL(dup)(oldfd);
1305   if (oldfd >= 0 && newfd >= 0 && newfd != oldfd)
1306     FdDup(thr, pc, oldfd, newfd);
1307   return newfd;
1308 }
1309
1310 TSAN_INTERCEPTOR(int, dup2, int oldfd, int newfd) {
1311   SCOPED_TSAN_INTERCEPTOR(dup2, oldfd, newfd);
1312   int newfd2 = REAL(dup2)(oldfd, newfd);
1313   if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
1314     FdDup(thr, pc, oldfd, newfd2);
1315   return newfd2;
1316 }
1317
1318 TSAN_INTERCEPTOR(int, dup3, int oldfd, int newfd, int flags) {
1319   SCOPED_TSAN_INTERCEPTOR(dup3, oldfd, newfd, flags);
1320   int newfd2 = REAL(dup3)(oldfd, newfd, flags);
1321   if (oldfd >= 0 && newfd2 >= 0 && newfd2 != oldfd)
1322     FdDup(thr, pc, oldfd, newfd2);
1323   return newfd2;
1324 }
1325
1326 TSAN_INTERCEPTOR(int, eventfd, unsigned initval, int flags) {
1327   SCOPED_TSAN_INTERCEPTOR(eventfd, initval, flags);
1328   int fd = REAL(eventfd)(initval, flags);
1329   if (fd >= 0)
1330     FdEventCreate(thr, pc, fd);
1331   return fd;
1332 }
1333
1334 TSAN_INTERCEPTOR(int, signalfd, int fd, void *mask, int flags) {
1335   SCOPED_TSAN_INTERCEPTOR(signalfd, fd, mask, flags);
1336   if (fd >= 0)
1337     FdClose(thr, pc, fd);
1338   fd = REAL(signalfd)(fd, mask, flags);
1339   if (fd >= 0)
1340     FdSignalCreate(thr, pc, fd);
1341   return fd;
1342 }
1343
1344 TSAN_INTERCEPTOR(int, inotify_init, int fake) {
1345   SCOPED_TSAN_INTERCEPTOR(inotify_init, fake);
1346   int fd = REAL(inotify_init)(fake);
1347   if (fd >= 0)
1348     FdInotifyCreate(thr, pc, fd);
1349   return fd;
1350 }
1351
1352 TSAN_INTERCEPTOR(int, inotify_init1, int flags) {
1353   SCOPED_TSAN_INTERCEPTOR(inotify_init1, flags);
1354   int fd = REAL(inotify_init1)(flags);
1355   if (fd >= 0)
1356     FdInotifyCreate(thr, pc, fd);
1357   return fd;
1358 }
1359
1360 TSAN_INTERCEPTOR(int, socket, int domain, int type, int protocol) {
1361   SCOPED_TSAN_INTERCEPTOR(socket, domain, type, protocol);
1362   int fd = REAL(socket)(domain, type, protocol);
1363   if (fd >= 0)
1364     FdSocketCreate(thr, pc, fd);
1365   return fd;
1366 }
1367
1368 TSAN_INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int *fd) {
1369   SCOPED_TSAN_INTERCEPTOR(socketpair, domain, type, protocol, fd);
1370   int res = REAL(socketpair)(domain, type, protocol, fd);
1371   if (res == 0 && fd[0] >= 0 && fd[1] >= 0)
1372     FdPipeCreate(thr, pc, fd[0], fd[1]);
1373   return res;
1374 }
1375
1376 TSAN_INTERCEPTOR(int, connect, int fd, void *addr, unsigned addrlen) {
1377   SCOPED_TSAN_INTERCEPTOR(connect, fd, addr, addrlen);
1378   FdSocketConnecting(thr, pc, fd);
1379   int res = REAL(connect)(fd, addr, addrlen);
1380   if (res == 0 && fd >= 0)
1381     FdSocketConnect(thr, pc, fd);
1382   return res;
1383 }
1384
1385 TSAN_INTERCEPTOR(int, bind, int fd, void *addr, unsigned addrlen) {
1386   SCOPED_TSAN_INTERCEPTOR(bind, fd, addr, addrlen);
1387   int res = REAL(bind)(fd, addr, addrlen);
1388   if (fd > 0 && res == 0)
1389     FdAccess(thr, pc, fd);
1390   return res;
1391 }
1392
1393 TSAN_INTERCEPTOR(int, listen, int fd, int backlog) {
1394   SCOPED_TSAN_INTERCEPTOR(listen, fd, backlog);
1395   int res = REAL(listen)(fd, backlog);
1396   if (fd > 0 && res == 0)
1397     FdAccess(thr, pc, fd);
1398   return res;
1399 }
1400
1401 TSAN_INTERCEPTOR(int, epoll_create, int size) {
1402   SCOPED_TSAN_INTERCEPTOR(epoll_create, size);
1403   int fd = REAL(epoll_create)(size);
1404   if (fd >= 0)
1405     FdPollCreate(thr, pc, fd);
1406   return fd;
1407 }
1408
1409 TSAN_INTERCEPTOR(int, epoll_create1, int flags) {
1410   SCOPED_TSAN_INTERCEPTOR(epoll_create1, flags);
1411   int fd = REAL(epoll_create1)(flags);
1412   if (fd >= 0)
1413     FdPollCreate(thr, pc, fd);
1414   return fd;
1415 }
1416
1417 TSAN_INTERCEPTOR(int, close, int fd) {
1418   SCOPED_TSAN_INTERCEPTOR(close, fd);
1419   if (fd >= 0)
1420     FdClose(thr, pc, fd);
1421   return REAL(close)(fd);
1422 }
1423
1424 TSAN_INTERCEPTOR(int, __close, int fd) {
1425   SCOPED_TSAN_INTERCEPTOR(__close, fd);
1426   if (fd >= 0)
1427     FdClose(thr, pc, fd);
1428   return REAL(__close)(fd);
1429 }
1430
1431 // glibc guts
1432 TSAN_INTERCEPTOR(void, __res_iclose, void *state, bool free_addr) {
1433   SCOPED_TSAN_INTERCEPTOR(__res_iclose, state, free_addr);
1434   int fds[64];
1435   int cnt = ExtractResolvFDs(state, fds, ARRAY_SIZE(fds));
1436   for (int i = 0; i < cnt; i++) {
1437     if (fds[i] > 0)
1438       FdClose(thr, pc, fds[i]);
1439   }
1440   REAL(__res_iclose)(state, free_addr);
1441 }
1442
1443 TSAN_INTERCEPTOR(int, pipe, int *pipefd) {
1444   SCOPED_TSAN_INTERCEPTOR(pipe, pipefd);
1445   int res = REAL(pipe)(pipefd);
1446   if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
1447     FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
1448   return res;
1449 }
1450
1451 TSAN_INTERCEPTOR(int, pipe2, int *pipefd, int flags) {
1452   SCOPED_TSAN_INTERCEPTOR(pipe2, pipefd, flags);
1453   int res = REAL(pipe2)(pipefd, flags);
1454   if (res == 0 && pipefd[0] >= 0 && pipefd[1] >= 0)
1455     FdPipeCreate(thr, pc, pipefd[0], pipefd[1]);
1456   return res;
1457 }
1458
1459 TSAN_INTERCEPTOR(long_t, send, int fd, void *buf, long_t len, int flags) {
1460   SCOPED_TSAN_INTERCEPTOR(send, fd, buf, len, flags);
1461   if (fd >= 0) {
1462     FdAccess(thr, pc, fd);
1463     FdRelease(thr, pc, fd);
1464   }
1465   int res = REAL(send)(fd, buf, len, flags);
1466   return res;
1467 }
1468
1469 TSAN_INTERCEPTOR(long_t, sendmsg, int fd, void *msg, int flags) {
1470   SCOPED_TSAN_INTERCEPTOR(sendmsg, fd, msg, flags);
1471   if (fd >= 0) {
1472     FdAccess(thr, pc, fd);
1473     FdRelease(thr, pc, fd);
1474   }
1475   int res = REAL(sendmsg)(fd, msg, flags);
1476   return res;
1477 }
1478
1479 TSAN_INTERCEPTOR(long_t, recv, int fd, void *buf, long_t len, int flags) {
1480   SCOPED_TSAN_INTERCEPTOR(recv, fd, buf, len, flags);
1481   if (fd >= 0)
1482     FdAccess(thr, pc, fd);
1483   int res = REAL(recv)(fd, buf, len, flags);
1484   if (res >= 0 && fd >= 0) {
1485     FdAcquire(thr, pc, fd);
1486   }
1487   return res;
1488 }
1489
1490 TSAN_INTERCEPTOR(int, unlink, char *path) {
1491   SCOPED_TSAN_INTERCEPTOR(unlink, path);
1492   Release(thr, pc, File2addr(path));
1493   int res = REAL(unlink)(path);
1494   return res;
1495 }
1496
1497 TSAN_INTERCEPTOR(void*, fopen, char *path, char *mode) {
1498   SCOPED_TSAN_INTERCEPTOR(fopen, path, mode);
1499   void *res = REAL(fopen)(path, mode);
1500   Acquire(thr, pc, File2addr(path));
1501   if (res) {
1502     int fd = fileno_unlocked(res);
1503     if (fd >= 0)
1504       FdFileCreate(thr, pc, fd);
1505   }
1506   return res;
1507 }
1508
1509 TSAN_INTERCEPTOR(void*, freopen, char *path, char *mode, void *stream) {
1510   SCOPED_TSAN_INTERCEPTOR(freopen, path, mode, stream);
1511   if (stream) {
1512     int fd = fileno_unlocked(stream);
1513     if (fd >= 0)
1514       FdClose(thr, pc, fd);
1515   }
1516   void *res = REAL(freopen)(path, mode, stream);
1517   Acquire(thr, pc, File2addr(path));
1518   if (res) {
1519     int fd = fileno_unlocked(res);
1520     if (fd >= 0)
1521       FdFileCreate(thr, pc, fd);
1522   }
1523   return res;
1524 }
1525
1526 TSAN_INTERCEPTOR(int, fclose, void *stream) {
1527   // libc file streams can call user-supplied functions, see fopencookie.
1528   {
1529     SCOPED_TSAN_INTERCEPTOR(fclose, stream);
1530     if (stream) {
1531       int fd = fileno_unlocked(stream);
1532       if (fd >= 0)
1533         FdClose(thr, pc, fd);
1534     }
1535   }
1536   return REAL(fclose)(stream);
1537 }
1538
1539 TSAN_INTERCEPTOR(uptr, fread, void *ptr, uptr size, uptr nmemb, void *f) {
1540   // libc file streams can call user-supplied functions, see fopencookie.
1541   {
1542     SCOPED_TSAN_INTERCEPTOR(fread, ptr, size, nmemb, f);
1543     MemoryAccessRange(thr, pc, (uptr)ptr, size * nmemb, true);
1544   }
1545   return REAL(fread)(ptr, size, nmemb, f);
1546 }
1547
1548 TSAN_INTERCEPTOR(uptr, fwrite, const void *p, uptr size, uptr nmemb, void *f) {
1549   // libc file streams can call user-supplied functions, see fopencookie.
1550   {
1551     SCOPED_TSAN_INTERCEPTOR(fwrite, p, size, nmemb, f);
1552     MemoryAccessRange(thr, pc, (uptr)p, size * nmemb, false);
1553   }
1554   return REAL(fwrite)(p, size, nmemb, f);
1555 }
1556
1557 TSAN_INTERCEPTOR(int, fflush, void *stream) {
1558   // libc file streams can call user-supplied functions, see fopencookie.
1559   {
1560     SCOPED_TSAN_INTERCEPTOR(fflush, stream);
1561   }
1562   return REAL(fflush)(stream);
1563 }
1564
1565 TSAN_INTERCEPTOR(void, abort, int fake) {
1566   SCOPED_TSAN_INTERCEPTOR(abort, fake);
1567   REAL(fflush)(0);
1568   REAL(abort)(fake);
1569 }
1570
1571 TSAN_INTERCEPTOR(int, puts, const char *s) {
1572   SCOPED_TSAN_INTERCEPTOR(puts, s);
1573   MemoryAccessRange(thr, pc, (uptr)s, internal_strlen(s), false);
1574   return REAL(puts)(s);
1575 }
1576
1577 TSAN_INTERCEPTOR(int, rmdir, char *path) {
1578   SCOPED_TSAN_INTERCEPTOR(rmdir, path);
1579   Release(thr, pc, Dir2addr(path));
1580   int res = REAL(rmdir)(path);
1581   return res;
1582 }
1583
1584 TSAN_INTERCEPTOR(void*, opendir, char *path) {
1585   SCOPED_TSAN_INTERCEPTOR(opendir, path);
1586   void *res = REAL(opendir)(path);
1587   if (res != 0)
1588     Acquire(thr, pc, Dir2addr(path));
1589   return res;
1590 }
1591
1592 TSAN_INTERCEPTOR(int, epoll_ctl, int epfd, int op, int fd, void *ev) {
1593   SCOPED_TSAN_INTERCEPTOR(epoll_ctl, epfd, op, fd, ev);
1594   if (epfd >= 0)
1595     FdAccess(thr, pc, epfd);
1596   if (epfd >= 0 && fd >= 0)
1597     FdAccess(thr, pc, fd);
1598   if (op == EPOLL_CTL_ADD && epfd >= 0)
1599     FdRelease(thr, pc, epfd);
1600   int res = REAL(epoll_ctl)(epfd, op, fd, ev);
1601   return res;
1602 }
1603
1604 TSAN_INTERCEPTOR(int, epoll_wait, int epfd, void *ev, int cnt, int timeout) {
1605   SCOPED_TSAN_INTERCEPTOR(epoll_wait, epfd, ev, cnt, timeout);
1606   if (epfd >= 0)
1607     FdAccess(thr, pc, epfd);
1608   int res = BLOCK_REAL(epoll_wait)(epfd, ev, cnt, timeout);
1609   if (res > 0 && epfd >= 0)
1610     FdAcquire(thr, pc, epfd);
1611   return res;
1612 }
1613
1614 namespace __tsan {
1615
1616 static void CallUserSignalHandler(ThreadState *thr, bool sync, bool sigact,
1617     int sig, my_siginfo_t *info, void *uctx) {
1618   // Ensure that the handler does not spoil errno.
1619   const int saved_errno = errno;
1620   errno = 99;
1621   // Need to remember pc before the call, because the handler can reset it.
1622   uptr pc = sigact ?
1623      (uptr)sigactions[sig].sa_sigaction :
1624      (uptr)sigactions[sig].sa_handler;
1625   pc += 1;  // return address is expected, OutputReport() will undo this
1626   if (sigact)
1627     sigactions[sig].sa_sigaction(sig, info, uctx);
1628   else
1629     sigactions[sig].sa_handler(sig);
1630   // We do not detect errno spoiling for SIGTERM,
1631   // because some SIGTERM handlers do spoil errno but reraise SIGTERM,
1632   // tsan reports false positive in such case.
1633   // It's difficult to properly detect this situation (reraise),
1634   // because in async signal processing case (when handler is called directly
1635   // from rtl_generic_sighandler) we have not yet received the reraised
1636   // signal; and it looks too fragile to intercept all ways to reraise a signal.
1637   if (flags()->report_bugs && !sync && sig != SIGTERM && errno != 99) {
1638     Context *ctx = CTX();
1639     __tsan::StackTrace stack;
1640     stack.ObtainCurrent(thr, pc);
1641     ThreadRegistryLock l(ctx->thread_registry);
1642     ScopedReport rep(ReportTypeErrnoInSignal);
1643     if (!IsFiredSuppression(ctx, rep, stack)) {
1644       rep.AddStack(&stack);
1645       OutputReport(ctx, rep, rep.GetReport()->stacks[0]);
1646     }
1647   }
1648   errno = saved_errno;
1649 }
1650
1651 void ProcessPendingSignals(ThreadState *thr) {
1652   SignalContext *sctx = SigCtx(thr);
1653   if (sctx == 0 || sctx->pending_signal_count == 0 || thr->in_signal_handler)
1654     return;
1655   thr->in_signal_handler = true;
1656   sctx->pending_signal_count = 0;
1657   // These are too big for stack.
1658   static THREADLOCAL __sanitizer_sigset_t emptyset, oldset;
1659   REAL(sigfillset)(&emptyset);
1660   pthread_sigmask(SIG_SETMASK, &emptyset, &oldset);
1661   for (int sig = 0; sig < kSigCount; sig++) {
1662     SignalDesc *signal = &sctx->pending_signals[sig];
1663     if (signal->armed) {
1664       signal->armed = false;
1665       if (sigactions[sig].sa_handler != SIG_DFL
1666           && sigactions[sig].sa_handler != SIG_IGN) {
1667         CallUserSignalHandler(thr, false, signal->sigaction,
1668             sig, &signal->siginfo, &signal->ctx);
1669       }
1670     }
1671   }
1672   pthread_sigmask(SIG_SETMASK, &oldset, 0);
1673   CHECK_EQ(thr->in_signal_handler, true);
1674   thr->in_signal_handler = false;
1675 }
1676
1677 }  // namespace __tsan
1678
1679 static bool is_sync_signal(SignalContext *sctx, int sig) {
1680   return sig == SIGSEGV || sig == SIGBUS || sig == SIGILL ||
1681       sig == SIGABRT || sig == SIGFPE || sig == SIGPIPE || sig == SIGSYS ||
1682       // If we are sending signal to ourselves, we must process it now.
1683       (sctx && sig == sctx->int_signal_send);
1684 }
1685
1686 void ALWAYS_INLINE rtl_generic_sighandler(bool sigact, int sig,
1687     my_siginfo_t *info, void *ctx) {
1688   ThreadState *thr = cur_thread();
1689   SignalContext *sctx = SigCtx(thr);
1690   if (sig < 0 || sig >= kSigCount) {
1691     VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig);
1692     return;
1693   }
1694   // Don't mess with synchronous signals.
1695   const bool sync = is_sync_signal(sctx, sig);
1696   if (sync ||
1697       // If we are in blocking function, we can safely process it now
1698       // (but check if we are in a recursive interceptor,
1699       // i.e. pthread_join()->munmap()).
1700       (sctx && sctx->in_blocking_func == 1)) {
1701     CHECK_EQ(thr->in_signal_handler, false);
1702     thr->in_signal_handler = true;
1703     if (sctx && sctx->in_blocking_func == 1) {
1704       // We ignore interceptors in blocking functions,
1705       // temporary enbled them again while we are calling user function.
1706       int const i = thr->ignore_interceptors;
1707       thr->ignore_interceptors = 0;
1708       CallUserSignalHandler(thr, sync, sigact, sig, info, ctx);
1709       thr->ignore_interceptors = i;
1710     } else {
1711       CallUserSignalHandler(thr, sync, sigact, sig, info, ctx);
1712     }
1713     CHECK_EQ(thr->in_signal_handler, true);
1714     thr->in_signal_handler = false;
1715     return;
1716   }
1717
1718   if (sctx == 0)
1719     return;
1720   SignalDesc *signal = &sctx->pending_signals[sig];
1721   if (signal->armed == false) {
1722     signal->armed = true;
1723     signal->sigaction = sigact;
1724     if (info)
1725       internal_memcpy(&signal->siginfo, info, sizeof(*info));
1726     if (ctx)
1727       internal_memcpy(&signal->ctx, ctx, sizeof(signal->ctx));
1728     sctx->pending_signal_count++;
1729   }
1730 }
1731
1732 static void rtl_sighandler(int sig) {
1733   rtl_generic_sighandler(false, sig, 0, 0);
1734 }
1735
1736 static void rtl_sigaction(int sig, my_siginfo_t *info, void *ctx) {
1737   rtl_generic_sighandler(true, sig, info, ctx);
1738 }
1739
1740 TSAN_INTERCEPTOR(int, sigaction, int sig, sigaction_t *act, sigaction_t *old) {
1741   SCOPED_TSAN_INTERCEPTOR(sigaction, sig, act, old);
1742   if (old)
1743     internal_memcpy(old, &sigactions[sig], sizeof(*old));
1744   if (act == 0)
1745     return 0;
1746   internal_memcpy(&sigactions[sig], act, sizeof(*act));
1747   sigaction_t newact;
1748   internal_memcpy(&newact, act, sizeof(newact));
1749   REAL(sigfillset)(&newact.sa_mask);
1750   if (act->sa_handler != SIG_IGN && act->sa_handler != SIG_DFL) {
1751     if (newact.sa_flags & SA_SIGINFO)
1752       newact.sa_sigaction = rtl_sigaction;
1753     else
1754       newact.sa_handler = rtl_sighandler;
1755   }
1756   int res = REAL(sigaction)(sig, &newact, 0);
1757   return res;
1758 }
1759
1760 TSAN_INTERCEPTOR(sighandler_t, signal, int sig, sighandler_t h) {
1761   sigaction_t act;
1762   act.sa_handler = h;
1763   REAL(memset)(&act.sa_mask, -1, sizeof(act.sa_mask));
1764   act.sa_flags = 0;
1765   sigaction_t old;
1766   int res = sigaction(sig, &act, &old);
1767   if (res)
1768     return SIG_ERR;
1769   return old.sa_handler;
1770 }
1771
1772 TSAN_INTERCEPTOR(int, sigsuspend, const __sanitizer_sigset_t *mask) {
1773   SCOPED_TSAN_INTERCEPTOR(sigsuspend, mask);
1774   return REAL(sigsuspend)(mask);
1775 }
1776
1777 TSAN_INTERCEPTOR(int, raise, int sig) {
1778   SCOPED_TSAN_INTERCEPTOR(raise, sig);
1779   SignalContext *sctx = SigCtx(thr);
1780   CHECK_NE(sctx, 0);
1781   int prev = sctx->int_signal_send;
1782   sctx->int_signal_send = sig;
1783   int res = REAL(raise)(sig);
1784   CHECK_EQ(sctx->int_signal_send, sig);
1785   sctx->int_signal_send = prev;
1786   return res;
1787 }
1788
1789 TSAN_INTERCEPTOR(int, kill, int pid, int sig) {
1790   SCOPED_TSAN_INTERCEPTOR(kill, pid, sig);
1791   SignalContext *sctx = SigCtx(thr);
1792   CHECK_NE(sctx, 0);
1793   int prev = sctx->int_signal_send;
1794   if (pid == (int)internal_getpid()) {
1795     sctx->int_signal_send = sig;
1796   }
1797   int res = REAL(kill)(pid, sig);
1798   if (pid == (int)internal_getpid()) {
1799     CHECK_EQ(sctx->int_signal_send, sig);
1800     sctx->int_signal_send = prev;
1801   }
1802   return res;
1803 }
1804
1805 TSAN_INTERCEPTOR(int, pthread_kill, void *tid, int sig) {
1806   SCOPED_TSAN_INTERCEPTOR(pthread_kill, tid, sig);
1807   SignalContext *sctx = SigCtx(thr);
1808   CHECK_NE(sctx, 0);
1809   int prev = sctx->int_signal_send;
1810   if (tid == pthread_self()) {
1811     sctx->int_signal_send = sig;
1812   }
1813   int res = REAL(pthread_kill)(tid, sig);
1814   if (tid == pthread_self()) {
1815     CHECK_EQ(sctx->int_signal_send, sig);
1816     sctx->int_signal_send = prev;
1817   }
1818   return res;
1819 }
1820
1821 TSAN_INTERCEPTOR(int, gettimeofday, void *tv, void *tz) {
1822   SCOPED_TSAN_INTERCEPTOR(gettimeofday, tv, tz);
1823   // It's intercepted merely to process pending signals.
1824   return REAL(gettimeofday)(tv, tz);
1825 }
1826
1827 TSAN_INTERCEPTOR(int, getaddrinfo, void *node, void *service,
1828     void *hints, void *rv) {
1829   SCOPED_TSAN_INTERCEPTOR(getaddrinfo, node, service, hints, rv);
1830   // We miss atomic synchronization in getaddrinfo,
1831   // and can report false race between malloc and free
1832   // inside of getaddrinfo. So ignore memory accesses.
1833   ThreadIgnoreBegin(thr, pc);
1834   int res = REAL(getaddrinfo)(node, service, hints, rv);
1835   ThreadIgnoreEnd(thr, pc);
1836   return res;
1837 }
1838
1839 // Linux kernel has a bug that leads to kernel deadlock if a process
1840 // maps TBs of memory and then calls mlock().
1841 static void MlockIsUnsupported() {
1842   static atomic_uint8_t printed;
1843   if (atomic_exchange(&printed, 1, memory_order_relaxed))
1844     return;
1845   if (flags()->verbosity > 0)
1846     Printf("INFO: ThreadSanitizer ignores mlock/mlockall/munlock/munlockall\n");
1847 }
1848
1849 TSAN_INTERCEPTOR(int, mlock, const void *addr, uptr len) {
1850   MlockIsUnsupported();
1851   return 0;
1852 }
1853
1854 TSAN_INTERCEPTOR(int, munlock, const void *addr, uptr len) {
1855   MlockIsUnsupported();
1856   return 0;
1857 }
1858
1859 TSAN_INTERCEPTOR(int, mlockall, int flags) {
1860   MlockIsUnsupported();
1861   return 0;
1862 }
1863
1864 TSAN_INTERCEPTOR(int, munlockall, void) {
1865   MlockIsUnsupported();
1866   return 0;
1867 }
1868
1869 TSAN_INTERCEPTOR(int, fork, int fake) {
1870   if (cur_thread()->in_symbolizer)
1871     return REAL(fork)(fake);
1872   SCOPED_INTERCEPTOR_RAW(fork, fake);
1873   ForkBefore(thr, pc);
1874   int pid = REAL(fork)(fake);
1875   if (pid == 0) {
1876     // child
1877     ForkChildAfter(thr, pc);
1878     FdOnFork(thr, pc);
1879   } else if (pid > 0) {
1880     // parent
1881     ForkParentAfter(thr, pc);
1882   } else {
1883     // error
1884     ForkParentAfter(thr, pc);
1885   }
1886   return pid;
1887 }
1888
1889 TSAN_INTERCEPTOR(int, vfork, int fake) {
1890   // Some programs (e.g. openjdk) call close for all file descriptors
1891   // in the child process. Under tsan it leads to false positives, because
1892   // address space is shared, so the parent process also thinks that
1893   // the descriptors are closed (while they are actually not).
1894   // This leads to false positives due to missed synchronization.
1895   // Strictly saying this is undefined behavior, because vfork child is not
1896   // allowed to call any functions other than exec/exit. But this is what
1897   // openjdk does, so we want to handle it.
1898   // We could disable interceptors in the child process. But it's not possible
1899   // to simply intercept and wrap vfork, because vfork child is not allowed
1900   // to return from the function that calls vfork, and that's exactly what
1901   // we would do. So this would require some assembly trickery as well.
1902   // Instead we simply turn vfork into fork.
1903   return WRAP(fork)(fake);
1904 }
1905
1906 static int OnExit(ThreadState *thr) {
1907   int status = Finalize(thr);
1908   REAL(fflush)(0);
1909   return status;
1910 }
1911
1912 struct TsanInterceptorContext {
1913   ThreadState *thr;
1914   const uptr caller_pc;
1915   const uptr pc;
1916 };
1917
1918 static void HandleRecvmsg(ThreadState *thr, uptr pc,
1919     __sanitizer_msghdr *msg) {
1920   int fds[64];
1921   int cnt = ExtractRecvmsgFDs(msg, fds, ARRAY_SIZE(fds));
1922   for (int i = 0; i < cnt; i++)
1923     FdEventCreate(thr, pc, fds[i]);
1924 }
1925
1926 #include "sanitizer_common/sanitizer_platform_interceptors.h"
1927 // Causes interceptor recursion (getaddrinfo() and fopen())
1928 #undef SANITIZER_INTERCEPT_GETADDRINFO
1929
1930 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
1931 #define COMMON_INTERCEPTOR_UNPOISON_PARAM(ctx, count) \
1932   do {                                                \
1933   } while (false)
1934
1935 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size)                    \
1936   MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr,                 \
1937                     ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
1938                     true)
1939
1940 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size)                       \
1941   MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr,                  \
1942                     ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
1943                     false)
1944
1945 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...)      \
1946   SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__);         \
1947   TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
1948   ctx = (void *)&_ctx;                                \
1949   (void) ctx;
1950
1951 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
1952   FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd)
1953
1954 #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
1955   FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd)
1956
1957 #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \
1958   FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd)
1959
1960 #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
1961   FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd)
1962
1963 #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
1964   ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
1965
1966 #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
1967   CTX()->thread_registry->SetThreadNameByUserId(thread, name)
1968
1969 #define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
1970
1971 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
1972   OnExit(((TsanInterceptorContext *) ctx)->thr)
1973
1974 #define COMMON_INTERCEPTOR_MUTEX_LOCK(ctx, m) \
1975   MutexLock(((TsanInterceptorContext *)ctx)->thr, \
1976             ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
1977
1978 #define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
1979   MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
1980             ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
1981
1982 #define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
1983   MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
1984             ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
1985
1986 #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \
1987   HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \
1988       ((TsanInterceptorContext *)ctx)->pc, msg)
1989
1990 #include "sanitizer_common/sanitizer_common_interceptors.inc"
1991
1992 #define TSAN_SYSCALL() \
1993   ThreadState *thr = cur_thread(); \
1994   if (thr->ignore_interceptors) \
1995     return; \
1996   ScopedSyscall scoped_syscall(thr) \
1997 /**/
1998
1999 struct ScopedSyscall {
2000   ThreadState *thr;
2001
2002   explicit ScopedSyscall(ThreadState *thr)
2003       : thr(thr) {
2004     Initialize(thr);
2005   }
2006
2007   ~ScopedSyscall() {
2008     ProcessPendingSignals(thr);
2009   }
2010 };
2011
2012 static void syscall_access_range(uptr pc, uptr p, uptr s, bool write) {
2013   TSAN_SYSCALL();
2014   MemoryAccessRange(thr, pc, p, s, write);
2015 }
2016
2017 static void syscall_acquire(uptr pc, uptr addr) {
2018   TSAN_SYSCALL();
2019   Acquire(thr, pc, addr);
2020   Printf("syscall_acquire(%p)\n", addr);
2021 }
2022
2023 static void syscall_release(uptr pc, uptr addr) {
2024   TSAN_SYSCALL();
2025   Printf("syscall_release(%p)\n", addr);
2026   Release(thr, pc, addr);
2027 }
2028
2029 static void syscall_fd_close(uptr pc, int fd) {
2030   TSAN_SYSCALL();
2031   FdClose(thr, pc, fd);
2032 }
2033
2034 static USED void syscall_fd_acquire(uptr pc, int fd) {
2035   TSAN_SYSCALL();
2036   FdAcquire(thr, pc, fd);
2037   Printf("syscall_fd_acquire(%p)\n", fd);
2038 }
2039
2040 static USED void syscall_fd_release(uptr pc, int fd) {
2041   TSAN_SYSCALL();
2042   Printf("syscall_fd_release(%p)\n", fd);
2043   FdRelease(thr, pc, fd);
2044 }
2045
2046 static void syscall_pre_fork(uptr pc) {
2047   TSAN_SYSCALL();
2048   ForkBefore(thr, pc);
2049 }
2050
2051 static void syscall_post_fork(uptr pc, int pid) {
2052   TSAN_SYSCALL();
2053   if (pid == 0) {
2054     // child
2055     ForkChildAfter(thr, pc);
2056     FdOnFork(thr, pc);
2057   } else if (pid > 0) {
2058     // parent
2059     ForkParentAfter(thr, pc);
2060   } else {
2061     // error
2062     ForkParentAfter(thr, pc);
2063   }
2064 }
2065
2066 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \
2067   syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false)
2068
2069 #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
2070   syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true)
2071
2072 #define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
2073   do {                                       \
2074     (void)(p);                               \
2075     (void)(s);                               \
2076   } while (false)
2077
2078 #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
2079   do {                                        \
2080     (void)(p);                                \
2081     (void)(s);                                \
2082   } while (false)
2083
2084 #define COMMON_SYSCALL_ACQUIRE(addr) \
2085     syscall_acquire(GET_CALLER_PC(), (uptr)(addr))
2086
2087 #define COMMON_SYSCALL_RELEASE(addr) \
2088     syscall_release(GET_CALLER_PC(), (uptr)(addr))
2089
2090 #define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd)
2091
2092 #define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd)
2093
2094 #define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd)
2095
2096 #define COMMON_SYSCALL_PRE_FORK() \
2097   syscall_pre_fork(GET_CALLER_PC())
2098
2099 #define COMMON_SYSCALL_POST_FORK(res) \
2100   syscall_post_fork(GET_CALLER_PC(), res)
2101
2102 #include "sanitizer_common/sanitizer_common_syscalls.inc"
2103
2104 namespace __tsan {
2105
2106 static void finalize(void *arg) {
2107   ThreadState *thr = cur_thread();
2108   uptr pc = 0;
2109   atexit_ctx->exit(thr, pc);
2110   int status = Finalize(thr);
2111   // Make sure the output is not lost.
2112   // Flushing all the streams here may freeze the process if a child thread is
2113   // performing file stream operations at the same time.
2114   REAL(fflush)(stdout);
2115   REAL(fflush)(stderr);
2116   if (status)
2117     REAL(_exit)(status);
2118 }
2119
2120 static void unreachable() {
2121   Printf("FATAL: ThreadSanitizer: unreachable called\n");
2122   Die();
2123 }
2124
2125 void InitializeInterceptors() {
2126   // We need to setup it early, because functions like dlsym() can call it.
2127   REAL(memset) = internal_memset;
2128   REAL(memcpy) = internal_memcpy;
2129   REAL(memcmp) = internal_memcmp;
2130
2131   // Instruct libc malloc to consume less memory.
2132   mallopt(1, 0);  // M_MXFAST
2133   mallopt(-3, 32*1024);  // M_MMAP_THRESHOLD
2134
2135   SANITIZER_COMMON_INTERCEPTORS_INIT;
2136
2137   // We can not use TSAN_INTERCEPT to get setjmp addr,
2138   // because it does &setjmp and setjmp is not present in some versions of libc.
2139   using __interception::GetRealFunctionAddress;
2140   GetRealFunctionAddress("setjmp", (uptr*)&REAL(setjmp), 0, 0);
2141   GetRealFunctionAddress("_setjmp", (uptr*)&REAL(_setjmp), 0, 0);
2142   GetRealFunctionAddress("sigsetjmp", (uptr*)&REAL(sigsetjmp), 0, 0);
2143   GetRealFunctionAddress("__sigsetjmp", (uptr*)&REAL(__sigsetjmp), 0, 0);
2144
2145   TSAN_INTERCEPT(longjmp);
2146   TSAN_INTERCEPT(siglongjmp);
2147
2148   TSAN_INTERCEPT(malloc);
2149   TSAN_INTERCEPT(__libc_memalign);
2150   TSAN_INTERCEPT(calloc);
2151   TSAN_INTERCEPT(realloc);
2152   TSAN_INTERCEPT(free);
2153   TSAN_INTERCEPT(cfree);
2154   TSAN_INTERCEPT(mmap);
2155   TSAN_INTERCEPT(mmap64);
2156   TSAN_INTERCEPT(munmap);
2157   TSAN_INTERCEPT(memalign);
2158   TSAN_INTERCEPT(valloc);
2159   TSAN_INTERCEPT(pvalloc);
2160   TSAN_INTERCEPT(posix_memalign);
2161
2162   TSAN_INTERCEPT(strlen);
2163   TSAN_INTERCEPT(memset);
2164   TSAN_INTERCEPT(memcpy);
2165   TSAN_INTERCEPT(memchr);
2166   TSAN_INTERCEPT(memrchr);
2167   TSAN_INTERCEPT(memmove);
2168   TSAN_INTERCEPT(memcmp);
2169   TSAN_INTERCEPT(strchr);
2170   TSAN_INTERCEPT(strchrnul);
2171   TSAN_INTERCEPT(strrchr);
2172   TSAN_INTERCEPT(strcpy);  // NOLINT
2173   TSAN_INTERCEPT(strncpy);
2174   TSAN_INTERCEPT(strstr);
2175   TSAN_INTERCEPT(strdup);
2176
2177   TSAN_INTERCEPT(pthread_create);
2178   TSAN_INTERCEPT(pthread_join);
2179   TSAN_INTERCEPT(pthread_detach);
2180
2181   TSAN_INTERCEPT(pthread_mutex_init);
2182   TSAN_INTERCEPT(pthread_mutex_destroy);
2183   TSAN_INTERCEPT(pthread_mutex_trylock);
2184   TSAN_INTERCEPT(pthread_mutex_timedlock);
2185
2186   TSAN_INTERCEPT(pthread_spin_init);
2187   TSAN_INTERCEPT(pthread_spin_destroy);
2188   TSAN_INTERCEPT(pthread_spin_lock);
2189   TSAN_INTERCEPT(pthread_spin_trylock);
2190   TSAN_INTERCEPT(pthread_spin_unlock);
2191
2192   TSAN_INTERCEPT(pthread_rwlock_init);
2193   TSAN_INTERCEPT(pthread_rwlock_destroy);
2194   TSAN_INTERCEPT(pthread_rwlock_rdlock);
2195   TSAN_INTERCEPT(pthread_rwlock_tryrdlock);
2196   TSAN_INTERCEPT(pthread_rwlock_timedrdlock);
2197   TSAN_INTERCEPT(pthread_rwlock_wrlock);
2198   TSAN_INTERCEPT(pthread_rwlock_trywrlock);
2199   TSAN_INTERCEPT(pthread_rwlock_timedwrlock);
2200   TSAN_INTERCEPT(pthread_rwlock_unlock);
2201
2202   TSAN_INTERCEPT(pthread_barrier_init);
2203   TSAN_INTERCEPT(pthread_barrier_destroy);
2204   TSAN_INTERCEPT(pthread_barrier_wait);
2205
2206   TSAN_INTERCEPT(pthread_once);
2207
2208   TSAN_INTERCEPT(sem_init);
2209   TSAN_INTERCEPT(sem_destroy);
2210   TSAN_INTERCEPT(sem_wait);
2211   TSAN_INTERCEPT(sem_trywait);
2212   TSAN_INTERCEPT(sem_timedwait);
2213   TSAN_INTERCEPT(sem_post);
2214   TSAN_INTERCEPT(sem_getvalue);
2215
2216   TSAN_INTERCEPT(stat);
2217   TSAN_INTERCEPT(__xstat);
2218   TSAN_INTERCEPT(stat64);
2219   TSAN_INTERCEPT(__xstat64);
2220   TSAN_INTERCEPT(lstat);
2221   TSAN_INTERCEPT(__lxstat);
2222   TSAN_INTERCEPT(lstat64);
2223   TSAN_INTERCEPT(__lxstat64);
2224   TSAN_INTERCEPT(fstat);
2225   TSAN_INTERCEPT(__fxstat);
2226   TSAN_INTERCEPT(fstat64);
2227   TSAN_INTERCEPT(__fxstat64);
2228   TSAN_INTERCEPT(open);
2229   TSAN_INTERCEPT(open64);
2230   TSAN_INTERCEPT(creat);
2231   TSAN_INTERCEPT(creat64);
2232   TSAN_INTERCEPT(dup);
2233   TSAN_INTERCEPT(dup2);
2234   TSAN_INTERCEPT(dup3);
2235   TSAN_INTERCEPT(eventfd);
2236   TSAN_INTERCEPT(signalfd);
2237   TSAN_INTERCEPT(inotify_init);
2238   TSAN_INTERCEPT(inotify_init1);
2239   TSAN_INTERCEPT(socket);
2240   TSAN_INTERCEPT(socketpair);
2241   TSAN_INTERCEPT(connect);
2242   TSAN_INTERCEPT(bind);
2243   TSAN_INTERCEPT(listen);
2244   TSAN_INTERCEPT(epoll_create);
2245   TSAN_INTERCEPT(epoll_create1);
2246   TSAN_INTERCEPT(close);
2247   TSAN_INTERCEPT(__close);
2248   TSAN_INTERCEPT(__res_iclose);
2249   TSAN_INTERCEPT(pipe);
2250   TSAN_INTERCEPT(pipe2);
2251
2252   TSAN_INTERCEPT(send);
2253   TSAN_INTERCEPT(sendmsg);
2254   TSAN_INTERCEPT(recv);
2255
2256   TSAN_INTERCEPT(unlink);
2257   TSAN_INTERCEPT(fopen);
2258   TSAN_INTERCEPT(freopen);
2259   TSAN_INTERCEPT(fclose);
2260   TSAN_INTERCEPT(fread);
2261   TSAN_INTERCEPT(fwrite);
2262   TSAN_INTERCEPT(fflush);
2263   TSAN_INTERCEPT(abort);
2264   TSAN_INTERCEPT(puts);
2265   TSAN_INTERCEPT(rmdir);
2266   TSAN_INTERCEPT(opendir);
2267
2268   TSAN_INTERCEPT(epoll_ctl);
2269   TSAN_INTERCEPT(epoll_wait);
2270
2271   TSAN_INTERCEPT(sigaction);
2272   TSAN_INTERCEPT(signal);
2273   TSAN_INTERCEPT(sigsuspend);
2274   TSAN_INTERCEPT(raise);
2275   TSAN_INTERCEPT(kill);
2276   TSAN_INTERCEPT(pthread_kill);
2277   TSAN_INTERCEPT(sleep);
2278   TSAN_INTERCEPT(usleep);
2279   TSAN_INTERCEPT(nanosleep);
2280   TSAN_INTERCEPT(gettimeofday);
2281   TSAN_INTERCEPT(getaddrinfo);
2282
2283   TSAN_INTERCEPT(mlock);
2284   TSAN_INTERCEPT(munlock);
2285   TSAN_INTERCEPT(mlockall);
2286   TSAN_INTERCEPT(munlockall);
2287
2288   TSAN_INTERCEPT(fork);
2289   TSAN_INTERCEPT(vfork);
2290   TSAN_INTERCEPT(dlopen);
2291   TSAN_INTERCEPT(dlclose);
2292   TSAN_INTERCEPT(on_exit);
2293   TSAN_INTERCEPT(__cxa_atexit);
2294   TSAN_INTERCEPT(_exit);
2295
2296   // Need to setup it, because interceptors check that the function is resolved.
2297   // But atexit is emitted directly into the module, so can't be resolved.
2298   REAL(atexit) = (int(*)(void(*)()))unreachable;
2299   atexit_ctx = new(internal_alloc(MBlockAtExit, sizeof(AtExitContext)))
2300       AtExitContext();
2301
2302   if (REAL(__cxa_atexit)(&finalize, 0, 0)) {
2303     Printf("ThreadSanitizer: failed to setup atexit callback\n");
2304     Die();
2305   }
2306
2307   if (pthread_key_create(&g_thread_finalize_key, &thread_finalize)) {
2308     Printf("ThreadSanitizer: failed to create thread key\n");
2309     Die();
2310   }
2311
2312   FdInit();
2313 }
2314
2315 void internal_start_thread(void(*func)(void *arg), void *arg) {
2316   // Start the thread with signals blocked, otherwise it can steal user signals.
2317   __sanitizer_sigset_t set, old;
2318   internal_sigfillset(&set);
2319   internal_sigprocmask(SIG_SETMASK, &set, &old);
2320   void *th;
2321   REAL(pthread_create)(&th, 0, (void*(*)(void *arg))func, arg);
2322   REAL(pthread_detach)(th);
2323   internal_sigprocmask(SIG_SETMASK, &old, 0);
2324 }
2325
2326 }  // namespace __tsan