1 //===-- tsan_interceptors_posix.cpp ---------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is a part of ThreadSanitizer (TSan), a race detector.
11 // FIXME: move as many interceptors as possible into
12 // sanitizer_common/sanitizer_common_interceptors.inc
13 //===----------------------------------------------------------------------===//
15 #include "sanitizer_common/sanitizer_atomic.h"
16 #include "sanitizer_common/sanitizer_errno.h"
17 #include "sanitizer_common/sanitizer_libc.h"
18 #include "sanitizer_common/sanitizer_linux.h"
19 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
20 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
21 #include "sanitizer_common/sanitizer_placement_new.h"
22 #include "sanitizer_common/sanitizer_posix.h"
23 #include "sanitizer_common/sanitizer_stacktrace.h"
24 #include "sanitizer_common/sanitizer_tls_get_addr.h"
25 #include "interception/interception.h"
26 #include "tsan_interceptors.h"
27 #include "tsan_interface.h"
28 #include "tsan_platform.h"
29 #include "tsan_suppressions.h"
31 #include "tsan_mman.h"
34 using namespace __tsan
;
36 #if SANITIZER_FREEBSD || SANITIZER_MAC
37 #define stdout __stdoutp
38 #define stderr __stderrp
42 #define dirfd(dirp) (*(int *)(dirp))
43 #define fileno_unlocked(fp) \
44 (((__sanitizer_FILE *)fp)->_file == -1 \
46 : (int)(unsigned short)(((__sanitizer_FILE *)fp)->_file))
48 #define stdout ((__sanitizer_FILE*)&__sF[1])
49 #define stderr ((__sanitizer_FILE*)&__sF[2])
51 #define nanosleep __nanosleep50
52 #define vfork __vfork14
60 const int kSigCount
= 129;
62 const int kSigCount
= 65;
67 u64 opaque
[768 / sizeof(u64
) + 1];
71 // The size is determined by looking at sizeof of real ucontext_t on linux.
72 u64 opaque
[936 / sizeof(u64
) + 1];
76 #if defined(__x86_64__) || defined(__mips__) || SANITIZER_PPC64V1
77 #define PTHREAD_ABI_BASE "GLIBC_2.3.2"
78 #elif defined(__aarch64__) || SANITIZER_PPC64V2
79 #define PTHREAD_ABI_BASE "GLIBC_2.17"
82 extern "C" int pthread_attr_init(void *attr
);
83 extern "C" int pthread_attr_destroy(void *attr
);
84 DECLARE_REAL(int, pthread_attr_getdetachstate
, void *, void *)
85 extern "C" int pthread_attr_setstacksize(void *attr
, uptr stacksize
);
86 extern "C" int pthread_key_create(unsigned *key
, void (*destructor
)(void* v
));
87 extern "C" int pthread_setspecific(unsigned key
, const void *v
);
88 DECLARE_REAL(int, pthread_mutexattr_gettype
, void *, void *)
89 DECLARE_REAL(int, fflush
, __sanitizer_FILE
*fp
)
90 DECLARE_REAL_AND_INTERCEPTOR(void *, malloc
, uptr size
)
91 DECLARE_REAL_AND_INTERCEPTOR(void, free
, void *ptr
)
92 extern "C" void *pthread_self();
93 extern "C" void _exit(int status
);
95 extern "C" int fileno_unlocked(void *stream
);
96 extern "C" int dirfd(void *dirp
);
98 #if !SANITIZER_FREEBSD && !SANITIZER_ANDROID && !SANITIZER_NETBSD
99 extern "C" int mallopt(int param
, int value
);
102 extern __sanitizer_FILE __sF
[];
104 extern __sanitizer_FILE
*stdout
, *stderr
;
106 #if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
107 const int PTHREAD_MUTEX_RECURSIVE
= 1;
108 const int PTHREAD_MUTEX_RECURSIVE_NP
= 1;
110 const int PTHREAD_MUTEX_RECURSIVE
= 2;
111 const int PTHREAD_MUTEX_RECURSIVE_NP
= 2;
113 #if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
114 const int EPOLL_CTL_ADD
= 1;
116 const int SIGILL
= 4;
117 const int SIGTRAP
= 5;
118 const int SIGABRT
= 6;
119 const int SIGFPE
= 8;
120 const int SIGSEGV
= 11;
121 const int SIGPIPE
= 13;
122 const int SIGTERM
= 15;
123 #if defined(__mips__) || SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
124 const int SIGBUS
= 10;
125 const int SIGSYS
= 12;
127 const int SIGBUS
= 7;
128 const int SIGSYS
= 31;
130 void *const MAP_FAILED
= (void*)-1;
132 const int PTHREAD_BARRIER_SERIAL_THREAD
= 1234567;
134 const int PTHREAD_BARRIER_SERIAL_THREAD
= -1;
136 const int MAP_FIXED
= 0x10;
139 // From /usr/include/unistd.h
140 # define F_ULOCK 0 /* Unlock a previously locked region. */
141 # define F_LOCK 1 /* Lock a region for exclusive use. */
142 # define F_TLOCK 2 /* Test and lock a region for exclusive use. */
143 # define F_TEST 3 /* Test a region for other processes locks. */
145 #if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_NETBSD
146 const int SA_SIGINFO
= 0x40;
147 const int SIG_SETMASK
= 3;
148 #elif defined(__mips__)
149 const int SA_SIGINFO
= 8;
150 const int SIG_SETMASK
= 3;
152 const int SA_SIGINFO
= 4;
153 const int SIG_SETMASK
= 2;
156 #define COMMON_INTERCEPTOR_NOTHING_IS_INITIALIZED \
157 (cur_thread_init(), !cur_thread()->is_inited)
163 __sanitizer_siginfo siginfo
;
167 struct ThreadSignalContext
{
169 atomic_uintptr_t in_blocking_func
;
170 atomic_uintptr_t have_pending_signals
;
171 SignalDesc pending_signals
[kSigCount
];
172 // emptyset and oldset are too big for stack.
173 __sanitizer_sigset_t emptyset
;
174 __sanitizer_sigset_t oldset
;
177 // The sole reason tsan wraps atexit callbacks is to establish synchronization
178 // between callback setup and callback execution.
184 // InterceptorContext holds all global data required for interceptors.
185 // It's explicitly constructed in InitializeInterceptors with placement new
186 // and is never destroyed. This allows usage of members with non-trivial
187 // constructors and destructors.
188 struct InterceptorContext
{
189 // The object is 64-byte aligned, because we want hot data to be located
190 // in a single cache line if possible (it's accessed in every interceptor).
191 ALIGNED(64) LibIgnore libignore
;
192 __sanitizer_sigaction sigactions
[kSigCount
];
193 #if !SANITIZER_MAC && !SANITIZER_NETBSD
194 unsigned finalize_key
;
197 BlockingMutex atexit_mu
;
198 Vector
<struct AtExitCtx
*> AtExitStack
;
201 : libignore(LINKER_INITIALIZED
), AtExitStack() {
205 static ALIGNED(64) char interceptor_placeholder
[sizeof(InterceptorContext
)];
206 InterceptorContext
*interceptor_ctx() {
207 return reinterpret_cast<InterceptorContext
*>(&interceptor_placeholder
[0]);
210 LibIgnore
*libignore() {
211 return &interceptor_ctx()->libignore
;
214 void InitializeLibIgnore() {
215 const SuppressionContext
&supp
= *Suppressions();
216 const uptr n
= supp
.SuppressionCount();
217 for (uptr i
= 0; i
< n
; i
++) {
218 const Suppression
*s
= supp
.SuppressionAt(i
);
219 if (0 == internal_strcmp(s
->type
, kSuppressionLib
))
220 libignore()->AddIgnoredLibrary(s
->templ
);
222 if (flags()->ignore_noninstrumented_modules
)
223 libignore()->IgnoreNoninstrumentedModules(true);
224 libignore()->OnLibraryLoaded(0);
227 // The following two hooks can be used by for cooperative scheduling when
229 #ifdef TSAN_EXTERNAL_HOOKS
230 void OnPotentiallyBlockingRegionBegin();
231 void OnPotentiallyBlockingRegionEnd();
233 SANITIZER_WEAK_CXX_DEFAULT_IMPL
void OnPotentiallyBlockingRegionBegin() {}
234 SANITIZER_WEAK_CXX_DEFAULT_IMPL
void OnPotentiallyBlockingRegionEnd() {}
237 } // namespace __tsan
239 static ThreadSignalContext
*SigCtx(ThreadState
*thr
) {
240 ThreadSignalContext
*ctx
= (ThreadSignalContext
*)thr
->signal_ctx
;
241 if (ctx
== 0 && !thr
->is_dead
) {
242 ctx
= (ThreadSignalContext
*)MmapOrDie(sizeof(*ctx
), "ThreadSignalContext");
243 MemoryResetRange(thr
, (uptr
)&SigCtx
, (uptr
)ctx
, sizeof(*ctx
));
244 thr
->signal_ctx
= ctx
;
249 ScopedInterceptor::ScopedInterceptor(ThreadState
*thr
, const char *fname
,
251 : thr_(thr
), pc_(pc
), in_ignored_lib_(false), ignoring_(false) {
253 if (!thr_
->is_inited
) return;
254 if (!thr_
->ignore_interceptors
) FuncEntry(thr
, pc
);
255 DPrintf("#%d: intercept %s()\n", thr_
->tid
, fname
);
257 !thr_
->in_ignored_lib
&& libignore()->IsIgnored(pc
, &in_ignored_lib_
);
261 ScopedInterceptor::~ScopedInterceptor() {
262 if (!thr_
->is_inited
) return;
264 if (!thr_
->ignore_interceptors
) {
265 ProcessPendingSignals(thr_
);
271 void ScopedInterceptor::EnableIgnores() {
273 ThreadIgnoreBegin(thr_
, pc_
, /*save_stack=*/false);
274 if (flags()->ignore_noninstrumented_modules
) thr_
->suppress_reports
++;
275 if (in_ignored_lib_
) {
276 DCHECK(!thr_
->in_ignored_lib
);
277 thr_
->in_ignored_lib
= true;
282 void ScopedInterceptor::DisableIgnores() {
284 ThreadIgnoreEnd(thr_
, pc_
);
285 if (flags()->ignore_noninstrumented_modules
) thr_
->suppress_reports
--;
286 if (in_ignored_lib_
) {
287 DCHECK(thr_
->in_ignored_lib
);
288 thr_
->in_ignored_lib
= false;
293 #define TSAN_INTERCEPT(func) INTERCEPT_FUNCTION(func)
294 #if SANITIZER_FREEBSD
295 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
296 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
297 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
298 #elif SANITIZER_NETBSD
299 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION(func)
300 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func) \
301 INTERCEPT_FUNCTION(__libc_##func)
302 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func) \
303 INTERCEPT_FUNCTION(__libc_thr_##func)
305 # define TSAN_INTERCEPT_VER(func, ver) INTERCEPT_FUNCTION_VER(func, ver)
306 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(func)
307 # define TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(func)
310 #define READ_STRING_OF_LEN(thr, pc, s, len, n) \
311 MemoryAccessRange((thr), (pc), (uptr)(s), \
312 common_flags()->strict_string_checks ? (len) + 1 : (n), false)
314 #define READ_STRING(thr, pc, s, n) \
315 READ_STRING_OF_LEN((thr), (pc), (s), internal_strlen(s), (n))
317 #define BLOCK_REAL(name) (BlockingCall(thr), REAL(name))
319 struct BlockingCall
{
320 explicit BlockingCall(ThreadState
*thr
)
324 atomic_store(&ctx
->in_blocking_func
, 1, memory_order_relaxed
);
325 if (atomic_load(&ctx
->have_pending_signals
, memory_order_relaxed
) == 0)
327 atomic_store(&ctx
->in_blocking_func
, 0, memory_order_relaxed
);
328 ProcessPendingSignals(thr
);
330 // When we are in a "blocking call", we process signals asynchronously
331 // (right when they arrive). In this context we do not expect to be
332 // executing any user/runtime code. The known interceptor sequence when
333 // this is not true is: pthread_join -> munmap(stack). It's fine
334 // to ignore munmap in this case -- we handle stack shadow separately.
335 thr
->ignore_interceptors
++;
339 thr
->ignore_interceptors
--;
340 atomic_store(&ctx
->in_blocking_func
, 0, memory_order_relaxed
);
344 ThreadSignalContext
*ctx
;
347 TSAN_INTERCEPTOR(unsigned, sleep
, unsigned sec
) {
348 SCOPED_TSAN_INTERCEPTOR(sleep
, sec
);
349 unsigned res
= BLOCK_REAL(sleep
)(sec
);
354 TSAN_INTERCEPTOR(int, usleep
, long_t usec
) {
355 SCOPED_TSAN_INTERCEPTOR(usleep
, usec
);
356 int res
= BLOCK_REAL(usleep
)(usec
);
361 TSAN_INTERCEPTOR(int, nanosleep
, void *req
, void *rem
) {
362 SCOPED_TSAN_INTERCEPTOR(nanosleep
, req
, rem
);
363 int res
= BLOCK_REAL(nanosleep
)(req
, rem
);
368 TSAN_INTERCEPTOR(int, pause
, int fake
) {
369 SCOPED_TSAN_INTERCEPTOR(pause
, fake
);
370 return BLOCK_REAL(pause
)(fake
);
373 static void at_exit_wrapper() {
376 // Ensure thread-safety.
377 BlockingMutexLock
l(&interceptor_ctx()->atexit_mu
);
379 // Pop AtExitCtx from the top of the stack of callback functions
380 uptr element
= interceptor_ctx()->AtExitStack
.Size() - 1;
381 ctx
= interceptor_ctx()->AtExitStack
[element
];
382 interceptor_ctx()->AtExitStack
.PopBack();
385 Acquire(cur_thread(), (uptr
)0, (uptr
)ctx
);
386 ((void(*)())ctx
->f
)();
390 static void cxa_at_exit_wrapper(void *arg
) {
391 Acquire(cur_thread(), 0, (uptr
)arg
);
392 AtExitCtx
*ctx
= (AtExitCtx
*)arg
;
393 ((void(*)(void *arg
))ctx
->f
)(ctx
->arg
);
397 static int setup_at_exit_wrapper(ThreadState
*thr
, uptr pc
, void(*f
)(),
398 void *arg
, void *dso
);
400 #if !SANITIZER_ANDROID
401 TSAN_INTERCEPTOR(int, atexit
, void (*f
)()) {
404 // We want to setup the atexit callback even if we are in ignored lib
406 SCOPED_INTERCEPTOR_RAW(atexit
, f
);
407 return setup_at_exit_wrapper(thr
, pc
, (void(*)())f
, 0, 0);
411 TSAN_INTERCEPTOR(int, __cxa_atexit
, void (*f
)(void *a
), void *arg
, void *dso
) {
414 SCOPED_TSAN_INTERCEPTOR(__cxa_atexit
, f
, arg
, dso
);
415 return setup_at_exit_wrapper(thr
, pc
, (void(*)())f
, arg
, dso
);
418 static int setup_at_exit_wrapper(ThreadState
*thr
, uptr pc
, void(*f
)(),
419 void *arg
, void *dso
) {
420 AtExitCtx
*ctx
= (AtExitCtx
*)InternalAlloc(sizeof(AtExitCtx
));
423 Release(thr
, pc
, (uptr
)ctx
);
424 // Memory allocation in __cxa_atexit will race with free during exit,
425 // because we do not see synchronization around atexit callback list.
426 ThreadIgnoreBegin(thr
, pc
);
429 // NetBSD does not preserve the 2nd argument if dso is equal to 0
430 // Store ctx in a local stack-like structure
432 // Ensure thread-safety.
433 BlockingMutexLock
l(&interceptor_ctx()->atexit_mu
);
435 res
= REAL(__cxa_atexit
)((void (*)(void *a
))at_exit_wrapper
, 0, 0);
436 // Push AtExitCtx on the top of the stack of callback functions
438 interceptor_ctx()->AtExitStack
.PushBack(ctx
);
441 res
= REAL(__cxa_atexit
)(cxa_at_exit_wrapper
, ctx
, dso
);
443 ThreadIgnoreEnd(thr
, pc
);
447 #if !SANITIZER_MAC && !SANITIZER_NETBSD
448 static void on_exit_wrapper(int status
, void *arg
) {
449 ThreadState
*thr
= cur_thread();
451 Acquire(thr
, pc
, (uptr
)arg
);
452 AtExitCtx
*ctx
= (AtExitCtx
*)arg
;
453 ((void(*)(int status
, void *arg
))ctx
->f
)(status
, ctx
->arg
);
457 TSAN_INTERCEPTOR(int, on_exit
, void(*f
)(int, void*), void *arg
) {
460 SCOPED_TSAN_INTERCEPTOR(on_exit
, f
, arg
);
461 AtExitCtx
*ctx
= (AtExitCtx
*)InternalAlloc(sizeof(AtExitCtx
));
462 ctx
->f
= (void(*)())f
;
464 Release(thr
, pc
, (uptr
)ctx
);
465 // Memory allocation in __cxa_atexit will race with free during exit,
466 // because we do not see synchronization around atexit callback list.
467 ThreadIgnoreBegin(thr
, pc
);
468 int res
= REAL(on_exit
)(on_exit_wrapper
, ctx
);
469 ThreadIgnoreEnd(thr
, pc
);
472 #define TSAN_MAYBE_INTERCEPT_ON_EXIT TSAN_INTERCEPT(on_exit)
474 #define TSAN_MAYBE_INTERCEPT_ON_EXIT
478 static void JmpBufGarbageCollect(ThreadState
*thr
, uptr sp
) {
479 for (uptr i
= 0; i
< thr
->jmp_bufs
.Size(); i
++) {
480 JmpBuf
*buf
= &thr
->jmp_bufs
[i
];
482 uptr sz
= thr
->jmp_bufs
.Size();
483 internal_memcpy(buf
, &thr
->jmp_bufs
[sz
- 1], sizeof(*buf
));
484 thr
->jmp_bufs
.PopBack();
490 static void SetJmp(ThreadState
*thr
, uptr sp
) {
491 if (!thr
->is_inited
) // called from libc guts during bootstrap
494 JmpBufGarbageCollect(thr
, sp
);
496 JmpBuf
*buf
= thr
->jmp_bufs
.PushBack();
498 buf
->shadow_stack_pos
= thr
->shadow_stack_pos
;
499 ThreadSignalContext
*sctx
= SigCtx(thr
);
500 buf
->int_signal_send
= sctx
? sctx
->int_signal_send
: 0;
501 buf
->in_blocking_func
= sctx
?
502 atomic_load(&sctx
->in_blocking_func
, memory_order_relaxed
) :
504 buf
->in_signal_handler
= atomic_load(&thr
->in_signal_handler
,
505 memory_order_relaxed
);
508 static void LongJmp(ThreadState
*thr
, uptr
*env
) {
509 uptr sp
= ExtractLongJmpSp(env
);
510 // Find the saved buf with matching sp.
511 for (uptr i
= 0; i
< thr
->jmp_bufs
.Size(); i
++) {
512 JmpBuf
*buf
= &thr
->jmp_bufs
[i
];
514 CHECK_GE(thr
->shadow_stack_pos
, buf
->shadow_stack_pos
);
516 while (thr
->shadow_stack_pos
> buf
->shadow_stack_pos
)
518 ThreadSignalContext
*sctx
= SigCtx(thr
);
520 sctx
->int_signal_send
= buf
->int_signal_send
;
521 atomic_store(&sctx
->in_blocking_func
, buf
->in_blocking_func
,
522 memory_order_relaxed
);
524 atomic_store(&thr
->in_signal_handler
, buf
->in_signal_handler
,
525 memory_order_relaxed
);
526 JmpBufGarbageCollect(thr
, buf
->sp
- 1); // do not collect buf->sp
530 Printf("ThreadSanitizer: can't find longjmp buf\n");
534 // FIXME: put everything below into a common extern "C" block?
535 extern "C" void __tsan_setjmp(uptr sp
) {
537 SetJmp(cur_thread(), sp
);
541 TSAN_INTERCEPTOR(int, setjmp
, void *env
);
542 TSAN_INTERCEPTOR(int, _setjmp
, void *env
);
543 TSAN_INTERCEPTOR(int, sigsetjmp
, void *env
);
544 #else // SANITIZER_MAC
547 #define setjmp_symname __setjmp14
548 #define sigsetjmp_symname __sigsetjmp14
550 #define setjmp_symname setjmp
551 #define sigsetjmp_symname sigsetjmp
554 #define TSAN_INTERCEPTOR_SETJMP_(x) __interceptor_ ## x
555 #define TSAN_INTERCEPTOR_SETJMP__(x) TSAN_INTERCEPTOR_SETJMP_(x)
556 #define TSAN_INTERCEPTOR_SETJMP TSAN_INTERCEPTOR_SETJMP__(setjmp_symname)
557 #define TSAN_INTERCEPTOR_SIGSETJMP TSAN_INTERCEPTOR_SETJMP__(sigsetjmp_symname)
559 #define TSAN_STRING_SETJMP SANITIZER_STRINGIFY(setjmp_symname)
560 #define TSAN_STRING_SIGSETJMP SANITIZER_STRINGIFY(sigsetjmp_symname)
562 // Not called. Merely to satisfy TSAN_INTERCEPT().
563 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
564 int TSAN_INTERCEPTOR_SETJMP(void *env
);
565 extern "C" int TSAN_INTERCEPTOR_SETJMP(void *env
) {
570 // FIXME: any reason to have a separate declaration?
571 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
572 int __interceptor__setjmp(void *env
);
573 extern "C" int __interceptor__setjmp(void *env
) {
578 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
579 int TSAN_INTERCEPTOR_SIGSETJMP(void *env
);
580 extern "C" int TSAN_INTERCEPTOR_SIGSETJMP(void *env
) {
585 #if !SANITIZER_NETBSD
586 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
587 int __interceptor___sigsetjmp(void *env
);
588 extern "C" int __interceptor___sigsetjmp(void *env
) {
594 extern "C" int setjmp_symname(void *env
);
595 extern "C" int _setjmp(void *env
);
596 extern "C" int sigsetjmp_symname(void *env
);
597 #if !SANITIZER_NETBSD
598 extern "C" int __sigsetjmp(void *env
);
600 DEFINE_REAL(int, setjmp_symname
, void *env
)
601 DEFINE_REAL(int, _setjmp
, void *env
)
602 DEFINE_REAL(int, sigsetjmp_symname
, void *env
)
603 #if !SANITIZER_NETBSD
604 DEFINE_REAL(int, __sigsetjmp
, void *env
)
606 #endif // SANITIZER_MAC
609 #define longjmp_symname __longjmp14
610 #define siglongjmp_symname __siglongjmp14
612 #define longjmp_symname longjmp
613 #define siglongjmp_symname siglongjmp
616 TSAN_INTERCEPTOR(void, longjmp_symname
, uptr
*env
, int val
) {
617 // Note: if we call REAL(longjmp) in the context of ScopedInterceptor,
618 // bad things will happen. We will jump over ScopedInterceptor dtor and can
619 // leave thr->in_ignored_lib set.
621 SCOPED_INTERCEPTOR_RAW(longjmp_symname
, env
, val
);
623 LongJmp(cur_thread(), env
);
624 REAL(longjmp_symname
)(env
, val
);
627 TSAN_INTERCEPTOR(void, siglongjmp_symname
, uptr
*env
, int val
) {
629 SCOPED_INTERCEPTOR_RAW(siglongjmp_symname
, env
, val
);
631 LongJmp(cur_thread(), env
);
632 REAL(siglongjmp_symname
)(env
, val
);
636 TSAN_INTERCEPTOR(void, _longjmp
, uptr
*env
, int val
) {
638 SCOPED_INTERCEPTOR_RAW(_longjmp
, env
, val
);
640 LongJmp(cur_thread(), env
);
641 REAL(_longjmp
)(env
, val
);
646 TSAN_INTERCEPTOR(void*, malloc
, uptr size
) {
648 return InternalAlloc(size
);
651 SCOPED_INTERCEPTOR_RAW(malloc
, size
);
652 p
= user_alloc(thr
, pc
, size
);
654 invoke_malloc_hook(p
, size
);
658 TSAN_INTERCEPTOR(void*, __libc_memalign
, uptr align
, uptr sz
) {
659 SCOPED_TSAN_INTERCEPTOR(__libc_memalign
, align
, sz
);
660 return user_memalign(thr
, pc
, align
, sz
);
663 TSAN_INTERCEPTOR(void*, calloc
, uptr size
, uptr n
) {
665 return InternalCalloc(size
, n
);
668 SCOPED_INTERCEPTOR_RAW(calloc
, size
, n
);
669 p
= user_calloc(thr
, pc
, size
, n
);
671 invoke_malloc_hook(p
, n
* size
);
675 TSAN_INTERCEPTOR(void*, realloc
, void *p
, uptr size
) {
677 return InternalRealloc(p
, size
);
681 SCOPED_INTERCEPTOR_RAW(realloc
, p
, size
);
682 p
= user_realloc(thr
, pc
, p
, size
);
684 invoke_malloc_hook(p
, size
);
688 TSAN_INTERCEPTOR(void*, reallocarray
, void *p
, uptr size
, uptr n
) {
690 return InternalReallocArray(p
, size
, n
);
694 SCOPED_INTERCEPTOR_RAW(reallocarray
, p
, size
, n
);
695 p
= user_reallocarray(thr
, pc
, p
, size
, n
);
697 invoke_malloc_hook(p
, size
);
701 TSAN_INTERCEPTOR(void, free
, void *p
) {
705 return InternalFree(p
);
707 SCOPED_INTERCEPTOR_RAW(free
, p
);
708 user_free(thr
, pc
, p
);
711 TSAN_INTERCEPTOR(void, cfree
, void *p
) {
715 return InternalFree(p
);
717 SCOPED_INTERCEPTOR_RAW(cfree
, p
);
718 user_free(thr
, pc
, p
);
721 TSAN_INTERCEPTOR(uptr
, malloc_usable_size
, void *p
) {
722 SCOPED_INTERCEPTOR_RAW(malloc_usable_size
, p
);
723 return user_alloc_usable_size(p
);
727 TSAN_INTERCEPTOR(char *, strcpy
, char *dst
, const char *src
) {
728 SCOPED_TSAN_INTERCEPTOR(strcpy
, dst
, src
);
729 uptr srclen
= internal_strlen(src
);
730 MemoryAccessRange(thr
, pc
, (uptr
)dst
, srclen
+ 1, true);
731 MemoryAccessRange(thr
, pc
, (uptr
)src
, srclen
+ 1, false);
732 return REAL(strcpy
)(dst
, src
);
735 TSAN_INTERCEPTOR(char*, strncpy
, char *dst
, char *src
, uptr n
) {
736 SCOPED_TSAN_INTERCEPTOR(strncpy
, dst
, src
, n
);
737 uptr srclen
= internal_strnlen(src
, n
);
738 MemoryAccessRange(thr
, pc
, (uptr
)dst
, n
, true);
739 MemoryAccessRange(thr
, pc
, (uptr
)src
, min(srclen
+ 1, n
), false);
740 return REAL(strncpy
)(dst
, src
, n
);
743 TSAN_INTERCEPTOR(char*, strdup
, const char *str
) {
744 SCOPED_TSAN_INTERCEPTOR(strdup
, str
);
745 // strdup will call malloc, so no instrumentation is required here.
746 return REAL(strdup
)(str
);
749 // Zero out addr if it points into shadow memory and was provided as a hint
750 // only, i.e., MAP_FIXED is not set.
751 static bool fix_mmap_addr(void **addr
, long_t sz
, int flags
) {
753 if (!IsAppMem((uptr
)*addr
) || !IsAppMem((uptr
)*addr
+ sz
- 1)) {
754 if (flags
& MAP_FIXED
) {
755 errno
= errno_EINVAL
;
765 template <class Mmap
>
766 static void *mmap_interceptor(ThreadState
*thr
, uptr pc
, Mmap real_mmap
,
767 void *addr
, SIZE_T sz
, int prot
, int flags
,
768 int fd
, OFF64_T off
) {
769 if (!fix_mmap_addr(&addr
, sz
, flags
)) return MAP_FAILED
;
770 void *res
= real_mmap(addr
, sz
, prot
, flags
, fd
, off
);
771 if (res
!= MAP_FAILED
) {
772 if (fd
> 0) FdAccess(thr
, pc
, fd
);
773 MemoryRangeImitateWriteOrResetRange(thr
, pc
, (uptr
)res
, sz
);
778 TSAN_INTERCEPTOR(int, munmap
, void *addr
, long_t sz
) {
779 SCOPED_TSAN_INTERCEPTOR(munmap
, addr
, sz
);
780 UnmapShadow(thr
, (uptr
)addr
, sz
);
781 int res
= REAL(munmap
)(addr
, sz
);
786 TSAN_INTERCEPTOR(void*, memalign
, uptr align
, uptr sz
) {
787 SCOPED_INTERCEPTOR_RAW(memalign
, align
, sz
);
788 return user_memalign(thr
, pc
, align
, sz
);
790 #define TSAN_MAYBE_INTERCEPT_MEMALIGN TSAN_INTERCEPT(memalign)
792 #define TSAN_MAYBE_INTERCEPT_MEMALIGN
796 TSAN_INTERCEPTOR(void*, aligned_alloc
, uptr align
, uptr sz
) {
798 return InternalAlloc(sz
, nullptr, align
);
799 SCOPED_INTERCEPTOR_RAW(aligned_alloc
, align
, sz
);
800 return user_aligned_alloc(thr
, pc
, align
, sz
);
803 TSAN_INTERCEPTOR(void*, valloc
, uptr sz
) {
805 return InternalAlloc(sz
, nullptr, GetPageSizeCached());
806 SCOPED_INTERCEPTOR_RAW(valloc
, sz
);
807 return user_valloc(thr
, pc
, sz
);
812 TSAN_INTERCEPTOR(void*, pvalloc
, uptr sz
) {
813 if (in_symbolizer()) {
814 uptr PageSize
= GetPageSizeCached();
815 sz
= sz
? RoundUpTo(sz
, PageSize
) : PageSize
;
816 return InternalAlloc(sz
, nullptr, PageSize
);
818 SCOPED_INTERCEPTOR_RAW(pvalloc
, sz
);
819 return user_pvalloc(thr
, pc
, sz
);
821 #define TSAN_MAYBE_INTERCEPT_PVALLOC TSAN_INTERCEPT(pvalloc)
823 #define TSAN_MAYBE_INTERCEPT_PVALLOC
827 TSAN_INTERCEPTOR(int, posix_memalign
, void **memptr
, uptr align
, uptr sz
) {
828 if (in_symbolizer()) {
829 void *p
= InternalAlloc(sz
, nullptr, align
);
835 SCOPED_INTERCEPTOR_RAW(posix_memalign
, memptr
, align
, sz
);
836 return user_posix_memalign(thr
, pc
, memptr
, align
, sz
);
840 // __cxa_guard_acquire and friends need to be intercepted in a special way -
841 // regular interceptors will break statically-linked libstdc++. Linux
842 // interceptors are especially defined as weak functions (so that they don't
843 // cause link errors when user defines them as well). So they silently
844 // auto-disable themselves when such symbol is already present in the binary. If
845 // we link libstdc++ statically, it will bring own __cxa_guard_acquire which
846 // will silently replace our interceptor. That's why on Linux we simply export
847 // these interceptors with INTERFACE_ATTRIBUTE.
848 // On OS X, we don't support statically linking, so we just use a regular
851 #define STDCXX_INTERCEPTOR TSAN_INTERCEPTOR
853 #define STDCXX_INTERCEPTOR(rettype, name, ...) \
854 extern "C" rettype INTERFACE_ATTRIBUTE name(__VA_ARGS__)
857 // Used in thread-safe function static initialization.
858 STDCXX_INTERCEPTOR(int, __cxa_guard_acquire
, atomic_uint32_t
*g
) {
859 SCOPED_INTERCEPTOR_RAW(__cxa_guard_acquire
, g
);
860 OnPotentiallyBlockingRegionBegin();
861 auto on_exit
= at_scope_exit(&OnPotentiallyBlockingRegionEnd
);
863 u32 cmp
= atomic_load(g
, memory_order_acquire
);
865 if (atomic_compare_exchange_strong(g
, &cmp
, 1<<16, memory_order_relaxed
))
867 } else if (cmp
== 1) {
868 Acquire(thr
, pc
, (uptr
)g
);
871 internal_sched_yield();
876 STDCXX_INTERCEPTOR(void, __cxa_guard_release
, atomic_uint32_t
*g
) {
877 SCOPED_INTERCEPTOR_RAW(__cxa_guard_release
, g
);
878 Release(thr
, pc
, (uptr
)g
);
879 atomic_store(g
, 1, memory_order_release
);
882 STDCXX_INTERCEPTOR(void, __cxa_guard_abort
, atomic_uint32_t
*g
) {
883 SCOPED_INTERCEPTOR_RAW(__cxa_guard_abort
, g
);
884 atomic_store(g
, 0, memory_order_relaxed
);
888 void DestroyThreadState() {
889 ThreadState
*thr
= cur_thread();
890 Processor
*proc
= thr
->proc();
892 ProcUnwire(proc
, thr
);
895 cur_thread_finalize();
898 void PlatformCleanUpThreadState(ThreadState
*thr
) {
899 ThreadSignalContext
*sctx
= thr
->signal_ctx
;
902 UnmapOrDie(sctx
, sizeof(*sctx
));
905 } // namespace __tsan
907 #if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
908 static void thread_finalize(void *v
) {
911 if (pthread_setspecific(interceptor_ctx()->finalize_key
,
912 (void*)(iter
- 1))) {
913 Printf("ThreadSanitizer: failed to set thread key\n");
918 DestroyThreadState();
924 void* (*callback
)(void *arg
);
926 atomic_uintptr_t tid
;
929 extern "C" void *__tsan_thread_start_func(void *arg
) {
930 ThreadParam
*p
= (ThreadParam
*)arg
;
931 void* (*callback
)(void *arg
) = p
->callback
;
932 void *param
= p
->param
;
936 ThreadState
*thr
= cur_thread();
937 // Thread-local state is not initialized yet.
938 ScopedIgnoreInterceptors ignore
;
939 #if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
940 ThreadIgnoreBegin(thr
, 0);
941 if (pthread_setspecific(interceptor_ctx()->finalize_key
,
942 (void *)GetPthreadDestructorIterations())) {
943 Printf("ThreadSanitizer: failed to set thread key\n");
946 ThreadIgnoreEnd(thr
, 0);
948 while ((tid
= atomic_load(&p
->tid
, memory_order_acquire
)) == 0)
949 internal_sched_yield();
950 Processor
*proc
= ProcCreate();
952 ThreadStart(thr
, tid
, GetTid(), ThreadType::Regular
);
953 atomic_store(&p
->tid
, 0, memory_order_release
);
955 void *res
= callback(param
);
956 // Prevent the callback from being tail called,
957 // it mixes up stack traces.
958 volatile int foo
= 42;
963 TSAN_INTERCEPTOR(int, pthread_create
,
964 void *th
, void *attr
, void *(*callback
)(void*), void * param
) {
965 SCOPED_INTERCEPTOR_RAW(pthread_create
, th
, attr
, callback
, param
);
967 MaybeSpawnBackgroundThread();
969 if (ctx
->after_multithreaded_fork
) {
970 if (flags()->die_after_fork
) {
971 Report("ThreadSanitizer: starting new threads after multi-threaded "
972 "fork is not supported. Dying (set die_after_fork=0 to override)\n");
975 VPrintf(1, "ThreadSanitizer: starting new threads after multi-threaded "
976 "fork is not supported (pid %d). Continuing because of "
977 "die_after_fork=0, but you are on your own\n", internal_getpid());
980 __sanitizer_pthread_attr_t myattr
;
982 pthread_attr_init(&myattr
);
986 REAL(pthread_attr_getdetachstate
)(attr
, &detached
);
987 AdjustStackSize(attr
);
990 p
.callback
= callback
;
992 atomic_store(&p
.tid
, 0, memory_order_relaxed
);
995 // Otherwise we see false positives in pthread stack manipulation.
996 ScopedIgnoreInterceptors ignore
;
997 ThreadIgnoreBegin(thr
, pc
);
998 res
= REAL(pthread_create
)(th
, attr
, __tsan_thread_start_func
, &p
);
999 ThreadIgnoreEnd(thr
, pc
);
1002 int tid
= ThreadCreate(thr
, pc
, *(uptr
*)th
, IsStateDetached(detached
));
1004 // Synchronization on p.tid serves two purposes:
1005 // 1. ThreadCreate must finish before the new thread starts.
1006 // Otherwise the new thread can call pthread_detach, but the pthread_t
1007 // identifier is not yet registered in ThreadRegistry by ThreadCreate.
1008 // 2. ThreadStart must finish before this thread continues.
1009 // Otherwise, this thread can call pthread_detach and reset thr->sync
1010 // before the new thread got a chance to acquire from it in ThreadStart.
1011 atomic_store(&p
.tid
, tid
, memory_order_release
);
1012 while (atomic_load(&p
.tid
, memory_order_acquire
) != 0)
1013 internal_sched_yield();
1015 if (attr
== &myattr
)
1016 pthread_attr_destroy(&myattr
);
1020 TSAN_INTERCEPTOR(int, pthread_join
, void *th
, void **ret
) {
1021 SCOPED_INTERCEPTOR_RAW(pthread_join
, th
, ret
);
1022 int tid
= ThreadConsumeTid(thr
, pc
, (uptr
)th
);
1023 ThreadIgnoreBegin(thr
, pc
);
1024 int res
= BLOCK_REAL(pthread_join
)(th
, ret
);
1025 ThreadIgnoreEnd(thr
, pc
);
1027 ThreadJoin(thr
, pc
, tid
);
1032 DEFINE_REAL_PTHREAD_FUNCTIONS
1034 TSAN_INTERCEPTOR(int, pthread_detach
, void *th
) {
1035 SCOPED_INTERCEPTOR_RAW(pthread_detach
, th
);
1036 int tid
= ThreadConsumeTid(thr
, pc
, (uptr
)th
);
1037 int res
= REAL(pthread_detach
)(th
);
1039 ThreadDetach(thr
, pc
, tid
);
1044 TSAN_INTERCEPTOR(void, pthread_exit
, void *retval
) {
1046 SCOPED_INTERCEPTOR_RAW(pthread_exit
, retval
);
1047 #if !SANITIZER_MAC && !SANITIZER_ANDROID
1048 CHECK_EQ(thr
, &cur_thread_placeholder
);
1051 REAL(pthread_exit
)(retval
);
1055 TSAN_INTERCEPTOR(int, pthread_tryjoin_np
, void *th
, void **ret
) {
1056 SCOPED_INTERCEPTOR_RAW(pthread_tryjoin_np
, th
, ret
);
1057 int tid
= ThreadConsumeTid(thr
, pc
, (uptr
)th
);
1058 ThreadIgnoreBegin(thr
, pc
);
1059 int res
= REAL(pthread_tryjoin_np
)(th
, ret
);
1060 ThreadIgnoreEnd(thr
, pc
);
1062 ThreadJoin(thr
, pc
, tid
);
1064 ThreadNotJoined(thr
, pc
, tid
, (uptr
)th
);
1068 TSAN_INTERCEPTOR(int, pthread_timedjoin_np
, void *th
, void **ret
,
1069 const struct timespec
*abstime
) {
1070 SCOPED_INTERCEPTOR_RAW(pthread_timedjoin_np
, th
, ret
, abstime
);
1071 int tid
= ThreadConsumeTid(thr
, pc
, (uptr
)th
);
1072 ThreadIgnoreBegin(thr
, pc
);
1073 int res
= BLOCK_REAL(pthread_timedjoin_np
)(th
, ret
, abstime
);
1074 ThreadIgnoreEnd(thr
, pc
);
1076 ThreadJoin(thr
, pc
, tid
);
1078 ThreadNotJoined(thr
, pc
, tid
, (uptr
)th
);
1084 // NPTL implementation of pthread_cond has 2 versions (2.2.5 and 2.3.2).
1085 // pthread_cond_t has different size in the different versions.
1086 // If call new REAL functions for old pthread_cond_t, they will corrupt memory
1087 // after pthread_cond_t (old cond is smaller).
1088 // If we call old REAL functions for new pthread_cond_t, we will lose some
1089 // functionality (e.g. old functions do not support waiting against
1091 // Proper handling would require to have 2 versions of interceptors as well.
1092 // But this is messy, in particular requires linker scripts when sanitizer
1093 // runtime is linked into a shared library.
1094 // Instead we assume we don't have dynamic libraries built against old
1095 // pthread (2.2.5 is dated by 2002). And provide legacy_pthread_cond flag
1096 // that allows to work with old libraries (but this mode does not support
1097 // some features, e.g. pthread_condattr_getpshared).
1098 static void *init_cond(void *c
, bool force
= false) {
1099 // sizeof(pthread_cond_t) >= sizeof(uptr) in both versions.
1100 // So we allocate additional memory on the side large enough to hold
1101 // any pthread_cond_t object. Always call new REAL functions, but pass
1102 // the aux object to them.
1103 // Note: the code assumes that PTHREAD_COND_INITIALIZER initializes
1104 // first word of pthread_cond_t to zero.
1105 // It's all relevant only for linux.
1106 if (!common_flags()->legacy_pthread_cond
)
1108 atomic_uintptr_t
*p
= (atomic_uintptr_t
*)c
;
1109 uptr cond
= atomic_load(p
, memory_order_acquire
);
1110 if (!force
&& cond
!= 0)
1112 void *newcond
= WRAP(malloc
)(pthread_cond_t_sz
);
1113 internal_memset(newcond
, 0, pthread_cond_t_sz
);
1114 if (atomic_compare_exchange_strong(p
, &cond
, (uptr
)newcond
,
1115 memory_order_acq_rel
))
1117 WRAP(free
)(newcond
);
1121 struct CondMutexUnlockCtx
{
1122 ScopedInterceptor
*si
;
1128 static void cond_mutex_unlock(CondMutexUnlockCtx
*arg
) {
1129 // pthread_cond_wait interceptor has enabled async signal delivery
1130 // (see BlockingCall below). Disable async signals since we are running
1131 // tsan code. Also ScopedInterceptor and BlockingCall destructors won't run
1132 // since the thread is cancelled, so we have to manually execute them
1133 // (the thread still can run some user code due to pthread_cleanup_push).
1134 ThreadSignalContext
*ctx
= SigCtx(arg
->thr
);
1135 CHECK_EQ(atomic_load(&ctx
->in_blocking_func
, memory_order_relaxed
), 1);
1136 atomic_store(&ctx
->in_blocking_func
, 0, memory_order_relaxed
);
1137 MutexPostLock(arg
->thr
, arg
->pc
, (uptr
)arg
->m
, MutexFlagDoPreLockOnPostLock
);
1138 // Undo BlockingCall ctor effects.
1139 arg
->thr
->ignore_interceptors
--;
1140 arg
->si
->~ScopedInterceptor();
1143 INTERCEPTOR(int, pthread_cond_init
, void *c
, void *a
) {
1144 void *cond
= init_cond(c
, true);
1145 SCOPED_TSAN_INTERCEPTOR(pthread_cond_init
, cond
, a
);
1146 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), true);
1147 return REAL(pthread_cond_init
)(cond
, a
);
1150 static int cond_wait(ThreadState
*thr
, uptr pc
, ScopedInterceptor
*si
,
1151 int (*fn
)(void *c
, void *m
, void *abstime
), void *c
,
1153 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), false);
1154 MutexUnlock(thr
, pc
, (uptr
)m
);
1155 CondMutexUnlockCtx arg
= {si
, thr
, pc
, m
};
1157 // This ensures that we handle mutex lock even in case of pthread_cancel.
1158 // See test/tsan/cond_cancel.cpp.
1160 // Enable signal delivery while the thread is blocked.
1161 BlockingCall
bc(thr
);
1162 res
= call_pthread_cancel_with_cleanup(
1163 fn
, c
, m
, t
, (void (*)(void *arg
))cond_mutex_unlock
, &arg
);
1165 if (res
== errno_EOWNERDEAD
) MutexRepair(thr
, pc
, (uptr
)m
);
1166 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagDoPreLockOnPostLock
);
1170 INTERCEPTOR(int, pthread_cond_wait
, void *c
, void *m
) {
1171 void *cond
= init_cond(c
);
1172 SCOPED_TSAN_INTERCEPTOR(pthread_cond_wait
, cond
, m
);
1173 return cond_wait(thr
, pc
, &si
, (int (*)(void *c
, void *m
, void *abstime
))REAL(
1178 INTERCEPTOR(int, pthread_cond_timedwait
, void *c
, void *m
, void *abstime
) {
1179 void *cond
= init_cond(c
);
1180 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait
, cond
, m
, abstime
);
1181 return cond_wait(thr
, pc
, &si
, REAL(pthread_cond_timedwait
), cond
, m
,
1186 INTERCEPTOR(int, pthread_cond_timedwait_relative_np
, void *c
, void *m
,
1188 void *cond
= init_cond(c
);
1189 SCOPED_TSAN_INTERCEPTOR(pthread_cond_timedwait_relative_np
, cond
, m
, reltime
);
1190 return cond_wait(thr
, pc
, &si
, REAL(pthread_cond_timedwait_relative_np
), cond
,
1195 INTERCEPTOR(int, pthread_cond_signal
, void *c
) {
1196 void *cond
= init_cond(c
);
1197 SCOPED_TSAN_INTERCEPTOR(pthread_cond_signal
, cond
);
1198 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), false);
1199 return REAL(pthread_cond_signal
)(cond
);
1202 INTERCEPTOR(int, pthread_cond_broadcast
, void *c
) {
1203 void *cond
= init_cond(c
);
1204 SCOPED_TSAN_INTERCEPTOR(pthread_cond_broadcast
, cond
);
1205 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), false);
1206 return REAL(pthread_cond_broadcast
)(cond
);
1209 INTERCEPTOR(int, pthread_cond_destroy
, void *c
) {
1210 void *cond
= init_cond(c
);
1211 SCOPED_TSAN_INTERCEPTOR(pthread_cond_destroy
, cond
);
1212 MemoryAccessRange(thr
, pc
, (uptr
)c
, sizeof(uptr
), true);
1213 int res
= REAL(pthread_cond_destroy
)(cond
);
1214 if (common_flags()->legacy_pthread_cond
) {
1215 // Free our aux cond and zero the pointer to not leave dangling pointers.
1217 atomic_store((atomic_uintptr_t
*)c
, 0, memory_order_relaxed
);
1222 TSAN_INTERCEPTOR(int, pthread_mutex_init
, void *m
, void *a
) {
1223 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_init
, m
, a
);
1224 int res
= REAL(pthread_mutex_init
)(m
, a
);
1229 if (REAL(pthread_mutexattr_gettype
)(a
, &type
) == 0)
1230 if (type
== PTHREAD_MUTEX_RECURSIVE
||
1231 type
== PTHREAD_MUTEX_RECURSIVE_NP
)
1232 flagz
|= MutexFlagWriteReentrant
;
1234 MutexCreate(thr
, pc
, (uptr
)m
, flagz
);
1239 TSAN_INTERCEPTOR(int, pthread_mutex_destroy
, void *m
) {
1240 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_destroy
, m
);
1241 int res
= REAL(pthread_mutex_destroy
)(m
);
1242 if (res
== 0 || res
== errno_EBUSY
) {
1243 MutexDestroy(thr
, pc
, (uptr
)m
);
1248 TSAN_INTERCEPTOR(int, pthread_mutex_trylock
, void *m
) {
1249 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_trylock
, m
);
1250 int res
= REAL(pthread_mutex_trylock
)(m
);
1251 if (res
== errno_EOWNERDEAD
)
1252 MutexRepair(thr
, pc
, (uptr
)m
);
1253 if (res
== 0 || res
== errno_EOWNERDEAD
)
1254 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagTryLock
);
1259 TSAN_INTERCEPTOR(int, pthread_mutex_timedlock
, void *m
, void *abstime
) {
1260 SCOPED_TSAN_INTERCEPTOR(pthread_mutex_timedlock
, m
, abstime
);
1261 int res
= REAL(pthread_mutex_timedlock
)(m
, abstime
);
1263 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagTryLock
);
1270 TSAN_INTERCEPTOR(int, pthread_spin_init
, void *m
, int pshared
) {
1271 SCOPED_TSAN_INTERCEPTOR(pthread_spin_init
, m
, pshared
);
1272 int res
= REAL(pthread_spin_init
)(m
, pshared
);
1274 MutexCreate(thr
, pc
, (uptr
)m
);
1279 TSAN_INTERCEPTOR(int, pthread_spin_destroy
, void *m
) {
1280 SCOPED_TSAN_INTERCEPTOR(pthread_spin_destroy
, m
);
1281 int res
= REAL(pthread_spin_destroy
)(m
);
1283 MutexDestroy(thr
, pc
, (uptr
)m
);
1288 TSAN_INTERCEPTOR(int, pthread_spin_lock
, void *m
) {
1289 SCOPED_TSAN_INTERCEPTOR(pthread_spin_lock
, m
);
1290 MutexPreLock(thr
, pc
, (uptr
)m
);
1291 int res
= REAL(pthread_spin_lock
)(m
);
1293 MutexPostLock(thr
, pc
, (uptr
)m
);
1298 TSAN_INTERCEPTOR(int, pthread_spin_trylock
, void *m
) {
1299 SCOPED_TSAN_INTERCEPTOR(pthread_spin_trylock
, m
);
1300 int res
= REAL(pthread_spin_trylock
)(m
);
1302 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagTryLock
);
1307 TSAN_INTERCEPTOR(int, pthread_spin_unlock
, void *m
) {
1308 SCOPED_TSAN_INTERCEPTOR(pthread_spin_unlock
, m
);
1309 MutexUnlock(thr
, pc
, (uptr
)m
);
1310 int res
= REAL(pthread_spin_unlock
)(m
);
1315 TSAN_INTERCEPTOR(int, pthread_rwlock_init
, void *m
, void *a
) {
1316 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_init
, m
, a
);
1317 int res
= REAL(pthread_rwlock_init
)(m
, a
);
1319 MutexCreate(thr
, pc
, (uptr
)m
);
1324 TSAN_INTERCEPTOR(int, pthread_rwlock_destroy
, void *m
) {
1325 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_destroy
, m
);
1326 int res
= REAL(pthread_rwlock_destroy
)(m
);
1328 MutexDestroy(thr
, pc
, (uptr
)m
);
1333 TSAN_INTERCEPTOR(int, pthread_rwlock_rdlock
, void *m
) {
1334 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_rdlock
, m
);
1335 MutexPreReadLock(thr
, pc
, (uptr
)m
);
1336 int res
= REAL(pthread_rwlock_rdlock
)(m
);
1338 MutexPostReadLock(thr
, pc
, (uptr
)m
);
1343 TSAN_INTERCEPTOR(int, pthread_rwlock_tryrdlock
, void *m
) {
1344 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_tryrdlock
, m
);
1345 int res
= REAL(pthread_rwlock_tryrdlock
)(m
);
1347 MutexPostReadLock(thr
, pc
, (uptr
)m
, MutexFlagTryLock
);
1353 TSAN_INTERCEPTOR(int, pthread_rwlock_timedrdlock
, void *m
, void *abstime
) {
1354 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedrdlock
, m
, abstime
);
1355 int res
= REAL(pthread_rwlock_timedrdlock
)(m
, abstime
);
1357 MutexPostReadLock(thr
, pc
, (uptr
)m
);
1363 TSAN_INTERCEPTOR(int, pthread_rwlock_wrlock
, void *m
) {
1364 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_wrlock
, m
);
1365 MutexPreLock(thr
, pc
, (uptr
)m
);
1366 int res
= REAL(pthread_rwlock_wrlock
)(m
);
1368 MutexPostLock(thr
, pc
, (uptr
)m
);
1373 TSAN_INTERCEPTOR(int, pthread_rwlock_trywrlock
, void *m
) {
1374 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_trywrlock
, m
);
1375 int res
= REAL(pthread_rwlock_trywrlock
)(m
);
1377 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagTryLock
);
1383 TSAN_INTERCEPTOR(int, pthread_rwlock_timedwrlock
, void *m
, void *abstime
) {
1384 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_timedwrlock
, m
, abstime
);
1385 int res
= REAL(pthread_rwlock_timedwrlock
)(m
, abstime
);
1387 MutexPostLock(thr
, pc
, (uptr
)m
, MutexFlagTryLock
);
1393 TSAN_INTERCEPTOR(int, pthread_rwlock_unlock
, void *m
) {
1394 SCOPED_TSAN_INTERCEPTOR(pthread_rwlock_unlock
, m
);
1395 MutexReadOrWriteUnlock(thr
, pc
, (uptr
)m
);
1396 int res
= REAL(pthread_rwlock_unlock
)(m
);
1401 TSAN_INTERCEPTOR(int, pthread_barrier_init
, void *b
, void *a
, unsigned count
) {
1402 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_init
, b
, a
, count
);
1403 MemoryWrite(thr
, pc
, (uptr
)b
, kSizeLog1
);
1404 int res
= REAL(pthread_barrier_init
)(b
, a
, count
);
1408 TSAN_INTERCEPTOR(int, pthread_barrier_destroy
, void *b
) {
1409 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_destroy
, b
);
1410 MemoryWrite(thr
, pc
, (uptr
)b
, kSizeLog1
);
1411 int res
= REAL(pthread_barrier_destroy
)(b
);
1415 TSAN_INTERCEPTOR(int, pthread_barrier_wait
, void *b
) {
1416 SCOPED_TSAN_INTERCEPTOR(pthread_barrier_wait
, b
);
1417 Release(thr
, pc
, (uptr
)b
);
1418 MemoryRead(thr
, pc
, (uptr
)b
, kSizeLog1
);
1419 int res
= REAL(pthread_barrier_wait
)(b
);
1420 MemoryRead(thr
, pc
, (uptr
)b
, kSizeLog1
);
1421 if (res
== 0 || res
== PTHREAD_BARRIER_SERIAL_THREAD
) {
1422 Acquire(thr
, pc
, (uptr
)b
);
1428 TSAN_INTERCEPTOR(int, pthread_once
, void *o
, void (*f
)()) {
1429 SCOPED_INTERCEPTOR_RAW(pthread_once
, o
, f
);
1430 if (o
== 0 || f
== 0)
1431 return errno_EINVAL
;
1435 a
= static_cast<atomic_uint32_t
*>((void *)((char *)o
+ sizeof(long_t
)));
1436 else if (SANITIZER_NETBSD
)
1437 a
= static_cast<atomic_uint32_t
*>
1438 ((void *)((char *)o
+ __sanitizer::pthread_mutex_t_sz
));
1440 a
= static_cast<atomic_uint32_t
*>(o
);
1442 u32 v
= atomic_load(a
, memory_order_acquire
);
1443 if (v
== 0 && atomic_compare_exchange_strong(a
, &v
, 1,
1444 memory_order_relaxed
)) {
1446 if (!thr
->in_ignored_lib
)
1447 Release(thr
, pc
, (uptr
)o
);
1448 atomic_store(a
, 2, memory_order_release
);
1451 internal_sched_yield();
1452 v
= atomic_load(a
, memory_order_acquire
);
1454 if (!thr
->in_ignored_lib
)
1455 Acquire(thr
, pc
, (uptr
)o
);
1460 #if SANITIZER_LINUX && !SANITIZER_ANDROID
1461 TSAN_INTERCEPTOR(int, __fxstat
, int version
, int fd
, void *buf
) {
1462 SCOPED_TSAN_INTERCEPTOR(__fxstat
, version
, fd
, buf
);
1464 FdAccess(thr
, pc
, fd
);
1465 return REAL(__fxstat
)(version
, fd
, buf
);
1467 #define TSAN_MAYBE_INTERCEPT___FXSTAT TSAN_INTERCEPT(__fxstat)
1469 #define TSAN_MAYBE_INTERCEPT___FXSTAT
1472 TSAN_INTERCEPTOR(int, fstat
, int fd
, void *buf
) {
1473 #if SANITIZER_FREEBSD || SANITIZER_MAC || SANITIZER_ANDROID || SANITIZER_NETBSD
1474 SCOPED_TSAN_INTERCEPTOR(fstat
, fd
, buf
);
1476 FdAccess(thr
, pc
, fd
);
1477 return REAL(fstat
)(fd
, buf
);
1479 SCOPED_TSAN_INTERCEPTOR(__fxstat
, 0, fd
, buf
);
1481 FdAccess(thr
, pc
, fd
);
1482 return REAL(__fxstat
)(0, fd
, buf
);
1486 #if SANITIZER_LINUX && !SANITIZER_ANDROID
1487 TSAN_INTERCEPTOR(int, __fxstat64
, int version
, int fd
, void *buf
) {
1488 SCOPED_TSAN_INTERCEPTOR(__fxstat64
, version
, fd
, buf
);
1490 FdAccess(thr
, pc
, fd
);
1491 return REAL(__fxstat64
)(version
, fd
, buf
);
1493 #define TSAN_MAYBE_INTERCEPT___FXSTAT64 TSAN_INTERCEPT(__fxstat64)
1495 #define TSAN_MAYBE_INTERCEPT___FXSTAT64
1498 #if SANITIZER_LINUX && !SANITIZER_ANDROID
1499 TSAN_INTERCEPTOR(int, fstat64
, int fd
, void *buf
) {
1500 SCOPED_TSAN_INTERCEPTOR(__fxstat64
, 0, fd
, buf
);
1502 FdAccess(thr
, pc
, fd
);
1503 return REAL(__fxstat64
)(0, fd
, buf
);
1505 #define TSAN_MAYBE_INTERCEPT_FSTAT64 TSAN_INTERCEPT(fstat64)
1507 #define TSAN_MAYBE_INTERCEPT_FSTAT64
1510 TSAN_INTERCEPTOR(int, open
, const char *name
, int flags
, int mode
) {
1511 SCOPED_TSAN_INTERCEPTOR(open
, name
, flags
, mode
);
1512 READ_STRING(thr
, pc
, name
, 0);
1513 int fd
= REAL(open
)(name
, flags
, mode
);
1515 FdFileCreate(thr
, pc
, fd
);
1520 TSAN_INTERCEPTOR(int, open64
, const char *name
, int flags
, int mode
) {
1521 SCOPED_TSAN_INTERCEPTOR(open64
, name
, flags
, mode
);
1522 READ_STRING(thr
, pc
, name
, 0);
1523 int fd
= REAL(open64
)(name
, flags
, mode
);
1525 FdFileCreate(thr
, pc
, fd
);
1528 #define TSAN_MAYBE_INTERCEPT_OPEN64 TSAN_INTERCEPT(open64)
1530 #define TSAN_MAYBE_INTERCEPT_OPEN64
1533 TSAN_INTERCEPTOR(int, creat
, const char *name
, int mode
) {
1534 SCOPED_TSAN_INTERCEPTOR(creat
, name
, mode
);
1535 READ_STRING(thr
, pc
, name
, 0);
1536 int fd
= REAL(creat
)(name
, mode
);
1538 FdFileCreate(thr
, pc
, fd
);
1543 TSAN_INTERCEPTOR(int, creat64
, const char *name
, int mode
) {
1544 SCOPED_TSAN_INTERCEPTOR(creat64
, name
, mode
);
1545 READ_STRING(thr
, pc
, name
, 0);
1546 int fd
= REAL(creat64
)(name
, mode
);
1548 FdFileCreate(thr
, pc
, fd
);
1551 #define TSAN_MAYBE_INTERCEPT_CREAT64 TSAN_INTERCEPT(creat64)
1553 #define TSAN_MAYBE_INTERCEPT_CREAT64
1556 TSAN_INTERCEPTOR(int, dup
, int oldfd
) {
1557 SCOPED_TSAN_INTERCEPTOR(dup
, oldfd
);
1558 int newfd
= REAL(dup
)(oldfd
);
1559 if (oldfd
>= 0 && newfd
>= 0 && newfd
!= oldfd
)
1560 FdDup(thr
, pc
, oldfd
, newfd
, true);
1564 TSAN_INTERCEPTOR(int, dup2
, int oldfd
, int newfd
) {
1565 SCOPED_TSAN_INTERCEPTOR(dup2
, oldfd
, newfd
);
1566 int newfd2
= REAL(dup2
)(oldfd
, newfd
);
1567 if (oldfd
>= 0 && newfd2
>= 0 && newfd2
!= oldfd
)
1568 FdDup(thr
, pc
, oldfd
, newfd2
, false);
1573 TSAN_INTERCEPTOR(int, dup3
, int oldfd
, int newfd
, int flags
) {
1574 SCOPED_TSAN_INTERCEPTOR(dup3
, oldfd
, newfd
, flags
);
1575 int newfd2
= REAL(dup3
)(oldfd
, newfd
, flags
);
1576 if (oldfd
>= 0 && newfd2
>= 0 && newfd2
!= oldfd
)
1577 FdDup(thr
, pc
, oldfd
, newfd2
, false);
1583 TSAN_INTERCEPTOR(int, eventfd
, unsigned initval
, int flags
) {
1584 SCOPED_TSAN_INTERCEPTOR(eventfd
, initval
, flags
);
1585 int fd
= REAL(eventfd
)(initval
, flags
);
1587 FdEventCreate(thr
, pc
, fd
);
1590 #define TSAN_MAYBE_INTERCEPT_EVENTFD TSAN_INTERCEPT(eventfd)
1592 #define TSAN_MAYBE_INTERCEPT_EVENTFD
1596 TSAN_INTERCEPTOR(int, signalfd
, int fd
, void *mask
, int flags
) {
1597 SCOPED_TSAN_INTERCEPTOR(signalfd
, fd
, mask
, flags
);
1599 FdClose(thr
, pc
, fd
);
1600 fd
= REAL(signalfd
)(fd
, mask
, flags
);
1602 FdSignalCreate(thr
, pc
, fd
);
1605 #define TSAN_MAYBE_INTERCEPT_SIGNALFD TSAN_INTERCEPT(signalfd)
1607 #define TSAN_MAYBE_INTERCEPT_SIGNALFD
1611 TSAN_INTERCEPTOR(int, inotify_init
, int fake
) {
1612 SCOPED_TSAN_INTERCEPTOR(inotify_init
, fake
);
1613 int fd
= REAL(inotify_init
)(fake
);
1615 FdInotifyCreate(thr
, pc
, fd
);
1618 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT TSAN_INTERCEPT(inotify_init)
1620 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT
1624 TSAN_INTERCEPTOR(int, inotify_init1
, int flags
) {
1625 SCOPED_TSAN_INTERCEPTOR(inotify_init1
, flags
);
1626 int fd
= REAL(inotify_init1
)(flags
);
1628 FdInotifyCreate(thr
, pc
, fd
);
1631 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1 TSAN_INTERCEPT(inotify_init1)
1633 #define TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1
1636 TSAN_INTERCEPTOR(int, socket
, int domain
, int type
, int protocol
) {
1637 SCOPED_TSAN_INTERCEPTOR(socket
, domain
, type
, protocol
);
1638 int fd
= REAL(socket
)(domain
, type
, protocol
);
1640 FdSocketCreate(thr
, pc
, fd
);
1644 TSAN_INTERCEPTOR(int, socketpair
, int domain
, int type
, int protocol
, int *fd
) {
1645 SCOPED_TSAN_INTERCEPTOR(socketpair
, domain
, type
, protocol
, fd
);
1646 int res
= REAL(socketpair
)(domain
, type
, protocol
, fd
);
1647 if (res
== 0 && fd
[0] >= 0 && fd
[1] >= 0)
1648 FdPipeCreate(thr
, pc
, fd
[0], fd
[1]);
1652 TSAN_INTERCEPTOR(int, connect
, int fd
, void *addr
, unsigned addrlen
) {
1653 SCOPED_TSAN_INTERCEPTOR(connect
, fd
, addr
, addrlen
);
1654 FdSocketConnecting(thr
, pc
, fd
);
1655 int res
= REAL(connect
)(fd
, addr
, addrlen
);
1656 if (res
== 0 && fd
>= 0)
1657 FdSocketConnect(thr
, pc
, fd
);
1661 TSAN_INTERCEPTOR(int, bind
, int fd
, void *addr
, unsigned addrlen
) {
1662 SCOPED_TSAN_INTERCEPTOR(bind
, fd
, addr
, addrlen
);
1663 int res
= REAL(bind
)(fd
, addr
, addrlen
);
1664 if (fd
> 0 && res
== 0)
1665 FdAccess(thr
, pc
, fd
);
1669 TSAN_INTERCEPTOR(int, listen
, int fd
, int backlog
) {
1670 SCOPED_TSAN_INTERCEPTOR(listen
, fd
, backlog
);
1671 int res
= REAL(listen
)(fd
, backlog
);
1672 if (fd
> 0 && res
== 0)
1673 FdAccess(thr
, pc
, fd
);
1677 TSAN_INTERCEPTOR(int, close
, int fd
) {
1678 SCOPED_TSAN_INTERCEPTOR(close
, fd
);
1680 FdClose(thr
, pc
, fd
);
1681 return REAL(close
)(fd
);
1685 TSAN_INTERCEPTOR(int, __close
, int fd
) {
1686 SCOPED_TSAN_INTERCEPTOR(__close
, fd
);
1688 FdClose(thr
, pc
, fd
);
1689 return REAL(__close
)(fd
);
1691 #define TSAN_MAYBE_INTERCEPT___CLOSE TSAN_INTERCEPT(__close)
1693 #define TSAN_MAYBE_INTERCEPT___CLOSE
1697 #if SANITIZER_LINUX && !SANITIZER_ANDROID
1698 TSAN_INTERCEPTOR(void, __res_iclose
, void *state
, bool free_addr
) {
1699 SCOPED_TSAN_INTERCEPTOR(__res_iclose
, state
, free_addr
);
1701 int cnt
= ExtractResolvFDs(state
, fds
, ARRAY_SIZE(fds
));
1702 for (int i
= 0; i
< cnt
; i
++) {
1704 FdClose(thr
, pc
, fds
[i
]);
1706 REAL(__res_iclose
)(state
, free_addr
);
1708 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE TSAN_INTERCEPT(__res_iclose)
1710 #define TSAN_MAYBE_INTERCEPT___RES_ICLOSE
1713 TSAN_INTERCEPTOR(int, pipe
, int *pipefd
) {
1714 SCOPED_TSAN_INTERCEPTOR(pipe
, pipefd
);
1715 int res
= REAL(pipe
)(pipefd
);
1716 if (res
== 0 && pipefd
[0] >= 0 && pipefd
[1] >= 0)
1717 FdPipeCreate(thr
, pc
, pipefd
[0], pipefd
[1]);
1722 TSAN_INTERCEPTOR(int, pipe2
, int *pipefd
, int flags
) {
1723 SCOPED_TSAN_INTERCEPTOR(pipe2
, pipefd
, flags
);
1724 int res
= REAL(pipe2
)(pipefd
, flags
);
1725 if (res
== 0 && pipefd
[0] >= 0 && pipefd
[1] >= 0)
1726 FdPipeCreate(thr
, pc
, pipefd
[0], pipefd
[1]);
1731 TSAN_INTERCEPTOR(int, unlink
, char *path
) {
1732 SCOPED_TSAN_INTERCEPTOR(unlink
, path
);
1733 Release(thr
, pc
, File2addr(path
));
1734 int res
= REAL(unlink
)(path
);
1738 TSAN_INTERCEPTOR(void*, tmpfile
, int fake
) {
1739 SCOPED_TSAN_INTERCEPTOR(tmpfile
, fake
);
1740 void *res
= REAL(tmpfile
)(fake
);
1742 int fd
= fileno_unlocked(res
);
1744 FdFileCreate(thr
, pc
, fd
);
1750 TSAN_INTERCEPTOR(void*, tmpfile64
, int fake
) {
1751 SCOPED_TSAN_INTERCEPTOR(tmpfile64
, fake
);
1752 void *res
= REAL(tmpfile64
)(fake
);
1754 int fd
= fileno_unlocked(res
);
1756 FdFileCreate(thr
, pc
, fd
);
1760 #define TSAN_MAYBE_INTERCEPT_TMPFILE64 TSAN_INTERCEPT(tmpfile64)
1762 #define TSAN_MAYBE_INTERCEPT_TMPFILE64
1765 static void FlushStreams() {
1766 // Flushing all the streams here may freeze the process if a child thread is
1767 // performing file stream operations at the same time.
1768 REAL(fflush
)(stdout
);
1769 REAL(fflush
)(stderr
);
1772 TSAN_INTERCEPTOR(void, abort
, int fake
) {
1773 SCOPED_TSAN_INTERCEPTOR(abort
, fake
);
1778 TSAN_INTERCEPTOR(int, rmdir
, char *path
) {
1779 SCOPED_TSAN_INTERCEPTOR(rmdir
, path
);
1780 Release(thr
, pc
, Dir2addr(path
));
1781 int res
= REAL(rmdir
)(path
);
1785 TSAN_INTERCEPTOR(int, closedir
, void *dirp
) {
1786 SCOPED_TSAN_INTERCEPTOR(closedir
, dirp
);
1788 int fd
= dirfd(dirp
);
1789 FdClose(thr
, pc
, fd
);
1791 return REAL(closedir
)(dirp
);
1795 TSAN_INTERCEPTOR(int, epoll_create
, int size
) {
1796 SCOPED_TSAN_INTERCEPTOR(epoll_create
, size
);
1797 int fd
= REAL(epoll_create
)(size
);
1799 FdPollCreate(thr
, pc
, fd
);
1803 TSAN_INTERCEPTOR(int, epoll_create1
, int flags
) {
1804 SCOPED_TSAN_INTERCEPTOR(epoll_create1
, flags
);
1805 int fd
= REAL(epoll_create1
)(flags
);
1807 FdPollCreate(thr
, pc
, fd
);
1811 TSAN_INTERCEPTOR(int, epoll_ctl
, int epfd
, int op
, int fd
, void *ev
) {
1812 SCOPED_TSAN_INTERCEPTOR(epoll_ctl
, epfd
, op
, fd
, ev
);
1814 FdAccess(thr
, pc
, epfd
);
1815 if (epfd
>= 0 && fd
>= 0)
1816 FdAccess(thr
, pc
, fd
);
1817 if (op
== EPOLL_CTL_ADD
&& epfd
>= 0)
1818 FdRelease(thr
, pc
, epfd
);
1819 int res
= REAL(epoll_ctl
)(epfd
, op
, fd
, ev
);
1823 TSAN_INTERCEPTOR(int, epoll_wait
, int epfd
, void *ev
, int cnt
, int timeout
) {
1824 SCOPED_TSAN_INTERCEPTOR(epoll_wait
, epfd
, ev
, cnt
, timeout
);
1826 FdAccess(thr
, pc
, epfd
);
1827 int res
= BLOCK_REAL(epoll_wait
)(epfd
, ev
, cnt
, timeout
);
1828 if (res
> 0 && epfd
>= 0)
1829 FdAcquire(thr
, pc
, epfd
);
1833 TSAN_INTERCEPTOR(int, epoll_pwait
, int epfd
, void *ev
, int cnt
, int timeout
,
1835 SCOPED_TSAN_INTERCEPTOR(epoll_pwait
, epfd
, ev
, cnt
, timeout
, sigmask
);
1837 FdAccess(thr
, pc
, epfd
);
1838 int res
= BLOCK_REAL(epoll_pwait
)(epfd
, ev
, cnt
, timeout
, sigmask
);
1839 if (res
> 0 && epfd
>= 0)
1840 FdAcquire(thr
, pc
, epfd
);
1844 #define TSAN_MAYBE_INTERCEPT_EPOLL \
1845 TSAN_INTERCEPT(epoll_create); \
1846 TSAN_INTERCEPT(epoll_create1); \
1847 TSAN_INTERCEPT(epoll_ctl); \
1848 TSAN_INTERCEPT(epoll_wait); \
1849 TSAN_INTERCEPT(epoll_pwait)
1851 #define TSAN_MAYBE_INTERCEPT_EPOLL
1854 // The following functions are intercepted merely to process pending signals.
1855 // If program blocks signal X, we must deliver the signal before the function
1856 // returns. Similarly, if program unblocks a signal (or returns from sigsuspend)
1857 // it's better to deliver the signal straight away.
1858 TSAN_INTERCEPTOR(int, sigsuspend
, const __sanitizer_sigset_t
*mask
) {
1859 SCOPED_TSAN_INTERCEPTOR(sigsuspend
, mask
);
1860 return REAL(sigsuspend
)(mask
);
1863 TSAN_INTERCEPTOR(int, sigblock
, int mask
) {
1864 SCOPED_TSAN_INTERCEPTOR(sigblock
, mask
);
1865 return REAL(sigblock
)(mask
);
1868 TSAN_INTERCEPTOR(int, sigsetmask
, int mask
) {
1869 SCOPED_TSAN_INTERCEPTOR(sigsetmask
, mask
);
1870 return REAL(sigsetmask
)(mask
);
1873 TSAN_INTERCEPTOR(int, pthread_sigmask
, int how
, const __sanitizer_sigset_t
*set
,
1874 __sanitizer_sigset_t
*oldset
) {
1875 SCOPED_TSAN_INTERCEPTOR(pthread_sigmask
, how
, set
, oldset
);
1876 return REAL(pthread_sigmask
)(how
, set
, oldset
);
1881 static void CallUserSignalHandler(ThreadState
*thr
, bool sync
, bool acquire
,
1882 bool sigact
, int sig
,
1883 __sanitizer_siginfo
*info
, void *uctx
) {
1884 __sanitizer_sigaction
*sigactions
= interceptor_ctx()->sigactions
;
1886 Acquire(thr
, 0, (uptr
)&sigactions
[sig
]);
1887 // Signals are generally asynchronous, so if we receive a signals when
1888 // ignores are enabled we should disable ignores. This is critical for sync
1889 // and interceptors, because otherwise we can miss syncronization and report
1891 int ignore_reads_and_writes
= thr
->ignore_reads_and_writes
;
1892 int ignore_interceptors
= thr
->ignore_interceptors
;
1893 int ignore_sync
= thr
->ignore_sync
;
1894 if (!ctx
->after_multithreaded_fork
) {
1895 thr
->ignore_reads_and_writes
= 0;
1896 thr
->fast_state
.ClearIgnoreBit();
1897 thr
->ignore_interceptors
= 0;
1898 thr
->ignore_sync
= 0;
1900 // Ensure that the handler does not spoil errno.
1901 const int saved_errno
= errno
;
1903 // This code races with sigaction. Be careful to not read sa_sigaction twice.
1904 // Also need to remember pc for reporting before the call,
1905 // because the handler can reset it.
1907 sigact
? (uptr
)sigactions
[sig
].sigaction
: (uptr
)sigactions
[sig
].handler
;
1908 if (pc
!= sig_dfl
&& pc
!= sig_ign
) {
1910 ((__sanitizer_sigactionhandler_ptr
)pc
)(sig
, info
, uctx
);
1912 ((__sanitizer_sighandler_ptr
)pc
)(sig
);
1914 if (!ctx
->after_multithreaded_fork
) {
1915 thr
->ignore_reads_and_writes
= ignore_reads_and_writes
;
1916 if (ignore_reads_and_writes
)
1917 thr
->fast_state
.SetIgnoreBit();
1918 thr
->ignore_interceptors
= ignore_interceptors
;
1919 thr
->ignore_sync
= ignore_sync
;
1921 // We do not detect errno spoiling for SIGTERM,
1922 // because some SIGTERM handlers do spoil errno but reraise SIGTERM,
1923 // tsan reports false positive in such case.
1924 // It's difficult to properly detect this situation (reraise),
1925 // because in async signal processing case (when handler is called directly
1926 // from rtl_generic_sighandler) we have not yet received the reraised
1927 // signal; and it looks too fragile to intercept all ways to reraise a signal.
1928 if (flags()->report_bugs
&& !sync
&& sig
!= SIGTERM
&& errno
!= 99) {
1929 VarSizeStackTrace stack
;
1930 // StackTrace::GetNestInstructionPc(pc) is used because return address is
1931 // expected, OutputReport() will undo this.
1932 ObtainCurrentStack(thr
, StackTrace::GetNextInstructionPc(pc
), &stack
);
1933 ThreadRegistryLock
l(ctx
->thread_registry
);
1934 ScopedReport
rep(ReportTypeErrnoInSignal
);
1935 if (!IsFiredSuppression(ctx
, ReportTypeErrnoInSignal
, stack
)) {
1936 rep
.AddStack(stack
, true);
1937 OutputReport(thr
, rep
);
1940 errno
= saved_errno
;
1943 void ProcessPendingSignals(ThreadState
*thr
) {
1944 ThreadSignalContext
*sctx
= SigCtx(thr
);
1946 atomic_load(&sctx
->have_pending_signals
, memory_order_relaxed
) == 0)
1948 atomic_store(&sctx
->have_pending_signals
, 0, memory_order_relaxed
);
1949 atomic_fetch_add(&thr
->in_signal_handler
, 1, memory_order_relaxed
);
1950 internal_sigfillset(&sctx
->emptyset
);
1951 int res
= REAL(pthread_sigmask
)(SIG_SETMASK
, &sctx
->emptyset
, &sctx
->oldset
);
1953 for (int sig
= 0; sig
< kSigCount
; sig
++) {
1954 SignalDesc
*signal
= &sctx
->pending_signals
[sig
];
1955 if (signal
->armed
) {
1956 signal
->armed
= false;
1957 CallUserSignalHandler(thr
, false, true, signal
->sigaction
, sig
,
1958 &signal
->siginfo
, &signal
->ctx
);
1961 res
= REAL(pthread_sigmask
)(SIG_SETMASK
, &sctx
->oldset
, 0);
1963 atomic_fetch_add(&thr
->in_signal_handler
, -1, memory_order_relaxed
);
1966 } // namespace __tsan
1968 static bool is_sync_signal(ThreadSignalContext
*sctx
, int sig
) {
1969 return sig
== SIGSEGV
|| sig
== SIGBUS
|| sig
== SIGILL
|| sig
== SIGTRAP
||
1970 sig
== SIGABRT
|| sig
== SIGFPE
|| sig
== SIGPIPE
|| sig
== SIGSYS
||
1971 // If we are sending signal to ourselves, we must process it now.
1972 (sctx
&& sig
== sctx
->int_signal_send
);
1975 void ALWAYS_INLINE
rtl_generic_sighandler(bool sigact
, int sig
,
1976 __sanitizer_siginfo
*info
,
1979 ThreadState
*thr
= cur_thread();
1980 ThreadSignalContext
*sctx
= SigCtx(thr
);
1981 if (sig
< 0 || sig
>= kSigCount
) {
1982 VPrintf(1, "ThreadSanitizer: ignoring signal %d\n", sig
);
1985 // Don't mess with synchronous signals.
1986 const bool sync
= is_sync_signal(sctx
, sig
);
1988 // If we are in blocking function, we can safely process it now
1989 // (but check if we are in a recursive interceptor,
1990 // i.e. pthread_join()->munmap()).
1991 (sctx
&& atomic_load(&sctx
->in_blocking_func
, memory_order_relaxed
))) {
1992 atomic_fetch_add(&thr
->in_signal_handler
, 1, memory_order_relaxed
);
1993 if (sctx
&& atomic_load(&sctx
->in_blocking_func
, memory_order_relaxed
)) {
1994 atomic_store(&sctx
->in_blocking_func
, 0, memory_order_relaxed
);
1995 CallUserSignalHandler(thr
, sync
, true, sigact
, sig
, info
, ctx
);
1996 atomic_store(&sctx
->in_blocking_func
, 1, memory_order_relaxed
);
1998 // Be very conservative with when we do acquire in this case.
1999 // It's unsafe to do acquire in async handlers, because ThreadState
2000 // can be in inconsistent state.
2001 // SIGSYS looks relatively safe -- it's synchronous and can actually
2002 // need some global state.
2003 bool acq
= (sig
== SIGSYS
);
2004 CallUserSignalHandler(thr
, sync
, acq
, sigact
, sig
, info
, ctx
);
2006 atomic_fetch_add(&thr
->in_signal_handler
, -1, memory_order_relaxed
);
2012 SignalDesc
*signal
= &sctx
->pending_signals
[sig
];
2013 if (signal
->armed
== false) {
2014 signal
->armed
= true;
2015 signal
->sigaction
= sigact
;
2017 internal_memcpy(&signal
->siginfo
, info
, sizeof(*info
));
2019 internal_memcpy(&signal
->ctx
, ctx
, sizeof(signal
->ctx
));
2020 atomic_store(&sctx
->have_pending_signals
, 1, memory_order_relaxed
);
2024 static void rtl_sighandler(int sig
) {
2025 rtl_generic_sighandler(false, sig
, 0, 0);
2028 static void rtl_sigaction(int sig
, __sanitizer_siginfo
*info
, void *ctx
) {
2029 rtl_generic_sighandler(true, sig
, info
, ctx
);
2032 TSAN_INTERCEPTOR(int, raise
, int sig
) {
2033 SCOPED_TSAN_INTERCEPTOR(raise
, sig
);
2034 ThreadSignalContext
*sctx
= SigCtx(thr
);
2036 int prev
= sctx
->int_signal_send
;
2037 sctx
->int_signal_send
= sig
;
2038 int res
= REAL(raise
)(sig
);
2039 CHECK_EQ(sctx
->int_signal_send
, sig
);
2040 sctx
->int_signal_send
= prev
;
2044 TSAN_INTERCEPTOR(int, kill
, int pid
, int sig
) {
2045 SCOPED_TSAN_INTERCEPTOR(kill
, pid
, sig
);
2046 ThreadSignalContext
*sctx
= SigCtx(thr
);
2048 int prev
= sctx
->int_signal_send
;
2049 if (pid
== (int)internal_getpid()) {
2050 sctx
->int_signal_send
= sig
;
2052 int res
= REAL(kill
)(pid
, sig
);
2053 if (pid
== (int)internal_getpid()) {
2054 CHECK_EQ(sctx
->int_signal_send
, sig
);
2055 sctx
->int_signal_send
= prev
;
2060 TSAN_INTERCEPTOR(int, pthread_kill
, void *tid
, int sig
) {
2061 SCOPED_TSAN_INTERCEPTOR(pthread_kill
, tid
, sig
);
2062 ThreadSignalContext
*sctx
= SigCtx(thr
);
2064 int prev
= sctx
->int_signal_send
;
2065 if (tid
== pthread_self()) {
2066 sctx
->int_signal_send
= sig
;
2068 int res
= REAL(pthread_kill
)(tid
, sig
);
2069 if (tid
== pthread_self()) {
2070 CHECK_EQ(sctx
->int_signal_send
, sig
);
2071 sctx
->int_signal_send
= prev
;
2076 TSAN_INTERCEPTOR(int, gettimeofday
, void *tv
, void *tz
) {
2077 SCOPED_TSAN_INTERCEPTOR(gettimeofday
, tv
, tz
);
2078 // It's intercepted merely to process pending signals.
2079 return REAL(gettimeofday
)(tv
, tz
);
2082 TSAN_INTERCEPTOR(int, getaddrinfo
, void *node
, void *service
,
2083 void *hints
, void *rv
) {
2084 SCOPED_TSAN_INTERCEPTOR(getaddrinfo
, node
, service
, hints
, rv
);
2085 // We miss atomic synchronization in getaddrinfo,
2086 // and can report false race between malloc and free
2087 // inside of getaddrinfo. So ignore memory accesses.
2088 ThreadIgnoreBegin(thr
, pc
);
2089 int res
= REAL(getaddrinfo
)(node
, service
, hints
, rv
);
2090 ThreadIgnoreEnd(thr
, pc
);
2094 TSAN_INTERCEPTOR(int, fork
, int fake
) {
2095 if (in_symbolizer())
2096 return REAL(fork
)(fake
);
2097 SCOPED_INTERCEPTOR_RAW(fork
, fake
);
2098 ForkBefore(thr
, pc
);
2101 // On OS X, REAL(fork) can call intercepted functions (OSSpinLockLock), and
2102 // we'll assert in CheckNoLocks() unless we ignore interceptors.
2103 ScopedIgnoreInterceptors ignore
;
2104 pid
= REAL(fork
)(fake
);
2108 ForkChildAfter(thr
, pc
);
2110 } else if (pid
> 0) {
2112 ForkParentAfter(thr
, pc
);
2115 ForkParentAfter(thr
, pc
);
2120 TSAN_INTERCEPTOR(int, vfork
, int fake
) {
2121 // Some programs (e.g. openjdk) call close for all file descriptors
2122 // in the child process. Under tsan it leads to false positives, because
2123 // address space is shared, so the parent process also thinks that
2124 // the descriptors are closed (while they are actually not).
2125 // This leads to false positives due to missed synchronization.
2126 // Strictly saying this is undefined behavior, because vfork child is not
2127 // allowed to call any functions other than exec/exit. But this is what
2128 // openjdk does, so we want to handle it.
2129 // We could disable interceptors in the child process. But it's not possible
2130 // to simply intercept and wrap vfork, because vfork child is not allowed
2131 // to return from the function that calls vfork, and that's exactly what
2132 // we would do. So this would require some assembly trickery as well.
2133 // Instead we simply turn vfork into fork.
2134 return WRAP(fork
)(fake
);
2137 #if !SANITIZER_MAC && !SANITIZER_ANDROID
2138 typedef int (*dl_iterate_phdr_cb_t
)(__sanitizer_dl_phdr_info
*info
, SIZE_T size
,
2140 struct dl_iterate_phdr_data
{
2143 dl_iterate_phdr_cb_t cb
;
2147 static bool IsAppNotRodata(uptr addr
) {
2148 return IsAppMem(addr
) && *(u64
*)MemToShadow(addr
) != kShadowRodata
;
2151 static int dl_iterate_phdr_cb(__sanitizer_dl_phdr_info
*info
, SIZE_T size
,
2153 dl_iterate_phdr_data
*cbdata
= (dl_iterate_phdr_data
*)data
;
2154 // dlopen/dlclose allocate/free dynamic-linker-internal memory, which is later
2155 // accessible in dl_iterate_phdr callback. But we don't see synchronization
2156 // inside of dynamic linker, so we "unpoison" it here in order to not
2157 // produce false reports. Ignoring malloc/free in dlopen/dlclose is not enough
2158 // because some libc functions call __libc_dlopen.
2159 if (info
&& IsAppNotRodata((uptr
)info
->dlpi_name
))
2160 MemoryResetRange(cbdata
->thr
, cbdata
->pc
, (uptr
)info
->dlpi_name
,
2161 internal_strlen(info
->dlpi_name
));
2162 int res
= cbdata
->cb(info
, size
, cbdata
->data
);
2163 // Perform the check one more time in case info->dlpi_name was overwritten
2164 // by user callback.
2165 if (info
&& IsAppNotRodata((uptr
)info
->dlpi_name
))
2166 MemoryResetRange(cbdata
->thr
, cbdata
->pc
, (uptr
)info
->dlpi_name
,
2167 internal_strlen(info
->dlpi_name
));
2171 TSAN_INTERCEPTOR(int, dl_iterate_phdr
, dl_iterate_phdr_cb_t cb
, void *data
) {
2172 SCOPED_TSAN_INTERCEPTOR(dl_iterate_phdr
, cb
, data
);
2173 dl_iterate_phdr_data cbdata
;
2178 int res
= REAL(dl_iterate_phdr
)(dl_iterate_phdr_cb
, &cbdata
);
2183 static int OnExit(ThreadState
*thr
) {
2184 int status
= Finalize(thr
);
2189 struct TsanInterceptorContext
{
2191 const uptr caller_pc
;
2196 static void HandleRecvmsg(ThreadState
*thr
, uptr pc
,
2197 __sanitizer_msghdr
*msg
) {
2199 int cnt
= ExtractRecvmsgFDs(msg
, fds
, ARRAY_SIZE(fds
));
2200 for (int i
= 0; i
< cnt
; i
++)
2201 FdEventCreate(thr
, pc
, fds
[i
]);
2205 #include "sanitizer_common/sanitizer_platform_interceptors.h"
2206 // Causes interceptor recursion (getaddrinfo() and fopen())
2207 #undef SANITIZER_INTERCEPT_GETADDRINFO
2208 // We define our own.
2209 #if SANITIZER_INTERCEPT_TLS_GET_ADDR
2210 #define NEED_TLS_GET_ADDR
2212 #undef SANITIZER_INTERCEPT_TLS_GET_ADDR
2213 #undef SANITIZER_INTERCEPT_PTHREAD_SIGMASK
2215 #define COMMON_INTERCEPT_FUNCTION(name) INTERCEPT_FUNCTION(name)
2216 #define COMMON_INTERCEPT_FUNCTION_VER(name, ver) \
2217 INTERCEPT_FUNCTION_VER(name, ver)
2219 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
2220 MemoryAccessRange(((TsanInterceptorContext *)ctx)->thr, \
2221 ((TsanInterceptorContext *)ctx)->pc, (uptr)ptr, size, \
2224 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
2225 MemoryAccessRange(((TsanInterceptorContext *) ctx)->thr, \
2226 ((TsanInterceptorContext *) ctx)->pc, (uptr) ptr, size, \
2229 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...) \
2230 SCOPED_TSAN_INTERCEPTOR(func, __VA_ARGS__); \
2231 TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
2232 ctx = (void *)&_ctx; \
2235 #define COMMON_INTERCEPTOR_ENTER_NOIGNORE(ctx, func, ...) \
2236 SCOPED_INTERCEPTOR_RAW(func, __VA_ARGS__); \
2237 TsanInterceptorContext _ctx = {thr, caller_pc, pc}; \
2238 ctx = (void *)&_ctx; \
2241 #define COMMON_INTERCEPTOR_FILE_OPEN(ctx, file, path) \
2243 Acquire(thr, pc, File2addr(path)); \
2245 int fd = fileno_unlocked(file); \
2246 if (fd >= 0) FdFileCreate(thr, pc, fd); \
2249 #define COMMON_INTERCEPTOR_FILE_CLOSE(ctx, file) \
2251 int fd = fileno_unlocked(file); \
2252 if (fd >= 0) FdClose(thr, pc, fd); \
2255 #define COMMON_INTERCEPTOR_LIBRARY_LOADED(filename, handle) \
2256 libignore()->OnLibraryLoaded(filename)
2258 #define COMMON_INTERCEPTOR_LIBRARY_UNLOADED() \
2259 libignore()->OnLibraryUnloaded()
2261 #define COMMON_INTERCEPTOR_ACQUIRE(ctx, u) \
2262 Acquire(((TsanInterceptorContext *) ctx)->thr, pc, u)
2264 #define COMMON_INTERCEPTOR_RELEASE(ctx, u) \
2265 Release(((TsanInterceptorContext *) ctx)->thr, pc, u)
2267 #define COMMON_INTERCEPTOR_DIR_ACQUIRE(ctx, path) \
2268 Acquire(((TsanInterceptorContext *) ctx)->thr, pc, Dir2addr(path))
2270 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
2271 FdAcquire(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2273 #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
2274 FdRelease(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2276 #define COMMON_INTERCEPTOR_FD_ACCESS(ctx, fd) \
2277 FdAccess(((TsanInterceptorContext *) ctx)->thr, pc, fd)
2279 #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
2280 FdSocketAccept(((TsanInterceptorContext *) ctx)->thr, pc, fd, newfd)
2282 #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
2283 ThreadSetName(((TsanInterceptorContext *) ctx)->thr, name)
2285 #define COMMON_INTERCEPTOR_SET_PTHREAD_NAME(ctx, thread, name) \
2286 __tsan::ctx->thread_registry->SetThreadNameByUserId(thread, name)
2288 #define COMMON_INTERCEPTOR_BLOCK_REAL(name) BLOCK_REAL(name)
2290 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) \
2291 OnExit(((TsanInterceptorContext *) ctx)->thr)
2293 #define COMMON_INTERCEPTOR_MUTEX_PRE_LOCK(ctx, m) \
2294 MutexPreLock(((TsanInterceptorContext *)ctx)->thr, \
2295 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2297 #define COMMON_INTERCEPTOR_MUTEX_POST_LOCK(ctx, m) \
2298 MutexPostLock(((TsanInterceptorContext *)ctx)->thr, \
2299 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2301 #define COMMON_INTERCEPTOR_MUTEX_UNLOCK(ctx, m) \
2302 MutexUnlock(((TsanInterceptorContext *)ctx)->thr, \
2303 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2305 #define COMMON_INTERCEPTOR_MUTEX_REPAIR(ctx, m) \
2306 MutexRepair(((TsanInterceptorContext *)ctx)->thr, \
2307 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2309 #define COMMON_INTERCEPTOR_MUTEX_INVALID(ctx, m) \
2310 MutexInvalidAccess(((TsanInterceptorContext *)ctx)->thr, \
2311 ((TsanInterceptorContext *)ctx)->pc, (uptr)m)
2313 #define COMMON_INTERCEPTOR_MMAP_IMPL(ctx, mmap, addr, sz, prot, flags, fd, \
2316 return mmap_interceptor(thr, pc, REAL(mmap), addr, sz, prot, flags, fd, \
2321 #define COMMON_INTERCEPTOR_HANDLE_RECVMSG(ctx, msg) \
2322 HandleRecvmsg(((TsanInterceptorContext *)ctx)->thr, \
2323 ((TsanInterceptorContext *)ctx)->pc, msg)
2326 #define COMMON_INTERCEPTOR_GET_TLS_RANGE(begin, end) \
2327 if (TsanThread *t = GetCurrentThread()) { \
2328 *begin = t->tls_begin(); \
2329 *end = t->tls_end(); \
2331 *begin = *end = 0; \
2334 #define COMMON_INTERCEPTOR_USER_CALLBACK_START() \
2335 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START()
2337 #define COMMON_INTERCEPTOR_USER_CALLBACK_END() \
2338 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END()
2340 #include "sanitizer_common/sanitizer_common_interceptors.inc"
2342 static int sigaction_impl(int sig
, const __sanitizer_sigaction
*act
,
2343 __sanitizer_sigaction
*old
);
2344 static __sanitizer_sighandler_ptr
signal_impl(int sig
,
2345 __sanitizer_sighandler_ptr h
);
2347 #define SIGNAL_INTERCEPTOR_SIGACTION_IMPL(signo, act, oldact) \
2348 { return sigaction_impl(signo, act, oldact); }
2350 #define SIGNAL_INTERCEPTOR_SIGNAL_IMPL(func, signo, handler) \
2351 { return (uptr)signal_impl(signo, (__sanitizer_sighandler_ptr)handler); }
2353 #include "sanitizer_common/sanitizer_signal_interceptors.inc"
2355 int sigaction_impl(int sig
, const __sanitizer_sigaction
*act
,
2356 __sanitizer_sigaction
*old
) {
2357 // Note: if we call REAL(sigaction) directly for any reason without proxying
2358 // the signal handler through rtl_sigaction, very bad things will happen.
2359 // The handler will run synchronously and corrupt tsan per-thread state.
2360 SCOPED_INTERCEPTOR_RAW(sigaction
, sig
, act
, old
);
2361 __sanitizer_sigaction
*sigactions
= interceptor_ctx()->sigactions
;
2362 __sanitizer_sigaction old_stored
;
2363 if (old
) internal_memcpy(&old_stored
, &sigactions
[sig
], sizeof(old_stored
));
2364 __sanitizer_sigaction newact
;
2366 // Copy act into sigactions[sig].
2367 // Can't use struct copy, because compiler can emit call to memcpy.
2368 // Can't use internal_memcpy, because it copies byte-by-byte,
2369 // and signal handler reads the handler concurrently. It it can read
2370 // some bytes from old value and some bytes from new value.
2371 // Use volatile to prevent insertion of memcpy.
2372 sigactions
[sig
].handler
=
2373 *(volatile __sanitizer_sighandler_ptr
const *)&act
->handler
;
2374 sigactions
[sig
].sa_flags
= *(volatile int const *)&act
->sa_flags
;
2375 internal_memcpy(&sigactions
[sig
].sa_mask
, &act
->sa_mask
,
2376 sizeof(sigactions
[sig
].sa_mask
));
2377 #if !SANITIZER_FREEBSD && !SANITIZER_MAC && !SANITIZER_NETBSD
2378 sigactions
[sig
].sa_restorer
= act
->sa_restorer
;
2380 internal_memcpy(&newact
, act
, sizeof(newact
));
2381 internal_sigfillset(&newact
.sa_mask
);
2382 if ((uptr
)act
->handler
!= sig_ign
&& (uptr
)act
->handler
!= sig_dfl
) {
2383 if (newact
.sa_flags
& SA_SIGINFO
)
2384 newact
.sigaction
= rtl_sigaction
;
2386 newact
.handler
= rtl_sighandler
;
2388 ReleaseStore(thr
, pc
, (uptr
)&sigactions
[sig
]);
2391 int res
= REAL(sigaction
)(sig
, act
, old
);
2392 if (res
== 0 && old
) {
2393 uptr cb
= (uptr
)old
->sigaction
;
2394 if (cb
== (uptr
)rtl_sigaction
|| cb
== (uptr
)rtl_sighandler
) {
2395 internal_memcpy(old
, &old_stored
, sizeof(*old
));
2401 static __sanitizer_sighandler_ptr
signal_impl(int sig
,
2402 __sanitizer_sighandler_ptr h
) {
2403 __sanitizer_sigaction act
;
2405 internal_memset(&act
.sa_mask
, -1, sizeof(act
.sa_mask
));
2407 __sanitizer_sigaction old
;
2408 int res
= sigaction_symname(sig
, &act
, &old
);
2409 if (res
) return (__sanitizer_sighandler_ptr
)sig_err
;
2413 #define TSAN_SYSCALL() \
2414 ThreadState *thr = cur_thread(); \
2415 if (thr->ignore_interceptors) \
2417 ScopedSyscall scoped_syscall(thr) \
2420 struct ScopedSyscall
{
2423 explicit ScopedSyscall(ThreadState
*thr
)
2429 ProcessPendingSignals(thr
);
2433 #if !SANITIZER_FREEBSD && !SANITIZER_MAC
2434 static void syscall_access_range(uptr pc
, uptr p
, uptr s
, bool write
) {
2436 MemoryAccessRange(thr
, pc
, p
, s
, write
);
2439 static void syscall_acquire(uptr pc
, uptr addr
) {
2441 Acquire(thr
, pc
, addr
);
2442 DPrintf("syscall_acquire(%p)\n", addr
);
2445 static void syscall_release(uptr pc
, uptr addr
) {
2447 DPrintf("syscall_release(%p)\n", addr
);
2448 Release(thr
, pc
, addr
);
2451 static void syscall_fd_close(uptr pc
, int fd
) {
2453 FdClose(thr
, pc
, fd
);
2456 static USED
void syscall_fd_acquire(uptr pc
, int fd
) {
2458 FdAcquire(thr
, pc
, fd
);
2459 DPrintf("syscall_fd_acquire(%p)\n", fd
);
2462 static USED
void syscall_fd_release(uptr pc
, int fd
) {
2464 DPrintf("syscall_fd_release(%p)\n", fd
);
2465 FdRelease(thr
, pc
, fd
);
2468 static void syscall_pre_fork(uptr pc
) {
2470 ForkBefore(thr
, pc
);
2473 static void syscall_post_fork(uptr pc
, int pid
) {
2477 ForkChildAfter(thr
, pc
);
2479 } else if (pid
> 0) {
2481 ForkParentAfter(thr
, pc
);
2484 ForkParentAfter(thr
, pc
);
2489 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) \
2490 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), false)
2492 #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
2493 syscall_access_range(GET_CALLER_PC(), (uptr)(p), (uptr)(s), true)
2495 #define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
2501 #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) \
2507 #define COMMON_SYSCALL_ACQUIRE(addr) \
2508 syscall_acquire(GET_CALLER_PC(), (uptr)(addr))
2510 #define COMMON_SYSCALL_RELEASE(addr) \
2511 syscall_release(GET_CALLER_PC(), (uptr)(addr))
2513 #define COMMON_SYSCALL_FD_CLOSE(fd) syscall_fd_close(GET_CALLER_PC(), fd)
2515 #define COMMON_SYSCALL_FD_ACQUIRE(fd) syscall_fd_acquire(GET_CALLER_PC(), fd)
2517 #define COMMON_SYSCALL_FD_RELEASE(fd) syscall_fd_release(GET_CALLER_PC(), fd)
2519 #define COMMON_SYSCALL_PRE_FORK() \
2520 syscall_pre_fork(GET_CALLER_PC())
2522 #define COMMON_SYSCALL_POST_FORK(res) \
2523 syscall_post_fork(GET_CALLER_PC(), res)
2525 #include "sanitizer_common/sanitizer_common_syscalls.inc"
2526 #include "sanitizer_common/sanitizer_syscalls_netbsd.inc"
2528 #ifdef NEED_TLS_GET_ADDR
2529 // Define own interceptor instead of sanitizer_common's for three reasons:
2530 // 1. It must not process pending signals.
2531 // Signal handlers may contain MOVDQA instruction (see below).
2532 // 2. It must be as simple as possible to not contain MOVDQA.
2533 // 3. Sanitizer_common version uses COMMON_INTERCEPTOR_INITIALIZE_RANGE which
2534 // is empty for tsan (meant only for msan).
2535 // Note: __tls_get_addr can be called with mis-aligned stack due to:
2536 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58066
2537 // So the interceptor must work with mis-aligned stack, in particular, does not
2538 // execute MOVDQA with stack addresses.
2539 TSAN_INTERCEPTOR(void *, __tls_get_addr
, void *arg
) {
2540 void *res
= REAL(__tls_get_addr
)(arg
);
2541 ThreadState
*thr
= cur_thread();
2544 DTLS::DTV
*dtv
= DTLS_on_tls_get_addr(arg
, res
, thr
->tls_addr
,
2545 thr
->tls_addr
+ thr
->tls_size
);
2548 // New DTLS block has been allocated.
2549 MemoryResetRange(thr
, 0, dtv
->beg
, dtv
->size
);
2554 #if SANITIZER_NETBSD
2555 TSAN_INTERCEPTOR(void, _lwp_exit
) {
2556 SCOPED_TSAN_INTERCEPTOR(_lwp_exit
);
2557 DestroyThreadState();
2560 #define TSAN_MAYBE_INTERCEPT__LWP_EXIT TSAN_INTERCEPT(_lwp_exit)
2562 #define TSAN_MAYBE_INTERCEPT__LWP_EXIT
2565 #if SANITIZER_FREEBSD
2566 TSAN_INTERCEPTOR(void, thr_exit
, tid_t
*state
) {
2567 SCOPED_TSAN_INTERCEPTOR(thr_exit
, state
);
2568 DestroyThreadState();
2569 REAL(thr_exit(state
));
2571 #define TSAN_MAYBE_INTERCEPT_THR_EXIT TSAN_INTERCEPT(thr_exit)
2573 #define TSAN_MAYBE_INTERCEPT_THR_EXIT
2576 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_init
, void *c
, void *a
)
2577 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_signal
, void *c
)
2578 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_broadcast
, void *c
)
2579 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_wait
, void *c
, void *m
)
2580 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, cond_destroy
, void *c
)
2581 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_init
, void *m
, void *a
)
2582 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_destroy
, void *m
)
2583 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, mutex_trylock
, void *m
)
2584 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_init
, void *m
, void *a
)
2585 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_destroy
, void *m
)
2586 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_rdlock
, void *m
)
2587 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_tryrdlock
, void *m
)
2588 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_wrlock
, void *m
)
2589 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_trywrlock
, void *m
)
2590 TSAN_INTERCEPTOR_NETBSD_ALIAS(int, rwlock_unlock
, void *m
)
2591 TSAN_INTERCEPTOR_NETBSD_ALIAS_THR(int, once
, void *o
, void (*f
)())
2592 TSAN_INTERCEPTOR_NETBSD_ALIAS_THR2(int, sigsetmask
, sigmask
, int a
, void *b
,
2597 static void finalize(void *arg
) {
2598 ThreadState
*thr
= cur_thread();
2599 int status
= Finalize(thr
);
2600 // Make sure the output is not lost.
2606 #if !SANITIZER_MAC && !SANITIZER_ANDROID
2607 static void unreachable() {
2608 Report("FATAL: ThreadSanitizer: unreachable called\n");
2613 // Define default implementation since interception of libdispatch is optional.
2614 SANITIZER_WEAK_ATTRIBUTE
void InitializeLibdispatchInterceptors() {}
2616 void InitializeInterceptors() {
2618 // We need to setup it early, because functions like dlsym() can call it.
2619 REAL(memset
) = internal_memset
;
2620 REAL(memcpy
) = internal_memcpy
;
2623 // Instruct libc malloc to consume less memory.
2625 mallopt(1, 0); // M_MXFAST
2626 mallopt(-3, 32*1024); // M_MMAP_THRESHOLD
2629 new(interceptor_ctx()) InterceptorContext();
2631 InitializeCommonInterceptors();
2632 InitializeSignalInterceptors();
2633 InitializeLibdispatchInterceptors();
2636 // We can not use TSAN_INTERCEPT to get setjmp addr,
2637 // because it does &setjmp and setjmp is not present in some versions of libc.
2638 using __interception::InterceptFunction
;
2639 InterceptFunction(TSAN_STRING_SETJMP
, (uptr
*)&REAL(setjmp_symname
), 0, 0);
2640 InterceptFunction("_setjmp", (uptr
*)&REAL(_setjmp
), 0, 0);
2641 InterceptFunction(TSAN_STRING_SIGSETJMP
, (uptr
*)&REAL(sigsetjmp_symname
), 0,
2643 #if !SANITIZER_NETBSD
2644 InterceptFunction("__sigsetjmp", (uptr
*)&REAL(__sigsetjmp
), 0, 0);
2648 TSAN_INTERCEPT(longjmp_symname
);
2649 TSAN_INTERCEPT(siglongjmp_symname
);
2650 #if SANITIZER_NETBSD
2651 TSAN_INTERCEPT(_longjmp
);
2654 TSAN_INTERCEPT(malloc
);
2655 TSAN_INTERCEPT(__libc_memalign
);
2656 TSAN_INTERCEPT(calloc
);
2657 TSAN_INTERCEPT(realloc
);
2658 TSAN_INTERCEPT(reallocarray
);
2659 TSAN_INTERCEPT(free
);
2660 TSAN_INTERCEPT(cfree
);
2661 TSAN_INTERCEPT(munmap
);
2662 TSAN_MAYBE_INTERCEPT_MEMALIGN
;
2663 TSAN_INTERCEPT(valloc
);
2664 TSAN_MAYBE_INTERCEPT_PVALLOC
;
2665 TSAN_INTERCEPT(posix_memalign
);
2667 TSAN_INTERCEPT(strcpy
);
2668 TSAN_INTERCEPT(strncpy
);
2669 TSAN_INTERCEPT(strdup
);
2671 TSAN_INTERCEPT(pthread_create
);
2672 TSAN_INTERCEPT(pthread_join
);
2673 TSAN_INTERCEPT(pthread_detach
);
2674 TSAN_INTERCEPT(pthread_exit
);
2676 TSAN_INTERCEPT(pthread_tryjoin_np
);
2677 TSAN_INTERCEPT(pthread_timedjoin_np
);
2680 TSAN_INTERCEPT_VER(pthread_cond_init
, PTHREAD_ABI_BASE
);
2681 TSAN_INTERCEPT_VER(pthread_cond_signal
, PTHREAD_ABI_BASE
);
2682 TSAN_INTERCEPT_VER(pthread_cond_broadcast
, PTHREAD_ABI_BASE
);
2683 TSAN_INTERCEPT_VER(pthread_cond_wait
, PTHREAD_ABI_BASE
);
2684 TSAN_INTERCEPT_VER(pthread_cond_timedwait
, PTHREAD_ABI_BASE
);
2685 TSAN_INTERCEPT_VER(pthread_cond_destroy
, PTHREAD_ABI_BASE
);
2687 TSAN_INTERCEPT(pthread_mutex_init
);
2688 TSAN_INTERCEPT(pthread_mutex_destroy
);
2689 TSAN_INTERCEPT(pthread_mutex_trylock
);
2690 TSAN_INTERCEPT(pthread_mutex_timedlock
);
2692 TSAN_INTERCEPT(pthread_spin_init
);
2693 TSAN_INTERCEPT(pthread_spin_destroy
);
2694 TSAN_INTERCEPT(pthread_spin_lock
);
2695 TSAN_INTERCEPT(pthread_spin_trylock
);
2696 TSAN_INTERCEPT(pthread_spin_unlock
);
2698 TSAN_INTERCEPT(pthread_rwlock_init
);
2699 TSAN_INTERCEPT(pthread_rwlock_destroy
);
2700 TSAN_INTERCEPT(pthread_rwlock_rdlock
);
2701 TSAN_INTERCEPT(pthread_rwlock_tryrdlock
);
2702 TSAN_INTERCEPT(pthread_rwlock_timedrdlock
);
2703 TSAN_INTERCEPT(pthread_rwlock_wrlock
);
2704 TSAN_INTERCEPT(pthread_rwlock_trywrlock
);
2705 TSAN_INTERCEPT(pthread_rwlock_timedwrlock
);
2706 TSAN_INTERCEPT(pthread_rwlock_unlock
);
2708 TSAN_INTERCEPT(pthread_barrier_init
);
2709 TSAN_INTERCEPT(pthread_barrier_destroy
);
2710 TSAN_INTERCEPT(pthread_barrier_wait
);
2712 TSAN_INTERCEPT(pthread_once
);
2714 TSAN_INTERCEPT(fstat
);
2715 TSAN_MAYBE_INTERCEPT___FXSTAT
;
2716 TSAN_MAYBE_INTERCEPT_FSTAT64
;
2717 TSAN_MAYBE_INTERCEPT___FXSTAT64
;
2718 TSAN_INTERCEPT(open
);
2719 TSAN_MAYBE_INTERCEPT_OPEN64
;
2720 TSAN_INTERCEPT(creat
);
2721 TSAN_MAYBE_INTERCEPT_CREAT64
;
2722 TSAN_INTERCEPT(dup
);
2723 TSAN_INTERCEPT(dup2
);
2724 TSAN_INTERCEPT(dup3
);
2725 TSAN_MAYBE_INTERCEPT_EVENTFD
;
2726 TSAN_MAYBE_INTERCEPT_SIGNALFD
;
2727 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT
;
2728 TSAN_MAYBE_INTERCEPT_INOTIFY_INIT1
;
2729 TSAN_INTERCEPT(socket
);
2730 TSAN_INTERCEPT(socketpair
);
2731 TSAN_INTERCEPT(connect
);
2732 TSAN_INTERCEPT(bind
);
2733 TSAN_INTERCEPT(listen
);
2734 TSAN_MAYBE_INTERCEPT_EPOLL
;
2735 TSAN_INTERCEPT(close
);
2736 TSAN_MAYBE_INTERCEPT___CLOSE
;
2737 TSAN_MAYBE_INTERCEPT___RES_ICLOSE
;
2738 TSAN_INTERCEPT(pipe
);
2739 TSAN_INTERCEPT(pipe2
);
2741 TSAN_INTERCEPT(unlink
);
2742 TSAN_INTERCEPT(tmpfile
);
2743 TSAN_MAYBE_INTERCEPT_TMPFILE64
;
2744 TSAN_INTERCEPT(abort
);
2745 TSAN_INTERCEPT(rmdir
);
2746 TSAN_INTERCEPT(closedir
);
2748 TSAN_INTERCEPT(sigsuspend
);
2749 TSAN_INTERCEPT(sigblock
);
2750 TSAN_INTERCEPT(sigsetmask
);
2751 TSAN_INTERCEPT(pthread_sigmask
);
2752 TSAN_INTERCEPT(raise
);
2753 TSAN_INTERCEPT(kill
);
2754 TSAN_INTERCEPT(pthread_kill
);
2755 TSAN_INTERCEPT(sleep
);
2756 TSAN_INTERCEPT(usleep
);
2757 TSAN_INTERCEPT(nanosleep
);
2758 TSAN_INTERCEPT(pause
);
2759 TSAN_INTERCEPT(gettimeofday
);
2760 TSAN_INTERCEPT(getaddrinfo
);
2762 TSAN_INTERCEPT(fork
);
2763 TSAN_INTERCEPT(vfork
);
2764 #if !SANITIZER_ANDROID
2765 TSAN_INTERCEPT(dl_iterate_phdr
);
2767 TSAN_MAYBE_INTERCEPT_ON_EXIT
;
2768 TSAN_INTERCEPT(__cxa_atexit
);
2769 TSAN_INTERCEPT(_exit
);
2771 #ifdef NEED_TLS_GET_ADDR
2772 TSAN_INTERCEPT(__tls_get_addr
);
2775 TSAN_MAYBE_INTERCEPT__LWP_EXIT
;
2776 TSAN_MAYBE_INTERCEPT_THR_EXIT
;
2778 #if !SANITIZER_MAC && !SANITIZER_ANDROID
2779 // Need to setup it, because interceptors check that the function is resolved.
2780 // But atexit is emitted directly into the module, so can't be resolved.
2781 REAL(atexit
) = (int(*)(void(*)()))unreachable
;
2784 if (REAL(__cxa_atexit
)(&finalize
, 0, 0)) {
2785 Printf("ThreadSanitizer: failed to setup atexit callback\n");
2789 #if !SANITIZER_MAC && !SANITIZER_NETBSD && !SANITIZER_FREEBSD
2790 if (pthread_key_create(&interceptor_ctx()->finalize_key
, &thread_finalize
)) {
2791 Printf("ThreadSanitizer: failed to create thread key\n");
2796 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_init
);
2797 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_signal
);
2798 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_broadcast
);
2799 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_wait
);
2800 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(cond_destroy
);
2801 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_init
);
2802 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_destroy
);
2803 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(mutex_trylock
);
2804 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_init
);
2805 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_destroy
);
2806 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_rdlock
);
2807 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_tryrdlock
);
2808 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_wrlock
);
2809 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_trywrlock
);
2810 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS(rwlock_unlock
);
2811 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(once
);
2812 TSAN_MAYBE_INTERCEPT_NETBSD_ALIAS_THR(sigsetmask
);
2817 } // namespace __tsan
2819 // Invisible barrier for tests.
2820 // There were several unsuccessful iterations for this functionality:
2821 // 1. Initially it was implemented in user code using
2822 // REAL(pthread_barrier_wait). But pthread_barrier_wait is not supported on
2823 // MacOS. Futexes are linux-specific for this matter.
2824 // 2. Then we switched to atomics+usleep(10). But usleep produced parasitic
2825 // "as-if synchronized via sleep" messages in reports which failed some
2827 // 3. Then we switched to atomics+sched_yield. But this produced tons of tsan-
2828 // visible events, which lead to "failed to restore stack trace" failures.
2829 // Note that no_sanitize_thread attribute does not turn off atomic interception
2830 // so attaching it to the function defined in user code does not help.
2831 // That's why we now have what we have.
2832 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
2833 void __tsan_testonly_barrier_init(u64
*barrier
, u32 count
) {
2834 if (count
>= (1 << 8)) {
2835 Printf("barrier_init: count is too large (%d)\n", count
);
2838 // 8 lsb is thread count, the remaining are count of entered threads.
2842 extern "C" SANITIZER_INTERFACE_ATTRIBUTE
2843 void __tsan_testonly_barrier_wait(u64
*barrier
) {
2844 unsigned old
= __atomic_fetch_add(barrier
, 1 << 8, __ATOMIC_RELAXED
);
2845 unsigned old_epoch
= (old
>> 8) / (old
& 0xff);
2847 unsigned cur
= __atomic_load_n(barrier
, __ATOMIC_RELAXED
);
2848 unsigned cur_epoch
= (cur
>> 8) / (cur
& 0xff);
2849 if (cur_epoch
!= old_epoch
)
2851 internal_sched_yield();