1 //===-- sanitizer_stoptheworld_linux_libcdep.cpp --------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // See sanitizer_stoptheworld.h for details.
10 // This implementation was inspired by Markus Gutschke's linuxthreads.cc.
12 //===----------------------------------------------------------------------===//
14 #include "sanitizer_platform.h"
16 #if SANITIZER_LINUX && \
17 (defined(__x86_64__) || defined(__mips__) || defined(__aarch64__) || \
18 defined(__powerpc64__) || defined(__s390__) || defined(__i386__) || \
19 defined(__arm__) || SANITIZER_RISCV64)
21 #include "sanitizer_stoptheworld.h"
23 #include "sanitizer_platform_limits_posix.h"
24 #include "sanitizer_atomic.h"
27 #include <sched.h> // for CLONE_* definitions
29 #include <sys/prctl.h> // for PR_* definitions
30 #include <sys/ptrace.h> // for PTRACE_* definitions
31 #include <sys/types.h> // for pid_t
32 #include <sys/uio.h> // for iovec
33 #include <elf.h> // for NT_PRSTATUS
34 #if (defined(__aarch64__) || SANITIZER_RISCV64) && !SANITIZER_ANDROID
35 // GLIBC 2.20+ sys/user does not include asm/ptrace.h
36 # include <asm/ptrace.h>
38 #include <sys/user.h> // for user_regs_struct
39 #if SANITIZER_ANDROID && SANITIZER_MIPS
40 # include <asm/reg.h> // for mips SP register in sys/user.h
42 #include <sys/wait.h> // for signal-related stuff
52 #include "sanitizer_common.h"
53 #include "sanitizer_flags.h"
54 #include "sanitizer_libc.h"
55 #include "sanitizer_linux.h"
56 #include "sanitizer_mutex.h"
57 #include "sanitizer_placement_new.h"
59 // Sufficiently old kernel headers don't provide this value, but we can still
60 // call prctl with it. If the runtime kernel is new enough, the prctl call will
61 // have the desired effect; if the kernel is too old, the call will error and we
62 // can ignore said error.
63 #ifndef PR_SET_PTRACER
64 #define PR_SET_PTRACER 0x59616d61
67 // This module works by spawning a Linux task which then attaches to every
68 // thread in the caller process with ptrace. This suspends the threads, and
69 // PTRACE_GETREGS can then be used to obtain their register state. The callback
70 // supplied to StopTheWorld() is run in the tracer task while the threads are
72 // The tracer task must be placed in a different thread group for ptrace to
73 // work, so it cannot be spawned as a pthread. Instead, we use the low-level
74 // clone() interface (we want to share the address space with the caller
75 // process, so we prefer clone() over fork()).
77 // We don't use any libc functions, relying instead on direct syscalls. There
78 // are two reasons for this:
79 // 1. calling a library function while threads are suspended could cause a
80 // deadlock, if one of the treads happens to be holding a libc lock;
81 // 2. it's generally not safe to call libc functions from the tracer task,
82 // because clone() does not set up a thread-local storage for it. Any
83 // thread-local variables used by libc will be shared between the tracer task
84 // and the thread which spawned it.
86 namespace __sanitizer
{
88 class SuspendedThreadsListLinux final
: public SuspendedThreadsList
{
90 SuspendedThreadsListLinux() { thread_ids_
.reserve(1024); }
92 tid_t
GetThreadID(uptr index
) const override
;
93 uptr
ThreadCount() const override
;
94 bool ContainsTid(tid_t thread_id
) const;
95 void Append(tid_t tid
);
97 PtraceRegistersStatus
GetRegistersAndSP(uptr index
,
98 InternalMmapVector
<uptr
> *buffer
,
99 uptr
*sp
) const override
;
102 InternalMmapVector
<tid_t
> thread_ids_
;
105 // Structure for passing arguments into the tracer thread.
106 struct TracerThreadArgument
{
107 StopTheWorldCallback callback
;
108 void *callback_argument
;
109 // The tracer thread waits on this mutex while the parent finishes its
112 // Tracer thread signals its completion by setting done.
113 atomic_uintptr_t done
;
117 // This class handles thread suspending/unsuspending in the tracer thread.
118 class ThreadSuspender
{
120 explicit ThreadSuspender(pid_t pid
, TracerThreadArgument
*arg
)
125 bool SuspendAllThreads();
126 void ResumeAllThreads();
127 void KillAllThreads();
128 SuspendedThreadsListLinux
&suspended_threads_list() {
129 return suspended_threads_list_
;
131 TracerThreadArgument
*arg
;
133 SuspendedThreadsListLinux suspended_threads_list_
;
135 bool SuspendThread(tid_t thread_id
);
138 bool ThreadSuspender::SuspendThread(tid_t tid
) {
139 // Are we already attached to this thread?
140 // Currently this check takes linear time, however the number of threads is
142 if (suspended_threads_list_
.ContainsTid(tid
)) return false;
144 if (internal_iserror(internal_ptrace(PTRACE_ATTACH
, tid
, nullptr, nullptr),
146 // Either the thread is dead, or something prevented us from attaching.
147 // Log this event and move on.
148 VReport(1, "Could not attach to thread %zu (errno %d).\n", (uptr
)tid
,
152 VReport(2, "Attached to thread %zu.\n", (uptr
)tid
);
153 // The thread is not guaranteed to stop before ptrace returns, so we must
154 // wait on it. Note: if the thread receives a signal concurrently,
155 // we can get notification about the signal before notification about stop.
156 // In such case we need to forward the signal to the thread, otherwise
157 // the signal will be missed (as we do PTRACE_DETACH with arg=0) and
158 // any logic relying on signals will break. After forwarding we need to
159 // continue to wait for stopping, because the thread is not stopped yet.
160 // We do ignore delivery of SIGSTOP, because we want to make stop-the-world
161 // as invisible as possible.
165 HANDLE_EINTR(waitpid_status
, internal_waitpid(tid
, &status
, __WALL
));
167 if (internal_iserror(waitpid_status
, &wperrno
)) {
168 // Got a ECHILD error. I don't think this situation is possible, but it
169 // doesn't hurt to report it.
170 VReport(1, "Waiting on thread %zu failed, detaching (errno %d).\n",
172 internal_ptrace(PTRACE_DETACH
, tid
, nullptr, nullptr);
175 if (WIFSTOPPED(status
) && WSTOPSIG(status
) != SIGSTOP
) {
176 internal_ptrace(PTRACE_CONT
, tid
, nullptr,
177 (void*)(uptr
)WSTOPSIG(status
));
182 suspended_threads_list_
.Append(tid
);
187 void ThreadSuspender::ResumeAllThreads() {
188 for (uptr i
= 0; i
< suspended_threads_list_
.ThreadCount(); i
++) {
189 pid_t tid
= suspended_threads_list_
.GetThreadID(i
);
191 if (!internal_iserror(internal_ptrace(PTRACE_DETACH
, tid
, nullptr, nullptr),
193 VReport(2, "Detached from thread %d.\n", tid
);
195 // Either the thread is dead, or we are already detached.
196 // The latter case is possible, for instance, if this function was called
197 // from a signal handler.
198 VReport(1, "Could not detach from thread %d (errno %d).\n", tid
, pterrno
);
203 void ThreadSuspender::KillAllThreads() {
204 for (uptr i
= 0; i
< suspended_threads_list_
.ThreadCount(); i
++)
205 internal_ptrace(PTRACE_KILL
, suspended_threads_list_
.GetThreadID(i
),
209 bool ThreadSuspender::SuspendAllThreads() {
210 ThreadLister
thread_lister(pid_
);
212 InternalMmapVector
<tid_t
> threads
;
213 threads
.reserve(128);
214 for (int i
= 0; i
< 30 && retry
; ++i
) {
216 switch (thread_lister
.ListThreads(&threads
)) {
217 case ThreadLister::Error
:
220 case ThreadLister::Incomplete
:
223 case ThreadLister::Ok
:
226 for (tid_t tid
: threads
) {
227 if (SuspendThread(tid
))
231 return suspended_threads_list_
.ThreadCount();
234 // Pointer to the ThreadSuspender instance for use in signal handler.
235 static ThreadSuspender
*thread_suspender_instance
= nullptr;
237 // Synchronous signals that should not be blocked.
238 static const int kSyncSignals
[] = { SIGABRT
, SIGILL
, SIGFPE
, SIGSEGV
, SIGBUS
,
241 static void TracerThreadDieCallback() {
242 // Generally a call to Die() in the tracer thread should be fatal to the
243 // parent process as well, because they share the address space.
244 // This really only works correctly if all the threads are suspended at this
245 // point. So we correctly handle calls to Die() from within the callback, but
246 // not those that happen before or after the callback. Hopefully there aren't
247 // a lot of opportunities for that to happen...
248 ThreadSuspender
*inst
= thread_suspender_instance
;
249 if (inst
&& stoptheworld_tracer_pid
== internal_getpid()) {
250 inst
->KillAllThreads();
251 thread_suspender_instance
= nullptr;
255 // Signal handler to wake up suspended threads when the tracer thread dies.
256 static void TracerThreadSignalHandler(int signum
, __sanitizer_siginfo
*siginfo
,
258 SignalContext
ctx(siginfo
, uctx
);
259 Printf("Tracer caught signal %d: addr=0x%zx pc=0x%zx sp=0x%zx\n", signum
,
260 ctx
.addr
, ctx
.pc
, ctx
.sp
);
261 ThreadSuspender
*inst
= thread_suspender_instance
;
263 if (signum
== SIGABRT
)
264 inst
->KillAllThreads();
266 inst
->ResumeAllThreads();
267 RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback
));
268 thread_suspender_instance
= nullptr;
269 atomic_store(&inst
->arg
->done
, 1, memory_order_relaxed
);
271 internal__exit((signum
== SIGABRT
) ? 1 : 2);
274 // Size of alternative stack for signal handlers in the tracer thread.
275 static const int kHandlerStackSize
= 8192;
277 // This function will be run as a cloned task.
278 static int TracerThread(void* argument
) {
279 TracerThreadArgument
*tracer_thread_argument
=
280 (TracerThreadArgument
*)argument
;
282 internal_prctl(PR_SET_PDEATHSIG
, SIGKILL
, 0, 0, 0);
283 // Check if parent is already dead.
284 if (internal_getppid() != tracer_thread_argument
->parent_pid
)
287 // Wait for the parent thread to finish preparations.
288 tracer_thread_argument
->mutex
.Lock();
289 tracer_thread_argument
->mutex
.Unlock();
291 RAW_CHECK(AddDieCallback(TracerThreadDieCallback
));
293 ThreadSuspender
thread_suspender(internal_getppid(), tracer_thread_argument
);
294 // Global pointer for the signal handler.
295 thread_suspender_instance
= &thread_suspender
;
297 // Alternate stack for signal handling.
298 InternalMmapVector
<char> handler_stack_memory(kHandlerStackSize
);
299 stack_t handler_stack
;
300 internal_memset(&handler_stack
, 0, sizeof(handler_stack
));
301 handler_stack
.ss_sp
= handler_stack_memory
.data();
302 handler_stack
.ss_size
= kHandlerStackSize
;
303 internal_sigaltstack(&handler_stack
, nullptr);
305 // Install our handler for synchronous signals. Other signals should be
306 // blocked by the mask we inherited from the parent thread.
307 for (uptr i
= 0; i
< ARRAY_SIZE(kSyncSignals
); i
++) {
308 __sanitizer_sigaction act
;
309 internal_memset(&act
, 0, sizeof(act
));
310 act
.sigaction
= TracerThreadSignalHandler
;
311 act
.sa_flags
= SA_ONSTACK
| SA_SIGINFO
;
312 internal_sigaction_norestorer(kSyncSignals
[i
], &act
, 0);
316 if (!thread_suspender
.SuspendAllThreads()) {
317 VReport(1, "Failed suspending threads.\n");
320 tracer_thread_argument
->callback(thread_suspender
.suspended_threads_list(),
321 tracer_thread_argument
->callback_argument
);
322 thread_suspender
.ResumeAllThreads();
325 RAW_CHECK(RemoveDieCallback(TracerThreadDieCallback
));
326 thread_suspender_instance
= nullptr;
327 atomic_store(&tracer_thread_argument
->done
, 1, memory_order_relaxed
);
331 class ScopedStackSpaceWithGuard
{
333 explicit ScopedStackSpaceWithGuard(uptr stack_size
) {
334 stack_size_
= stack_size
;
335 guard_size_
= GetPageSizeCached();
336 // FIXME: Omitting MAP_STACK here works in current kernels but might break
338 guard_start_
= (uptr
)MmapOrDie(stack_size_
+ guard_size_
,
339 "ScopedStackWithGuard");
340 CHECK(MprotectNoAccess((uptr
)guard_start_
, guard_size_
));
342 ~ScopedStackSpaceWithGuard() {
343 UnmapOrDie((void *)guard_start_
, stack_size_
+ guard_size_
);
345 void *Bottom() const {
346 return (void *)(guard_start_
+ stack_size_
+ guard_size_
);
355 // We have a limitation on the stack frame size, so some stuff had to be moved
357 static __sanitizer_sigset_t blocked_sigset
;
358 static __sanitizer_sigset_t old_sigset
;
360 class StopTheWorldScope
{
362 StopTheWorldScope() {
363 // Make this process dumpable. Processes that are not dumpable cannot be
365 process_was_dumpable_
= internal_prctl(PR_GET_DUMPABLE
, 0, 0, 0, 0);
366 if (!process_was_dumpable_
)
367 internal_prctl(PR_SET_DUMPABLE
, 1, 0, 0, 0);
370 ~StopTheWorldScope() {
371 // Restore the dumpable flag.
372 if (!process_was_dumpable_
)
373 internal_prctl(PR_SET_DUMPABLE
, 0, 0, 0, 0);
377 int process_was_dumpable_
;
380 // When sanitizer output is being redirected to file (i.e. by using log_path),
381 // the tracer should write to the parent's log instead of trying to open a new
382 // file. Alert the logging code to the fact that we have a tracer.
383 struct ScopedSetTracerPID
{
384 explicit ScopedSetTracerPID(uptr tracer_pid
) {
385 stoptheworld_tracer_pid
= tracer_pid
;
386 stoptheworld_tracer_ppid
= internal_getpid();
388 ~ScopedSetTracerPID() {
389 stoptheworld_tracer_pid
= 0;
390 stoptheworld_tracer_ppid
= 0;
394 void StopTheWorld(StopTheWorldCallback callback
, void *argument
) {
395 StopTheWorldScope in_stoptheworld
;
396 // Prepare the arguments for TracerThread.
397 struct TracerThreadArgument tracer_thread_argument
;
398 tracer_thread_argument
.callback
= callback
;
399 tracer_thread_argument
.callback_argument
= argument
;
400 tracer_thread_argument
.parent_pid
= internal_getpid();
401 atomic_store(&tracer_thread_argument
.done
, 0, memory_order_relaxed
);
402 const uptr kTracerStackSize
= 2 * 1024 * 1024;
403 ScopedStackSpaceWithGuard
tracer_stack(kTracerStackSize
);
404 // Block the execution of TracerThread until after we have set ptrace
406 tracer_thread_argument
.mutex
.Lock();
407 // Signal handling story.
408 // We don't want async signals to be delivered to the tracer thread,
409 // so we block all async signals before creating the thread. An async signal
410 // handler can temporary modify errno, which is shared with this thread.
411 // We ought to use pthread_sigmask here, because sigprocmask has undefined
412 // behavior in multithreaded programs. However, on linux sigprocmask is
413 // equivalent to pthread_sigmask with the exception that pthread_sigmask
414 // does not allow to block some signals used internally in pthread
415 // implementation. We are fine with blocking them here, we are really not
416 // going to pthread_cancel the thread.
417 // The tracer thread should not raise any synchronous signals. But in case it
418 // does, we setup a special handler for sync signals that properly kills the
419 // parent as well. Note: we don't pass CLONE_SIGHAND to clone, so handlers
420 // in the tracer thread won't interfere with user program. Double note: if a
421 // user does something along the lines of 'kill -11 pid', that can kill the
422 // process even if user setup own handler for SEGV.
423 // Thing to watch out for: this code should not change behavior of user code
424 // in any observable way. In particular it should not override user signal
426 internal_sigfillset(&blocked_sigset
);
427 for (uptr i
= 0; i
< ARRAY_SIZE(kSyncSignals
); i
++)
428 internal_sigdelset(&blocked_sigset
, kSyncSignals
[i
]);
429 int rv
= internal_sigprocmask(SIG_BLOCK
, &blocked_sigset
, &old_sigset
);
431 uptr tracer_pid
= internal_clone(
432 TracerThread
, tracer_stack
.Bottom(),
433 CLONE_VM
| CLONE_FS
| CLONE_FILES
| CLONE_UNTRACED
,
434 &tracer_thread_argument
, nullptr /* parent_tidptr */,
435 nullptr /* newtls */, nullptr /* child_tidptr */);
436 internal_sigprocmask(SIG_SETMASK
, &old_sigset
, 0);
438 if (internal_iserror(tracer_pid
, &local_errno
)) {
439 VReport(1, "Failed spawning a tracer thread (errno %d).\n", local_errno
);
440 tracer_thread_argument
.mutex
.Unlock();
442 ScopedSetTracerPID
scoped_set_tracer_pid(tracer_pid
);
443 // On some systems we have to explicitly declare that we want to be traced
444 // by the tracer thread.
445 internal_prctl(PR_SET_PTRACER
, tracer_pid
, 0, 0, 0);
446 // Allow the tracer thread to start.
447 tracer_thread_argument
.mutex
.Unlock();
448 // NOTE: errno is shared between this thread and the tracer thread.
449 // internal_waitpid() may call syscall() which can access/spoil errno,
450 // so we can't call it now. Instead we for the tracer thread to finish using
451 // the spin loop below. Man page for sched_yield() says "In the Linux
452 // implementation, sched_yield() always succeeds", so let's hope it does not
453 // spoil errno. Note that this spin loop runs only for brief periods before
454 // the tracer thread has suspended us and when it starts unblocking threads.
455 while (atomic_load(&tracer_thread_argument
.done
, memory_order_relaxed
) == 0)
457 // Now the tracer thread is about to exit and does not touch errno,
460 uptr waitpid_status
= internal_waitpid(tracer_pid
, nullptr, __WALL
);
461 if (!internal_iserror(waitpid_status
, &local_errno
))
463 if (local_errno
== EINTR
)
465 VReport(1, "Waiting on the tracer thread failed (errno %d).\n",
472 // Platform-specific methods from SuspendedThreadsList.
473 #if SANITIZER_ANDROID && defined(__arm__)
474 typedef pt_regs regs_struct
;
475 #define REG_SP ARM_sp
477 #elif SANITIZER_LINUX && defined(__arm__)
478 typedef user_regs regs_struct
;
479 #define REG_SP uregs[13]
481 #elif defined(__i386__) || defined(__x86_64__)
482 typedef user_regs_struct regs_struct
;
483 #if defined(__i386__)
488 #define ARCH_IOVEC_FOR_GETREGSET
489 // Support ptrace extensions even when compiled without required kernel support
490 #ifndef NT_X86_XSTATE
491 #define NT_X86_XSTATE 0x202
493 // Compiler may use FP registers to store pointers.
494 static constexpr uptr kExtraRegs
[] = {NT_X86_XSTATE
, NT_FPREGSET
};
496 #elif defined(__powerpc__) || defined(__powerpc64__)
497 typedef pt_regs regs_struct
;
498 #define REG_SP gpr[PT_R1]
500 #elif defined(__mips__)
501 typedef struct user regs_struct
;
502 # if SANITIZER_ANDROID
503 # define REG_SP regs[EF_R29]
505 # define REG_SP regs[EF_REG29]
508 #elif defined(__aarch64__)
509 typedef struct user_pt_regs regs_struct
;
511 static constexpr uptr kExtraRegs
[] = {0};
512 #define ARCH_IOVEC_FOR_GETREGSET
514 #elif SANITIZER_RISCV64
515 typedef struct user_regs_struct regs_struct
;
517 static constexpr uptr kExtraRegs
[] = {0};
518 #define ARCH_IOVEC_FOR_GETREGSET
520 #elif defined(__s390__)
521 typedef _user_regs_struct regs_struct
;
522 #define REG_SP gprs[15]
523 static constexpr uptr kExtraRegs
[] = {0};
524 #define ARCH_IOVEC_FOR_GETREGSET
527 #error "Unsupported architecture"
528 #endif // SANITIZER_ANDROID && defined(__arm__)
530 tid_t
SuspendedThreadsListLinux::GetThreadID(uptr index
) const {
531 CHECK_LT(index
, thread_ids_
.size());
532 return thread_ids_
[index
];
535 uptr
SuspendedThreadsListLinux::ThreadCount() const {
536 return thread_ids_
.size();
539 bool SuspendedThreadsListLinux::ContainsTid(tid_t thread_id
) const {
540 for (uptr i
= 0; i
< thread_ids_
.size(); i
++) {
541 if (thread_ids_
[i
] == thread_id
) return true;
546 void SuspendedThreadsListLinux::Append(tid_t tid
) {
547 thread_ids_
.push_back(tid
);
550 PtraceRegistersStatus
SuspendedThreadsListLinux::GetRegistersAndSP(
551 uptr index
, InternalMmapVector
<uptr
> *buffer
, uptr
*sp
) const {
552 pid_t tid
= GetThreadID(index
);
553 constexpr uptr uptr_sz
= sizeof(uptr
);
555 #ifdef ARCH_IOVEC_FOR_GETREGSET
556 auto append
= [&](uptr regset
) {
557 uptr size
= buffer
->size();
558 // NT_X86_XSTATE requires 64bit alignment.
559 uptr size_up
= RoundUpTo(size
, 8 / uptr_sz
);
560 buffer
->reserve(Max
<uptr
>(1024, size_up
));
561 struct iovec regset_io
;
562 for (;; buffer
->resize(buffer
->capacity() * 2)) {
563 buffer
->resize(buffer
->capacity());
564 uptr available_bytes
= (buffer
->size() - size_up
) * uptr_sz
;
565 regset_io
.iov_base
= buffer
->data() + size_up
;
566 regset_io
.iov_len
= available_bytes
;
568 internal_iserror(internal_ptrace(PTRACE_GETREGSET
, tid
,
569 (void *)regset
, (void *)®set_io
),
572 VReport(1, "Could not get regset %p from thread %d (errno %d).\n",
573 (void *)regset
, tid
, pterrno
);
574 buffer
->resize(size
);
578 // Far enough from the buffer size, no need to resize and repeat.
579 if (regset_io
.iov_len
+ 64 < available_bytes
)
582 buffer
->resize(size_up
+ RoundUpTo(regset_io
.iov_len
, uptr_sz
) / uptr_sz
);
587 bool fail
= !append(NT_PRSTATUS
);
589 // Accept the first available and do not report errors.
590 for (uptr regs
: kExtraRegs
)
591 if (regs
&& append(regs
))
595 buffer
->resize(RoundUpTo(sizeof(regs_struct
), uptr_sz
) / uptr_sz
);
596 bool fail
= internal_iserror(
597 internal_ptrace(PTRACE_GETREGS
, tid
, nullptr, buffer
->data()), &pterrno
);
599 VReport(1, "Could not get registers from thread %d (errno %d).\n", tid
,
603 // ESRCH means that the given thread is not suspended or already dead.
604 // Therefore it's unsafe to inspect its data (e.g. walk through stack) and
605 // we should notify caller about this.
606 return pterrno
== ESRCH
? REGISTERS_UNAVAILABLE_FATAL
607 : REGISTERS_UNAVAILABLE
;
610 *sp
= reinterpret_cast<regs_struct
*>(buffer
->data())[0].REG_SP
;
611 return REGISTERS_AVAILABLE
;
614 } // namespace __sanitizer
616 #endif // SANITIZER_LINUX && (defined(__x86_64__) || defined(__mips__)
617 // || defined(__aarch64__) || defined(__powerpc64__)
618 // || defined(__s390__) || defined(__i386__) || defined(__arm__)