1 //===-- tsan_platform_linux.cpp -------------------------------------------===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is a part of ThreadSanitizer (TSan), a race detector.
11 // Linux- and BSD-specific code.
12 //===----------------------------------------------------------------------===//
14 #include "sanitizer_common/sanitizer_platform.h"
15 #if SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD
17 #include "sanitizer_common/sanitizer_common.h"
18 #include "sanitizer_common/sanitizer_libc.h"
19 #include "sanitizer_common/sanitizer_linux.h"
20 #include "sanitizer_common/sanitizer_platform_limits_netbsd.h"
21 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
22 #include "sanitizer_common/sanitizer_posix.h"
23 #include "sanitizer_common/sanitizer_procmaps.h"
24 #include "sanitizer_common/sanitizer_stackdepot.h"
25 #include "sanitizer_common/sanitizer_stoptheworld.h"
26 #include "tsan_flags.h"
27 #include "tsan_platform.h"
39 #include <sys/personality.h>
42 #include <sys/syscall.h>
43 #include <sys/socket.h>
45 #include <sys/types.h>
46 #include <sys/resource.h>
52 #define __need_res_state
65 extern "C" void *__libc_stack_end
;
66 void *__libc_stack_end
= 0;
69 #if SANITIZER_LINUX && defined(__aarch64__) && !SANITIZER_GO
70 # define INIT_LONGJMP_XOR_KEY 1
72 # define INIT_LONGJMP_XOR_KEY 0
75 #if INIT_LONGJMP_XOR_KEY
76 #include "interception/interception.h"
77 // Must be declared outside of other namespaces.
78 DECLARE_REAL(int, _setjmp
, void *env
)
83 #if INIT_LONGJMP_XOR_KEY
84 static void InitializeLongjmpXorKey();
85 static uptr longjmp_xor_key
;
88 // Runtime detected VMA size.
103 void FillProfileCallback(uptr p
, uptr rss
, bool file
,
104 uptr
*mem
, uptr stats_size
) {
105 mem
[MemTotal
] += rss
;
106 if (p
>= ShadowBeg() && p
< ShadowEnd())
107 mem
[MemShadow
] += rss
;
108 else if (p
>= MetaShadowBeg() && p
< MetaShadowEnd())
110 else if ((p
>= LoAppMemBeg() && p
< LoAppMemEnd()) ||
111 (p
>= MidAppMemBeg() && p
< MidAppMemEnd()) ||
112 (p
>= HiAppMemBeg() && p
< HiAppMemEnd()))
113 mem
[file
? MemFile
: MemMmap
] += rss
;
114 else if (p
>= HeapMemBeg() && p
< HeapMemEnd())
116 else if (p
>= TraceMemBeg() && p
< TraceMemEnd())
117 mem
[MemTrace
] += rss
;
119 mem
[MemOther
] += rss
;
122 void WriteMemoryProfile(char *buf
, uptr buf_size
, u64 uptime_ns
) {
124 internal_memset(mem
, 0, sizeof(mem
));
125 GetMemoryProfile(FillProfileCallback
, mem
, MemCount
);
126 auto meta
= ctx
->metamap
.GetMemoryStats();
127 StackDepotStats stacks
= StackDepotGetStats();
129 ctx
->thread_registry
.GetNumberOfThreads(&nthread
, &nlive
);
130 uptr internal_stats
[AllocatorStatCount
];
131 internal_allocator()->GetStats(internal_stats
);
132 // All these are allocated from the common mmap region.
133 mem
[MemMmap
] -= meta
.mem_block
+ meta
.sync_obj
+ stacks
.allocated
+
134 internal_stats
[AllocatorStatMapped
];
135 if (s64(mem
[MemMmap
]) < 0)
139 "%llus: RSS %zd MB: shadow:%zd meta:%zd file:%zd mmap:%zd"
140 " trace:%zd heap:%zd other:%zd intalloc:%zd memblocks:%zd syncobj:%zu"
141 " stacks=%zd[%zd] nthr=%zd/%zd\n",
142 uptime_ns
/ (1000 * 1000 * 1000), mem
[MemTotal
] >> 20,
143 mem
[MemShadow
] >> 20, mem
[MemMeta
] >> 20, mem
[MemFile
] >> 20,
144 mem
[MemMmap
] >> 20, mem
[MemTrace
] >> 20, mem
[MemHeap
] >> 20,
145 mem
[MemOther
] >> 20, internal_stats
[AllocatorStatMapped
] >> 20,
146 meta
.mem_block
>> 20, meta
.sync_obj
>> 20, stacks
.allocated
>> 20,
147 stacks
.n_uniq_ids
, nlive
, nthread
);
151 void FlushShadowMemoryCallback(
152 const SuspendedThreadsList
&suspended_threads_list
,
154 ReleaseMemoryPagesToOS(ShadowBeg(), ShadowEnd());
158 void FlushShadowMemory() {
160 StopTheWorld(FlushShadowMemoryCallback
, 0);
165 // Mark shadow for .rodata sections with the special kShadowRodata marker.
166 // Accesses to .rodata can't race, so this saves time, memory and trace space.
167 static void MapRodata() {
168 // First create temp file.
169 const char *tmpdir
= GetEnv("TMPDIR");
171 tmpdir
= GetEnv("TEST_TMPDIR");
179 internal_snprintf(name
, sizeof(name
), "%s/tsan.rodata.%d",
180 tmpdir
, (int)internal_getpid());
181 uptr openrv
= internal_open(name
, O_RDWR
| O_CREAT
| O_EXCL
, 0600);
182 if (internal_iserror(openrv
))
184 internal_unlink(name
); // Unlink it now, so that we can reuse the buffer.
186 // Fill the file with kShadowRodata.
187 const uptr kMarkerSize
= 512 * 1024 / sizeof(RawShadow
);
188 InternalMmapVector
<RawShadow
> marker(kMarkerSize
);
189 // volatile to prevent insertion of memset
190 for (volatile RawShadow
*p
= marker
.data(); p
< marker
.data() + kMarkerSize
;
193 internal_write(fd
, marker
.data(), marker
.size() * sizeof(RawShadow
));
194 // Map the file into memory.
195 uptr page
= internal_mmap(0, GetPageSizeCached(), PROT_READ
| PROT_WRITE
,
196 MAP_PRIVATE
| MAP_ANONYMOUS
, fd
, 0);
197 if (internal_iserror(page
)) {
201 // Map the file into shadow of .rodata sections.
202 MemoryMappingLayout
proc_maps(/*cache_enabled*/true);
203 // Reusing the buffer 'name'.
204 MemoryMappedSegment
segment(name
, ARRAY_SIZE(name
));
205 while (proc_maps
.Next(&segment
)) {
206 if (segment
.filename
[0] != 0 && segment
.filename
[0] != '[' &&
207 segment
.IsReadable() && segment
.IsExecutable() &&
208 !segment
.IsWritable() && IsAppMem(segment
.start
)) {
209 // Assume it's .rodata
210 char *shadow_start
= (char *)MemToShadow(segment
.start
);
211 char *shadow_end
= (char *)MemToShadow(segment
.end
);
212 for (char *p
= shadow_start
; p
< shadow_end
;
213 p
+= marker
.size() * sizeof(RawShadow
)) {
215 p
, Min
<uptr
>(marker
.size() * sizeof(RawShadow
), shadow_end
- p
),
216 PROT_READ
, MAP_PRIVATE
| MAP_FIXED
, fd
, 0);
223 void InitializeShadowMemoryPlatform() {
227 #endif // #if !SANITIZER_GO
229 void InitializePlatformEarly() {
231 (MostSignificantSetBitIndex(GET_CURRENT_FRAME()) + 1);
232 #if defined(__aarch64__)
234 if (vmaSize
!= 39 && vmaSize
!= 42 && vmaSize
!= 48) {
235 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
236 Printf("FATAL: Found %zd - Supported 39, 42 and 48\n", vmaSize
);
241 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
242 Printf("FATAL: Found %zd - Supported 48\n", vmaSize
);
246 #elif defined(__powerpc64__)
248 if (vmaSize
!= 44 && vmaSize
!= 46 && vmaSize
!= 47) {
249 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
250 Printf("FATAL: Found %zd - Supported 44, 46, and 47\n", vmaSize
);
254 if (vmaSize
!= 46 && vmaSize
!= 47) {
255 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
256 Printf("FATAL: Found %zd - Supported 46, and 47\n", vmaSize
);
260 #elif defined(__mips64)
263 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
264 Printf("FATAL: Found %zd - Supported 40\n", vmaSize
);
269 Printf("FATAL: ThreadSanitizer: unsupported VMA range\n");
270 Printf("FATAL: Found %zd - Supported 47\n", vmaSize
);
277 void InitializePlatform() {
278 DisableCoreDumperIfNecessary();
280 // Go maps shadow memory lazily and works fine with limited address space.
281 // Unlimited stack is not a problem as well, because the executable
282 // is not compiled with -pie.
286 // TSan doesn't play well with unlimited stack size (as stack
287 // overlaps with shadow memory). If we detect unlimited stack size,
288 // we re-exec the program with limited stack size as a best effort.
289 if (StackSizeIsUnlimited()) {
290 const uptr kMaxStackSize
= 32 * 1024 * 1024;
291 VReport(1, "Program is run with unlimited stack size, which wouldn't "
292 "work with ThreadSanitizer.\n"
293 "Re-execing with stack size limited to %zd bytes.\n",
295 SetStackSizeLimitInBytes(kMaxStackSize
);
299 if (!AddressSpaceIsUnlimited()) {
300 Report("WARNING: Program is run with limited virtual address space,"
301 " which wouldn't work with ThreadSanitizer.\n");
302 Report("Re-execing with unlimited virtual address space.\n");
303 SetAddressSpaceUnlimited();
306 #if SANITIZER_LINUX && defined(__aarch64__)
307 // After patch "arm64: mm: support ARCH_MMAP_RND_BITS." is introduced in
308 // linux kernel, the random gap between stack and mapped area is increased
309 // from 128M to 36G on 39-bit aarch64. As it is almost impossible to cover
310 // this big range, we should disable randomized virtual space on aarch64.
311 int old_personality
= personality(0xffffffff);
312 if (old_personality
!= -1 && (old_personality
& ADDR_NO_RANDOMIZE
) == 0) {
313 VReport(1, "WARNING: Program is run with randomized virtual address "
314 "space, which wouldn't work with ThreadSanitizer.\n"
315 "Re-execing with fixed virtual address space.\n");
316 CHECK_NE(personality(old_personality
| ADDR_NO_RANDOMIZE
), -1);
319 // Initialize the xor key used in {sig}{set,long}jump.
320 InitializeLongjmpXorKey();
328 #endif // !SANITIZER_GO
332 // Extract file descriptors passed to glibc internal __res_iclose function.
333 // This is required to properly "close" the fds, because we do not see internal
334 // closes within glibc. The code is a pure hack.
335 int ExtractResolvFDs(void *state
, int *fds
, int nfd
) {
336 #if SANITIZER_LINUX && !SANITIZER_ANDROID
338 struct __res_state
*statp
= (struct __res_state
*)state
;
339 for (int i
= 0; i
< MAXNS
&& cnt
< nfd
; i
++) {
340 if (statp
->_u
._ext
.nsaddrs
[i
] && statp
->_u
._ext
.nssocks
[i
] != -1)
341 fds
[cnt
++] = statp
->_u
._ext
.nssocks
[i
];
349 // Extract file descriptors passed via UNIX domain sockets.
350 // This is required to properly handle "open" of these fds.
351 // see 'man recvmsg' and 'man 3 cmsg'.
352 int ExtractRecvmsgFDs(void *msgp
, int *fds
, int nfd
) {
354 msghdr
*msg
= (msghdr
*)msgp
;
355 struct cmsghdr
*cmsg
= CMSG_FIRSTHDR(msg
);
356 for (; cmsg
; cmsg
= CMSG_NXTHDR(msg
, cmsg
)) {
357 if (cmsg
->cmsg_level
!= SOL_SOCKET
|| cmsg
->cmsg_type
!= SCM_RIGHTS
)
359 int n
= (cmsg
->cmsg_len
- CMSG_LEN(0)) / sizeof(fds
[0]);
360 for (int i
= 0; i
< n
; i
++) {
361 fds
[res
++] = ((int*)CMSG_DATA(cmsg
))[i
];
369 // Reverse operation of libc stack pointer mangling
370 static uptr
UnmangleLongJmpSp(uptr mangled_sp
) {
371 #if defined(__x86_64__)
374 // xor %fs:0x30, %rsi
377 asm("ror $0x11, %0 \n"
378 "xor %%fs:0x30, %0 \n"
385 #elif defined(__aarch64__)
387 return mangled_sp
^ longjmp_xor_key
;
391 #elif defined(__powerpc64__)
393 // ld r4, -28696(r13)
396 asm("ld %0, -28696(%%r13)" : "=r" (xor_key
));
397 return mangled_sp
^ xor_key
;
398 #elif defined(__mips__)
400 #elif defined(__s390x__)
401 // tcbhead_t.stack_guard
402 uptr xor_key
= ((uptr
*)__builtin_thread_pointer())[5];
403 return mangled_sp
^ xor_key
;
405 #error "Unknown platform"
411 # define LONG_JMP_SP_ENV_SLOT 6
415 #elif defined(__powerpc__)
416 # define LONG_JMP_SP_ENV_SLOT 0
417 #elif SANITIZER_FREEBSD
418 # define LONG_JMP_SP_ENV_SLOT 2
419 #elif SANITIZER_LINUX
421 # define LONG_JMP_SP_ENV_SLOT 13
422 # elif defined(__mips64)
423 # define LONG_JMP_SP_ENV_SLOT 1
424 # elif defined(__s390x__)
425 # define LONG_JMP_SP_ENV_SLOT 9
427 # define LONG_JMP_SP_ENV_SLOT 6
431 uptr
ExtractLongJmpSp(uptr
*env
) {
432 uptr mangled_sp
= env
[LONG_JMP_SP_ENV_SLOT
];
433 return UnmangleLongJmpSp(mangled_sp
);
436 #if INIT_LONGJMP_XOR_KEY
437 // GLIBC mangles the function pointers in jmp_buf (used in {set,long}*jmp
438 // functions) by XORing them with a random key. For AArch64 it is a global
439 // variable rather than a TCB one (as for x86_64/powerpc). We obtain the key by
440 // issuing a setjmp and XORing the SP pointer values to derive the key.
441 static void InitializeLongjmpXorKey() {
442 // 1. Call REAL(setjmp), which stores the mangled SP in env.
446 // 2. Retrieve vanilla/mangled SP.
448 asm("mov %0, sp" : "=r" (sp
));
449 uptr mangled_sp
= ((uptr
*)&env
)[LONG_JMP_SP_ENV_SLOT
];
451 // 3. xor SPs to obtain key.
452 longjmp_xor_key
= mangled_sp
^ sp
;
456 extern "C" void __tsan_tls_initialization() {}
458 void ImitateTlsWrite(ThreadState
*thr
, uptr tls_addr
, uptr tls_size
) {
459 // Check that the thr object is in tls;
460 const uptr thr_beg
= (uptr
)thr
;
461 const uptr thr_end
= (uptr
)thr
+ sizeof(*thr
);
462 CHECK_GE(thr_beg
, tls_addr
);
463 CHECK_LE(thr_beg
, tls_addr
+ tls_size
);
464 CHECK_GE(thr_end
, tls_addr
);
465 CHECK_LE(thr_end
, tls_addr
+ tls_size
);
466 // Since the thr object is huge, skip it.
467 const uptr pc
= StackTrace::GetNextInstructionPc(
468 reinterpret_cast<uptr
>(__tsan_tls_initialization
));
469 MemoryRangeImitateWrite(thr
, pc
, tls_addr
, thr_beg
- tls_addr
);
470 MemoryRangeImitateWrite(thr
, pc
, thr_end
, tls_addr
+ tls_size
- thr_end
);
473 // Note: this function runs with async signals enabled,
474 // so it must not touch any tsan state.
475 int call_pthread_cancel_with_cleanup(int (*fn
)(void *arg
),
476 void (*cleanup
)(void *arg
), void *arg
) {
477 // pthread_cleanup_push/pop are hardcore macros mess.
478 // We can't intercept nor call them w/o including pthread.h.
480 pthread_cleanup_push(cleanup
, arg
);
482 pthread_cleanup_pop(0);
485 #endif // !SANITIZER_GO
488 void ReplaceSystemMalloc() { }
492 #if SANITIZER_ANDROID
493 // On Android, one thread can call intercepted functions after
494 // DestroyThreadState(), so add a fake thread state for "dead" threads.
495 static ThreadState
*dead_thread_state
= nullptr;
497 ThreadState
*cur_thread() {
498 ThreadState
* thr
= reinterpret_cast<ThreadState
*>(*get_android_tls_ptr());
499 if (thr
== nullptr) {
500 __sanitizer_sigset_t emptyset
;
501 internal_sigfillset(&emptyset
);
502 __sanitizer_sigset_t oldset
;
503 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK
, &emptyset
, &oldset
));
504 thr
= reinterpret_cast<ThreadState
*>(*get_android_tls_ptr());
505 if (thr
== nullptr) {
506 thr
= reinterpret_cast<ThreadState
*>(MmapOrDie(sizeof(ThreadState
),
508 *get_android_tls_ptr() = reinterpret_cast<uptr
>(thr
);
509 if (dead_thread_state
== nullptr) {
510 dead_thread_state
= reinterpret_cast<ThreadState
*>(
511 MmapOrDie(sizeof(ThreadState
), "ThreadState"));
512 dead_thread_state
->fast_state
.SetIgnoreBit();
513 dead_thread_state
->ignore_interceptors
= 1;
514 dead_thread_state
->is_dead
= true;
515 *const_cast<u32
*>(&dead_thread_state
->tid
) = -1;
516 CHECK_EQ(0, internal_mprotect(dead_thread_state
, sizeof(ThreadState
),
520 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK
, &oldset
, nullptr));
525 void set_cur_thread(ThreadState
*thr
) {
526 *get_android_tls_ptr() = reinterpret_cast<uptr
>(thr
);
529 void cur_thread_finalize() {
530 __sanitizer_sigset_t emptyset
;
531 internal_sigfillset(&emptyset
);
532 __sanitizer_sigset_t oldset
;
533 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK
, &emptyset
, &oldset
));
534 ThreadState
* thr
= reinterpret_cast<ThreadState
*>(*get_android_tls_ptr());
535 if (thr
!= dead_thread_state
) {
536 *get_android_tls_ptr() = reinterpret_cast<uptr
>(dead_thread_state
);
537 UnmapOrDie(thr
, sizeof(ThreadState
));
539 CHECK_EQ(0, internal_sigprocmask(SIG_SETMASK
, &oldset
, nullptr));
541 #endif // SANITIZER_ANDROID
542 #endif // if !SANITIZER_GO
544 } // namespace __tsan
546 #endif // SANITIZER_LINUX || SANITIZER_FREEBSD || SANITIZER_NETBSD