1 //===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
9 // This file is a part of ThreadSanitizer (TSan), a race detector.
11 // Platform-specific code.
12 //===----------------------------------------------------------------------===//
14 #ifndef TSAN_PLATFORM_H
15 #define TSAN_PLATFORM_H
17 #if !defined(__LP64__) && !defined(_WIN64)
18 # error "Only 64-bit is supported"
21 #include "sanitizer_common/sanitizer_common.h"
22 #include "tsan_defs.h"
27 // App memory is not mapped onto shadow memory range.
28 kBrokenMapping
= 1 << 0,
29 // Mapping app memory and back does not produce the same address,
30 // this can lead to wrong addresses in reports and potentially
31 // other bad consequences.
32 kBrokenReverseMapping
= 1 << 1,
33 // Mapping is non-linear for linear user range.
34 // This is bad and can lead to unpredictable memory corruptions, etc
35 // because range access functions assume linearity.
36 kBrokenLinearity
= 1 << 2,
37 // Meta for an app region overlaps with the meta of another app region.
38 // This is determined by recomputing the individual meta regions for
41 // N.B. There is no "kBrokenReverseMetaMapping" constant because there
42 // is no MetaToMem function. However, note that (!kBrokenLinearity
43 // && !kBrokenAliasedMetas) implies that MemToMeta is invertible.
44 kBrokenAliasedMetas
= 1 << 3,
48 C/C++ on linux/x86_64 and freebsd/x86_64
49 0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
50 0040 0000 0000 - 0100 0000 0000: -
51 0100 0000 0000 - 1000 0000 0000: shadow
52 1000 0000 0000 - 3000 0000 0000: -
53 3000 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)
54 3400 0000 0000 - 5500 0000 0000: -
55 5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
56 5680 0000 0000 - 7d00 0000 0000: -
57 7b00 0000 0000 - 7c00 0000 0000: heap
58 7c00 0000 0000 - 7e80 0000 0000: -
59 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
61 C/C++ on netbsd/amd64 can reuse the same mapping:
62 * The address space starts from 0x1000 (option with 0x0) and ends with
64 * LoAppMem-kHeapMemEnd can be reused as it is.
66 * No MidAppMem region.
67 * No additional HeapMem region.
68 * HiAppMem contains the stack, loader, shared libraries and heap.
69 * Stack on NetBSD/amd64 has prereserved 128MB.
70 * Heap grows downwards (top-down).
71 * ASLR must be disabled per-process or globally.
73 struct Mapping48AddressSpace
{
74 static const uptr kMetaShadowBeg
= 0x300000000000ull
;
75 static const uptr kMetaShadowEnd
= 0x340000000000ull
;
76 static const uptr kShadowBeg
= 0x010000000000ull
;
77 static const uptr kShadowEnd
= 0x100000000000ull
;
78 static const uptr kHeapMemBeg
= 0x7b0000000000ull
;
79 static const uptr kHeapMemEnd
= 0x7c0000000000ull
;
80 static const uptr kLoAppMemBeg
= 0x000000001000ull
;
81 static const uptr kLoAppMemEnd
= 0x008000000000ull
;
82 static const uptr kMidAppMemBeg
= 0x550000000000ull
;
83 static const uptr kMidAppMemEnd
= 0x568000000000ull
;
84 static const uptr kHiAppMemBeg
= 0x7e8000000000ull
;
85 static const uptr kHiAppMemEnd
= 0x800000000000ull
;
86 static const uptr kShadowMsk
= 0x780000000000ull
;
87 static const uptr kShadowXor
= 0x040000000000ull
;
88 static const uptr kShadowAdd
= 0x000000000000ull
;
89 static const uptr kVdsoBeg
= 0xf000000000000000ull
;
93 C/C++ on linux/mips64 (40-bit VMA)
94 0000 0000 00 - 0100 0000 00: - (4 GB)
95 0100 0000 00 - 0200 0000 00: main binary (4 GB)
96 0200 0000 00 - 1200 0000 00: - (64 GB)
97 1200 0000 00 - 2200 0000 00: shadow (64 GB)
98 2200 0000 00 - 4000 0000 00: - (120 GB)
99 4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB)
100 5000 0000 00 - aa00 0000 00: - (360 GB)
101 aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB)
102 ab00 0000 00 - fe00 0000 00: - (332 GB)
103 fe00 0000 00 - ff00 0000 00: heap (4 GB)
104 ff00 0000 00 - ff80 0000 00: - (2 GB)
105 ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB)
107 struct MappingMips64_40
{
108 static const uptr kMetaShadowBeg
= 0x4000000000ull
;
109 static const uptr kMetaShadowEnd
= 0x5000000000ull
;
110 static const uptr kShadowBeg
= 0x1200000000ull
;
111 static const uptr kShadowEnd
= 0x2200000000ull
;
112 static const uptr kHeapMemBeg
= 0xfe00000000ull
;
113 static const uptr kHeapMemEnd
= 0xff00000000ull
;
114 static const uptr kLoAppMemBeg
= 0x0100000000ull
;
115 static const uptr kLoAppMemEnd
= 0x0200000000ull
;
116 static const uptr kMidAppMemBeg
= 0xaa00000000ull
;
117 static const uptr kMidAppMemEnd
= 0xab00000000ull
;
118 static const uptr kHiAppMemBeg
= 0xff80000000ull
;
119 static const uptr kHiAppMemEnd
= 0xffffffffffull
;
120 static const uptr kShadowMsk
= 0xf800000000ull
;
121 static const uptr kShadowXor
= 0x0800000000ull
;
122 static const uptr kShadowAdd
= 0x0000000000ull
;
123 static const uptr kVdsoBeg
= 0xfffff00000ull
;
127 C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
128 0000 0000 00 - 0100 0000 00: - (4 GB)
129 0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB)
130 0200 0000 00 - 0300 0000 00: heap (4 GB)
131 0300 0000 00 - 0400 0000 00: - (4 GB)
132 0400 0000 00 - 0800 0000 00: shadow memory (16 GB)
133 0800 0000 00 - 0d00 0000 00: - (20 GB)
134 0d00 0000 00 - 0e00 0000 00: metainfo (4 GB)
135 0e00 0000 00 - 1000 0000 00: -
137 struct MappingAppleAarch64
{
138 static const uptr kLoAppMemBeg
= 0x0100000000ull
;
139 static const uptr kLoAppMemEnd
= 0x0200000000ull
;
140 static const uptr kHeapMemBeg
= 0x0200000000ull
;
141 static const uptr kHeapMemEnd
= 0x0300000000ull
;
142 static const uptr kShadowBeg
= 0x0400000000ull
;
143 static const uptr kShadowEnd
= 0x0800000000ull
;
144 static const uptr kMetaShadowBeg
= 0x0d00000000ull
;
145 static const uptr kMetaShadowEnd
= 0x0e00000000ull
;
146 static const uptr kHiAppMemBeg
= 0x0fc0000000ull
;
147 static const uptr kHiAppMemEnd
= 0x0fc0000000ull
;
148 static const uptr kShadowMsk
= 0x0ull
;
149 static const uptr kShadowXor
= 0x0ull
;
150 static const uptr kShadowAdd
= 0x0200000000ull
;
151 static const uptr kVdsoBeg
= 0x7000000000000000ull
;
152 static const uptr kMidAppMemBeg
= 0;
153 static const uptr kMidAppMemEnd
= 0;
157 C/C++ on linux/aarch64 (39-bit VMA)
158 0000 0010 00 - 0500 0000 00: main binary (20 GB)
159 0100 0000 00 - 2000 0000 00: -
160 2000 0000 00 - 4000 0000 00: shadow memory (128 GB)
161 4000 0000 00 - 4800 0000 00: metainfo (32 GB)
162 4800 0000 00 - 5500 0000 00: -
163 5500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB)
164 5600 0000 00 - 7c00 0000 00: -
165 7a00 0000 00 - 7d00 0000 00: heap (12 GB)
166 7d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB)
168 struct MappingAarch64_39
{
169 static const uptr kLoAppMemBeg
= 0x0000001000ull
;
170 static const uptr kLoAppMemEnd
= 0x0500000000ull
;
171 static const uptr kShadowBeg
= 0x2000000000ull
;
172 static const uptr kShadowEnd
= 0x4000000000ull
;
173 static const uptr kMetaShadowBeg
= 0x4000000000ull
;
174 static const uptr kMetaShadowEnd
= 0x4800000000ull
;
175 static const uptr kMidAppMemBeg
= 0x5500000000ull
;
176 static const uptr kMidAppMemEnd
= 0x5a00000000ull
;
177 static const uptr kHeapMemBeg
= 0x7a00000000ull
;
178 static const uptr kHeapMemEnd
= 0x7d00000000ull
;
179 static const uptr kHiAppMemBeg
= 0x7d00000000ull
;
180 static const uptr kHiAppMemEnd
= 0x7fffffffffull
;
181 static const uptr kShadowMsk
= 0x7000000000ull
;
182 static const uptr kShadowXor
= 0x1000000000ull
;
183 static const uptr kShadowAdd
= 0x0000000000ull
;
184 static const uptr kVdsoBeg
= 0x7f00000000ull
;
188 C/C++ on linux/aarch64 (42-bit VMA)
189 00000 0010 00 - 02000 0000 00: main binary (128 GB)
190 02000 0000 00 - 08000 0000 00: -
191 10000 0000 00 - 20000 0000 00: shadow memory (1024 GB)
192 20000 0000 00 - 24000 0000 00: metainfo (256 GB)
193 24000 0000 00 - 2aa00 0000 00: -
194 2aa00 0000 00 - 2c000 0000 00: main binary (PIE) (88 GB)
195 2c000 0000 00 - 3c000 0000 00: -
196 3c000 0000 00 - 3f000 0000 00: heap (192 GB)
197 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack (64 GB)
199 struct MappingAarch64_42
{
200 static const uptr kLoAppMemBeg
= 0x00000001000ull
;
201 static const uptr kLoAppMemEnd
= 0x02000000000ull
;
202 static const uptr kShadowBeg
= 0x10000000000ull
;
203 static const uptr kShadowEnd
= 0x20000000000ull
;
204 static const uptr kMetaShadowBeg
= 0x20000000000ull
;
205 static const uptr kMetaShadowEnd
= 0x24000000000ull
;
206 static const uptr kMidAppMemBeg
= 0x2aa00000000ull
;
207 static const uptr kMidAppMemEnd
= 0x2c000000000ull
;
208 static const uptr kHeapMemBeg
= 0x3c000000000ull
;
209 static const uptr kHeapMemEnd
= 0x3f000000000ull
;
210 static const uptr kHiAppMemBeg
= 0x3f000000000ull
;
211 static const uptr kHiAppMemEnd
= 0x3ffffffffffull
;
212 static const uptr kShadowMsk
= 0x38000000000ull
;
213 static const uptr kShadowXor
= 0x08000000000ull
;
214 static const uptr kShadowAdd
= 0x00000000000ull
;
215 static const uptr kVdsoBeg
= 0x37f00000000ull
;
219 C/C++ on linux/aarch64 (48-bit VMA)
220 0000 0000 1000 - 0a00 0000 0000: main binary (10240 GB)
221 0a00 0000 1000 - 1554 0000 0000: -
222 1554 0000 1000 - 5400 0000 0000: shadow memory (64176 GB)
223 5400 0000 1000 - 8000 0000 0000: -
224 8000 0000 1000 - 0a00 0000 0000: metainfo (32768 GB)
225 a000 0000 1000 - aaaa 0000 0000: -
226 aaaa 0000 1000 - ac00 0000 0000: main binary (PIE) (1368 GB)
227 ac00 0000 1000 - fc00 0000 0000: -
228 fc00 0000 1000 - ffff ffff ffff: modules and main thread stack (4096 GB)
230 N.B. the shadow memory region has a strange start address, because it
231 contains the shadows for the mid, high and low app regions (in this
234 struct MappingAarch64_48
{
235 static const uptr kLoAppMemBeg
= 0x0000000001000ull
;
236 static const uptr kLoAppMemEnd
= 0x00a0000000000ull
;
237 static const uptr kShadowBeg
= 0x0155400000000ull
;
238 static const uptr kShadowEnd
= 0x0540000000000ull
;
239 static const uptr kMetaShadowBeg
= 0x0800000000000ull
;
240 static const uptr kMetaShadowEnd
= 0x0a00000000000ull
;
241 static const uptr kMidAppMemBeg
= 0x0aaaa00000000ull
;
242 static const uptr kMidAppMemEnd
= 0x0ac0000000000ull
;
243 static const uptr kHiAppMemBeg
= 0x0fc0000000000ull
;
244 static const uptr kHiAppMemEnd
= 0x1000000000000ull
;
245 static const uptr kHeapMemBeg
= 0x0fc0000000000ull
;
246 static const uptr kHeapMemEnd
= 0x0fc0000000000ull
;
247 static const uptr kShadowMsk
= 0x0c00000000000ull
;
248 static const uptr kShadowXor
= 0x0200000000000ull
;
249 static const uptr kShadowAdd
= 0x0000000000000ull
;
250 static const uptr kVdsoBeg
= 0xffff000000000ull
;
253 /* C/C++ on linux/loongarch64 (47-bit VMA)
254 0000 0000 4000 - 0080 0000 0000: main binary
255 0080 0000 0000 - 0100 0000 0000: -
256 0100 0000 0000 - 1000 0000 0000: shadow memory
257 1000 0000 0000 - 3000 0000 0000: -
258 3000 0000 0000 - 3400 0000 0000: metainfo
259 3400 0000 0000 - 5555 0000 0000: -
260 5555 0000 0000 - 5556 0000 0000: main binary (PIE)
261 5556 0000 0000 - 7ffe 0000 0000: -
262 7ffe 0000 0000 - 7fff 0000 0000: heap
263 7fff 0000 0000 - 7fff 8000 0000: -
264 7fff 8000 0000 - 8000 0000 0000: modules and main thread stack
266 struct MappingLoongArch64_47
{
267 static const uptr kMetaShadowBeg
= 0x300000000000ull
;
268 static const uptr kMetaShadowEnd
= 0x340000000000ull
;
269 static const uptr kShadowBeg
= 0x010000000000ull
;
270 static const uptr kShadowEnd
= 0x100000000000ull
;
271 static const uptr kHeapMemBeg
= 0x7ffe00000000ull
;
272 static const uptr kHeapMemEnd
= 0x7fff00000000ull
;
273 static const uptr kLoAppMemBeg
= 0x000000004000ull
;
274 static const uptr kLoAppMemEnd
= 0x008000000000ull
;
275 static const uptr kMidAppMemBeg
= 0x555500000000ull
;
276 static const uptr kMidAppMemEnd
= 0x555600000000ull
;
277 static const uptr kHiAppMemBeg
= 0x7fff80000000ull
;
278 static const uptr kHiAppMemEnd
= 0x800000000000ull
;
279 static const uptr kShadowMsk
= 0x780000000000ull
;
280 static const uptr kShadowXor
= 0x040000000000ull
;
281 static const uptr kShadowAdd
= 0x000000000000ull
;
282 static const uptr kVdsoBeg
= 0x7fffffffc000ull
;
286 C/C++ on linux/powerpc64 (44-bit VMA)
287 0000 0000 0100 - 0001 0000 0000: main binary
288 0001 0000 0000 - 0001 0000 0000: -
289 0001 0000 0000 - 0b00 0000 0000: shadow
290 0b00 0000 0000 - 0b00 0000 0000: -
291 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
292 0d00 0000 0000 - 0f00 0000 0000: -
293 0f00 0000 0000 - 0f50 0000 0000: heap
294 0f50 0000 0000 - 0f60 0000 0000: -
295 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
297 struct MappingPPC64_44
{
298 static const uptr kBroken
= kBrokenMapping
| kBrokenReverseMapping
|
299 kBrokenLinearity
| kBrokenAliasedMetas
;
300 static const uptr kMetaShadowBeg
= 0x0b0000000000ull
;
301 static const uptr kMetaShadowEnd
= 0x0d0000000000ull
;
302 static const uptr kShadowBeg
= 0x000100000000ull
;
303 static const uptr kShadowEnd
= 0x0b0000000000ull
;
304 static const uptr kLoAppMemBeg
= 0x000000000100ull
;
305 static const uptr kLoAppMemEnd
= 0x000100000000ull
;
306 static const uptr kHeapMemBeg
= 0x0f0000000000ull
;
307 static const uptr kHeapMemEnd
= 0x0f5000000000ull
;
308 static const uptr kHiAppMemBeg
= 0x0f6000000000ull
;
309 static const uptr kHiAppMemEnd
= 0x100000000000ull
; // 44 bits
310 static const uptr kShadowMsk
= 0x0f0000000000ull
;
311 static const uptr kShadowXor
= 0x002100000000ull
;
312 static const uptr kShadowAdd
= 0x000000000000ull
;
313 static const uptr kVdsoBeg
= 0x3c0000000000000ull
;
314 static const uptr kMidAppMemBeg
= 0;
315 static const uptr kMidAppMemEnd
= 0;
319 C/C++ on linux/powerpc64 (46-bit VMA)
320 0000 0000 1000 - 0100 0000 0000: main binary
321 0100 0000 0000 - 0200 0000 0000: -
322 0100 0000 0000 - 0800 0000 0000: shadow
323 0800 0000 0000 - 1000 0000 0000: -
324 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
325 1200 0000 0000 - 3d00 0000 0000: -
326 3d00 0000 0000 - 3e00 0000 0000: heap
327 3e00 0000 0000 - 3e80 0000 0000: -
328 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
330 struct MappingPPC64_46
{
331 static const uptr kMetaShadowBeg
= 0x100000000000ull
;
332 static const uptr kMetaShadowEnd
= 0x120000000000ull
;
333 static const uptr kShadowBeg
= 0x010000000000ull
;
334 static const uptr kShadowEnd
= 0x080000000000ull
;
335 static const uptr kHeapMemBeg
= 0x3d0000000000ull
;
336 static const uptr kHeapMemEnd
= 0x3e0000000000ull
;
337 static const uptr kLoAppMemBeg
= 0x000000001000ull
;
338 static const uptr kLoAppMemEnd
= 0x010000000000ull
;
339 static const uptr kHiAppMemBeg
= 0x3e8000000000ull
;
340 static const uptr kHiAppMemEnd
= 0x400000000000ull
; // 46 bits
341 static const uptr kShadowMsk
= 0x3c0000000000ull
;
342 static const uptr kShadowXor
= 0x020000000000ull
;
343 static const uptr kShadowAdd
= 0x000000000000ull
;
344 static const uptr kVdsoBeg
= 0x7800000000000000ull
;
345 static const uptr kMidAppMemBeg
= 0;
346 static const uptr kMidAppMemEnd
= 0;
350 C/C++ on linux/powerpc64 (47-bit VMA)
351 0000 0000 1000 - 0100 0000 0000: main binary
352 0100 0000 0000 - 0200 0000 0000: -
353 0100 0000 0000 - 0800 0000 0000: shadow
354 0800 0000 0000 - 1000 0000 0000: -
355 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
356 1200 0000 0000 - 7d00 0000 0000: -
357 7d00 0000 0000 - 7e00 0000 0000: heap
358 7e00 0000 0000 - 7e80 0000 0000: -
359 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
361 struct MappingPPC64_47
{
362 static const uptr kMetaShadowBeg
= 0x100000000000ull
;
363 static const uptr kMetaShadowEnd
= 0x120000000000ull
;
364 static const uptr kShadowBeg
= 0x010000000000ull
;
365 static const uptr kShadowEnd
= 0x080000000000ull
;
366 static const uptr kHeapMemBeg
= 0x7d0000000000ull
;
367 static const uptr kHeapMemEnd
= 0x7e0000000000ull
;
368 static const uptr kLoAppMemBeg
= 0x000000001000ull
;
369 static const uptr kLoAppMemEnd
= 0x010000000000ull
;
370 static const uptr kHiAppMemBeg
= 0x7e8000000000ull
;
371 static const uptr kHiAppMemEnd
= 0x800000000000ull
; // 47 bits
372 static const uptr kShadowMsk
= 0x7c0000000000ull
;
373 static const uptr kShadowXor
= 0x020000000000ull
;
374 static const uptr kShadowAdd
= 0x000000000000ull
;
375 static const uptr kVdsoBeg
= 0x7800000000000000ull
;
376 static const uptr kMidAppMemBeg
= 0;
377 static const uptr kMidAppMemEnd
= 0;
382 While the kernel provides a 64-bit address space, we have to restrict ourselves
383 to 48 bits due to how e.g. SyncVar::GetId() works.
384 0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
385 0e00 0000 0000 - 2000 0000 0000: -
386 2000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app)
387 4000 0000 0000 - 9000 0000 0000: -
388 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
389 9800 0000 0000 - be00 0000 0000: -
390 be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
392 struct MappingS390x
{
393 static const uptr kMetaShadowBeg
= 0x900000000000ull
;
394 static const uptr kMetaShadowEnd
= 0x980000000000ull
;
395 static const uptr kShadowBeg
= 0x200000000000ull
;
396 static const uptr kShadowEnd
= 0x400000000000ull
;
397 static const uptr kHeapMemBeg
= 0xbe0000000000ull
;
398 static const uptr kHeapMemEnd
= 0xc00000000000ull
;
399 static const uptr kLoAppMemBeg
= 0x000000001000ull
;
400 static const uptr kLoAppMemEnd
= 0x0e0000000000ull
;
401 static const uptr kHiAppMemBeg
= 0xc00000004000ull
;
402 static const uptr kHiAppMemEnd
= 0xc00000004000ull
;
403 static const uptr kShadowMsk
= 0xb00000000000ull
;
404 static const uptr kShadowXor
= 0x100000000000ull
;
405 static const uptr kShadowAdd
= 0x000000000000ull
;
406 static const uptr kVdsoBeg
= 0xfffffffff000ull
;
407 static const uptr kMidAppMemBeg
= 0;
408 static const uptr kMidAppMemEnd
= 0;
411 /* Go on linux, darwin and freebsd on x86_64
412 0000 0000 1000 - 0000 1000 0000: executable
413 0000 1000 0000 - 00c0 0000 0000: -
414 00c0 0000 0000 - 00e0 0000 0000: heap
415 00e0 0000 0000 - 2000 0000 0000: -
416 2000 0000 0000 - 21c0 0000 0000: shadow
417 21c0 0000 0000 - 3000 0000 0000: -
418 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
419 4000 0000 0000 - 8000 0000 0000: -
423 static const uptr kMetaShadowBeg
= 0x300000000000ull
;
424 static const uptr kMetaShadowEnd
= 0x400000000000ull
;
425 static const uptr kShadowBeg
= 0x200000000000ull
;
426 static const uptr kShadowEnd
= 0x21c000000000ull
;
427 static const uptr kLoAppMemBeg
= 0x000000001000ull
;
428 static const uptr kLoAppMemEnd
= 0x00e000000000ull
;
429 static const uptr kMidAppMemBeg
= 0;
430 static const uptr kMidAppMemEnd
= 0;
431 static const uptr kHiAppMemBeg
= 0;
432 static const uptr kHiAppMemEnd
= 0;
433 static const uptr kHeapMemBeg
= 0;
434 static const uptr kHeapMemEnd
= 0;
435 static const uptr kVdsoBeg
= 0;
436 static const uptr kShadowMsk
= 0;
437 static const uptr kShadowXor
= 0;
438 static const uptr kShadowAdd
= 0x200000000000ull
;
442 0000 0000 1000 - 0000 1000 0000: executable
443 0000 1000 0000 - 00f8 0000 0000: -
444 00c0 0000 0000 - 00e0 0000 0000: heap
445 00e0 0000 0000 - 0100 0000 0000: -
446 0100 0000 0000 - 0300 0000 0000: shadow
447 0300 0000 0000 - 0700 0000 0000: -
448 0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
449 07d0 0000 0000 - 8000 0000 0000: -
450 PIE binaries currently not supported, but it should be theoretically possible.
453 struct MappingGoWindows
{
454 static const uptr kMetaShadowBeg
= 0x070000000000ull
;
455 static const uptr kMetaShadowEnd
= 0x077000000000ull
;
456 static const uptr kShadowBeg
= 0x010000000000ull
;
457 static const uptr kShadowEnd
= 0x030000000000ull
;
458 static const uptr kLoAppMemBeg
= 0x000000001000ull
;
459 static const uptr kLoAppMemEnd
= 0x00e000000000ull
;
460 static const uptr kMidAppMemBeg
= 0;
461 static const uptr kMidAppMemEnd
= 0;
462 static const uptr kHiAppMemBeg
= 0;
463 static const uptr kHiAppMemEnd
= 0;
464 static const uptr kHeapMemBeg
= 0;
465 static const uptr kHeapMemEnd
= 0;
466 static const uptr kVdsoBeg
= 0;
467 static const uptr kShadowMsk
= 0;
468 static const uptr kShadowXor
= 0;
469 static const uptr kShadowAdd
= 0x010000000000ull
;
472 /* Go on linux/powerpc64 (46-bit VMA)
473 0000 0000 1000 - 0000 1000 0000: executable
474 0000 1000 0000 - 00c0 0000 0000: -
475 00c0 0000 0000 - 00e0 0000 0000: heap
476 00e0 0000 0000 - 2000 0000 0000: -
477 2000 0000 0000 - 21c0 0000 0000: shadow
478 21c0 0000 0000 - 2400 0000 0000: -
479 2400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects)
480 2470 0000 0000 - 4000 0000 0000: -
483 struct MappingGoPPC64_46
{
484 static const uptr kMetaShadowBeg
= 0x240000000000ull
;
485 static const uptr kMetaShadowEnd
= 0x247000000000ull
;
486 static const uptr kShadowBeg
= 0x200000000000ull
;
487 static const uptr kShadowEnd
= 0x21c000000000ull
;
488 static const uptr kLoAppMemBeg
= 0x000000001000ull
;
489 static const uptr kLoAppMemEnd
= 0x00e000000000ull
;
490 static const uptr kMidAppMemBeg
= 0;
491 static const uptr kMidAppMemEnd
= 0;
492 static const uptr kHiAppMemBeg
= 0;
493 static const uptr kHiAppMemEnd
= 0;
494 static const uptr kHeapMemBeg
= 0;
495 static const uptr kHeapMemEnd
= 0;
496 static const uptr kVdsoBeg
= 0;
497 static const uptr kShadowMsk
= 0;
498 static const uptr kShadowXor
= 0;
499 static const uptr kShadowAdd
= 0x200000000000ull
;
502 /* Go on linux/powerpc64 (47-bit VMA)
503 0000 0000 1000 - 0000 1000 0000: executable
504 0000 1000 0000 - 00c0 0000 0000: -
505 00c0 0000 0000 - 00e0 0000 0000: heap
506 00e0 0000 0000 - 2000 0000 0000: -
507 2000 0000 0000 - 2800 0000 0000: shadow
508 2800 0000 0000 - 3000 0000 0000: -
509 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
510 3200 0000 0000 - 8000 0000 0000: -
513 struct MappingGoPPC64_47
{
514 static const uptr kMetaShadowBeg
= 0x300000000000ull
;
515 static const uptr kMetaShadowEnd
= 0x320000000000ull
;
516 static const uptr kShadowBeg
= 0x200000000000ull
;
517 static const uptr kShadowEnd
= 0x280000000000ull
;
518 static const uptr kLoAppMemBeg
= 0x000000001000ull
;
519 static const uptr kLoAppMemEnd
= 0x00e000000000ull
;
520 static const uptr kMidAppMemBeg
= 0;
521 static const uptr kMidAppMemEnd
= 0;
522 static const uptr kHiAppMemBeg
= 0;
523 static const uptr kHiAppMemEnd
= 0;
524 static const uptr kHeapMemBeg
= 0;
525 static const uptr kHeapMemEnd
= 0;
526 static const uptr kVdsoBeg
= 0;
527 static const uptr kShadowMsk
= 0;
528 static const uptr kShadowXor
= 0;
529 static const uptr kShadowAdd
= 0x200000000000ull
;
532 /* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
533 0000 0000 1000 - 0000 1000 0000: executable
534 0000 1000 0000 - 00c0 0000 0000: -
535 00c0 0000 0000 - 00e0 0000 0000: heap
536 00e0 0000 0000 - 2000 0000 0000: -
537 2000 0000 0000 - 2800 0000 0000: shadow
538 2800 0000 0000 - 3000 0000 0000: -
539 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
540 3200 0000 0000 - 8000 0000 0000: -
542 struct MappingGoAarch64
{
543 static const uptr kMetaShadowBeg
= 0x300000000000ull
;
544 static const uptr kMetaShadowEnd
= 0x320000000000ull
;
545 static const uptr kShadowBeg
= 0x200000000000ull
;
546 static const uptr kShadowEnd
= 0x280000000000ull
;
547 static const uptr kLoAppMemBeg
= 0x000000001000ull
;
548 static const uptr kLoAppMemEnd
= 0x00e000000000ull
;
549 static const uptr kMidAppMemBeg
= 0;
550 static const uptr kMidAppMemEnd
= 0;
551 static const uptr kHiAppMemBeg
= 0;
552 static const uptr kHiAppMemEnd
= 0;
553 static const uptr kHeapMemBeg
= 0;
554 static const uptr kHeapMemEnd
= 0;
555 static const uptr kVdsoBeg
= 0;
556 static const uptr kShadowMsk
= 0;
557 static const uptr kShadowXor
= 0;
558 static const uptr kShadowAdd
= 0x200000000000ull
;
562 Go on linux/mips64 (47-bit VMA)
563 0000 0000 1000 - 0000 1000 0000: executable
564 0000 1000 0000 - 00c0 0000 0000: -
565 00c0 0000 0000 - 00e0 0000 0000: heap
566 00e0 0000 0000 - 2000 0000 0000: -
567 2000 0000 0000 - 2800 0000 0000: shadow
568 2800 0000 0000 - 3000 0000 0000: -
569 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
570 3200 0000 0000 - 8000 0000 0000: -
572 struct MappingGoMips64_47
{
573 static const uptr kMetaShadowBeg
= 0x300000000000ull
;
574 static const uptr kMetaShadowEnd
= 0x320000000000ull
;
575 static const uptr kShadowBeg
= 0x200000000000ull
;
576 static const uptr kShadowEnd
= 0x280000000000ull
;
577 static const uptr kLoAppMemBeg
= 0x000000001000ull
;
578 static const uptr kLoAppMemEnd
= 0x00e000000000ull
;
579 static const uptr kMidAppMemBeg
= 0;
580 static const uptr kMidAppMemEnd
= 0;
581 static const uptr kHiAppMemBeg
= 0;
582 static const uptr kHiAppMemEnd
= 0;
583 static const uptr kHeapMemBeg
= 0;
584 static const uptr kHeapMemEnd
= 0;
585 static const uptr kVdsoBeg
= 0;
586 static const uptr kShadowMsk
= 0;
587 static const uptr kShadowXor
= 0;
588 static const uptr kShadowAdd
= 0x200000000000ull
;
593 0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
594 1000 0000 0000 - 4000 0000 0000: -
595 4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app)
596 6000 0000 0000 - 9000 0000 0000: -
597 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
599 struct MappingGoS390x
{
600 static const uptr kMetaShadowBeg
= 0x900000000000ull
;
601 static const uptr kMetaShadowEnd
= 0x980000000000ull
;
602 static const uptr kShadowBeg
= 0x400000000000ull
;
603 static const uptr kShadowEnd
= 0x600000000000ull
;
604 static const uptr kLoAppMemBeg
= 0x000000001000ull
;
605 static const uptr kLoAppMemEnd
= 0x100000000000ull
;
606 static const uptr kMidAppMemBeg
= 0;
607 static const uptr kMidAppMemEnd
= 0;
608 static const uptr kHiAppMemBeg
= 0;
609 static const uptr kHiAppMemEnd
= 0;
610 static const uptr kHeapMemBeg
= 0;
611 static const uptr kHeapMemEnd
= 0;
612 static const uptr kVdsoBeg
= 0;
613 static const uptr kShadowMsk
= 0;
614 static const uptr kShadowXor
= 0;
615 static const uptr kShadowAdd
= 0x400000000000ull
;
620 template <typename Func
, typename Arg
>
621 ALWAYS_INLINE
auto SelectMapping(Arg arg
) {
623 # if defined(__powerpc64__)
626 return Func::template Apply
<MappingGoPPC64_46
>(arg
);
628 return Func::template Apply
<MappingGoPPC64_47
>(arg
);
630 # elif defined(__mips64)
631 return Func::template Apply
<MappingGoMips64_47
>(arg
);
632 # elif defined(__s390x__)
633 return Func::template Apply
<MappingGoS390x
>(arg
);
634 # elif defined(__aarch64__)
635 return Func::template Apply
<MappingGoAarch64
>(arg
);
636 # elif SANITIZER_WINDOWS
637 return Func::template Apply
<MappingGoWindows
>(arg
);
639 return Func::template Apply
<MappingGo48
>(arg
);
641 #else // SANITIZER_GO
642 # if SANITIZER_IOS && !SANITIZER_IOSSIM
643 return Func::template Apply
<MappingAppleAarch64
>(arg
);
644 # elif defined(__x86_64__) || SANITIZER_APPLE
645 return Func::template Apply
<Mapping48AddressSpace
>(arg
);
646 # elif defined(__aarch64__)
649 return Func::template Apply
<MappingAarch64_39
>(arg
);
651 return Func::template Apply
<MappingAarch64_42
>(arg
);
653 return Func::template Apply
<MappingAarch64_48
>(arg
);
655 # elif SANITIZER_LOONGARCH64
656 return Func::template Apply
<MappingLoongArch64_47
>(arg
);
657 # elif defined(__powerpc64__)
660 return Func::template Apply
<MappingPPC64_44
>(arg
);
662 return Func::template Apply
<MappingPPC64_46
>(arg
);
664 return Func::template Apply
<MappingPPC64_47
>(arg
);
666 # elif defined(__mips64)
667 return Func::template Apply
<MappingMips64_40
>(arg
);
668 # elif defined(__s390x__)
669 return Func::template Apply
<MappingS390x
>(arg
);
671 # error "unsupported platform"
677 template <typename Func
>
678 void ForEachMapping() {
679 Func::template Apply
<Mapping48AddressSpace
>();
680 Func::template Apply
<MappingMips64_40
>();
681 Func::template Apply
<MappingAppleAarch64
>();
682 Func::template Apply
<MappingAarch64_39
>();
683 Func::template Apply
<MappingAarch64_42
>();
684 Func::template Apply
<MappingAarch64_48
>();
685 Func::template Apply
<MappingLoongArch64_47
>();
686 Func::template Apply
<MappingPPC64_44
>();
687 Func::template Apply
<MappingPPC64_46
>();
688 Func::template Apply
<MappingPPC64_47
>();
689 Func::template Apply
<MappingS390x
>();
690 Func::template Apply
<MappingGo48
>();
691 Func::template Apply
<MappingGoWindows
>();
692 Func::template Apply
<MappingGoPPC64_46
>();
693 Func::template Apply
<MappingGoPPC64_47
>();
694 Func::template Apply
<MappingGoAarch64
>();
695 Func::template Apply
<MappingGoMips64_47
>();
696 Func::template Apply
<MappingGoS390x
>();
715 struct MappingField
{
716 template <typename Mapping
>
717 static uptr
Apply(MappingType type
) {
720 return Mapping::kLoAppMemBeg
;
722 return Mapping::kLoAppMemEnd
;
724 return Mapping::kMidAppMemBeg
;
726 return Mapping::kMidAppMemEnd
;
728 return Mapping::kHiAppMemBeg
;
730 return Mapping::kHiAppMemEnd
;
732 return Mapping::kHeapMemBeg
;
734 return Mapping::kHeapMemEnd
;
736 return Mapping::kVdsoBeg
;
738 return Mapping::kShadowBeg
;
740 return Mapping::kShadowEnd
;
742 return Mapping::kMetaShadowBeg
;
744 return Mapping::kMetaShadowEnd
;
751 uptr
LoAppMemBeg(void) { return SelectMapping
<MappingField
>(kLoAppMemBeg
); }
753 uptr
LoAppMemEnd(void) { return SelectMapping
<MappingField
>(kLoAppMemEnd
); }
756 uptr
MidAppMemBeg(void) { return SelectMapping
<MappingField
>(kMidAppMemBeg
); }
758 uptr
MidAppMemEnd(void) { return SelectMapping
<MappingField
>(kMidAppMemEnd
); }
761 uptr
HeapMemBeg(void) { return SelectMapping
<MappingField
>(kHeapMemBeg
); }
763 uptr
HeapMemEnd(void) { return SelectMapping
<MappingField
>(kHeapMemEnd
); }
766 uptr
HiAppMemBeg(void) { return SelectMapping
<MappingField
>(kHiAppMemBeg
); }
768 uptr
HiAppMemEnd(void) { return SelectMapping
<MappingField
>(kHiAppMemEnd
); }
771 uptr
VdsoBeg(void) { return SelectMapping
<MappingField
>(kVdsoBeg
); }
774 uptr
ShadowBeg(void) { return SelectMapping
<MappingField
>(kShadowBeg
); }
776 uptr
ShadowEnd(void) { return SelectMapping
<MappingField
>(kShadowEnd
); }
779 uptr
MetaShadowBeg(void) { return SelectMapping
<MappingField
>(kMetaShadowBeg
); }
781 uptr
MetaShadowEnd(void) { return SelectMapping
<MappingField
>(kMetaShadowEnd
); }
783 struct IsAppMemImpl
{
784 template <typename Mapping
>
785 static bool Apply(uptr mem
) {
786 return (mem
>= Mapping::kHeapMemBeg
&& mem
< Mapping::kHeapMemEnd
) ||
787 (mem
>= Mapping::kMidAppMemBeg
&& mem
< Mapping::kMidAppMemEnd
) ||
788 (mem
>= Mapping::kLoAppMemBeg
&& mem
< Mapping::kLoAppMemEnd
) ||
789 (mem
>= Mapping::kHiAppMemBeg
&& mem
< Mapping::kHiAppMemEnd
);
794 bool IsAppMem(uptr mem
) { return SelectMapping
<IsAppMemImpl
>(mem
); }
796 struct IsShadowMemImpl
{
797 template <typename Mapping
>
798 static bool Apply(uptr mem
) {
799 return mem
>= Mapping::kShadowBeg
&& mem
<= Mapping::kShadowEnd
;
804 bool IsShadowMem(RawShadow
*p
) {
805 return SelectMapping
<IsShadowMemImpl
>(reinterpret_cast<uptr
>(p
));
808 struct IsMetaMemImpl
{
809 template <typename Mapping
>
810 static bool Apply(uptr mem
) {
811 return mem
>= Mapping::kMetaShadowBeg
&& mem
<= Mapping::kMetaShadowEnd
;
816 bool IsMetaMem(const u32
*p
) {
817 return SelectMapping
<IsMetaMemImpl
>(reinterpret_cast<uptr
>(p
));
820 struct MemToShadowImpl
{
821 template <typename Mapping
>
822 static uptr
Apply(uptr x
) {
823 DCHECK(IsAppMemImpl::Apply
<Mapping
>(x
));
824 return (((x
) & ~(Mapping::kShadowMsk
| (kShadowCell
- 1))) ^
825 Mapping::kShadowXor
) *
832 RawShadow
*MemToShadow(uptr x
) {
833 return reinterpret_cast<RawShadow
*>(SelectMapping
<MemToShadowImpl
>(x
));
836 struct MemToMetaImpl
{
837 template <typename Mapping
>
838 static u32
*Apply(uptr x
) {
839 DCHECK(IsAppMemImpl::Apply
<Mapping
>(x
));
840 return (u32
*)(((((x
) & ~(Mapping::kShadowMsk
| (kMetaShadowCell
- 1)))) /
841 kMetaShadowCell
* kMetaShadowSize
) |
842 Mapping::kMetaShadowBeg
);
847 u32
*MemToMeta(uptr x
) { return SelectMapping
<MemToMetaImpl
>(x
); }
849 struct ShadowToMemImpl
{
850 template <typename Mapping
>
851 static uptr
Apply(uptr sp
) {
852 if (!IsShadowMemImpl::Apply
<Mapping
>(sp
))
854 // The shadow mapping is non-linear and we've lost some bits, so we don't
855 // have an easy way to restore the original app address. But the mapping is
856 // a bijection, so we try to restore the address as belonging to
857 // low/mid/high range consecutively and see if shadow->app->shadow mapping
858 // gives us the same address.
860 ((sp
- Mapping::kShadowAdd
) / kShadowMultiplier
) ^ Mapping::kShadowXor
;
861 if (p
>= Mapping::kLoAppMemBeg
&& p
< Mapping::kLoAppMemEnd
&&
862 MemToShadowImpl::Apply
<Mapping
>(p
) == sp
)
864 if (Mapping::kMidAppMemBeg
) {
865 uptr p_mid
= p
+ (Mapping::kMidAppMemBeg
& Mapping::kShadowMsk
);
866 if (p_mid
>= Mapping::kMidAppMemBeg
&& p_mid
< Mapping::kMidAppMemEnd
&&
867 MemToShadowImpl::Apply
<Mapping
>(p_mid
) == sp
)
870 return p
| Mapping::kShadowMsk
;
875 uptr
ShadowToMem(RawShadow
*s
) {
876 return SelectMapping
<ShadowToMemImpl
>(reinterpret_cast<uptr
>(s
));
879 // Compresses addr to kCompressedAddrBits stored in least significant bits.
880 ALWAYS_INLINE uptr
CompressAddr(uptr addr
) {
881 return addr
& ((1ull << kCompressedAddrBits
) - 1);
884 struct RestoreAddrImpl
{
886 template <typename Mapping
>
887 static Result
Apply(uptr addr
) {
888 // To restore the address we go over all app memory ranges and check if top
889 // 3 bits of the compressed addr match that of the app range. If yes, we
890 // assume that the compressed address come from that range and restore the
891 // missing top bits to match the app range address.
892 const uptr ranges
[] = {
893 Mapping::kLoAppMemBeg
, Mapping::kLoAppMemEnd
, Mapping::kMidAppMemBeg
,
894 Mapping::kMidAppMemEnd
, Mapping::kHiAppMemBeg
, Mapping::kHiAppMemEnd
,
895 Mapping::kHeapMemBeg
, Mapping::kHeapMemEnd
,
897 const uptr indicator
= 0x0e0000000000ull
;
898 const uptr ind_lsb
= 1ull << LeastSignificantSetBitIndex(indicator
);
899 for (uptr i
= 0; i
< ARRAY_SIZE(ranges
); i
+= 2) {
900 uptr beg
= ranges
[i
];
901 uptr end
= ranges
[i
+ 1];
904 for (uptr p
= beg
; p
< end
; p
= RoundDown(p
+ ind_lsb
, ind_lsb
)) {
905 if ((addr
& indicator
) == (p
& indicator
))
906 return addr
| (p
& ~(ind_lsb
- 1));
909 Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr
);
914 // Restores compressed addr from kCompressedAddrBits to full representation.
915 // This is called only during reporting and is not performance-critical.
916 inline uptr
RestoreAddr(uptr addr
) {
917 return SelectMapping
<RestoreAddrImpl
>(addr
);
920 void InitializePlatform();
921 void InitializePlatformEarly();
922 void CheckAndProtect();
923 void InitializeShadowMemoryPlatform();
924 void WriteMemoryProfile(char *buf
, uptr buf_size
, u64 uptime_ns
);
925 int ExtractResolvFDs(void *state
, int *fds
, int nfd
);
926 int ExtractRecvmsgFDs(void *msg
, int *fds
, int nfd
);
927 uptr
ExtractLongJmpSp(uptr
*env
);
928 void ImitateTlsWrite(ThreadState
*thr
, uptr tls_addr
, uptr tls_size
);
930 int call_pthread_cancel_with_cleanup(int (*fn
)(void *arg
),
931 void (*cleanup
)(void *arg
), void *arg
);
933 void DestroyThreadState();
934 void PlatformCleanUpThreadState(ThreadState
*thr
);
936 } // namespace __tsan
938 #endif // TSAN_PLATFORM_H