libsanitizer/tsan/tsan_platform.h

   1 //===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of ThreadSanitizer (TSan), a race detector.
  10 //
  11 // Platform-specific code.
  12 //===----------------------------------------------------------------------===//
  13
  14 #ifndef TSAN_PLATFORM_H
  15 #define TSAN_PLATFORM_H
  16
  17 #if !defined(__LP64__) && !defined(_WIN64)
  18 # error "Only 64-bit is supported"
  19 #endif
  20
  21 #include "sanitizer_common/sanitizer_common.h"
  22 #include "tsan_defs.h"
  23
  24 namespace __tsan {
  25
  26 enum {
  27   // App memory is not mapped onto shadow memory range.
  28   kBrokenMapping = 1 << 0,
  29   // Mapping app memory and back does not produce the same address,
  30   // this can lead to wrong addresses in reports and potentially
  31   // other bad consequences.
  32   kBrokenReverseMapping = 1 << 1,
  33   // Mapping is non-linear for linear user range.
  34   // This is bad and can lead to unpredictable memory corruptions, etc
  35   // because range access functions assume linearity.
  36   kBrokenLinearity = 1 << 2,
  37 };
  38
  39 /*
  40 C/C++ on linux/x86_64 and freebsd/x86_64
  41 0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
  42 0040 0000 0000 - 0100 0000 0000: -
  43 0100 0000 0000 - 1000 0000 0000: shadow
  44 1000 0000 0000 - 3000 0000 0000: -
  45 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
  46 4000 0000 0000 - 5500 0000 0000: -
  47 5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
  48 5680 0000 0000 - 7d00 0000 0000: -
  49 7b00 0000 0000 - 7c00 0000 0000: heap
  50 7c00 0000 0000 - 7e80 0000 0000: -
  51 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
  52
  53 C/C++ on netbsd/amd64 can reuse the same mapping:
  54  * The address space starts from 0x1000 (option with 0x0) and ends with
  55    0x7f7ffffff000.
  56  * LoAppMem-kHeapMemEnd can be reused as it is.
  57  * No VDSO support.
  58  * No MidAppMem region.
  59  * No additional HeapMem region.
  60  * HiAppMem contains the stack, loader, shared libraries and heap.
  61  * Stack on NetBSD/amd64 has prereserved 128MB.
  62  * Heap grows downwards (top-down).
  63  * ASLR must be disabled per-process or globally.
  64 */
  65 struct Mapping48AddressSpace {
  66   static const uptr kMetaShadowBeg = 0x300000000000ull;
  67   static const uptr kMetaShadowEnd = 0x340000000000ull;
  68   static const uptr kShadowBeg     = 0x010000000000ull;
  69   static const uptr kShadowEnd = 0x100000000000ull;
  70   static const uptr kHeapMemBeg    = 0x7b0000000000ull;
  71   static const uptr kHeapMemEnd    = 0x7c0000000000ull;
  72   static const uptr kLoAppMemBeg   = 0x000000001000ull;
  73   static const uptr kLoAppMemEnd   = 0x008000000000ull;
  74   static const uptr kMidAppMemBeg  = 0x550000000000ull;
  75   static const uptr kMidAppMemEnd  = 0x568000000000ull;
  76   static const uptr kHiAppMemBeg   = 0x7e8000000000ull;
  77   static const uptr kHiAppMemEnd   = 0x800000000000ull;
  78   static const uptr kShadowMsk = 0x780000000000ull;
  79   static const uptr kShadowXor = 0x040000000000ull;
  80   static const uptr kShadowAdd = 0x000000000000ull;
  81   static const uptr kVdsoBeg       = 0xf000000000000000ull;
  82 };
  83
  84 /*
  85 C/C++ on linux/mips64 (40-bit VMA)
  86 0000 0000 00 - 0100 0000 00: -                                           (4 GB)
  87 0100 0000 00 - 0200 0000 00: main binary                                 (4 GB)
  88 0200 0000 00 - 1200 0000 00: -                                          (64 GB)
  89 1200 0000 00 - 2200 0000 00: shadow                                     (64 GB)
  90 2200 0000 00 - 4000 0000 00: -                                         (120 GB)
  91 4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects)  (64 GB)
  92 5000 0000 00 - aa00 0000 00: -                                         (360 GB)
  93 aa00 0000 00 - ab00 0000 00: main binary (PIE)                           (4 GB)
  94 ab00 0000 00 - fe00 0000 00: -                                         (332 GB)
  95 fe00 0000 00 - ff00 0000 00: heap                                        (4 GB)
  96 ff00 0000 00 - ff80 0000 00: -                                           (2 GB)
  97 ff80 0000 00 - ffff ffff ff: modules and main thread stack              (<2 GB)
  98 */
  99 struct MappingMips64_40 {
 100   static const uptr kMetaShadowBeg = 0x4000000000ull;
 101   static const uptr kMetaShadowEnd = 0x5000000000ull;
 102   static const uptr kShadowBeg = 0x1200000000ull;
 103   static const uptr kShadowEnd = 0x2200000000ull;
 104   static const uptr kHeapMemBeg    = 0xfe00000000ull;
 105   static const uptr kHeapMemEnd    = 0xff00000000ull;
 106   static const uptr kLoAppMemBeg   = 0x0100000000ull;
 107   static const uptr kLoAppMemEnd   = 0x0200000000ull;
 108   static const uptr kMidAppMemBeg  = 0xaa00000000ull;
 109   static const uptr kMidAppMemEnd  = 0xab00000000ull;
 110   static const uptr kHiAppMemBeg   = 0xff80000000ull;
 111   static const uptr kHiAppMemEnd   = 0xffffffffffull;
 112   static const uptr kShadowMsk = 0xf800000000ull;
 113   static const uptr kShadowXor = 0x0800000000ull;
 114   static const uptr kShadowAdd = 0x0000000000ull;
 115   static const uptr kVdsoBeg       = 0xfffff00000ull;
 116 };
 117
 118 /*
 119 C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
 120 0000 0000 00 - 0100 0000 00: -                                    (4 GB)
 121 0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks  (4 GB)
 122 0200 0000 00 - 0300 0000 00: heap                                 (4 GB)
 123 0300 0000 00 - 0400 0000 00: -                                    (4 GB)
 124 0400 0000 00 - 0800 0000 00: shadow memory                       (16 GB)
 125 0800 0000 00 - 0d00 0000 00: -                                   (20 GB)
 126 0d00 0000 00 - 0e00 0000 00: metainfo                             (4 GB)
 127 0e00 0000 00 - 1000 0000 00: -
 128 */
 129 struct MappingAppleAarch64 {
 130   static const uptr kLoAppMemBeg   = 0x0100000000ull;
 131   static const uptr kLoAppMemEnd   = 0x0200000000ull;
 132   static const uptr kHeapMemBeg    = 0x0200000000ull;
 133   static const uptr kHeapMemEnd    = 0x0300000000ull;
 134   static const uptr kShadowBeg     = 0x0400000000ull;
 135   static const uptr kShadowEnd = 0x0800000000ull;
 136   static const uptr kMetaShadowBeg = 0x0d00000000ull;
 137   static const uptr kMetaShadowEnd = 0x0e00000000ull;
 138   static const uptr kHiAppMemBeg   = 0x0fc0000000ull;
 139   static const uptr kHiAppMemEnd   = 0x0fc0000000ull;
 140   static const uptr kShadowMsk = 0x0ull;
 141   static const uptr kShadowXor = 0x0ull;
 142   static const uptr kShadowAdd = 0x0200000000ull;
 143   static const uptr kVdsoBeg       = 0x7000000000000000ull;
 144   static const uptr kMidAppMemBeg = 0;
 145   static const uptr kMidAppMemEnd = 0;
 146 };
 147
 148 /*
 149 C/C++ on linux/aarch64 (39-bit VMA)
 150 0000 0010 00 - 0100 0000 00: main binary
 151 0100 0000 00 - 0400 0000 00: -
 152 0400 0000 00 - 1000 0000 00: shadow memory
 153 2000 0000 00 - 3100 0000 00: -
 154 3100 0000 00 - 3400 0000 00: metainfo
 155 3400 0000 00 - 5500 0000 00: -
 156 5500 0000 00 - 5600 0000 00: main binary (PIE)
 157 5600 0000 00 - 7c00 0000 00: -
 158 7c00 0000 00 - 7d00 0000 00: heap
 159 7d00 0000 00 - 7fff ffff ff: modules and main thread stack
 160 */
 161 struct MappingAarch64_39 {
 162   static const uptr kLoAppMemBeg   = 0x0000001000ull;
 163   static const uptr kLoAppMemEnd   = 0x0100000000ull;
 164   static const uptr kShadowBeg = 0x0400000000ull;
 165   static const uptr kShadowEnd = 0x1000000000ull;
 166   static const uptr kMetaShadowBeg = 0x3100000000ull;
 167   static const uptr kMetaShadowEnd = 0x3400000000ull;
 168   static const uptr kMidAppMemBeg  = 0x5500000000ull;
 169   static const uptr kMidAppMemEnd = 0x5600000000ull;
 170   static const uptr kHeapMemBeg    = 0x7c00000000ull;
 171   static const uptr kHeapMemEnd    = 0x7d00000000ull;
 172   static const uptr kHiAppMemBeg   = 0x7e00000000ull;
 173   static const uptr kHiAppMemEnd   = 0x7fffffffffull;
 174   static const uptr kShadowMsk = 0x7800000000ull;
 175   static const uptr kShadowXor = 0x0200000000ull;
 176   static const uptr kShadowAdd = 0x0000000000ull;
 177   static const uptr kVdsoBeg       = 0x7f00000000ull;
 178 };
 179
 180 /*
 181 C/C++ on linux/aarch64 (42-bit VMA)
 182 00000 0010 00 - 01000 0000 00: main binary
 183 01000 0000 00 - 08000 0000 00: -
 184 08000 0000 00 - 10000 0000 00: shadow memory
 185 10000 0000 00 - 26000 0000 00: -
 186 26000 0000 00 - 28000 0000 00: metainfo
 187 28000 0000 00 - 2aa00 0000 00: -
 188 2aa00 0000 00 - 2ab00 0000 00: main binary (PIE)
 189 2ab00 0000 00 - 3e000 0000 00: -
 190 3e000 0000 00 - 3f000 0000 00: heap
 191 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack
 192 */
 193 struct MappingAarch64_42 {
 194   static const uptr kBroken = kBrokenReverseMapping;
 195   static const uptr kLoAppMemBeg   = 0x00000001000ull;
 196   static const uptr kLoAppMemEnd   = 0x01000000000ull;
 197   static const uptr kShadowBeg = 0x08000000000ull;
 198   static const uptr kShadowEnd = 0x10000000000ull;
 199   static const uptr kMetaShadowBeg = 0x26000000000ull;
 200   static const uptr kMetaShadowEnd = 0x28000000000ull;
 201   static const uptr kMidAppMemBeg  = 0x2aa00000000ull;
 202   static const uptr kMidAppMemEnd = 0x2ab00000000ull;
 203   static const uptr kHeapMemBeg    = 0x3e000000000ull;
 204   static const uptr kHeapMemEnd    = 0x3f000000000ull;
 205   static const uptr kHiAppMemBeg   = 0x3f000000000ull;
 206   static const uptr kHiAppMemEnd   = 0x3ffffffffffull;
 207   static const uptr kShadowMsk = 0x3c000000000ull;
 208   static const uptr kShadowXor = 0x04000000000ull;
 209   static const uptr kShadowAdd = 0x00000000000ull;
 210   static const uptr kVdsoBeg       = 0x37f00000000ull;
 211 };
 212
 213 struct MappingAarch64_48 {
 214   static const uptr kLoAppMemBeg   = 0x0000000001000ull;
 215   static const uptr kLoAppMemEnd   = 0x0000200000000ull;
 216   static const uptr kShadowBeg = 0x0001000000000ull;
 217   static const uptr kShadowEnd = 0x0002000000000ull;
 218   static const uptr kMetaShadowBeg = 0x0005000000000ull;
 219   static const uptr kMetaShadowEnd = 0x0006000000000ull;
 220   static const uptr kMidAppMemBeg  = 0x0aaaa00000000ull;
 221   static const uptr kMidAppMemEnd = 0x0aaaf00000000ull;
 222   static const uptr kHeapMemBeg    = 0x0ffff00000000ull;
 223   static const uptr kHeapMemEnd    = 0x0ffff00000000ull;
 224   static const uptr kHiAppMemBeg   = 0x0ffff00000000ull;
 225   static const uptr kHiAppMemEnd   = 0x1000000000000ull;
 226   static const uptr kShadowMsk = 0x0fff800000000ull;
 227   static const uptr kShadowXor = 0x0000800000000ull;
 228   static const uptr kShadowAdd = 0x0000000000000ull;
 229   static const uptr kVdsoBeg       = 0xffff000000000ull;
 230 };
 231
 232 /*
 233 C/C++ on linux/powerpc64 (44-bit VMA)
 234 0000 0000 0100 - 0001 0000 0000: main binary
 235 0001 0000 0000 - 0001 0000 0000: -
 236 0001 0000 0000 - 0b00 0000 0000: shadow
 237 0b00 0000 0000 - 0b00 0000 0000: -
 238 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
 239 0d00 0000 0000 - 0f00 0000 0000: -
 240 0f00 0000 0000 - 0f50 0000 0000: heap
 241 0f50 0000 0000 - 0f60 0000 0000: -
 242 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
 243 */
 244 struct MappingPPC64_44 {
 245   static const uptr kBroken =
 246       kBrokenMapping | kBrokenReverseMapping | kBrokenLinearity;
 247   static const uptr kMetaShadowBeg = 0x0b0000000000ull;
 248   static const uptr kMetaShadowEnd = 0x0d0000000000ull;
 249   static const uptr kShadowBeg     = 0x000100000000ull;
 250   static const uptr kShadowEnd     = 0x0b0000000000ull;
 251   static const uptr kLoAppMemBeg   = 0x000000000100ull;
 252   static const uptr kLoAppMemEnd   = 0x000100000000ull;
 253   static const uptr kHeapMemBeg    = 0x0f0000000000ull;
 254   static const uptr kHeapMemEnd    = 0x0f5000000000ull;
 255   static const uptr kHiAppMemBeg   = 0x0f6000000000ull;
 256   static const uptr kHiAppMemEnd   = 0x100000000000ull; // 44 bits
 257   static const uptr kShadowMsk = 0x0f0000000000ull;
 258   static const uptr kShadowXor = 0x002100000000ull;
 259   static const uptr kShadowAdd = 0x000000000000ull;
 260   static const uptr kVdsoBeg       = 0x3c0000000000000ull;
 261   static const uptr kMidAppMemBeg = 0;
 262   static const uptr kMidAppMemEnd = 0;
 263 };
 264
 265 /*
 266 C/C++ on linux/powerpc64 (46-bit VMA)
 267 0000 0000 1000 - 0100 0000 0000: main binary
 268 0100 0000 0000 - 0200 0000 0000: -
 269 0100 0000 0000 - 0800 0000 0000: shadow
 270 0800 0000 0000 - 1000 0000 0000: -
 271 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
 272 1200 0000 0000 - 3d00 0000 0000: -
 273 3d00 0000 0000 - 3e00 0000 0000: heap
 274 3e00 0000 0000 - 3e80 0000 0000: -
 275 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
 276 */
 277 struct MappingPPC64_46 {
 278   static const uptr kMetaShadowBeg = 0x100000000000ull;
 279   static const uptr kMetaShadowEnd = 0x120000000000ull;
 280   static const uptr kShadowBeg     = 0x010000000000ull;
 281   static const uptr kShadowEnd = 0x080000000000ull;
 282   static const uptr kHeapMemBeg    = 0x3d0000000000ull;
 283   static const uptr kHeapMemEnd    = 0x3e0000000000ull;
 284   static const uptr kLoAppMemBeg   = 0x000000001000ull;
 285   static const uptr kLoAppMemEnd   = 0x010000000000ull;
 286   static const uptr kHiAppMemBeg   = 0x3e8000000000ull;
 287   static const uptr kHiAppMemEnd   = 0x400000000000ull; // 46 bits
 288   static const uptr kShadowMsk = 0x3c0000000000ull;
 289   static const uptr kShadowXor = 0x020000000000ull;
 290   static const uptr kShadowAdd = 0x000000000000ull;
 291   static const uptr kVdsoBeg       = 0x7800000000000000ull;
 292   static const uptr kMidAppMemBeg = 0;
 293   static const uptr kMidAppMemEnd = 0;
 294 };
 295
 296 /*
 297 C/C++ on linux/powerpc64 (47-bit VMA)
 298 0000 0000 1000 - 0100 0000 0000: main binary
 299 0100 0000 0000 - 0200 0000 0000: -
 300 0100 0000 0000 - 0800 0000 0000: shadow
 301 0800 0000 0000 - 1000 0000 0000: -
 302 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects)
 303 1200 0000 0000 - 7d00 0000 0000: -
 304 7d00 0000 0000 - 7e00 0000 0000: heap
 305 7e00 0000 0000 - 7e80 0000 0000: -
 306 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
 307 */
 308 struct MappingPPC64_47 {
 309   static const uptr kMetaShadowBeg = 0x100000000000ull;
 310   static const uptr kMetaShadowEnd = 0x120000000000ull;
 311   static const uptr kShadowBeg     = 0x010000000000ull;
 312   static const uptr kShadowEnd = 0x080000000000ull;
 313   static const uptr kHeapMemBeg    = 0x7d0000000000ull;
 314   static const uptr kHeapMemEnd    = 0x7e0000000000ull;
 315   static const uptr kLoAppMemBeg   = 0x000000001000ull;
 316   static const uptr kLoAppMemEnd   = 0x010000000000ull;
 317   static const uptr kHiAppMemBeg   = 0x7e8000000000ull;
 318   static const uptr kHiAppMemEnd   = 0x800000000000ull; // 47 bits
 319   static const uptr kShadowMsk = 0x7c0000000000ull;
 320   static const uptr kShadowXor = 0x020000000000ull;
 321   static const uptr kShadowAdd = 0x000000000000ull;
 322   static const uptr kVdsoBeg       = 0x7800000000000000ull;
 323   static const uptr kMidAppMemBeg = 0;
 324   static const uptr kMidAppMemEnd = 0;
 325 };
 326
 327 /*
 328 C/C++ on linux/s390x
 329 While the kernel provides a 64-bit address space, we have to restrict ourselves
 330 to 48 bits due to how e.g. SyncVar::GetId() works.
 331 0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
 332 0e00 0000 0000 - 2000 0000 0000: -
 333 2000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app)
 334 4000 0000 0000 - 9000 0000 0000: -
 335 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
 336 9800 0000 0000 - be00 0000 0000: -
 337 be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
 338 */
 339 struct MappingS390x {
 340   static const uptr kMetaShadowBeg = 0x900000000000ull;
 341   static const uptr kMetaShadowEnd = 0x980000000000ull;
 342   static const uptr kShadowBeg = 0x200000000000ull;
 343   static const uptr kShadowEnd = 0x400000000000ull;
 344   static const uptr kHeapMemBeg    = 0xbe0000000000ull;
 345   static const uptr kHeapMemEnd    = 0xc00000000000ull;
 346   static const uptr kLoAppMemBeg   = 0x000000001000ull;
 347   static const uptr kLoAppMemEnd   = 0x0e0000000000ull;
 348   static const uptr kHiAppMemBeg   = 0xc00000004000ull;
 349   static const uptr kHiAppMemEnd   = 0xc00000004000ull;
 350   static const uptr kShadowMsk = 0xb00000000000ull;
 351   static const uptr kShadowXor = 0x100000000000ull;
 352   static const uptr kShadowAdd = 0x000000000000ull;
 353   static const uptr kVdsoBeg       = 0xfffffffff000ull;
 354   static const uptr kMidAppMemBeg = 0;
 355   static const uptr kMidAppMemEnd = 0;
 356 };
 357
 358 /* Go on linux, darwin and freebsd on x86_64
 359 0000 0000 1000 - 0000 1000 0000: executable
 360 0000 1000 0000 - 00c0 0000 0000: -
 361 00c0 0000 0000 - 00e0 0000 0000: heap
 362 00e0 0000 0000 - 2000 0000 0000: -
 363 2000 0000 0000 - 21c0 0000 0000: shadow
 364 21c0 0000 0000 - 3000 0000 0000: -
 365 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
 366 4000 0000 0000 - 8000 0000 0000: -
 367 */
 368
 369 struct MappingGo48 {
 370   static const uptr kMetaShadowBeg = 0x300000000000ull;
 371   static const uptr kMetaShadowEnd = 0x400000000000ull;
 372   static const uptr kShadowBeg     = 0x200000000000ull;
 373   static const uptr kShadowEnd = 0x21c000000000ull;
 374   static const uptr kLoAppMemBeg = 0x000000001000ull;
 375   static const uptr kLoAppMemEnd = 0x00e000000000ull;
 376   static const uptr kMidAppMemBeg = 0;
 377   static const uptr kMidAppMemEnd = 0;
 378   static const uptr kHiAppMemBeg = 0;
 379   static const uptr kHiAppMemEnd = 0;
 380   static const uptr kHeapMemBeg = 0;
 381   static const uptr kHeapMemEnd = 0;
 382   static const uptr kVdsoBeg = 0;
 383   static const uptr kShadowMsk = 0;
 384   static const uptr kShadowXor = 0;
 385   static const uptr kShadowAdd = 0x200000000000ull;
 386 };
 387
 388 /* Go on windows
 389 0000 0000 1000 - 0000 1000 0000: executable
 390 0000 1000 0000 - 00f8 0000 0000: -
 391 00c0 0000 0000 - 00e0 0000 0000: heap
 392 00e0 0000 0000 - 0100 0000 0000: -
 393 0100 0000 0000 - 0300 0000 0000: shadow
 394 0300 0000 0000 - 0700 0000 0000: -
 395 0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
 396 07d0 0000 0000 - 8000 0000 0000: -
 397 PIE binaries currently not supported, but it should be theoretically possible.
 398 */
 399
 400 struct MappingGoWindows {
 401   static const uptr kMetaShadowBeg = 0x070000000000ull;
 402   static const uptr kMetaShadowEnd = 0x077000000000ull;
 403   static const uptr kShadowBeg     = 0x010000000000ull;
 404   static const uptr kShadowEnd = 0x030000000000ull;
 405   static const uptr kLoAppMemBeg = 0x000000001000ull;
 406   static const uptr kLoAppMemEnd = 0x00e000000000ull;
 407   static const uptr kMidAppMemBeg = 0;
 408   static const uptr kMidAppMemEnd = 0;
 409   static const uptr kHiAppMemBeg = 0;
 410   static const uptr kHiAppMemEnd = 0;
 411   static const uptr kHeapMemBeg = 0;
 412   static const uptr kHeapMemEnd = 0;
 413   static const uptr kVdsoBeg = 0;
 414   static const uptr kShadowMsk = 0;
 415   static const uptr kShadowXor = 0;
 416   static const uptr kShadowAdd = 0x010000000000ull;
 417 };
 418
 419 /* Go on linux/powerpc64 (46-bit VMA)
 420 0000 0000 1000 - 0000 1000 0000: executable
 421 0000 1000 0000 - 00c0 0000 0000: -
 422 00c0 0000 0000 - 00e0 0000 0000: heap
 423 00e0 0000 0000 - 2000 0000 0000: -
 424 2000 0000 0000 - 21c0 0000 0000: shadow
 425 21c0 0000 0000 - 2400 0000 0000: -
 426 2400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects)
 427 2470 0000 0000 - 4000 0000 0000: -
 428 */
 429
 430 struct MappingGoPPC64_46 {
 431   static const uptr kMetaShadowBeg = 0x240000000000ull;
 432   static const uptr kMetaShadowEnd = 0x247000000000ull;
 433   static const uptr kShadowBeg     = 0x200000000000ull;
 434   static const uptr kShadowEnd = 0x21c000000000ull;
 435   static const uptr kLoAppMemBeg = 0x000000001000ull;
 436   static const uptr kLoAppMemEnd = 0x00e000000000ull;
 437   static const uptr kMidAppMemBeg = 0;
 438   static const uptr kMidAppMemEnd = 0;
 439   static const uptr kHiAppMemBeg = 0;
 440   static const uptr kHiAppMemEnd = 0;
 441   static const uptr kHeapMemBeg = 0;
 442   static const uptr kHeapMemEnd = 0;
 443   static const uptr kVdsoBeg = 0;
 444   static const uptr kShadowMsk = 0;
 445   static const uptr kShadowXor = 0;
 446   static const uptr kShadowAdd = 0x200000000000ull;
 447 };
 448
 449 /* Go on linux/powerpc64 (47-bit VMA)
 450 0000 0000 1000 - 0000 1000 0000: executable
 451 0000 1000 0000 - 00c0 0000 0000: -
 452 00c0 0000 0000 - 00e0 0000 0000: heap
 453 00e0 0000 0000 - 2000 0000 0000: -
 454 2000 0000 0000 - 2800 0000 0000: shadow
 455 2800 0000 0000 - 3000 0000 0000: -
 456 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
 457 3200 0000 0000 - 8000 0000 0000: -
 458 */
 459
 460 struct MappingGoPPC64_47 {
 461   static const uptr kMetaShadowBeg = 0x300000000000ull;
 462   static const uptr kMetaShadowEnd = 0x320000000000ull;
 463   static const uptr kShadowBeg     = 0x200000000000ull;
 464   static const uptr kShadowEnd = 0x280000000000ull;
 465   static const uptr kLoAppMemBeg = 0x000000001000ull;
 466   static const uptr kLoAppMemEnd = 0x00e000000000ull;
 467   static const uptr kMidAppMemBeg = 0;
 468   static const uptr kMidAppMemEnd = 0;
 469   static const uptr kHiAppMemBeg = 0;
 470   static const uptr kHiAppMemEnd = 0;
 471   static const uptr kHeapMemBeg = 0;
 472   static const uptr kHeapMemEnd = 0;
 473   static const uptr kVdsoBeg = 0;
 474   static const uptr kShadowMsk = 0;
 475   static const uptr kShadowXor = 0;
 476   static const uptr kShadowAdd = 0x200000000000ull;
 477 };
 478
 479 /* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
 480 0000 0000 1000 - 0000 1000 0000: executable
 481 0000 1000 0000 - 00c0 0000 0000: -
 482 00c0 0000 0000 - 00e0 0000 0000: heap
 483 00e0 0000 0000 - 2000 0000 0000: -
 484 2000 0000 0000 - 2800 0000 0000: shadow
 485 2800 0000 0000 - 3000 0000 0000: -
 486 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
 487 3200 0000 0000 - 8000 0000 0000: -
 488 */
 489 struct MappingGoAarch64 {
 490   static const uptr kMetaShadowBeg = 0x300000000000ull;
 491   static const uptr kMetaShadowEnd = 0x320000000000ull;
 492   static const uptr kShadowBeg     = 0x200000000000ull;
 493   static const uptr kShadowEnd = 0x280000000000ull;
 494   static const uptr kLoAppMemBeg = 0x000000001000ull;
 495   static const uptr kLoAppMemEnd = 0x00e000000000ull;
 496   static const uptr kMidAppMemBeg = 0;
 497   static const uptr kMidAppMemEnd = 0;
 498   static const uptr kHiAppMemBeg = 0;
 499   static const uptr kHiAppMemEnd = 0;
 500   static const uptr kHeapMemBeg = 0;
 501   static const uptr kHeapMemEnd = 0;
 502   static const uptr kVdsoBeg = 0;
 503   static const uptr kShadowMsk = 0;
 504   static const uptr kShadowXor = 0;
 505   static const uptr kShadowAdd = 0x200000000000ull;
 506 };
 507
 508 /*
 509 Go on linux/mips64 (47-bit VMA)
 510 0000 0000 1000 - 0000 1000 0000: executable
 511 0000 1000 0000 - 00c0 0000 0000: -
 512 00c0 0000 0000 - 00e0 0000 0000: heap
 513 00e0 0000 0000 - 2000 0000 0000: -
 514 2000 0000 0000 - 2800 0000 0000: shadow
 515 2800 0000 0000 - 3000 0000 0000: -
 516 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects)
 517 3200 0000 0000 - 8000 0000 0000: -
 518 */
 519 struct MappingGoMips64_47 {
 520   static const uptr kMetaShadowBeg = 0x300000000000ull;
 521   static const uptr kMetaShadowEnd = 0x320000000000ull;
 522   static const uptr kShadowBeg = 0x200000000000ull;
 523   static const uptr kShadowEnd = 0x280000000000ull;
 524   static const uptr kLoAppMemBeg = 0x000000001000ull;
 525   static const uptr kLoAppMemEnd = 0x00e000000000ull;
 526   static const uptr kMidAppMemBeg = 0;
 527   static const uptr kMidAppMemEnd = 0;
 528   static const uptr kHiAppMemBeg = 0;
 529   static const uptr kHiAppMemEnd = 0;
 530   static const uptr kHeapMemBeg = 0;
 531   static const uptr kHeapMemEnd = 0;
 532   static const uptr kVdsoBeg = 0;
 533   static const uptr kShadowMsk = 0;
 534   static const uptr kShadowXor = 0;
 535   static const uptr kShadowAdd = 0x200000000000ull;
 536 };
 537
 538 /*
 539 Go on linux/s390x
 540 0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
 541 1000 0000 0000 - 4000 0000 0000: -
 542 4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app)
 543 6000 0000 0000 - 9000 0000 0000: -
 544 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
 545 */
 546 struct MappingGoS390x {
 547   static const uptr kMetaShadowBeg = 0x900000000000ull;
 548   static const uptr kMetaShadowEnd = 0x980000000000ull;
 549   static const uptr kShadowBeg     = 0x400000000000ull;
 550   static const uptr kShadowEnd = 0x600000000000ull;
 551   static const uptr kLoAppMemBeg = 0x000000001000ull;
 552   static const uptr kLoAppMemEnd = 0x100000000000ull;
 553   static const uptr kMidAppMemBeg = 0;
 554   static const uptr kMidAppMemEnd = 0;
 555   static const uptr kHiAppMemBeg = 0;
 556   static const uptr kHiAppMemEnd = 0;
 557   static const uptr kHeapMemBeg = 0;
 558   static const uptr kHeapMemEnd = 0;
 559   static const uptr kVdsoBeg = 0;
 560   static const uptr kShadowMsk = 0;
 561   static const uptr kShadowXor = 0;
 562   static const uptr kShadowAdd = 0x400000000000ull;
 563 };
 564
 565 extern uptr vmaSize;
 566
 567 template <typename Func, typename Arg>
 568 ALWAYS_INLINE auto SelectMapping(Arg arg) {
 569 #if SANITIZER_GO
 570 #  if defined(__powerpc64__)
 571   switch (vmaSize) {
 572     case 46:
 573       return Func::template Apply<MappingGoPPC64_46>(arg);
 574     case 47:
 575       return Func::template Apply<MappingGoPPC64_47>(arg);
 576   }
 577 #  elif defined(__mips64)
 578   return Func::template Apply<MappingGoMips64_47>(arg);
 579 #  elif defined(__s390x__)
 580   return Func::template Apply<MappingGoS390x>(arg);
 581 #  elif defined(__aarch64__)
 582   return Func::template Apply<MappingGoAarch64>(arg);
 583 #  elif SANITIZER_WINDOWS
 584   return Func::template Apply<MappingGoWindows>(arg);
 585 #  else
 586   return Func::template Apply<MappingGo48>(arg);
 587 #  endif
 588 #else  // SANITIZER_GO
 589 #  if SANITIZER_IOS && !SANITIZER_IOSSIM
 590   return Func::template Apply<MappingAppleAarch64>(arg);
 591 #  elif defined(__x86_64__) || SANITIZER_APPLE
 592   return Func::template Apply<Mapping48AddressSpace>(arg);
 593 #  elif defined(__aarch64__)
 594   switch (vmaSize) {
 595     case 39:
 596       return Func::template Apply<MappingAarch64_39>(arg);
 597     case 42:
 598       return Func::template Apply<MappingAarch64_42>(arg);
 599     case 48:
 600       return Func::template Apply<MappingAarch64_48>(arg);
 601   }
 602 #  elif defined(__powerpc64__)
 603   switch (vmaSize) {
 604     case 44:
 605       return Func::template Apply<MappingPPC64_44>(arg);
 606     case 46:
 607       return Func::template Apply<MappingPPC64_46>(arg);
 608     case 47:
 609       return Func::template Apply<MappingPPC64_47>(arg);
 610   }
 611 #  elif defined(__mips64)
 612   return Func::template Apply<MappingMips64_40>(arg);
 613 #  elif defined(__s390x__)
 614   return Func::template Apply<MappingS390x>(arg);
 615 #  else
 616 #    error "unsupported platform"
 617 #  endif
 618 #endif
 619   Die();
 620 }
 621
 622 template <typename Func>
 623 void ForEachMapping() {
 624   Func::template Apply<Mapping48AddressSpace>();
 625   Func::template Apply<MappingMips64_40>();
 626   Func::template Apply<MappingAppleAarch64>();
 627   Func::template Apply<MappingAarch64_39>();
 628   Func::template Apply<MappingAarch64_42>();
 629   Func::template Apply<MappingAarch64_48>();
 630   Func::template Apply<MappingPPC64_44>();
 631   Func::template Apply<MappingPPC64_46>();
 632   Func::template Apply<MappingPPC64_47>();
 633   Func::template Apply<MappingS390x>();
 634   Func::template Apply<MappingGo48>();
 635   Func::template Apply<MappingGoWindows>();
 636   Func::template Apply<MappingGoPPC64_46>();
 637   Func::template Apply<MappingGoPPC64_47>();
 638   Func::template Apply<MappingGoAarch64>();
 639   Func::template Apply<MappingGoMips64_47>();
 640   Func::template Apply<MappingGoS390x>();
 641 }
 642
 643 enum MappingType {
 644   kLoAppMemBeg,
 645   kLoAppMemEnd,
 646   kHiAppMemBeg,
 647   kHiAppMemEnd,
 648   kMidAppMemBeg,
 649   kMidAppMemEnd,
 650   kHeapMemBeg,
 651   kHeapMemEnd,
 652   kShadowBeg,
 653   kShadowEnd,
 654   kMetaShadowBeg,
 655   kMetaShadowEnd,
 656   kVdsoBeg,
 657 };
 658
 659 struct MappingField {
 660   template <typename Mapping>
 661   static uptr Apply(MappingType type) {
 662     switch (type) {
 663       case kLoAppMemBeg:
 664         return Mapping::kLoAppMemBeg;
 665       case kLoAppMemEnd:
 666         return Mapping::kLoAppMemEnd;
 667       case kMidAppMemBeg:
 668         return Mapping::kMidAppMemBeg;
 669       case kMidAppMemEnd:
 670         return Mapping::kMidAppMemEnd;
 671       case kHiAppMemBeg:
 672         return Mapping::kHiAppMemBeg;
 673       case kHiAppMemEnd:
 674         return Mapping::kHiAppMemEnd;
 675       case kHeapMemBeg:
 676         return Mapping::kHeapMemBeg;
 677       case kHeapMemEnd:
 678         return Mapping::kHeapMemEnd;
 679       case kVdsoBeg:
 680         return Mapping::kVdsoBeg;
 681       case kShadowBeg:
 682         return Mapping::kShadowBeg;
 683       case kShadowEnd:
 684         return Mapping::kShadowEnd;
 685       case kMetaShadowBeg:
 686         return Mapping::kMetaShadowBeg;
 687       case kMetaShadowEnd:
 688         return Mapping::kMetaShadowEnd;
 689     }
 690     Die();
 691   }
 692 };
 693
 694 ALWAYS_INLINE
 695 uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); }
 696 ALWAYS_INLINE
 697 uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); }
 698
 699 ALWAYS_INLINE
 700 uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); }
 701 ALWAYS_INLINE
 702 uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); }
 703
 704 ALWAYS_INLINE
 705 uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); }
 706 ALWAYS_INLINE
 707 uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); }
 708
 709 ALWAYS_INLINE
 710 uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); }
 711 ALWAYS_INLINE
 712 uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); }
 713
 714 ALWAYS_INLINE
 715 uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); }
 716
 717 ALWAYS_INLINE
 718 uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); }
 719 ALWAYS_INLINE
 720 uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); }
 721
 722 ALWAYS_INLINE
 723 uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); }
 724 ALWAYS_INLINE
 725 uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); }
 726
 727 struct IsAppMemImpl {
 728   template <typename Mapping>
 729   static bool Apply(uptr mem) {
 730   return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
 731          (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
 732          (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
 733          (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
 734   }
 735 };
 736
 737 ALWAYS_INLINE
 738 bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); }
 739
 740 struct IsShadowMemImpl {
 741   template <typename Mapping>
 742   static bool Apply(uptr mem) {
 743     return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
 744   }
 745 };
 746
 747 ALWAYS_INLINE
 748 bool IsShadowMem(RawShadow *p) {
 749   return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p));
 750 }
 751
 752 struct IsMetaMemImpl {
 753   template <typename Mapping>
 754   static bool Apply(uptr mem) {
 755     return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
 756   }
 757 };
 758
 759 ALWAYS_INLINE
 760 bool IsMetaMem(const u32 *p) {
 761   return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p));
 762 }
 763
 764 struct MemToShadowImpl {
 765   template <typename Mapping>
 766   static uptr Apply(uptr x) {
 767     DCHECK(IsAppMemImpl::Apply<Mapping>(x));
 768     return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^
 769             Mapping::kShadowXor) *
 770                kShadowMultiplier +
 771            Mapping::kShadowAdd;
 772   }
 773 };
 774
 775 ALWAYS_INLINE
 776 RawShadow *MemToShadow(uptr x) {
 777   return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x));
 778 }
 779
 780 struct MemToMetaImpl {
 781   template <typename Mapping>
 782   static u32 *Apply(uptr x) {
 783     DCHECK(IsAppMemImpl::Apply<Mapping>(x));
 784     return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) /
 785                     kMetaShadowCell * kMetaShadowSize) |
 786                    Mapping::kMetaShadowBeg);
 787   }
 788 };
 789
 790 ALWAYS_INLINE
 791 u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); }
 792
 793 struct ShadowToMemImpl {
 794   template <typename Mapping>
 795   static uptr Apply(uptr sp) {
 796     if (!IsShadowMemImpl::Apply<Mapping>(sp))
 797       return 0;
 798     // The shadow mapping is non-linear and we've lost some bits, so we don't
 799     // have an easy way to restore the original app address. But the mapping is
 800     // a bijection, so we try to restore the address as belonging to
 801     // low/mid/high range consecutively and see if shadow->app->shadow mapping
 802     // gives us the same address.
 803     uptr p =
 804         ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor;
 805     if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
 806         MemToShadowImpl::Apply<Mapping>(p) == sp)
 807       return p;
 808     if (Mapping::kMidAppMemBeg) {
 809       uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk);
 810       if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd &&
 811           MemToShadowImpl::Apply<Mapping>(p_mid) == sp)
 812         return p_mid;
 813     }
 814     return p | Mapping::kShadowMsk;
 815   }
 816 };
 817
 818 ALWAYS_INLINE
 819 uptr ShadowToMem(RawShadow *s) {
 820   return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s));
 821 }
 822
 823 // Compresses addr to kCompressedAddrBits stored in least significant bits.
 824 ALWAYS_INLINE uptr CompressAddr(uptr addr) {
 825   return addr & ((1ull << kCompressedAddrBits) - 1);
 826 }
 827
 828 struct RestoreAddrImpl {
 829   typedef uptr Result;
 830   template <typename Mapping>
 831   static Result Apply(uptr addr) {
 832     // To restore the address we go over all app memory ranges and check if top
 833     // 3 bits of the compressed addr match that of the app range. If yes, we
 834     // assume that the compressed address come from that range and restore the
 835     // missing top bits to match the app range address.
 836     const uptr ranges[] = {
 837         Mapping::kLoAppMemBeg,  Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
 838         Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
 839         Mapping::kHeapMemBeg,   Mapping::kHeapMemEnd,
 840     };
 841     const uptr indicator = 0x0e0000000000ull;
 842     const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator);
 843     for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
 844       uptr beg = ranges[i];
 845       uptr end = ranges[i + 1];
 846       if (beg == end)
 847         continue;
 848       for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) {
 849         if ((addr & indicator) == (p & indicator))
 850           return addr | (p & ~(ind_lsb - 1));
 851       }
 852     }
 853     Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr);
 854     Die();
 855   }
 856 };
 857
 858 // Restores compressed addr from kCompressedAddrBits to full representation.
 859 // This is called only during reporting and is not performance-critical.
 860 inline uptr RestoreAddr(uptr addr) {
 861   return SelectMapping<RestoreAddrImpl>(addr);
 862 }
 863
 864 void InitializePlatform();
 865 void InitializePlatformEarly();
 866 void CheckAndProtect();
 867 void InitializeShadowMemoryPlatform();
 868 void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
 869 int ExtractResolvFDs(void *state, int *fds, int nfd);
 870 int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
 871 uptr ExtractLongJmpSp(uptr *env);
 872 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
 873
 874 int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
 875                                      void (*cleanup)(void *arg), void *arg);
 876
 877 void DestroyThreadState();
 878 void PlatformCleanUpThreadState(ThreadState *thr);
 879
 880 }  // namespace __tsan
 881
 882 #endif  // TSAN_PLATFORM_H