libsanitizer/tsan/tsan_platform.h

   1 //===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer (TSan), a race detector.
   9 //
  10 // Platform-specific code.
  11 //===----------------------------------------------------------------------===//
  12
  13 #ifndef TSAN_PLATFORM_H
  14 #define TSAN_PLATFORM_H
  15
  16 #if !defined(__LP64__) && !defined(_WIN64)
  17 # error "Only 64-bit is supported"
  18 #endif
  19
  20 #include "tsan_defs.h"
  21 #include "tsan_trace.h"
  22
  23 namespace __tsan {
  24
  25 #if !SANITIZER_GO
  26
  27 #if defined(__x86_64__)
  28 /*
  29 C/C++ on linux/x86_64 and freebsd/x86_64
  30 0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
  31 0040 0000 0000 - 0100 0000 0000: -
  32 0100 0000 0000 - 2000 0000 0000: shadow
  33 2000 0000 0000 - 3000 0000 0000: -
  34 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
  35 4000 0000 0000 - 5500 0000 0000: -
  36 5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
  37 5680 0000 0000 - 6000 0000 0000: -
  38 6000 0000 0000 - 6200 0000 0000: traces
  39 6200 0000 0000 - 7d00 0000 0000: -
  40 7b00 0000 0000 - 7c00 0000 0000: heap
  41 7c00 0000 0000 - 7e80 0000 0000: -
  42 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
  43 */
  44 struct Mapping {
  45   static const uptr kMetaShadowBeg = 0x300000000000ull;
  46   static const uptr kMetaShadowEnd = 0x340000000000ull;
  47   static const uptr kTraceMemBeg   = 0x600000000000ull;
  48   static const uptr kTraceMemEnd   = 0x620000000000ull;
  49   static const uptr kShadowBeg     = 0x010000000000ull;
  50   static const uptr kShadowEnd     = 0x200000000000ull;
  51   static const uptr kHeapMemBeg    = 0x7b0000000000ull;
  52   static const uptr kHeapMemEnd    = 0x7c0000000000ull;
  53   static const uptr kLoAppMemBeg   = 0x000000001000ull;
  54   static const uptr kLoAppMemEnd   = 0x008000000000ull;
  55   static const uptr kMidAppMemBeg  = 0x550000000000ull;
  56   static const uptr kMidAppMemEnd  = 0x568000000000ull;
  57   static const uptr kHiAppMemBeg   = 0x7e8000000000ull;
  58   static const uptr kHiAppMemEnd   = 0x800000000000ull;
  59   static const uptr kAppMemMsk     = 0x780000000000ull;
  60   static const uptr kAppMemXor     = 0x040000000000ull;
  61   static const uptr kVdsoBeg       = 0xf000000000000000ull;
  62 };
  63
  64 #define TSAN_MID_APP_RANGE 1
  65 #elif defined(__mips64)
  66 /*
  67 C/C++ on linux/mips64
  68 0100 0000 00 - 0200 0000 00: main binary
  69 0200 0000 00 - 1400 0000 00: -
  70 1400 0000 00 - 2400 0000 00: shadow
  71 2400 0000 00 - 3000 0000 00: -
  72 3000 0000 00 - 4000 0000 00: metainfo (memory blocks and sync objects)
  73 4000 0000 00 - 6000 0000 00: -
  74 6000 0000 00 - 6200 0000 00: traces
  75 6200 0000 00 - fe00 0000 00: -
  76 fe00 0000 00 - ff00 0000 00: heap
  77 ff00 0000 00 - ff80 0000 00: -
  78 ff80 0000 00 - ffff ffff ff: modules and main thread stack
  79 */
  80 struct Mapping {
  81   static const uptr kMetaShadowBeg = 0x4000000000ull;
  82   static const uptr kMetaShadowEnd = 0x5000000000ull;
  83   static const uptr kTraceMemBeg   = 0xb000000000ull;
  84   static const uptr kTraceMemEnd   = 0xb200000000ull;
  85   static const uptr kShadowBeg     = 0x2400000000ull;
  86   static const uptr kShadowEnd     = 0x4000000000ull;
  87   static const uptr kHeapMemBeg    = 0xfe00000000ull;
  88   static const uptr kHeapMemEnd    = 0xff00000000ull;
  89   static const uptr kLoAppMemBeg   = 0x0100000000ull;
  90   static const uptr kLoAppMemEnd   = 0x0200000000ull;
  91   static const uptr kMidAppMemBeg  = 0xaa00000000ull;
  92   static const uptr kMidAppMemEnd  = 0xab00000000ull;
  93   static const uptr kHiAppMemBeg   = 0xff80000000ull;
  94   static const uptr kHiAppMemEnd   = 0xffffffffffull;
  95   static const uptr kAppMemMsk     = 0xf800000000ull;
  96   static const uptr kAppMemXor     = 0x0800000000ull;
  97   static const uptr kVdsoBeg       = 0xfffff00000ull;
  98 };
  99
 100 #define TSAN_MID_APP_RANGE 1
 101 #elif defined(__aarch64__)
 102 // AArch64 supports multiple VMA which leads to multiple address transformation
 103 // functions.  To support these multiple VMAS transformations and mappings TSAN
 104 // runtime for AArch64 uses an external memory read (vmaSize) to select which
 105 // mapping to use.  Although slower, it make a same instrumented binary run on
 106 // multiple kernels.
 107
 108 /*
 109 C/C++ on linux/aarch64 (39-bit VMA)
 110 0000 0010 00 - 0100 0000 00: main binary
 111 0100 0000 00 - 0800 0000 00: -
 112 0800 0000 00 - 2000 0000 00: shadow memory
 113 2000 0000 00 - 3100 0000 00: -
 114 3100 0000 00 - 3400 0000 00: metainfo
 115 3400 0000 00 - 5500 0000 00: -
 116 5500 0000 00 - 5600 0000 00: main binary (PIE)
 117 5600 0000 00 - 6000 0000 00: -
 118 6000 0000 00 - 6200 0000 00: traces
 119 6200 0000 00 - 7d00 0000 00: -
 120 7c00 0000 00 - 7d00 0000 00: heap
 121 7d00 0000 00 - 7fff ffff ff: modules and main thread stack
 122 */
 123 struct Mapping39 {
 124   static const uptr kLoAppMemBeg   = 0x0000001000ull;
 125   static const uptr kLoAppMemEnd   = 0x0100000000ull;
 126   static const uptr kShadowBeg     = 0x0800000000ull;
 127   static const uptr kShadowEnd     = 0x2000000000ull;
 128   static const uptr kMetaShadowBeg = 0x3100000000ull;
 129   static const uptr kMetaShadowEnd = 0x3400000000ull;
 130   static const uptr kMidAppMemBeg  = 0x5500000000ull;
 131   static const uptr kMidAppMemEnd  = 0x5600000000ull;
 132   static const uptr kTraceMemBeg   = 0x6000000000ull;
 133   static const uptr kTraceMemEnd   = 0x6200000000ull;
 134   static const uptr kHeapMemBeg    = 0x7c00000000ull;
 135   static const uptr kHeapMemEnd    = 0x7d00000000ull;
 136   static const uptr kHiAppMemBeg   = 0x7e00000000ull;
 137   static const uptr kHiAppMemEnd   = 0x7fffffffffull;
 138   static const uptr kAppMemMsk     = 0x7800000000ull;
 139   static const uptr kAppMemXor     = 0x0200000000ull;
 140   static const uptr kVdsoBeg       = 0x7f00000000ull;
 141 };
 142
 143 /*
 144 C/C++ on linux/aarch64 (42-bit VMA)
 145 00000 0010 00 - 01000 0000 00: main binary
 146 01000 0000 00 - 10000 0000 00: -
 147 10000 0000 00 - 20000 0000 00: shadow memory
 148 20000 0000 00 - 26000 0000 00: -
 149 26000 0000 00 - 28000 0000 00: metainfo
 150 28000 0000 00 - 2aa00 0000 00: -
 151 2aa00 0000 00 - 2ab00 0000 00: main binary (PIE)
 152 2ab00 0000 00 - 36200 0000 00: -
 153 36200 0000 00 - 36240 0000 00: traces
 154 36240 0000 00 - 3e000 0000 00: -
 155 3e000 0000 00 - 3f000 0000 00: heap
 156 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack
 157 */
 158 struct Mapping42 {
 159   static const uptr kLoAppMemBeg   = 0x00000001000ull;
 160   static const uptr kLoAppMemEnd   = 0x01000000000ull;
 161   static const uptr kShadowBeg     = 0x10000000000ull;
 162   static const uptr kShadowEnd     = 0x20000000000ull;
 163   static const uptr kMetaShadowBeg = 0x26000000000ull;
 164   static const uptr kMetaShadowEnd = 0x28000000000ull;
 165   static const uptr kMidAppMemBeg  = 0x2aa00000000ull;
 166   static const uptr kMidAppMemEnd  = 0x2ab00000000ull;
 167   static const uptr kTraceMemBeg   = 0x36200000000ull;
 168   static const uptr kTraceMemEnd   = 0x36400000000ull;
 169   static const uptr kHeapMemBeg    = 0x3e000000000ull;
 170   static const uptr kHeapMemEnd    = 0x3f000000000ull;
 171   static const uptr kHiAppMemBeg   = 0x3f000000000ull;
 172   static const uptr kHiAppMemEnd   = 0x3ffffffffffull;
 173   static const uptr kAppMemMsk     = 0x3c000000000ull;
 174   static const uptr kAppMemXor     = 0x04000000000ull;
 175   static const uptr kVdsoBeg       = 0x37f00000000ull;
 176 };
 177
 178 struct Mapping48 {
 179   static const uptr kLoAppMemBeg   = 0x0000000001000ull;
 180   static const uptr kLoAppMemEnd   = 0x0000200000000ull;
 181   static const uptr kShadowBeg     = 0x0002000000000ull;
 182   static const uptr kShadowEnd     = 0x0004000000000ull;
 183   static const uptr kMetaShadowBeg = 0x0005000000000ull;
 184   static const uptr kMetaShadowEnd = 0x0006000000000ull;
 185   static const uptr kMidAppMemBeg  = 0x0aaaa00000000ull;
 186   static const uptr kMidAppMemEnd  = 0x0aaaf00000000ull;
 187   static const uptr kTraceMemBeg   = 0x0f06000000000ull;
 188   static const uptr kTraceMemEnd   = 0x0f06200000000ull;
 189   static const uptr kHeapMemBeg    = 0x0ffff00000000ull;
 190   static const uptr kHeapMemEnd    = 0x0ffff00000000ull;
 191   static const uptr kHiAppMemBeg   = 0x0ffff00000000ull;
 192   static const uptr kHiAppMemEnd   = 0x1000000000000ull;
 193   static const uptr kAppMemMsk     = 0x0fff800000000ull;
 194   static const uptr kAppMemXor     = 0x0000800000000ull;
 195   static const uptr kVdsoBeg       = 0xffff000000000ull;
 196 };
 197
 198 // Indicates the runtime will define the memory regions at runtime.
 199 #define TSAN_RUNTIME_VMA 1
 200 // Indicates that mapping defines a mid range memory segment.
 201 #define TSAN_MID_APP_RANGE 1
 202 #elif defined(__powerpc64__)
 203 // PPC64 supports multiple VMA which leads to multiple address transformation
 204 // functions.  To support these multiple VMAS transformations and mappings TSAN
 205 // runtime for PPC64 uses an external memory read (vmaSize) to select which
 206 // mapping to use.  Although slower, it make a same instrumented binary run on
 207 // multiple kernels.
 208
 209 /*
 210 C/C++ on linux/powerpc64 (44-bit VMA)
 211 0000 0000 0100 - 0001 0000 0000: main binary
 212 0001 0000 0000 - 0001 0000 0000: -
 213 0001 0000 0000 - 0b00 0000 0000: shadow
 214 0b00 0000 0000 - 0b00 0000 0000: -
 215 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
 216 0d00 0000 0000 - 0d00 0000 0000: -
 217 0d00 0000 0000 - 0f00 0000 0000: traces
 218 0f00 0000 0000 - 0f00 0000 0000: -
 219 0f00 0000 0000 - 0f50 0000 0000: heap
 220 0f50 0000 0000 - 0f60 0000 0000: -
 221 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
 222 */
 223 struct Mapping44 {
 224   static const uptr kMetaShadowBeg = 0x0b0000000000ull;
 225   static const uptr kMetaShadowEnd = 0x0d0000000000ull;
 226   static const uptr kTraceMemBeg   = 0x0d0000000000ull;
 227   static const uptr kTraceMemEnd   = 0x0f0000000000ull;
 228   static const uptr kShadowBeg     = 0x000100000000ull;
 229   static const uptr kShadowEnd     = 0x0b0000000000ull;
 230   static const uptr kLoAppMemBeg   = 0x000000000100ull;
 231   static const uptr kLoAppMemEnd   = 0x000100000000ull;
 232   static const uptr kHeapMemBeg    = 0x0f0000000000ull;
 233   static const uptr kHeapMemEnd    = 0x0f5000000000ull;
 234   static const uptr kHiAppMemBeg   = 0x0f6000000000ull;
 235   static const uptr kHiAppMemEnd   = 0x100000000000ull; // 44 bits
 236   static const uptr kAppMemMsk     = 0x0f0000000000ull;
 237   static const uptr kAppMemXor     = 0x002100000000ull;
 238   static const uptr kVdsoBeg       = 0x3c0000000000000ull;
 239 };
 240
 241 /*
 242 C/C++ on linux/powerpc64 (46-bit VMA)
 243 0000 0000 1000 - 0100 0000 0000: main binary
 244 0100 0000 0000 - 0200 0000 0000: -
 245 0100 0000 0000 - 1000 0000 0000: shadow
 246 1000 0000 0000 - 1000 0000 0000: -
 247 1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
 248 2000 0000 0000 - 2000 0000 0000: -
 249 2000 0000 0000 - 2200 0000 0000: traces
 250 2200 0000 0000 - 3d00 0000 0000: -
 251 3d00 0000 0000 - 3e00 0000 0000: heap
 252 3e00 0000 0000 - 3e80 0000 0000: -
 253 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
 254 */
 255 struct Mapping46 {
 256   static const uptr kMetaShadowBeg = 0x100000000000ull;
 257   static const uptr kMetaShadowEnd = 0x200000000000ull;
 258   static const uptr kTraceMemBeg   = 0x200000000000ull;
 259   static const uptr kTraceMemEnd   = 0x220000000000ull;
 260   static const uptr kShadowBeg     = 0x010000000000ull;
 261   static const uptr kShadowEnd     = 0x100000000000ull;
 262   static const uptr kHeapMemBeg    = 0x3d0000000000ull;
 263   static const uptr kHeapMemEnd    = 0x3e0000000000ull;
 264   static const uptr kLoAppMemBeg   = 0x000000001000ull;
 265   static const uptr kLoAppMemEnd   = 0x010000000000ull;
 266   static const uptr kHiAppMemBeg   = 0x3e8000000000ull;
 267   static const uptr kHiAppMemEnd   = 0x400000000000ull; // 46 bits
 268   static const uptr kAppMemMsk     = 0x3c0000000000ull;
 269   static const uptr kAppMemXor     = 0x020000000000ull;
 270   static const uptr kVdsoBeg       = 0x7800000000000000ull;
 271 };
 272
 273 // Indicates the runtime will define the memory regions at runtime.
 274 #define TSAN_RUNTIME_VMA 1
 275 #endif
 276
 277 #elif SANITIZER_GO && !SANITIZER_WINDOWS
 278
 279 /* Go on linux, darwin and freebsd
 280 0000 0000 1000 - 0000 1000 0000: executable
 281 0000 1000 0000 - 00c0 0000 0000: -
 282 00c0 0000 0000 - 00e0 0000 0000: heap
 283 00e0 0000 0000 - 2000 0000 0000: -
 284 2000 0000 0000 - 2380 0000 0000: shadow
 285 2380 0000 0000 - 3000 0000 0000: -
 286 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
 287 4000 0000 0000 - 6000 0000 0000: -
 288 6000 0000 0000 - 6200 0000 0000: traces
 289 6200 0000 0000 - 8000 0000 0000: -
 290 */
 291
 292 struct Mapping {
 293   static const uptr kMetaShadowBeg = 0x300000000000ull;
 294   static const uptr kMetaShadowEnd = 0x400000000000ull;
 295   static const uptr kTraceMemBeg   = 0x600000000000ull;
 296   static const uptr kTraceMemEnd   = 0x620000000000ull;
 297   static const uptr kShadowBeg     = 0x200000000000ull;
 298   static const uptr kShadowEnd     = 0x238000000000ull;
 299   static const uptr kAppMemBeg     = 0x000000001000ull;
 300   static const uptr kAppMemEnd     = 0x00e000000000ull;
 301 };
 302
 303 #elif SANITIZER_GO && SANITIZER_WINDOWS
 304
 305 /* Go on windows
 306 0000 0000 1000 - 0000 1000 0000: executable
 307 0000 1000 0000 - 00f8 0000 0000: -
 308 00c0 0000 0000 - 00e0 0000 0000: heap
 309 00e0 0000 0000 - 0100 0000 0000: -
 310 0100 0000 0000 - 0500 0000 0000: shadow
 311 0500 0000 0000 - 0560 0000 0000: -
 312 0560 0000 0000 - 0760 0000 0000: traces
 313 0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects)
 314 07d0 0000 0000 - 8000 0000 0000: -
 315 */
 316
 317 struct Mapping {
 318   static const uptr kMetaShadowBeg = 0x076000000000ull;
 319   static const uptr kMetaShadowEnd = 0x07d000000000ull;
 320   static const uptr kTraceMemBeg   = 0x056000000000ull;
 321   static const uptr kTraceMemEnd   = 0x076000000000ull;
 322   static const uptr kShadowBeg     = 0x010000000000ull;
 323   static const uptr kShadowEnd     = 0x050000000000ull;
 324   static const uptr kAppMemBeg     = 0x000000001000ull;
 325   static const uptr kAppMemEnd     = 0x00e000000000ull;
 326 };
 327
 328 #else
 329 # error "Unknown platform"
 330 #endif
 331
 332
 333 #ifdef TSAN_RUNTIME_VMA
 334 extern uptr vmaSize;
 335 #endif
 336
 337
 338 enum MappingType {
 339   MAPPING_LO_APP_BEG,
 340   MAPPING_LO_APP_END,
 341   MAPPING_HI_APP_BEG,
 342   MAPPING_HI_APP_END,
 343 #ifdef TSAN_MID_APP_RANGE
 344   MAPPING_MID_APP_BEG,
 345   MAPPING_MID_APP_END,
 346 #endif
 347   MAPPING_HEAP_BEG,
 348   MAPPING_HEAP_END,
 349   MAPPING_APP_BEG,
 350   MAPPING_APP_END,
 351   MAPPING_SHADOW_BEG,
 352   MAPPING_SHADOW_END,
 353   MAPPING_META_SHADOW_BEG,
 354   MAPPING_META_SHADOW_END,
 355   MAPPING_TRACE_BEG,
 356   MAPPING_TRACE_END,
 357   MAPPING_VDSO_BEG,
 358 };
 359
 360 template<typename Mapping, int Type>
 361 uptr MappingImpl(void) {
 362   switch (Type) {
 363 #if !SANITIZER_GO
 364     case MAPPING_LO_APP_BEG: return Mapping::kLoAppMemBeg;
 365     case MAPPING_LO_APP_END: return Mapping::kLoAppMemEnd;
 366 # ifdef TSAN_MID_APP_RANGE
 367     case MAPPING_MID_APP_BEG: return Mapping::kMidAppMemBeg;
 368     case MAPPING_MID_APP_END: return Mapping::kMidAppMemEnd;
 369 # endif
 370     case MAPPING_HI_APP_BEG: return Mapping::kHiAppMemBeg;
 371     case MAPPING_HI_APP_END: return Mapping::kHiAppMemEnd;
 372     case MAPPING_HEAP_BEG: return Mapping::kHeapMemBeg;
 373     case MAPPING_HEAP_END: return Mapping::kHeapMemEnd;
 374     case MAPPING_VDSO_BEG: return Mapping::kVdsoBeg;
 375 #else
 376     case MAPPING_APP_BEG: return Mapping::kAppMemBeg;
 377     case MAPPING_APP_END: return Mapping::kAppMemEnd;
 378 #endif
 379     case MAPPING_SHADOW_BEG: return Mapping::kShadowBeg;
 380     case MAPPING_SHADOW_END: return Mapping::kShadowEnd;
 381     case MAPPING_META_SHADOW_BEG: return Mapping::kMetaShadowBeg;
 382     case MAPPING_META_SHADOW_END: return Mapping::kMetaShadowEnd;
 383     case MAPPING_TRACE_BEG: return Mapping::kTraceMemBeg;
 384     case MAPPING_TRACE_END: return Mapping::kTraceMemEnd;
 385   }
 386 }
 387
 388 template<int Type>
 389 uptr MappingArchImpl(void) {
 390 #ifdef __aarch64__
 391   switch (vmaSize) {
 392     case 39: return MappingImpl<Mapping39, Type>();
 393     case 42: return MappingImpl<Mapping42, Type>();
 394     case 48: return MappingImpl<Mapping48, Type>();
 395   }
 396   DCHECK(0);
 397   return 0;
 398 #elif defined(__powerpc64__)
 399   if (vmaSize == 44)
 400     return MappingImpl<Mapping44, Type>();
 401   else
 402     return MappingImpl<Mapping46, Type>();
 403   DCHECK(0);
 404 #else
 405   return MappingImpl<Mapping, Type>();
 406 #endif
 407 }
 408
 409 #if !SANITIZER_GO
 410 ALWAYS_INLINE
 411 uptr LoAppMemBeg(void) {
 412   return MappingArchImpl<MAPPING_LO_APP_BEG>();
 413 }
 414 ALWAYS_INLINE
 415 uptr LoAppMemEnd(void) {
 416   return MappingArchImpl<MAPPING_LO_APP_END>();
 417 }
 418
 419 #ifdef TSAN_MID_APP_RANGE
 420 ALWAYS_INLINE
 421 uptr MidAppMemBeg(void) {
 422   return MappingArchImpl<MAPPING_MID_APP_BEG>();
 423 }
 424 ALWAYS_INLINE
 425 uptr MidAppMemEnd(void) {
 426   return MappingArchImpl<MAPPING_MID_APP_END>();
 427 }
 428 #endif
 429
 430 ALWAYS_INLINE
 431 uptr HeapMemBeg(void) {
 432   return MappingArchImpl<MAPPING_HEAP_BEG>();
 433 }
 434 ALWAYS_INLINE
 435 uptr HeapMemEnd(void) {
 436   return MappingArchImpl<MAPPING_HEAP_END>();
 437 }
 438
 439 ALWAYS_INLINE
 440 uptr HiAppMemBeg(void) {
 441   return MappingArchImpl<MAPPING_HI_APP_BEG>();
 442 }
 443 ALWAYS_INLINE
 444 uptr HiAppMemEnd(void) {
 445   return MappingArchImpl<MAPPING_HI_APP_END>();
 446 }
 447
 448 ALWAYS_INLINE
 449 uptr VdsoBeg(void) {
 450   return MappingArchImpl<MAPPING_VDSO_BEG>();
 451 }
 452
 453 #else
 454
 455 ALWAYS_INLINE
 456 uptr AppMemBeg(void) {
 457   return MappingArchImpl<MAPPING_APP_BEG>();
 458 }
 459 ALWAYS_INLINE
 460 uptr AppMemEnd(void) {
 461   return MappingArchImpl<MAPPING_APP_END>();
 462 }
 463
 464 #endif
 465
 466 static inline
 467 bool GetUserRegion(int i, uptr *start, uptr *end) {
 468   switch (i) {
 469   default:
 470     return false;
 471 #if !SANITIZER_GO
 472   case 0:
 473     *start = LoAppMemBeg();
 474     *end = LoAppMemEnd();
 475     return true;
 476   case 1:
 477     *start = HiAppMemBeg();
 478     *end = HiAppMemEnd();
 479     return true;
 480   case 2:
 481     *start = HeapMemBeg();
 482     *end = HeapMemEnd();
 483     return true;
 484 # ifdef TSAN_MID_APP_RANGE
 485   case 3:
 486     *start = MidAppMemBeg();
 487     *end = MidAppMemEnd();
 488     return true;
 489 # endif
 490 #else
 491   case 0:
 492     *start = AppMemBeg();
 493     *end = AppMemEnd();
 494     return true;
 495 #endif
 496   }
 497 }
 498
 499 ALWAYS_INLINE
 500 uptr ShadowBeg(void) {
 501   return MappingArchImpl<MAPPING_SHADOW_BEG>();
 502 }
 503 ALWAYS_INLINE
 504 uptr ShadowEnd(void) {
 505   return MappingArchImpl<MAPPING_SHADOW_END>();
 506 }
 507
 508 ALWAYS_INLINE
 509 uptr MetaShadowBeg(void) {
 510   return MappingArchImpl<MAPPING_META_SHADOW_BEG>();
 511 }
 512 ALWAYS_INLINE
 513 uptr MetaShadowEnd(void) {
 514   return MappingArchImpl<MAPPING_META_SHADOW_END>();
 515 }
 516
 517 ALWAYS_INLINE
 518 uptr TraceMemBeg(void) {
 519   return MappingArchImpl<MAPPING_TRACE_BEG>();
 520 }
 521 ALWAYS_INLINE
 522 uptr TraceMemEnd(void) {
 523   return MappingArchImpl<MAPPING_TRACE_END>();
 524 }
 525
 526
 527 template<typename Mapping>
 528 bool IsAppMemImpl(uptr mem) {
 529 #if !SANITIZER_GO
 530   return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
 531 # ifdef TSAN_MID_APP_RANGE
 532          (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
 533 # endif
 534          (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
 535          (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
 536 #else
 537   return mem >= Mapping::kAppMemBeg && mem < Mapping::kAppMemEnd;
 538 #endif
 539 }
 540
 541 ALWAYS_INLINE
 542 bool IsAppMem(uptr mem) {
 543 #ifdef __aarch64__
 544   switch (vmaSize) {
 545     case 39: return IsAppMemImpl<Mapping39>(mem);
 546     case 42: return IsAppMemImpl<Mapping42>(mem);
 547     case 48: return IsAppMemImpl<Mapping48>(mem);
 548   }
 549   DCHECK(0);
 550   return false;
 551 #elif defined(__powerpc64__)
 552   if (vmaSize == 44)
 553     return IsAppMemImpl<Mapping44>(mem);
 554   else
 555     return IsAppMemImpl<Mapping46>(mem);
 556   DCHECK(0);
 557 #else
 558   return IsAppMemImpl<Mapping>(mem);
 559 #endif
 560 }
 561
 562
 563 template<typename Mapping>
 564 bool IsShadowMemImpl(uptr mem) {
 565   return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
 566 }
 567
 568 ALWAYS_INLINE
 569 bool IsShadowMem(uptr mem) {
 570 #ifdef __aarch64__
 571   switch (vmaSize) {
 572     case 39: return IsShadowMemImpl<Mapping39>(mem);
 573     case 42: return IsShadowMemImpl<Mapping42>(mem);
 574     case 48: return IsShadowMemImpl<Mapping48>(mem);
 575   }
 576   DCHECK(0);
 577   return false;
 578 #elif defined(__powerpc64__)
 579   if (vmaSize == 44)
 580     return IsShadowMemImpl<Mapping44>(mem);
 581   else
 582     return IsShadowMemImpl<Mapping46>(mem);
 583   DCHECK(0);
 584 #else
 585   return IsShadowMemImpl<Mapping>(mem);
 586 #endif
 587 }
 588
 589
 590 template<typename Mapping>
 591 bool IsMetaMemImpl(uptr mem) {
 592   return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
 593 }
 594
 595 ALWAYS_INLINE
 596 bool IsMetaMem(uptr mem) {
 597 #ifdef __aarch64__
 598   switch (vmaSize) {
 599     case 39: return IsMetaMemImpl<Mapping39>(mem);
 600     case 42: return IsMetaMemImpl<Mapping42>(mem);
 601     case 48: return IsMetaMemImpl<Mapping48>(mem);
 602   }
 603   DCHECK(0);
 604   return false;
 605 #elif defined(__powerpc64__)
 606   if (vmaSize == 44)
 607     return IsMetaMemImpl<Mapping44>(mem);
 608   else
 609     return IsMetaMemImpl<Mapping46>(mem);
 610   DCHECK(0);
 611 #else
 612   return IsMetaMemImpl<Mapping>(mem);
 613 #endif
 614 }
 615
 616
 617 template<typename Mapping>
 618 uptr MemToShadowImpl(uptr x) {
 619   DCHECK(IsAppMem(x));
 620 #if !SANITIZER_GO
 621   return (((x) & ~(Mapping::kAppMemMsk | (kShadowCell - 1)))
 622       ^ Mapping::kAppMemXor) * kShadowCnt;
 623 #else
 624 # ifndef SANITIZER_WINDOWS
 625   return ((x & ~(kShadowCell - 1)) * kShadowCnt) | Mapping::kShadowBeg;
 626 # else
 627   return ((x & ~(kShadowCell - 1)) * kShadowCnt) + Mapping::kShadowBeg;
 628 # endif
 629 #endif
 630 }
 631
 632 ALWAYS_INLINE
 633 uptr MemToShadow(uptr x) {
 634 #ifdef __aarch64__
 635   switch (vmaSize) {
 636     case 39: return MemToShadowImpl<Mapping39>(x);
 637     case 42: return MemToShadowImpl<Mapping42>(x);
 638     case 48: return MemToShadowImpl<Mapping48>(x);
 639   }
 640   DCHECK(0);
 641   return 0;
 642 #elif defined(__powerpc64__)
 643   if (vmaSize == 44)
 644     return MemToShadowImpl<Mapping44>(x);
 645   else
 646     return MemToShadowImpl<Mapping46>(x);
 647   DCHECK(0);
 648 #else
 649   return MemToShadowImpl<Mapping>(x);
 650 #endif
 651 }
 652
 653
 654 template<typename Mapping>
 655 u32 *MemToMetaImpl(uptr x) {
 656   DCHECK(IsAppMem(x));
 657 #if !SANITIZER_GO
 658   return (u32*)(((((x) & ~(Mapping::kAppMemMsk | (kMetaShadowCell - 1)))) /
 659       kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg);
 660 #else
 661 # ifndef SANITIZER_WINDOWS
 662   return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
 663       kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg);
 664 # else
 665   return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
 666       kMetaShadowCell * kMetaShadowSize) + Mapping::kMetaShadowBeg);
 667 # endif
 668 #endif
 669 }
 670
 671 ALWAYS_INLINE
 672 u32 *MemToMeta(uptr x) {
 673 #ifdef __aarch64__
 674   switch (vmaSize) {
 675     case 39: return MemToMetaImpl<Mapping39>(x);
 676     case 42: return MemToMetaImpl<Mapping42>(x);
 677     case 48: return MemToMetaImpl<Mapping48>(x);
 678   }
 679   DCHECK(0);
 680   return 0;
 681 #elif defined(__powerpc64__)
 682   if (vmaSize == 44)
 683     return MemToMetaImpl<Mapping44>(x);
 684   else
 685     return MemToMetaImpl<Mapping46>(x);
 686   DCHECK(0);
 687 #else
 688   return MemToMetaImpl<Mapping>(x);
 689 #endif
 690 }
 691
 692
 693 template<typename Mapping>
 694 uptr ShadowToMemImpl(uptr s) {
 695   DCHECK(IsShadowMem(s));
 696 #if !SANITIZER_GO
 697   // The shadow mapping is non-linear and we've lost some bits, so we don't have
 698   // an easy way to restore the original app address. But the mapping is a
 699   // bijection, so we try to restore the address as belonging to low/mid/high
 700   // range consecutively and see if shadow->app->shadow mapping gives us the
 701   // same address.
 702   uptr p = (s / kShadowCnt) ^ Mapping::kAppMemXor;
 703   if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
 704       MemToShadow(p) == s)
 705     return p;
 706 # ifdef TSAN_MID_APP_RANGE
 707   p = ((s / kShadowCnt) ^ Mapping::kAppMemXor) +
 708       (Mapping::kMidAppMemBeg & Mapping::kAppMemMsk);
 709   if (p >= Mapping::kMidAppMemBeg && p < Mapping::kMidAppMemEnd &&
 710       MemToShadow(p) == s)
 711     return p;
 712 # endif
 713   return ((s / kShadowCnt) ^ Mapping::kAppMemXor) | Mapping::kAppMemMsk;
 714 #else  // #if !SANITIZER_GO
 715 # ifndef SANITIZER_WINDOWS
 716   return (s & ~Mapping::kShadowBeg) / kShadowCnt;
 717 # else
 718   return (s - Mapping::kShadowBeg) / kShadowCnt;
 719 # endif // SANITIZER_WINDOWS
 720 #endif
 721 }
 722
 723 ALWAYS_INLINE
 724 uptr ShadowToMem(uptr s) {
 725 #ifdef __aarch64__
 726   switch (vmaSize) {
 727     case 39: return ShadowToMemImpl<Mapping39>(s);
 728     case 42: return ShadowToMemImpl<Mapping42>(s);
 729     case 48: return ShadowToMemImpl<Mapping48>(s);
 730   }
 731   DCHECK(0);
 732   return 0;
 733 #elif defined(__powerpc64__)
 734   if (vmaSize == 44)
 735     return ShadowToMemImpl<Mapping44>(s);
 736   else
 737     return ShadowToMemImpl<Mapping46>(s);
 738   DCHECK(0);
 739 #else
 740   return ShadowToMemImpl<Mapping>(s);
 741 #endif
 742 }
 743
 744
 745
 746 // The additional page is to catch shadow stack overflow as paging fault.
 747 // Windows wants 64K alignment for mmaps.
 748 const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace)
 749     + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1);
 750
 751 template<typename Mapping>
 752 uptr GetThreadTraceImpl(int tid) {
 753   uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize;
 754   DCHECK_LT(p, Mapping::kTraceMemEnd);
 755   return p;
 756 }
 757
 758 ALWAYS_INLINE
 759 uptr GetThreadTrace(int tid) {
 760 #ifdef __aarch64__
 761   switch (vmaSize) {
 762     case 39: return GetThreadTraceImpl<Mapping39>(tid);
 763     case 42: return GetThreadTraceImpl<Mapping42>(tid);
 764     case 48: return GetThreadTraceImpl<Mapping48>(tid);
 765   }
 766   DCHECK(0);
 767   return 0;
 768 #elif defined(__powerpc64__)
 769   if (vmaSize == 44)
 770     return GetThreadTraceImpl<Mapping44>(tid);
 771   else
 772     return GetThreadTraceImpl<Mapping46>(tid);
 773   DCHECK(0);
 774 #else
 775   return GetThreadTraceImpl<Mapping>(tid);
 776 #endif
 777 }
 778
 779
 780 template<typename Mapping>
 781 uptr GetThreadTraceHeaderImpl(int tid) {
 782   uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize
 783       + kTraceSize * sizeof(Event);
 784   DCHECK_LT(p, Mapping::kTraceMemEnd);
 785   return p;
 786 }
 787
 788 ALWAYS_INLINE
 789 uptr GetThreadTraceHeader(int tid) {
 790 #ifdef __aarch64__
 791   switch (vmaSize) {
 792     case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid);
 793     case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid);
 794     case 48: return GetThreadTraceHeaderImpl<Mapping48>(tid);
 795   }
 796   DCHECK(0);
 797   return 0;
 798 #elif defined(__powerpc64__)
 799   if (vmaSize == 44)
 800     return GetThreadTraceHeaderImpl<Mapping44>(tid);
 801   else
 802     return GetThreadTraceHeaderImpl<Mapping46>(tid);
 803   DCHECK(0);
 804 #else
 805   return GetThreadTraceHeaderImpl<Mapping>(tid);
 806 #endif
 807 }
 808
 809 void InitializePlatform();
 810 void InitializePlatformEarly();
 811 void CheckAndProtect();
 812 void InitializeShadowMemoryPlatform();
 813 void FlushShadowMemory();
 814 void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive);
 815 int ExtractResolvFDs(void *state, int *fds, int nfd);
 816 int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
 817
 818 int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
 819     void *abstime), void *c, void *m, void *abstime,
 820     void(*cleanup)(void *arg), void *arg);
 821
 822 void DestroyThreadState();
 823
 824 }  // namespace __tsan
 825
 826 #endif  // TSAN_PLATFORM_H