libsanitizer/tsan/tsan_platform.h

   1 //===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
   2 //
   3 // This file is distributed under the University of Illinois Open Source
   4 // License. See LICENSE.TXT for details.
   5 //
   6 //===----------------------------------------------------------------------===//
   7 //
   8 // This file is a part of ThreadSanitizer (TSan), a race detector.
   9 //
  10 // Platform-specific code.
  11 //===----------------------------------------------------------------------===//
  12
  13 #ifndef TSAN_PLATFORM_H
  14 #define TSAN_PLATFORM_H
  15
  16 #if !defined(__LP64__) && !defined(_WIN64)
  17 # error "Only 64-bit is supported"
  18 #endif
  19
  20 #include "tsan_defs.h"
  21 #include "tsan_trace.h"
  22
  23 namespace __tsan {
  24
  25 #if !SANITIZER_GO
  26
  27 #if defined(__x86_64__)
  28 /*
  29 C/C++ on linux/x86_64 and freebsd/x86_64
  30 0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
  31 0040 0000 0000 - 0100 0000 0000: -
  32 0100 0000 0000 - 2000 0000 0000: shadow
  33 2000 0000 0000 - 3000 0000 0000: -
  34 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
  35 4000 0000 0000 - 5500 0000 0000: -
  36 5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
  37 5680 0000 0000 - 6000 0000 0000: -
  38 6000 0000 0000 - 6200 0000 0000: traces
  39 6200 0000 0000 - 7d00 0000 0000: -
  40 7b00 0000 0000 - 7c00 0000 0000: heap
  41 7c00 0000 0000 - 7e80 0000 0000: -
  42 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
  43 */
  44 struct Mapping {
  45   static const uptr kMetaShadowBeg = 0x300000000000ull;
  46   static const uptr kMetaShadowEnd = 0x340000000000ull;
  47   static const uptr kTraceMemBeg   = 0x600000000000ull;
  48   static const uptr kTraceMemEnd   = 0x620000000000ull;
  49   static const uptr kShadowBeg     = 0x010000000000ull;
  50   static const uptr kShadowEnd     = 0x200000000000ull;
  51   static const uptr kHeapMemBeg    = 0x7b0000000000ull;
  52   static const uptr kHeapMemEnd    = 0x7c0000000000ull;
  53   static const uptr kLoAppMemBeg   = 0x000000001000ull;
  54   static const uptr kLoAppMemEnd   = 0x008000000000ull;
  55   static const uptr kMidAppMemBeg  = 0x550000000000ull;
  56   static const uptr kMidAppMemEnd  = 0x568000000000ull;
  57   static const uptr kHiAppMemBeg   = 0x7e8000000000ull;
  58   static const uptr kHiAppMemEnd   = 0x800000000000ull;
  59   static const uptr kAppMemMsk     = 0x780000000000ull;
  60   static const uptr kAppMemXor     = 0x040000000000ull;
  61   static const uptr kVdsoBeg       = 0xf000000000000000ull;
  62 };
  63
  64 #define TSAN_MID_APP_RANGE 1
  65 #elif defined(__mips64)
  66 /*
  67 C/C++ on linux/mips64
  68 0100 0000 00 - 0200 0000 00: main binary
  69 0200 0000 00 - 1400 0000 00: -
  70 1400 0000 00 - 2400 0000 00: shadow
  71 2400 0000 00 - 3000 0000 00: -
  72 3000 0000 00 - 4000 0000 00: metainfo (memory blocks and sync objects)
  73 4000 0000 00 - 6000 0000 00: -
  74 6000 0000 00 - 6200 0000 00: traces
  75 6200 0000 00 - fe00 0000 00: -
  76 fe00 0000 00 - ff00 0000 00: heap
  77 ff00 0000 00 - ff80 0000 00: -
  78 ff80 0000 00 - ffff ffff ff: modules and main thread stack
  79 */
  80 struct Mapping {
  81   static const uptr kMetaShadowBeg = 0x4000000000ull;
  82   static const uptr kMetaShadowEnd = 0x5000000000ull;
  83   static const uptr kTraceMemBeg   = 0xb000000000ull;
  84   static const uptr kTraceMemEnd   = 0xb200000000ull;
  85   static const uptr kShadowBeg     = 0x2400000000ull;
  86   static const uptr kShadowEnd     = 0x4000000000ull;
  87   static const uptr kHeapMemBeg    = 0xfe00000000ull;
  88   static const uptr kHeapMemEnd    = 0xff00000000ull;
  89   static const uptr kLoAppMemBeg   = 0x0100000000ull;
  90   static const uptr kLoAppMemEnd   = 0x0200000000ull;
  91   static const uptr kMidAppMemBeg  = 0xaa00000000ull;
  92   static const uptr kMidAppMemEnd  = 0xab00000000ull;
  93   static const uptr kHiAppMemBeg   = 0xff80000000ull;
  94   static const uptr kHiAppMemEnd   = 0xffffffffffull;
  95   static const uptr kAppMemMsk     = 0xf800000000ull;
  96   static const uptr kAppMemXor     = 0x0800000000ull;
  97   static const uptr kVdsoBeg       = 0xfffff00000ull;
  98 };
  99
 100 #define TSAN_MID_APP_RANGE 1
 101 #elif defined(__aarch64__) && defined(__APPLE__)
 102 /*
 103 C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
 104 0000 0000 00 - 0100 0000 00: -                                    (4 GB)
 105 0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks  (4 GB)
 106 0200 0000 00 - 0300 0000 00: heap                                 (4 GB)
 107 0300 0000 00 - 0400 0000 00: -                                    (4 GB)
 108 0400 0000 00 - 0c00 0000 00: shadow memory                       (32 GB)
 109 0c00 0000 00 - 0d00 0000 00: -                                    (4 GB)
 110 0d00 0000 00 - 0e00 0000 00: metainfo                             (4 GB)
 111 0e00 0000 00 - 0f00 0000 00: -                                    (4 GB)
 112 0f00 0000 00 - 1000 0000 00: traces                               (4 GB)
 113 */
 114 struct Mapping {
 115   static const uptr kLoAppMemBeg   = 0x0100000000ull;
 116   static const uptr kLoAppMemEnd   = 0x0200000000ull;
 117   static const uptr kHeapMemBeg    = 0x0200000000ull;
 118   static const uptr kHeapMemEnd    = 0x0300000000ull;
 119   static const uptr kShadowBeg     = 0x0400000000ull;
 120   static const uptr kShadowEnd     = 0x0c00000000ull;
 121   static const uptr kMetaShadowBeg = 0x0d00000000ull;
 122   static const uptr kMetaShadowEnd = 0x0e00000000ull;
 123   static const uptr kTraceMemBeg   = 0x0f00000000ull;
 124   static const uptr kTraceMemEnd   = 0x1000000000ull;
 125   static const uptr kHiAppMemBeg   = 0x1000000000ull;
 126   static const uptr kHiAppMemEnd   = 0x1000000000ull;
 127   static const uptr kAppMemMsk     =          0x0ull;
 128   static const uptr kAppMemXor     =          0x0ull;
 129   static const uptr kVdsoBeg       = 0x7000000000000000ull;
 130 };
 131
 132 #elif defined(__aarch64__)
 133 // AArch64 supports multiple VMA which leads to multiple address transformation
 134 // functions.  To support these multiple VMAS transformations and mappings TSAN
 135 // runtime for AArch64 uses an external memory read (vmaSize) to select which
 136 // mapping to use.  Although slower, it make a same instrumented binary run on
 137 // multiple kernels.
 138
 139 /*
 140 C/C++ on linux/aarch64 (39-bit VMA)
 141 0000 0010 00 - 0100 0000 00: main binary
 142 0100 0000 00 - 0800 0000 00: -
 143 0800 0000 00 - 2000 0000 00: shadow memory
 144 2000 0000 00 - 3100 0000 00: -
 145 3100 0000 00 - 3400 0000 00: metainfo
 146 3400 0000 00 - 5500 0000 00: -
 147 5500 0000 00 - 5600 0000 00: main binary (PIE)
 148 5600 0000 00 - 6000 0000 00: -
 149 6000 0000 00 - 6200 0000 00: traces
 150 6200 0000 00 - 7d00 0000 00: -
 151 7c00 0000 00 - 7d00 0000 00: heap
 152 7d00 0000 00 - 7fff ffff ff: modules and main thread stack
 153 */
 154 struct Mapping39 {
 155   static const uptr kLoAppMemBeg   = 0x0000001000ull;
 156   static const uptr kLoAppMemEnd   = 0x0100000000ull;
 157   static const uptr kShadowBeg     = 0x0800000000ull;
 158   static const uptr kShadowEnd     = 0x2000000000ull;
 159   static const uptr kMetaShadowBeg = 0x3100000000ull;
 160   static const uptr kMetaShadowEnd = 0x3400000000ull;
 161   static const uptr kMidAppMemBeg  = 0x5500000000ull;
 162   static const uptr kMidAppMemEnd  = 0x5600000000ull;
 163   static const uptr kTraceMemBeg   = 0x6000000000ull;
 164   static const uptr kTraceMemEnd   = 0x6200000000ull;
 165   static const uptr kHeapMemBeg    = 0x7c00000000ull;
 166   static const uptr kHeapMemEnd    = 0x7d00000000ull;
 167   static const uptr kHiAppMemBeg   = 0x7e00000000ull;
 168   static const uptr kHiAppMemEnd   = 0x7fffffffffull;
 169   static const uptr kAppMemMsk     = 0x7800000000ull;
 170   static const uptr kAppMemXor     = 0x0200000000ull;
 171   static const uptr kVdsoBeg       = 0x7f00000000ull;
 172 };
 173
 174 /*
 175 C/C++ on linux/aarch64 (42-bit VMA)
 176 00000 0010 00 - 01000 0000 00: main binary
 177 01000 0000 00 - 10000 0000 00: -
 178 10000 0000 00 - 20000 0000 00: shadow memory
 179 20000 0000 00 - 26000 0000 00: -
 180 26000 0000 00 - 28000 0000 00: metainfo
 181 28000 0000 00 - 2aa00 0000 00: -
 182 2aa00 0000 00 - 2ab00 0000 00: main binary (PIE)
 183 2ab00 0000 00 - 36200 0000 00: -
 184 36200 0000 00 - 36240 0000 00: traces
 185 36240 0000 00 - 3e000 0000 00: -
 186 3e000 0000 00 - 3f000 0000 00: heap
 187 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack
 188 */
 189 struct Mapping42 {
 190   static const uptr kLoAppMemBeg   = 0x00000001000ull;
 191   static const uptr kLoAppMemEnd   = 0x01000000000ull;
 192   static const uptr kShadowBeg     = 0x10000000000ull;
 193   static const uptr kShadowEnd     = 0x20000000000ull;
 194   static const uptr kMetaShadowBeg = 0x26000000000ull;
 195   static const uptr kMetaShadowEnd = 0x28000000000ull;
 196   static const uptr kMidAppMemBeg  = 0x2aa00000000ull;
 197   static const uptr kMidAppMemEnd  = 0x2ab00000000ull;
 198   static const uptr kTraceMemBeg   = 0x36200000000ull;
 199   static const uptr kTraceMemEnd   = 0x36400000000ull;
 200   static const uptr kHeapMemBeg    = 0x3e000000000ull;
 201   static const uptr kHeapMemEnd    = 0x3f000000000ull;
 202   static const uptr kHiAppMemBeg   = 0x3f000000000ull;
 203   static const uptr kHiAppMemEnd   = 0x3ffffffffffull;
 204   static const uptr kAppMemMsk     = 0x3c000000000ull;
 205   static const uptr kAppMemXor     = 0x04000000000ull;
 206   static const uptr kVdsoBeg       = 0x37f00000000ull;
 207 };
 208
 209 struct Mapping48 {
 210   static const uptr kLoAppMemBeg   = 0x0000000001000ull;
 211   static const uptr kLoAppMemEnd   = 0x0000200000000ull;
 212   static const uptr kShadowBeg     = 0x0002000000000ull;
 213   static const uptr kShadowEnd     = 0x0004000000000ull;
 214   static const uptr kMetaShadowBeg = 0x0005000000000ull;
 215   static const uptr kMetaShadowEnd = 0x0006000000000ull;
 216   static const uptr kMidAppMemBeg  = 0x0aaaa00000000ull;
 217   static const uptr kMidAppMemEnd  = 0x0aaaf00000000ull;
 218   static const uptr kTraceMemBeg   = 0x0f06000000000ull;
 219   static const uptr kTraceMemEnd   = 0x0f06200000000ull;
 220   static const uptr kHeapMemBeg    = 0x0ffff00000000ull;
 221   static const uptr kHeapMemEnd    = 0x0ffff00000000ull;
 222   static const uptr kHiAppMemBeg   = 0x0ffff00000000ull;
 223   static const uptr kHiAppMemEnd   = 0x1000000000000ull;
 224   static const uptr kAppMemMsk     = 0x0fff800000000ull;
 225   static const uptr kAppMemXor     = 0x0000800000000ull;
 226   static const uptr kVdsoBeg       = 0xffff000000000ull;
 227 };
 228
 229 // Indicates the runtime will define the memory regions at runtime.
 230 #define TSAN_RUNTIME_VMA 1
 231 // Indicates that mapping defines a mid range memory segment.
 232 #define TSAN_MID_APP_RANGE 1
 233 #elif defined(__powerpc64__)
 234 // PPC64 supports multiple VMA which leads to multiple address transformation
 235 // functions.  To support these multiple VMAS transformations and mappings TSAN
 236 // runtime for PPC64 uses an external memory read (vmaSize) to select which
 237 // mapping to use.  Although slower, it make a same instrumented binary run on
 238 // multiple kernels.
 239
 240 /*
 241 C/C++ on linux/powerpc64 (44-bit VMA)
 242 0000 0000 0100 - 0001 0000 0000: main binary
 243 0001 0000 0000 - 0001 0000 0000: -
 244 0001 0000 0000 - 0b00 0000 0000: shadow
 245 0b00 0000 0000 - 0b00 0000 0000: -
 246 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
 247 0d00 0000 0000 - 0d00 0000 0000: -
 248 0d00 0000 0000 - 0f00 0000 0000: traces
 249 0f00 0000 0000 - 0f00 0000 0000: -
 250 0f00 0000 0000 - 0f50 0000 0000: heap
 251 0f50 0000 0000 - 0f60 0000 0000: -
 252 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
 253 */
 254 struct Mapping44 {
 255   static const uptr kMetaShadowBeg = 0x0b0000000000ull;
 256   static const uptr kMetaShadowEnd = 0x0d0000000000ull;
 257   static const uptr kTraceMemBeg   = 0x0d0000000000ull;
 258   static const uptr kTraceMemEnd   = 0x0f0000000000ull;
 259   static const uptr kShadowBeg     = 0x000100000000ull;
 260   static const uptr kShadowEnd     = 0x0b0000000000ull;
 261   static const uptr kLoAppMemBeg   = 0x000000000100ull;
 262   static const uptr kLoAppMemEnd   = 0x000100000000ull;
 263   static const uptr kHeapMemBeg    = 0x0f0000000000ull;
 264   static const uptr kHeapMemEnd    = 0x0f5000000000ull;
 265   static const uptr kHiAppMemBeg   = 0x0f6000000000ull;
 266   static const uptr kHiAppMemEnd   = 0x100000000000ull; // 44 bits
 267   static const uptr kAppMemMsk     = 0x0f0000000000ull;
 268   static const uptr kAppMemXor     = 0x002100000000ull;
 269   static const uptr kVdsoBeg       = 0x3c0000000000000ull;
 270 };
 271
 272 /*
 273 C/C++ on linux/powerpc64 (46-bit VMA)
 274 0000 0000 1000 - 0100 0000 0000: main binary
 275 0100 0000 0000 - 0200 0000 0000: -
 276 0100 0000 0000 - 1000 0000 0000: shadow
 277 1000 0000 0000 - 1000 0000 0000: -
 278 1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
 279 2000 0000 0000 - 2000 0000 0000: -
 280 2000 0000 0000 - 2200 0000 0000: traces
 281 2200 0000 0000 - 3d00 0000 0000: -
 282 3d00 0000 0000 - 3e00 0000 0000: heap
 283 3e00 0000 0000 - 3e80 0000 0000: -
 284 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
 285 */
 286 struct Mapping46 {
 287   static const uptr kMetaShadowBeg = 0x100000000000ull;
 288   static const uptr kMetaShadowEnd = 0x200000000000ull;
 289   static const uptr kTraceMemBeg   = 0x200000000000ull;
 290   static const uptr kTraceMemEnd   = 0x220000000000ull;
 291   static const uptr kShadowBeg     = 0x010000000000ull;
 292   static const uptr kShadowEnd     = 0x100000000000ull;
 293   static const uptr kHeapMemBeg    = 0x3d0000000000ull;
 294   static const uptr kHeapMemEnd    = 0x3e0000000000ull;
 295   static const uptr kLoAppMemBeg   = 0x000000001000ull;
 296   static const uptr kLoAppMemEnd   = 0x010000000000ull;
 297   static const uptr kHiAppMemBeg   = 0x3e8000000000ull;
 298   static const uptr kHiAppMemEnd   = 0x400000000000ull; // 46 bits
 299   static const uptr kAppMemMsk     = 0x3c0000000000ull;
 300   static const uptr kAppMemXor     = 0x020000000000ull;
 301   static const uptr kVdsoBeg       = 0x7800000000000000ull;
 302 };
 303
 304 /*
 305 C/C++ on linux/powerpc64 (47-bit VMA)
 306 0000 0000 1000 - 0100 0000 0000: main binary
 307 0100 0000 0000 - 0200 0000 0000: -
 308 0100 0000 0000 - 1000 0000 0000: shadow
 309 1000 0000 0000 - 1000 0000 0000: -
 310 1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
 311 2000 0000 0000 - 2000 0000 0000: -
 312 2000 0000 0000 - 2200 0000 0000: traces
 313 2200 0000 0000 - 7d00 0000 0000: -
 314 7d00 0000 0000 - 7e00 0000 0000: heap
 315 7e00 0000 0000 - 7e80 0000 0000: -
 316 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
 317 */
 318 struct Mapping47 {
 319   static const uptr kMetaShadowBeg = 0x100000000000ull;
 320   static const uptr kMetaShadowEnd = 0x200000000000ull;
 321   static const uptr kTraceMemBeg   = 0x200000000000ull;
 322   static const uptr kTraceMemEnd   = 0x220000000000ull;
 323   static const uptr kShadowBeg     = 0x010000000000ull;
 324   static const uptr kShadowEnd     = 0x100000000000ull;
 325   static const uptr kHeapMemBeg    = 0x7d0000000000ull;
 326   static const uptr kHeapMemEnd    = 0x7e0000000000ull;
 327   static const uptr kLoAppMemBeg   = 0x000000001000ull;
 328   static const uptr kLoAppMemEnd   = 0x010000000000ull;
 329   static const uptr kHiAppMemBeg   = 0x7e8000000000ull;
 330   static const uptr kHiAppMemEnd   = 0x800000000000ull; // 47 bits
 331   static const uptr kAppMemMsk     = 0x7c0000000000ull;
 332   static const uptr kAppMemXor     = 0x020000000000ull;
 333   static const uptr kVdsoBeg       = 0x7800000000000000ull;
 334 };
 335
 336 // Indicates the runtime will define the memory regions at runtime.
 337 #define TSAN_RUNTIME_VMA 1
 338 #endif
 339
 340 #elif SANITIZER_GO && !SANITIZER_WINDOWS
 341
 342 /* Go on linux, darwin and freebsd
 343 0000 0000 1000 - 0000 1000 0000: executable
 344 0000 1000 0000 - 00c0 0000 0000: -
 345 00c0 0000 0000 - 00e0 0000 0000: heap
 346 00e0 0000 0000 - 2000 0000 0000: -
 347 2000 0000 0000 - 2380 0000 0000: shadow
 348 2380 0000 0000 - 3000 0000 0000: -
 349 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
 350 4000 0000 0000 - 6000 0000 0000: -
 351 6000 0000 0000 - 6200 0000 0000: traces
 352 6200 0000 0000 - 8000 0000 0000: -
 353 */
 354
 355 struct Mapping {
 356   static const uptr kMetaShadowBeg = 0x300000000000ull;
 357   static const uptr kMetaShadowEnd = 0x400000000000ull;
 358   static const uptr kTraceMemBeg   = 0x600000000000ull;
 359   static const uptr kTraceMemEnd   = 0x620000000000ull;
 360   static const uptr kShadowBeg     = 0x200000000000ull;
 361   static const uptr kShadowEnd     = 0x238000000000ull;
 362   static const uptr kAppMemBeg     = 0x000000001000ull;
 363   static const uptr kAppMemEnd     = 0x00e000000000ull;
 364 };
 365
 366 #elif SANITIZER_GO && SANITIZER_WINDOWS
 367
 368 /* Go on windows
 369 0000 0000 1000 - 0000 1000 0000: executable
 370 0000 1000 0000 - 00f8 0000 0000: -
 371 00c0 0000 0000 - 00e0 0000 0000: heap
 372 00e0 0000 0000 - 0100 0000 0000: -
 373 0100 0000 0000 - 0500 0000 0000: shadow
 374 0500 0000 0000 - 0560 0000 0000: -
 375 0560 0000 0000 - 0760 0000 0000: traces
 376 0760 0000 0000 - 07d0 0000 0000: metainfo (memory blocks and sync objects)
 377 07d0 0000 0000 - 8000 0000 0000: -
 378 */
 379
 380 struct Mapping {
 381   static const uptr kMetaShadowBeg = 0x076000000000ull;
 382   static const uptr kMetaShadowEnd = 0x07d000000000ull;
 383   static const uptr kTraceMemBeg   = 0x056000000000ull;
 384   static const uptr kTraceMemEnd   = 0x076000000000ull;
 385   static const uptr kShadowBeg     = 0x010000000000ull;
 386   static const uptr kShadowEnd     = 0x050000000000ull;
 387   static const uptr kAppMemBeg     = 0x000000001000ull;
 388   static const uptr kAppMemEnd     = 0x00e000000000ull;
 389 };
 390
 391 #else
 392 # error "Unknown platform"
 393 #endif
 394
 395
 396 #ifdef TSAN_RUNTIME_VMA
 397 extern uptr vmaSize;
 398 #endif
 399
 400
 401 enum MappingType {
 402   MAPPING_LO_APP_BEG,
 403   MAPPING_LO_APP_END,
 404   MAPPING_HI_APP_BEG,
 405   MAPPING_HI_APP_END,
 406 #ifdef TSAN_MID_APP_RANGE
 407   MAPPING_MID_APP_BEG,
 408   MAPPING_MID_APP_END,
 409 #endif
 410   MAPPING_HEAP_BEG,
 411   MAPPING_HEAP_END,
 412   MAPPING_APP_BEG,
 413   MAPPING_APP_END,
 414   MAPPING_SHADOW_BEG,
 415   MAPPING_SHADOW_END,
 416   MAPPING_META_SHADOW_BEG,
 417   MAPPING_META_SHADOW_END,
 418   MAPPING_TRACE_BEG,
 419   MAPPING_TRACE_END,
 420   MAPPING_VDSO_BEG,
 421 };
 422
 423 template<typename Mapping, int Type>
 424 uptr MappingImpl(void) {
 425   switch (Type) {
 426 #if !SANITIZER_GO
 427     case MAPPING_LO_APP_BEG: return Mapping::kLoAppMemBeg;
 428     case MAPPING_LO_APP_END: return Mapping::kLoAppMemEnd;
 429 # ifdef TSAN_MID_APP_RANGE
 430     case MAPPING_MID_APP_BEG: return Mapping::kMidAppMemBeg;
 431     case MAPPING_MID_APP_END: return Mapping::kMidAppMemEnd;
 432 # endif
 433     case MAPPING_HI_APP_BEG: return Mapping::kHiAppMemBeg;
 434     case MAPPING_HI_APP_END: return Mapping::kHiAppMemEnd;
 435     case MAPPING_HEAP_BEG: return Mapping::kHeapMemBeg;
 436     case MAPPING_HEAP_END: return Mapping::kHeapMemEnd;
 437     case MAPPING_VDSO_BEG: return Mapping::kVdsoBeg;
 438 #else
 439     case MAPPING_APP_BEG: return Mapping::kAppMemBeg;
 440     case MAPPING_APP_END: return Mapping::kAppMemEnd;
 441 #endif
 442     case MAPPING_SHADOW_BEG: return Mapping::kShadowBeg;
 443     case MAPPING_SHADOW_END: return Mapping::kShadowEnd;
 444     case MAPPING_META_SHADOW_BEG: return Mapping::kMetaShadowBeg;
 445     case MAPPING_META_SHADOW_END: return Mapping::kMetaShadowEnd;
 446     case MAPPING_TRACE_BEG: return Mapping::kTraceMemBeg;
 447     case MAPPING_TRACE_END: return Mapping::kTraceMemEnd;
 448   }
 449 }
 450
 451 template<int Type>
 452 uptr MappingArchImpl(void) {
 453 #if defined(__aarch64__) && !defined(__APPLE__)
 454   switch (vmaSize) {
 455     case 39: return MappingImpl<Mapping39, Type>();
 456     case 42: return MappingImpl<Mapping42, Type>();
 457     case 48: return MappingImpl<Mapping48, Type>();
 458   }
 459   DCHECK(0);
 460   return 0;
 461 #elif defined(__powerpc64__)
 462   switch (vmaSize) {
 463     case 44: return MappingImpl<Mapping44, Type>();
 464     case 46: return MappingImpl<Mapping46, Type>();
 465     case 47: return MappingImpl<Mapping47, Type>();
 466   }
 467   DCHECK(0);
 468   return 0;
 469 #else
 470   return MappingImpl<Mapping, Type>();
 471 #endif
 472 }
 473
 474 #if !SANITIZER_GO
 475 ALWAYS_INLINE
 476 uptr LoAppMemBeg(void) {
 477   return MappingArchImpl<MAPPING_LO_APP_BEG>();
 478 }
 479 ALWAYS_INLINE
 480 uptr LoAppMemEnd(void) {
 481   return MappingArchImpl<MAPPING_LO_APP_END>();
 482 }
 483
 484 #ifdef TSAN_MID_APP_RANGE
 485 ALWAYS_INLINE
 486 uptr MidAppMemBeg(void) {
 487   return MappingArchImpl<MAPPING_MID_APP_BEG>();
 488 }
 489 ALWAYS_INLINE
 490 uptr MidAppMemEnd(void) {
 491   return MappingArchImpl<MAPPING_MID_APP_END>();
 492 }
 493 #endif
 494
 495 ALWAYS_INLINE
 496 uptr HeapMemBeg(void) {
 497   return MappingArchImpl<MAPPING_HEAP_BEG>();
 498 }
 499 ALWAYS_INLINE
 500 uptr HeapMemEnd(void) {
 501   return MappingArchImpl<MAPPING_HEAP_END>();
 502 }
 503
 504 ALWAYS_INLINE
 505 uptr HiAppMemBeg(void) {
 506   return MappingArchImpl<MAPPING_HI_APP_BEG>();
 507 }
 508 ALWAYS_INLINE
 509 uptr HiAppMemEnd(void) {
 510   return MappingArchImpl<MAPPING_HI_APP_END>();
 511 }
 512
 513 ALWAYS_INLINE
 514 uptr VdsoBeg(void) {
 515   return MappingArchImpl<MAPPING_VDSO_BEG>();
 516 }
 517
 518 #else
 519
 520 ALWAYS_INLINE
 521 uptr AppMemBeg(void) {
 522   return MappingArchImpl<MAPPING_APP_BEG>();
 523 }
 524 ALWAYS_INLINE
 525 uptr AppMemEnd(void) {
 526   return MappingArchImpl<MAPPING_APP_END>();
 527 }
 528
 529 #endif
 530
 531 static inline
 532 bool GetUserRegion(int i, uptr *start, uptr *end) {
 533   switch (i) {
 534   default:
 535     return false;
 536 #if !SANITIZER_GO
 537   case 0:
 538     *start = LoAppMemBeg();
 539     *end = LoAppMemEnd();
 540     return true;
 541   case 1:
 542     *start = HiAppMemBeg();
 543     *end = HiAppMemEnd();
 544     return true;
 545   case 2:
 546     *start = HeapMemBeg();
 547     *end = HeapMemEnd();
 548     return true;
 549 # ifdef TSAN_MID_APP_RANGE
 550   case 3:
 551     *start = MidAppMemBeg();
 552     *end = MidAppMemEnd();
 553     return true;
 554 # endif
 555 #else
 556   case 0:
 557     *start = AppMemBeg();
 558     *end = AppMemEnd();
 559     return true;
 560 #endif
 561   }
 562 }
 563
 564 ALWAYS_INLINE
 565 uptr ShadowBeg(void) {
 566   return MappingArchImpl<MAPPING_SHADOW_BEG>();
 567 }
 568 ALWAYS_INLINE
 569 uptr ShadowEnd(void) {
 570   return MappingArchImpl<MAPPING_SHADOW_END>();
 571 }
 572
 573 ALWAYS_INLINE
 574 uptr MetaShadowBeg(void) {
 575   return MappingArchImpl<MAPPING_META_SHADOW_BEG>();
 576 }
 577 ALWAYS_INLINE
 578 uptr MetaShadowEnd(void) {
 579   return MappingArchImpl<MAPPING_META_SHADOW_END>();
 580 }
 581
 582 ALWAYS_INLINE
 583 uptr TraceMemBeg(void) {
 584   return MappingArchImpl<MAPPING_TRACE_BEG>();
 585 }
 586 ALWAYS_INLINE
 587 uptr TraceMemEnd(void) {
 588   return MappingArchImpl<MAPPING_TRACE_END>();
 589 }
 590
 591
 592 template<typename Mapping>
 593 bool IsAppMemImpl(uptr mem) {
 594 #if !SANITIZER_GO
 595   return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
 596 # ifdef TSAN_MID_APP_RANGE
 597          (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
 598 # endif
 599          (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
 600          (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
 601 #else
 602   return mem >= Mapping::kAppMemBeg && mem < Mapping::kAppMemEnd;
 603 #endif
 604 }
 605
 606 ALWAYS_INLINE
 607 bool IsAppMem(uptr mem) {
 608 #if defined(__aarch64__) && !defined(__APPLE__)
 609   switch (vmaSize) {
 610     case 39: return IsAppMemImpl<Mapping39>(mem);
 611     case 42: return IsAppMemImpl<Mapping42>(mem);
 612     case 48: return IsAppMemImpl<Mapping48>(mem);
 613   }
 614   DCHECK(0);
 615   return false;
 616 #elif defined(__powerpc64__)
 617   switch (vmaSize) {
 618     case 44: return IsAppMemImpl<Mapping44>(mem);
 619     case 46: return IsAppMemImpl<Mapping46>(mem);
 620     case 47: return IsAppMemImpl<Mapping47>(mem);
 621   }
 622   DCHECK(0);
 623   return false;
 624 #else
 625   return IsAppMemImpl<Mapping>(mem);
 626 #endif
 627 }
 628
 629
 630 template<typename Mapping>
 631 bool IsShadowMemImpl(uptr mem) {
 632   return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
 633 }
 634
 635 ALWAYS_INLINE
 636 bool IsShadowMem(uptr mem) {
 637 #if defined(__aarch64__) && !defined(__APPLE__)
 638   switch (vmaSize) {
 639     case 39: return IsShadowMemImpl<Mapping39>(mem);
 640     case 42: return IsShadowMemImpl<Mapping42>(mem);
 641     case 48: return IsShadowMemImpl<Mapping48>(mem);
 642   }
 643   DCHECK(0);
 644   return false;
 645 #elif defined(__powerpc64__)
 646   switch (vmaSize) {
 647     case 44: return IsShadowMemImpl<Mapping44>(mem);
 648     case 46: return IsShadowMemImpl<Mapping46>(mem);
 649     case 47: return IsShadowMemImpl<Mapping47>(mem);
 650   }
 651   DCHECK(0);
 652   return false;
 653 #else
 654   return IsShadowMemImpl<Mapping>(mem);
 655 #endif
 656 }
 657
 658
 659 template<typename Mapping>
 660 bool IsMetaMemImpl(uptr mem) {
 661   return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
 662 }
 663
 664 ALWAYS_INLINE
 665 bool IsMetaMem(uptr mem) {
 666 #if defined(__aarch64__) && !defined(__APPLE__)
 667   switch (vmaSize) {
 668     case 39: return IsMetaMemImpl<Mapping39>(mem);
 669     case 42: return IsMetaMemImpl<Mapping42>(mem);
 670     case 48: return IsMetaMemImpl<Mapping48>(mem);
 671   }
 672   DCHECK(0);
 673   return false;
 674 #elif defined(__powerpc64__)
 675   switch (vmaSize) {
 676     case 44: return IsMetaMemImpl<Mapping44>(mem);
 677     case 46: return IsMetaMemImpl<Mapping46>(mem);
 678     case 47: return IsMetaMemImpl<Mapping47>(mem);
 679   }
 680   DCHECK(0);
 681   return false;
 682 #else
 683   return IsMetaMemImpl<Mapping>(mem);
 684 #endif
 685 }
 686
 687
 688 template<typename Mapping>
 689 uptr MemToShadowImpl(uptr x) {
 690   DCHECK(IsAppMem(x));
 691 #if !SANITIZER_GO
 692   return (((x) & ~(Mapping::kAppMemMsk | (kShadowCell - 1)))
 693       ^ Mapping::kAppMemXor) * kShadowCnt;
 694 #else
 695 # ifndef SANITIZER_WINDOWS
 696   return ((x & ~(kShadowCell - 1)) * kShadowCnt) | Mapping::kShadowBeg;
 697 # else
 698   return ((x & ~(kShadowCell - 1)) * kShadowCnt) + Mapping::kShadowBeg;
 699 # endif
 700 #endif
 701 }
 702
 703 ALWAYS_INLINE
 704 uptr MemToShadow(uptr x) {
 705 #if defined(__aarch64__) && !defined(__APPLE__)
 706   switch (vmaSize) {
 707     case 39: return MemToShadowImpl<Mapping39>(x);
 708     case 42: return MemToShadowImpl<Mapping42>(x);
 709     case 48: return MemToShadowImpl<Mapping48>(x);
 710   }
 711   DCHECK(0);
 712   return 0;
 713 #elif defined(__powerpc64__)
 714   switch (vmaSize) {
 715     case 44: return MemToShadowImpl<Mapping44>(x);
 716     case 46: return MemToShadowImpl<Mapping46>(x);
 717     case 47: return MemToShadowImpl<Mapping47>(x);
 718   }
 719   DCHECK(0);
 720   return 0;
 721 #else
 722   return MemToShadowImpl<Mapping>(x);
 723 #endif
 724 }
 725
 726
 727 template<typename Mapping>
 728 u32 *MemToMetaImpl(uptr x) {
 729   DCHECK(IsAppMem(x));
 730 #if !SANITIZER_GO
 731   return (u32*)(((((x) & ~(Mapping::kAppMemMsk | (kMetaShadowCell - 1)))) /
 732       kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg);
 733 #else
 734 # ifndef SANITIZER_WINDOWS
 735   return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
 736       kMetaShadowCell * kMetaShadowSize) | Mapping::kMetaShadowBeg);
 737 # else
 738   return (u32*)(((x & ~(kMetaShadowCell - 1)) / \
 739       kMetaShadowCell * kMetaShadowSize) + Mapping::kMetaShadowBeg);
 740 # endif
 741 #endif
 742 }
 743
 744 ALWAYS_INLINE
 745 u32 *MemToMeta(uptr x) {
 746 #if defined(__aarch64__) && !defined(__APPLE__)
 747   switch (vmaSize) {
 748     case 39: return MemToMetaImpl<Mapping39>(x);
 749     case 42: return MemToMetaImpl<Mapping42>(x);
 750     case 48: return MemToMetaImpl<Mapping48>(x);
 751   }
 752   DCHECK(0);
 753   return 0;
 754 #elif defined(__powerpc64__)
 755   switch (vmaSize) {
 756     case 44: return MemToMetaImpl<Mapping44>(x);
 757     case 46: return MemToMetaImpl<Mapping46>(x);
 758     case 47: return MemToMetaImpl<Mapping47>(x);
 759   }
 760   DCHECK(0);
 761   return 0;
 762 #else
 763   return MemToMetaImpl<Mapping>(x);
 764 #endif
 765 }
 766
 767
 768 template<typename Mapping>
 769 uptr ShadowToMemImpl(uptr s) {
 770   DCHECK(IsShadowMem(s));
 771 #if !SANITIZER_GO
 772   // The shadow mapping is non-linear and we've lost some bits, so we don't have
 773   // an easy way to restore the original app address. But the mapping is a
 774   // bijection, so we try to restore the address as belonging to low/mid/high
 775   // range consecutively and see if shadow->app->shadow mapping gives us the
 776   // same address.
 777   uptr p = (s / kShadowCnt) ^ Mapping::kAppMemXor;
 778   if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
 779       MemToShadow(p) == s)
 780     return p;
 781 # ifdef TSAN_MID_APP_RANGE
 782   p = ((s / kShadowCnt) ^ Mapping::kAppMemXor) +
 783       (Mapping::kMidAppMemBeg & Mapping::kAppMemMsk);
 784   if (p >= Mapping::kMidAppMemBeg && p < Mapping::kMidAppMemEnd &&
 785       MemToShadow(p) == s)
 786     return p;
 787 # endif
 788   return ((s / kShadowCnt) ^ Mapping::kAppMemXor) | Mapping::kAppMemMsk;
 789 #else  // #if !SANITIZER_GO
 790 # ifndef SANITIZER_WINDOWS
 791   return (s & ~Mapping::kShadowBeg) / kShadowCnt;
 792 # else
 793   return (s - Mapping::kShadowBeg) / kShadowCnt;
 794 # endif // SANITIZER_WINDOWS
 795 #endif
 796 }
 797
 798 ALWAYS_INLINE
 799 uptr ShadowToMem(uptr s) {
 800 #if defined(__aarch64__) && !defined(__APPLE__)
 801   switch (vmaSize) {
 802     case 39: return ShadowToMemImpl<Mapping39>(s);
 803     case 42: return ShadowToMemImpl<Mapping42>(s);
 804     case 48: return ShadowToMemImpl<Mapping48>(s);
 805   }
 806   DCHECK(0);
 807   return 0;
 808 #elif defined(__powerpc64__)
 809   switch (vmaSize) {
 810     case 44: return ShadowToMemImpl<Mapping44>(s);
 811     case 46: return ShadowToMemImpl<Mapping46>(s);
 812     case 47: return ShadowToMemImpl<Mapping47>(s);
 813   }
 814   DCHECK(0);
 815   return 0;
 816 #else
 817   return ShadowToMemImpl<Mapping>(s);
 818 #endif
 819 }
 820
 821
 822
 823 // The additional page is to catch shadow stack overflow as paging fault.
 824 // Windows wants 64K alignment for mmaps.
 825 const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace)
 826     + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1);
 827
 828 template<typename Mapping>
 829 uptr GetThreadTraceImpl(int tid) {
 830   uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize;
 831   DCHECK_LT(p, Mapping::kTraceMemEnd);
 832   return p;
 833 }
 834
 835 ALWAYS_INLINE
 836 uptr GetThreadTrace(int tid) {
 837 #if defined(__aarch64__) && !defined(__APPLE__)
 838   switch (vmaSize) {
 839     case 39: return GetThreadTraceImpl<Mapping39>(tid);
 840     case 42: return GetThreadTraceImpl<Mapping42>(tid);
 841     case 48: return GetThreadTraceImpl<Mapping48>(tid);
 842   }
 843   DCHECK(0);
 844   return 0;
 845 #elif defined(__powerpc64__)
 846   switch (vmaSize) {
 847     case 44: return GetThreadTraceImpl<Mapping44>(tid);
 848     case 46: return GetThreadTraceImpl<Mapping46>(tid);
 849     case 47: return GetThreadTraceImpl<Mapping47>(tid);
 850   }
 851   DCHECK(0);
 852   return 0;
 853 #else
 854   return GetThreadTraceImpl<Mapping>(tid);
 855 #endif
 856 }
 857
 858
 859 template<typename Mapping>
 860 uptr GetThreadTraceHeaderImpl(int tid) {
 861   uptr p = Mapping::kTraceMemBeg + (uptr)tid * kTotalTraceSize
 862       + kTraceSize * sizeof(Event);
 863   DCHECK_LT(p, Mapping::kTraceMemEnd);
 864   return p;
 865 }
 866
 867 ALWAYS_INLINE
 868 uptr GetThreadTraceHeader(int tid) {
 869 #if defined(__aarch64__) && !defined(__APPLE__)
 870   switch (vmaSize) {
 871     case 39: return GetThreadTraceHeaderImpl<Mapping39>(tid);
 872     case 42: return GetThreadTraceHeaderImpl<Mapping42>(tid);
 873     case 48: return GetThreadTraceHeaderImpl<Mapping48>(tid);
 874   }
 875   DCHECK(0);
 876   return 0;
 877 #elif defined(__powerpc64__)
 878   switch (vmaSize) {
 879     case 44: return GetThreadTraceHeaderImpl<Mapping44>(tid);
 880     case 46: return GetThreadTraceHeaderImpl<Mapping46>(tid);
 881     case 47: return GetThreadTraceHeaderImpl<Mapping47>(tid);
 882   }
 883   DCHECK(0);
 884   return 0;
 885 #else
 886   return GetThreadTraceHeaderImpl<Mapping>(tid);
 887 #endif
 888 }
 889
 890 void InitializePlatform();
 891 void InitializePlatformEarly();
 892 void CheckAndProtect();
 893 void InitializeShadowMemoryPlatform();
 894 void FlushShadowMemory();
 895 void WriteMemoryProfile(char *buf, uptr buf_size, uptr nthread, uptr nlive);
 896 int ExtractResolvFDs(void *state, int *fds, int nfd);
 897 int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
 898 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
 899
 900 int call_pthread_cancel_with_cleanup(int(*fn)(void *c, void *m,
 901     void *abstime), void *c, void *m, void *abstime,
 902     void(*cleanup)(void *arg), void *arg);
 903
 904 void DestroyThreadState();
 905
 906 }  // namespace __tsan
 907
 908 #endif  // TSAN_PLATFORM_H