libsanitizer/tsan/tsan_platform.h

   1 //===-- tsan_platform.h -----------------------------------------*- C++ -*-===//
   2 //
   3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
   4 // See https://llvm.org/LICENSE.txt for license information.
   5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
   6 //
   7 //===----------------------------------------------------------------------===//
   8 //
   9 // This file is a part of ThreadSanitizer (TSan), a race detector.
  10 //
  11 // Platform-specific code.
  12 //===----------------------------------------------------------------------===//
  13
  14 #ifndef TSAN_PLATFORM_H
  15 #define TSAN_PLATFORM_H
  16
  17 #if !defined(__LP64__) && !defined(_WIN64)
  18 # error "Only 64-bit is supported"
  19 #endif
  20
  21 #include "tsan_defs.h"
  22 #include "tsan_trace.h"
  23
  24 namespace __tsan {
  25
  26 enum {
  27   // App memory is not mapped onto shadow memory range.
  28   kBrokenMapping = 1 << 0,
  29   // Mapping app memory and back does not produce the same address,
  30   // this can lead to wrong addresses in reports and potentially
  31   // other bad consequences.
  32   kBrokenReverseMapping = 1 << 1,
  33   // Mapping is non-linear for linear user range.
  34   // This is bad and can lead to unpredictable memory corruptions, etc
  35   // because range access functions assume linearity.
  36   kBrokenLinearity = 1 << 2,
  37 };
  38
  39 /*
  40 C/C++ on linux/x86_64 and freebsd/x86_64
  41 0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB)
  42 0040 0000 0000 - 0100 0000 0000: -
  43 0100 0000 0000 - 2000 0000 0000: shadow
  44 2000 0000 0000 - 3000 0000 0000: -
  45 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
  46 4000 0000 0000 - 5500 0000 0000: -
  47 5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels
  48 5680 0000 0000 - 6000 0000 0000: -
  49 6000 0000 0000 - 6200 0000 0000: traces
  50 6200 0000 0000 - 7d00 0000 0000: -
  51 7b00 0000 0000 - 7c00 0000 0000: heap
  52 7c00 0000 0000 - 7e80 0000 0000: -
  53 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
  54
  55 C/C++ on netbsd/amd64 can reuse the same mapping:
  56  * The address space starts from 0x1000 (option with 0x0) and ends with
  57    0x7f7ffffff000.
  58  * LoAppMem-kHeapMemEnd can be reused as it is.
  59  * No VDSO support.
  60  * No MidAppMem region.
  61  * No additional HeapMem region.
  62  * HiAppMem contains the stack, loader, shared libraries and heap.
  63  * Stack on NetBSD/amd64 has prereserved 128MB.
  64  * Heap grows downwards (top-down).
  65  * ASLR must be disabled per-process or globally.
  66 */
  67 struct Mapping48AddressSpace {
  68   static const uptr kMetaShadowBeg = 0x300000000000ull;
  69   static const uptr kMetaShadowEnd = 0x340000000000ull;
  70   static const uptr kTraceMemBeg   = 0x600000000000ull;
  71   static const uptr kTraceMemEnd   = 0x620000000000ull;
  72   static const uptr kShadowBeg     = 0x010000000000ull;
  73   static const uptr kShadowEnd     = 0x200000000000ull;
  74   static const uptr kHeapMemBeg    = 0x7b0000000000ull;
  75   static const uptr kHeapMemEnd    = 0x7c0000000000ull;
  76   static const uptr kLoAppMemBeg   = 0x000000001000ull;
  77   static const uptr kLoAppMemEnd   = 0x008000000000ull;
  78   static const uptr kMidAppMemBeg  = 0x550000000000ull;
  79   static const uptr kMidAppMemEnd  = 0x568000000000ull;
  80   static const uptr kHiAppMemBeg   = 0x7e8000000000ull;
  81   static const uptr kHiAppMemEnd   = 0x800000000000ull;
  82   static const uptr kShadowMsk = 0x780000000000ull;
  83   static const uptr kShadowXor = 0x040000000000ull;
  84   static const uptr kShadowAdd = 0x000000000000ull;
  85   static const uptr kVdsoBeg       = 0xf000000000000000ull;
  86 };
  87
  88 /*
  89 C/C++ on linux/mips64 (40-bit VMA)
  90 0000 0000 00 - 0100 0000 00: -                                           (4 GB)
  91 0100 0000 00 - 0200 0000 00: main binary                                 (4 GB)
  92 0200 0000 00 - 2000 0000 00: -                                         (120 GB)
  93 2000 0000 00 - 4000 0000 00: shadow                                    (128 GB)
  94 4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects)  (64 GB)
  95 5000 0000 00 - aa00 0000 00: -                                         (360 GB)
  96 aa00 0000 00 - ab00 0000 00: main binary (PIE)                           (4 GB)
  97 ab00 0000 00 - b000 0000 00: -                                          (20 GB)
  98 b000 0000 00 - b200 0000 00: traces                                      (8 GB)
  99 b200 0000 00 - fe00 0000 00: -                                         (304 GB)
 100 fe00 0000 00 - ff00 0000 00: heap                                        (4 GB)
 101 ff00 0000 00 - ff80 0000 00: -                                           (2 GB)
 102 ff80 0000 00 - ffff ffff ff: modules and main thread stack              (<2 GB)
 103 */
 104 struct MappingMips64_40 {
 105   static const uptr kMetaShadowBeg = 0x4000000000ull;
 106   static const uptr kMetaShadowEnd = 0x5000000000ull;
 107   static const uptr kTraceMemBeg   = 0xb000000000ull;
 108   static const uptr kTraceMemEnd   = 0xb200000000ull;
 109   static const uptr kShadowBeg     = 0x2000000000ull;
 110   static const uptr kShadowEnd     = 0x4000000000ull;
 111   static const uptr kHeapMemBeg    = 0xfe00000000ull;
 112   static const uptr kHeapMemEnd    = 0xff00000000ull;
 113   static const uptr kLoAppMemBeg   = 0x0100000000ull;
 114   static const uptr kLoAppMemEnd   = 0x0200000000ull;
 115   static const uptr kMidAppMemBeg  = 0xaa00000000ull;
 116   static const uptr kMidAppMemEnd  = 0xab00000000ull;
 117   static const uptr kHiAppMemBeg   = 0xff80000000ull;
 118   static const uptr kHiAppMemEnd   = 0xffffffffffull;
 119   static const uptr kShadowMsk = 0xf800000000ull;
 120   static const uptr kShadowXor = 0x0800000000ull;
 121   static const uptr kShadowAdd = 0x0000000000ull;
 122   static const uptr kVdsoBeg       = 0xfffff00000ull;
 123 };
 124
 125 /*
 126 C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM)
 127 0000 0000 00 - 0100 0000 00: -                                    (4 GB)
 128 0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks  (4 GB)
 129 0200 0000 00 - 0300 0000 00: heap                                 (4 GB)
 130 0300 0000 00 - 0400 0000 00: -                                    (4 GB)
 131 0400 0000 00 - 0c00 0000 00: shadow memory                       (32 GB)
 132 0c00 0000 00 - 0d00 0000 00: -                                    (4 GB)
 133 0d00 0000 00 - 0e00 0000 00: metainfo                             (4 GB)
 134 0e00 0000 00 - 0f00 0000 00: -                                    (4 GB)
 135 0f00 0000 00 - 0fc0 0000 00: traces                               (3 GB)
 136 0fc0 0000 00 - 1000 0000 00: -
 137 */
 138 struct MappingAppleAarch64 {
 139   static const uptr kLoAppMemBeg   = 0x0100000000ull;
 140   static const uptr kLoAppMemEnd   = 0x0200000000ull;
 141   static const uptr kHeapMemBeg    = 0x0200000000ull;
 142   static const uptr kHeapMemEnd    = 0x0300000000ull;
 143   static const uptr kShadowBeg     = 0x0400000000ull;
 144   static const uptr kShadowEnd     = 0x0c00000000ull;
 145   static const uptr kMetaShadowBeg = 0x0d00000000ull;
 146   static const uptr kMetaShadowEnd = 0x0e00000000ull;
 147   static const uptr kTraceMemBeg   = 0x0f00000000ull;
 148   static const uptr kTraceMemEnd   = 0x0fc0000000ull;
 149   static const uptr kHiAppMemBeg   = 0x0fc0000000ull;
 150   static const uptr kHiAppMemEnd   = 0x0fc0000000ull;
 151   static const uptr kShadowMsk = 0x0ull;
 152   static const uptr kShadowXor = 0x0ull;
 153   static const uptr kShadowAdd = 0x0ull;
 154   static const uptr kVdsoBeg       = 0x7000000000000000ull;
 155   static const uptr kMidAppMemBeg = 0;
 156   static const uptr kMidAppMemEnd = 0;
 157 };
 158
 159 /*
 160 C/C++ on linux/aarch64 (39-bit VMA)
 161 0000 0010 00 - 0100 0000 00: main binary
 162 0100 0000 00 - 0800 0000 00: -
 163 0800 0000 00 - 2000 0000 00: shadow memory
 164 2000 0000 00 - 3100 0000 00: -
 165 3100 0000 00 - 3400 0000 00: metainfo
 166 3400 0000 00 - 5500 0000 00: -
 167 5500 0000 00 - 5600 0000 00: main binary (PIE)
 168 5600 0000 00 - 6000 0000 00: -
 169 6000 0000 00 - 6200 0000 00: traces
 170 6200 0000 00 - 7d00 0000 00: -
 171 7c00 0000 00 - 7d00 0000 00: heap
 172 7d00 0000 00 - 7fff ffff ff: modules and main thread stack
 173 */
 174 struct MappingAarch64_39 {
 175   static const uptr kLoAppMemBeg   = 0x0000001000ull;
 176   static const uptr kLoAppMemEnd   = 0x0100000000ull;
 177   static const uptr kShadowBeg     = 0x0800000000ull;
 178   static const uptr kShadowEnd     = 0x2000000000ull;
 179   static const uptr kMetaShadowBeg = 0x3100000000ull;
 180   static const uptr kMetaShadowEnd = 0x3400000000ull;
 181   static const uptr kMidAppMemBeg  = 0x5500000000ull;
 182   static const uptr kMidAppMemEnd  = 0x5600000000ull;
 183   static const uptr kTraceMemBeg   = 0x6000000000ull;
 184   static const uptr kTraceMemEnd   = 0x6200000000ull;
 185   static const uptr kHeapMemBeg    = 0x7c00000000ull;
 186   static const uptr kHeapMemEnd    = 0x7d00000000ull;
 187   static const uptr kHiAppMemBeg   = 0x7e00000000ull;
 188   static const uptr kHiAppMemEnd   = 0x7fffffffffull;
 189   static const uptr kShadowMsk = 0x7800000000ull;
 190   static const uptr kShadowXor = 0x0200000000ull;
 191   static const uptr kShadowAdd = 0x0000000000ull;
 192   static const uptr kVdsoBeg       = 0x7f00000000ull;
 193 };
 194
 195 /*
 196 C/C++ on linux/aarch64 (42-bit VMA)
 197 00000 0010 00 - 01000 0000 00: main binary
 198 01000 0000 00 - 10000 0000 00: -
 199 10000 0000 00 - 20000 0000 00: shadow memory
 200 20000 0000 00 - 26000 0000 00: -
 201 26000 0000 00 - 28000 0000 00: metainfo
 202 28000 0000 00 - 2aa00 0000 00: -
 203 2aa00 0000 00 - 2ab00 0000 00: main binary (PIE)
 204 2ab00 0000 00 - 36200 0000 00: -
 205 36200 0000 00 - 36240 0000 00: traces
 206 36240 0000 00 - 3e000 0000 00: -
 207 3e000 0000 00 - 3f000 0000 00: heap
 208 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack
 209 */
 210 struct MappingAarch64_42 {
 211   static const uptr kBroken = kBrokenReverseMapping;
 212   static const uptr kLoAppMemBeg   = 0x00000001000ull;
 213   static const uptr kLoAppMemEnd   = 0x01000000000ull;
 214   static const uptr kShadowBeg     = 0x10000000000ull;
 215   static const uptr kShadowEnd     = 0x20000000000ull;
 216   static const uptr kMetaShadowBeg = 0x26000000000ull;
 217   static const uptr kMetaShadowEnd = 0x28000000000ull;
 218   static const uptr kMidAppMemBeg  = 0x2aa00000000ull;
 219   static const uptr kMidAppMemEnd  = 0x2ab00000000ull;
 220   static const uptr kTraceMemBeg   = 0x36200000000ull;
 221   static const uptr kTraceMemEnd   = 0x36400000000ull;
 222   static const uptr kHeapMemBeg    = 0x3e000000000ull;
 223   static const uptr kHeapMemEnd    = 0x3f000000000ull;
 224   static const uptr kHiAppMemBeg   = 0x3f000000000ull;
 225   static const uptr kHiAppMemEnd   = 0x3ffffffffffull;
 226   static const uptr kShadowMsk = 0x3c000000000ull;
 227   static const uptr kShadowXor = 0x04000000000ull;
 228   static const uptr kShadowAdd = 0x00000000000ull;
 229   static const uptr kVdsoBeg       = 0x37f00000000ull;
 230 };
 231
 232 struct MappingAarch64_48 {
 233   static const uptr kLoAppMemBeg   = 0x0000000001000ull;
 234   static const uptr kLoAppMemEnd   = 0x0000200000000ull;
 235   static const uptr kShadowBeg     = 0x0002000000000ull;
 236   static const uptr kShadowEnd     = 0x0004000000000ull;
 237   static const uptr kMetaShadowBeg = 0x0005000000000ull;
 238   static const uptr kMetaShadowEnd = 0x0006000000000ull;
 239   static const uptr kMidAppMemBeg  = 0x0aaaa00000000ull;
 240   static const uptr kMidAppMemEnd  = 0x0aaaf00000000ull;
 241   static const uptr kTraceMemBeg   = 0x0f06000000000ull;
 242   static const uptr kTraceMemEnd   = 0x0f06200000000ull;
 243   static const uptr kHeapMemBeg    = 0x0ffff00000000ull;
 244   static const uptr kHeapMemEnd    = 0x0ffff00000000ull;
 245   static const uptr kHiAppMemBeg   = 0x0ffff00000000ull;
 246   static const uptr kHiAppMemEnd   = 0x1000000000000ull;
 247   static const uptr kShadowMsk = 0x0fff800000000ull;
 248   static const uptr kShadowXor = 0x0000800000000ull;
 249   static const uptr kShadowAdd = 0x0000000000000ull;
 250   static const uptr kVdsoBeg       = 0xffff000000000ull;
 251 };
 252
 253 /*
 254 C/C++ on linux/powerpc64 (44-bit VMA)
 255 0000 0000 0100 - 0001 0000 0000: main binary
 256 0001 0000 0000 - 0001 0000 0000: -
 257 0001 0000 0000 - 0b00 0000 0000: shadow
 258 0b00 0000 0000 - 0b00 0000 0000: -
 259 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects)
 260 0d00 0000 0000 - 0d00 0000 0000: -
 261 0d00 0000 0000 - 0f00 0000 0000: traces
 262 0f00 0000 0000 - 0f00 0000 0000: -
 263 0f00 0000 0000 - 0f50 0000 0000: heap
 264 0f50 0000 0000 - 0f60 0000 0000: -
 265 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack
 266 */
 267 struct MappingPPC64_44 {
 268   static const uptr kBroken =
 269       kBrokenMapping | kBrokenReverseMapping | kBrokenLinearity;
 270   static const uptr kMetaShadowBeg = 0x0b0000000000ull;
 271   static const uptr kMetaShadowEnd = 0x0d0000000000ull;
 272   static const uptr kTraceMemBeg   = 0x0d0000000000ull;
 273   static const uptr kTraceMemEnd   = 0x0f0000000000ull;
 274   static const uptr kShadowBeg     = 0x000100000000ull;
 275   static const uptr kShadowEnd     = 0x0b0000000000ull;
 276   static const uptr kLoAppMemBeg   = 0x000000000100ull;
 277   static const uptr kLoAppMemEnd   = 0x000100000000ull;
 278   static const uptr kHeapMemBeg    = 0x0f0000000000ull;
 279   static const uptr kHeapMemEnd    = 0x0f5000000000ull;
 280   static const uptr kHiAppMemBeg   = 0x0f6000000000ull;
 281   static const uptr kHiAppMemEnd   = 0x100000000000ull; // 44 bits
 282   static const uptr kShadowMsk = 0x0f0000000000ull;
 283   static const uptr kShadowXor = 0x002100000000ull;
 284   static const uptr kShadowAdd = 0x000000000000ull;
 285   static const uptr kVdsoBeg       = 0x3c0000000000000ull;
 286   static const uptr kMidAppMemBeg = 0;
 287   static const uptr kMidAppMemEnd = 0;
 288 };
 289
 290 /*
 291 C/C++ on linux/powerpc64 (46-bit VMA)
 292 0000 0000 1000 - 0100 0000 0000: main binary
 293 0100 0000 0000 - 0200 0000 0000: -
 294 0100 0000 0000 - 1000 0000 0000: shadow
 295 1000 0000 0000 - 1000 0000 0000: -
 296 1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
 297 2000 0000 0000 - 2000 0000 0000: -
 298 2000 0000 0000 - 2200 0000 0000: traces
 299 2200 0000 0000 - 3d00 0000 0000: -
 300 3d00 0000 0000 - 3e00 0000 0000: heap
 301 3e00 0000 0000 - 3e80 0000 0000: -
 302 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack
 303 */
 304 struct MappingPPC64_46 {
 305   static const uptr kMetaShadowBeg = 0x100000000000ull;
 306   static const uptr kMetaShadowEnd = 0x200000000000ull;
 307   static const uptr kTraceMemBeg   = 0x200000000000ull;
 308   static const uptr kTraceMemEnd   = 0x220000000000ull;
 309   static const uptr kShadowBeg     = 0x010000000000ull;
 310   static const uptr kShadowEnd     = 0x100000000000ull;
 311   static const uptr kHeapMemBeg    = 0x3d0000000000ull;
 312   static const uptr kHeapMemEnd    = 0x3e0000000000ull;
 313   static const uptr kLoAppMemBeg   = 0x000000001000ull;
 314   static const uptr kLoAppMemEnd   = 0x010000000000ull;
 315   static const uptr kHiAppMemBeg   = 0x3e8000000000ull;
 316   static const uptr kHiAppMemEnd   = 0x400000000000ull; // 46 bits
 317   static const uptr kShadowMsk = 0x3c0000000000ull;
 318   static const uptr kShadowXor = 0x020000000000ull;
 319   static const uptr kShadowAdd = 0x000000000000ull;
 320   static const uptr kVdsoBeg       = 0x7800000000000000ull;
 321   static const uptr kMidAppMemBeg = 0;
 322   static const uptr kMidAppMemEnd = 0;
 323 };
 324
 325 /*
 326 C/C++ on linux/powerpc64 (47-bit VMA)
 327 0000 0000 1000 - 0100 0000 0000: main binary
 328 0100 0000 0000 - 0200 0000 0000: -
 329 0100 0000 0000 - 1000 0000 0000: shadow
 330 1000 0000 0000 - 1000 0000 0000: -
 331 1000 0000 0000 - 2000 0000 0000: metainfo (memory blocks and sync objects)
 332 2000 0000 0000 - 2000 0000 0000: -
 333 2000 0000 0000 - 2200 0000 0000: traces
 334 2200 0000 0000 - 7d00 0000 0000: -
 335 7d00 0000 0000 - 7e00 0000 0000: heap
 336 7e00 0000 0000 - 7e80 0000 0000: -
 337 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack
 338 */
 339 struct MappingPPC64_47 {
 340   static const uptr kMetaShadowBeg = 0x100000000000ull;
 341   static const uptr kMetaShadowEnd = 0x200000000000ull;
 342   static const uptr kTraceMemBeg   = 0x200000000000ull;
 343   static const uptr kTraceMemEnd   = 0x220000000000ull;
 344   static const uptr kShadowBeg     = 0x010000000000ull;
 345   static const uptr kShadowEnd     = 0x100000000000ull;
 346   static const uptr kHeapMemBeg    = 0x7d0000000000ull;
 347   static const uptr kHeapMemEnd    = 0x7e0000000000ull;
 348   static const uptr kLoAppMemBeg   = 0x000000001000ull;
 349   static const uptr kLoAppMemEnd   = 0x010000000000ull;
 350   static const uptr kHiAppMemBeg   = 0x7e8000000000ull;
 351   static const uptr kHiAppMemEnd   = 0x800000000000ull; // 47 bits
 352   static const uptr kShadowMsk = 0x7c0000000000ull;
 353   static const uptr kShadowXor = 0x020000000000ull;
 354   static const uptr kShadowAdd = 0x000000000000ull;
 355   static const uptr kVdsoBeg       = 0x7800000000000000ull;
 356   static const uptr kMidAppMemBeg = 0;
 357   static const uptr kMidAppMemEnd = 0;
 358 };
 359
 360 /*
 361 C/C++ on linux/s390x
 362 While the kernel provides a 64-bit address space, we have to restrict ourselves
 363 to 48 bits due to how e.g. SyncVar::GetId() works.
 364 0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB
 365 0e00 0000 0000 - 4000 0000 0000: -
 366 4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
 367 8000 0000 0000 - 9000 0000 0000: -
 368 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
 369 9800 0000 0000 - a000 0000 0000: -
 370 a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
 371 b000 0000 0000 - be00 0000 0000: -
 372 be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator)
 373 */
 374 struct MappingS390x {
 375   static const uptr kMetaShadowBeg = 0x900000000000ull;
 376   static const uptr kMetaShadowEnd = 0x980000000000ull;
 377   static const uptr kTraceMemBeg   = 0xa00000000000ull;
 378   static const uptr kTraceMemEnd   = 0xb00000000000ull;
 379   static const uptr kShadowBeg     = 0x400000000000ull;
 380   static const uptr kShadowEnd     = 0x800000000000ull;
 381   static const uptr kHeapMemBeg    = 0xbe0000000000ull;
 382   static const uptr kHeapMemEnd    = 0xc00000000000ull;
 383   static const uptr kLoAppMemBeg   = 0x000000001000ull;
 384   static const uptr kLoAppMemEnd   = 0x0e0000000000ull;
 385   static const uptr kHiAppMemBeg   = 0xc00000004000ull;
 386   static const uptr kHiAppMemEnd   = 0xc00000004000ull;
 387   static const uptr kShadowMsk = 0xb00000000000ull;
 388   static const uptr kShadowXor = 0x100000000000ull;
 389   static const uptr kShadowAdd = 0x000000000000ull;
 390   static const uptr kVdsoBeg       = 0xfffffffff000ull;
 391   static const uptr kMidAppMemBeg = 0;
 392   static const uptr kMidAppMemEnd = 0;
 393 };
 394
 395 /* Go on linux, darwin and freebsd on x86_64
 396 0000 0000 1000 - 0000 1000 0000: executable
 397 0000 1000 0000 - 00c0 0000 0000: -
 398 00c0 0000 0000 - 00e0 0000 0000: heap
 399 00e0 0000 0000 - 2000 0000 0000: -
 400 2000 0000 0000 - 2380 0000 0000: shadow
 401 2380 0000 0000 - 3000 0000 0000: -
 402 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
 403 4000 0000 0000 - 6000 0000 0000: -
 404 6000 0000 0000 - 6200 0000 0000: traces
 405 6200 0000 0000 - 8000 0000 0000: -
 406 */
 407
 408 struct MappingGo48 {
 409   static const uptr kMetaShadowBeg = 0x300000000000ull;
 410   static const uptr kMetaShadowEnd = 0x400000000000ull;
 411   static const uptr kTraceMemBeg   = 0x600000000000ull;
 412   static const uptr kTraceMemEnd   = 0x620000000000ull;
 413   static const uptr kShadowBeg     = 0x200000000000ull;
 414   static const uptr kShadowEnd     = 0x238000000000ull;
 415   static const uptr kLoAppMemBeg = 0x000000001000ull;
 416   static const uptr kLoAppMemEnd = 0x00e000000000ull;
 417   static const uptr kMidAppMemBeg = 0;
 418   static const uptr kMidAppMemEnd = 0;
 419   static const uptr kHiAppMemBeg = 0;
 420   static const uptr kHiAppMemEnd = 0;
 421   static const uptr kHeapMemBeg = 0;
 422   static const uptr kHeapMemEnd = 0;
 423   static const uptr kVdsoBeg = 0;
 424   static const uptr kShadowMsk = 0;
 425   static const uptr kShadowXor = 0;
 426   static const uptr kShadowAdd = 0x200000000000ull;
 427 };
 428
 429 /* Go on windows
 430 0000 0000 1000 - 0000 1000 0000: executable
 431 0000 1000 0000 - 00f8 0000 0000: -
 432 00c0 0000 0000 - 00e0 0000 0000: heap
 433 00e0 0000 0000 - 0100 0000 0000: -
 434 0100 0000 0000 - 0500 0000 0000: shadow
 435 0500 0000 0000 - 0700 0000 0000: traces
 436 0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects)
 437 07d0 0000 0000 - 8000 0000 0000: -
 438 */
 439
 440 struct MappingGoWindows {
 441   static const uptr kMetaShadowBeg = 0x070000000000ull;
 442   static const uptr kMetaShadowEnd = 0x077000000000ull;
 443   static const uptr kTraceMemBeg = 0x050000000000ull;
 444   static const uptr kTraceMemEnd = 0x070000000000ull;
 445   static const uptr kShadowBeg     = 0x010000000000ull;
 446   static const uptr kShadowEnd     = 0x050000000000ull;
 447   static const uptr kLoAppMemBeg = 0x000000001000ull;
 448   static const uptr kLoAppMemEnd = 0x00e000000000ull;
 449   static const uptr kMidAppMemBeg = 0;
 450   static const uptr kMidAppMemEnd = 0;
 451   static const uptr kHiAppMemBeg = 0;
 452   static const uptr kHiAppMemEnd = 0;
 453   static const uptr kHeapMemBeg = 0;
 454   static const uptr kHeapMemEnd = 0;
 455   static const uptr kVdsoBeg = 0;
 456   static const uptr kShadowMsk = 0;
 457   static const uptr kShadowXor = 0;
 458   static const uptr kShadowAdd = 0x010000000000ull;
 459 };
 460
 461 /* Go on linux/powerpc64 (46-bit VMA)
 462 0000 0000 1000 - 0000 1000 0000: executable
 463 0000 1000 0000 - 00c0 0000 0000: -
 464 00c0 0000 0000 - 00e0 0000 0000: heap
 465 00e0 0000 0000 - 2000 0000 0000: -
 466 2000 0000 0000 - 2380 0000 0000: shadow
 467 2380 0000 0000 - 2400 0000 0000: -
 468 2400 0000 0000 - 3400 0000 0000: metainfo (memory blocks and sync objects)
 469 3400 0000 0000 - 3600 0000 0000: -
 470 3600 0000 0000 - 3800 0000 0000: traces
 471 3800 0000 0000 - 4000 0000 0000: -
 472 */
 473
 474 struct MappingGoPPC64_46 {
 475   static const uptr kMetaShadowBeg = 0x240000000000ull;
 476   static const uptr kMetaShadowEnd = 0x340000000000ull;
 477   static const uptr kTraceMemBeg   = 0x360000000000ull;
 478   static const uptr kTraceMemEnd   = 0x380000000000ull;
 479   static const uptr kShadowBeg     = 0x200000000000ull;
 480   static const uptr kShadowEnd     = 0x238000000000ull;
 481   static const uptr kLoAppMemBeg = 0x000000001000ull;
 482   static const uptr kLoAppMemEnd = 0x00e000000000ull;
 483   static const uptr kMidAppMemBeg = 0;
 484   static const uptr kMidAppMemEnd = 0;
 485   static const uptr kHiAppMemBeg = 0;
 486   static const uptr kHiAppMemEnd = 0;
 487   static const uptr kHeapMemBeg = 0;
 488   static const uptr kHeapMemEnd = 0;
 489   static const uptr kVdsoBeg = 0;
 490   static const uptr kShadowMsk = 0;
 491   static const uptr kShadowXor = 0;
 492   static const uptr kShadowAdd = 0x200000000000ull;
 493 };
 494
 495 /* Go on linux/powerpc64 (47-bit VMA)
 496 0000 0000 1000 - 0000 1000 0000: executable
 497 0000 1000 0000 - 00c0 0000 0000: -
 498 00c0 0000 0000 - 00e0 0000 0000: heap
 499 00e0 0000 0000 - 2000 0000 0000: -
 500 2000 0000 0000 - 3000 0000 0000: shadow
 501 3000 0000 0000 - 3000 0000 0000: -
 502 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
 503 4000 0000 0000 - 6000 0000 0000: -
 504 6000 0000 0000 - 6200 0000 0000: traces
 505 6200 0000 0000 - 8000 0000 0000: -
 506 */
 507
 508 struct MappingGoPPC64_47 {
 509   static const uptr kMetaShadowBeg = 0x300000000000ull;
 510   static const uptr kMetaShadowEnd = 0x400000000000ull;
 511   static const uptr kTraceMemBeg   = 0x600000000000ull;
 512   static const uptr kTraceMemEnd   = 0x620000000000ull;
 513   static const uptr kShadowBeg     = 0x200000000000ull;
 514   static const uptr kShadowEnd     = 0x300000000000ull;
 515   static const uptr kLoAppMemBeg = 0x000000001000ull;
 516   static const uptr kLoAppMemEnd = 0x00e000000000ull;
 517   static const uptr kMidAppMemBeg = 0;
 518   static const uptr kMidAppMemEnd = 0;
 519   static const uptr kHiAppMemBeg = 0;
 520   static const uptr kHiAppMemEnd = 0;
 521   static const uptr kHeapMemBeg = 0;
 522   static const uptr kHeapMemEnd = 0;
 523   static const uptr kVdsoBeg = 0;
 524   static const uptr kShadowMsk = 0;
 525   static const uptr kShadowXor = 0;
 526   static const uptr kShadowAdd = 0x200000000000ull;
 527 };
 528
 529 /* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA)
 530 0000 0000 1000 - 0000 1000 0000: executable
 531 0000 1000 0000 - 00c0 0000 0000: -
 532 00c0 0000 0000 - 00e0 0000 0000: heap
 533 00e0 0000 0000 - 2000 0000 0000: -
 534 2000 0000 0000 - 3000 0000 0000: shadow
 535 3000 0000 0000 - 3000 0000 0000: -
 536 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
 537 4000 0000 0000 - 6000 0000 0000: -
 538 6000 0000 0000 - 6200 0000 0000: traces
 539 6200 0000 0000 - 8000 0000 0000: -
 540 */
 541 struct MappingGoAarch64 {
 542   static const uptr kMetaShadowBeg = 0x300000000000ull;
 543   static const uptr kMetaShadowEnd = 0x400000000000ull;
 544   static const uptr kTraceMemBeg   = 0x600000000000ull;
 545   static const uptr kTraceMemEnd   = 0x620000000000ull;
 546   static const uptr kShadowBeg     = 0x200000000000ull;
 547   static const uptr kShadowEnd     = 0x300000000000ull;
 548   static const uptr kLoAppMemBeg = 0x000000001000ull;
 549   static const uptr kLoAppMemEnd = 0x00e000000000ull;
 550   static const uptr kMidAppMemBeg = 0;
 551   static const uptr kMidAppMemEnd = 0;
 552   static const uptr kHiAppMemBeg = 0;
 553   static const uptr kHiAppMemEnd = 0;
 554   static const uptr kHeapMemBeg = 0;
 555   static const uptr kHeapMemEnd = 0;
 556   static const uptr kVdsoBeg = 0;
 557   static const uptr kShadowMsk = 0;
 558   static const uptr kShadowXor = 0;
 559   static const uptr kShadowAdd = 0x200000000000ull;
 560 };
 561
 562 /*
 563 Go on linux/mips64 (47-bit VMA)
 564 0000 0000 1000 - 0000 1000 0000: executable
 565 0000 1000 0000 - 00c0 0000 0000: -
 566 00c0 0000 0000 - 00e0 0000 0000: heap
 567 00e0 0000 0000 - 2000 0000 0000: -
 568 2000 0000 0000 - 3000 0000 0000: shadow
 569 3000 0000 0000 - 3000 0000 0000: -
 570 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects)
 571 4000 0000 0000 - 6000 0000 0000: -
 572 6000 0000 0000 - 6200 0000 0000: traces
 573 6200 0000 0000 - 8000 0000 0000: -
 574 */
 575 struct MappingGoMips64_47 {
 576   static const uptr kMetaShadowBeg = 0x300000000000ull;
 577   static const uptr kMetaShadowEnd = 0x400000000000ull;
 578   static const uptr kTraceMemBeg = 0x600000000000ull;
 579   static const uptr kTraceMemEnd = 0x620000000000ull;
 580   static const uptr kShadowBeg = 0x200000000000ull;
 581   static const uptr kShadowEnd = 0x300000000000ull;
 582   static const uptr kLoAppMemBeg = 0x000000001000ull;
 583   static const uptr kLoAppMemEnd = 0x00e000000000ull;
 584   static const uptr kMidAppMemBeg = 0;
 585   static const uptr kMidAppMemEnd = 0;
 586   static const uptr kHiAppMemBeg = 0;
 587   static const uptr kHiAppMemEnd = 0;
 588   static const uptr kHeapMemBeg = 0;
 589   static const uptr kHeapMemEnd = 0;
 590   static const uptr kVdsoBeg = 0;
 591   static const uptr kShadowMsk = 0;
 592   static const uptr kShadowXor = 0;
 593   static const uptr kShadowAdd = 0x200000000000ull;
 594 };
 595
 596 /*
 597 Go on linux/s390x
 598 0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB
 599 1000 0000 0000 - 4000 0000 0000: -
 600 4000 0000 0000 - 8000 0000 0000: shadow - 64TiB (4 * app)
 601 8000 0000 0000 - 9000 0000 0000: -
 602 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app)
 603 9800 0000 0000 - a000 0000 0000: -
 604 a000 0000 0000 - b000 0000 0000: traces - 16TiB (max history * 128k threads)
 605 */
 606 struct MappingGoS390x {
 607   static const uptr kMetaShadowBeg = 0x900000000000ull;
 608   static const uptr kMetaShadowEnd = 0x980000000000ull;
 609   static const uptr kTraceMemBeg   = 0xa00000000000ull;
 610   static const uptr kTraceMemEnd   = 0xb00000000000ull;
 611   static const uptr kShadowBeg     = 0x400000000000ull;
 612   static const uptr kShadowEnd     = 0x800000000000ull;
 613   static const uptr kLoAppMemBeg = 0x000000001000ull;
 614   static const uptr kLoAppMemEnd = 0x100000000000ull;
 615   static const uptr kMidAppMemBeg = 0;
 616   static const uptr kMidAppMemEnd = 0;
 617   static const uptr kHiAppMemBeg = 0;
 618   static const uptr kHiAppMemEnd = 0;
 619   static const uptr kHeapMemBeg = 0;
 620   static const uptr kHeapMemEnd = 0;
 621   static const uptr kVdsoBeg = 0;
 622   static const uptr kShadowMsk = 0;
 623   static const uptr kShadowXor = 0;
 624   static const uptr kShadowAdd = 0x400000000000ull;
 625 };
 626
 627 extern uptr vmaSize;
 628
 629 template <typename Func, typename Arg>
 630 ALWAYS_INLINE auto SelectMapping(Arg arg) {
 631 #if SANITIZER_GO
 632 #  if defined(__powerpc64__)
 633   switch (vmaSize) {
 634     case 46:
 635       return Func::template Apply<MappingGoPPC64_46>(arg);
 636     case 47:
 637       return Func::template Apply<MappingGoPPC64_47>(arg);
 638   }
 639 #  elif defined(__mips64)
 640   return Func::template Apply<MappingGoMips64_47>(arg);
 641 #  elif defined(__s390x__)
 642   return Func::template Apply<MappingGoS390x>(arg);
 643 #  elif defined(__aarch64__)
 644   return Func::template Apply<MappingGoAarch64>(arg);
 645 #  elif SANITIZER_WINDOWS
 646   return Func::template Apply<MappingGoWindows>(arg);
 647 #  else
 648   return Func::template Apply<MappingGo48>(arg);
 649 #  endif
 650 #else  // SANITIZER_GO
 651 #  if defined(__x86_64__) || SANITIZER_IOSSIM || SANITIZER_MAC && !SANITIZER_IOS
 652   return Func::template Apply<Mapping48AddressSpace>(arg);
 653 #  elif defined(__aarch64__) && defined(__APPLE__)
 654   return Func::template Apply<MappingAppleAarch64>(arg);
 655 #  elif defined(__aarch64__) && !defined(__APPLE__)
 656   switch (vmaSize) {
 657     case 39:
 658       return Func::template Apply<MappingAarch64_39>(arg);
 659     case 42:
 660       return Func::template Apply<MappingAarch64_42>(arg);
 661     case 48:
 662       return Func::template Apply<MappingAarch64_48>(arg);
 663   }
 664 #  elif defined(__powerpc64__)
 665   switch (vmaSize) {
 666     case 44:
 667       return Func::template Apply<MappingPPC64_44>(arg);
 668     case 46:
 669       return Func::template Apply<MappingPPC64_46>(arg);
 670     case 47:
 671       return Func::template Apply<MappingPPC64_47>(arg);
 672   }
 673 #  elif defined(__mips64)
 674   return Func::template Apply<MappingMips64_40>(arg);
 675 #  elif defined(__s390x__)
 676   return Func::template Apply<MappingS390x>(arg);
 677 #  else
 678 #    error "unsupported platform"
 679 #  endif
 680 #endif
 681   Die();
 682 }
 683
 684 template <typename Func>
 685 void ForEachMapping() {
 686   Func::template Apply<Mapping48AddressSpace>();
 687   Func::template Apply<MappingMips64_40>();
 688   Func::template Apply<MappingAppleAarch64>();
 689   Func::template Apply<MappingAarch64_39>();
 690   Func::template Apply<MappingAarch64_42>();
 691   Func::template Apply<MappingAarch64_48>();
 692   Func::template Apply<MappingPPC64_44>();
 693   Func::template Apply<MappingPPC64_46>();
 694   Func::template Apply<MappingPPC64_47>();
 695   Func::template Apply<MappingS390x>();
 696   Func::template Apply<MappingGo48>();
 697   Func::template Apply<MappingGoWindows>();
 698   Func::template Apply<MappingGoPPC64_46>();
 699   Func::template Apply<MappingGoPPC64_47>();
 700   Func::template Apply<MappingGoAarch64>();
 701   Func::template Apply<MappingGoMips64_47>();
 702   Func::template Apply<MappingGoS390x>();
 703 }
 704
 705 enum MappingType {
 706   kLoAppMemBeg,
 707   kLoAppMemEnd,
 708   kHiAppMemBeg,
 709   kHiAppMemEnd,
 710   kMidAppMemBeg,
 711   kMidAppMemEnd,
 712   kHeapMemBeg,
 713   kHeapMemEnd,
 714   kShadowBeg,
 715   kShadowEnd,
 716   kMetaShadowBeg,
 717   kMetaShadowEnd,
 718   kTraceMemBeg,
 719   kTraceMemEnd,
 720   kVdsoBeg,
 721 };
 722
 723 struct MappingField {
 724   template <typename Mapping>
 725   static uptr Apply(MappingType type) {
 726     switch (type) {
 727       case kLoAppMemBeg:
 728         return Mapping::kLoAppMemBeg;
 729       case kLoAppMemEnd:
 730         return Mapping::kLoAppMemEnd;
 731       case kMidAppMemBeg:
 732         return Mapping::kMidAppMemBeg;
 733       case kMidAppMemEnd:
 734         return Mapping::kMidAppMemEnd;
 735       case kHiAppMemBeg:
 736         return Mapping::kHiAppMemBeg;
 737       case kHiAppMemEnd:
 738         return Mapping::kHiAppMemEnd;
 739       case kHeapMemBeg:
 740         return Mapping::kHeapMemBeg;
 741       case kHeapMemEnd:
 742         return Mapping::kHeapMemEnd;
 743       case kVdsoBeg:
 744         return Mapping::kVdsoBeg;
 745       case kShadowBeg:
 746         return Mapping::kShadowBeg;
 747       case kShadowEnd:
 748         return Mapping::kShadowEnd;
 749       case kMetaShadowBeg:
 750         return Mapping::kMetaShadowBeg;
 751       case kMetaShadowEnd:
 752         return Mapping::kMetaShadowEnd;
 753       case kTraceMemBeg:
 754         return Mapping::kTraceMemBeg;
 755       case kTraceMemEnd:
 756         return Mapping::kTraceMemEnd;
 757     }
 758     Die();
 759   }
 760 };
 761
 762 ALWAYS_INLINE
 763 uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); }
 764 ALWAYS_INLINE
 765 uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); }
 766
 767 ALWAYS_INLINE
 768 uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); }
 769 ALWAYS_INLINE
 770 uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); }
 771
 772 ALWAYS_INLINE
 773 uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); }
 774 ALWAYS_INLINE
 775 uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); }
 776
 777 ALWAYS_INLINE
 778 uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); }
 779 ALWAYS_INLINE
 780 uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); }
 781
 782 ALWAYS_INLINE
 783 uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); }
 784
 785 ALWAYS_INLINE
 786 uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); }
 787 ALWAYS_INLINE
 788 uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); }
 789
 790 ALWAYS_INLINE
 791 uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); }
 792 ALWAYS_INLINE
 793 uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); }
 794
 795 ALWAYS_INLINE
 796 uptr TraceMemBeg(void) { return SelectMapping<MappingField>(kTraceMemBeg); }
 797 ALWAYS_INLINE
 798 uptr TraceMemEnd(void) { return SelectMapping<MappingField>(kTraceMemEnd); }
 799
 800 struct IsAppMemImpl {
 801   template <typename Mapping>
 802   static bool Apply(uptr mem) {
 803   return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) ||
 804          (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) ||
 805          (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) ||
 806          (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd);
 807   }
 808 };
 809
 810 ALWAYS_INLINE
 811 bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); }
 812
 813 struct IsShadowMemImpl {
 814   template <typename Mapping>
 815   static bool Apply(uptr mem) {
 816     return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd;
 817   }
 818 };
 819
 820 ALWAYS_INLINE
 821 bool IsShadowMem(RawShadow *p) {
 822   return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p));
 823 }
 824
 825 struct IsMetaMemImpl {
 826   template <typename Mapping>
 827   static bool Apply(uptr mem) {
 828     return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd;
 829   }
 830 };
 831
 832 ALWAYS_INLINE
 833 bool IsMetaMem(const u32 *p) {
 834   return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p));
 835 }
 836
 837 struct MemToShadowImpl {
 838   template <typename Mapping>
 839   static uptr Apply(uptr x) {
 840     DCHECK(IsAppMemImpl::Apply<Mapping>(x));
 841     return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^
 842             Mapping::kShadowXor) *
 843                kShadowMultiplier +
 844            Mapping::kShadowAdd;
 845   }
 846 };
 847
 848 ALWAYS_INLINE
 849 RawShadow *MemToShadow(uptr x) {
 850   return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x));
 851 }
 852
 853 struct MemToMetaImpl {
 854   template <typename Mapping>
 855   static u32 *Apply(uptr x) {
 856     DCHECK(IsAppMemImpl::Apply<Mapping>(x));
 857     return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) /
 858                     kMetaShadowCell * kMetaShadowSize) |
 859                    Mapping::kMetaShadowBeg);
 860   }
 861 };
 862
 863 ALWAYS_INLINE
 864 u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); }
 865
 866 struct ShadowToMemImpl {
 867   template <typename Mapping>
 868   static uptr Apply(uptr sp) {
 869     if (!IsShadowMemImpl::Apply<Mapping>(sp))
 870       return 0;
 871     // The shadow mapping is non-linear and we've lost some bits, so we don't
 872     // have an easy way to restore the original app address. But the mapping is
 873     // a bijection, so we try to restore the address as belonging to
 874     // low/mid/high range consecutively and see if shadow->app->shadow mapping
 875     // gives us the same address.
 876     uptr p =
 877         ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor;
 878     if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd &&
 879         MemToShadowImpl::Apply<Mapping>(p) == sp)
 880       return p;
 881     if (Mapping::kMidAppMemBeg) {
 882       uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk);
 883       if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd &&
 884           MemToShadowImpl::Apply<Mapping>(p_mid) == sp)
 885         return p_mid;
 886     }
 887     return p | Mapping::kShadowMsk;
 888   }
 889 };
 890
 891 ALWAYS_INLINE
 892 uptr ShadowToMem(RawShadow *s) {
 893   return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s));
 894 }
 895
 896 // Compresses addr to kCompressedAddrBits stored in least significant bits.
 897 ALWAYS_INLINE uptr CompressAddr(uptr addr) {
 898   return addr & ((1ull << kCompressedAddrBits) - 1);
 899 }
 900
 901 struct RestoreAddrImpl {
 902   typedef uptr Result;
 903   template <typename Mapping>
 904   static Result Apply(uptr addr) {
 905     // To restore the address we go over all app memory ranges and check if top
 906     // 3 bits of the compressed addr match that of the app range. If yes, we
 907     // assume that the compressed address come from that range and restore the
 908     // missing top bits to match the app range address.
 909     static constexpr uptr ranges[] = {
 910         Mapping::kLoAppMemBeg,  Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg,
 911         Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd,
 912         Mapping::kHeapMemBeg,   Mapping::kHeapMemEnd,
 913     };
 914     const uptr indicator = 0x0e0000000000ull;
 915     const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator);
 916     for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) {
 917       uptr beg = ranges[i];
 918       uptr end = ranges[i + 1];
 919       if (beg == end)
 920         continue;
 921       for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) {
 922         if ((addr & indicator) == (p & indicator))
 923           return addr | (p & ~(ind_lsb - 1));
 924       }
 925     }
 926     Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr);
 927     Die();
 928   }
 929 };
 930
 931 // Restores compressed addr from kCompressedAddrBits to full representation.
 932 // This is called only during reporting and is not performance-critical.
 933 inline uptr RestoreAddr(uptr addr) {
 934   return SelectMapping<RestoreAddrImpl>(addr);
 935 }
 936
 937 // The additional page is to catch shadow stack overflow as paging fault.
 938 // Windows wants 64K alignment for mmaps.
 939 const uptr kTotalTraceSize = (kTraceSize * sizeof(Event) + sizeof(Trace)
 940     + (64 << 10) + (64 << 10) - 1) & ~((64 << 10) - 1);
 941
 942 struct GetThreadTraceImpl {
 943   template <typename Mapping>
 944   static uptr Apply(uptr tid) {
 945     uptr p = Mapping::kTraceMemBeg + tid * kTotalTraceSize;
 946     DCHECK_LT(p, Mapping::kTraceMemEnd);
 947     return p;
 948   }
 949 };
 950
 951 ALWAYS_INLINE
 952 uptr GetThreadTrace(int tid) { return SelectMapping<GetThreadTraceImpl>(tid); }
 953
 954 struct GetThreadTraceHeaderImpl {
 955   template <typename Mapping>
 956   static uptr Apply(uptr tid) {
 957     uptr p = Mapping::kTraceMemBeg + tid * kTotalTraceSize +
 958              kTraceSize * sizeof(Event);
 959     DCHECK_LT(p, Mapping::kTraceMemEnd);
 960     return p;
 961   }
 962 };
 963
 964 ALWAYS_INLINE
 965 uptr GetThreadTraceHeader(int tid) {
 966   return SelectMapping<GetThreadTraceHeaderImpl>(tid);
 967 }
 968
 969 void InitializePlatform();
 970 void InitializePlatformEarly();
 971 void CheckAndProtect();
 972 void InitializeShadowMemoryPlatform();
 973 void FlushShadowMemory();
 974 void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns);
 975 int ExtractResolvFDs(void *state, int *fds, int nfd);
 976 int ExtractRecvmsgFDs(void *msg, int *fds, int nfd);
 977 uptr ExtractLongJmpSp(uptr *env);
 978 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size);
 979
 980 int call_pthread_cancel_with_cleanup(int (*fn)(void *arg),
 981                                      void (*cleanup)(void *arg), void *arg);
 982
 983 void DestroyThreadState();
 984 void PlatformCleanUpThreadState(ThreadState *thr);
 985
 986 }  // namespace __tsan
 987
 988 #endif  // TSAN_PLATFORM_H