lib/msan/msan_interceptors.cc

   1 //===-- msan_interceptors.cc ----------------------------------------------===//
   2 //
   3 //                     The LLVM Compiler Infrastructure
   4 //
   5 // This file is distributed under the University of Illinois Open Source
   6 // License. See LICENSE.TXT for details.
   7 //
   8 //===----------------------------------------------------------------------===//
   9 //
  10 // This file is a part of MemorySanitizer.
  11 //
  12 // Interceptors for standard library functions.
  13 //
  14 // FIXME: move as many interceptors as possible into
  15 // sanitizer_common/sanitizer_common_interceptors.h
  16 //===----------------------------------------------------------------------===//
  17
  18 #include "interception/interception.h"
  19 #include "msan.h"
  20 #include "sanitizer_common/sanitizer_platform_limits_posix.h"
  21 #include "sanitizer_common/sanitizer_allocator.h"
  22 #include "sanitizer_common/sanitizer_allocator_internal.h"
  23 #include "sanitizer_common/sanitizer_atomic.h"
  24 #include "sanitizer_common/sanitizer_common.h"
  25 #include "sanitizer_common/sanitizer_stackdepot.h"
  26 #include "sanitizer_common/sanitizer_libc.h"
  27 #include "sanitizer_common/sanitizer_linux.h"
  28
  29 #include <stdarg.h>
  30 // ACHTUNG! No other system header includes in this file.
  31 // Ideally, we should get rid of stdarg.h as well.
  32
  33 using namespace __msan;
  34
  35 using __sanitizer::memory_order;
  36 using __sanitizer::atomic_load;
  37 using __sanitizer::atomic_store;
  38 using __sanitizer::atomic_uintptr_t;
  39
  40 // True if this is a nested interceptor.
  41 static THREADLOCAL int in_interceptor_scope;
  42
  43 struct InterceptorScope {
  44   InterceptorScope() { ++in_interceptor_scope; }
  45   ~InterceptorScope() { --in_interceptor_scope; }
  46 };
  47
  48 bool IsInInterceptorScope() {
  49   return in_interceptor_scope;
  50 }
  51
  52 #define ENSURE_MSAN_INITED() do { \
  53   CHECK(!msan_init_is_running); \
  54   if (!msan_inited) { \
  55     __msan_init(); \
  56   } \
  57 } while (0)
  58
  59 // Check that [x, x+n) range is unpoisoned.
  60 #define CHECK_UNPOISONED_0(x, n)                                             \
  61   do {                                                                       \
  62     sptr offset = __msan_test_shadow(x, n);                                  \
  63     if (__msan::IsInSymbolizer()) break;                                     \
  64     if (offset >= 0 && __msan::flags()->report_umrs) {                       \
  65       GET_CALLER_PC_BP_SP;                                                   \
  66       (void) sp;                                                             \
  67       Printf("UMR in %s at offset %d inside [%p, +%d) \n", __FUNCTION__,     \
  68              offset, x, n);                                                  \
  69       __msan::PrintWarningWithOrigin(pc, bp,                                 \
  70                                      __msan_get_origin((char *)x + offset)); \
  71       if (__msan::flags()->halt_on_error) {                                  \
  72         Printf("Exiting\n");                                                 \
  73         Die();                                                               \
  74       }                                                                      \
  75     }                                                                        \
  76   } while (0)
  77
  78 // Check that [x, x+n) range is unpoisoned unless we are in a nested
  79 // interceptor.
  80 #define CHECK_UNPOISONED(x, n)                             \
  81   do {                                                     \
  82     if (!IsInInterceptorScope()) CHECK_UNPOISONED_0(x, n); \
  83   } while (0);
  84
  85 static void *fast_memset(void *ptr, int c, SIZE_T n);
  86 static void *fast_memcpy(void *dst, const void *src, SIZE_T n);
  87
  88 INTERCEPTOR(SIZE_T, fread, void *ptr, SIZE_T size, SIZE_T nmemb, void *file) {
  89   ENSURE_MSAN_INITED();
  90   SIZE_T res = REAL(fread)(ptr, size, nmemb, file);
  91   if (res > 0)
  92     __msan_unpoison(ptr, res *size);
  93   return res;
  94 }
  95
  96 INTERCEPTOR(SIZE_T, fread_unlocked, void *ptr, SIZE_T size, SIZE_T nmemb,
  97             void *file) {
  98   ENSURE_MSAN_INITED();
  99   SIZE_T res = REAL(fread_unlocked)(ptr, size, nmemb, file);
 100   if (res > 0)
 101     __msan_unpoison(ptr, res *size);
 102   return res;
 103 }
 104
 105 INTERCEPTOR(SSIZE_T, readlink, const char *path, char *buf, SIZE_T bufsiz) {
 106   ENSURE_MSAN_INITED();
 107   SSIZE_T res = REAL(readlink)(path, buf, bufsiz);
 108   if (res > 0)
 109     __msan_unpoison(buf, res);
 110   return res;
 111 }
 112
 113 INTERCEPTOR(void *, memcpy, void *dest, const void *src, SIZE_T n) {
 114   return __msan_memcpy(dest, src, n);
 115 }
 116
 117 INTERCEPTOR(void *, mempcpy, void *dest, const void *src, SIZE_T n) {
 118   return (char *)__msan_memcpy(dest, src, n) + n;
 119 }
 120
 121 INTERCEPTOR(void *, memmove, void *dest, const void *src, SIZE_T n) {
 122   return __msan_memmove(dest, src, n);
 123 }
 124
 125 INTERCEPTOR(void *, memset, void *s, int c, SIZE_T n) {
 126   return __msan_memset(s, c, n);
 127 }
 128
 129 INTERCEPTOR(void *, bcopy, const void *src, void *dest, SIZE_T n) {
 130   return __msan_memmove(dest, src, n);
 131 }
 132
 133 INTERCEPTOR(int, posix_memalign, void **memptr, SIZE_T alignment, SIZE_T size) {
 134   GET_MALLOC_STACK_TRACE;
 135   CHECK_EQ(alignment & (alignment - 1), 0);
 136   CHECK_NE(memptr, 0);
 137   *memptr = MsanReallocate(&stack, 0, size, alignment, false);
 138   CHECK_NE(*memptr, 0);
 139   __msan_unpoison(memptr, sizeof(*memptr));
 140   return 0;
 141 }
 142
 143 INTERCEPTOR(void *, memalign, SIZE_T boundary, SIZE_T size) {
 144   GET_MALLOC_STACK_TRACE;
 145   CHECK_EQ(boundary & (boundary - 1), 0);
 146   void *ptr = MsanReallocate(&stack, 0, size, boundary, false);
 147   return ptr;
 148 }
 149
 150 INTERCEPTOR(void *, valloc, SIZE_T size) {
 151   GET_MALLOC_STACK_TRACE;
 152   void *ptr = MsanReallocate(&stack, 0, size, GetPageSizeCached(), false);
 153   return ptr;
 154 }
 155
 156 INTERCEPTOR(void *, pvalloc, SIZE_T size) {
 157   GET_MALLOC_STACK_TRACE;
 158   uptr PageSize = GetPageSizeCached();
 159   size = RoundUpTo(size, PageSize);
 160   if (size == 0) {
 161     // pvalloc(0) should allocate one page.
 162     size = PageSize;
 163   }
 164   void *ptr = MsanReallocate(&stack, 0, size, PageSize, false);
 165   return ptr;
 166 }
 167
 168 INTERCEPTOR(void, free, void *ptr) {
 169   GET_MALLOC_STACK_TRACE;
 170   if (ptr == 0) return;
 171   MsanDeallocate(&stack, ptr);
 172 }
 173
 174 INTERCEPTOR(SIZE_T, strlen, const char *s) {
 175   ENSURE_MSAN_INITED();
 176   SIZE_T res = REAL(strlen)(s);
 177   CHECK_UNPOISONED(s, res + 1);
 178   return res;
 179 }
 180
 181 INTERCEPTOR(SIZE_T, strnlen, const char *s, SIZE_T n) {
 182   ENSURE_MSAN_INITED();
 183   SIZE_T res = REAL(strnlen)(s, n);
 184   SIZE_T scan_size = (res == n) ? res : res + 1;
 185   CHECK_UNPOISONED(s, scan_size);
 186   return res;
 187 }
 188
 189 // FIXME: Add stricter shadow checks in str* interceptors (ex.: strcpy should
 190 // check the shadow of the terminating \0 byte).
 191
 192 INTERCEPTOR(char *, strcpy, char *dest, const char *src) {  // NOLINT
 193   ENSURE_MSAN_INITED();
 194   SIZE_T n = REAL(strlen)(src);
 195   char *res = REAL(strcpy)(dest, src);  // NOLINT
 196   __msan_copy_poison(dest, src, n + 1);
 197   return res;
 198 }
 199
 200 INTERCEPTOR(char *, strncpy, char *dest, const char *src, SIZE_T n) {  // NOLINT
 201   ENSURE_MSAN_INITED();
 202   SIZE_T copy_size = REAL(strnlen)(src, n);
 203   if (copy_size < n)
 204     copy_size++;  // trailing \0
 205   char *res = REAL(strncpy)(dest, src, n);  // NOLINT
 206   __msan_copy_poison(dest, src, copy_size);
 207   return res;
 208 }
 209
 210 INTERCEPTOR(char *, stpcpy, char *dest, const char *src) {  // NOLINT
 211   ENSURE_MSAN_INITED();
 212   SIZE_T n = REAL(strlen)(src);
 213   char *res = REAL(stpcpy)(dest, src);  // NOLINT
 214   __msan_copy_poison(dest, src, n + 1);
 215   return res;
 216 }
 217
 218 INTERCEPTOR(char *, strdup, char *src) {
 219   ENSURE_MSAN_INITED();
 220   SIZE_T n = REAL(strlen)(src);
 221   char *res = REAL(strdup)(src);
 222   __msan_copy_poison(res, src, n + 1);
 223   return res;
 224 }
 225
 226 INTERCEPTOR(char *, __strdup, char *src) {
 227   ENSURE_MSAN_INITED();
 228   SIZE_T n = REAL(strlen)(src);
 229   char *res = REAL(__strdup)(src);
 230   __msan_copy_poison(res, src, n + 1);
 231   return res;
 232 }
 233
 234 INTERCEPTOR(char *, strndup, char *src, SIZE_T n) {
 235   ENSURE_MSAN_INITED();
 236   SIZE_T copy_size = REAL(strnlen)(src, n);
 237   char *res = REAL(strndup)(src, n);
 238   __msan_copy_poison(res, src, copy_size);
 239   __msan_unpoison(res + copy_size, 1); // \0
 240   return res;
 241 }
 242
 243 INTERCEPTOR(char *, __strndup, char *src, SIZE_T n) {
 244   ENSURE_MSAN_INITED();
 245   SIZE_T copy_size = REAL(strnlen)(src, n);
 246   char *res = REAL(__strndup)(src, n);
 247   __msan_copy_poison(res, src, copy_size);
 248   __msan_unpoison(res + copy_size, 1); // \0
 249   return res;
 250 }
 251
 252 INTERCEPTOR(char *, gcvt, double number, SIZE_T ndigit, char *buf) {
 253   ENSURE_MSAN_INITED();
 254   char *res = REAL(gcvt)(number, ndigit, buf);
 255   // DynamoRio tool will take care of unpoisoning gcvt result for us.
 256   if (!__msan_has_dynamic_component()) {
 257     SIZE_T n = REAL(strlen)(buf);
 258     __msan_unpoison(buf, n + 1);
 259   }
 260   return res;
 261 }
 262
 263 INTERCEPTOR(char *, strcat, char *dest, const char *src) {  // NOLINT
 264   ENSURE_MSAN_INITED();
 265   SIZE_T src_size = REAL(strlen)(src);
 266   SIZE_T dest_size = REAL(strlen)(dest);
 267   char *res = REAL(strcat)(dest, src);  // NOLINT
 268   __msan_copy_poison(dest + dest_size, src, src_size + 1);
 269   return res;
 270 }
 271
 272 INTERCEPTOR(char *, strncat, char *dest, const char *src, SIZE_T n) {  // NOLINT
 273   ENSURE_MSAN_INITED();
 274   SIZE_T dest_size = REAL(strlen)(dest);
 275   SIZE_T copy_size = REAL(strlen)(src);
 276   if (copy_size < n)
 277     copy_size++;  // trailing \0
 278   char *res = REAL(strncat)(dest, src, n);  // NOLINT
 279   __msan_copy_poison(dest + dest_size, src, copy_size);
 280   return res;
 281 }
 282
 283 INTERCEPTOR(long, strtol, const char *nptr, char **endptr,  // NOLINT
 284             int base) {
 285   ENSURE_MSAN_INITED();
 286   long res = REAL(strtol)(nptr, endptr, base);  // NOLINT
 287   if (!__msan_has_dynamic_component()) {
 288     __msan_unpoison(endptr, sizeof(*endptr));
 289   }
 290   return res;
 291 }
 292
 293 INTERCEPTOR(long long, strtoll, const char *nptr, char **endptr,  // NOLINT
 294             int base) {
 295   ENSURE_MSAN_INITED();
 296   long res = REAL(strtoll)(nptr, endptr, base);  //NOLINT
 297   if (!__msan_has_dynamic_component()) {
 298     __msan_unpoison(endptr, sizeof(*endptr));
 299   }
 300   return res;
 301 }
 302
 303 INTERCEPTOR(unsigned long, strtoul, const char *nptr, char **endptr,  // NOLINT
 304             int base) {
 305   ENSURE_MSAN_INITED();
 306   unsigned long res = REAL(strtoul)(nptr, endptr, base);  // NOLINT
 307   if (!__msan_has_dynamic_component()) {
 308     __msan_unpoison(endptr, sizeof(*endptr));
 309   }
 310   return res;
 311 }
 312
 313 INTERCEPTOR(unsigned long long, strtoull, const char *nptr,  // NOLINT
 314             char **endptr, int base) {
 315   ENSURE_MSAN_INITED();
 316   unsigned long res = REAL(strtoull)(nptr, endptr, base);  // NOLINT
 317   if (!__msan_has_dynamic_component()) {
 318     __msan_unpoison(endptr, sizeof(*endptr));
 319   }
 320   return res;
 321 }
 322
 323 INTERCEPTOR(double, strtod, const char *nptr, char **endptr) {  // NOLINT
 324   ENSURE_MSAN_INITED();
 325   double res = REAL(strtod)(nptr, endptr);  // NOLINT
 326   if (!__msan_has_dynamic_component()) {
 327     __msan_unpoison(endptr, sizeof(*endptr));
 328   }
 329   return res;
 330 }
 331
 332 INTERCEPTOR(double, strtod_l, const char *nptr, char **endptr,
 333             void *loc) {  // NOLINT
 334   ENSURE_MSAN_INITED();
 335   double res = REAL(strtod_l)(nptr, endptr, loc);  // NOLINT
 336   if (!__msan_has_dynamic_component()) {
 337     __msan_unpoison(endptr, sizeof(*endptr));
 338   }
 339   return res;
 340 }
 341
 342 INTERCEPTOR(double, __strtod_l, const char *nptr, char **endptr,
 343             void *loc) {  // NOLINT
 344   ENSURE_MSAN_INITED();
 345   double res = REAL(__strtod_l)(nptr, endptr, loc);  // NOLINT
 346   if (!__msan_has_dynamic_component()) {
 347     __msan_unpoison(endptr, sizeof(*endptr));
 348   }
 349   return res;
 350 }
 351
 352 INTERCEPTOR(float, strtof, const char *nptr, char **endptr) {  // NOLINT
 353   ENSURE_MSAN_INITED();
 354   float res = REAL(strtof)(nptr, endptr);  // NOLINT
 355   if (!__msan_has_dynamic_component()) {
 356     __msan_unpoison(endptr, sizeof(*endptr));
 357   }
 358   return res;
 359 }
 360
 361 INTERCEPTOR(float, strtof_l, const char *nptr, char **endptr,
 362             void *loc) {  // NOLINT
 363   ENSURE_MSAN_INITED();
 364   float res = REAL(strtof_l)(nptr, endptr, loc);  // NOLINT
 365   if (!__msan_has_dynamic_component()) {
 366     __msan_unpoison(endptr, sizeof(*endptr));
 367   }
 368   return res;
 369 }
 370
 371 INTERCEPTOR(float, __strtof_l, const char *nptr, char **endptr,
 372             void *loc) {  // NOLINT
 373   ENSURE_MSAN_INITED();
 374   float res = REAL(__strtof_l)(nptr, endptr, loc);  // NOLINT
 375   if (!__msan_has_dynamic_component()) {
 376     __msan_unpoison(endptr, sizeof(*endptr));
 377   }
 378   return res;
 379 }
 380
 381 INTERCEPTOR(long double, strtold, const char *nptr, char **endptr) {  // NOLINT
 382   ENSURE_MSAN_INITED();
 383   long double res = REAL(strtold)(nptr, endptr);  // NOLINT
 384   if (!__msan_has_dynamic_component()) {
 385     __msan_unpoison(endptr, sizeof(*endptr));
 386   }
 387   return res;
 388 }
 389
 390 INTERCEPTOR(long double, strtold_l, const char *nptr, char **endptr,
 391             void *loc) {  // NOLINT
 392   ENSURE_MSAN_INITED();
 393   long double res = REAL(strtold_l)(nptr, endptr, loc);  // NOLINT
 394   if (!__msan_has_dynamic_component()) {
 395     __msan_unpoison(endptr, sizeof(*endptr));
 396   }
 397   return res;
 398 }
 399
 400 INTERCEPTOR(long double, __strtold_l, const char *nptr, char **endptr,
 401             void *loc) {  // NOLINT
 402   ENSURE_MSAN_INITED();
 403   long double res = REAL(__strtold_l)(nptr, endptr, loc);  // NOLINT
 404   if (!__msan_has_dynamic_component()) {
 405     __msan_unpoison(endptr, sizeof(*endptr));
 406   }
 407   return res;
 408 }
 409
 410 INTERCEPTOR(int, vasprintf, char **strp, const char *format, va_list ap) {
 411   ENSURE_MSAN_INITED();
 412   int res = REAL(vasprintf)(strp, format, ap);
 413   if (res >= 0 && !__msan_has_dynamic_component()) {
 414     __msan_unpoison(strp, sizeof(*strp));
 415     __msan_unpoison(*strp, res + 1);
 416   }
 417   return res;
 418 }
 419
 420 INTERCEPTOR(int, asprintf, char **strp, const char *format, ...) {  // NOLINT
 421   ENSURE_MSAN_INITED();
 422   va_list ap;
 423   va_start(ap, format);
 424   int res = vasprintf(strp, format, ap);  // NOLINT
 425   va_end(ap);
 426   return res;
 427 }
 428
 429 INTERCEPTOR(int, vsnprintf, char *str, uptr size,
 430             const char *format, va_list ap) {
 431   ENSURE_MSAN_INITED();
 432   int res = REAL(vsnprintf)(str, size, format, ap);
 433   if (res >= 0 && !__msan_has_dynamic_component()) {
 434     __msan_unpoison(str, res + 1);
 435   }
 436   return res;
 437 }
 438
 439 INTERCEPTOR(int, vsprintf, char *str, const char *format, va_list ap) {
 440   ENSURE_MSAN_INITED();
 441   int res = REAL(vsprintf)(str, format, ap);
 442   if (res >= 0 && !__msan_has_dynamic_component()) {
 443     __msan_unpoison(str, res + 1);
 444   }
 445   return res;
 446 }
 447
 448 INTERCEPTOR(int, vswprintf, void *str, uptr size, void *format, va_list ap) {
 449   ENSURE_MSAN_INITED();
 450   int res = REAL(vswprintf)(str, size, format, ap);
 451   if (res >= 0 && !__msan_has_dynamic_component()) {
 452     __msan_unpoison(str, 4 * (res + 1));
 453   }
 454   return res;
 455 }
 456
 457 INTERCEPTOR(int, sprintf, char *str, const char *format, ...) {  // NOLINT
 458   ENSURE_MSAN_INITED();
 459   va_list ap;
 460   va_start(ap, format);
 461   int res = vsprintf(str, format, ap);  // NOLINT
 462   va_end(ap);
 463   return res;
 464 }
 465
 466 INTERCEPTOR(int, snprintf, char *str, uptr size, const char *format, ...) {
 467   ENSURE_MSAN_INITED();
 468   va_list ap;
 469   va_start(ap, format);
 470   int res = vsnprintf(str, size, format, ap);
 471   va_end(ap);
 472   return res;
 473 }
 474
 475 INTERCEPTOR(int, swprintf, void *str, uptr size, void *format, ...) {
 476   ENSURE_MSAN_INITED();
 477   va_list ap;
 478   va_start(ap, format);
 479   int res = vswprintf(str, size, format, ap);
 480   va_end(ap);
 481   return res;
 482 }
 483
 484 // SIZE_T strftime(char *s, SIZE_T max, const char *format,const struct tm *tm);
 485 INTERCEPTOR(SIZE_T, strftime, char *s, SIZE_T max, const char *format,
 486             void *tm) {
 487   ENSURE_MSAN_INITED();
 488   SIZE_T res = REAL(strftime)(s, max, format, tm);
 489   if (res) __msan_unpoison(s, res + 1);
 490   return res;
 491 }
 492
 493 INTERCEPTOR(int, mbtowc, wchar_t *dest, const char *src, SIZE_T n) {
 494   ENSURE_MSAN_INITED();
 495   int res = REAL(mbtowc)(dest, src, n);
 496   if (res != -1 && dest) __msan_unpoison(dest, sizeof(wchar_t));
 497   return res;
 498 }
 499
 500 INTERCEPTOR(int, mbrtowc, wchar_t *dest, const char *src, SIZE_T n, void *ps) {
 501   ENSURE_MSAN_INITED();
 502   SIZE_T res = REAL(mbrtowc)(dest, src, n, ps);
 503   if (res != (SIZE_T)-1 && dest) __msan_unpoison(dest, sizeof(wchar_t));
 504   return res;
 505 }
 506
 507 INTERCEPTOR(SIZE_T, wcslen, const wchar_t *s) {
 508   ENSURE_MSAN_INITED();
 509   SIZE_T res = REAL(wcslen)(s);
 510   CHECK_UNPOISONED(s, sizeof(wchar_t) * (res + 1));
 511   return res;
 512 }
 513
 514 // wchar_t *wcschr(const wchar_t *wcs, wchar_t wc);
 515 INTERCEPTOR(wchar_t *, wcschr, void *s, wchar_t wc, void *ps) {
 516   ENSURE_MSAN_INITED();
 517   wchar_t *res = REAL(wcschr)(s, wc, ps);
 518   return res;
 519 }
 520
 521 // wchar_t *wcscpy(wchar_t *dest, const wchar_t *src);
 522 INTERCEPTOR(wchar_t *, wcscpy, wchar_t *dest, const wchar_t *src) {
 523   ENSURE_MSAN_INITED();
 524   wchar_t *res = REAL(wcscpy)(dest, src);
 525   __msan_copy_poison(dest, src, sizeof(wchar_t) * (REAL(wcslen)(src) + 1));
 526   return res;
 527 }
 528
 529 // wchar_t *wmemcpy(wchar_t *dest, const wchar_t *src, SIZE_T n);
 530 INTERCEPTOR(wchar_t *, wmemcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) {
 531   ENSURE_MSAN_INITED();
 532   wchar_t *res = REAL(wmemcpy)(dest, src, n);
 533   __msan_copy_poison(dest, src, n * sizeof(wchar_t));
 534   return res;
 535 }
 536
 537 INTERCEPTOR(wchar_t *, wmempcpy, wchar_t *dest, const wchar_t *src, SIZE_T n) {
 538   ENSURE_MSAN_INITED();
 539   wchar_t *res = REAL(wmempcpy)(dest, src, n);
 540   __msan_copy_poison(dest, src, n * sizeof(wchar_t));
 541   return res;
 542 }
 543
 544 INTERCEPTOR(wchar_t *, wmemset, wchar_t *s, wchar_t c, SIZE_T n) {
 545   CHECK(MEM_IS_APP(s));
 546   ENSURE_MSAN_INITED();
 547   wchar_t *res = (wchar_t *)fast_memset(s, c, n * sizeof(wchar_t));
 548   __msan_unpoison(s, n * sizeof(wchar_t));
 549   return res;
 550 }
 551
 552 INTERCEPTOR(wchar_t *, wmemmove, wchar_t *dest, const wchar_t *src, SIZE_T n) {
 553   ENSURE_MSAN_INITED();
 554   wchar_t *res = REAL(wmemmove)(dest, src, n);
 555   __msan_move_poison(dest, src, n * sizeof(wchar_t));
 556   return res;
 557 }
 558
 559 INTERCEPTOR(int, wcscmp, const wchar_t *s1, const wchar_t *s2) {
 560   ENSURE_MSAN_INITED();
 561   int res = REAL(wcscmp)(s1, s2);
 562   return res;
 563 }
 564
 565 INTERCEPTOR(double, wcstod, const wchar_t *nptr, wchar_t **endptr) {
 566   ENSURE_MSAN_INITED();
 567   double res = REAL(wcstod)(nptr, endptr);
 568   __msan_unpoison(endptr, sizeof(*endptr));
 569   return res;
 570 }
 571
 572 INTERCEPTOR(int, gettimeofday, void *tv, void *tz) {
 573   ENSURE_MSAN_INITED();
 574   int res = REAL(gettimeofday)(tv, tz);
 575   if (tv)
 576     __msan_unpoison(tv, 16);
 577   if (tz)
 578     __msan_unpoison(tz, 8);
 579   return res;
 580 }
 581
 582 INTERCEPTOR(char *, fcvt, double x, int a, int *b, int *c) {
 583   ENSURE_MSAN_INITED();
 584   char *res = REAL(fcvt)(x, a, b, c);
 585   if (!__msan_has_dynamic_component()) {
 586     __msan_unpoison(b, sizeof(*b));
 587     __msan_unpoison(c, sizeof(*c));
 588   }
 589   return res;
 590 }
 591
 592 INTERCEPTOR(char *, getenv, char *name) {
 593   ENSURE_MSAN_INITED();
 594   char *res = REAL(getenv)(name);
 595   if (!__msan_has_dynamic_component()) {
 596     if (res)
 597       __msan_unpoison(res, REAL(strlen)(res) + 1);
 598   }
 599   return res;
 600 }
 601
 602 extern char **environ;
 603
 604 static void UnpoisonEnviron() {
 605   char **envp = environ;
 606   for (; *envp; ++envp) {
 607     __msan_unpoison(envp, sizeof(*envp));
 608     __msan_unpoison(*envp, REAL(strlen)(*envp) + 1);
 609   }
 610   // Trailing NULL pointer.
 611   __msan_unpoison(envp, sizeof(*envp));
 612 }
 613
 614 INTERCEPTOR(int, setenv, const char *name, const char *value, int overwrite) {
 615   ENSURE_MSAN_INITED();
 616   int res = REAL(setenv)(name, value, overwrite);
 617   if (!res) UnpoisonEnviron();
 618   return res;
 619 }
 620
 621 INTERCEPTOR(int, putenv, char *string) {
 622   ENSURE_MSAN_INITED();
 623   int res = REAL(putenv)(string);
 624   if (!res) UnpoisonEnviron();
 625   return res;
 626 }
 627
 628 INTERCEPTOR(int, __fxstat, int magic, int fd, void *buf) {
 629   ENSURE_MSAN_INITED();
 630   int res = REAL(__fxstat)(magic, fd, buf);
 631   if (!res)
 632     __msan_unpoison(buf, __sanitizer::struct_stat_sz);
 633   return res;
 634 }
 635
 636 INTERCEPTOR(int, __fxstat64, int magic, int fd, void *buf) {
 637   ENSURE_MSAN_INITED();
 638   int res = REAL(__fxstat64)(magic, fd, buf);
 639   if (!res)
 640     __msan_unpoison(buf, __sanitizer::struct_stat64_sz);
 641   return res;
 642 }
 643
 644 INTERCEPTOR(int, __fxstatat, int magic, int fd, char *pathname, void *buf,
 645             int flags) {
 646   ENSURE_MSAN_INITED();
 647   int res = REAL(__fxstatat)(magic, fd, pathname, buf, flags);
 648   if (!res) __msan_unpoison(buf, __sanitizer::struct_stat_sz);
 649   return res;
 650 }
 651
 652 INTERCEPTOR(int, __fxstatat64, int magic, int fd, char *pathname, void *buf,
 653             int flags) {
 654   ENSURE_MSAN_INITED();
 655   int res = REAL(__fxstatat64)(magic, fd, pathname, buf, flags);
 656   if (!res) __msan_unpoison(buf, __sanitizer::struct_stat64_sz);
 657   return res;
 658 }
 659
 660 INTERCEPTOR(int, __xstat, int magic, char *path, void *buf) {
 661   ENSURE_MSAN_INITED();
 662   int res = REAL(__xstat)(magic, path, buf);
 663   if (!res)
 664     __msan_unpoison(buf, __sanitizer::struct_stat_sz);
 665   return res;
 666 }
 667
 668 INTERCEPTOR(int, __xstat64, int magic, char *path, void *buf) {
 669   ENSURE_MSAN_INITED();
 670   int res = REAL(__xstat64)(magic, path, buf);
 671   if (!res)
 672     __msan_unpoison(buf, __sanitizer::struct_stat64_sz);
 673   return res;
 674 }
 675
 676 INTERCEPTOR(int, __lxstat, int magic, char *path, void *buf) {
 677   ENSURE_MSAN_INITED();
 678   int res = REAL(__lxstat)(magic, path, buf);
 679   if (!res)
 680     __msan_unpoison(buf, __sanitizer::struct_stat_sz);
 681   return res;
 682 }
 683
 684 INTERCEPTOR(int, __lxstat64, int magic, char *path, void *buf) {
 685   ENSURE_MSAN_INITED();
 686   int res = REAL(__lxstat64)(magic, path, buf);
 687   if (!res)
 688     __msan_unpoison(buf, __sanitizer::struct_stat64_sz);
 689   return res;
 690 }
 691
 692 INTERCEPTOR(int, pipe, int pipefd[2]) {
 693   if (msan_init_is_running)
 694     return REAL(pipe)(pipefd);
 695   ENSURE_MSAN_INITED();
 696   int res = REAL(pipe)(pipefd);
 697   if (!res)
 698     __msan_unpoison(pipefd, sizeof(int[2]));
 699   return res;
 700 }
 701
 702 INTERCEPTOR(int, pipe2, int pipefd[2], int flags) {
 703   ENSURE_MSAN_INITED();
 704   int res = REAL(pipe2)(pipefd, flags);
 705   if (!res)
 706     __msan_unpoison(pipefd, sizeof(int[2]));
 707   return res;
 708 }
 709
 710 INTERCEPTOR(int, socketpair, int domain, int type, int protocol, int sv[2]) {
 711   ENSURE_MSAN_INITED();
 712   int res = REAL(socketpair)(domain, type, protocol, sv);
 713   if (!res)
 714     __msan_unpoison(sv, sizeof(int[2]));
 715   return res;
 716 }
 717
 718 INTERCEPTOR(char *, fgets, char *s, int size, void *stream) {
 719   ENSURE_MSAN_INITED();
 720   char *res = REAL(fgets)(s, size, stream);
 721   if (res)
 722     __msan_unpoison(s, REAL(strlen)(s) + 1);
 723   return res;
 724 }
 725
 726 INTERCEPTOR(char *, fgets_unlocked, char *s, int size, void *stream) {
 727   ENSURE_MSAN_INITED();
 728   char *res = REAL(fgets_unlocked)(s, size, stream);
 729   if (res)
 730     __msan_unpoison(s, REAL(strlen)(s) + 1);
 731   return res;
 732 }
 733
 734 INTERCEPTOR(int, getrlimit, int resource, void *rlim) {
 735   if (msan_init_is_running)
 736     return REAL(getrlimit)(resource, rlim);
 737   ENSURE_MSAN_INITED();
 738   int res = REAL(getrlimit)(resource, rlim);
 739   if (!res)
 740     __msan_unpoison(rlim, __sanitizer::struct_rlimit_sz);
 741   return res;
 742 }
 743
 744 INTERCEPTOR(int, getrlimit64, int resource, void *rlim) {
 745   if (msan_init_is_running)
 746     return REAL(getrlimit64)(resource, rlim);
 747   ENSURE_MSAN_INITED();
 748   int res = REAL(getrlimit64)(resource, rlim);
 749   if (!res)
 750     __msan_unpoison(rlim, __sanitizer::struct_rlimit64_sz);
 751   return res;
 752 }
 753
 754 INTERCEPTOR(int, statfs, const char *s, void *buf) {
 755   ENSURE_MSAN_INITED();
 756   int res = REAL(statfs)(s, buf);
 757   if (!res)
 758     __msan_unpoison(buf, __sanitizer::struct_statfs_sz);
 759   return res;
 760 }
 761
 762 INTERCEPTOR(int, fstatfs, int fd, void *buf) {
 763   ENSURE_MSAN_INITED();
 764   int res = REAL(fstatfs)(fd, buf);
 765   if (!res)
 766     __msan_unpoison(buf, __sanitizer::struct_statfs_sz);
 767   return res;
 768 }
 769
 770 INTERCEPTOR(int, statfs64, const char *s, void *buf) {
 771   ENSURE_MSAN_INITED();
 772   int res = REAL(statfs64)(s, buf);
 773   if (!res)
 774     __msan_unpoison(buf, __sanitizer::struct_statfs64_sz);
 775   return res;
 776 }
 777
 778 INTERCEPTOR(int, fstatfs64, int fd, void *buf) {
 779   ENSURE_MSAN_INITED();
 780   int res = REAL(fstatfs64)(fd, buf);
 781   if (!res)
 782     __msan_unpoison(buf, __sanitizer::struct_statfs64_sz);
 783   return res;
 784 }
 785
 786 INTERCEPTOR(int, uname, void *utsname) {
 787   ENSURE_MSAN_INITED();
 788   int res = REAL(uname)(utsname);
 789   if (!res) {
 790     __msan_unpoison(utsname, __sanitizer::struct_utsname_sz);
 791   }
 792   return res;
 793 }
 794
 795 INTERCEPTOR(int, gethostname, char *name, SIZE_T len) {
 796   ENSURE_MSAN_INITED();
 797   int res = REAL(gethostname)(name, len);
 798   if (!res) {
 799     SIZE_T real_len = REAL(strnlen)(name, len);
 800     if (real_len < len)
 801       ++real_len;
 802     __msan_unpoison(name, real_len);
 803   }
 804   return res;
 805 }
 806
 807 INTERCEPTOR(int, epoll_wait, int epfd, void *events, int maxevents,
 808     int timeout) {
 809   ENSURE_MSAN_INITED();
 810   int res = REAL(epoll_wait)(epfd, events, maxevents, timeout);
 811   if (res > 0) {
 812     __msan_unpoison(events, __sanitizer::struct_epoll_event_sz * res);
 813   }
 814   return res;
 815 }
 816
 817 INTERCEPTOR(int, epoll_pwait, int epfd, void *events, int maxevents,
 818     int timeout, void *sigmask) {
 819   ENSURE_MSAN_INITED();
 820   int res = REAL(epoll_pwait)(epfd, events, maxevents, timeout, sigmask);
 821   if (res > 0) {
 822     __msan_unpoison(events, __sanitizer::struct_epoll_event_sz * res);
 823   }
 824   return res;
 825 }
 826
 827 INTERCEPTOR(SSIZE_T, recv, int fd, void *buf, SIZE_T len, int flags) {
 828   ENSURE_MSAN_INITED();
 829   SSIZE_T res = REAL(recv)(fd, buf, len, flags);
 830   if (res > 0)
 831     __msan_unpoison(buf, res);
 832   return res;
 833 }
 834
 835 INTERCEPTOR(SSIZE_T, recvfrom, int fd, void *buf, SIZE_T len, int flags,
 836             void *srcaddr, int *addrlen) {
 837   ENSURE_MSAN_INITED();
 838   SIZE_T srcaddr_sz;
 839   if (srcaddr) srcaddr_sz = *addrlen;
 840   SSIZE_T res = REAL(recvfrom)(fd, buf, len, flags, srcaddr, addrlen);
 841   if (res > 0) {
 842     __msan_unpoison(buf, res);
 843     if (srcaddr) {
 844       SIZE_T sz = *addrlen;
 845       __msan_unpoison(srcaddr, (sz < srcaddr_sz) ? sz : srcaddr_sz);
 846     }
 847   }
 848   return res;
 849 }
 850
 851 INTERCEPTOR(void *, calloc, SIZE_T nmemb, SIZE_T size) {
 852   if (CallocShouldReturnNullDueToOverflow(size, nmemb))
 853     return AllocatorReturnNull();
 854   GET_MALLOC_STACK_TRACE;
 855   if (!msan_inited) {
 856     // Hack: dlsym calls calloc before REAL(calloc) is retrieved from dlsym.
 857     const SIZE_T kCallocPoolSize = 1024;
 858     static uptr calloc_memory_for_dlsym[kCallocPoolSize];
 859     static SIZE_T allocated;
 860     SIZE_T size_in_words = ((nmemb * size) + kWordSize - 1) / kWordSize;
 861     void *mem = (void*)&calloc_memory_for_dlsym[allocated];
 862     allocated += size_in_words;
 863     CHECK(allocated < kCallocPoolSize);
 864     return mem;
 865   }
 866   return MsanReallocate(&stack, 0, nmemb * size, sizeof(u64), true);
 867 }
 868
 869 INTERCEPTOR(void *, realloc, void *ptr, SIZE_T size) {
 870   GET_MALLOC_STACK_TRACE;
 871   return MsanReallocate(&stack, ptr, size, sizeof(u64), false);
 872 }
 873
 874 INTERCEPTOR(void *, malloc, SIZE_T size) {
 875   GET_MALLOC_STACK_TRACE;
 876   return MsanReallocate(&stack, 0, size, sizeof(u64), false);
 877 }
 878
 879 void __msan_allocated_memory(const void* data, uptr size) {
 880   GET_MALLOC_STACK_TRACE;
 881   if (flags()->poison_in_malloc)
 882     __msan_poison(data, size);
 883   if (__msan_get_track_origins()) {
 884     u32 stack_id = StackDepotPut(stack.trace, stack.size);
 885     CHECK(stack_id);
 886     CHECK_EQ((stack_id >> 31), 0);  // Higher bit is occupied by stack origins.
 887     __msan_set_origin(data, size, stack_id);
 888   }
 889 }
 890
 891 INTERCEPTOR(void *, mmap, void *addr, SIZE_T length, int prot, int flags,
 892             int fd, OFF_T offset) {
 893   ENSURE_MSAN_INITED();
 894   void *res = REAL(mmap)(addr, length, prot, flags, fd, offset);
 895   if (res != (void*)-1)
 896     __msan_unpoison(res, RoundUpTo(length, GetPageSize()));
 897   return res;
 898 }
 899
 900 INTERCEPTOR(void *, mmap64, void *addr, SIZE_T length, int prot, int flags,
 901             int fd, OFF64_T offset) {
 902   ENSURE_MSAN_INITED();
 903   void *res = REAL(mmap64)(addr, length, prot, flags, fd, offset);
 904   if (res != (void*)-1)
 905     __msan_unpoison(res, RoundUpTo(length, GetPageSize()));
 906   return res;
 907 }
 908
 909 struct dlinfo {
 910   char *dli_fname;
 911   void *dli_fbase;
 912   char *dli_sname;
 913   void *dli_saddr;
 914 };
 915
 916 INTERCEPTOR(int, dladdr, void *addr, dlinfo *info) {
 917   ENSURE_MSAN_INITED();
 918   int res = REAL(dladdr)(addr, info);
 919   if (res != 0) {
 920     __msan_unpoison(info, sizeof(*info));
 921     if (info->dli_fname)
 922       __msan_unpoison(info->dli_fname, REAL(strlen)(info->dli_fname) + 1);
 923     if (info->dli_sname)
 924       __msan_unpoison(info->dli_sname, REAL(strlen)(info->dli_sname) + 1);
 925   }
 926   return res;
 927 }
 928
 929 // dlopen() ultimately calls mmap() down inside the loader, which generally
 930 // doesn't participate in dynamic symbol resolution.  Therefore we won't
 931 // intercept its calls to mmap, and we have to hook it here.  The loader
 932 // initializes the module before returning, so without the dynamic component, we
 933 // won't be able to clear the shadow before the initializers.  Fixing this would
 934 // require putting our own initializer first to clear the shadow.
 935 INTERCEPTOR(void *, dlopen, const char *filename, int flag) {
 936   ENSURE_MSAN_INITED();
 937   EnterLoader();
 938   link_map *map = (link_map *)REAL(dlopen)(filename, flag);
 939   ExitLoader();
 940   if (!__msan_has_dynamic_component() && map) {
 941     // If msandr didn't clear the shadow before the initializers ran, we do it
 942     // ourselves afterwards.
 943     ForEachMappedRegion(map, __msan_unpoison);
 944   }
 945   return (void *)map;
 946 }
 947
 948 typedef int (*dl_iterate_phdr_cb)(__sanitizer_dl_phdr_info *info, SIZE_T size,
 949                                   void *data);
 950 struct dl_iterate_phdr_data {
 951   dl_iterate_phdr_cb callback;
 952   void *data;
 953 };
 954
 955 static int msan_dl_iterate_phdr_cb(__sanitizer_dl_phdr_info *info, SIZE_T size,
 956                                    void *data) {
 957   if (info) {
 958     __msan_unpoison(info, size);
 959     if (info->dlpi_name)
 960       __msan_unpoison(info->dlpi_name, REAL(strlen)(info->dlpi_name) + 1);
 961   }
 962   dl_iterate_phdr_data *cbdata = (dl_iterate_phdr_data *)data;
 963   UnpoisonParam(3);
 964   return cbdata->callback(info, size, cbdata->data);
 965 }
 966
 967 INTERCEPTOR(int, dl_iterate_phdr, dl_iterate_phdr_cb callback, void *data) {
 968   ENSURE_MSAN_INITED();
 969   EnterLoader();
 970   dl_iterate_phdr_data cbdata;
 971   cbdata.callback = callback;
 972   cbdata.data = data;
 973   int res = REAL(dl_iterate_phdr)(msan_dl_iterate_phdr_cb, (void *)&cbdata);
 974   ExitLoader();
 975   return res;
 976 }
 977
 978 INTERCEPTOR(int, getrusage, int who, void *usage) {
 979   ENSURE_MSAN_INITED();
 980   int res = REAL(getrusage)(who, usage);
 981   if (res == 0) {
 982     __msan_unpoison(usage, __sanitizer::struct_rusage_sz);
 983   }
 984   return res;
 985 }
 986
 987 // sigactions_mu guarantees atomicity of sigaction() and signal() calls.
 988 // Access to sigactions[] is gone with relaxed atomics to avoid data race with
 989 // the signal handler.
 990 const int kMaxSignals = 1024;
 991 static atomic_uintptr_t sigactions[kMaxSignals];
 992 static StaticSpinMutex sigactions_mu;
 993
 994 static void SignalHandler(int signo) {
 995   ScopedThreadLocalStateBackup stlsb;
 996   UnpoisonParam(1);
 997
 998   typedef void (*signal_cb)(int x);
 999   signal_cb cb =
1000       (signal_cb)atomic_load(&sigactions[signo], memory_order_relaxed);
1001   cb(signo);
1002 }
1003
1004 static void SignalAction(int signo, void *si, void *uc) {
1005   ScopedThreadLocalStateBackup stlsb;
1006   UnpoisonParam(3);
1007   __msan_unpoison(si, sizeof(__sanitizer_sigaction));
1008   __msan_unpoison(uc, __sanitizer::ucontext_t_sz);
1009
1010   typedef void (*sigaction_cb)(int, void *, void *);
1011   sigaction_cb cb =
1012       (sigaction_cb)atomic_load(&sigactions[signo], memory_order_relaxed);
1013   cb(signo, si, uc);
1014 }
1015
1016 INTERCEPTOR(int, sigaction, int signo, const __sanitizer_sigaction *act,
1017             __sanitizer_sigaction *oldact) {
1018   ENSURE_MSAN_INITED();
1019   // FIXME: check that *act is unpoisoned.
1020   // That requires intercepting all of sigemptyset, sigfillset, etc.
1021   int res;
1022   if (flags()->wrap_signals) {
1023     SpinMutexLock lock(&sigactions_mu);
1024     CHECK_LT(signo, kMaxSignals);
1025     uptr old_cb = atomic_load(&sigactions[signo], memory_order_relaxed);
1026     __sanitizer_sigaction new_act;
1027     __sanitizer_sigaction *pnew_act = act ? &new_act : 0;
1028     if (act) {
1029       internal_memcpy(pnew_act, act, sizeof(__sanitizer_sigaction));
1030       uptr cb = (uptr)pnew_act->sa_sigaction;
1031       uptr new_cb = (pnew_act->sa_flags & __sanitizer::sa_siginfo)
1032                         ? (uptr)SignalAction
1033                         : (uptr)SignalHandler;
1034       if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) {
1035         atomic_store(&sigactions[signo], cb, memory_order_relaxed);
1036         pnew_act->sa_sigaction = (void (*)(int, void *, void *))new_cb;
1037       }
1038     }
1039     res = REAL(sigaction)(signo, pnew_act, oldact);
1040     if (res == 0 && oldact) {
1041       uptr cb = (uptr)oldact->sa_sigaction;
1042       if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) {
1043         oldact->sa_sigaction = (void (*)(int, void *, void *))old_cb;
1044       }
1045     }
1046   } else {
1047     res = REAL(sigaction)(signo, act, oldact);
1048   }
1049
1050   if (res == 0 && oldact) {
1051     __msan_unpoison(oldact, sizeof(__sanitizer_sigaction));
1052   }
1053   return res;
1054 }
1055
1056 INTERCEPTOR(int, signal, int signo, uptr cb) {
1057   ENSURE_MSAN_INITED();
1058   if (flags()->wrap_signals) {
1059     CHECK_LT(signo, kMaxSignals);
1060     SpinMutexLock lock(&sigactions_mu);
1061     if (cb != __sanitizer::sig_ign && cb != __sanitizer::sig_dfl) {
1062       atomic_store(&sigactions[signo], cb, memory_order_relaxed);
1063       cb = (uptr) SignalHandler;
1064     }
1065     return REAL(signal)(signo, cb);
1066   } else {
1067     return REAL(signal)(signo, cb);
1068   }
1069 }
1070
1071 extern "C" int pthread_attr_init(void *attr);
1072 extern "C" int pthread_attr_destroy(void *attr);
1073 extern "C" int pthread_attr_setstacksize(void *attr, uptr stacksize);
1074 extern "C" int pthread_attr_getstack(void *attr, uptr *stack, uptr *stacksize);
1075
1076 INTERCEPTOR(int, pthread_create, void *th, void *attr, void *(*callback)(void*),
1077             void * param) {
1078   ENSURE_MSAN_INITED(); // for GetTlsSize()
1079   __sanitizer_pthread_attr_t myattr;
1080   if (attr == 0) {
1081     pthread_attr_init(&myattr);
1082     attr = &myattr;
1083   }
1084
1085   AdjustStackSizeLinux(attr, common_flags()->verbosity);
1086
1087   int res = REAL(pthread_create)(th, attr, callback, param);
1088   if (attr == &myattr)
1089     pthread_attr_destroy(&myattr);
1090   if (!res) {
1091     __msan_unpoison(th, __sanitizer::pthread_t_sz);
1092   }
1093   return res;
1094 }
1095
1096 INTERCEPTOR(int, pthread_key_create, __sanitizer_pthread_key_t *key,
1097             void (*dtor)(void *value)) {
1098   ENSURE_MSAN_INITED();
1099   int res = REAL(pthread_key_create)(key, dtor);
1100   if (!res && key)
1101     __msan_unpoison(key, sizeof(*key));
1102   return res;
1103 }
1104
1105 INTERCEPTOR(int, pthread_join, void *th, void **retval) {
1106   ENSURE_MSAN_INITED();
1107   int res = REAL(pthread_join)(th, retval);
1108   if (!res && retval)
1109     __msan_unpoison(retval, sizeof(*retval));
1110   return res;
1111 }
1112
1113 extern char *tzname[2];
1114
1115 INTERCEPTOR(void, tzset) {
1116   ENSURE_MSAN_INITED();
1117   REAL(tzset)();
1118   if (tzname[0])
1119     __msan_unpoison(tzname[0], REAL(strlen)(tzname[0]) + 1);
1120   if (tzname[1])
1121     __msan_unpoison(tzname[1], REAL(strlen)(tzname[1]) + 1);
1122   return;
1123 }
1124
1125 struct MSanAtExitRecord {
1126   void (*func)(void *arg);
1127   void *arg;
1128 };
1129
1130 void MSanAtExitWrapper(void *arg) {
1131   UnpoisonParam(1);
1132   MSanAtExitRecord *r = (MSanAtExitRecord *)arg;
1133   r->func(r->arg);
1134   InternalFree(r);
1135 }
1136
1137 // Unpoison argument shadow for C++ module destructors.
1138 INTERCEPTOR(int, __cxa_atexit, void (*func)(void *), void *arg,
1139             void *dso_handle) {
1140   if (msan_init_is_running) return REAL(__cxa_atexit)(func, arg, dso_handle);
1141   ENSURE_MSAN_INITED();
1142   MSanAtExitRecord *r =
1143       (MSanAtExitRecord *)InternalAlloc(sizeof(MSanAtExitRecord));
1144   r->func = func;
1145   r->arg = arg;
1146   return REAL(__cxa_atexit)(MSanAtExitWrapper, r, dso_handle);
1147 }
1148
1149 struct MSanInterceptorContext {
1150   bool in_interceptor_scope;
1151 };
1152
1153 namespace __msan {
1154
1155 int OnExit() {
1156   // FIXME: ask frontend whether we need to return failure.
1157   return 0;
1158 }
1159
1160 }  // namespace __msan
1161
1162 // A version of CHECK_UNPOISED using a saved scope value. Used in common
1163 // interceptors.
1164 #define CHECK_UNPOISONED_CTX(ctx, x, n)                         \
1165   do {                                                          \
1166     if (!((MSanInterceptorContext *)ctx)->in_interceptor_scope) \
1167       CHECK_UNPOISONED_0(x, n);                                 \
1168   } while (0)
1169
1170 #define COMMON_INTERCEPTOR_UNPOISON_PARAM(ctx, count)  \
1171   UnpoisonParam(count)
1172 #define COMMON_INTERCEPTOR_WRITE_RANGE(ctx, ptr, size) \
1173   __msan_unpoison(ptr, size)
1174 #define COMMON_INTERCEPTOR_READ_RANGE(ctx, ptr, size) \
1175   CHECK_UNPOISONED_CTX(ctx, ptr, size)
1176 #define COMMON_INTERCEPTOR_INITIALIZE_RANGE(ctx, ptr, size) \
1177   __msan_unpoison(ptr, size)
1178 #define COMMON_INTERCEPTOR_ENTER(ctx, func, ...)              \
1179   if (msan_init_is_running) return REAL(func)(__VA_ARGS__);   \
1180   MSanInterceptorContext msan_ctx = {IsInInterceptorScope()}; \
1181   ctx = (void *)&msan_ctx;                                    \
1182   (void)ctx;                                                  \
1183   InterceptorScope interceptor_scope;                         \
1184   ENSURE_MSAN_INITED();
1185 #define COMMON_INTERCEPTOR_FD_ACQUIRE(ctx, fd) \
1186   do {                                         \
1187   } while (false)
1188 #define COMMON_INTERCEPTOR_FD_RELEASE(ctx, fd) \
1189   do {                                         \
1190   } while (false)
1191 #define COMMON_INTERCEPTOR_FD_SOCKET_ACCEPT(ctx, fd, newfd) \
1192   do {                                                      \
1193   } while (false)
1194 #define COMMON_INTERCEPTOR_SET_THREAD_NAME(ctx, name) \
1195   do {                                                \
1196   } while (false)  // FIXME
1197 #define COMMON_INTERCEPTOR_BLOCK_REAL(name) REAL(name)
1198 #define COMMON_INTERCEPTOR_ON_EXIT(ctx) OnExit()
1199 #include "sanitizer_common/sanitizer_common_interceptors.inc"
1200
1201 #define COMMON_SYSCALL_PRE_READ_RANGE(p, s) CHECK_UNPOISONED(p, s)
1202 #define COMMON_SYSCALL_PRE_WRITE_RANGE(p, s) \
1203   do {                                       \
1204   } while (false)
1205 #define COMMON_SYSCALL_POST_READ_RANGE(p, s) \
1206   do {                                       \
1207   } while (false)
1208 #define COMMON_SYSCALL_POST_WRITE_RANGE(p, s) __msan_unpoison(p, s)
1209 #include "sanitizer_common/sanitizer_common_syscalls.inc"
1210
1211 // static
1212 void *fast_memset(void *ptr, int c, SIZE_T n) {
1213   // hack until we have a really fast internal_memset
1214   if (sizeof(uptr) == 8 &&
1215       (n % 8) == 0 &&
1216       ((uptr)ptr % 8) == 0 &&
1217       (c == 0 || c == -1)) {
1218     // Printf("memset %p %zd %x\n", ptr, n, c);
1219     uptr to_store = c ? -1L : 0L;
1220     uptr *p = (uptr*)ptr;
1221     for (SIZE_T i = 0; i < n / 8; i++)
1222       p[i] = to_store;
1223     return ptr;
1224   }
1225   return internal_memset(ptr, c, n);
1226 }
1227
1228 // static
1229 void *fast_memcpy(void *dst, const void *src, SIZE_T n) {
1230   // Same hack as in fast_memset above.
1231   if (sizeof(uptr) == 8 &&
1232       (n % 8) == 0 &&
1233       ((uptr)dst % 8) == 0 &&
1234       ((uptr)src % 8) == 0) {
1235     uptr *d = (uptr*)dst;
1236     uptr *s = (uptr*)src;
1237     for (SIZE_T i = 0; i < n / 8; i++)
1238       d[i] = s[i];
1239     return dst;
1240   }
1241   return internal_memcpy(dst, src, n);
1242 }
1243
1244 // These interface functions reside here so that they can use
1245 // fast_memset, etc.
1246 void __msan_unpoison(const void *a, uptr size) {
1247   if (!MEM_IS_APP(a)) return;
1248   fast_memset((void*)MEM_TO_SHADOW((uptr)a), 0, size);
1249 }
1250
1251 void __msan_poison(const void *a, uptr size) {
1252   if (!MEM_IS_APP(a)) return;
1253   fast_memset((void*)MEM_TO_SHADOW((uptr)a),
1254               __msan::flags()->poison_heap_with_zeroes ? 0 : -1, size);
1255 }
1256
1257 void __msan_poison_stack(void *a, uptr size) {
1258   if (!MEM_IS_APP(a)) return;
1259   fast_memset((void*)MEM_TO_SHADOW((uptr)a),
1260               __msan::flags()->poison_stack_with_zeroes ? 0 : -1, size);
1261 }
1262
1263 void __msan_clear_and_unpoison(void *a, uptr size) {
1264   fast_memset(a, 0, size);
1265   fast_memset((void*)MEM_TO_SHADOW((uptr)a), 0, size);
1266 }
1267
1268 void __msan_copy_origin(void *dst, const void *src, uptr size) {
1269   if (!__msan_get_track_origins()) return;
1270   if (!MEM_IS_APP(dst) || !MEM_IS_APP(src)) return;
1271   uptr d = MEM_TO_ORIGIN(dst);
1272   uptr s = MEM_TO_ORIGIN(src);
1273   uptr beg = d & ~3UL;  // align down.
1274   uptr end = (d + size + 3) & ~3UL;  // align up.
1275   s = s & ~3UL;  // align down.
1276   fast_memcpy((void*)beg, (void*)s, end - beg);
1277 }
1278
1279 void __msan_copy_poison(void *dst, const void *src, uptr size) {
1280   if (!MEM_IS_APP(dst)) return;
1281   if (!MEM_IS_APP(src)) return;
1282   fast_memcpy((void*)MEM_TO_SHADOW((uptr)dst),
1283               (void*)MEM_TO_SHADOW((uptr)src), size);
1284   __msan_copy_origin(dst, src, size);
1285 }
1286
1287 void __msan_move_poison(void *dst, const void *src, uptr size) {
1288   if (!MEM_IS_APP(dst)) return;
1289   if (!MEM_IS_APP(src)) return;
1290   internal_memmove((void*)MEM_TO_SHADOW((uptr)dst),
1291          (void*)MEM_TO_SHADOW((uptr)src), size);
1292   __msan_copy_origin(dst, src, size);
1293 }
1294
1295 void *__msan_memcpy(void *dest, const void *src, SIZE_T n) {
1296   ENSURE_MSAN_INITED();
1297   void *res = fast_memcpy(dest, src, n);
1298   __msan_copy_poison(dest, src, n);
1299   return res;
1300 }
1301
1302 void *__msan_memset(void *s, int c, SIZE_T n) {
1303   ENSURE_MSAN_INITED();
1304   void *res = fast_memset(s, c, n);
1305   __msan_unpoison(s, n);
1306   return res;
1307 }
1308
1309 void *__msan_memmove(void *dest, const void *src, SIZE_T n) {
1310   ENSURE_MSAN_INITED();
1311   void *res = REAL(memmove)(dest, src, n);
1312   __msan_move_poison(dest, src, n);
1313   return res;
1314 }
1315
1316 namespace __msan {
1317 void InitializeInterceptors() {
1318   static int inited = 0;
1319   CHECK_EQ(inited, 0);
1320   SANITIZER_COMMON_INTERCEPTORS_INIT;
1321
1322   INTERCEPT_FUNCTION(mmap);
1323   INTERCEPT_FUNCTION(mmap64);
1324   INTERCEPT_FUNCTION(posix_memalign);
1325   INTERCEPT_FUNCTION(memalign);
1326   INTERCEPT_FUNCTION(valloc);
1327   INTERCEPT_FUNCTION(pvalloc);
1328   INTERCEPT_FUNCTION(malloc);
1329   INTERCEPT_FUNCTION(calloc);
1330   INTERCEPT_FUNCTION(realloc);
1331   INTERCEPT_FUNCTION(free);
1332   INTERCEPT_FUNCTION(fread);
1333   INTERCEPT_FUNCTION(fread_unlocked);
1334   INTERCEPT_FUNCTION(readlink);
1335   INTERCEPT_FUNCTION(memcpy);
1336   INTERCEPT_FUNCTION(mempcpy);
1337   INTERCEPT_FUNCTION(memset);
1338   INTERCEPT_FUNCTION(memmove);
1339   INTERCEPT_FUNCTION(bcopy);
1340   INTERCEPT_FUNCTION(wmemset);
1341   INTERCEPT_FUNCTION(wmemcpy);
1342   INTERCEPT_FUNCTION(wmempcpy);
1343   INTERCEPT_FUNCTION(wmemmove);
1344   INTERCEPT_FUNCTION(strcpy);  // NOLINT
1345   INTERCEPT_FUNCTION(stpcpy);  // NOLINT
1346   INTERCEPT_FUNCTION(strdup);
1347   INTERCEPT_FUNCTION(__strdup);
1348   INTERCEPT_FUNCTION(strndup);
1349   INTERCEPT_FUNCTION(__strndup);
1350   INTERCEPT_FUNCTION(strncpy);  // NOLINT
1351   INTERCEPT_FUNCTION(strlen);
1352   INTERCEPT_FUNCTION(strnlen);
1353   INTERCEPT_FUNCTION(gcvt);
1354   INTERCEPT_FUNCTION(strcat);  // NOLINT
1355   INTERCEPT_FUNCTION(strncat);  // NOLINT
1356   INTERCEPT_FUNCTION(strtol);
1357   INTERCEPT_FUNCTION(strtoll);
1358   INTERCEPT_FUNCTION(strtoul);
1359   INTERCEPT_FUNCTION(strtoull);
1360   INTERCEPT_FUNCTION(strtod);
1361   INTERCEPT_FUNCTION(strtod_l);
1362   INTERCEPT_FUNCTION(__strtod_l);
1363   INTERCEPT_FUNCTION(strtof);
1364   INTERCEPT_FUNCTION(strtof_l);
1365   INTERCEPT_FUNCTION(__strtof_l);
1366   INTERCEPT_FUNCTION(strtold);
1367   INTERCEPT_FUNCTION(strtold_l);
1368   INTERCEPT_FUNCTION(__strtold_l);
1369   INTERCEPT_FUNCTION(vasprintf);
1370   INTERCEPT_FUNCTION(asprintf);
1371   INTERCEPT_FUNCTION(vsprintf);
1372   INTERCEPT_FUNCTION(vsnprintf);
1373   INTERCEPT_FUNCTION(vswprintf);
1374   INTERCEPT_FUNCTION(sprintf);  // NOLINT
1375   INTERCEPT_FUNCTION(snprintf);
1376   INTERCEPT_FUNCTION(swprintf);
1377   INTERCEPT_FUNCTION(strftime);
1378   INTERCEPT_FUNCTION(mbtowc);
1379   INTERCEPT_FUNCTION(mbrtowc);
1380   INTERCEPT_FUNCTION(wcslen);
1381   INTERCEPT_FUNCTION(wcschr);
1382   INTERCEPT_FUNCTION(wcscpy);
1383   INTERCEPT_FUNCTION(wcscmp);
1384   INTERCEPT_FUNCTION(wcstod);
1385   INTERCEPT_FUNCTION(getenv);
1386   INTERCEPT_FUNCTION(setenv);
1387   INTERCEPT_FUNCTION(putenv);
1388   INTERCEPT_FUNCTION(gettimeofday);
1389   INTERCEPT_FUNCTION(fcvt);
1390   INTERCEPT_FUNCTION(__fxstat);
1391   INTERCEPT_FUNCTION(__fxstatat);
1392   INTERCEPT_FUNCTION(__xstat);
1393   INTERCEPT_FUNCTION(__lxstat);
1394   INTERCEPT_FUNCTION(__fxstat64);
1395   INTERCEPT_FUNCTION(__fxstatat64);
1396   INTERCEPT_FUNCTION(__xstat64);
1397   INTERCEPT_FUNCTION(__lxstat64);
1398   INTERCEPT_FUNCTION(pipe);
1399   INTERCEPT_FUNCTION(pipe2);
1400   INTERCEPT_FUNCTION(socketpair);
1401   INTERCEPT_FUNCTION(fgets);
1402   INTERCEPT_FUNCTION(fgets_unlocked);
1403   INTERCEPT_FUNCTION(getrlimit);
1404   INTERCEPT_FUNCTION(getrlimit64);
1405   INTERCEPT_FUNCTION(statfs);
1406   INTERCEPT_FUNCTION(fstatfs);
1407   INTERCEPT_FUNCTION(statfs64);
1408   INTERCEPT_FUNCTION(fstatfs64);
1409   INTERCEPT_FUNCTION(uname);
1410   INTERCEPT_FUNCTION(gethostname);
1411   INTERCEPT_FUNCTION(epoll_wait);
1412   INTERCEPT_FUNCTION(epoll_pwait);
1413   INTERCEPT_FUNCTION(recv);
1414   INTERCEPT_FUNCTION(recvfrom);
1415   INTERCEPT_FUNCTION(dladdr);
1416   INTERCEPT_FUNCTION(dlopen);
1417   INTERCEPT_FUNCTION(dl_iterate_phdr);
1418   INTERCEPT_FUNCTION(getrusage);
1419   INTERCEPT_FUNCTION(sigaction);
1420   INTERCEPT_FUNCTION(signal);
1421   INTERCEPT_FUNCTION(pthread_create);
1422   INTERCEPT_FUNCTION(pthread_key_create);
1423   INTERCEPT_FUNCTION(pthread_join);
1424   INTERCEPT_FUNCTION(tzset);
1425   INTERCEPT_FUNCTION(__cxa_atexit);
1426   inited = 1;
1427 }
1428 }  // namespace __msan