js/src/jit/ProcessExecutableMemory.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "jit/ProcessExecutableMemory.h"
   8
   9 #include "mozilla/Array.h"
  10 #include "mozilla/Atomics.h"
  11 #include "mozilla/DebugOnly.h"
  12 #include "mozilla/Maybe.h"
  13 #include "mozilla/TaggedAnonymousMemory.h"
  14 #include "mozilla/XorShift128PlusRNG.h"
  15
  16 #include <errno.h>
  17
  18 #include "jsfriendapi.h"
  19 #include "jsmath.h"
  20
  21 #include "gc/Memory.h"
  22 #include "jit/FlushICache.h"  // js::jit::FlushICache
  23 #include "jit/JitOptions.h"
  24 #include "threading/LockGuard.h"
  25 #include "threading/Mutex.h"
  26 #include "util/Memory.h"
  27 #include "util/Poison.h"
  28 #include "util/WindowsWrapper.h"
  29 #include "vm/MutexIDs.h"
  30
  31 #ifdef XP_WIN
  32 #  include "mozilla/StackWalk_windows.h"
  33 #  include "mozilla/WindowsVersion.h"
  34 #elif defined(__wasi__)
  35 #  if defined(JS_CODEGEN_WASM32)
  36 #    include <cstdlib>
  37 #  else
  38 // Nothing.
  39 #  endif
  40 #else
  41 #  include <sys/mman.h>
  42 #  include <unistd.h>
  43 #endif
  44
  45 #ifdef MOZ_VALGRIND
  46 #  include <valgrind/valgrind.h>
  47 #endif
  48
  49 using namespace js;
  50 using namespace js::jit;
  51
  52 #ifdef XP_WIN
  53 #  if defined(HAVE_64BIT_BUILD)
  54 #    define NEED_JIT_UNWIND_HANDLING
  55 #  endif
  56
  57 static void* ComputeRandomAllocationAddress() {
  58   /*
  59    * Inspiration is V8's OS::Allocate in platform-win32.cc.
  60    *
  61    * VirtualAlloc takes 64K chunks out of the virtual address space, so we
  62    * keep 16b alignment.
  63    *
  64    * x86: V8 comments say that keeping addresses in the [64MiB, 1GiB) range
  65    * tries to avoid system default DLL mapping space. In the end, we get 13
  66    * bits of randomness in our selection.
  67    * x64: [2GiB, 4TiB), with 25 bits of randomness.
  68    */
  69 #  ifdef HAVE_64BIT_BUILD
  70   static const uintptr_t base = 0x0000000080000000;
  71   static const uintptr_t mask = 0x000003ffffff0000;
  72 #  elif defined(_M_IX86) || defined(__i386__)
  73   static const uintptr_t base = 0x04000000;
  74   static const uintptr_t mask = 0x3fff0000;
  75 #  else
  76 #    error "Unsupported architecture"
  77 #  endif
  78
  79   uint64_t rand = js::GenerateRandomSeed();
  80   return (void*)(base | (rand & mask));
  81 }
  82
  83 #  ifdef NEED_JIT_UNWIND_HANDLING
  84 static js::JitExceptionHandler sJitExceptionHandler;
  85 static bool sHasInstalledFunctionTable = false;
  86 #  endif
  87
  88 JS_PUBLIC_API void js::SetJitExceptionHandler(JitExceptionHandler handler) {
  89 #  ifdef NEED_JIT_UNWIND_HANDLING
  90   MOZ_ASSERT(!sJitExceptionHandler);
  91   sJitExceptionHandler = handler;
  92 #  else
  93   // Just do nothing if unwind handling is disabled.
  94 #  endif
  95 }
  96
  97 #  ifdef NEED_JIT_UNWIND_HANDLING
  98 #    if defined(_M_ARM64)
  99 // See the ".xdata records" section of
 100 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
 101 // These records can have various fields present or absent depending on the
 102 // bits set in the header. Our struct will use one 32-bit slot for unwind codes,
 103 // and no slots for epilog scopes.
 104 struct UnwindData {
 105   uint32_t functionLength : 18;
 106   uint32_t version : 2;
 107   uint32_t hasExceptionHandler : 1;
 108   uint32_t packedEpilog : 1;
 109   uint32_t epilogCount : 5;
 110   uint32_t codeWords : 5;
 111   uint8_t unwindCodes[4];
 112   uint32_t exceptionHandler;
 113 };
 114
 115 static const unsigned ThunkLength = 20;
 116 #    else
 117 // From documentation for UNWIND_INFO on
 118 // https://learn.microsoft.com/en-us/cpp/build/exception-handling-x64
 119 struct UnwindInfo {
 120   uint8_t version : 3;
 121   uint8_t flags : 5;
 122   uint8_t sizeOfPrologue;
 123   uint8_t countOfUnwindCodes;
 124   uint8_t frameRegister : 4;
 125   uint8_t frameOffset : 4;
 126 };
 127 static const unsigned ThunkLength = 12;
 128 union UnwindCode {
 129   struct {
 130     uint8_t codeOffset;
 131     uint8_t unwindOp : 4;
 132     uint8_t opInfo : 4;
 133   };
 134   uint16_t frameOffset;
 135 };
 136
 137 static constexpr int kNumberOfUnwindCodes = 2;
 138 static constexpr int kPushRbpInstructionLength = 1;
 139 static constexpr int kMovRbpRspInstructionLength = 3;
 140 static constexpr int kRbpPrefixCodes = 2;
 141 static constexpr int kRbpPrefixLength =
 142     kPushRbpInstructionLength + kMovRbpRspInstructionLength;
 143
 144 struct UnwindData {
 145   UnwindInfo unwindInfo;
 146   UnwindCode unwindCodes[kNumberOfUnwindCodes];
 147   uint32_t exceptionHandler;
 148
 149   UnwindData() {
 150     static constexpr int kOpPushNonvol = 0;
 151     static constexpr int kOpSetFPReg = 3;
 152
 153     unwindInfo.version = 1;
 154     unwindInfo.flags = UNW_FLAG_EHANDLER;
 155     unwindInfo.sizeOfPrologue = kRbpPrefixLength;
 156     unwindInfo.countOfUnwindCodes = kRbpPrefixCodes;
 157     unwindInfo.frameRegister = 5;
 158     unwindInfo.frameOffset = 0;
 159
 160     // Offset here are specified to beginning of the -next- instruction.
 161     unwindCodes[0].codeOffset = kRbpPrefixLength;  // movq rbp, rsp
 162     unwindCodes[0].unwindOp = kOpSetFPReg;
 163     unwindCodes[0].opInfo = 0;
 164
 165     unwindCodes[1].codeOffset = kPushRbpInstructionLength;  // push rbp
 166     unwindCodes[1].unwindOp = kOpPushNonvol;
 167     unwindCodes[1].opInfo = 5;
 168   }
 169 };
 170 #    endif
 171
 172 struct ExceptionHandlerRecord {
 173   void* dynamicTable;
 174   UnwindData unwindData;
 175   uint8_t thunk[ThunkLength];
 176   RUNTIME_FUNCTION runtimeFunction;
 177 };
 178
 179 // This function must match the function pointer type PEXCEPTION_HANDLER
 180 // mentioned in:
 181 //   http://msdn.microsoft.com/en-us/library/ssa62fwe.aspx.
 182 // This type is rather elusive in documentation; Wine is the best I've found:
 183 //   http://source.winehq.org/source/include/winnt.h
 184 static DWORD ExceptionHandler(PEXCEPTION_RECORD exceptionRecord,
 185                               _EXCEPTION_REGISTRATION_RECORD*, PCONTEXT context,
 186                               _EXCEPTION_REGISTRATION_RECORD**) {
 187   if (sJitExceptionHandler) {
 188     return sJitExceptionHandler(exceptionRecord, context);
 189   }
 190
 191   return ExceptionContinueSearch;
 192 }
 193
 194 // Required for enabling Stackwalking on windows using external tools.
 195 extern "C" NTSYSAPI DWORD NTAPI RtlAddGrowableFunctionTable(
 196     PVOID* DynamicTable, PRUNTIME_FUNCTION FunctionTable, DWORD EntryCount,
 197     DWORD MaximumEntryCount, ULONG_PTR RangeBase, ULONG_PTR RangeEnd);
 198
 199 // For an explanation of the problem being solved here, see
 200 // SetJitExceptionFilter in jsfriendapi.h.
 201 static bool RegisterExecutableMemory(void* p, size_t bytes, size_t pageSize) {
 202   if (!VirtualAlloc(p, pageSize, MEM_COMMIT, PAGE_READWRITE)) {
 203     MOZ_CRASH();
 204   }
 205
 206   // A page was reserved inside this structure for the record. This is because
 207   // all entries in the record are describes as an offset from the start of the
 208   // memory region. We construct the record there.
 209   ExceptionHandlerRecord* r = new (p) ExceptionHandlerRecord();
 210   void* handler = JS_FUNC_TO_DATA_PTR(void*, ExceptionHandler);
 211
 212   // Because the .xdata format on ARM64 can only encode sizes up to 1M (much
 213   // too small for our JIT code regions), we register a function table callback
 214   // to provide RUNTIME_FUNCTIONs at runtime. Windows doesn't seem to care about
 215   // the size fields on RUNTIME_FUNCTIONs that are created in this way, so the
 216   // same RUNTIME_FUNCTION can work for any address in the region. We'll set up
 217   // a generic one now and the callback can just return a pointer to it.
 218
 219   // All these fields are specified to be offsets from the base of the
 220   // executable code (which is 'p'), even if they have 'Address' in their
 221   // names. In particular, exceptionHandler is a ULONG offset which is a
 222   // 32-bit integer. Since 'p' can be farther than INT32_MAX away from
 223   // sJitExceptionHandler, we must generate a little thunk inside the
 224   // record. The record is put on its own page so that we can take away write
 225   // access to protect against accidental clobbering.
 226
 227 #    if defined(_M_ARM64)
 228   if (!sJitExceptionHandler) {
 229     return false;
 230   }
 231
 232   r->runtimeFunction.BeginAddress = pageSize;
 233   r->runtimeFunction.UnwindData = offsetof(ExceptionHandlerRecord, unwindData);
 234   static_assert(offsetof(ExceptionHandlerRecord, unwindData) % 4 == 0,
 235                 "The ARM64 .pdata format requires that exception information "
 236                 "RVAs be 4-byte aligned.");
 237
 238   memset(&r->unwindData, 0, sizeof(r->unwindData));
 239   r->unwindData.hasExceptionHandler = true;
 240   r->unwindData.exceptionHandler = offsetof(ExceptionHandlerRecord, thunk);
 241
 242   // Use a fake unwind code to make the Windows unwinder do _something_. If the
 243   // PC and SP both stay unchanged, we'll fail the unwinder's sanity checks and
 244   // it won't call our exception handler.
 245   r->unwindData.codeWords = 1;  // one 32-bit word gives us up to 4 codes
 246   r->unwindData.unwindCodes[0] =
 247       0b00000001;  // alloc_s small stack of size 1*16
 248   r->unwindData.unwindCodes[1] = 0b11100100;  // end
 249
 250   uint32_t* thunk = (uint32_t*)r->thunk;
 251   uint16_t* addr = (uint16_t*)&handler;
 252
 253   // xip0/r16 should be safe to clobber: Windows just used it to call our thunk.
 254   const uint8_t reg = 16;
 255
 256   // Say `handler` is 0x4444333322221111, then:
 257   thunk[0] = 0xd2800000 | addr[0] << 5 | reg;  // mov  xip0, 1111
 258   thunk[1] = 0xf2a00000 | addr[1] << 5 | reg;  // movk xip0, 2222 lsl #0x10
 259   thunk[2] = 0xf2c00000 | addr[2] << 5 | reg;  // movk xip0, 3333 lsl #0x20
 260   thunk[3] = 0xf2e00000 | addr[3] << 5 | reg;  // movk xip0, 4444 lsl #0x30
 261   thunk[4] = 0xd61f0000 | reg << 5;            // br xip0
 262 #    else
 263   r->runtimeFunction.BeginAddress = pageSize;
 264   r->runtimeFunction.EndAddress = (DWORD)bytes;
 265   r->runtimeFunction.UnwindData = offsetof(ExceptionHandlerRecord, unwindData);
 266   r->unwindData.exceptionHandler = offsetof(ExceptionHandlerRecord, thunk);
 267
 268   // mov imm64, rax
 269   r->thunk[0] = 0x48;
 270   r->thunk[1] = 0xb8;
 271   memcpy(&r->thunk[2], &handler, 8);
 272
 273   // jmp rax
 274   r->thunk[10] = 0xff;
 275   r->thunk[11] = 0xe0;
 276 #    endif
 277
 278   // RtlAddGrowableFunctionTable will write into the region. We must therefore
 279   // only write-protect is after this has been called.
 280
 281   // XXX NB: The profiler believes this function is only called from the main
 282   // thread. If that ever becomes untrue, the profiler must be updated
 283   // immediately.
 284   {
 285     AutoSuppressStackWalking suppress;
 286     DWORD result = RtlAddGrowableFunctionTable(
 287         &r->dynamicTable, &r->runtimeFunction, 1, 1, (ULONG_PTR)p,
 288         (ULONG_PTR)p + bytes - pageSize);
 289     if (result != S_OK) {
 290       return false;
 291     }
 292   }
 293
 294   DWORD oldProtect;
 295   if (!VirtualProtect(p, pageSize, PAGE_EXECUTE_READ, &oldProtect)) {
 296     MOZ_CRASH();
 297   }
 298
 299   return true;
 300 }
 301
 302 static void UnregisterExecutableMemory(void* p, size_t bytes, size_t pageSize) {
 303   // There's no such thing as RtlUninstallFunctionTableCallback, so there's
 304   // nothing to do here.
 305 }
 306 #  endif
 307
 308 static void* ReserveProcessExecutableMemory(size_t bytes) {
 309 #  ifdef NEED_JIT_UNWIND_HANDLING
 310   size_t pageSize = gc::SystemPageSize();
 311   // Always reserve space for the unwind information.
 312   bytes += pageSize;
 313 #  endif
 314
 315   void* p = nullptr;
 316   for (size_t i = 0; i < 10; i++) {
 317     void* randomAddr = ComputeRandomAllocationAddress();
 318     p = VirtualAlloc(randomAddr, bytes, MEM_RESERVE, PAGE_NOACCESS);
 319     if (p) {
 320       break;
 321     }
 322   }
 323
 324   if (!p) {
 325     // Try again without randomization.
 326     p = VirtualAlloc(nullptr, bytes, MEM_RESERVE, PAGE_NOACCESS);
 327     if (!p) {
 328       return nullptr;
 329     }
 330   }
 331
 332 #  ifdef NEED_JIT_UNWIND_HANDLING
 333   if (RegisterExecutableMemory(p, bytes, pageSize)) {
 334     sHasInstalledFunctionTable = true;
 335   } else {
 336     if (sJitExceptionHandler) {
 337       // This should have succeeded if we have an exception handler. Bail.
 338       VirtualFree(p, 0, MEM_RELEASE);
 339       return nullptr;
 340     }
 341   }
 342
 343   // Skip the first page where we might have allocated an exception handler
 344   // record.
 345   p = (uint8_t*)p + pageSize;
 346   bytes -= pageSize;
 347
 348   RegisterJitCodeRegion((uint8_t*)p, bytes);
 349 #  endif
 350   return p;
 351 }
 352
 353 static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
 354 #  ifdef NEED_JIT_UNWIND_HANDLING
 355   UnregisterJitCodeRegion((uint8_t*)addr, bytes);
 356
 357   size_t pageSize = gc::SystemPageSize();
 358   addr = (uint8_t*)addr - pageSize;
 359
 360   if (sHasInstalledFunctionTable) {
 361     UnregisterExecutableMemory(addr, bytes, pageSize);
 362   }
 363 #  endif
 364
 365   VirtualFree(addr, 0, MEM_RELEASE);
 366 }
 367
 368 static DWORD ProtectionSettingToFlags(ProtectionSetting protection) {
 369   if (!JitOptions.writeProtectCode) {
 370     return PAGE_EXECUTE_READWRITE;
 371   }
 372   switch (protection) {
 373     case ProtectionSetting::Writable:
 374       return PAGE_READWRITE;
 375     case ProtectionSetting::Executable:
 376       return PAGE_EXECUTE_READ;
 377   }
 378   MOZ_CRASH();
 379 }
 380
 381 [[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
 382                                       ProtectionSetting protection) {
 383   void* p = VirtualAlloc(addr, bytes, MEM_COMMIT,
 384                          ProtectionSettingToFlags(protection));
 385   if (!p) {
 386     return false;
 387   }
 388   MOZ_RELEASE_ASSERT(p == addr);
 389   return true;
 390 }
 391
 392 static void DecommitPages(void* addr, size_t bytes) {
 393   if (!VirtualFree(addr, bytes, MEM_DECOMMIT)) {
 394     MOZ_CRASH("DecommitPages failed");
 395   }
 396 }
 397 #elif defined(__wasi__)
 398 #  if defined(JS_CODEGEN_WASM32)
 399 static void* ReserveProcessExecutableMemory(size_t bytes) {
 400   return malloc(bytes);
 401 }
 402
 403 static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
 404   free(addr);
 405 }
 406
 407 [[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
 408                                       ProtectionSetting protection) {
 409   return true;
 410 }
 411
 412 static void DecommitPages(void* addr, size_t bytes) {}
 413
 414 #  else
 415 static void* ReserveProcessExecutableMemory(size_t bytes) {
 416   MOZ_CRASH("NYI for WASI.");
 417   return nullptr;
 418 }
 419 static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
 420   MOZ_CRASH("NYI for WASI.");
 421 }
 422 [[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
 423                                       ProtectionSetting protection) {
 424   MOZ_CRASH("NYI for WASI.");
 425   return false;
 426 }
 427 static void DecommitPages(void* addr, size_t bytes) {
 428   MOZ_CRASH("NYI for WASI.");
 429 }
 430 #  endif
 431 #else  // !XP_WIN && !__wasi__
 432 #  ifndef MAP_NORESERVE
 433 #    define MAP_NORESERVE 0
 434 #  endif
 435
 436 static void* ComputeRandomAllocationAddress() {
 437 #  ifdef __OpenBSD__
 438   // OpenBSD already has random mmap and the idea that all x64 cpus
 439   // have 48-bit address space is not correct. Returning nullptr
 440   // allows OpenBSD do to the right thing.
 441   return nullptr;
 442 #  else
 443   uint64_t rand = js::GenerateRandomSeed();
 444
 445 #    ifdef HAVE_64BIT_BUILD
 446   // x64 CPUs have a 48-bit address space and on some platforms the OS will
 447   // give us access to 47 bits, so to be safe we right shift by 18 to leave
 448   // 46 bits.
 449   rand >>= 18;
 450 #    else
 451   // On 32-bit, right shift by 34 to leave 30 bits, range [0, 1GiB). Then add
 452   // 512MiB to get range [512MiB, 1.5GiB), or [0x20000000, 0x60000000). This
 453   // is based on V8 comments in platform-posix.cc saying this range is
 454   // relatively unpopulated across a variety of kernels.
 455   rand >>= 34;
 456   rand += 512 * 1024 * 1024;
 457 #    endif
 458
 459   // Ensure page alignment.
 460   uintptr_t mask = ~uintptr_t(gc::SystemPageSize() - 1);
 461   return (void*)uintptr_t(rand & mask);
 462 #  endif
 463 }
 464
 465 static void DecommitPages(void* addr, size_t bytes);
 466
 467 static void* ReserveProcessExecutableMemory(size_t bytes) {
 468   // On most Unix platforms our strategy is as follows:
 469   //
 470   // * Reserve:  mmap with PROT_NONE
 471   // * Commit:   mmap with MAP_FIXED, PROT_READ | ...
 472   // * Decommit: mmap with MAP_FIXED, PROT_NONE
 473   //
 474   // On Apple Silicon this only works if we use mprotect to implement W^X. To
 475   // use RWX pages with the faster pthread_jit_write_protect_np API for
 476   // thread-local writable/executable switching, the kernel enforces the
 477   // following rules:
 478   //
 479   // * The initial mmap must be called with MAP_JIT.
 480   // * MAP_FIXED can't be used with MAP_JIT.
 481   // * Since macOS 11.2, mprotect can't be used to change permissions of RWX JIT
 482   //   pages (even PROT_NONE fails).
 483   //   See https://developer.apple.com/forums/thread/672804.
 484   //
 485   // This means we have to use the following strategy on Apple Silicon:
 486   //
 487   // * Reserve:  1) mmap with PROT_READ | PROT_WRITE | PROT_EXEC and MAP_JIT
 488   //             2) decommit
 489   // * Commit:   madvise with MADV_FREE_REUSE
 490   // * Decommit: madvise with MADV_FREE_REUSABLE
 491   //
 492   // On Intel Macs we also need to use MAP_JIT, to be compatible with the
 493   // Hardened Runtime (with com.apple.security.cs.allow-jit = true). The
 494   // pthread_jit_write_protect_np API is not available on Intel and MAP_JIT
 495   // can't be used with MAP_FIXED, so we have to use a hybrid of the above two
 496   // strategies:
 497   //
 498   // * Reserve:  1) mmap with PROT_NONE and MAP_JIT
 499   //             2) decommit
 500   // * Commit:   1) madvise with MADV_FREE_REUSE
 501   //             2) mprotect with PROT_READ | ...
 502   // * Decommit: 1) mprotect with PROT_NONE
 503   //             2) madvise with MADV_FREE_REUSABLE
 504   //
 505   // This is inspired by V8's code in OS::SetPermissions.
 506
 507   // Note that randomAddr is just a hint: if the address is not available
 508   // mmap will pick a different address.
 509   void* randomAddr = ComputeRandomAllocationAddress();
 510   unsigned protection = PROT_NONE;
 511   unsigned flags = MAP_NORESERVE | MAP_PRIVATE | MAP_ANON;
 512 #  if defined(XP_DARWIN)
 513   flags |= MAP_JIT;
 514 #    if defined(JS_USE_APPLE_FAST_WX)
 515   protection = PROT_READ | PROT_WRITE | PROT_EXEC;
 516 #    endif
 517 #  endif
 518   void* p = MozTaggedAnonymousMmap(randomAddr, bytes, protection, flags, -1, 0,
 519                                    "js-executable-memory");
 520   if (p == MAP_FAILED) {
 521     return nullptr;
 522   }
 523 #  if defined(XP_DARWIN)
 524   DecommitPages(p, bytes);
 525 #  endif
 526   return p;
 527 }
 528
 529 static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
 530   mozilla::DebugOnly<int> result = munmap(addr, bytes);
 531   MOZ_ASSERT(!result || errno == ENOMEM);
 532 }
 533
 534 static unsigned ProtectionSettingToFlags(ProtectionSetting protection) {
 535   if (!JitOptions.writeProtectCode) {
 536     return PROT_READ | PROT_WRITE | PROT_EXEC;
 537   }
 538 #  ifdef MOZ_VALGRIND
 539   // If we're configured for Valgrind and running on it, use a slacker
 540   // scheme that doesn't change execute permissions, since doing so causes
 541   // Valgrind a lot of extra overhead re-JITting code that loses and later
 542   // regains execute permission.  See bug 1338179.
 543   if (RUNNING_ON_VALGRIND) {
 544     switch (protection) {
 545       case ProtectionSetting::Writable:
 546         return PROT_READ | PROT_WRITE | PROT_EXEC;
 547       case ProtectionSetting::Executable:
 548         return PROT_READ | PROT_EXEC;
 549     }
 550     MOZ_CRASH();
 551   }
 552   // If we get here, we're configured for Valgrind but not running on
 553   // it, so use the standard scheme.
 554 #  endif
 555   switch (protection) {
 556     case ProtectionSetting::Writable:
 557       return PROT_READ | PROT_WRITE;
 558     case ProtectionSetting::Executable:
 559       return PROT_READ | PROT_EXEC;
 560   }
 561   MOZ_CRASH();
 562 }
 563
 564 [[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
 565                                       ProtectionSetting protection) {
 566   // See the comment in ReserveProcessExecutableMemory.
 567 #  if defined(XP_DARWIN)
 568   int ret;
 569   do {
 570     ret = madvise(addr, bytes, MADV_FREE_REUSE);
 571   } while (ret != 0 && errno == EAGAIN);
 572   if (ret != 0) {
 573     return false;
 574   }
 575 #    if !defined(JS_USE_APPLE_FAST_WX)
 576   unsigned flags = ProtectionSettingToFlags(protection);
 577   if (mprotect(addr, bytes, flags)) {
 578     return false;
 579   }
 580 #    endif
 581   return true;
 582 #  else
 583   unsigned flags = ProtectionSettingToFlags(protection);
 584   void* p = MozTaggedAnonymousMmap(addr, bytes, flags,
 585                                    MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0,
 586                                    "js-executable-memory");
 587   if (p == MAP_FAILED) {
 588     return false;
 589   }
 590   MOZ_RELEASE_ASSERT(p == addr);
 591   return true;
 592 #  endif
 593 }
 594
 595 static void DecommitPages(void* addr, size_t bytes) {
 596   // See the comment in ReserveProcessExecutableMemory.
 597 #  if defined(XP_DARWIN)
 598   int ret;
 599 #    if !defined(JS_USE_APPLE_FAST_WX)
 600   ret = mprotect(addr, bytes, PROT_NONE);
 601   MOZ_RELEASE_ASSERT(ret == 0);
 602 #    endif
 603   do {
 604     ret = madvise(addr, bytes, MADV_FREE_REUSABLE);
 605   } while (ret != 0 && errno == EAGAIN);
 606   MOZ_RELEASE_ASSERT(ret == 0);
 607 #  else
 608   // Use mmap with MAP_FIXED and PROT_NONE. Inspired by jemalloc's
 609   // pages_decommit.
 610   void* p = MozTaggedAnonymousMmap(addr, bytes, PROT_NONE,
 611                                    MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0,
 612                                    "js-executable-memory");
 613   MOZ_RELEASE_ASSERT(addr == p);
 614 #  endif
 615 }
 616 #endif
 617
 618 template <size_t NumBits>
 619 class PageBitSet {
 620   using WordType = uint32_t;
 621   static const size_t BitsPerWord = sizeof(WordType) * 8;
 622
 623   static_assert((NumBits % BitsPerWord) == 0,
 624                 "NumBits must be a multiple of BitsPerWord");
 625   static const size_t NumWords = NumBits / BitsPerWord;
 626
 627   mozilla::Array<WordType, NumWords> words_;
 628
 629   uint32_t indexToWord(uint32_t index) const {
 630     MOZ_ASSERT(index < NumBits);
 631     return index / BitsPerWord;
 632   }
 633   WordType indexToBit(uint32_t index) const {
 634     MOZ_ASSERT(index < NumBits);
 635     return WordType(1) << (index % BitsPerWord);
 636   }
 637
 638  public:
 639   void init() { mozilla::PodArrayZero(words_); }
 640   bool contains(size_t index) const {
 641     uint32_t word = indexToWord(index);
 642     return words_[word] & indexToBit(index);
 643   }
 644   void insert(size_t index) {
 645     MOZ_ASSERT(!contains(index));
 646     uint32_t word = indexToWord(index);
 647     words_[word] |= indexToBit(index);
 648   }
 649   void remove(size_t index) {
 650     MOZ_ASSERT(contains(index));
 651     uint32_t word = indexToWord(index);
 652     words_[word] &= ~indexToBit(index);
 653   }
 654
 655 #ifdef DEBUG
 656   bool empty() const {
 657     for (size_t i = 0; i < NumWords; i++) {
 658       if (words_[i] != 0) {
 659         return false;
 660       }
 661     }
 662     return true;
 663   }
 664 #endif
 665 };
 666
 667 // Per-process executable memory allocator. It reserves a block of memory of
 668 // MaxCodeBytesPerProcess bytes, then allocates/deallocates pages from that.
 669 //
 670 // This has a number of benefits compared to raw mmap/VirtualAlloc:
 671 //
 672 // * More resillient against certain attacks.
 673 //
 674 // * Behaves more consistently across platforms: it avoids the 64K granularity
 675 //   issues on Windows, for instance.
 676 //
 677 // * On x64, near jumps can be used for jumps to other JIT pages.
 678 //
 679 // * On Win64, we have to register the exception handler only once (at process
 680 //   startup). This saves some memory and avoids RtlAddFunctionTable profiler
 681 //   deadlocks.
 682 class ProcessExecutableMemory {
 683   static_assert(
 684       (MaxCodeBytesPerProcess % ExecutableCodePageSize) == 0,
 685       "MaxCodeBytesPerProcess must be a multiple of ExecutableCodePageSize");
 686   static const size_t MaxCodePages =
 687       MaxCodeBytesPerProcess / ExecutableCodePageSize;
 688
 689   // Start of the MaxCodeBytesPerProcess memory block or nullptr if
 690   // uninitialized. Note that this is NOT guaranteed to be aligned to
 691   // ExecutableCodePageSize.
 692   uint8_t* base_;
 693
 694   // The fields below should only be accessed while we hold the lock.
 695   Mutex lock_ MOZ_UNANNOTATED;
 696
 697   // pagesAllocated_ is an Atomic so that bytesAllocated does not have to
 698   // take the lock.
 699   mozilla::Atomic<size_t, mozilla::ReleaseAcquire> pagesAllocated_;
 700
 701   // Page where we should try to allocate next.
 702   size_t cursor_;
 703
 704   mozilla::Maybe<mozilla::non_crypto::XorShift128PlusRNG> rng_;
 705   PageBitSet<MaxCodePages> pages_;
 706
 707  public:
 708   ProcessExecutableMemory()
 709       : base_(nullptr),
 710         lock_(mutexid::ProcessExecutableRegion),
 711         pagesAllocated_(0),
 712         cursor_(0),
 713         pages_() {}
 714
 715   [[nodiscard]] bool init() {
 716     pages_.init();
 717
 718     MOZ_RELEASE_ASSERT(!initialized());
 719     MOZ_RELEASE_ASSERT(HasJitBackend());
 720     MOZ_RELEASE_ASSERT(gc::SystemPageSize() <= ExecutableCodePageSize);
 721
 722     void* p = ReserveProcessExecutableMemory(MaxCodeBytesPerProcess);
 723     if (!p) {
 724       return false;
 725     }
 726
 727     base_ = static_cast<uint8_t*>(p);
 728
 729     mozilla::Array<uint64_t, 2> seed;
 730     GenerateXorShift128PlusSeed(seed);
 731     rng_.emplace(seed[0], seed[1]);
 732     return true;
 733   }
 734
 735   uint8_t* base() const { return base_; }
 736
 737   bool initialized() const { return base_ != nullptr; }
 738
 739   size_t bytesAllocated() const {
 740     MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
 741     return pagesAllocated_ * ExecutableCodePageSize;
 742   }
 743
 744   void release() {
 745     MOZ_ASSERT(initialized());
 746     MOZ_ASSERT(pages_.empty());
 747     MOZ_ASSERT(pagesAllocated_ == 0);
 748     DeallocateProcessExecutableMemory(base_, MaxCodeBytesPerProcess);
 749     base_ = nullptr;
 750     rng_.reset();
 751     MOZ_ASSERT(!initialized());
 752   }
 753
 754   void assertValidAddress(void* p, size_t bytes) const {
 755     MOZ_RELEASE_ASSERT(p >= base_ &&
 756                        uintptr_t(p) + bytes <=
 757                            uintptr_t(base_) + MaxCodeBytesPerProcess);
 758   }
 759
 760   bool containsAddress(const void* p) const {
 761     return p >= base_ &&
 762            uintptr_t(p) < uintptr_t(base_) + MaxCodeBytesPerProcess;
 763   }
 764
 765   void* allocate(size_t bytes, ProtectionSetting protection,
 766                  MemCheckKind checkKind);
 767   void deallocate(void* addr, size_t bytes, bool decommit);
 768 };
 769
 770 void* ProcessExecutableMemory::allocate(size_t bytes,
 771                                         ProtectionSetting protection,
 772                                         MemCheckKind checkKind) {
 773   MOZ_ASSERT(initialized());
 774   MOZ_ASSERT(HasJitBackend());
 775   MOZ_ASSERT(bytes > 0);
 776   MOZ_ASSERT((bytes % ExecutableCodePageSize) == 0);
 777
 778   size_t numPages = bytes / ExecutableCodePageSize;
 779
 780   // Take the lock and try to allocate.
 781   void* p = nullptr;
 782   {
 783     LockGuard<Mutex> guard(lock_);
 784     MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
 785
 786     // Check if we have enough pages available.
 787     if (pagesAllocated_ + numPages >= MaxCodePages) {
 788       return nullptr;
 789     }
 790
 791     MOZ_ASSERT(bytes <= MaxCodeBytesPerProcess);
 792
 793     // Maybe skip a page to make allocations less predictable.
 794     size_t page = cursor_ + (rng_.ref().next() % 2);
 795
 796     for (size_t i = 0; i < MaxCodePages; i++) {
 797       // Make sure page + numPages - 1 is a valid index.
 798       if (page + numPages > MaxCodePages) {
 799         page = 0;
 800       }
 801
 802       bool available = true;
 803       for (size_t j = 0; j < numPages; j++) {
 804         if (pages_.contains(page + j)) {
 805           available = false;
 806           break;
 807         }
 808       }
 809       if (!available) {
 810         page++;
 811         continue;
 812       }
 813
 814       // Mark the pages as unavailable.
 815       for (size_t j = 0; j < numPages; j++) {
 816         pages_.insert(page + j);
 817       }
 818
 819       pagesAllocated_ += numPages;
 820       MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
 821
 822       // If we allocated a small number of pages, move cursor_ to the
 823       // next page. We don't do this for larger allocations to avoid
 824       // skipping a large number of small holes.
 825       if (numPages <= 2) {
 826         cursor_ = page + numPages;
 827       }
 828
 829       p = base_ + page * ExecutableCodePageSize;
 830       break;
 831     }
 832     if (!p) {
 833       return nullptr;
 834     }
 835   }
 836
 837   // Commit the pages after releasing the lock.
 838   if (!CommitPages(p, bytes, protection)) {
 839     deallocate(p, bytes, /* decommit = */ false);
 840     return nullptr;
 841   }
 842
 843   SetMemCheckKind(p, bytes, checkKind);
 844
 845   return p;
 846 }
 847
 848 void ProcessExecutableMemory::deallocate(void* addr, size_t bytes,
 849                                          bool decommit) {
 850   MOZ_ASSERT(initialized());
 851   MOZ_ASSERT(addr);
 852   MOZ_ASSERT((uintptr_t(addr) % gc::SystemPageSize()) == 0);
 853   MOZ_ASSERT(bytes > 0);
 854   MOZ_ASSERT((bytes % ExecutableCodePageSize) == 0);
 855
 856   assertValidAddress(addr, bytes);
 857
 858   size_t firstPage =
 859       (static_cast<uint8_t*>(addr) - base_) / ExecutableCodePageSize;
 860   size_t numPages = bytes / ExecutableCodePageSize;
 861
 862   // Decommit before taking the lock.
 863   MOZ_MAKE_MEM_NOACCESS(addr, bytes);
 864   if (decommit) {
 865     DecommitPages(addr, bytes);
 866   }
 867
 868   LockGuard<Mutex> guard(lock_);
 869   MOZ_ASSERT(numPages <= pagesAllocated_);
 870   pagesAllocated_ -= numPages;
 871
 872   for (size_t i = 0; i < numPages; i++) {
 873     pages_.remove(firstPage + i);
 874   }
 875
 876   // Move the cursor back so we can reuse pages instead of fragmenting the
 877   // whole region.
 878   if (firstPage < cursor_) {
 879     cursor_ = firstPage;
 880   }
 881 }
 882
 883 static ProcessExecutableMemory execMemory;
 884
 885 void* js::jit::AllocateExecutableMemory(size_t bytes,
 886                                         ProtectionSetting protection,
 887                                         MemCheckKind checkKind) {
 888   return execMemory.allocate(bytes, protection, checkKind);
 889 }
 890
 891 void js::jit::DeallocateExecutableMemory(void* addr, size_t bytes) {
 892   execMemory.deallocate(addr, bytes, /* decommit = */ true);
 893 }
 894
 895 bool js::jit::InitProcessExecutableMemory() { return execMemory.init(); }
 896
 897 void js::jit::ReleaseProcessExecutableMemory() { execMemory.release(); }
 898
 899 size_t js::jit::LikelyAvailableExecutableMemory() {
 900   // Round down available memory to the closest MB.
 901   return MaxCodeBytesPerProcess -
 902          AlignBytes(execMemory.bytesAllocated(), 0x100000U);
 903 }
 904
 905 bool js::jit::CanLikelyAllocateMoreExecutableMemory() {
 906   // Use a 8 MB buffer.
 907   static const size_t BufferSize = 8 * 1024 * 1024;
 908
 909   MOZ_ASSERT(execMemory.bytesAllocated() <= MaxCodeBytesPerProcess);
 910
 911   return execMemory.bytesAllocated() + BufferSize <= MaxCodeBytesPerProcess;
 912 }
 913
 914 bool js::jit::AddressIsInExecutableMemory(const void* p) {
 915   return execMemory.containsAddress(p);
 916 }
 917
 918 bool js::jit::ReprotectRegion(void* start, size_t size,
 919                               ProtectionSetting protection,
 920                               MustFlushICache flushICache) {
 921 #if defined(JS_CODEGEN_WASM32)
 922   return true;
 923 #endif
 924
 925   // Flush ICache when making code executable, before we modify |size|.
 926   if (flushICache == MustFlushICache::Yes) {
 927     MOZ_ASSERT(protection == ProtectionSetting::Executable);
 928     jit::FlushICache(start, size);
 929   }
 930
 931   // Calculate the start of the page containing this region,
 932   // and account for this extra memory within size.
 933   size_t pageSize = gc::SystemPageSize();
 934   intptr_t startPtr = reinterpret_cast<intptr_t>(start);
 935   intptr_t pageStartPtr = startPtr & ~(pageSize - 1);
 936   void* pageStart = reinterpret_cast<void*>(pageStartPtr);
 937   size += (startPtr - pageStartPtr);
 938
 939   // Round size up
 940   size += (pageSize - 1);
 941   size &= ~(pageSize - 1);
 942
 943   MOZ_ASSERT((uintptr_t(pageStart) % pageSize) == 0);
 944
 945   execMemory.assertValidAddress(pageStart, size);
 946
 947   // On weak memory systems, make sure new code is visible on all cores before
 948   // addresses of the code are made public.  Now is the latest moment in time
 949   // when we can do that, and we're assuming that every other thread that has
 950   // written into the memory that is being reprotected here has synchronized
 951   // with this thread in such a way that the memory writes have become visible
 952   // and we therefore only need to execute the fence once here.  See bug 1529933
 953   // for a longer discussion of why this is both necessary and sufficient.
 954   //
 955   // We use the C++ fence here -- and not AtomicOperations::fenceSeqCst() --
 956   // primarily because ReprotectRegion will be called while we construct our own
 957   // jitted atomics.  But the C++ fence is sufficient and correct, too.
 958 #ifdef __wasi__
 959   MOZ_CRASH("NYI FOR WASI.");
 960 #else
 961   std::atomic_thread_fence(std::memory_order_seq_cst);
 962
 963   if (!JitOptions.writeProtectCode) {
 964     return true;
 965   }
 966
 967 #  ifdef JS_USE_APPLE_FAST_WX
 968   MOZ_CRASH("writeProtectCode should always be false on Apple Silicon");
 969 #  endif
 970
 971 #  ifdef XP_WIN
 972   DWORD flags = ProtectionSettingToFlags(protection);
 973   // This is a essentially a VirtualProtect, but with lighter impact on
 974   // antivirus analysis. See bug 1823634.
 975   if (!VirtualAlloc(pageStart, size, MEM_COMMIT, flags)) {
 976     return false;
 977   }
 978 #  else
 979   unsigned flags = ProtectionSettingToFlags(protection);
 980   if (mprotect(pageStart, size, flags)) {
 981     return false;
 982   }
 983 #  endif
 984 #endif  // __wasi__
 985
 986   execMemory.assertValidAddress(pageStart, size);
 987   return true;
 988 }
 989
 990 #ifdef JS_USE_APPLE_FAST_WX
 991 void js::jit::AutoMarkJitCodeWritableForThread::markExecutable(
 992     bool executable) {
 993   if (__builtin_available(macOS 11.0, *)) {
 994     pthread_jit_write_protect_np(executable);
 995   } else {
 996     MOZ_CRASH("pthread_jit_write_protect_np must be available");
 997   }
 998 }
 999 #endif
1000
1001 #ifdef DEBUG
1002 static MOZ_THREAD_LOCAL(bool) sMarkingWritable;
1003
1004 void js::jit::AutoMarkJitCodeWritableForThread::checkConstructor() {
1005   if (!sMarkingWritable.initialized()) {
1006     sMarkingWritable.infallibleInit();
1007   }
1008   MOZ_ASSERT(!sMarkingWritable.get(),
1009              "AutoMarkJitCodeWritableForThread shouldn't be nested");
1010   sMarkingWritable.set(true);
1011 }
1012
1013 void js::jit::AutoMarkJitCodeWritableForThread::checkDestructor() {
1014   MOZ_ASSERT(sMarkingWritable.get());
1015   sMarkingWritable.set(false);
1016 }
1017 #endif