js/src/jit/ProcessExecutableMemory.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "jit/ProcessExecutableMemory.h"
   8
   9 #include "mozilla/Array.h"
  10 #include "mozilla/Atomics.h"
  11 #include "mozilla/DebugOnly.h"
  12 #include "mozilla/Maybe.h"
  13 #include "mozilla/TaggedAnonymousMemory.h"
  14 #include "mozilla/XorShift128PlusRNG.h"
  15
  16 #include <errno.h>
  17
  18 #include "jsfriendapi.h"
  19 #include "jsmath.h"
  20
  21 #include "gc/Memory.h"
  22 #include "jit/FlushICache.h"  // js::jit::FlushICache
  23 #include "jit/JitOptions.h"
  24 #include "threading/LockGuard.h"
  25 #include "threading/Mutex.h"
  26 #include "util/Memory.h"
  27 #include "util/Poison.h"
  28 #include "util/WindowsWrapper.h"
  29 #include "vm/MutexIDs.h"
  30
  31 #ifdef XP_WIN
  32 #  include "mozilla/StackWalk_windows.h"
  33 #  include "mozilla/WindowsVersion.h"
  34 #elif defined(__wasi__)
  35 #  if defined(JS_CODEGEN_WASM32)
  36 #    include <cstdlib>
  37 #  else
  38 // Nothing.
  39 #  endif
  40 #else
  41 #  include <sys/mman.h>
  42 #  include <unistd.h>
  43 #endif
  44
  45 #ifdef MOZ_VALGRIND
  46 #  include <valgrind/valgrind.h>
  47 #endif
  48
  49 #if defined(XP_IOS)
  50 #  include <BrowserEngineCore/BEMemory.h>
  51 #endif
  52
  53 using namespace js;
  54 using namespace js::jit;
  55
  56 #ifdef XP_WIN
  57 #  if defined(HAVE_64BIT_BUILD)
  58 #    define NEED_JIT_UNWIND_HANDLING
  59 #  endif
  60
  61 static void* ComputeRandomAllocationAddress() {
  62   /*
  63    * Inspiration is V8's OS::Allocate in platform-win32.cc.
  64    *
  65    * VirtualAlloc takes 64K chunks out of the virtual address space, so we
  66    * keep 16b alignment.
  67    *
  68    * x86: V8 comments say that keeping addresses in the [64MiB, 1GiB) range
  69    * tries to avoid system default DLL mapping space. In the end, we get 13
  70    * bits of randomness in our selection.
  71    * x64: [2GiB, 4TiB), with 25 bits of randomness.
  72    */
  73 #  ifdef HAVE_64BIT_BUILD
  74   static const uintptr_t base = 0x0000000080000000;
  75   static const uintptr_t mask = 0x000003ffffff0000;
  76 #  elif defined(_M_IX86) || defined(__i386__)
  77   static const uintptr_t base = 0x04000000;
  78   static const uintptr_t mask = 0x3fff0000;
  79 #  else
  80 #    error "Unsupported architecture"
  81 #  endif
  82
  83   uint64_t rand = js::GenerateRandomSeed();
  84   return (void*)(base | (rand & mask));
  85 }
  86
  87 #  ifdef NEED_JIT_UNWIND_HANDLING
  88 static js::JitExceptionHandler sJitExceptionHandler;
  89 static bool sHasInstalledFunctionTable = false;
  90 #  endif
  91
  92 JS_PUBLIC_API void js::SetJitExceptionHandler(JitExceptionHandler handler) {
  93 #  ifdef NEED_JIT_UNWIND_HANDLING
  94   MOZ_ASSERT(!sJitExceptionHandler);
  95   sJitExceptionHandler = handler;
  96 #  else
  97   // Just do nothing if unwind handling is disabled.
  98 #  endif
  99 }
 100
 101 #  ifdef NEED_JIT_UNWIND_HANDLING
 102 #    if defined(_M_ARM64)
 103 // See the ".xdata records" section of
 104 // https://docs.microsoft.com/en-us/cpp/build/arm64-exception-handling
 105 // These records can have various fields present or absent depending on the
 106 // bits set in the header. Our struct will use one 32-bit slot for unwind codes,
 107 // and no slots for epilog scopes.
 108 struct UnwindData {
 109   uint32_t functionLength : 18;
 110   uint32_t version : 2;
 111   uint32_t hasExceptionHandler : 1;
 112   uint32_t packedEpilog : 1;
 113   uint32_t epilogCount : 5;
 114   uint32_t codeWords : 5;
 115   uint8_t unwindCodes[4];
 116   uint32_t exceptionHandler;
 117 };
 118
 119 static const unsigned ThunkLength = 20;
 120 #    else
 121 // From documentation for UNWIND_INFO on
 122 // https://learn.microsoft.com/en-us/cpp/build/exception-handling-x64
 123 struct UnwindInfo {
 124   uint8_t version : 3;
 125   uint8_t flags : 5;
 126   uint8_t sizeOfPrologue;
 127   uint8_t countOfUnwindCodes;
 128   uint8_t frameRegister : 4;
 129   uint8_t frameOffset : 4;
 130 };
 131 static const unsigned ThunkLength = 12;
 132 union UnwindCode {
 133   struct {
 134     uint8_t codeOffset;
 135     uint8_t unwindOp : 4;
 136     uint8_t opInfo : 4;
 137   };
 138   uint16_t frameOffset;
 139 };
 140
 141 static constexpr int kNumberOfUnwindCodes = 2;
 142 static constexpr int kPushRbpInstructionLength = 1;
 143 static constexpr int kMovRbpRspInstructionLength = 3;
 144 static constexpr int kRbpPrefixCodes = 2;
 145 static constexpr int kRbpPrefixLength =
 146     kPushRbpInstructionLength + kMovRbpRspInstructionLength;
 147
 148 struct UnwindData {
 149   UnwindInfo unwindInfo;
 150   UnwindCode unwindCodes[kNumberOfUnwindCodes];
 151   uint32_t exceptionHandler;
 152
 153   UnwindData() {
 154     static constexpr int kOpPushNonvol = 0;
 155     static constexpr int kOpSetFPReg = 3;
 156
 157     unwindInfo.version = 1;
 158     unwindInfo.flags = UNW_FLAG_EHANDLER;
 159     unwindInfo.sizeOfPrologue = kRbpPrefixLength;
 160     unwindInfo.countOfUnwindCodes = kRbpPrefixCodes;
 161     unwindInfo.frameRegister = 5;
 162     unwindInfo.frameOffset = 0;
 163
 164     // Offset here are specified to beginning of the -next- instruction.
 165     unwindCodes[0].codeOffset = kRbpPrefixLength;  // movq rbp, rsp
 166     unwindCodes[0].unwindOp = kOpSetFPReg;
 167     unwindCodes[0].opInfo = 0;
 168
 169     unwindCodes[1].codeOffset = kPushRbpInstructionLength;  // push rbp
 170     unwindCodes[1].unwindOp = kOpPushNonvol;
 171     unwindCodes[1].opInfo = 5;
 172   }
 173 };
 174 #    endif
 175
 176 struct ExceptionHandlerRecord {
 177   void* dynamicTable;
 178   UnwindData unwindData;
 179   uint8_t thunk[ThunkLength];
 180   RUNTIME_FUNCTION runtimeFunction;
 181 };
 182
 183 // This function must match the function pointer type PEXCEPTION_HANDLER
 184 // mentioned in:
 185 //   http://msdn.microsoft.com/en-us/library/ssa62fwe.aspx.
 186 // This type is rather elusive in documentation; Wine is the best I've found:
 187 //   http://source.winehq.org/source/include/winnt.h
 188 static DWORD ExceptionHandler(PEXCEPTION_RECORD exceptionRecord,
 189                               _EXCEPTION_REGISTRATION_RECORD*, PCONTEXT context,
 190                               _EXCEPTION_REGISTRATION_RECORD**) {
 191   if (sJitExceptionHandler) {
 192     return sJitExceptionHandler(exceptionRecord, context);
 193   }
 194
 195   return ExceptionContinueSearch;
 196 }
 197
 198 // Required for enabling Stackwalking on windows using external tools.
 199 extern "C" NTSYSAPI DWORD NTAPI RtlAddGrowableFunctionTable(
 200     PVOID* DynamicTable, PRUNTIME_FUNCTION FunctionTable, DWORD EntryCount,
 201     DWORD MaximumEntryCount, ULONG_PTR RangeBase, ULONG_PTR RangeEnd);
 202
 203 // For an explanation of the problem being solved here, see
 204 // SetJitExceptionFilter in jsfriendapi.h.
 205 static bool RegisterExecutableMemory(void* p, size_t bytes, size_t pageSize) {
 206   if (!VirtualAlloc(p, pageSize, MEM_COMMIT, PAGE_READWRITE)) {
 207     MOZ_CRASH();
 208   }
 209
 210   // A page was reserved inside this structure for the record. This is because
 211   // all entries in the record are describes as an offset from the start of the
 212   // memory region. We construct the record there.
 213   ExceptionHandlerRecord* r = new (p) ExceptionHandlerRecord();
 214   void* handler = JS_FUNC_TO_DATA_PTR(void*, ExceptionHandler);
 215
 216   // Because the .xdata format on ARM64 can only encode sizes up to 1M (much
 217   // too small for our JIT code regions), we register a function table callback
 218   // to provide RUNTIME_FUNCTIONs at runtime. Windows doesn't seem to care about
 219   // the size fields on RUNTIME_FUNCTIONs that are created in this way, so the
 220   // same RUNTIME_FUNCTION can work for any address in the region. We'll set up
 221   // a generic one now and the callback can just return a pointer to it.
 222
 223   // All these fields are specified to be offsets from the base of the
 224   // executable code (which is 'p'), even if they have 'Address' in their
 225   // names. In particular, exceptionHandler is a ULONG offset which is a
 226   // 32-bit integer. Since 'p' can be farther than INT32_MAX away from
 227   // sJitExceptionHandler, we must generate a little thunk inside the
 228   // record. The record is put on its own page so that we can take away write
 229   // access to protect against accidental clobbering.
 230
 231 #    if defined(_M_ARM64)
 232   if (!sJitExceptionHandler) {
 233     return false;
 234   }
 235
 236   r->runtimeFunction.BeginAddress = pageSize;
 237   r->runtimeFunction.UnwindData = offsetof(ExceptionHandlerRecord, unwindData);
 238   static_assert(offsetof(ExceptionHandlerRecord, unwindData) % 4 == 0,
 239                 "The ARM64 .pdata format requires that exception information "
 240                 "RVAs be 4-byte aligned.");
 241
 242   memset(&r->unwindData, 0, sizeof(r->unwindData));
 243   r->unwindData.hasExceptionHandler = true;
 244   r->unwindData.exceptionHandler = offsetof(ExceptionHandlerRecord, thunk);
 245
 246   // Use a fake unwind code to make the Windows unwinder do _something_. If the
 247   // PC and SP both stay unchanged, we'll fail the unwinder's sanity checks and
 248   // it won't call our exception handler.
 249   r->unwindData.codeWords = 1;  // one 32-bit word gives us up to 4 codes
 250   r->unwindData.unwindCodes[0] =
 251       0b00000001;  // alloc_s small stack of size 1*16
 252   r->unwindData.unwindCodes[1] = 0b11100100;  // end
 253
 254   uint32_t* thunk = (uint32_t*)r->thunk;
 255   uint16_t* addr = (uint16_t*)&handler;
 256
 257   // xip0/r16 should be safe to clobber: Windows just used it to call our thunk.
 258   const uint8_t reg = 16;
 259
 260   // Say `handler` is 0x4444333322221111, then:
 261   thunk[0] = 0xd2800000 | addr[0] << 5 | reg;  // mov  xip0, 1111
 262   thunk[1] = 0xf2a00000 | addr[1] << 5 | reg;  // movk xip0, 2222 lsl #0x10
 263   thunk[2] = 0xf2c00000 | addr[2] << 5 | reg;  // movk xip0, 3333 lsl #0x20
 264   thunk[3] = 0xf2e00000 | addr[3] << 5 | reg;  // movk xip0, 4444 lsl #0x30
 265   thunk[4] = 0xd61f0000 | reg << 5;            // br xip0
 266 #    else
 267   r->runtimeFunction.BeginAddress = pageSize;
 268   r->runtimeFunction.EndAddress = (DWORD)bytes;
 269   r->runtimeFunction.UnwindData = offsetof(ExceptionHandlerRecord, unwindData);
 270   r->unwindData.exceptionHandler = offsetof(ExceptionHandlerRecord, thunk);
 271
 272   // mov imm64, rax
 273   r->thunk[0] = 0x48;
 274   r->thunk[1] = 0xb8;
 275   memcpy(&r->thunk[2], &handler, 8);
 276
 277   // jmp rax
 278   r->thunk[10] = 0xff;
 279   r->thunk[11] = 0xe0;
 280 #    endif
 281
 282   // RtlAddGrowableFunctionTable will write into the region. We must therefore
 283   // only write-protect is after this has been called.
 284
 285   // XXX NB: The profiler believes this function is only called from the main
 286   // thread. If that ever becomes untrue, the profiler must be updated
 287   // immediately.
 288   {
 289     AutoSuppressStackWalking suppress;
 290     DWORD result = RtlAddGrowableFunctionTable(
 291         &r->dynamicTable, &r->runtimeFunction, 1, 1, (ULONG_PTR)p,
 292         (ULONG_PTR)p + bytes - pageSize);
 293     if (result != S_OK) {
 294       return false;
 295     }
 296   }
 297
 298   DWORD oldProtect;
 299   if (!VirtualProtect(p, pageSize, PAGE_EXECUTE_READ, &oldProtect)) {
 300     MOZ_CRASH();
 301   }
 302
 303   return true;
 304 }
 305
 306 static void UnregisterExecutableMemory(void* p, size_t bytes, size_t pageSize) {
 307   // There's no such thing as RtlUninstallFunctionTableCallback, so there's
 308   // nothing to do here.
 309 }
 310 #  endif
 311
 312 static void* ReserveProcessExecutableMemory(size_t bytes) {
 313 #  ifdef NEED_JIT_UNWIND_HANDLING
 314   size_t pageSize = gc::SystemPageSize();
 315   // Always reserve space for the unwind information.
 316   bytes += pageSize;
 317 #  endif
 318
 319   void* p = nullptr;
 320   for (size_t i = 0; i < 10; i++) {
 321     void* randomAddr = ComputeRandomAllocationAddress();
 322     p = VirtualAlloc(randomAddr, bytes, MEM_RESERVE, PAGE_NOACCESS);
 323     if (p) {
 324       break;
 325     }
 326   }
 327
 328   if (!p) {
 329     // Try again without randomization.
 330     p = VirtualAlloc(nullptr, bytes, MEM_RESERVE, PAGE_NOACCESS);
 331     if (!p) {
 332       return nullptr;
 333     }
 334   }
 335
 336 #  ifdef NEED_JIT_UNWIND_HANDLING
 337   if (RegisterExecutableMemory(p, bytes, pageSize)) {
 338     sHasInstalledFunctionTable = true;
 339   } else {
 340     if (sJitExceptionHandler) {
 341       // This should have succeeded if we have an exception handler. Bail.
 342       VirtualFree(p, 0, MEM_RELEASE);
 343       return nullptr;
 344     }
 345   }
 346
 347   // Skip the first page where we might have allocated an exception handler
 348   // record.
 349   p = (uint8_t*)p + pageSize;
 350   bytes -= pageSize;
 351
 352   RegisterJitCodeRegion((uint8_t*)p, bytes);
 353 #  endif
 354   return p;
 355 }
 356
 357 static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
 358 #  ifdef NEED_JIT_UNWIND_HANDLING
 359   UnregisterJitCodeRegion((uint8_t*)addr, bytes);
 360
 361   size_t pageSize = gc::SystemPageSize();
 362   addr = (uint8_t*)addr - pageSize;
 363
 364   if (sHasInstalledFunctionTable) {
 365     UnregisterExecutableMemory(addr, bytes, pageSize);
 366   }
 367 #  endif
 368
 369   VirtualFree(addr, 0, MEM_RELEASE);
 370 }
 371
 372 static DWORD ProtectionSettingToFlags(ProtectionSetting protection) {
 373   if (!JitOptions.writeProtectCode) {
 374     return PAGE_EXECUTE_READWRITE;
 375   }
 376   switch (protection) {
 377     case ProtectionSetting::Writable:
 378       return PAGE_READWRITE;
 379     case ProtectionSetting::Executable:
 380       return PAGE_EXECUTE_READ;
 381   }
 382   MOZ_CRASH();
 383 }
 384
 385 [[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
 386                                       ProtectionSetting protection) {
 387   void* p = VirtualAlloc(addr, bytes, MEM_COMMIT,
 388                          ProtectionSettingToFlags(protection));
 389   if (!p) {
 390     return false;
 391   }
 392   MOZ_RELEASE_ASSERT(p == addr);
 393   return true;
 394 }
 395
 396 static void DecommitPages(void* addr, size_t bytes) {
 397   if (!VirtualFree(addr, bytes, MEM_DECOMMIT)) {
 398     MOZ_CRASH("DecommitPages failed");
 399   }
 400 }
 401 #elif defined(__wasi__)
 402 #  if defined(JS_CODEGEN_WASM32)
 403 static void* ReserveProcessExecutableMemory(size_t bytes) {
 404   return malloc(bytes);
 405 }
 406
 407 static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
 408   free(addr);
 409 }
 410
 411 [[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
 412                                       ProtectionSetting protection) {
 413   return true;
 414 }
 415
 416 static void DecommitPages(void* addr, size_t bytes) {}
 417
 418 #  else
 419 static void* ReserveProcessExecutableMemory(size_t bytes) {
 420   MOZ_CRASH("NYI for WASI.");
 421   return nullptr;
 422 }
 423 static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
 424   MOZ_CRASH("NYI for WASI.");
 425 }
 426 [[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
 427                                       ProtectionSetting protection) {
 428   MOZ_CRASH("NYI for WASI.");
 429   return false;
 430 }
 431 static void DecommitPages(void* addr, size_t bytes) {
 432   MOZ_CRASH("NYI for WASI.");
 433 }
 434 #  endif
 435 #else  // !XP_WIN && !__wasi__
 436 #  ifndef MAP_NORESERVE
 437 #    define MAP_NORESERVE 0
 438 #  endif
 439
 440 static void* ComputeRandomAllocationAddress() {
 441 #  ifdef __OpenBSD__
 442   // OpenBSD already has random mmap and the idea that all x64 cpus
 443   // have 48-bit address space is not correct. Returning nullptr
 444   // allows OpenBSD do to the right thing.
 445   return nullptr;
 446 #  else
 447   uint64_t rand = js::GenerateRandomSeed();
 448
 449 #    ifdef HAVE_64BIT_BUILD
 450   // x64 CPUs have a 48-bit address space and on some platforms the OS will
 451   // give us access to 47 bits, so to be safe we right shift by 18 to leave
 452   // 46 bits.
 453   rand >>= 18;
 454 #    else
 455   // On 32-bit, right shift by 34 to leave 30 bits, range [0, 1GiB). Then add
 456   // 512MiB to get range [512MiB, 1.5GiB), or [0x20000000, 0x60000000). This
 457   // is based on V8 comments in platform-posix.cc saying this range is
 458   // relatively unpopulated across a variety of kernels.
 459   rand >>= 34;
 460   rand += 512 * 1024 * 1024;
 461 #    endif
 462
 463   // Ensure page alignment.
 464   uintptr_t mask = ~uintptr_t(gc::SystemPageSize() - 1);
 465   return (void*)uintptr_t(rand & mask);
 466 #  endif
 467 }
 468
 469 static void DecommitPages(void* addr, size_t bytes);
 470
 471 static void* ReserveProcessExecutableMemory(size_t bytes) {
 472   // On most Unix platforms our strategy is as follows:
 473   //
 474   // * Reserve:  mmap with PROT_NONE
 475   // * Commit:   mmap with MAP_FIXED, PROT_READ | ...
 476   // * Decommit: mmap with MAP_FIXED, PROT_NONE
 477   //
 478   // On Apple Silicon this only works if we use mprotect to implement W^X. To
 479   // use RWX pages with the faster pthread_jit_write_protect_np API for
 480   // thread-local writable/executable switching, the kernel enforces the
 481   // following rules:
 482   //
 483   // * The initial mmap must be called with MAP_JIT.
 484   // * MAP_FIXED can't be used with MAP_JIT.
 485   // * Since macOS 11.2, mprotect can't be used to change permissions of RWX JIT
 486   //   pages (even PROT_NONE fails).
 487   //   See https://developer.apple.com/forums/thread/672804.
 488   //
 489   // This means we have to use the following strategy on Apple Silicon:
 490   //
 491   // * Reserve:  1) mmap with PROT_READ | PROT_WRITE | PROT_EXEC and MAP_JIT
 492   //             2) decommit
 493   // * Commit:   madvise with MADV_FREE_REUSE
 494   // * Decommit: madvise with MADV_FREE_REUSABLE
 495   //
 496   // On Intel Macs we also need to use MAP_JIT, to be compatible with the
 497   // Hardened Runtime (with com.apple.security.cs.allow-jit = true). The
 498   // pthread_jit_write_protect_np API is not available on Intel and MAP_JIT
 499   // can't be used with MAP_FIXED, so we have to use a hybrid of the above two
 500   // strategies:
 501   //
 502   // * Reserve:  1) mmap with PROT_NONE and MAP_JIT
 503   //             2) decommit
 504   // * Commit:   1) madvise with MADV_FREE_REUSE
 505   //             2) mprotect with PROT_READ | ...
 506   // * Decommit: 1) mprotect with PROT_NONE
 507   //             2) madvise with MADV_FREE_REUSABLE
 508   //
 509   // This is inspired by V8's code in OS::SetPermissions.
 510
 511   // Note that randomAddr is just a hint: if the address is not available
 512   // mmap will pick a different address.
 513   void* randomAddr = ComputeRandomAllocationAddress();
 514   unsigned protection = PROT_NONE;
 515   unsigned flags = MAP_NORESERVE | MAP_PRIVATE | MAP_ANON;
 516 #  if defined(XP_DARWIN)
 517   flags |= MAP_JIT;
 518 #    if defined(JS_USE_APPLE_FAST_WX)
 519   protection = PROT_READ | PROT_WRITE | PROT_EXEC;
 520 #    endif
 521 #  endif
 522   void* p = MozTaggedAnonymousMmap(randomAddr, bytes, protection, flags, -1, 0,
 523                                    "js-executable-memory");
 524   if (p == MAP_FAILED) {
 525     return nullptr;
 526   }
 527 #  if defined(XP_DARWIN)
 528   DecommitPages(p, bytes);
 529 #  endif
 530   return p;
 531 }
 532
 533 static void DeallocateProcessExecutableMemory(void* addr, size_t bytes) {
 534   mozilla::DebugOnly<int> result = munmap(addr, bytes);
 535   MOZ_ASSERT(!result || errno == ENOMEM);
 536 }
 537
 538 static unsigned ProtectionSettingToFlags(ProtectionSetting protection) {
 539   if (!JitOptions.writeProtectCode) {
 540     return PROT_READ | PROT_WRITE | PROT_EXEC;
 541   }
 542 #  ifdef MOZ_VALGRIND
 543   // If we're configured for Valgrind and running on it, use a slacker
 544   // scheme that doesn't change execute permissions, since doing so causes
 545   // Valgrind a lot of extra overhead re-JITting code that loses and later
 546   // regains execute permission.  See bug 1338179.
 547   if (RUNNING_ON_VALGRIND) {
 548     switch (protection) {
 549       case ProtectionSetting::Writable:
 550         return PROT_READ | PROT_WRITE | PROT_EXEC;
 551       case ProtectionSetting::Executable:
 552         return PROT_READ | PROT_EXEC;
 553     }
 554     MOZ_CRASH();
 555   }
 556   // If we get here, we're configured for Valgrind but not running on
 557   // it, so use the standard scheme.
 558 #  endif
 559   switch (protection) {
 560     case ProtectionSetting::Writable:
 561       return PROT_READ | PROT_WRITE;
 562     case ProtectionSetting::Executable:
 563       return PROT_READ | PROT_EXEC;
 564   }
 565   MOZ_CRASH();
 566 }
 567
 568 [[nodiscard]] static bool CommitPages(void* addr, size_t bytes,
 569                                       ProtectionSetting protection) {
 570   // See the comment in ReserveProcessExecutableMemory.
 571 #  if defined(XP_DARWIN)
 572   int ret;
 573   do {
 574     ret = madvise(addr, bytes, MADV_FREE_REUSE);
 575   } while (ret != 0 && errno == EAGAIN);
 576   if (ret != 0) {
 577     return false;
 578   }
 579 #    if !defined(JS_USE_APPLE_FAST_WX)
 580   unsigned flags = ProtectionSettingToFlags(protection);
 581   if (mprotect(addr, bytes, flags)) {
 582     return false;
 583   }
 584 #    endif
 585   return true;
 586 #  else
 587   unsigned flags = ProtectionSettingToFlags(protection);
 588   void* p = MozTaggedAnonymousMmap(addr, bytes, flags,
 589                                    MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0,
 590                                    "js-executable-memory");
 591   if (p == MAP_FAILED) {
 592     return false;
 593   }
 594   MOZ_RELEASE_ASSERT(p == addr);
 595   return true;
 596 #  endif
 597 }
 598
 599 static void DecommitPages(void* addr, size_t bytes) {
 600   // See the comment in ReserveProcessExecutableMemory.
 601 #  if defined(XP_DARWIN)
 602   int ret;
 603 #    if !defined(JS_USE_APPLE_FAST_WX)
 604   ret = mprotect(addr, bytes, PROT_NONE);
 605   MOZ_RELEASE_ASSERT(ret == 0);
 606 #    endif
 607   do {
 608     ret = madvise(addr, bytes, MADV_FREE_REUSABLE);
 609   } while (ret != 0 && errno == EAGAIN);
 610   MOZ_RELEASE_ASSERT(ret == 0);
 611 #  else
 612   // Use mmap with MAP_FIXED and PROT_NONE. Inspired by jemalloc's
 613   // pages_decommit.
 614   void* p = MozTaggedAnonymousMmap(addr, bytes, PROT_NONE,
 615                                    MAP_FIXED | MAP_PRIVATE | MAP_ANON, -1, 0,
 616                                    "js-executable-memory");
 617   MOZ_RELEASE_ASSERT(addr == p);
 618 #  endif
 619 }
 620 #endif
 621
 622 template <size_t NumBits>
 623 class PageBitSet {
 624   using WordType = uint32_t;
 625   static const size_t BitsPerWord = sizeof(WordType) * 8;
 626
 627   static_assert((NumBits % BitsPerWord) == 0,
 628                 "NumBits must be a multiple of BitsPerWord");
 629   static const size_t NumWords = NumBits / BitsPerWord;
 630
 631   mozilla::Array<WordType, NumWords> words_;
 632
 633   uint32_t indexToWord(uint32_t index) const {
 634     MOZ_ASSERT(index < NumBits);
 635     return index / BitsPerWord;
 636   }
 637   WordType indexToBit(uint32_t index) const {
 638     MOZ_ASSERT(index < NumBits);
 639     return WordType(1) << (index % BitsPerWord);
 640   }
 641
 642  public:
 643   void init() { mozilla::PodArrayZero(words_); }
 644   bool contains(size_t index) const {
 645     uint32_t word = indexToWord(index);
 646     return words_[word] & indexToBit(index);
 647   }
 648   void insert(size_t index) {
 649     MOZ_ASSERT(!contains(index));
 650     uint32_t word = indexToWord(index);
 651     words_[word] |= indexToBit(index);
 652   }
 653   void remove(size_t index) {
 654     MOZ_ASSERT(contains(index));
 655     uint32_t word = indexToWord(index);
 656     words_[word] &= ~indexToBit(index);
 657   }
 658
 659 #ifdef DEBUG
 660   bool empty() const {
 661     for (size_t i = 0; i < NumWords; i++) {
 662       if (words_[i] != 0) {
 663         return false;
 664       }
 665     }
 666     return true;
 667   }
 668 #endif
 669 };
 670
 671 // Per-process executable memory allocator. It reserves a block of memory of
 672 // MaxCodeBytesPerProcess bytes, then allocates/deallocates pages from that.
 673 //
 674 // This has a number of benefits compared to raw mmap/VirtualAlloc:
 675 //
 676 // * More resillient against certain attacks.
 677 //
 678 // * Behaves more consistently across platforms: it avoids the 64K granularity
 679 //   issues on Windows, for instance.
 680 //
 681 // * On x64, near jumps can be used for jumps to other JIT pages.
 682 //
 683 // * On Win64, we have to register the exception handler only once (at process
 684 //   startup). This saves some memory and avoids RtlAddFunctionTable profiler
 685 //   deadlocks.
 686 class ProcessExecutableMemory {
 687   static_assert(
 688       (MaxCodeBytesPerProcess % ExecutableCodePageSize) == 0,
 689       "MaxCodeBytesPerProcess must be a multiple of ExecutableCodePageSize");
 690   static const size_t MaxCodePages =
 691       MaxCodeBytesPerProcess / ExecutableCodePageSize;
 692
 693   // Start of the MaxCodeBytesPerProcess memory block or nullptr if
 694   // uninitialized. Note that this is NOT guaranteed to be aligned to
 695   // ExecutableCodePageSize.
 696   uint8_t* base_;
 697
 698   // The fields below should only be accessed while we hold the lock.
 699   Mutex lock_ MOZ_UNANNOTATED;
 700
 701   // pagesAllocated_ is an Atomic so that bytesAllocated does not have to
 702   // take the lock.
 703   mozilla::Atomic<size_t, mozilla::ReleaseAcquire> pagesAllocated_;
 704
 705   // Page where we should try to allocate next.
 706   size_t cursor_;
 707
 708   mozilla::Maybe<mozilla::non_crypto::XorShift128PlusRNG> rng_;
 709   PageBitSet<MaxCodePages> pages_;
 710
 711  public:
 712   ProcessExecutableMemory()
 713       : base_(nullptr),
 714         lock_(mutexid::ProcessExecutableRegion),
 715         pagesAllocated_(0),
 716         cursor_(0),
 717         pages_() {}
 718
 719   [[nodiscard]] bool init() {
 720     pages_.init();
 721
 722     MOZ_RELEASE_ASSERT(!initialized());
 723     MOZ_RELEASE_ASSERT(HasJitBackend());
 724     MOZ_RELEASE_ASSERT(gc::SystemPageSize() <= ExecutableCodePageSize);
 725
 726     void* p = ReserveProcessExecutableMemory(MaxCodeBytesPerProcess);
 727     if (!p) {
 728       return false;
 729     }
 730
 731     base_ = static_cast<uint8_t*>(p);
 732
 733     mozilla::Array<uint64_t, 2> seed;
 734     GenerateXorShift128PlusSeed(seed);
 735     rng_.emplace(seed[0], seed[1]);
 736     return true;
 737   }
 738
 739   uint8_t* base() const { return base_; }
 740
 741   bool initialized() const { return base_ != nullptr; }
 742
 743   size_t bytesAllocated() const {
 744     MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
 745     return pagesAllocated_ * ExecutableCodePageSize;
 746   }
 747
 748   void release() {
 749     MOZ_ASSERT(initialized());
 750     MOZ_ASSERT(pages_.empty());
 751     MOZ_ASSERT(pagesAllocated_ == 0);
 752     DeallocateProcessExecutableMemory(base_, MaxCodeBytesPerProcess);
 753     base_ = nullptr;
 754     rng_.reset();
 755     MOZ_ASSERT(!initialized());
 756   }
 757
 758   void assertValidAddress(void* p, size_t bytes) const {
 759     MOZ_RELEASE_ASSERT(p >= base_ &&
 760                        uintptr_t(p) + bytes <=
 761                            uintptr_t(base_) + MaxCodeBytesPerProcess);
 762   }
 763
 764   bool containsAddress(const void* p) const {
 765     return p >= base_ &&
 766            uintptr_t(p) < uintptr_t(base_) + MaxCodeBytesPerProcess;
 767   }
 768
 769   void* allocate(size_t bytes, ProtectionSetting protection,
 770                  MemCheckKind checkKind);
 771   void deallocate(void* addr, size_t bytes, bool decommit);
 772 };
 773
 774 void* ProcessExecutableMemory::allocate(size_t bytes,
 775                                         ProtectionSetting protection,
 776                                         MemCheckKind checkKind) {
 777   MOZ_ASSERT(initialized());
 778   MOZ_ASSERT(HasJitBackend());
 779   MOZ_ASSERT(bytes > 0);
 780   MOZ_ASSERT((bytes % ExecutableCodePageSize) == 0);
 781
 782   size_t numPages = bytes / ExecutableCodePageSize;
 783
 784   // Take the lock and try to allocate.
 785   void* p = nullptr;
 786   {
 787     LockGuard<Mutex> guard(lock_);
 788     MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
 789
 790     // Check if we have enough pages available.
 791     if (pagesAllocated_ + numPages >= MaxCodePages) {
 792       return nullptr;
 793     }
 794
 795     MOZ_ASSERT(bytes <= MaxCodeBytesPerProcess);
 796
 797     // Maybe skip a page to make allocations less predictable.
 798     size_t page = cursor_ + (rng_.ref().next() % 2);
 799
 800     for (size_t i = 0; i < MaxCodePages; i++) {
 801       // Make sure page + numPages - 1 is a valid index.
 802       if (page + numPages > MaxCodePages) {
 803         page = 0;
 804       }
 805
 806       bool available = true;
 807       for (size_t j = 0; j < numPages; j++) {
 808         if (pages_.contains(page + j)) {
 809           available = false;
 810           break;
 811         }
 812       }
 813       if (!available) {
 814         page++;
 815         continue;
 816       }
 817
 818       // Mark the pages as unavailable.
 819       for (size_t j = 0; j < numPages; j++) {
 820         pages_.insert(page + j);
 821       }
 822
 823       pagesAllocated_ += numPages;
 824       MOZ_ASSERT(pagesAllocated_ <= MaxCodePages);
 825
 826       // If we allocated a small number of pages, move cursor_ to the
 827       // next page. We don't do this for larger allocations to avoid
 828       // skipping a large number of small holes.
 829       if (numPages <= 2) {
 830         cursor_ = page + numPages;
 831       }
 832
 833       p = base_ + page * ExecutableCodePageSize;
 834       break;
 835     }
 836     if (!p) {
 837       return nullptr;
 838     }
 839   }
 840
 841   // Commit the pages after releasing the lock.
 842   if (!CommitPages(p, bytes, protection)) {
 843     deallocate(p, bytes, /* decommit = */ false);
 844     return nullptr;
 845   }
 846
 847   SetMemCheckKind(p, bytes, checkKind);
 848
 849   return p;
 850 }
 851
 852 void ProcessExecutableMemory::deallocate(void* addr, size_t bytes,
 853                                          bool decommit) {
 854   MOZ_ASSERT(initialized());
 855   MOZ_ASSERT(addr);
 856   MOZ_ASSERT((uintptr_t(addr) % gc::SystemPageSize()) == 0);
 857   MOZ_ASSERT(bytes > 0);
 858   MOZ_ASSERT((bytes % ExecutableCodePageSize) == 0);
 859
 860   assertValidAddress(addr, bytes);
 861
 862   size_t firstPage =
 863       (static_cast<uint8_t*>(addr) - base_) / ExecutableCodePageSize;
 864   size_t numPages = bytes / ExecutableCodePageSize;
 865
 866   // Decommit before taking the lock.
 867   MOZ_MAKE_MEM_NOACCESS(addr, bytes);
 868   if (decommit) {
 869     DecommitPages(addr, bytes);
 870   }
 871
 872   LockGuard<Mutex> guard(lock_);
 873   MOZ_ASSERT(numPages <= pagesAllocated_);
 874   pagesAllocated_ -= numPages;
 875
 876   for (size_t i = 0; i < numPages; i++) {
 877     pages_.remove(firstPage + i);
 878   }
 879
 880   // Move the cursor back so we can reuse pages instead of fragmenting the
 881   // whole region.
 882   if (firstPage < cursor_) {
 883     cursor_ = firstPage;
 884   }
 885 }
 886
 887 static ProcessExecutableMemory execMemory;
 888
 889 void* js::jit::AllocateExecutableMemory(size_t bytes,
 890                                         ProtectionSetting protection,
 891                                         MemCheckKind checkKind) {
 892   return execMemory.allocate(bytes, protection, checkKind);
 893 }
 894
 895 void js::jit::DeallocateExecutableMemory(void* addr, size_t bytes) {
 896   execMemory.deallocate(addr, bytes, /* decommit = */ true);
 897 }
 898
 899 bool js::jit::InitProcessExecutableMemory() { return execMemory.init(); }
 900
 901 void js::jit::ReleaseProcessExecutableMemory() { execMemory.release(); }
 902
 903 size_t js::jit::LikelyAvailableExecutableMemory() {
 904   // Round down available memory to the closest MB.
 905   return MaxCodeBytesPerProcess -
 906          AlignBytes(execMemory.bytesAllocated(), 0x100000U);
 907 }
 908
 909 bool js::jit::CanLikelyAllocateMoreExecutableMemory() {
 910   // Use a 8 MB buffer.
 911   static const size_t BufferSize = 8 * 1024 * 1024;
 912
 913   MOZ_ASSERT(execMemory.bytesAllocated() <= MaxCodeBytesPerProcess);
 914
 915   return execMemory.bytesAllocated() + BufferSize <= MaxCodeBytesPerProcess;
 916 }
 917
 918 bool js::jit::AddressIsInExecutableMemory(const void* p) {
 919   return execMemory.containsAddress(p);
 920 }
 921
 922 bool js::jit::ReprotectRegion(void* start, size_t size,
 923                               ProtectionSetting protection,
 924                               MustFlushICache flushICache) {
 925 #if defined(JS_CODEGEN_WASM32)
 926   return true;
 927 #endif
 928
 929   // Flush ICache when making code executable, before we modify |size|.
 930   if (flushICache == MustFlushICache::Yes) {
 931     MOZ_ASSERT(protection == ProtectionSetting::Executable);
 932     jit::FlushICache(start, size);
 933   }
 934
 935   // Calculate the start of the page containing this region,
 936   // and account for this extra memory within size.
 937   size_t pageSize = gc::SystemPageSize();
 938   intptr_t startPtr = reinterpret_cast<intptr_t>(start);
 939   intptr_t pageStartPtr = startPtr & ~(pageSize - 1);
 940   void* pageStart = reinterpret_cast<void*>(pageStartPtr);
 941   size += (startPtr - pageStartPtr);
 942
 943   // Round size up
 944   size += (pageSize - 1);
 945   size &= ~(pageSize - 1);
 946
 947   MOZ_ASSERT((uintptr_t(pageStart) % pageSize) == 0);
 948
 949   execMemory.assertValidAddress(pageStart, size);
 950
 951   // On weak memory systems, make sure new code is visible on all cores before
 952   // addresses of the code are made public.  Now is the latest moment in time
 953   // when we can do that, and we're assuming that every other thread that has
 954   // written into the memory that is being reprotected here has synchronized
 955   // with this thread in such a way that the memory writes have become visible
 956   // and we therefore only need to execute the fence once here.  See bug 1529933
 957   // for a longer discussion of why this is both necessary and sufficient.
 958   //
 959   // We use the C++ fence here -- and not AtomicOperations::fenceSeqCst() --
 960   // primarily because ReprotectRegion will be called while we construct our own
 961   // jitted atomics.  But the C++ fence is sufficient and correct, too.
 962 #ifdef __wasi__
 963   MOZ_CRASH("NYI FOR WASI.");
 964 #else
 965   std::atomic_thread_fence(std::memory_order_seq_cst);
 966
 967   if (!JitOptions.writeProtectCode) {
 968     return true;
 969   }
 970
 971 #  ifdef JS_USE_APPLE_FAST_WX
 972   MOZ_CRASH("writeProtectCode should always be false on Apple Silicon");
 973 #  endif
 974
 975 #  ifdef XP_WIN
 976   DWORD flags = ProtectionSettingToFlags(protection);
 977   // This is a essentially a VirtualProtect, but with lighter impact on
 978   // antivirus analysis. See bug 1823634.
 979   if (!VirtualAlloc(pageStart, size, MEM_COMMIT, flags)) {
 980     return false;
 981   }
 982 #  else
 983   unsigned flags = ProtectionSettingToFlags(protection);
 984   if (mprotect(pageStart, size, flags)) {
 985     return false;
 986   }
 987 #  endif
 988 #endif  // __wasi__
 989
 990   execMemory.assertValidAddress(pageStart, size);
 991   return true;
 992 }
 993
 994 #ifdef JS_USE_APPLE_FAST_WX
 995 void js::jit::AutoMarkJitCodeWritableForThread::markExecutable(
 996     bool executable) {
 997 #  if defined(XP_IOS)
 998   if (executable) {
 999     be_memory_inline_jit_restrict_rwx_to_rx_with_witness();
1000   } else {
1001     be_memory_inline_jit_restrict_rwx_to_rw_with_witness();
1002   }
1003 #  else
1004   if (__builtin_available(macOS 11.0, *)) {
1005     pthread_jit_write_protect_np(executable);
1006   } else {
1007     MOZ_CRASH("pthread_jit_write_protect_np must be available");
1008   }
1009 #  endif
1010 }
1011 #endif
1012
1013 #ifdef DEBUG
1014 static MOZ_THREAD_LOCAL(bool) sMarkingWritable;
1015
1016 void js::jit::AutoMarkJitCodeWritableForThread::checkConstructor() {
1017   if (!sMarkingWritable.initialized()) {
1018     sMarkingWritable.infallibleInit();
1019   }
1020   MOZ_ASSERT(!sMarkingWritable.get(),
1021              "AutoMarkJitCodeWritableForThread shouldn't be nested");
1022   sMarkingWritable.set(true);
1023 }
1024
1025 void js::jit::AutoMarkJitCodeWritableForThread::checkDestructor() {
1026   MOZ_ASSERT(sMarkingWritable.get());
1027   sMarkingWritable.set(false);
1028 }
1029 #endif