js/src/jit/JitRuntime.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef jit_JitRuntime_h
   8 #define jit_JitRuntime_h
   9
  10 #include "mozilla/Assertions.h"
  11 #include "mozilla/Atomics.h"
  12 #include "mozilla/EnumeratedArray.h"
  13 #include "mozilla/LinkedList.h"
  14
  15 #include <stddef.h>
  16 #include <stdint.h>
  17
  18 #include "jstypes.h"
  19
  20 #include "jit/ABIFunctions.h"
  21 #include "jit/BaselineICList.h"
  22 #include "jit/BaselineJIT.h"
  23 #include "jit/CalleeToken.h"
  24 #include "jit/InterpreterEntryTrampoline.h"
  25 #include "jit/IonCompileTask.h"
  26 #include "jit/IonTypes.h"
  27 #include "jit/JitCode.h"
  28 #include "jit/JitHints.h"
  29 #include "jit/shared/Assembler-shared.h"
  30 #include "js/AllocPolicy.h"
  31 #include "js/ProfilingFrameIterator.h"
  32 #include "js/TypeDecls.h"
  33 #include "js/UniquePtr.h"
  34 #include "js/Vector.h"
  35 #include "threading/ProtectedData.h"
  36 #include "vm/GeckoProfiler.h"
  37 #include "vm/Runtime.h"
  38
  39 class JS_PUBLIC_API JSTracer;
  40
  41 namespace js {
  42
  43 class AutoLockHelperThreadState;
  44 class GCMarker;
  45
  46 namespace jit {
  47
  48 class FrameSizeClass;
  49 class Label;
  50 class MacroAssembler;
  51 struct VMFunctionData;
  52
  53 enum class VMFunctionId;
  54
  55 enum class BaselineICFallbackKind : uint8_t {
  56 #define DEF_ENUM_KIND(kind) kind,
  57   IC_BASELINE_FALLBACK_CODE_KIND_LIST(DEF_ENUM_KIND)
  58 #undef DEF_ENUM_KIND
  59       Count
  60 };
  61
  62 enum class BailoutReturnKind {
  63   GetProp,
  64   GetPropSuper,
  65   SetProp,
  66   GetElem,
  67   GetElemSuper,
  68   Call,
  69   New,
  70   Count
  71 };
  72
  73 // Class storing code and offsets for all Baseline IC fallback trampolines. This
  74 // is stored in JitRuntime and generated when creating the JitRuntime.
  75 class BaselineICFallbackCode {
  76   JitCode* code_ = nullptr;
  77   using OffsetArray =
  78       mozilla::EnumeratedArray<BaselineICFallbackKind,
  79                                BaselineICFallbackKind::Count, uint32_t>;
  80   OffsetArray offsets_ = {};
  81
  82   // Keep track of offset into various baseline stubs' code at return
  83   // point from called script.
  84   using BailoutReturnArray =
  85       mozilla::EnumeratedArray<BailoutReturnKind, BailoutReturnKind::Count,
  86                                uint32_t>;
  87   BailoutReturnArray bailoutReturnOffsets_ = {};
  88
  89  public:
  90   BaselineICFallbackCode() = default;
  91   BaselineICFallbackCode(const BaselineICFallbackCode&) = delete;
  92   void operator=(const BaselineICFallbackCode&) = delete;
  93
  94   void initOffset(BaselineICFallbackKind kind, uint32_t offset) {
  95     offsets_[kind] = offset;
  96   }
  97   void initCode(JitCode* code) { code_ = code; }
  98   void initBailoutReturnOffset(BailoutReturnKind kind, uint32_t offset) {
  99     bailoutReturnOffsets_[kind] = offset;
 100   }
 101   TrampolinePtr addr(BaselineICFallbackKind kind) const {
 102     return TrampolinePtr(code_->raw() + offsets_[kind]);
 103   }
 104   uint8_t* bailoutReturnAddr(BailoutReturnKind kind) const {
 105     return code_->raw() + bailoutReturnOffsets_[kind];
 106   }
 107 };
 108
 109 enum class ArgumentsRectifierKind { Normal, TrialInlining };
 110
 111 enum class DebugTrapHandlerKind { Interpreter, Compiler, Count };
 112
 113 enum class IonGenericCallKind { Call, Construct, Count };
 114
 115 using EnterJitCode = void (*)(void*, unsigned int, Value*, InterpreterFrame*,
 116                               CalleeToken, JSObject*, size_t, Value*);
 117
 118 class JitcodeGlobalTable;
 119 class PerfSpewerRangeRecorder;
 120
 121 class JitRuntime {
 122  private:
 123   MainThreadData<uint64_t> nextCompilationId_{0};
 124
 125   // Buffer for OSR from baseline to Ion. To avoid holding on to this for too
 126   // long it's also freed in EnterBaseline and EnterJit (after returning from
 127   // JIT code).
 128   MainThreadData<js::UniquePtr<uint8_t>> ionOsrTempData_{nullptr};
 129
 130   // Shared exception-handler tail.
 131   WriteOnceData<uint32_t> exceptionTailOffset_{0};
 132
 133   // Shared profiler exit frame tail.
 134   WriteOnceData<uint32_t> profilerExitFrameTailOffset_{0};
 135
 136   // Trampoline for entering JIT code.
 137   WriteOnceData<uint32_t> enterJITOffset_{0};
 138
 139   // Generic bailout table; used if the bailout table overflows.
 140   WriteOnceData<uint32_t> bailoutHandlerOffset_{0};
 141
 142   // Argument-rectifying thunks, in the case of insufficient arguments passed
 143   // to a function call site. The return offset is used to rebuild stack frames
 144   // when bailing out.
 145   WriteOnceData<uint32_t> argumentsRectifierOffset_{0};
 146   WriteOnceData<uint32_t> trialInliningArgumentsRectifierOffset_{0};
 147   WriteOnceData<uint32_t> argumentsRectifierReturnOffset_{0};
 148
 149   // Thunk that invalides an (Ion compiled) caller on the Ion stack.
 150   WriteOnceData<uint32_t> invalidatorOffset_{0};
 151
 152   // Thunk that calls the GC pre barrier.
 153   WriteOnceData<uint32_t> valuePreBarrierOffset_{0};
 154   WriteOnceData<uint32_t> stringPreBarrierOffset_{0};
 155   WriteOnceData<uint32_t> objectPreBarrierOffset_{0};
 156   WriteOnceData<uint32_t> shapePreBarrierOffset_{0};
 157   WriteOnceData<uint32_t> wasmAnyRefPreBarrierOffset_{0};
 158
 159   // Thunk to call malloc/free.
 160   WriteOnceData<uint32_t> freeStubOffset_{0};
 161
 162   // Thunk called to finish compilation of an IonScript.
 163   WriteOnceData<uint32_t> lazyLinkStubOffset_{0};
 164
 165   // Thunk to enter the interpreter from JIT code.
 166   WriteOnceData<uint32_t> interpreterStubOffset_{0};
 167
 168   // Thunk to convert the value in R0 to int32 if it's a double.
 169   // Note: this stub treats -0 as +0 and may clobber R1.scratchReg().
 170   WriteOnceData<uint32_t> doubleToInt32ValueStubOffset_{0};
 171
 172   // Thunk to do a generic call from Ion.
 173   mozilla::EnumeratedArray<IonGenericCallKind, IonGenericCallKind::Count,
 174                            WriteOnceData<uint32_t>>
 175       ionGenericCallStubOffset_;
 176
 177   // Thunk used by the debugger for breakpoint and step mode.
 178   mozilla::EnumeratedArray<DebugTrapHandlerKind, DebugTrapHandlerKind::Count,
 179                            WriteOnceData<JitCode*>>
 180       debugTrapHandlers_;
 181
 182   // BaselineInterpreter state.
 183   BaselineInterpreter baselineInterpreter_;
 184
 185   // Code for trampolines and VMFunction wrappers.
 186   WriteOnceData<JitCode*> trampolineCode_{nullptr};
 187
 188   // Thunk that calls into the C++ interpreter from the interpreter
 189   // entry trampoline that is generated with --emit-interpreter-entry
 190   WriteOnceData<uint32_t> vmInterpreterEntryOffset_{0};
 191
 192   // Maps VMFunctionId to the offset of the wrapper code in trampolineCode_.
 193   using VMWrapperOffsets = Vector<uint32_t, 0, SystemAllocPolicy>;
 194   VMWrapperOffsets functionWrapperOffsets_;
 195
 196   MainThreadData<BaselineICFallbackCode> baselineICFallbackCode_;
 197
 198   // Global table of jitcode native address => bytecode address mappings.
 199   UnprotectedData<JitcodeGlobalTable*> jitcodeGlobalTable_{nullptr};
 200
 201   // Map that stores Jit Hints for each script.
 202   MainThreadData<JitHintsMap*> jitHintsMap_{nullptr};
 203
 204   // Map used to collect entry trampolines for the Interpreters which is used
 205   // for external profiling to identify which functions are being interpreted.
 206   MainThreadData<EntryTrampolineMap*> interpreterEntryMap_{nullptr};
 207
 208 #ifdef DEBUG
 209   // The number of possible bailing places encountered before forcefully bailing
 210   // in that place if the counter reaches zero. Note that zero also means
 211   // inactive.
 212   MainThreadData<uint32_t> ionBailAfterCounter_{0};
 213
 214   // Whether the bailAfter mechanism is enabled. Used to avoid generating the
 215   // Ion code instrumentation for ionBailAfterCounter_ if the testing function
 216   // isn't used.
 217   MainThreadData<bool> ionBailAfterEnabled_{false};
 218 #endif
 219
 220   // Number of Ion compilations which were finished off thread and are
 221   // waiting to be lazily linked. This is only set while holding the helper
 222   // thread state lock, but may be read from at other times.
 223   typedef mozilla::Atomic<size_t, mozilla::SequentiallyConsistent>
 224       NumFinishedOffThreadTasksType;
 225   NumFinishedOffThreadTasksType numFinishedOffThreadTasks_{0};
 226
 227   // List of Ion compilation waiting to get linked.
 228   using IonCompileTaskList = mozilla::LinkedList<js::jit::IonCompileTask>;
 229   MainThreadData<IonCompileTaskList> ionLazyLinkList_;
 230   MainThreadData<size_t> ionLazyLinkListSize_{0};
 231
 232 #ifdef DEBUG
 233   // Flag that can be set from JIT code to indicate it's invalid to call
 234   // arbitrary JS code in a particular region. This is checked in RunScript.
 235   MainThreadData<uint32_t> disallowArbitraryCode_{false};
 236 #endif
 237
 238   bool generateTrampolines(JSContext* cx);
 239   bool generateBaselineICFallbackCode(JSContext* cx);
 240
 241   void generateLazyLinkStub(MacroAssembler& masm);
 242   void generateInterpreterStub(MacroAssembler& masm);
 243   void generateDoubleToInt32ValueStub(MacroAssembler& masm);
 244   void generateProfilerExitFrameTailStub(MacroAssembler& masm,
 245                                          Label* profilerExitTail);
 246   void generateExceptionTailStub(MacroAssembler& masm, Label* profilerExitTail,
 247                                  Label* bailoutTail);
 248   void generateBailoutTailStub(MacroAssembler& masm, Label* bailoutTail);
 249   void generateEnterJIT(JSContext* cx, MacroAssembler& masm);
 250   void generateArgumentsRectifier(MacroAssembler& masm,
 251                                   ArgumentsRectifierKind kind);
 252   void generateBailoutHandler(MacroAssembler& masm, Label* bailoutTail);
 253   void generateInvalidator(MacroAssembler& masm, Label* bailoutTail);
 254   uint32_t generatePreBarrier(JSContext* cx, MacroAssembler& masm,
 255                               MIRType type);
 256   void generateFreeStub(MacroAssembler& masm);
 257   void generateIonGenericCallStub(MacroAssembler& masm,
 258                                   IonGenericCallKind kind);
 259
 260   // Helper functions for generateIonGenericCallStub
 261   void generateIonGenericCallBoundFunction(MacroAssembler& masm, Label* entry,
 262                                            Label* vmCall);
 263   void generateIonGenericCallNativeFunction(MacroAssembler& masm,
 264                                             bool isConstructing);
 265   void generateIonGenericCallFunCall(MacroAssembler& masm, Label* entry,
 266                                      Label* vmCall);
 267   void generateIonGenericCallArgumentsShift(MacroAssembler& masm, Register argc,
 268                                             Register curr, Register end,
 269                                             Register scratch, Label* done);
 270
 271   JitCode* generateDebugTrapHandler(JSContext* cx, DebugTrapHandlerKind kind);
 272
 273   bool generateVMWrapper(JSContext* cx, MacroAssembler& masm, VMFunctionId id,
 274                          const VMFunctionData& f, DynFn nativeFun,
 275                          uint32_t* wrapperOffset);
 276
 277   bool generateVMWrappers(JSContext* cx, MacroAssembler& masm,
 278                           PerfSpewerRangeRecorder& rangeRecorder);
 279
 280   uint32_t startTrampolineCode(MacroAssembler& masm);
 281
 282   TrampolinePtr trampolineCode(uint32_t offset) const {
 283     MOZ_ASSERT(offset > 0);
 284     MOZ_ASSERT(offset < trampolineCode_->instructionsSize());
 285     return TrampolinePtr(trampolineCode_->raw() + offset);
 286   }
 287
 288   void generateBaselineInterpreterEntryTrampoline(MacroAssembler& masm);
 289   void generateInterpreterEntryTrampoline(MacroAssembler& masm);
 290
 291   void bindLabelToOffset(Label* label, uint32_t offset) {
 292     MOZ_ASSERT(!trampolineCode_);
 293     label->bind(offset);
 294   }
 295
 296  public:
 297   JitCode* generateEntryTrampolineForScript(JSContext* cx, JSScript* script);
 298
 299   JitRuntime() = default;
 300   ~JitRuntime();
 301   [[nodiscard]] bool initialize(JSContext* cx);
 302
 303   static void TraceAtomZoneRoots(JSTracer* trc);
 304   [[nodiscard]] static bool MarkJitcodeGlobalTableIteratively(GCMarker* marker);
 305   static void TraceWeakJitcodeGlobalTable(JSRuntime* rt, JSTracer* trc);
 306
 307   const BaselineICFallbackCode& baselineICFallbackCode() const {
 308     return baselineICFallbackCode_.ref();
 309   }
 310
 311   IonCompilationId nextCompilationId() {
 312     return IonCompilationId(nextCompilationId_++);
 313   }
 314
 315 #ifdef DEBUG
 316   bool disallowArbitraryCode() const { return disallowArbitraryCode_; }
 317   void clearDisallowArbitraryCode() { disallowArbitraryCode_ = false; }
 318   const void* addressOfDisallowArbitraryCode() const {
 319     return &disallowArbitraryCode_.refNoCheck();
 320   }
 321 #endif
 322
 323   uint8_t* allocateIonOsrTempData(size_t size);
 324   void freeIonOsrTempData();
 325
 326   TrampolinePtr getVMWrapper(VMFunctionId funId) const {
 327     MOZ_ASSERT(trampolineCode_);
 328     return trampolineCode(functionWrapperOffsets_[size_t(funId)]);
 329   }
 330
 331   JitCode* debugTrapHandler(JSContext* cx, DebugTrapHandlerKind kind);
 332
 333   BaselineInterpreter& baselineInterpreter() { return baselineInterpreter_; }
 334
 335   TrampolinePtr getGenericBailoutHandler() const {
 336     return trampolineCode(bailoutHandlerOffset_);
 337   }
 338
 339   TrampolinePtr getExceptionTail() const {
 340     return trampolineCode(exceptionTailOffset_);
 341   }
 342
 343   TrampolinePtr getProfilerExitFrameTail() const {
 344     return trampolineCode(profilerExitFrameTailOffset_);
 345   }
 346
 347   TrampolinePtr getArgumentsRectifier(
 348       ArgumentsRectifierKind kind = ArgumentsRectifierKind::Normal) const {
 349     if (kind == ArgumentsRectifierKind::TrialInlining) {
 350       return trampolineCode(trialInliningArgumentsRectifierOffset_);
 351     }
 352     return trampolineCode(argumentsRectifierOffset_);
 353   }
 354
 355   uint32_t vmInterpreterEntryOffset() { return vmInterpreterEntryOffset_; }
 356
 357   TrampolinePtr getArgumentsRectifierReturnAddr() const {
 358     return trampolineCode(argumentsRectifierReturnOffset_);
 359   }
 360
 361   TrampolinePtr getInvalidationThunk() const {
 362     return trampolineCode(invalidatorOffset_);
 363   }
 364
 365   EnterJitCode enterJit() const {
 366     return JS_DATA_TO_FUNC_PTR(EnterJitCode,
 367                                trampolineCode(enterJITOffset_).value);
 368   }
 369
 370   // Return the registers from the native caller frame of the given JIT frame.
 371   // Nothing{} if frameStackAddress is NOT pointing at a native-to-JIT entry
 372   // frame, or if the information is not accessible/implemented on this
 373   // platform.
 374   static mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
 375   getCppEntryRegisters(JitFrameLayout* frameStackAddress);
 376
 377   TrampolinePtr preBarrier(MIRType type) const {
 378     switch (type) {
 379       case MIRType::Value:
 380         return trampolineCode(valuePreBarrierOffset_);
 381       case MIRType::String:
 382         return trampolineCode(stringPreBarrierOffset_);
 383       case MIRType::Object:
 384         return trampolineCode(objectPreBarrierOffset_);
 385       case MIRType::Shape:
 386         return trampolineCode(shapePreBarrierOffset_);
 387       case MIRType::WasmAnyRef:
 388         return trampolineCode(wasmAnyRefPreBarrierOffset_);
 389       default:
 390         MOZ_CRASH();
 391     }
 392   }
 393
 394   TrampolinePtr freeStub() const { return trampolineCode(freeStubOffset_); }
 395
 396   TrampolinePtr lazyLinkStub() const {
 397     return trampolineCode(lazyLinkStubOffset_);
 398   }
 399   TrampolinePtr interpreterStub() const {
 400     return trampolineCode(interpreterStubOffset_);
 401   }
 402
 403   TrampolinePtr getDoubleToInt32ValueStub() const {
 404     return trampolineCode(doubleToInt32ValueStubOffset_);
 405   }
 406
 407   TrampolinePtr getIonGenericCallStub(IonGenericCallKind kind) const {
 408     return trampolineCode(ionGenericCallStubOffset_[kind]);
 409   }
 410
 411   bool hasJitcodeGlobalTable() const { return jitcodeGlobalTable_ != nullptr; }
 412
 413   JitcodeGlobalTable* getJitcodeGlobalTable() {
 414     MOZ_ASSERT(hasJitcodeGlobalTable());
 415     return jitcodeGlobalTable_;
 416   }
 417
 418   bool hasJitHintsMap() const { return jitHintsMap_ != nullptr; }
 419
 420   JitHintsMap* getJitHintsMap() {
 421     MOZ_ASSERT(hasJitHintsMap());
 422     return jitHintsMap_;
 423   }
 424
 425   bool hasInterpreterEntryMap() const {
 426     return interpreterEntryMap_ != nullptr;
 427   }
 428
 429   EntryTrampolineMap* getInterpreterEntryMap() {
 430     MOZ_ASSERT(hasInterpreterEntryMap());
 431     return interpreterEntryMap_;
 432   }
 433
 434   bool isProfilerInstrumentationEnabled(JSRuntime* rt) {
 435     return rt->geckoProfiler().enabled();
 436   }
 437
 438   bool isOptimizationTrackingEnabled(JSRuntime* rt) {
 439     return isProfilerInstrumentationEnabled(rt);
 440   }
 441
 442 #ifdef DEBUG
 443   void* addressOfIonBailAfterCounter() { return &ionBailAfterCounter_; }
 444
 445   // Set after how many bailing places we should forcefully bail.
 446   // Zero disables this feature.
 447   void setIonBailAfterCounter(uint32_t after) { ionBailAfterCounter_ = after; }
 448   bool ionBailAfterEnabled() const { return ionBailAfterEnabled_; }
 449   void setIonBailAfterEnabled(bool enabled) { ionBailAfterEnabled_ = enabled; }
 450 #endif
 451
 452   size_t numFinishedOffThreadTasks() const {
 453     return numFinishedOffThreadTasks_;
 454   }
 455   NumFinishedOffThreadTasksType& numFinishedOffThreadTasksRef(
 456       const AutoLockHelperThreadState& locked) {
 457     return numFinishedOffThreadTasks_;
 458   }
 459
 460   IonCompileTaskList& ionLazyLinkList(JSRuntime* rt);
 461
 462   size_t ionLazyLinkListSize() const { return ionLazyLinkListSize_; }
 463
 464   void ionLazyLinkListRemove(JSRuntime* rt, js::jit::IonCompileTask* task);
 465   void ionLazyLinkListAdd(JSRuntime* rt, js::jit::IonCompileTask* task);
 466 };
 467
 468 }  // namespace jit
 469 }  // namespace js
 470
 471 #endif /* jit_JitRuntime_h */