js/src/jit/JitRuntime.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef jit_JitRuntime_h
   8 #define jit_JitRuntime_h
   9
  10 #include "mozilla/Assertions.h"
  11 #include "mozilla/Atomics.h"
  12 #include "mozilla/EnumeratedArray.h"
  13 #include "mozilla/LinkedList.h"
  14
  15 #include <stddef.h>
  16 #include <stdint.h>
  17
  18 #include "jstypes.h"
  19
  20 #include "jit/ABIFunctions.h"
  21 #include "jit/BaselineICList.h"
  22 #include "jit/BaselineJIT.h"
  23 #include "jit/CalleeToken.h"
  24 #include "jit/InterpreterEntryTrampoline.h"
  25 #include "jit/IonCompileTask.h"
  26 #include "jit/IonTypes.h"
  27 #include "jit/JitCode.h"
  28 #include "jit/JitHints.h"
  29 #include "jit/shared/Assembler-shared.h"
  30 #include "jit/TrampolineNatives.h"
  31 #include "js/AllocPolicy.h"
  32 #include "js/ProfilingFrameIterator.h"
  33 #include "js/TypeDecls.h"
  34 #include "js/UniquePtr.h"
  35 #include "js/Vector.h"
  36 #include "threading/ProtectedData.h"
  37 #include "vm/GeckoProfiler.h"
  38 #include "vm/Runtime.h"
  39
  40 class JS_PUBLIC_API JSTracer;
  41
  42 namespace js {
  43
  44 class AutoLockHelperThreadState;
  45 class GCMarker;
  46
  47 namespace jit {
  48
  49 class FrameSizeClass;
  50 class Label;
  51 class MacroAssembler;
  52 struct VMFunctionData;
  53
  54 enum class VMFunctionId;
  55
  56 enum class BaselineICFallbackKind : uint8_t {
  57 #define DEF_ENUM_KIND(kind) kind,
  58   IC_BASELINE_FALLBACK_CODE_KIND_LIST(DEF_ENUM_KIND)
  59 #undef DEF_ENUM_KIND
  60       Count
  61 };
  62
  63 enum class BailoutReturnKind {
  64   GetProp,
  65   GetPropSuper,
  66   SetProp,
  67   GetElem,
  68   GetElemSuper,
  69   Call,
  70   New,
  71   Count
  72 };
  73
  74 // Class storing code and offsets for all Baseline IC fallback trampolines. This
  75 // is stored in JitRuntime and generated when creating the JitRuntime.
  76 class BaselineICFallbackCode {
  77   JitCode* code_ = nullptr;
  78   using OffsetArray =
  79       mozilla::EnumeratedArray<BaselineICFallbackKind, uint32_t,
  80                                size_t(BaselineICFallbackKind::Count)>;
  81   OffsetArray offsets_ = {};
  82
  83   // Keep track of offset into various baseline stubs' code at return
  84   // point from called script.
  85   using BailoutReturnArray =
  86       mozilla::EnumeratedArray<BailoutReturnKind, uint32_t,
  87                                size_t(BailoutReturnKind::Count)>;
  88   BailoutReturnArray bailoutReturnOffsets_ = {};
  89
  90  public:
  91   BaselineICFallbackCode() = default;
  92   BaselineICFallbackCode(const BaselineICFallbackCode&) = delete;
  93   void operator=(const BaselineICFallbackCode&) = delete;
  94
  95   void initOffset(BaselineICFallbackKind kind, uint32_t offset) {
  96     offsets_[kind] = offset;
  97   }
  98   void initCode(JitCode* code) { code_ = code; }
  99   void initBailoutReturnOffset(BailoutReturnKind kind, uint32_t offset) {
 100     bailoutReturnOffsets_[kind] = offset;
 101   }
 102   TrampolinePtr addr(BaselineICFallbackKind kind) const {
 103     return TrampolinePtr(code_->raw() + offsets_[kind]);
 104   }
 105   uint8_t* bailoutReturnAddr(BailoutReturnKind kind) const {
 106     return code_->raw() + bailoutReturnOffsets_[kind];
 107   }
 108 };
 109
 110 enum class ArgumentsRectifierKind { Normal, TrialInlining };
 111
 112 enum class DebugTrapHandlerKind { Interpreter, Compiler, Count };
 113
 114 enum class IonGenericCallKind { Call, Construct, Count };
 115
 116 using EnterJitCode = void (*)(void*, unsigned int, Value*, InterpreterFrame*,
 117                               CalleeToken, JSObject*, size_t, Value*);
 118
 119 class JitcodeGlobalTable;
 120 class PerfSpewerRangeRecorder;
 121
 122 class JitRuntime {
 123  private:
 124   MainThreadData<uint64_t> nextCompilationId_{0};
 125
 126   // Buffer for OSR from baseline to Ion. To avoid holding on to this for too
 127   // long it's also freed in EnterBaseline and EnterJit (after returning from
 128   // JIT code).
 129   MainThreadData<js::UniquePtr<uint8_t>> ionOsrTempData_{nullptr};
 130   MainThreadData<uint32_t> ionOsrTempDataSize_{0};
 131
 132   // List of Ion compile tasks that should be freed. Used to batch multiple
 133   // tasks into a single IonFreeTask.
 134   MainThreadData<IonFreeCompileTasks> ionFreeTaskBatch_;
 135
 136   // Shared exception-handler tail.
 137   WriteOnceData<uint32_t> exceptionTailOffset_{0};
 138
 139   // Shared profiler exit frame tail.
 140   WriteOnceData<uint32_t> profilerExitFrameTailOffset_{0};
 141
 142   // Trampoline for entering JIT code.
 143   WriteOnceData<uint32_t> enterJITOffset_{0};
 144
 145   // Generic bailout table; used if the bailout table overflows.
 146   WriteOnceData<uint32_t> bailoutHandlerOffset_{0};
 147
 148   // Argument-rectifying thunks, in the case of insufficient arguments passed
 149   // to a function call site. The return offset is used to rebuild stack frames
 150   // when bailing out.
 151   WriteOnceData<uint32_t> argumentsRectifierOffset_{0};
 152   WriteOnceData<uint32_t> trialInliningArgumentsRectifierOffset_{0};
 153   WriteOnceData<uint32_t> argumentsRectifierReturnOffset_{0};
 154
 155   // Thunk that invalides an (Ion compiled) caller on the Ion stack.
 156   WriteOnceData<uint32_t> invalidatorOffset_{0};
 157
 158   // Thunk that calls the GC pre barrier.
 159   WriteOnceData<uint32_t> valuePreBarrierOffset_{0};
 160   WriteOnceData<uint32_t> stringPreBarrierOffset_{0};
 161   WriteOnceData<uint32_t> objectPreBarrierOffset_{0};
 162   WriteOnceData<uint32_t> shapePreBarrierOffset_{0};
 163   WriteOnceData<uint32_t> wasmAnyRefPreBarrierOffset_{0};
 164
 165   // Thunk to call malloc/free.
 166   WriteOnceData<uint32_t> freeStubOffset_{0};
 167
 168   // Thunk called to finish compilation of an IonScript.
 169   WriteOnceData<uint32_t> lazyLinkStubOffset_{0};
 170
 171   // Thunk to enter the interpreter from JIT code.
 172   WriteOnceData<uint32_t> interpreterStubOffset_{0};
 173
 174   // Thunk to convert the value in R0 to int32 if it's a double.
 175   // Note: this stub treats -0 as +0 and may clobber R1.scratchReg().
 176   WriteOnceData<uint32_t> doubleToInt32ValueStubOffset_{0};
 177
 178   // Thunk to do a generic call from Ion.
 179   mozilla::EnumeratedArray<IonGenericCallKind, WriteOnceData<uint32_t>,
 180                            size_t(IonGenericCallKind::Count)>
 181       ionGenericCallStubOffset_;
 182
 183   // Thunk used by the debugger for breakpoint and step mode.
 184   mozilla::EnumeratedArray<DebugTrapHandlerKind, WriteOnceData<JitCode*>,
 185                            size_t(DebugTrapHandlerKind::Count)>
 186       debugTrapHandlers_;
 187
 188   // BaselineInterpreter state.
 189   BaselineInterpreter baselineInterpreter_;
 190
 191   // Code for trampolines and VMFunction wrappers.
 192   WriteOnceData<JitCode*> trampolineCode_{nullptr};
 193
 194   // Thunk that calls into the C++ interpreter from the interpreter
 195   // entry trampoline that is generated with --emit-interpreter-entry
 196   WriteOnceData<uint32_t> vmInterpreterEntryOffset_{0};
 197
 198   // Maps VMFunctionId to the offset of the wrapper code in trampolineCode_.
 199   using VMWrapperOffsets = Vector<uint32_t, 0, SystemAllocPolicy>;
 200   VMWrapperOffsets functionWrapperOffsets_;
 201
 202   MainThreadData<BaselineICFallbackCode> baselineICFallbackCode_;
 203
 204   // Global table of jitcode native address => bytecode address mappings.
 205   UnprotectedData<JitcodeGlobalTable*> jitcodeGlobalTable_{nullptr};
 206
 207   // Map that stores Jit Hints for each script.
 208   MainThreadData<JitHintsMap*> jitHintsMap_{nullptr};
 209
 210   // Map used to collect entry trampolines for the Interpreters which is used
 211   // for external profiling to identify which functions are being interpreted.
 212   MainThreadData<EntryTrampolineMap*> interpreterEntryMap_{nullptr};
 213
 214 #ifdef DEBUG
 215   // The number of possible bailing places encountered before forcefully bailing
 216   // in that place if the counter reaches zero. Note that zero also means
 217   // inactive.
 218   MainThreadData<uint32_t> ionBailAfterCounter_{0};
 219
 220   // Whether the bailAfter mechanism is enabled. Used to avoid generating the
 221   // Ion code instrumentation for ionBailAfterCounter_ if the testing function
 222   // isn't used.
 223   MainThreadData<bool> ionBailAfterEnabled_{false};
 224 #endif
 225
 226   // Number of Ion compilations which were finished off thread and are
 227   // waiting to be lazily linked. This is only set while holding the helper
 228   // thread state lock, but may be read from at other times.
 229   typedef mozilla::Atomic<size_t, mozilla::SequentiallyConsistent>
 230       NumFinishedOffThreadTasksType;
 231   NumFinishedOffThreadTasksType numFinishedOffThreadTasks_{0};
 232
 233   // List of Ion compilation waiting to get linked.
 234   using IonCompileTaskList = mozilla::LinkedList<js::jit::IonCompileTask>;
 235   MainThreadData<IonCompileTaskList> ionLazyLinkList_;
 236   MainThreadData<size_t> ionLazyLinkListSize_{0};
 237
 238   // Pointer to trampoline code for each TrampolineNative. The JSFunction has
 239   // a JitEntry pointer that points to an item in this array.
 240   using TrampolineNativeJitEntryArray =
 241       mozilla::EnumeratedArray<TrampolineNative, void*,
 242                                size_t(TrampolineNative::Count)>;
 243   TrampolineNativeJitEntryArray trampolineNativeJitEntries_{};
 244
 245 #ifdef DEBUG
 246   // Flag that can be set from JIT code to indicate it's invalid to call
 247   // arbitrary JS code in a particular region. This is checked in RunScript.
 248   MainThreadData<uint32_t> disallowArbitraryCode_{false};
 249 #endif
 250
 251   bool generateTrampolines(JSContext* cx);
 252   bool generateBaselineICFallbackCode(JSContext* cx);
 253
 254   void generateLazyLinkStub(MacroAssembler& masm);
 255   void generateInterpreterStub(MacroAssembler& masm);
 256   void generateDoubleToInt32ValueStub(MacroAssembler& masm);
 257   void generateProfilerExitFrameTailStub(MacroAssembler& masm,
 258                                          Label* profilerExitTail);
 259   void generateExceptionTailStub(MacroAssembler& masm, Label* profilerExitTail,
 260                                  Label* bailoutTail);
 261   void generateBailoutTailStub(MacroAssembler& masm, Label* bailoutTail);
 262   void generateEnterJIT(JSContext* cx, MacroAssembler& masm);
 263   void generateArgumentsRectifier(MacroAssembler& masm,
 264                                   ArgumentsRectifierKind kind);
 265   void generateBailoutHandler(MacroAssembler& masm, Label* bailoutTail);
 266   void generateInvalidator(MacroAssembler& masm, Label* bailoutTail);
 267   uint32_t generatePreBarrier(JSContext* cx, MacroAssembler& masm,
 268                               MIRType type);
 269   void generateFreeStub(MacroAssembler& masm);
 270   void generateIonGenericCallStub(MacroAssembler& masm,
 271                                   IonGenericCallKind kind);
 272
 273   // Helper functions for generateIonGenericCallStub
 274   void generateIonGenericCallBoundFunction(MacroAssembler& masm, Label* entry,
 275                                            Label* vmCall);
 276   void generateIonGenericCallNativeFunction(MacroAssembler& masm,
 277                                             bool isConstructing);
 278   void generateIonGenericCallFunCall(MacroAssembler& masm, Label* entry,
 279                                      Label* vmCall);
 280   void generateIonGenericCallArgumentsShift(MacroAssembler& masm, Register argc,
 281                                             Register curr, Register end,
 282                                             Register scratch, Label* done);
 283
 284   JitCode* generateDebugTrapHandler(JSContext* cx, DebugTrapHandlerKind kind);
 285
 286   bool generateVMWrapper(JSContext* cx, MacroAssembler& masm, VMFunctionId id,
 287                          const VMFunctionData& f, DynFn nativeFun,
 288                          uint32_t* wrapperOffset);
 289
 290   bool generateVMWrappers(JSContext* cx, MacroAssembler& masm,
 291                           PerfSpewerRangeRecorder& rangeRecorder);
 292
 293   uint32_t startTrampolineCode(MacroAssembler& masm);
 294
 295   TrampolinePtr trampolineCode(uint32_t offset) const {
 296     MOZ_ASSERT(offset > 0);
 297     MOZ_ASSERT(offset < trampolineCode_->instructionsSize());
 298     return TrampolinePtr(trampolineCode_->raw() + offset);
 299   }
 300
 301   void generateBaselineInterpreterEntryTrampoline(MacroAssembler& masm);
 302   void generateInterpreterEntryTrampoline(MacroAssembler& masm);
 303
 304   using TrampolineNativeJitEntryOffsets =
 305       mozilla::EnumeratedArray<TrampolineNative, uint32_t,
 306                                size_t(TrampolineNative::Count)>;
 307   void generateTrampolineNatives(MacroAssembler& masm,
 308                                  TrampolineNativeJitEntryOffsets& offsets,
 309                                  PerfSpewerRangeRecorder& rangeRecorder);
 310   uint32_t generateArraySortTrampoline(MacroAssembler& masm);
 311
 312   void bindLabelToOffset(Label* label, uint32_t offset) {
 313     MOZ_ASSERT(!trampolineCode_);
 314     label->bind(offset);
 315   }
 316
 317  public:
 318   JitCode* generateEntryTrampolineForScript(JSContext* cx, JSScript* script);
 319
 320   JitRuntime() = default;
 321   ~JitRuntime();
 322   [[nodiscard]] bool initialize(JSContext* cx);
 323
 324   static void TraceAtomZoneRoots(JSTracer* trc);
 325   [[nodiscard]] static bool MarkJitcodeGlobalTableIteratively(GCMarker* marker);
 326   static void TraceWeakJitcodeGlobalTable(JSRuntime* rt, JSTracer* trc);
 327
 328   const BaselineICFallbackCode& baselineICFallbackCode() const {
 329     return baselineICFallbackCode_.ref();
 330   }
 331
 332   IonCompilationId nextCompilationId() {
 333     return IonCompilationId(nextCompilationId_++);
 334   }
 335
 336   [[nodiscard]] bool addIonCompileToFreeTaskBatch(IonCompileTask* task) {
 337     return ionFreeTaskBatch_.ref().append(task);
 338   }
 339   void maybeStartIonFreeTask(bool force);
 340
 341 #ifdef DEBUG
 342   bool disallowArbitraryCode() const { return disallowArbitraryCode_; }
 343   void clearDisallowArbitraryCode() { disallowArbitraryCode_ = false; }
 344   const void* addressOfDisallowArbitraryCode() const {
 345     return &disallowArbitraryCode_.refNoCheck();
 346   }
 347 #endif
 348
 349   uint8_t* allocateIonOsrTempData(size_t size);
 350   void freeIonOsrTempData();
 351
 352   TrampolinePtr getVMWrapper(VMFunctionId funId) const {
 353     MOZ_ASSERT(trampolineCode_);
 354     return trampolineCode(functionWrapperOffsets_[size_t(funId)]);
 355   }
 356
 357   JitCode* debugTrapHandler(JSContext* cx, DebugTrapHandlerKind kind);
 358
 359   BaselineInterpreter& baselineInterpreter() { return baselineInterpreter_; }
 360
 361   TrampolinePtr getGenericBailoutHandler() const {
 362     return trampolineCode(bailoutHandlerOffset_);
 363   }
 364
 365   TrampolinePtr getExceptionTail() const {
 366     return trampolineCode(exceptionTailOffset_);
 367   }
 368
 369   TrampolinePtr getProfilerExitFrameTail() const {
 370     return trampolineCode(profilerExitFrameTailOffset_);
 371   }
 372
 373   TrampolinePtr getArgumentsRectifier(
 374       ArgumentsRectifierKind kind = ArgumentsRectifierKind::Normal) const {
 375     if (kind == ArgumentsRectifierKind::TrialInlining) {
 376       return trampolineCode(trialInliningArgumentsRectifierOffset_);
 377     }
 378     return trampolineCode(argumentsRectifierOffset_);
 379   }
 380
 381   uint32_t vmInterpreterEntryOffset() { return vmInterpreterEntryOffset_; }
 382
 383   TrampolinePtr getArgumentsRectifierReturnAddr() const {
 384     return trampolineCode(argumentsRectifierReturnOffset_);
 385   }
 386
 387   TrampolinePtr getInvalidationThunk() const {
 388     return trampolineCode(invalidatorOffset_);
 389   }
 390
 391   EnterJitCode enterJit() const {
 392     return JS_DATA_TO_FUNC_PTR(EnterJitCode,
 393                                trampolineCode(enterJITOffset_).value);
 394   }
 395
 396   // Return the registers from the native caller frame of the given JIT frame.
 397   // Nothing{} if frameStackAddress is NOT pointing at a native-to-JIT entry
 398   // frame, or if the information is not accessible/implemented on this
 399   // platform.
 400   static mozilla::Maybe<::JS::ProfilingFrameIterator::RegisterState>
 401   getCppEntryRegisters(JitFrameLayout* frameStackAddress);
 402
 403   TrampolinePtr preBarrier(MIRType type) const {
 404     switch (type) {
 405       case MIRType::Value:
 406         return trampolineCode(valuePreBarrierOffset_);
 407       case MIRType::String:
 408         return trampolineCode(stringPreBarrierOffset_);
 409       case MIRType::Object:
 410         return trampolineCode(objectPreBarrierOffset_);
 411       case MIRType::Shape:
 412         return trampolineCode(shapePreBarrierOffset_);
 413       case MIRType::WasmAnyRef:
 414         return trampolineCode(wasmAnyRefPreBarrierOffset_);
 415       default:
 416         MOZ_CRASH();
 417     }
 418   }
 419
 420   TrampolinePtr freeStub() const { return trampolineCode(freeStubOffset_); }
 421
 422   TrampolinePtr lazyLinkStub() const {
 423     return trampolineCode(lazyLinkStubOffset_);
 424   }
 425   TrampolinePtr interpreterStub() const {
 426     return trampolineCode(interpreterStubOffset_);
 427   }
 428
 429   TrampolinePtr getDoubleToInt32ValueStub() const {
 430     return trampolineCode(doubleToInt32ValueStubOffset_);
 431   }
 432
 433   TrampolinePtr getIonGenericCallStub(IonGenericCallKind kind) const {
 434     return trampolineCode(ionGenericCallStubOffset_[kind]);
 435   }
 436
 437   void** trampolineNativeJitEntry(TrampolineNative native) {
 438     void** jitEntry = &trampolineNativeJitEntries_[native];
 439     MOZ_ASSERT(*jitEntry >= trampolineCode_->raw());
 440     MOZ_ASSERT(*jitEntry <
 441                trampolineCode_->raw() + trampolineCode_->instructionsSize());
 442     return jitEntry;
 443   }
 444   TrampolineNative trampolineNativeForJitEntry(void** entry) {
 445     MOZ_RELEASE_ASSERT(entry >= trampolineNativeJitEntries_.begin());
 446     size_t index = entry - trampolineNativeJitEntries_.begin();
 447     MOZ_RELEASE_ASSERT(index < size_t(TrampolineNative::Count));
 448     return TrampolineNative(index);
 449   }
 450
 451   bool hasJitcodeGlobalTable() const { return jitcodeGlobalTable_ != nullptr; }
 452
 453   JitcodeGlobalTable* getJitcodeGlobalTable() {
 454     MOZ_ASSERT(hasJitcodeGlobalTable());
 455     return jitcodeGlobalTable_;
 456   }
 457
 458   bool hasJitHintsMap() const { return jitHintsMap_ != nullptr; }
 459
 460   JitHintsMap* getJitHintsMap() {
 461     MOZ_ASSERT(hasJitHintsMap());
 462     return jitHintsMap_;
 463   }
 464
 465   bool hasInterpreterEntryMap() const {
 466     return interpreterEntryMap_ != nullptr;
 467   }
 468
 469   EntryTrampolineMap* getInterpreterEntryMap() {
 470     MOZ_ASSERT(hasInterpreterEntryMap());
 471     return interpreterEntryMap_;
 472   }
 473
 474   bool isProfilerInstrumentationEnabled(JSRuntime* rt) {
 475     return rt->geckoProfiler().enabled();
 476   }
 477
 478   bool isOptimizationTrackingEnabled(JSRuntime* rt) {
 479     return isProfilerInstrumentationEnabled(rt);
 480   }
 481
 482 #ifdef DEBUG
 483   void* addressOfIonBailAfterCounter() { return &ionBailAfterCounter_; }
 484
 485   // Set after how many bailing places we should forcefully bail.
 486   // Zero disables this feature.
 487   void setIonBailAfterCounter(uint32_t after) { ionBailAfterCounter_ = after; }
 488   bool ionBailAfterEnabled() const { return ionBailAfterEnabled_; }
 489   void setIonBailAfterEnabled(bool enabled) { ionBailAfterEnabled_ = enabled; }
 490 #endif
 491
 492   size_t numFinishedOffThreadTasks() const {
 493     return numFinishedOffThreadTasks_;
 494   }
 495   NumFinishedOffThreadTasksType& numFinishedOffThreadTasksRef(
 496       const AutoLockHelperThreadState& locked) {
 497     return numFinishedOffThreadTasks_;
 498   }
 499
 500   IonCompileTaskList& ionLazyLinkList(JSRuntime* rt);
 501
 502   size_t ionLazyLinkListSize() const { return ionLazyLinkListSize_; }
 503
 504   void ionLazyLinkListRemove(JSRuntime* rt, js::jit::IonCompileTask* task);
 505   void ionLazyLinkListAdd(JSRuntime* rt, js::jit::IonCompileTask* task);
 506 };
 507
 508 }  // namespace jit
 509 }  // namespace js
 510
 511 #endif /* jit_JitRuntime_h */