1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifndef jit_JitRuntime_h
8 #define jit_JitRuntime_h
10 #include "mozilla/Assertions.h"
11 #include "mozilla/Atomics.h"
12 #include "mozilla/EnumeratedArray.h"
13 #include "mozilla/LinkedList.h"
20 #include "jit/ABIFunctions.h"
21 #include "jit/BaselineICList.h"
22 #include "jit/BaselineJIT.h"
23 #include "jit/CalleeToken.h"
24 #include "jit/InterpreterEntryTrampoline.h"
25 #include "jit/IonCompileTask.h"
26 #include "jit/IonTypes.h"
27 #include "jit/JitCode.h"
28 #include "jit/JitHints.h"
29 #include "jit/shared/Assembler-shared.h"
30 #include "js/AllocPolicy.h"
31 #include "js/ProfilingFrameIterator.h"
32 #include "js/TypeDecls.h"
33 #include "js/UniquePtr.h"
34 #include "js/Vector.h"
35 #include "threading/ProtectedData.h"
36 #include "vm/GeckoProfiler.h"
37 #include "vm/Runtime.h"
39 class JS_PUBLIC_API JSTracer
;
43 class AutoLockHelperThreadState
;
51 struct VMFunctionData
;
53 enum class VMFunctionId
;
55 enum class BaselineICFallbackKind
: uint8_t {
56 #define DEF_ENUM_KIND(kind) kind,
57 IC_BASELINE_FALLBACK_CODE_KIND_LIST(DEF_ENUM_KIND
)
62 enum class BailoutReturnKind
{
73 // Class storing code and offsets for all Baseline IC fallback trampolines. This
74 // is stored in JitRuntime and generated when creating the JitRuntime.
75 class BaselineICFallbackCode
{
76 JitCode
* code_
= nullptr;
78 mozilla::EnumeratedArray
<BaselineICFallbackKind
,
79 BaselineICFallbackKind::Count
, uint32_t>;
80 OffsetArray offsets_
= {};
82 // Keep track of offset into various baseline stubs' code at return
83 // point from called script.
84 using BailoutReturnArray
=
85 mozilla::EnumeratedArray
<BailoutReturnKind
, BailoutReturnKind::Count
,
87 BailoutReturnArray bailoutReturnOffsets_
= {};
90 BaselineICFallbackCode() = default;
91 BaselineICFallbackCode(const BaselineICFallbackCode
&) = delete;
92 void operator=(const BaselineICFallbackCode
&) = delete;
94 void initOffset(BaselineICFallbackKind kind
, uint32_t offset
) {
95 offsets_
[kind
] = offset
;
97 void initCode(JitCode
* code
) { code_
= code
; }
98 void initBailoutReturnOffset(BailoutReturnKind kind
, uint32_t offset
) {
99 bailoutReturnOffsets_
[kind
] = offset
;
101 TrampolinePtr
addr(BaselineICFallbackKind kind
) const {
102 return TrampolinePtr(code_
->raw() + offsets_
[kind
]);
104 uint8_t* bailoutReturnAddr(BailoutReturnKind kind
) const {
105 return code_
->raw() + bailoutReturnOffsets_
[kind
];
109 enum class ArgumentsRectifierKind
{ Normal
, TrialInlining
};
111 enum class DebugTrapHandlerKind
{ Interpreter
, Compiler
, Count
};
113 enum class IonGenericCallKind
{ Call
, Construct
, Count
};
115 using EnterJitCode
= void (*)(void*, unsigned int, Value
*, InterpreterFrame
*,
116 CalleeToken
, JSObject
*, size_t, Value
*);
118 class JitcodeGlobalTable
;
119 class PerfSpewerRangeRecorder
;
123 MainThreadData
<uint64_t> nextCompilationId_
{0};
125 // Buffer for OSR from baseline to Ion. To avoid holding on to this for too
126 // long it's also freed in EnterBaseline and EnterJit (after returning from
128 MainThreadData
<js::UniquePtr
<uint8_t>> ionOsrTempData_
{nullptr};
130 // Shared exception-handler tail.
131 WriteOnceData
<uint32_t> exceptionTailOffset_
{0};
133 // Shared profiler exit frame tail.
134 WriteOnceData
<uint32_t> profilerExitFrameTailOffset_
{0};
136 // Trampoline for entering JIT code.
137 WriteOnceData
<uint32_t> enterJITOffset_
{0};
139 // Generic bailout table; used if the bailout table overflows.
140 WriteOnceData
<uint32_t> bailoutHandlerOffset_
{0};
142 // Argument-rectifying thunks, in the case of insufficient arguments passed
143 // to a function call site. The return offset is used to rebuild stack frames
145 WriteOnceData
<uint32_t> argumentsRectifierOffset_
{0};
146 WriteOnceData
<uint32_t> trialInliningArgumentsRectifierOffset_
{0};
147 WriteOnceData
<uint32_t> argumentsRectifierReturnOffset_
{0};
149 // Thunk that invalides an (Ion compiled) caller on the Ion stack.
150 WriteOnceData
<uint32_t> invalidatorOffset_
{0};
152 // Thunk that calls the GC pre barrier.
153 WriteOnceData
<uint32_t> valuePreBarrierOffset_
{0};
154 WriteOnceData
<uint32_t> stringPreBarrierOffset_
{0};
155 WriteOnceData
<uint32_t> objectPreBarrierOffset_
{0};
156 WriteOnceData
<uint32_t> shapePreBarrierOffset_
{0};
157 WriteOnceData
<uint32_t> wasmAnyRefPreBarrierOffset_
{0};
159 // Thunk to call malloc/free.
160 WriteOnceData
<uint32_t> freeStubOffset_
{0};
162 // Thunk called to finish compilation of an IonScript.
163 WriteOnceData
<uint32_t> lazyLinkStubOffset_
{0};
165 // Thunk to enter the interpreter from JIT code.
166 WriteOnceData
<uint32_t> interpreterStubOffset_
{0};
168 // Thunk to convert the value in R0 to int32 if it's a double.
169 // Note: this stub treats -0 as +0 and may clobber R1.scratchReg().
170 WriteOnceData
<uint32_t> doubleToInt32ValueStubOffset_
{0};
172 // Thunk to do a generic call from Ion.
173 mozilla::EnumeratedArray
<IonGenericCallKind
, IonGenericCallKind::Count
,
174 WriteOnceData
<uint32_t>>
175 ionGenericCallStubOffset_
;
177 // Thunk used by the debugger for breakpoint and step mode.
178 mozilla::EnumeratedArray
<DebugTrapHandlerKind
, DebugTrapHandlerKind::Count
,
179 WriteOnceData
<JitCode
*>>
182 // BaselineInterpreter state.
183 BaselineInterpreter baselineInterpreter_
;
185 // Code for trampolines and VMFunction wrappers.
186 WriteOnceData
<JitCode
*> trampolineCode_
{nullptr};
188 // Thunk that calls into the C++ interpreter from the interpreter
189 // entry trampoline that is generated with --emit-interpreter-entry
190 WriteOnceData
<uint32_t> vmInterpreterEntryOffset_
{0};
192 // Maps VMFunctionId to the offset of the wrapper code in trampolineCode_.
193 using VMWrapperOffsets
= Vector
<uint32_t, 0, SystemAllocPolicy
>;
194 VMWrapperOffsets functionWrapperOffsets_
;
196 MainThreadData
<BaselineICFallbackCode
> baselineICFallbackCode_
;
198 // Global table of jitcode native address => bytecode address mappings.
199 UnprotectedData
<JitcodeGlobalTable
*> jitcodeGlobalTable_
{nullptr};
201 // Map that stores Jit Hints for each script.
202 MainThreadData
<JitHintsMap
*> jitHintsMap_
{nullptr};
204 // Map used to collect entry trampolines for the Interpreters which is used
205 // for external profiling to identify which functions are being interpreted.
206 MainThreadData
<EntryTrampolineMap
*> interpreterEntryMap_
{nullptr};
209 // The number of possible bailing places encountered before forcefully bailing
210 // in that place if the counter reaches zero. Note that zero also means
212 MainThreadData
<uint32_t> ionBailAfterCounter_
{0};
214 // Whether the bailAfter mechanism is enabled. Used to avoid generating the
215 // Ion code instrumentation for ionBailAfterCounter_ if the testing function
217 MainThreadData
<bool> ionBailAfterEnabled_
{false};
220 // Number of Ion compilations which were finished off thread and are
221 // waiting to be lazily linked. This is only set while holding the helper
222 // thread state lock, but may be read from at other times.
223 typedef mozilla::Atomic
<size_t, mozilla::SequentiallyConsistent
>
224 NumFinishedOffThreadTasksType
;
225 NumFinishedOffThreadTasksType numFinishedOffThreadTasks_
{0};
227 // List of Ion compilation waiting to get linked.
228 using IonCompileTaskList
= mozilla::LinkedList
<js::jit::IonCompileTask
>;
229 MainThreadData
<IonCompileTaskList
> ionLazyLinkList_
;
230 MainThreadData
<size_t> ionLazyLinkListSize_
{0};
233 // Flag that can be set from JIT code to indicate it's invalid to call
234 // arbitrary JS code in a particular region. This is checked in RunScript.
235 MainThreadData
<uint32_t> disallowArbitraryCode_
{false};
238 bool generateTrampolines(JSContext
* cx
);
239 bool generateBaselineICFallbackCode(JSContext
* cx
);
241 void generateLazyLinkStub(MacroAssembler
& masm
);
242 void generateInterpreterStub(MacroAssembler
& masm
);
243 void generateDoubleToInt32ValueStub(MacroAssembler
& masm
);
244 void generateProfilerExitFrameTailStub(MacroAssembler
& masm
,
245 Label
* profilerExitTail
);
246 void generateExceptionTailStub(MacroAssembler
& masm
, Label
* profilerExitTail
,
248 void generateBailoutTailStub(MacroAssembler
& masm
, Label
* bailoutTail
);
249 void generateEnterJIT(JSContext
* cx
, MacroAssembler
& masm
);
250 void generateArgumentsRectifier(MacroAssembler
& masm
,
251 ArgumentsRectifierKind kind
);
252 void generateBailoutHandler(MacroAssembler
& masm
, Label
* bailoutTail
);
253 void generateInvalidator(MacroAssembler
& masm
, Label
* bailoutTail
);
254 uint32_t generatePreBarrier(JSContext
* cx
, MacroAssembler
& masm
,
256 void generateFreeStub(MacroAssembler
& masm
);
257 void generateIonGenericCallStub(MacroAssembler
& masm
,
258 IonGenericCallKind kind
);
260 // Helper functions for generateIonGenericCallStub
261 void generateIonGenericCallBoundFunction(MacroAssembler
& masm
, Label
* entry
,
263 void generateIonGenericCallNativeFunction(MacroAssembler
& masm
,
264 bool isConstructing
);
265 void generateIonGenericCallFunCall(MacroAssembler
& masm
, Label
* entry
,
267 void generateIonGenericCallArgumentsShift(MacroAssembler
& masm
, Register argc
,
268 Register curr
, Register end
,
269 Register scratch
, Label
* done
);
271 JitCode
* generateDebugTrapHandler(JSContext
* cx
, DebugTrapHandlerKind kind
);
273 bool generateVMWrapper(JSContext
* cx
, MacroAssembler
& masm
, VMFunctionId id
,
274 const VMFunctionData
& f
, DynFn nativeFun
,
275 uint32_t* wrapperOffset
);
277 bool generateVMWrappers(JSContext
* cx
, MacroAssembler
& masm
,
278 PerfSpewerRangeRecorder
& rangeRecorder
);
280 uint32_t startTrampolineCode(MacroAssembler
& masm
);
282 TrampolinePtr
trampolineCode(uint32_t offset
) const {
283 MOZ_ASSERT(offset
> 0);
284 MOZ_ASSERT(offset
< trampolineCode_
->instructionsSize());
285 return TrampolinePtr(trampolineCode_
->raw() + offset
);
288 void generateBaselineInterpreterEntryTrampoline(MacroAssembler
& masm
);
289 void generateInterpreterEntryTrampoline(MacroAssembler
& masm
);
291 void bindLabelToOffset(Label
* label
, uint32_t offset
) {
292 MOZ_ASSERT(!trampolineCode_
);
297 JitCode
* generateEntryTrampolineForScript(JSContext
* cx
, JSScript
* script
);
299 JitRuntime() = default;
301 [[nodiscard
]] bool initialize(JSContext
* cx
);
303 static void TraceAtomZoneRoots(JSTracer
* trc
);
304 [[nodiscard
]] static bool MarkJitcodeGlobalTableIteratively(GCMarker
* marker
);
305 static void TraceWeakJitcodeGlobalTable(JSRuntime
* rt
, JSTracer
* trc
);
307 const BaselineICFallbackCode
& baselineICFallbackCode() const {
308 return baselineICFallbackCode_
.ref();
311 IonCompilationId
nextCompilationId() {
312 return IonCompilationId(nextCompilationId_
++);
316 bool disallowArbitraryCode() const { return disallowArbitraryCode_
; }
317 void clearDisallowArbitraryCode() { disallowArbitraryCode_
= false; }
318 const void* addressOfDisallowArbitraryCode() const {
319 return &disallowArbitraryCode_
.refNoCheck();
323 uint8_t* allocateIonOsrTempData(size_t size
);
324 void freeIonOsrTempData();
326 TrampolinePtr
getVMWrapper(VMFunctionId funId
) const {
327 MOZ_ASSERT(trampolineCode_
);
328 return trampolineCode(functionWrapperOffsets_
[size_t(funId
)]);
331 JitCode
* debugTrapHandler(JSContext
* cx
, DebugTrapHandlerKind kind
);
333 BaselineInterpreter
& baselineInterpreter() { return baselineInterpreter_
; }
335 TrampolinePtr
getGenericBailoutHandler() const {
336 return trampolineCode(bailoutHandlerOffset_
);
339 TrampolinePtr
getExceptionTail() const {
340 return trampolineCode(exceptionTailOffset_
);
343 TrampolinePtr
getProfilerExitFrameTail() const {
344 return trampolineCode(profilerExitFrameTailOffset_
);
347 TrampolinePtr
getArgumentsRectifier(
348 ArgumentsRectifierKind kind
= ArgumentsRectifierKind::Normal
) const {
349 if (kind
== ArgumentsRectifierKind::TrialInlining
) {
350 return trampolineCode(trialInliningArgumentsRectifierOffset_
);
352 return trampolineCode(argumentsRectifierOffset_
);
355 uint32_t vmInterpreterEntryOffset() { return vmInterpreterEntryOffset_
; }
357 TrampolinePtr
getArgumentsRectifierReturnAddr() const {
358 return trampolineCode(argumentsRectifierReturnOffset_
);
361 TrampolinePtr
getInvalidationThunk() const {
362 return trampolineCode(invalidatorOffset_
);
365 EnterJitCode
enterJit() const {
366 return JS_DATA_TO_FUNC_PTR(EnterJitCode
,
367 trampolineCode(enterJITOffset_
).value
);
370 // Return the registers from the native caller frame of the given JIT frame.
371 // Nothing{} if frameStackAddress is NOT pointing at a native-to-JIT entry
372 // frame, or if the information is not accessible/implemented on this
374 static mozilla::Maybe
<::JS::ProfilingFrameIterator::RegisterState
>
375 getCppEntryRegisters(JitFrameLayout
* frameStackAddress
);
377 TrampolinePtr
preBarrier(MIRType type
) const {
380 return trampolineCode(valuePreBarrierOffset_
);
381 case MIRType::String
:
382 return trampolineCode(stringPreBarrierOffset_
);
383 case MIRType::Object
:
384 return trampolineCode(objectPreBarrierOffset_
);
386 return trampolineCode(shapePreBarrierOffset_
);
387 case MIRType::WasmAnyRef
:
388 return trampolineCode(wasmAnyRefPreBarrierOffset_
);
394 TrampolinePtr
freeStub() const { return trampolineCode(freeStubOffset_
); }
396 TrampolinePtr
lazyLinkStub() const {
397 return trampolineCode(lazyLinkStubOffset_
);
399 TrampolinePtr
interpreterStub() const {
400 return trampolineCode(interpreterStubOffset_
);
403 TrampolinePtr
getDoubleToInt32ValueStub() const {
404 return trampolineCode(doubleToInt32ValueStubOffset_
);
407 TrampolinePtr
getIonGenericCallStub(IonGenericCallKind kind
) const {
408 return trampolineCode(ionGenericCallStubOffset_
[kind
]);
411 bool hasJitcodeGlobalTable() const { return jitcodeGlobalTable_
!= nullptr; }
413 JitcodeGlobalTable
* getJitcodeGlobalTable() {
414 MOZ_ASSERT(hasJitcodeGlobalTable());
415 return jitcodeGlobalTable_
;
418 bool hasJitHintsMap() const { return jitHintsMap_
!= nullptr; }
420 JitHintsMap
* getJitHintsMap() {
421 MOZ_ASSERT(hasJitHintsMap());
425 bool hasInterpreterEntryMap() const {
426 return interpreterEntryMap_
!= nullptr;
429 EntryTrampolineMap
* getInterpreterEntryMap() {
430 MOZ_ASSERT(hasInterpreterEntryMap());
431 return interpreterEntryMap_
;
434 bool isProfilerInstrumentationEnabled(JSRuntime
* rt
) {
435 return rt
->geckoProfiler().enabled();
438 bool isOptimizationTrackingEnabled(JSRuntime
* rt
) {
439 return isProfilerInstrumentationEnabled(rt
);
443 void* addressOfIonBailAfterCounter() { return &ionBailAfterCounter_
; }
445 // Set after how many bailing places we should forcefully bail.
446 // Zero disables this feature.
447 void setIonBailAfterCounter(uint32_t after
) { ionBailAfterCounter_
= after
; }
448 bool ionBailAfterEnabled() const { return ionBailAfterEnabled_
; }
449 void setIonBailAfterEnabled(bool enabled
) { ionBailAfterEnabled_
= enabled
; }
452 size_t numFinishedOffThreadTasks() const {
453 return numFinishedOffThreadTasks_
;
455 NumFinishedOffThreadTasksType
& numFinishedOffThreadTasksRef(
456 const AutoLockHelperThreadState
& locked
) {
457 return numFinishedOffThreadTasks_
;
460 IonCompileTaskList
& ionLazyLinkList(JSRuntime
* rt
);
462 size_t ionLazyLinkListSize() const { return ionLazyLinkListSize_
; }
464 void ionLazyLinkListRemove(JSRuntime
* rt
, js::jit::IonCompileTask
* task
);
465 void ionLazyLinkListAdd(JSRuntime
* rt
, js::jit::IonCompileTask
* task
);
471 #endif /* jit_JitRuntime_h */