Bug 1867190 - Add prefs for PHC probablities r=glandium
[gecko.git] / js / src / jit / BaselineJIT.h
blobfddeffd3b50deb9279b27859835541618b59962f
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifndef jit_BaselineJIT_h
8 #define jit_BaselineJIT_h
10 #include "mozilla/Assertions.h"
11 #include "mozilla/Likely.h"
12 #include "mozilla/Maybe.h"
13 #include "mozilla/MemoryReporting.h"
14 #include "mozilla/Span.h"
16 #include <stddef.h>
17 #include <stdint.h>
19 #include "jsfriendapi.h"
21 #include "jit/IonTypes.h"
22 #include "jit/JitCode.h"
23 #include "jit/JitContext.h"
24 #include "jit/JitOptions.h"
25 #include "jit/shared/Assembler-shared.h"
26 #include "js/Principals.h"
27 #include "js/TypeDecls.h"
28 #include "js/Vector.h"
29 #include "threading/ProtectedData.h"
30 #include "util/TrailingArray.h"
31 #include "vm/JSScript.h"
33 namespace js {
35 class InterpreterFrame;
36 class RunState;
38 namespace jit {
40 class BaselineFrame;
41 class ExceptionBailoutInfo;
42 class IonCompileTask;
43 class JitActivation;
44 class JSJitFrameIter;
46 // Base class for entries mapping a pc offset to a native code offset.
47 class BasePCToNativeEntry {
48 uint32_t pcOffset_;
49 uint32_t nativeOffset_;
51 public:
52 BasePCToNativeEntry(uint32_t pcOffset, uint32_t nativeOffset)
53 : pcOffset_(pcOffset), nativeOffset_(nativeOffset) {}
54 uint32_t pcOffset() const { return pcOffset_; }
55 uint32_t nativeOffset() const { return nativeOffset_; }
58 // Class used during Baseline compilation to store the native code offset for
59 // resume offset ops.
60 class ResumeOffsetEntry : public BasePCToNativeEntry {
61 public:
62 using BasePCToNativeEntry::BasePCToNativeEntry;
65 using ResumeOffsetEntryVector =
66 Vector<ResumeOffsetEntry, 16, SystemAllocPolicy>;
68 // Largest script that the baseline compiler will attempt to compile.
69 #if defined(JS_CODEGEN_ARM)
70 // ARM branches can only reach 32MB, and the macroassembler doesn't mitigate
71 // that limitation. Use a stricter limit on the acceptable script size to
72 // avoid crashing when branches go out of range.
73 static constexpr uint32_t BaselineMaxScriptLength = 1000000u;
74 #else
75 static constexpr uint32_t BaselineMaxScriptLength = 0x0fffffffu;
76 #endif
78 // Limit the locals on a given script so that stack check on baseline frames
79 // doesn't overflow a uint32_t value.
80 // (BaselineMaxScriptSlots * sizeof(Value)) must fit within a uint32_t.
82 // This also applies to the Baseline Interpreter: it ensures we don't run out
83 // of stack space (and throw over-recursion exceptions) for scripts with a huge
84 // number of locals. The C++ interpreter avoids this by having heap-allocated
85 // stack frames.
86 static constexpr uint32_t BaselineMaxScriptSlots = 0xffffu;
88 // An entry in the BaselineScript return address table. These entries are used
89 // to determine the bytecode pc for a return address into Baseline code.
91 // There must be an entry for each location where we can end up calling into
92 // C++ (directly or via script/trampolines) and C++ can request the current
93 // bytecode pc (this includes anything that may throw an exception, GC, or walk
94 // the stack). We currently add entries for each:
96 // * callVM
97 // * IC
98 // * DebugTrap (trampoline call)
99 // * JSOp::Resume (because this is like a scripted call)
101 // Note: see also BaselineFrame::HAS_OVERRIDE_PC.
102 class RetAddrEntry {
103 // Offset from the start of the JIT code where call instruction is.
104 uint32_t returnOffset_;
106 // The offset of this bytecode op within the JSScript.
107 uint32_t pcOffset_ : 28;
109 public:
110 enum class Kind : uint32_t {
111 // An IC for a JOF_IC op.
114 // A callVM for an op.
115 CallVM,
117 // A callVM not for an op (e.g., in the prologue) that can't
118 // trigger debug mode.
119 NonOpCallVM,
121 // A callVM for the over-recursion check on function entry.
122 StackCheck,
124 // A callVM for an interrupt check.
125 InterruptCheck,
127 // DebugTrapHandler (for debugger breakpoints/stepping).
128 DebugTrap,
130 // A callVM for Debug{Prologue,AfterYield,Epilogue}.
131 DebugPrologue,
132 DebugAfterYield,
133 DebugEpilogue,
135 Invalid
138 private:
139 // What this entry is for.
140 uint32_t kind_ : 4;
142 public:
143 RetAddrEntry(uint32_t pcOffset, Kind kind, CodeOffset retOffset)
144 : returnOffset_(uint32_t(retOffset.offset())),
145 pcOffset_(pcOffset),
146 kind_(uint32_t(kind)) {
147 MOZ_ASSERT(returnOffset_ == retOffset.offset(),
148 "retOffset must fit in returnOffset_");
150 // The pc offset must fit in at least 28 bits, since we shave off 4 for
151 // the Kind enum.
152 MOZ_ASSERT(pcOffset_ == pcOffset);
153 static_assert(BaselineMaxScriptLength <= (1u << 28) - 1);
154 MOZ_ASSERT(pcOffset <= BaselineMaxScriptLength);
156 MOZ_ASSERT(kind < Kind::Invalid);
157 MOZ_ASSERT(this->kind() == kind, "kind must fit in kind_ bit field");
160 CodeOffset returnOffset() const { return CodeOffset(returnOffset_); }
162 uint32_t pcOffset() const { return pcOffset_; }
164 jsbytecode* pc(JSScript* script) const {
165 return script->offsetToPC(pcOffset_);
168 Kind kind() const {
169 MOZ_ASSERT(kind_ < uint32_t(Kind::Invalid));
170 return Kind(kind_);
174 // [SMDOC] BaselineScript
176 // This holds the metadata generated by the BaselineCompiler. The machine code
177 // associated with this is owned by a JitCode instance. This class instance is
178 // followed by several arrays:
180 // <BaselineScript itself>
181 // --
182 // uint8_t*[] resumeEntryList()
183 // RetAddrEntry[] retAddrEntries()
184 // OSREntry[] osrEntries()
185 // DebugTrapEntry[] debugTrapEntries()
187 // Note: The arrays are arranged in order of descending alignment requires so
188 // that padding is not required.
189 class alignas(uintptr_t) BaselineScript final : public TrailingArray {
190 private:
191 // Code pointer containing the actual method.
192 HeapPtr<JitCode*> method_ = nullptr;
194 // An ion compilation that is ready, but isn't linked yet.
195 MainThreadData<IonCompileTask*> pendingIonCompileTask_{nullptr};
197 // Baseline Interpreter can enter Baseline Compiler code at this address. This
198 // is right after the warm-up counter check in the prologue.
199 uint32_t warmUpCheckPrologueOffset_ = 0;
201 // The offsets for the toggledJump instructions for profiler instrumentation.
202 uint32_t profilerEnterToggleOffset_ = 0;
203 uint32_t profilerExitToggleOffset_ = 0;
205 private:
206 // Offset (in bytes) from `this` to the start of each trailing array. Each
207 // array ends where following one begins. There is no implicit padding (except
208 // possible at very end).
209 Offset resumeEntriesOffset_ = 0;
210 Offset retAddrEntriesOffset_ = 0;
211 Offset osrEntriesOffset_ = 0;
212 Offset debugTrapEntriesOffset_ = 0;
213 Offset allocBytes_ = 0;
215 // See `Flag` type below.
216 uint8_t flags_ = 0;
218 // End of fields.
220 public:
221 enum Flag {
222 // Flag set when compiled for use with Debugger. Handles various
223 // Debugger hooks and compiles toggled calls for traps.
224 HAS_DEBUG_INSTRUMENTATION = 1 << 0,
226 // Flag is set if this script has profiling instrumentation turned on.
227 PROFILER_INSTRUMENTATION_ON = 1 << 1,
230 // Native code offset for OSR from Baseline Interpreter into Baseline JIT at
231 // JSOp::LoopHead ops.
232 class OSREntry : public BasePCToNativeEntry {
233 public:
234 using BasePCToNativeEntry::BasePCToNativeEntry;
237 // Native code offset for a debug trap when the script is compiled with debug
238 // instrumentation.
239 class DebugTrapEntry : public BasePCToNativeEntry {
240 public:
241 using BasePCToNativeEntry::BasePCToNativeEntry;
244 private:
245 // Layout helpers
246 Offset resumeEntriesOffset() const { return resumeEntriesOffset_; }
247 Offset retAddrEntriesOffset() const { return retAddrEntriesOffset_; }
248 Offset osrEntriesOffset() const { return osrEntriesOffset_; }
249 Offset debugTrapEntriesOffset() const { return debugTrapEntriesOffset_; }
250 Offset endOffset() const { return allocBytes_; }
252 // Use BaselineScript::New to create new instances. It will properly
253 // allocate trailing objects.
254 BaselineScript(uint32_t warmUpCheckPrologueOffset,
255 uint32_t profilerEnterToggleOffset,
256 uint32_t profilerExitToggleOffset)
257 : warmUpCheckPrologueOffset_(warmUpCheckPrologueOffset),
258 profilerEnterToggleOffset_(profilerEnterToggleOffset),
259 profilerExitToggleOffset_(profilerExitToggleOffset) {}
261 template <typename T>
262 mozilla::Span<T> makeSpan(Offset start, Offset end) {
263 return mozilla::Span{offsetToPointer<T>(start), numElements<T>(start, end)};
266 // We store the native code address corresponding to each bytecode offset in
267 // the script's resumeOffsets list.
268 mozilla::Span<uint8_t*> resumeEntryList() {
269 return makeSpan<uint8_t*>(resumeEntriesOffset(), retAddrEntriesOffset());
272 // See each type for documentation of these arrays.
273 mozilla::Span<RetAddrEntry> retAddrEntries() {
274 return makeSpan<RetAddrEntry>(retAddrEntriesOffset(), osrEntriesOffset());
276 mozilla::Span<OSREntry> osrEntries() {
277 return makeSpan<OSREntry>(osrEntriesOffset(), debugTrapEntriesOffset());
279 mozilla::Span<DebugTrapEntry> debugTrapEntries() {
280 return makeSpan<DebugTrapEntry>(debugTrapEntriesOffset(), endOffset());
283 public:
284 static BaselineScript* New(JSContext* cx, uint32_t warmUpCheckPrologueOffset,
285 uint32_t profilerEnterToggleOffset,
286 uint32_t profilerExitToggleOffset,
287 size_t retAddrEntries, size_t osrEntries,
288 size_t debugTrapEntries, size_t resumeEntries);
290 static void Destroy(JS::GCContext* gcx, BaselineScript* script);
292 void trace(JSTracer* trc);
294 static inline size_t offsetOfMethod() {
295 return offsetof(BaselineScript, method_);
298 void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf,
299 size_t* data) const {
300 *data += mallocSizeOf(this);
303 void setHasDebugInstrumentation() { flags_ |= HAS_DEBUG_INSTRUMENTATION; }
304 bool hasDebugInstrumentation() const {
305 return flags_ & HAS_DEBUG_INSTRUMENTATION;
308 uint8_t* warmUpCheckPrologueAddr() const {
309 return method_->raw() + warmUpCheckPrologueOffset_;
312 JitCode* method() const { return method_; }
313 void setMethod(JitCode* code) {
314 MOZ_ASSERT(!method_);
315 method_ = code;
318 bool containsCodeAddress(uint8_t* addr) const {
319 return method()->raw() <= addr &&
320 addr <= method()->raw() + method()->instructionsSize();
323 uint8_t* returnAddressForEntry(const RetAddrEntry& ent);
325 const RetAddrEntry& retAddrEntryFromPCOffset(uint32_t pcOffset,
326 RetAddrEntry::Kind kind);
327 const RetAddrEntry& prologueRetAddrEntry(RetAddrEntry::Kind kind);
328 const RetAddrEntry& retAddrEntryFromReturnOffset(CodeOffset returnOffset);
329 const RetAddrEntry& retAddrEntryFromReturnAddress(const uint8_t* returnAddr);
331 uint8_t* nativeCodeForOSREntry(uint32_t pcOffset);
333 void copyRetAddrEntries(const RetAddrEntry* entries);
334 void copyOSREntries(const OSREntry* entries);
335 void copyDebugTrapEntries(const DebugTrapEntry* entries);
337 // Copy resumeOffsets list from |script| and convert the pcOffsets
338 // to native addresses in the Baseline code based on |entries|.
339 void computeResumeNativeOffsets(JSScript* script,
340 const ResumeOffsetEntryVector& entries);
342 // Return the bytecode offset for a given native code address. Be careful
343 // when using this method: it's an approximation and not guaranteed to be the
344 // correct pc.
345 jsbytecode* approximatePcForNativeAddress(JSScript* script,
346 uint8_t* nativeAddress);
348 // Toggle debug traps (used for breakpoints and step mode) in the script.
349 // If |pc| is nullptr, toggle traps for all ops in the script. Else, only
350 // toggle traps at |pc|.
351 void toggleDebugTraps(JSScript* script, jsbytecode* pc);
353 void toggleProfilerInstrumentation(bool enable);
354 bool isProfilerInstrumentationOn() const {
355 return flags_ & PROFILER_INSTRUMENTATION_ON;
358 static size_t offsetOfResumeEntriesOffset() {
359 static_assert(sizeof(Offset) == sizeof(uint32_t),
360 "JIT expect Offset to be uint32_t");
361 return offsetof(BaselineScript, resumeEntriesOffset_);
364 bool hasPendingIonCompileTask() const { return !!pendingIonCompileTask_; }
366 js::jit::IonCompileTask* pendingIonCompileTask() {
367 MOZ_ASSERT(hasPendingIonCompileTask());
368 return pendingIonCompileTask_;
370 void setPendingIonCompileTask(JSRuntime* rt, JSScript* script,
371 js::jit::IonCompileTask* task);
372 void removePendingIonCompileTask(JSRuntime* rt, JSScript* script);
374 size_t allocBytes() const { return allocBytes_; }
376 static_assert(
377 sizeof(BaselineScript) % sizeof(uintptr_t) == 0,
378 "The data attached to the script must be aligned for fast JIT access.");
380 enum class BaselineTier { Interpreter, Compiler };
382 template <BaselineTier Tier>
383 MethodStatus CanEnterBaselineMethod(JSContext* cx, RunState& state);
385 MethodStatus CanEnterBaselineInterpreterAtBranch(JSContext* cx,
386 InterpreterFrame* fp);
388 JitExecStatus EnterBaselineInterpreterAtBranch(JSContext* cx,
389 InterpreterFrame* fp,
390 jsbytecode* pc);
392 bool CanBaselineInterpretScript(JSScript* script);
394 // Called by the Baseline Interpreter to compile a script for the Baseline JIT.
395 // |res| is set to the native code address in the BaselineScript to jump to, or
396 // nullptr if we were unable to compile this script.
397 bool BaselineCompileFromBaselineInterpreter(JSContext* cx, BaselineFrame* frame,
398 uint8_t** res);
400 void FinishDiscardBaselineScript(JS::GCContext* gcx, JSScript* script);
402 void AddSizeOfBaselineData(JSScript* script, mozilla::MallocSizeOf mallocSizeOf,
403 size_t* data);
405 void ToggleBaselineProfiling(JSContext* cx, bool enable);
407 struct alignas(uintptr_t) BaselineBailoutInfo {
408 // Pointer into the current C stack, where overwriting will start.
409 uint8_t* incomingStack = nullptr;
411 // The top and bottom heapspace addresses of the reconstructed stack
412 // which will be copied to the bottom.
413 uint8_t* copyStackTop = nullptr;
414 uint8_t* copyStackBottom = nullptr;
416 // The value of the frame pointer register on resume.
417 void* resumeFramePtr = nullptr;
419 // The native code address to resume into.
420 void* resumeAddr = nullptr;
422 // The bytecode pc of try block and fault block.
423 jsbytecode* tryPC = nullptr;
424 jsbytecode* faultPC = nullptr;
426 // We use this to transfer exception information out from
427 // buildExpressionStack, since it would be too risky to throw from
428 // there.
429 jsid tempId = PropertyKey::Void();
431 // Number of baseline frames to push on the stack.
432 uint32_t numFrames = 0;
434 // The bailout kind.
435 mozilla::Maybe<BailoutKind> bailoutKind = {};
437 BaselineBailoutInfo() = default;
438 BaselineBailoutInfo(const BaselineBailoutInfo&) = default;
440 void operator=(const BaselineBailoutInfo&) = delete;
442 void trace(JSTracer* aTrc);
445 enum class BailoutReason {
446 Normal,
447 ExceptionHandler,
448 Invalidate,
451 [[nodiscard]] bool BailoutIonToBaseline(
452 JSContext* cx, JitActivation* activation, const JSJitFrameIter& iter,
453 BaselineBailoutInfo** bailoutInfo,
454 const ExceptionBailoutInfo* exceptionInfo, BailoutReason reason);
456 MethodStatus BaselineCompile(JSContext* cx, JSScript* script,
457 bool forceDebugInstrumentation = false);
459 // Class storing the generated Baseline Interpreter code for the runtime.
460 class BaselineInterpreter {
461 public:
462 struct CallVMOffsets {
463 uint32_t debugPrologueOffset = 0;
464 uint32_t debugEpilogueOffset = 0;
465 uint32_t debugAfterYieldOffset = 0;
467 struct ICReturnOffset {
468 uint32_t offset;
469 JSOp op;
470 ICReturnOffset(uint32_t offset, JSOp op) : offset(offset), op(op) {}
472 using ICReturnOffsetVector = Vector<ICReturnOffset, 0, SystemAllocPolicy>;
474 private:
475 // The interpreter code.
476 JitCode* code_ = nullptr;
478 // Offset of the code to start interpreting a bytecode op.
479 uint32_t interpretOpOffset_ = 0;
481 // Like interpretOpOffset_ but skips the debug trap for the current op.
482 uint32_t interpretOpNoDebugTrapOffset_ = 0;
484 // Early Ion bailouts will enter at this address. This is after frame
485 // construction and environment initialization.
486 uint32_t bailoutPrologueOffset_ = 0;
488 // The offsets for the toggledJump instructions for profiler instrumentation.
489 uint32_t profilerEnterToggleOffset_ = 0;
490 uint32_t profilerExitToggleOffset_ = 0;
492 // Offset of the jump (tail call) to the debug trap handler trampoline code.
493 // When the debugger is enabled, NOPs are patched to calls to this location.
494 uint32_t debugTrapHandlerOffset_ = 0;
496 // The offsets of toggled jumps for debugger instrumentation.
497 using CodeOffsetVector = Vector<uint32_t, 0, SystemAllocPolicy>;
498 CodeOffsetVector debugInstrumentationOffsets_;
500 // Offsets of toggled calls to the DebugTrapHandler trampoline (for
501 // breakpoints and stepping).
502 CodeOffsetVector debugTrapOffsets_;
504 // Offsets of toggled jumps for code coverage.
505 CodeOffsetVector codeCoverageOffsets_;
507 // Offsets of IC calls for IsIonInlinableOp ops, for Ion bailouts.
508 ICReturnOffsetVector icReturnOffsets_;
510 // Offsets of some callVMs for BaselineDebugModeOSR.
511 CallVMOffsets callVMOffsets_;
513 uint8_t* codeAtOffset(uint32_t offset) const {
514 MOZ_ASSERT(offset > 0);
515 MOZ_ASSERT(offset < code_->instructionsSize());
516 return codeRaw() + offset;
519 public:
520 BaselineInterpreter() = default;
522 BaselineInterpreter(const BaselineInterpreter&) = delete;
523 void operator=(const BaselineInterpreter&) = delete;
525 void init(JitCode* code, uint32_t interpretOpOffset,
526 uint32_t interpretOpNoDebugTrapOffset,
527 uint32_t bailoutPrologueOffset, uint32_t profilerEnterToggleOffset,
528 uint32_t profilerExitToggleOffset, uint32_t debugTrapHandlerOffset,
529 CodeOffsetVector&& debugInstrumentationOffsets,
530 CodeOffsetVector&& debugTrapOffsets,
531 CodeOffsetVector&& codeCoverageOffsets,
532 ICReturnOffsetVector&& icReturnOffsets,
533 const CallVMOffsets& callVMOffsets);
535 uint8_t* codeRaw() const { return code_->raw(); }
537 uint8_t* retAddrForDebugPrologueCallVM() const {
538 return codeAtOffset(callVMOffsets_.debugPrologueOffset);
540 uint8_t* retAddrForDebugEpilogueCallVM() const {
541 return codeAtOffset(callVMOffsets_.debugEpilogueOffset);
543 uint8_t* retAddrForDebugAfterYieldCallVM() const {
544 return codeAtOffset(callVMOffsets_.debugAfterYieldOffset);
546 uint8_t* bailoutPrologueEntryAddr() const {
547 return codeAtOffset(bailoutPrologueOffset_);
550 uint8_t* retAddrForIC(JSOp op) const;
552 TrampolinePtr interpretOpAddr() const {
553 return TrampolinePtr(codeAtOffset(interpretOpOffset_));
555 TrampolinePtr interpretOpNoDebugTrapAddr() const {
556 return TrampolinePtr(codeAtOffset(interpretOpNoDebugTrapOffset_));
559 void toggleProfilerInstrumentation(bool enable);
560 void toggleDebuggerInstrumentation(bool enable);
562 void toggleCodeCoverageInstrumentationUnchecked(bool enable);
563 void toggleCodeCoverageInstrumentation(bool enable);
566 [[nodiscard]] bool GenerateBaselineInterpreter(
567 JSContext* cx, BaselineInterpreter& interpreter);
569 inline bool IsBaselineJitEnabled(JSContext* cx) {
570 if (MOZ_UNLIKELY(!IsBaselineInterpreterEnabled())) {
571 return false;
573 if (MOZ_LIKELY(JitOptions.baselineJit)) {
574 return true;
576 if (JitOptions.jitForTrustedPrincipals) {
577 JS::Realm* realm = js::GetContextRealm(cx);
578 return realm && JS::GetRealmPrincipals(realm) &&
579 JS::GetRealmPrincipals(realm)->isSystemOrAddonPrincipal();
581 return false;
584 } // namespace jit
585 } // namespace js
587 namespace JS {
589 template <>
590 struct DeletePolicy<js::jit::BaselineScript> {
591 explicit DeletePolicy(JSRuntime* rt) : rt_(rt) {}
592 void operator()(const js::jit::BaselineScript* script);
594 private:
595 JSRuntime* rt_;
598 } // namespace JS
600 #endif /* jit_BaselineJIT_h */