Bug 1890750 - Part 1: Include NATIVE_JIT_ENTRY in FunctionFlags::HasJitEntryFlags...
[gecko.git] / js / src / jit / JitScript.h
blob5810e92263d507b063e2f3a521b18d8475414c00
1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifndef jit_JitScript_h
8 #define jit_JitScript_h
10 #include "mozilla/Assertions.h"
11 #include "mozilla/Atomics.h"
12 #include "mozilla/Attributes.h"
13 #include "mozilla/HashFunctions.h"
14 #include "mozilla/LinkedList.h"
15 #include "mozilla/Maybe.h"
16 #include "mozilla/MemoryReporting.h"
18 #include <stddef.h>
19 #include <stdint.h>
21 #include "jstypes.h"
22 #include "NamespaceImports.h"
24 #include "ds/LifoAlloc.h"
25 #include "gc/Barrier.h"
26 #include "jit/BaselineIC.h"
27 #include "js/TypeDecls.h"
28 #include "js/UniquePtr.h"
29 #include "js/Vector.h"
30 #include "util/TrailingArray.h"
31 #include "vm/EnvironmentObject.h"
33 class JS_PUBLIC_API JSScript;
34 class JS_PUBLIC_API JSTracer;
35 struct JS_PUBLIC_API JSContext;
37 namespace JS {
38 class Zone;
41 namespace js {
43 class SystemAllocPolicy;
45 namespace gc {
46 class AllocSite;
49 namespace jit {
51 class BaselineScript;
52 class ICStubSpace;
53 class InliningRoot;
54 class IonScript;
55 class JitScript;
56 class JitZone;
58 // Magic BaselineScript value indicating Baseline compilation has been disabled.
59 static constexpr uintptr_t BaselineDisabledScript = 0x1;
61 static BaselineScript* const BaselineDisabledScriptPtr =
62 reinterpret_cast<BaselineScript*>(BaselineDisabledScript);
64 // Magic IonScript values indicating Ion compilation has been disabled or the
65 // script is being Ion-compiled off-thread.
66 static constexpr uintptr_t IonDisabledScript = 0x1;
67 static constexpr uintptr_t IonCompilingScript = 0x2;
69 static IonScript* const IonDisabledScriptPtr =
70 reinterpret_cast<IonScript*>(IonDisabledScript);
71 static IonScript* const IonCompilingScriptPtr =
72 reinterpret_cast<IonScript*>(IonCompilingScript);
74 /* [SMDOC] ICScript Lifetimes
76 * An ICScript owns an array of ICEntries, each of which owns a linked
77 * list of ICStubs.
79 * A JitScript contains an embedded ICScript. If it has done any trial
80 * inlining, it also owns an InliningRoot. The InliningRoot owns all
81 * of the ICScripts that have been created for inlining into the
82 * corresponding JitScript. This ties the lifetime of the inlined
83 * ICScripts to the lifetime of the JitScript itself.
85 * We store pointers to ICScripts in two other places: on the stack in
86 * BaselineFrame, and in IC stubs for CallInlinedFunction.
88 * The ICScript pointer in a BaselineFrame either points to the
89 * ICScript embedded in the JitScript for that frame, or to an inlined
90 * ICScript owned by a caller. In each case, there must be a frame on
91 * the stack corresponding to the JitScript that owns the current
92 * ICScript, which will keep the ICScript alive.
94 * Each ICStub is owned by an ICScript and, indirectly, a
95 * JitScript. An ICStub that uses CallInlinedFunction contains an
96 * ICScript for use by the callee. The ICStub and the callee ICScript
97 * are always owned by the same JitScript, so the callee ICScript will
98 * not be freed while the ICStub is alive.
100 * The lifetime of an ICScript is independent of the lifetimes of the
101 * BaselineScript and IonScript/WarpScript to which it
102 * corresponds. They can be destroyed and recreated, and the ICScript
103 * will remain valid.
105 * When we discard JIT code, we mark ICScripts that are active on the stack as
106 * active and then purge all of the inactive ICScripts. We also purge ICStubs,
107 * including the CallInlinedFunction stub at the trial inining call site, and
108 * reset the ICStates to allow trial inlining again later.
110 * If there's a BaselineFrame for an inlined ICScript, we'll preserve both this
111 * ICScript and the IC chain for the call site in the caller's ICScript.
112 * See ICScript::purgeStubs and ICScript::purgeInactiveICScripts.
115 class alignas(uintptr_t) ICScript final : public TrailingArray<ICScript> {
116 public:
117 ICScript(uint32_t warmUpCount, Offset fallbackStubsOffset, Offset endOffset,
118 uint32_t depth, uint32_t bytecodeSize,
119 InliningRoot* inliningRoot = nullptr)
120 : inliningRoot_(inliningRoot),
121 warmUpCount_(warmUpCount),
122 fallbackStubsOffset_(fallbackStubsOffset),
123 endOffset_(endOffset),
124 depth_(depth),
125 bytecodeSize_(bytecodeSize) {}
127 ~ICScript();
129 bool isInlined() const { return depth_ > 0; }
131 void initICEntries(JSContext* cx, JSScript* script);
133 ICEntry& icEntry(size_t index) {
134 MOZ_ASSERT(index < numICEntries());
135 return icEntries()[index];
138 ICFallbackStub* fallbackStub(size_t index) {
139 MOZ_ASSERT(index < numICEntries());
140 return fallbackStubs() + index;
143 ICEntry* icEntryForStub(const ICFallbackStub* stub) {
144 size_t index = stub - fallbackStubs();
145 MOZ_ASSERT(index < numICEntries());
146 return &icEntry(index);
148 ICFallbackStub* fallbackStubForICEntry(const ICEntry* entry) {
149 size_t index = entry - icEntries();
150 MOZ_ASSERT(index < numICEntries());
151 return fallbackStub(index);
154 InliningRoot* inliningRoot() const { return inliningRoot_; }
155 uint32_t depth() const { return depth_; }
157 uint32_t bytecodeSize() const { return bytecodeSize_; }
159 void resetWarmUpCount(uint32_t count) { warmUpCount_ = count; }
161 static constexpr size_t offsetOfFirstStub(uint32_t entryIndex) {
162 return sizeof(ICScript) + entryIndex * sizeof(ICEntry) +
163 ICEntry::offsetOfFirstStub();
166 static constexpr Offset offsetOfWarmUpCount() {
167 return offsetof(ICScript, warmUpCount_);
169 static constexpr Offset offsetOfDepth() { return offsetof(ICScript, depth_); }
171 static constexpr Offset offsetOfICEntries() { return sizeof(ICScript); }
172 uint32_t numICEntries() const {
173 return numElements<ICEntry>(icEntriesOffset(), fallbackStubsOffset());
176 ICEntry* interpreterICEntryFromPCOffset(uint32_t pcOffset);
178 ICEntry& icEntryFromPCOffset(uint32_t pcOffset);
180 [[nodiscard]] bool addInlinedChild(JSContext* cx,
181 js::UniquePtr<ICScript> child,
182 uint32_t pcOffset);
183 ICScript* findInlinedChild(uint32_t pcOffset);
184 void removeInlinedChild(uint32_t pcOffset);
185 bool hasInlinedChild(uint32_t pcOffset);
187 void purgeStubs(Zone* zone, ICStubSpace& newStubSpace);
189 void purgeInactiveICScripts();
191 bool active() const { return active_; }
192 void setActive() { active_ = true; }
193 void resetActive() { active_ = false; }
195 gc::AllocSite* getOrCreateAllocSite(JSScript* outerScript, uint32_t pcOffset);
197 void prepareForDestruction(Zone* zone);
199 void trace(JSTracer* trc);
200 bool traceWeak(JSTracer* trc);
202 #ifdef DEBUG
203 mozilla::HashNumber hash();
204 #endif
206 private:
207 class CallSite {
208 public:
209 CallSite(ICScript* callee, uint32_t pcOffset)
210 : callee_(callee), pcOffset_(pcOffset) {}
211 ICScript* callee_;
212 uint32_t pcOffset_;
215 // If this ICScript was created for trial inlining or has another
216 // ICScript inlined into it, a pointer to the root of the inlining
217 // tree. Otherwise, nullptr.
218 InliningRoot* inliningRoot_ = nullptr;
220 // ICScripts that have been inlined into this ICScript.
221 js::UniquePtr<Vector<CallSite>> inlinedChildren_;
223 // List of allocation sites referred to by ICs in this script.
224 static constexpr size_t AllocSiteChunkSize = 256;
225 LifoAlloc allocSitesSpace_{AllocSiteChunkSize};
226 Vector<gc::AllocSite*, 0, SystemAllocPolicy> allocSites_;
228 // Number of times this copy of the script has been called or has had
229 // backedges taken. Reset if the script's JIT code is forcibly discarded.
230 // See also the ScriptWarmUpData class.
231 mozilla::Atomic<uint32_t, mozilla::Relaxed> warmUpCount_ = {};
233 // The offset of the ICFallbackStub array.
234 Offset fallbackStubsOffset_;
236 // The size of this allocation.
237 Offset endOffset_;
239 // The inlining depth of this ICScript. 0 for the inlining root.
240 uint32_t depth_;
242 // Bytecode size of the JSScript corresponding to this ICScript.
243 uint32_t bytecodeSize_;
245 // Flag set when discarding JIT code to indicate this script is on the stack
246 // and should not be discarded.
247 bool active_ = false;
249 Offset icEntriesOffset() const { return offsetOfICEntries(); }
250 Offset fallbackStubsOffset() const { return fallbackStubsOffset_; }
251 Offset endOffset() const { return endOffset_; }
253 public:
254 ICEntry* icEntries() { return offsetToPointer<ICEntry>(icEntriesOffset()); }
256 private:
257 ICFallbackStub* fallbackStubs() {
258 return offsetToPointer<ICFallbackStub>(fallbackStubsOffset());
261 JitScript* outerJitScript();
263 friend class JitScript;
266 // [SMDOC] JitScript
268 // JitScript stores type inference data, Baseline ICs and other JIT-related data
269 // for a script. Scripts with a JitScript can run in the Baseline Interpreter.
271 // IC Data
272 // =======
273 // All IC data for Baseline (Interpreter and JIT) is stored in an ICScript. Each
274 // JitScript contains an ICScript as the last field. Additional free-standing
275 // ICScripts may be created during trial inlining. Ion has its own IC chains
276 // stored in IonScript.
278 // For each IC we store an ICEntry, which points to the first ICStub in the
279 // chain, and an ICFallbackStub. Note that multiple stubs in the same zone can
280 // share Baseline IC code. This works because the stub data is stored in the
281 // ICStub instead of baked in in the stub code.
283 // Storing this separate from BaselineScript allows us to use the same ICs in
284 // the Baseline Interpreter and Baseline JIT. It also simplifies debug mode OSR
285 // because the JitScript can be reused when we have to recompile the
286 // BaselineScript.
288 // An ICScript contains a list of IC entries and a list of fallback stubs.
289 // There's one ICEntry and ICFallbackStub for each JOF_IC bytecode op.
291 // The ICScript also contains the warmUpCount for the script.
293 // Inlining Data
294 // =============
295 // JitScript also contains a list of Warp compilations inlining this script, for
296 // invalidation.
298 // Memory Layout
299 // =============
300 // JitScript contains an ICScript as the last field. ICScript has trailing
301 // (variable length) arrays for ICEntry and ICFallbackStub. The memory layout is
302 // as follows:
304 // Item | Offset
305 // ------------------------+------------------------
306 // JitScript | 0
307 // -->ICScript (field) |
308 // ICEntry[] | icEntriesOffset()
309 // ICFallbackStub[] | fallbackStubsOffset()
311 // These offsets are also used to compute numICEntries.
312 class alignas(uintptr_t) JitScript final
313 : public mozilla::LinkedListElement<JitScript>,
314 public TrailingArray<JitScript> {
315 friend class ::JSScript;
317 // Profile string used by the profiler for Baseline Interpreter frames.
318 const char* profileString_ = nullptr;
320 HeapPtr<JSScript*> owningScript_;
322 // Baseline code for the script. Either nullptr, BaselineDisabledScriptPtr or
323 // a valid BaselineScript*.
324 GCStructPtr<BaselineScript*> baselineScript_;
326 // Ion code for this script. Either nullptr, IonDisabledScriptPtr,
327 // IonCompilingScriptPtr or a valid IonScript*.
328 GCStructPtr<IonScript*> ionScript_;
330 // For functions that need a CallObject and/or NamedLambdaObject, the template
331 // objects used by the Baseline JIT and Ion. If the function needs both a
332 // named lambda object and a call object, the named lambda object template is
333 // linked via the call object's enclosing environment. This field is set the
334 // first time the Baseline JIT compiles this script.
335 mozilla::Maybe<HeapPtr<EnvironmentObject*>> templateEnv_;
337 // Analysis data computed lazily the first time this script is compiled or
338 // inlined by WarpBuilder.
339 mozilla::Maybe<bool> usesEnvironmentChain_;
341 // The size of this allocation.
342 Offset endOffset_ = 0;
344 struct Flags {
345 // True if this script entered Ion via OSR at a loop header.
346 bool hadIonOSR : 1;
348 Flags flags_ = {}; // Zero-initialize flags.
350 js::UniquePtr<InliningRoot> inliningRoot_;
352 #ifdef DEBUG
353 // If the last warp compilation invalidated because of TranspiledCacheIR
354 // bailouts, this is a hash of the ICScripts used in that compilation.
355 // When recompiling, we assert that the hash has changed.
356 mozilla::Maybe<mozilla::HashNumber> failedICHash_;
358 // To avoid pathological cases, we skip the check if we have purged
359 // stubs due to GC pressure.
360 bool hasPurgedStubs_ = false;
361 #endif
363 // Value of the warmup counter when the last IC stub was attached,
364 // used for Ion hints.
365 uint32_t warmUpCountAtLastICStub_ = 0;
367 ICScript icScript_;
368 // End of fields.
370 Offset endOffset() const { return endOffset_; }
372 public:
373 JitScript(JSScript* script, Offset fallbackStubsOffset, Offset endOffset,
374 const char* profileString);
376 ~JitScript();
378 JSScript* owningScript() const { return owningScript_; }
380 [[nodiscard]] bool ensureHasCachedBaselineJitData(JSContext* cx,
381 HandleScript script);
382 [[nodiscard]] bool ensureHasCachedIonData(JSContext* cx, HandleScript script);
384 void setHadIonOSR() { flags_.hadIonOSR = true; }
385 bool hadIonOSR() const { return flags_.hadIonOSR; }
387 uint32_t numICEntries() const { return icScript_.numICEntries(); }
389 #ifdef DEBUG
390 bool hasActiveICScript() const;
391 #endif
392 void resetAllActiveFlags();
394 void ensureProfileString(JSContext* cx, JSScript* script);
396 const char* profileString() const {
397 MOZ_ASSERT(profileString_);
398 return profileString_;
401 static void Destroy(Zone* zone, JitScript* script);
403 static constexpr Offset offsetOfICEntries() { return sizeof(JitScript); }
405 static constexpr size_t offsetOfBaselineScript() {
406 return offsetof(JitScript, baselineScript_);
408 static constexpr size_t offsetOfIonScript() {
409 return offsetof(JitScript, ionScript_);
411 static constexpr size_t offsetOfICScript() {
412 return offsetof(JitScript, icScript_);
414 static constexpr size_t offsetOfWarmUpCount() {
415 return offsetOfICScript() + ICScript::offsetOfWarmUpCount();
418 uint32_t warmUpCount() const { return icScript_.warmUpCount_; }
419 void incWarmUpCount() { icScript_.warmUpCount_++; }
420 void resetWarmUpCount(uint32_t count);
422 void prepareForDestruction(Zone* zone);
424 void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf, size_t* data,
425 size_t* allocSites) const;
427 ICEntry& icEntry(size_t index) { return icScript_.icEntry(index); }
429 ICFallbackStub* fallbackStub(size_t index) {
430 return icScript_.fallbackStub(index);
433 ICEntry* icEntryForStub(const ICFallbackStub* stub) {
434 return icScript_.icEntryForStub(stub);
436 ICFallbackStub* fallbackStubForICEntry(const ICEntry* entry) {
437 return icScript_.fallbackStubForICEntry(entry);
440 void trace(JSTracer* trc);
441 void traceWeak(JSTracer* trc);
442 void purgeStubs(JSScript* script, ICStubSpace& newStubSpace);
444 void purgeInactiveICScripts();
446 ICEntry& icEntryFromPCOffset(uint32_t pcOffset) {
447 return icScript_.icEntryFromPCOffset(pcOffset);
450 size_t allocBytes() const { return endOffset(); }
452 EnvironmentObject* templateEnvironment() const { return templateEnv_.ref(); }
454 bool usesEnvironmentChain() const { return *usesEnvironmentChain_; }
456 bool resetAllocSites(bool resetNurserySites, bool resetPretenuredSites);
458 void updateLastICStubCounter() { warmUpCountAtLastICStub_ = warmUpCount(); }
459 uint32_t warmUpCountAtLastICStub() const { return warmUpCountAtLastICStub_; }
461 private:
462 // Methods to set baselineScript_ to a BaselineScript*, nullptr, or
463 // BaselineDisabledScriptPtr.
464 void setBaselineScriptImpl(JSScript* script, BaselineScript* baselineScript);
465 void setBaselineScriptImpl(JS::GCContext* gcx, JSScript* script,
466 BaselineScript* baselineScript);
468 public:
469 // Methods for getting/setting/clearing a BaselineScript*.
470 bool hasBaselineScript() const {
471 bool res = baselineScript_ && baselineScript_ != BaselineDisabledScriptPtr;
472 MOZ_ASSERT_IF(!res, !hasIonScript());
473 return res;
475 BaselineScript* baselineScript() const {
476 MOZ_ASSERT(hasBaselineScript());
477 return baselineScript_;
479 void setBaselineScript(JSScript* script, BaselineScript* baselineScript) {
480 MOZ_ASSERT(!hasBaselineScript());
481 setBaselineScriptImpl(script, baselineScript);
482 MOZ_ASSERT(hasBaselineScript());
484 [[nodiscard]] BaselineScript* clearBaselineScript(JS::GCContext* gcx,
485 JSScript* script) {
486 BaselineScript* baseline = baselineScript();
487 setBaselineScriptImpl(gcx, script, nullptr);
488 return baseline;
491 private:
492 // Methods to set ionScript_ to an IonScript*, nullptr, or one of the special
493 // Ion{Disabled,Compiling}ScriptPtr values.
494 void setIonScriptImpl(JS::GCContext* gcx, JSScript* script,
495 IonScript* ionScript);
496 void setIonScriptImpl(JSScript* script, IonScript* ionScript);
498 // Helper that calls the passed function for the outer ICScript and for each
499 // inlined ICScript.
500 template <typename F>
501 void forEachICScript(const F& f);
502 template <typename F>
503 void forEachICScript(const F& f) const;
505 public:
506 // Methods for getting/setting/clearing an IonScript*.
507 bool hasIonScript() const {
508 bool res = ionScript_ && ionScript_ != IonDisabledScriptPtr &&
509 ionScript_ != IonCompilingScriptPtr;
510 MOZ_ASSERT_IF(res, baselineScript_);
511 return res;
513 IonScript* ionScript() const {
514 MOZ_ASSERT(hasIonScript());
515 return ionScript_;
517 void setIonScript(JSScript* script, IonScript* ionScript) {
518 MOZ_ASSERT(!hasIonScript());
519 setIonScriptImpl(script, ionScript);
520 MOZ_ASSERT(hasIonScript());
522 [[nodiscard]] IonScript* clearIonScript(JS::GCContext* gcx,
523 JSScript* script) {
524 IonScript* ion = ionScript();
525 setIonScriptImpl(gcx, script, nullptr);
526 return ion;
529 // Methods for off-thread compilation.
530 bool isIonCompilingOffThread() const {
531 return ionScript_ == IonCompilingScriptPtr;
533 void setIsIonCompilingOffThread(JSScript* script) {
534 MOZ_ASSERT(ionScript_ == nullptr);
535 setIonScriptImpl(script, IonCompilingScriptPtr);
537 void clearIsIonCompilingOffThread(JSScript* script) {
538 MOZ_ASSERT(isIonCompilingOffThread());
539 setIonScriptImpl(script, nullptr);
541 ICScript* icScript() { return &icScript_; }
543 bool hasInliningRoot() const { return !!inliningRoot_; }
544 InliningRoot* inliningRoot() const { return inliningRoot_.get(); }
545 InliningRoot* getOrCreateInliningRoot(JSContext* cx, JSScript* script);
547 inline void notePurgedStubs() {
548 #ifdef DEBUG
549 failedICHash_.reset();
550 hasPurgedStubs_ = true;
551 #endif
554 #ifdef DEBUG
555 bool hasPurgedStubs() const { return hasPurgedStubs_; }
556 bool hasFailedICHash() const { return failedICHash_.isSome(); }
557 mozilla::HashNumber getFailedICHash() { return failedICHash_.extract(); }
558 void setFailedICHash(mozilla::HashNumber hash) {
559 MOZ_ASSERT(failedICHash_.isNothing());
560 if (!hasPurgedStubs_) {
561 failedICHash_.emplace(hash);
564 #endif
567 // Ensures no JitScripts are purged in the current zone.
568 class MOZ_RAII AutoKeepJitScripts {
569 jit::JitZone* zone_;
570 bool prev_;
572 AutoKeepJitScripts(const AutoKeepJitScripts&) = delete;
573 void operator=(const AutoKeepJitScripts&) = delete;
575 public:
576 explicit inline AutoKeepJitScripts(JSContext* cx);
577 inline ~AutoKeepJitScripts();
580 // Mark ICScripts on the stack as active, so that they are not discarded
581 // during GC, and copy active Baseline IC stubs to the new stub space.
582 void MarkActiveICScriptsAndCopyStubs(Zone* zone, ICStubSpace& newStubSpace);
584 #ifdef JS_STRUCTURED_SPEW
585 void JitSpewBaselineICStats(JSScript* script, const char* dumpReason);
586 #endif
588 } // namespace jit
589 } // namespace js
591 #endif /* jit_JitScript_h */