1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifndef jit_JitScript_h
8 #define jit_JitScript_h
10 #include "mozilla/Assertions.h"
11 #include "mozilla/Atomics.h"
12 #include "mozilla/Attributes.h"
13 #include "mozilla/HashFunctions.h"
14 #include "mozilla/LinkedList.h"
15 #include "mozilla/Maybe.h"
16 #include "mozilla/MemoryReporting.h"
22 #include "NamespaceImports.h"
24 #include "ds/LifoAlloc.h"
25 #include "gc/Barrier.h"
26 #include "jit/BaselineIC.h"
27 #include "js/TypeDecls.h"
28 #include "js/UniquePtr.h"
29 #include "js/Vector.h"
30 #include "util/TrailingArray.h"
31 #include "vm/EnvironmentObject.h"
33 class JS_PUBLIC_API JSScript
;
34 class JS_PUBLIC_API JSTracer
;
35 struct JS_PUBLIC_API JSContext
;
43 class SystemAllocPolicy
;
58 // Magic BaselineScript value indicating Baseline compilation has been disabled.
59 static constexpr uintptr_t BaselineDisabledScript
= 0x1;
61 static BaselineScript
* const BaselineDisabledScriptPtr
=
62 reinterpret_cast<BaselineScript
*>(BaselineDisabledScript
);
64 // Magic IonScript values indicating Ion compilation has been disabled or the
65 // script is being Ion-compiled off-thread.
66 static constexpr uintptr_t IonDisabledScript
= 0x1;
67 static constexpr uintptr_t IonCompilingScript
= 0x2;
69 static IonScript
* const IonDisabledScriptPtr
=
70 reinterpret_cast<IonScript
*>(IonDisabledScript
);
71 static IonScript
* const IonCompilingScriptPtr
=
72 reinterpret_cast<IonScript
*>(IonCompilingScript
);
74 /* [SMDOC] ICScript Lifetimes
76 * An ICScript owns an array of ICEntries, each of which owns a linked
79 * A JitScript contains an embedded ICScript. If it has done any trial
80 * inlining, it also owns an InliningRoot. The InliningRoot owns all
81 * of the ICScripts that have been created for inlining into the
82 * corresponding JitScript. This ties the lifetime of the inlined
83 * ICScripts to the lifetime of the JitScript itself.
85 * We store pointers to ICScripts in two other places: on the stack in
86 * BaselineFrame, and in IC stubs for CallInlinedFunction.
88 * The ICScript pointer in a BaselineFrame either points to the
89 * ICScript embedded in the JitScript for that frame, or to an inlined
90 * ICScript owned by a caller. In each case, there must be a frame on
91 * the stack corresponding to the JitScript that owns the current
92 * ICScript, which will keep the ICScript alive.
94 * Each ICStub is owned by an ICScript and, indirectly, a
95 * JitScript. An ICStub that uses CallInlinedFunction contains an
96 * ICScript for use by the callee. The ICStub and the callee ICScript
97 * are always owned by the same JitScript, so the callee ICScript will
98 * not be freed while the ICStub is alive.
100 * The lifetime of an ICScript is independent of the lifetimes of the
101 * BaselineScript and IonScript/WarpScript to which it
102 * corresponds. They can be destroyed and recreated, and the ICScript
105 * When we discard JIT code, we mark ICScripts that are active on the stack as
106 * active and then purge all of the inactive ICScripts. We also purge ICStubs,
107 * including the CallInlinedFunction stub at the trial inining call site, and
108 * reset the ICStates to allow trial inlining again later.
110 * If there's a BaselineFrame for an inlined ICScript, we'll preserve both this
111 * ICScript and the IC chain for the call site in the caller's ICScript.
112 * See ICScript::purgeStubs and ICScript::purgeInactiveICScripts.
115 class alignas(uintptr_t) ICScript final
: public TrailingArray
<ICScript
> {
117 ICScript(uint32_t warmUpCount
, Offset fallbackStubsOffset
, Offset endOffset
,
118 uint32_t depth
, uint32_t bytecodeSize
,
119 InliningRoot
* inliningRoot
= nullptr)
120 : inliningRoot_(inliningRoot
),
121 warmUpCount_(warmUpCount
),
122 fallbackStubsOffset_(fallbackStubsOffset
),
123 endOffset_(endOffset
),
125 bytecodeSize_(bytecodeSize
) {}
129 bool isInlined() const { return depth_
> 0; }
131 void initICEntries(JSContext
* cx
, JSScript
* script
);
133 ICEntry
& icEntry(size_t index
) {
134 MOZ_ASSERT(index
< numICEntries());
135 return icEntries()[index
];
138 ICFallbackStub
* fallbackStub(size_t index
) {
139 MOZ_ASSERT(index
< numICEntries());
140 return fallbackStubs() + index
;
143 ICEntry
* icEntryForStub(const ICFallbackStub
* stub
) {
144 size_t index
= stub
- fallbackStubs();
145 MOZ_ASSERT(index
< numICEntries());
146 return &icEntry(index
);
148 ICFallbackStub
* fallbackStubForICEntry(const ICEntry
* entry
) {
149 size_t index
= entry
- icEntries();
150 MOZ_ASSERT(index
< numICEntries());
151 return fallbackStub(index
);
154 InliningRoot
* inliningRoot() const { return inliningRoot_
; }
155 uint32_t depth() const { return depth_
; }
157 uint32_t bytecodeSize() const { return bytecodeSize_
; }
159 void resetWarmUpCount(uint32_t count
) { warmUpCount_
= count
; }
161 static constexpr size_t offsetOfFirstStub(uint32_t entryIndex
) {
162 return sizeof(ICScript
) + entryIndex
* sizeof(ICEntry
) +
163 ICEntry::offsetOfFirstStub();
166 static constexpr Offset
offsetOfWarmUpCount() {
167 return offsetof(ICScript
, warmUpCount_
);
169 static constexpr Offset
offsetOfDepth() { return offsetof(ICScript
, depth_
); }
171 static constexpr Offset
offsetOfICEntries() { return sizeof(ICScript
); }
172 uint32_t numICEntries() const {
173 return numElements
<ICEntry
>(icEntriesOffset(), fallbackStubsOffset());
176 ICEntry
* interpreterICEntryFromPCOffset(uint32_t pcOffset
);
178 ICEntry
& icEntryFromPCOffset(uint32_t pcOffset
);
180 [[nodiscard
]] bool addInlinedChild(JSContext
* cx
,
181 js::UniquePtr
<ICScript
> child
,
183 ICScript
* findInlinedChild(uint32_t pcOffset
);
184 void removeInlinedChild(uint32_t pcOffset
);
185 bool hasInlinedChild(uint32_t pcOffset
);
187 void purgeStubs(Zone
* zone
, ICStubSpace
& newStubSpace
);
189 void purgeInactiveICScripts();
191 bool active() const { return active_
; }
192 void setActive() { active_
= true; }
193 void resetActive() { active_
= false; }
195 gc::AllocSite
* getOrCreateAllocSite(JSScript
* outerScript
, uint32_t pcOffset
);
197 void prepareForDestruction(Zone
* zone
);
199 void trace(JSTracer
* trc
);
200 bool traceWeak(JSTracer
* trc
);
203 mozilla::HashNumber
hash();
209 CallSite(ICScript
* callee
, uint32_t pcOffset
)
210 : callee_(callee
), pcOffset_(pcOffset
) {}
215 // If this ICScript was created for trial inlining or has another
216 // ICScript inlined into it, a pointer to the root of the inlining
217 // tree. Otherwise, nullptr.
218 InliningRoot
* inliningRoot_
= nullptr;
220 // ICScripts that have been inlined into this ICScript.
221 js::UniquePtr
<Vector
<CallSite
>> inlinedChildren_
;
223 // List of allocation sites referred to by ICs in this script.
224 static constexpr size_t AllocSiteChunkSize
= 256;
225 LifoAlloc allocSitesSpace_
{AllocSiteChunkSize
};
226 Vector
<gc::AllocSite
*, 0, SystemAllocPolicy
> allocSites_
;
228 // Number of times this copy of the script has been called or has had
229 // backedges taken. Reset if the script's JIT code is forcibly discarded.
230 // See also the ScriptWarmUpData class.
231 mozilla::Atomic
<uint32_t, mozilla::Relaxed
> warmUpCount_
= {};
233 // The offset of the ICFallbackStub array.
234 Offset fallbackStubsOffset_
;
236 // The size of this allocation.
239 // The inlining depth of this ICScript. 0 for the inlining root.
242 // Bytecode size of the JSScript corresponding to this ICScript.
243 uint32_t bytecodeSize_
;
245 // Flag set when discarding JIT code to indicate this script is on the stack
246 // and should not be discarded.
247 bool active_
= false;
249 Offset
icEntriesOffset() const { return offsetOfICEntries(); }
250 Offset
fallbackStubsOffset() const { return fallbackStubsOffset_
; }
251 Offset
endOffset() const { return endOffset_
; }
254 ICEntry
* icEntries() { return offsetToPointer
<ICEntry
>(icEntriesOffset()); }
257 ICFallbackStub
* fallbackStubs() {
258 return offsetToPointer
<ICFallbackStub
>(fallbackStubsOffset());
261 JitScript
* outerJitScript();
263 friend class JitScript
;
268 // JitScript stores type inference data, Baseline ICs and other JIT-related data
269 // for a script. Scripts with a JitScript can run in the Baseline Interpreter.
273 // All IC data for Baseline (Interpreter and JIT) is stored in an ICScript. Each
274 // JitScript contains an ICScript as the last field. Additional free-standing
275 // ICScripts may be created during trial inlining. Ion has its own IC chains
276 // stored in IonScript.
278 // For each IC we store an ICEntry, which points to the first ICStub in the
279 // chain, and an ICFallbackStub. Note that multiple stubs in the same zone can
280 // share Baseline IC code. This works because the stub data is stored in the
281 // ICStub instead of baked in in the stub code.
283 // Storing this separate from BaselineScript allows us to use the same ICs in
284 // the Baseline Interpreter and Baseline JIT. It also simplifies debug mode OSR
285 // because the JitScript can be reused when we have to recompile the
288 // An ICScript contains a list of IC entries and a list of fallback stubs.
289 // There's one ICEntry and ICFallbackStub for each JOF_IC bytecode op.
291 // The ICScript also contains the warmUpCount for the script.
295 // JitScript also contains a list of Warp compilations inlining this script, for
300 // JitScript contains an ICScript as the last field. ICScript has trailing
301 // (variable length) arrays for ICEntry and ICFallbackStub. The memory layout is
305 // ------------------------+------------------------
307 // -->ICScript (field) |
308 // ICEntry[] | icEntriesOffset()
309 // ICFallbackStub[] | fallbackStubsOffset()
311 // These offsets are also used to compute numICEntries.
312 class alignas(uintptr_t) JitScript final
313 : public mozilla::LinkedListElement
<JitScript
>,
314 public TrailingArray
<JitScript
> {
315 friend class ::JSScript
;
317 // Profile string used by the profiler for Baseline Interpreter frames.
318 const char* profileString_
= nullptr;
320 HeapPtr
<JSScript
*> owningScript_
;
322 // Baseline code for the script. Either nullptr, BaselineDisabledScriptPtr or
323 // a valid BaselineScript*.
324 GCStructPtr
<BaselineScript
*> baselineScript_
;
326 // Ion code for this script. Either nullptr, IonDisabledScriptPtr,
327 // IonCompilingScriptPtr or a valid IonScript*.
328 GCStructPtr
<IonScript
*> ionScript_
;
330 // For functions that need a CallObject and/or NamedLambdaObject, the template
331 // objects used by the Baseline JIT and Ion. If the function needs both a
332 // named lambda object and a call object, the named lambda object template is
333 // linked via the call object's enclosing environment. This field is set the
334 // first time the Baseline JIT compiles this script.
335 mozilla::Maybe
<HeapPtr
<EnvironmentObject
*>> templateEnv_
;
337 // Analysis data computed lazily the first time this script is compiled or
338 // inlined by WarpBuilder.
339 mozilla::Maybe
<bool> usesEnvironmentChain_
;
341 // The size of this allocation.
342 Offset endOffset_
= 0;
345 // True if this script entered Ion via OSR at a loop header.
348 Flags flags_
= {}; // Zero-initialize flags.
350 js::UniquePtr
<InliningRoot
> inliningRoot_
;
353 // If the last warp compilation invalidated because of TranspiledCacheIR
354 // bailouts, this is a hash of the ICScripts used in that compilation.
355 // When recompiling, we assert that the hash has changed.
356 mozilla::Maybe
<mozilla::HashNumber
> failedICHash_
;
358 // To avoid pathological cases, we skip the check if we have purged
359 // stubs due to GC pressure.
360 bool hasPurgedStubs_
= false;
363 // Value of the warmup counter when the last IC stub was attached,
364 // used for Ion hints.
365 uint32_t warmUpCountAtLastICStub_
= 0;
370 Offset
endOffset() const { return endOffset_
; }
373 JitScript(JSScript
* script
, Offset fallbackStubsOffset
, Offset endOffset
,
374 const char* profileString
);
378 JSScript
* owningScript() const { return owningScript_
; }
380 [[nodiscard
]] bool ensureHasCachedBaselineJitData(JSContext
* cx
,
381 HandleScript script
);
382 [[nodiscard
]] bool ensureHasCachedIonData(JSContext
* cx
, HandleScript script
);
384 void setHadIonOSR() { flags_
.hadIonOSR
= true; }
385 bool hadIonOSR() const { return flags_
.hadIonOSR
; }
387 uint32_t numICEntries() const { return icScript_
.numICEntries(); }
390 bool hasActiveICScript() const;
392 void resetAllActiveFlags();
394 void ensureProfileString(JSContext
* cx
, JSScript
* script
);
396 const char* profileString() const {
397 MOZ_ASSERT(profileString_
);
398 return profileString_
;
401 static void Destroy(Zone
* zone
, JitScript
* script
);
403 static constexpr Offset
offsetOfICEntries() { return sizeof(JitScript
); }
405 static constexpr size_t offsetOfBaselineScript() {
406 return offsetof(JitScript
, baselineScript_
);
408 static constexpr size_t offsetOfIonScript() {
409 return offsetof(JitScript
, ionScript_
);
411 static constexpr size_t offsetOfICScript() {
412 return offsetof(JitScript
, icScript_
);
414 static constexpr size_t offsetOfWarmUpCount() {
415 return offsetOfICScript() + ICScript::offsetOfWarmUpCount();
418 uint32_t warmUpCount() const { return icScript_
.warmUpCount_
; }
419 void incWarmUpCount() { icScript_
.warmUpCount_
++; }
420 void resetWarmUpCount(uint32_t count
);
422 void prepareForDestruction(Zone
* zone
);
424 void addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf
, size_t* data
,
425 size_t* allocSites
) const;
427 ICEntry
& icEntry(size_t index
) { return icScript_
.icEntry(index
); }
429 ICFallbackStub
* fallbackStub(size_t index
) {
430 return icScript_
.fallbackStub(index
);
433 ICEntry
* icEntryForStub(const ICFallbackStub
* stub
) {
434 return icScript_
.icEntryForStub(stub
);
436 ICFallbackStub
* fallbackStubForICEntry(const ICEntry
* entry
) {
437 return icScript_
.fallbackStubForICEntry(entry
);
440 void trace(JSTracer
* trc
);
441 void traceWeak(JSTracer
* trc
);
442 void purgeStubs(JSScript
* script
, ICStubSpace
& newStubSpace
);
444 void purgeInactiveICScripts();
446 ICEntry
& icEntryFromPCOffset(uint32_t pcOffset
) {
447 return icScript_
.icEntryFromPCOffset(pcOffset
);
450 size_t allocBytes() const { return endOffset(); }
452 EnvironmentObject
* templateEnvironment() const { return templateEnv_
.ref(); }
454 bool usesEnvironmentChain() const { return *usesEnvironmentChain_
; }
456 bool resetAllocSites(bool resetNurserySites
, bool resetPretenuredSites
);
458 void updateLastICStubCounter() { warmUpCountAtLastICStub_
= warmUpCount(); }
459 uint32_t warmUpCountAtLastICStub() const { return warmUpCountAtLastICStub_
; }
462 // Methods to set baselineScript_ to a BaselineScript*, nullptr, or
463 // BaselineDisabledScriptPtr.
464 void setBaselineScriptImpl(JSScript
* script
, BaselineScript
* baselineScript
);
465 void setBaselineScriptImpl(JS::GCContext
* gcx
, JSScript
* script
,
466 BaselineScript
* baselineScript
);
469 // Methods for getting/setting/clearing a BaselineScript*.
470 bool hasBaselineScript() const {
471 bool res
= baselineScript_
&& baselineScript_
!= BaselineDisabledScriptPtr
;
472 MOZ_ASSERT_IF(!res
, !hasIonScript());
475 BaselineScript
* baselineScript() const {
476 MOZ_ASSERT(hasBaselineScript());
477 return baselineScript_
;
479 void setBaselineScript(JSScript
* script
, BaselineScript
* baselineScript
) {
480 MOZ_ASSERT(!hasBaselineScript());
481 setBaselineScriptImpl(script
, baselineScript
);
482 MOZ_ASSERT(hasBaselineScript());
484 [[nodiscard
]] BaselineScript
* clearBaselineScript(JS::GCContext
* gcx
,
486 BaselineScript
* baseline
= baselineScript();
487 setBaselineScriptImpl(gcx
, script
, nullptr);
492 // Methods to set ionScript_ to an IonScript*, nullptr, or one of the special
493 // Ion{Disabled,Compiling}ScriptPtr values.
494 void setIonScriptImpl(JS::GCContext
* gcx
, JSScript
* script
,
495 IonScript
* ionScript
);
496 void setIonScriptImpl(JSScript
* script
, IonScript
* ionScript
);
498 // Helper that calls the passed function for the outer ICScript and for each
500 template <typename F
>
501 void forEachICScript(const F
& f
);
502 template <typename F
>
503 void forEachICScript(const F
& f
) const;
506 // Methods for getting/setting/clearing an IonScript*.
507 bool hasIonScript() const {
508 bool res
= ionScript_
&& ionScript_
!= IonDisabledScriptPtr
&&
509 ionScript_
!= IonCompilingScriptPtr
;
510 MOZ_ASSERT_IF(res
, baselineScript_
);
513 IonScript
* ionScript() const {
514 MOZ_ASSERT(hasIonScript());
517 void setIonScript(JSScript
* script
, IonScript
* ionScript
) {
518 MOZ_ASSERT(!hasIonScript());
519 setIonScriptImpl(script
, ionScript
);
520 MOZ_ASSERT(hasIonScript());
522 [[nodiscard
]] IonScript
* clearIonScript(JS::GCContext
* gcx
,
524 IonScript
* ion
= ionScript();
525 setIonScriptImpl(gcx
, script
, nullptr);
529 // Methods for off-thread compilation.
530 bool isIonCompilingOffThread() const {
531 return ionScript_
== IonCompilingScriptPtr
;
533 void setIsIonCompilingOffThread(JSScript
* script
) {
534 MOZ_ASSERT(ionScript_
== nullptr);
535 setIonScriptImpl(script
, IonCompilingScriptPtr
);
537 void clearIsIonCompilingOffThread(JSScript
* script
) {
538 MOZ_ASSERT(isIonCompilingOffThread());
539 setIonScriptImpl(script
, nullptr);
541 ICScript
* icScript() { return &icScript_
; }
543 bool hasInliningRoot() const { return !!inliningRoot_
; }
544 InliningRoot
* inliningRoot() const { return inliningRoot_
.get(); }
545 InliningRoot
* getOrCreateInliningRoot(JSContext
* cx
, JSScript
* script
);
547 inline void notePurgedStubs() {
549 failedICHash_
.reset();
550 hasPurgedStubs_
= true;
555 bool hasPurgedStubs() const { return hasPurgedStubs_
; }
556 bool hasFailedICHash() const { return failedICHash_
.isSome(); }
557 mozilla::HashNumber
getFailedICHash() { return failedICHash_
.extract(); }
558 void setFailedICHash(mozilla::HashNumber hash
) {
559 MOZ_ASSERT(failedICHash_
.isNothing());
560 if (!hasPurgedStubs_
) {
561 failedICHash_
.emplace(hash
);
567 // Ensures no JitScripts are purged in the current zone.
568 class MOZ_RAII AutoKeepJitScripts
{
572 AutoKeepJitScripts(const AutoKeepJitScripts
&) = delete;
573 void operator=(const AutoKeepJitScripts
&) = delete;
576 explicit inline AutoKeepJitScripts(JSContext
* cx
);
577 inline ~AutoKeepJitScripts();
580 // Mark ICScripts on the stack as active, so that they are not discarded
581 // during GC, and copy active Baseline IC stubs to the new stub space.
582 void MarkActiveICScriptsAndCopyStubs(Zone
* zone
, ICStubSpace
& newStubSpace
);
584 #ifdef JS_STRUCTURED_SPEW
585 void JitSpewBaselineICStats(JSScript
* script
, const char* dumpReason
);
591 #endif /* jit_JitScript_h */