1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifndef jit_MacroAssembler_inl_h
8 #define jit_MacroAssembler_inl_h
10 #include "jit/MacroAssembler.h"
12 #include "mozilla/FloatingPoint.h"
13 #include "mozilla/MathAlgorithms.h"
16 #include "jit/CalleeToken.h"
17 #include "jit/CompileWrappers.h"
18 #include "jit/JitFrames.h"
19 #include "jit/JSJitFrameIter.h"
21 #include "util/DifferentialTesting.h"
22 #include "vm/BigIntType.h"
23 #include "vm/JSObject.h"
24 #include "vm/ProxyObject.h"
25 #include "vm/Runtime.h"
26 #include "vm/StringType.h"
28 #include "jit/ABIFunctionList-inl.h"
30 #if defined(JS_CODEGEN_X86)
31 # include "jit/x86/MacroAssembler-x86-inl.h"
32 #elif defined(JS_CODEGEN_X64)
33 # include "jit/x64/MacroAssembler-x64-inl.h"
34 #elif defined(JS_CODEGEN_ARM)
35 # include "jit/arm/MacroAssembler-arm-inl.h"
36 #elif defined(JS_CODEGEN_ARM64)
37 # include "jit/arm64/MacroAssembler-arm64-inl.h"
38 #elif defined(JS_CODEGEN_MIPS32)
39 # include "jit/mips32/MacroAssembler-mips32-inl.h"
40 #elif defined(JS_CODEGEN_MIPS64)
41 # include "jit/mips64/MacroAssembler-mips64-inl.h"
42 #elif defined(JS_CODEGEN_LOONG64)
43 # include "jit/loong64/MacroAssembler-loong64-inl.h"
44 #elif defined(JS_CODEGEN_RISCV64)
45 # include "jit/riscv64/MacroAssembler-riscv64-inl.h"
46 #elif defined(JS_CODEGEN_WASM32)
47 # include "jit/wasm32/MacroAssembler-wasm32-inl.h"
48 #elif !defined(JS_CODEGEN_NONE)
49 # error "Unknown architecture!"
52 #include "wasm/WasmBuiltins.h"
57 template <typename Sig
>
58 DynFn
DynamicFunction(Sig fun
) {
59 ABIFunctionSignature
<Sig
> sig
;
60 return DynFn
{sig
.address(fun
)};
63 // Helper for generatePreBarrier.
64 inline DynFn
JitPreWriteBarrier(MIRType type
) {
66 case MIRType::Value
: {
67 using Fn
= void (*)(JSRuntime
* rt
, Value
* vp
);
68 return DynamicFunction
<Fn
>(JitValuePreWriteBarrier
);
70 case MIRType::String
: {
71 using Fn
= void (*)(JSRuntime
* rt
, JSString
** stringp
);
72 return DynamicFunction
<Fn
>(JitStringPreWriteBarrier
);
74 case MIRType::Object
: {
75 using Fn
= void (*)(JSRuntime
* rt
, JSObject
** objp
);
76 return DynamicFunction
<Fn
>(JitObjectPreWriteBarrier
);
78 case MIRType::Shape
: {
79 using Fn
= void (*)(JSRuntime
* rt
, Shape
** shapep
);
80 return DynamicFunction
<Fn
>(JitShapePreWriteBarrier
);
82 case MIRType::WasmAnyRef
: {
83 using Fn
= void (*)(JSRuntime
* rt
, wasm::AnyRef
* refp
);
84 return DynamicFunction
<Fn
>(JitWasmAnyRefPreWriteBarrier
);
91 //{{{ check_macroassembler_style
92 // ===============================================================
93 // Stack manipulation functions.
95 CodeOffset
MacroAssembler::PushWithPatch(ImmWord word
) {
96 framePushed_
+= sizeof(word
.value
);
97 return pushWithPatch(word
);
100 CodeOffset
MacroAssembler::PushWithPatch(ImmPtr imm
) {
101 return PushWithPatch(ImmWord(uintptr_t(imm
.value
)));
104 // ===============================================================
105 // Simple call functions.
107 void MacroAssembler::call(TrampolinePtr code
) { call(ImmPtr(code
.value
)); }
109 CodeOffset
MacroAssembler::call(const wasm::CallSiteDesc
& desc
,
110 const Register reg
) {
111 CodeOffset l
= call(reg
);
116 CodeOffset
MacroAssembler::call(const wasm::CallSiteDesc
& desc
,
117 uint32_t funcIndex
) {
118 CodeOffset l
= callWithPatch();
119 append(desc
, l
, funcIndex
);
123 void MacroAssembler::call(const wasm::CallSiteDesc
& desc
, wasm::Trap trap
) {
124 CodeOffset l
= callWithPatch();
125 append(desc
, l
, trap
);
128 CodeOffset
MacroAssembler::call(const wasm::CallSiteDesc
& desc
,
129 wasm::SymbolicAddress imm
) {
130 MOZ_ASSERT(wasm::NeedsBuiltinThunk(imm
),
131 "only for functions which may appear in profiler");
132 CodeOffset raOffset
= call(imm
);
133 append(desc
, raOffset
);
137 // ===============================================================
138 // ABI function calls.
140 void MacroAssembler::passABIArg(Register reg
) {
141 passABIArg(MoveOperand(reg
), ABIType::General
);
144 void MacroAssembler::passABIArg(FloatRegister reg
, ABIType type
) {
145 passABIArg(MoveOperand(reg
), type
);
148 void MacroAssembler::callWithABI(DynFn fun
, ABIType result
,
149 CheckUnsafeCallWithABI check
) {
150 AutoProfilerCallInstrumentation
profiler(*this);
151 callWithABINoProfiler(fun
.address
, result
, check
);
154 template <typename Sig
, Sig fun
>
155 void MacroAssembler::callWithABI(ABIType result
, CheckUnsafeCallWithABI check
) {
156 ABIFunction
<Sig
, fun
> abiFun
;
157 AutoProfilerCallInstrumentation
profiler(*this);
158 callWithABINoProfiler(abiFun
.address(), result
, check
);
161 void MacroAssembler::callWithABI(Register fun
, ABIType result
) {
162 AutoProfilerCallInstrumentation
profiler(*this);
163 callWithABINoProfiler(fun
, result
);
166 void MacroAssembler::callWithABI(const Address
& fun
, ABIType result
) {
167 AutoProfilerCallInstrumentation
profiler(*this);
168 callWithABINoProfiler(fun
, result
);
171 void MacroAssembler::appendSignatureType(ABIType type
) {
173 signature_
<<= ABITypeArgShift
;
174 signature_
|= uint32_t(type
);
178 ABIFunctionType
MacroAssembler::signature() const {
181 switch (signature_
) {
191 case Args_Double_None
:
192 case Args_Int_Double
:
193 case Args_Float32_Float32
:
194 case Args_Int_Float32
:
195 case Args_Double_Double
:
196 case Args_Double_Int
:
197 case Args_Double_DoubleInt
:
198 case Args_Double_DoubleDouble
:
199 case Args_Double_IntDouble
:
200 case Args_Int_IntDouble
:
201 case Args_Int_DoubleInt
:
202 case Args_Int_DoubleIntInt
:
203 case Args_Int_IntDoubleIntInt
:
204 case Args_Double_DoubleDoubleDouble
:
205 case Args_Double_DoubleDoubleDoubleDouble
:
206 case Args_Int64_GeneralGeneral
:
209 MOZ_CRASH("Unexpected type");
213 return ABIFunctionType(signature_
);
215 // No simulator enabled.
216 MOZ_CRASH("Only available for making calls within a simulator.");
220 // ===============================================================
223 uint32_t MacroAssembler::callJitNoProfiler(Register callee
) {
224 #ifdef JS_USE_LINK_REGISTER
225 // The return address is pushed by the callee.
228 callAndPushReturnAddress(callee
);
230 return currentOffset();
233 uint32_t MacroAssembler::callJit(Register callee
) {
234 AutoProfilerCallInstrumentation
profiler(*this);
235 uint32_t ret
= callJitNoProfiler(callee
);
239 uint32_t MacroAssembler::callJit(JitCode
* callee
) {
240 AutoProfilerCallInstrumentation
profiler(*this);
242 return currentOffset();
245 uint32_t MacroAssembler::callJit(TrampolinePtr code
) {
246 AutoProfilerCallInstrumentation
profiler(*this);
248 return currentOffset();
251 uint32_t MacroAssembler::callJit(ImmPtr callee
) {
252 AutoProfilerCallInstrumentation
profiler(*this);
254 return currentOffset();
257 void MacroAssembler::pushFrameDescriptor(FrameType type
) {
258 uint32_t descriptor
= MakeFrameDescriptor(type
);
259 push(Imm32(descriptor
));
262 void MacroAssembler::PushFrameDescriptor(FrameType type
) {
263 uint32_t descriptor
= MakeFrameDescriptor(type
);
264 Push(Imm32(descriptor
));
267 void MacroAssembler::pushFrameDescriptorForJitCall(FrameType type
,
269 uint32_t descriptor
= MakeFrameDescriptorForJitCall(type
, argc
);
270 push(Imm32(descriptor
));
273 void MacroAssembler::PushFrameDescriptorForJitCall(FrameType type
,
275 uint32_t descriptor
= MakeFrameDescriptorForJitCall(type
, argc
);
276 Push(Imm32(descriptor
));
279 void MacroAssembler::pushFrameDescriptorForJitCall(FrameType type
,
282 if (argc
!= scratch
) {
285 lshift32(Imm32(NUMACTUALARGS_SHIFT
), scratch
);
286 or32(Imm32(int32_t(type
)), scratch
);
290 void MacroAssembler::PushFrameDescriptorForJitCall(FrameType type
,
293 pushFrameDescriptorForJitCall(type
, argc
, scratch
);
294 framePushed_
+= sizeof(uintptr_t);
297 void MacroAssembler::loadNumActualArgs(Register framePtr
, Register dest
) {
298 loadPtr(Address(framePtr
, JitFrameLayout::offsetOfDescriptor()), dest
);
299 rshift32(Imm32(NUMACTUALARGS_SHIFT
), dest
);
302 void MacroAssembler::PushCalleeToken(Register callee
, bool constructing
) {
304 orPtr(Imm32(CalleeToken_FunctionConstructing
), callee
);
306 andPtr(Imm32(uint32_t(CalleeTokenMask
)), callee
);
308 static_assert(CalleeToken_Function
== 0,
309 "Non-constructing call requires no tagging");
314 void MacroAssembler::loadFunctionFromCalleeToken(Address token
, Register dest
) {
317 loadPtr(token
, dest
);
318 andPtr(Imm32(uint32_t(~CalleeTokenMask
)), dest
);
319 branchPtr(Assembler::Equal
, dest
, Imm32(CalleeToken_Function
), &ok
);
320 branchPtr(Assembler::Equal
, dest
, Imm32(CalleeToken_FunctionConstructing
),
322 assumeUnreachable("Unexpected CalleeToken tag");
325 loadPtr(token
, dest
);
326 andPtr(Imm32(uint32_t(CalleeTokenMask
)), dest
);
329 uint32_t MacroAssembler::buildFakeExitFrame(Register scratch
) {
330 mozilla::DebugOnly
<uint32_t> initialDepth
= framePushed();
332 PushFrameDescriptor(FrameType::IonJS
);
333 uint32_t retAddr
= pushFakeReturnAddress(scratch
);
336 MOZ_ASSERT(framePushed() == initialDepth
+ ExitFrameLayout::Size());
340 // ===============================================================
341 // Exit frame footer.
343 void MacroAssembler::enterExitFrame(Register cxreg
, Register scratch
,
345 linkExitFrame(cxreg
, scratch
);
346 // Push `ExitFrameType::VMFunction + VMFunctionId`, for marking the arguments.
347 // See ExitFooterFrame::data_.
348 uintptr_t type
= uintptr_t(ExitFrameType::VMFunction
) + uintptr_t(f
);
349 MOZ_ASSERT(type
<= INT32_MAX
);
353 void MacroAssembler::enterFakeExitFrame(Register cxreg
, Register scratch
,
354 ExitFrameType type
) {
355 linkExitFrame(cxreg
, scratch
);
356 Push(Imm32(int32_t(type
)));
359 void MacroAssembler::enterFakeExitFrameForNative(Register cxreg
,
361 bool isConstructing
) {
362 enterFakeExitFrame(cxreg
, scratch
,
363 isConstructing
? ExitFrameType::ConstructNative
364 : ExitFrameType::CallNative
);
367 void MacroAssembler::leaveExitFrame(size_t extraFrame
) {
368 freeStack(ExitFooterFrame::Size() + extraFrame
);
371 // ===============================================================
374 void MacroAssembler::moveValue(const ConstantOrRegister
& src
,
375 const ValueOperand
& dest
) {
376 if (src
.constant()) {
377 moveValue(src
.value(), dest
);
381 moveValue(src
.reg(), dest
);
384 // ===============================================================
387 void MacroAssembler::copy64(const Address
& src
, const Address
& dest
,
389 #if JS_BITS_PER_WORD == 32
390 MOZ_RELEASE_ASSERT(src
.base
!= scratch
&& dest
.base
!= scratch
);
391 load32(LowWord(src
), scratch
);
392 store32(scratch
, LowWord(dest
));
393 load32(HighWord(src
), scratch
);
394 store32(scratch
, HighWord(dest
));
396 Register64
scratch64(scratch
);
397 load64(src
, scratch64
);
398 store64(scratch64
, dest
);
402 // ===============================================================
403 // Arithmetic functions
405 void MacroAssembler::addPtr(ImmPtr imm
, Register dest
) {
406 addPtr(ImmWord(uintptr_t(imm
.value
)), dest
);
409 // ===============================================================
413 void MacroAssembler::branchIfFalseBool(Register reg
, L label
) {
414 // Note that C++ bool is only 1 byte, so ignore the higher-order bits.
415 branchTest32(Assembler::Zero
, reg
, Imm32(0xFF), label
);
418 void MacroAssembler::branchIfTrueBool(Register reg
, Label
* label
) {
419 // Note that C++ bool is only 1 byte, so ignore the higher-order bits.
420 branchTest32(Assembler::NonZero
, reg
, Imm32(0xFF), label
);
423 void MacroAssembler::branchIfRope(Register str
, Label
* label
) {
424 Address
flags(str
, JSString::offsetOfFlags());
425 branchTest32(Assembler::Zero
, flags
, Imm32(JSString::LINEAR_BIT
), label
);
428 void MacroAssembler::branchIfNotRope(Register str
, Label
* label
) {
429 Address
flags(str
, JSString::offsetOfFlags());
430 branchTest32(Assembler::NonZero
, flags
, Imm32(JSString::LINEAR_BIT
), label
);
433 void MacroAssembler::branchLatin1String(Register string
, Label
* label
) {
434 branchTest32(Assembler::NonZero
, Address(string
, JSString::offsetOfFlags()),
435 Imm32(JSString::LATIN1_CHARS_BIT
), label
);
438 void MacroAssembler::branchTwoByteString(Register string
, Label
* label
) {
439 branchTest32(Assembler::Zero
, Address(string
, JSString::offsetOfFlags()),
440 Imm32(JSString::LATIN1_CHARS_BIT
), label
);
443 void MacroAssembler::branchIfBigIntIsNegative(Register bigInt
, Label
* label
) {
444 branchTest32(Assembler::NonZero
, Address(bigInt
, BigInt::offsetOfFlags()),
445 Imm32(BigInt::signBitMask()), label
);
448 void MacroAssembler::branchIfBigIntIsNonNegative(Register bigInt
,
450 branchTest32(Assembler::Zero
, Address(bigInt
, BigInt::offsetOfFlags()),
451 Imm32(BigInt::signBitMask()), label
);
454 void MacroAssembler::branchIfBigIntIsZero(Register bigInt
, Label
* label
) {
455 branch32(Assembler::Equal
, Address(bigInt
, BigInt::offsetOfLength()),
459 void MacroAssembler::branchIfBigIntIsNonZero(Register bigInt
, Label
* label
) {
460 branch32(Assembler::NotEqual
, Address(bigInt
, BigInt::offsetOfLength()),
464 void MacroAssembler::branchTestFunctionFlags(Register fun
, uint32_t flags
,
465 Condition cond
, Label
* label
) {
466 Address
address(fun
, JSFunction::offsetOfFlagsAndArgCount());
467 branchTest32(cond
, address
, Imm32(flags
), label
);
470 void MacroAssembler::branchIfNotFunctionIsNonBuiltinCtor(Register fun
,
473 // Guard the function has the BASESCRIPT and CONSTRUCTOR flags and does NOT
474 // have the SELF_HOSTED flag.
475 // This is equivalent to JSFunction::isNonBuiltinConstructor.
476 constexpr int32_t mask
= FunctionFlags::BASESCRIPT
|
477 FunctionFlags::SELF_HOSTED
|
478 FunctionFlags::CONSTRUCTOR
;
479 constexpr int32_t expected
=
480 FunctionFlags::BASESCRIPT
| FunctionFlags::CONSTRUCTOR
;
482 load32(Address(fun
, JSFunction::offsetOfFlagsAndArgCount()), scratch
);
483 and32(Imm32(mask
), scratch
);
484 branch32(Assembler::NotEqual
, scratch
, Imm32(expected
), label
);
487 void MacroAssembler::branchIfFunctionHasNoJitEntry(Register fun
, Label
* label
) {
488 uint16_t flags
= FunctionFlags::HasJitEntryFlags();
489 branchTestFunctionFlags(fun
, flags
, Assembler::Zero
, label
);
492 void MacroAssembler::branchIfFunctionHasJitEntry(Register fun
, Label
* label
) {
493 uint16_t flags
= FunctionFlags::HasJitEntryFlags();
494 branchTestFunctionFlags(fun
, flags
, Assembler::NonZero
, label
);
497 void MacroAssembler::branchIfScriptHasJitScript(Register script
, Label
* label
) {
498 static_assert(ScriptWarmUpData::JitScriptTag
== 0,
499 "Code below depends on tag value");
500 branchTestPtr(Assembler::Zero
,
501 Address(script
, JSScript::offsetOfWarmUpData()),
502 Imm32(ScriptWarmUpData::TagMask
), label
);
505 void MacroAssembler::branchIfScriptHasNoJitScript(Register script
,
507 static_assert(ScriptWarmUpData::JitScriptTag
== 0,
508 "Code below depends on tag value");
509 static_assert(BaseScript::offsetOfWarmUpData() ==
510 SelfHostedLazyScript::offsetOfWarmUpData(),
511 "SelfHostedLazyScript and BaseScript must use same layout for "
513 branchTestPtr(Assembler::NonZero
,
514 Address(script
, JSScript::offsetOfWarmUpData()),
515 Imm32(ScriptWarmUpData::TagMask
), label
);
518 void MacroAssembler::loadJitScript(Register script
, Register dest
) {
521 branchIfScriptHasJitScript(script
, &ok
);
522 assumeUnreachable("Script has no JitScript!");
526 static_assert(ScriptWarmUpData::JitScriptTag
== 0,
527 "Code below depends on tag value");
528 loadPtr(Address(script
, JSScript::offsetOfWarmUpData()), dest
);
531 void MacroAssembler::loadFunctionArgCount(Register func
, Register output
) {
532 load32(Address(func
, JSFunction::offsetOfFlagsAndArgCount()), output
);
533 rshift32(Imm32(JSFunction::ArgCountShift
), output
);
536 void MacroAssembler::branchIfObjectEmulatesUndefined(Register objReg
,
540 MOZ_ASSERT(objReg
!= scratch
);
543 if (JS::Prefs::use_emulates_undefined_fuse()) {
544 loadPtr(AbsoluteAddress(
545 runtime()->addressOfHasSeenObjectEmulateUndefinedFuse()),
547 branchPtr(Assembler::Equal
, scratch
, ImmPtr(nullptr), &done
);
550 // The branches to out-of-line code here implement a conservative version
551 // of the JSObject::isWrapper test performed in EmulatesUndefined.
552 loadObjClassUnsafe(objReg
, scratch
);
554 branchTestClassIsProxy(true, scratch
, slowCheck
);
556 Address
flags(scratch
, JSClass::offsetOfFlags());
557 branchTest32(Assembler::NonZero
, flags
, Imm32(JSCLASS_EMULATES_UNDEFINED
),
562 void MacroAssembler::branchFunctionKind(Condition cond
,
563 FunctionFlags::FunctionKind kind
,
564 Register fun
, Register scratch
,
566 Address
address(fun
, JSFunction::offsetOfFlagsAndArgCount());
567 load32(address
, scratch
);
568 and32(Imm32(FunctionFlags::FUNCTION_KIND_MASK
), scratch
);
569 branch32(cond
, scratch
, Imm32(kind
), label
);
572 void MacroAssembler::branchTestObjClass(Condition cond
, Register obj
,
573 const JSClass
* clasp
, Register scratch
,
574 Register spectreRegToZero
,
576 MOZ_ASSERT(obj
!= scratch
);
577 MOZ_ASSERT(scratch
!= spectreRegToZero
);
579 loadPtr(Address(obj
, JSObject::offsetOfShape()), scratch
);
580 loadPtr(Address(scratch
, Shape::offsetOfBaseShape()), scratch
);
581 branchPtr(cond
, Address(scratch
, BaseShape::offsetOfClasp()), ImmPtr(clasp
),
584 if (JitOptions
.spectreObjectMitigations
) {
585 spectreZeroRegister(cond
, scratch
, spectreRegToZero
);
589 void MacroAssembler::branchTestObjClassNoSpectreMitigations(
590 Condition cond
, Register obj
, const JSClass
* clasp
, Register scratch
,
592 loadPtr(Address(obj
, JSObject::offsetOfShape()), scratch
);
593 loadPtr(Address(scratch
, Shape::offsetOfBaseShape()), scratch
);
594 branchPtr(cond
, Address(scratch
, BaseShape::offsetOfClasp()), ImmPtr(clasp
),
598 void MacroAssembler::branchTestObjClass(Condition cond
, Register obj
,
599 const Address
& clasp
, Register scratch
,
600 Register spectreRegToZero
,
602 MOZ_ASSERT(obj
!= scratch
);
603 MOZ_ASSERT(scratch
!= spectreRegToZero
);
605 loadObjClassUnsafe(obj
, scratch
);
606 branchPtr(cond
, clasp
, scratch
, label
);
608 if (JitOptions
.spectreObjectMitigations
) {
609 spectreZeroRegister(cond
, scratch
, spectreRegToZero
);
613 void MacroAssembler::branchTestObjClassNoSpectreMitigations(
614 Condition cond
, Register obj
, const Address
& clasp
, Register scratch
,
616 MOZ_ASSERT(obj
!= scratch
);
617 loadObjClassUnsafe(obj
, scratch
);
618 branchPtr(cond
, clasp
, scratch
, label
);
621 void MacroAssembler::branchTestObjClass(Condition cond
, Register obj
,
622 Register clasp
, Register scratch
,
623 Register spectreRegToZero
,
625 MOZ_ASSERT(obj
!= scratch
);
626 MOZ_ASSERT(scratch
!= spectreRegToZero
);
628 loadObjClassUnsafe(obj
, scratch
);
629 branchPtr(cond
, clasp
, scratch
, label
);
631 if (JitOptions
.spectreObjectMitigations
) {
632 spectreZeroRegister(cond
, scratch
, spectreRegToZero
);
636 void MacroAssembler::branchTestClass(
637 Condition cond
, Register clasp
,
638 std::pair
<const JSClass
*, const JSClass
*> classes
, Label
* label
) {
639 MOZ_ASSERT(cond
== Assembler::Equal
|| cond
== Assembler::NotEqual
);
641 if (cond
== Assembler::Equal
) {
642 branchPtr(Assembler::Equal
, clasp
, ImmPtr(classes
.first
), label
);
643 branchPtr(Assembler::Equal
, clasp
, ImmPtr(classes
.second
), label
);
648 branchPtr(Assembler::Equal
, clasp
, ImmPtr(classes
.first
), &isClass
);
649 branchPtr(Assembler::NotEqual
, clasp
, ImmPtr(classes
.second
), label
);
653 void MacroAssembler::branchTestObjClass(
654 Condition cond
, Register obj
,
655 std::pair
<const JSClass
*, const JSClass
*> classes
, Register scratch
,
656 Register spectreRegToZero
, Label
* label
) {
657 MOZ_ASSERT(scratch
!= spectreRegToZero
);
659 branchTestObjClassNoSpectreMitigations(cond
, obj
, classes
, scratch
, label
);
661 if (JitOptions
.spectreObjectMitigations
) {
662 spectreZeroRegister(cond
, scratch
, spectreRegToZero
);
666 void MacroAssembler::branchTestObjClassNoSpectreMitigations(
667 Condition cond
, Register obj
,
668 std::pair
<const JSClass
*, const JSClass
*> classes
, Register scratch
,
670 MOZ_ASSERT(cond
== Assembler::Equal
|| cond
== Assembler::NotEqual
);
671 MOZ_ASSERT(obj
!= scratch
);
673 loadObjClassUnsafe(obj
, scratch
);
674 branchTestClass(cond
, scratch
, classes
, label
);
677 void MacroAssembler::branchTestClassIsFunction(Condition cond
, Register clasp
,
679 return branchTestClass(cond
, clasp
, {&FunctionClass
, &ExtendedFunctionClass
},
683 void MacroAssembler::branchTestObjIsFunction(Condition cond
, Register obj
,
685 Register spectreRegToZero
,
687 MOZ_ASSERT(scratch
!= spectreRegToZero
);
689 branchTestObjIsFunctionNoSpectreMitigations(cond
, obj
, scratch
, label
);
691 if (JitOptions
.spectreObjectMitigations
) {
692 spectreZeroRegister(cond
, scratch
, spectreRegToZero
);
696 void MacroAssembler::branchTestObjIsFunctionNoSpectreMitigations(
697 Condition cond
, Register obj
, Register scratch
, Label
* label
) {
698 MOZ_ASSERT(cond
== Assembler::Equal
|| cond
== Assembler::NotEqual
);
699 MOZ_ASSERT(obj
!= scratch
);
701 loadObjClassUnsafe(obj
, scratch
);
702 branchTestClassIsFunction(cond
, scratch
, label
);
705 void MacroAssembler::branchTestObjShape(Condition cond
, Register obj
,
706 const Shape
* shape
, Register scratch
,
707 Register spectreRegToZero
,
709 MOZ_ASSERT(obj
!= scratch
);
710 MOZ_ASSERT(spectreRegToZero
!= scratch
);
712 if (JitOptions
.spectreObjectMitigations
) {
713 move32(Imm32(0), scratch
);
716 branchPtr(cond
, Address(obj
, JSObject::offsetOfShape()), ImmGCPtr(shape
),
719 if (JitOptions
.spectreObjectMitigations
) {
720 spectreMovePtr(cond
, scratch
, spectreRegToZero
);
724 void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond
,
728 branchPtr(cond
, Address(obj
, JSObject::offsetOfShape()), ImmGCPtr(shape
),
732 void MacroAssembler::branchTestObjShape(Condition cond
, Register obj
,
733 Register shape
, Register scratch
,
734 Register spectreRegToZero
,
736 MOZ_ASSERT(obj
!= scratch
);
737 MOZ_ASSERT(obj
!= shape
);
738 MOZ_ASSERT(spectreRegToZero
!= scratch
);
740 if (JitOptions
.spectreObjectMitigations
) {
741 move32(Imm32(0), scratch
);
744 branchPtr(cond
, Address(obj
, JSObject::offsetOfShape()), shape
, label
);
746 if (JitOptions
.spectreObjectMitigations
) {
747 spectreMovePtr(cond
, scratch
, spectreRegToZero
);
751 void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond
,
755 branchPtr(cond
, Address(obj
, JSObject::offsetOfShape()), shape
, label
);
758 void MacroAssembler::branchTestObjShapeUnsafe(Condition cond
, Register obj
,
759 Register shape
, Label
* label
) {
760 branchTestObjShapeNoSpectreMitigations(cond
, obj
, shape
, label
);
763 void MacroAssembler::branchTestClassIsProxy(bool proxy
, Register clasp
,
765 branchTest32(proxy
? Assembler::NonZero
: Assembler::Zero
,
766 Address(clasp
, JSClass::offsetOfFlags()),
767 Imm32(JSCLASS_IS_PROXY
), label
);
770 void MacroAssembler::branchTestObjectIsProxy(bool proxy
, Register object
,
771 Register scratch
, Label
* label
) {
772 constexpr uint32_t ShiftedMask
= (Shape::kindMask() << Shape::kindShift());
773 static_assert(uint32_t(Shape::Kind::Proxy
) == 0,
774 "branchTest32 below depends on proxy kind being 0");
775 loadPtr(Address(object
, JSObject::offsetOfShape()), scratch
);
776 branchTest32(proxy
? Assembler::Zero
: Assembler::NonZero
,
777 Address(scratch
, Shape::offsetOfImmutableFlags()),
778 Imm32(ShiftedMask
), label
);
781 void MacroAssembler::branchTestProxyHandlerFamily(Condition cond
,
784 const void* handlerp
,
788 branchTestObjectIsProxy(true, proxy
, scratch
, &ok
);
789 assumeUnreachable("Expected ProxyObject in branchTestProxyHandlerFamily");
793 Address
handlerAddr(proxy
, ProxyObject::offsetOfHandler());
794 loadPtr(handlerAddr
, scratch
);
795 Address
familyAddr(scratch
, BaseProxyHandler::offsetOfFamily());
796 branchPtr(cond
, familyAddr
, ImmPtr(handlerp
), label
);
799 void MacroAssembler::branchTestNeedsIncrementalBarrier(Condition cond
,
801 MOZ_ASSERT(cond
== Zero
|| cond
== NonZero
);
802 CompileZone
* zone
= realm()->zone();
803 const uint32_t* needsBarrierAddr
= zone
->addressOfNeedsIncrementalBarrier();
804 branchTest32(cond
, AbsoluteAddress(needsBarrierAddr
), Imm32(0x1), label
);
807 void MacroAssembler::branchTestNeedsIncrementalBarrierAnyZone(
808 Condition cond
, Label
* label
, Register scratch
) {
809 MOZ_ASSERT(cond
== Zero
|| cond
== NonZero
);
811 branchTestNeedsIncrementalBarrier(cond
, label
);
813 // We are compiling the interpreter or another runtime-wide trampoline, so
814 // we have to load cx->zone.
815 loadPtr(AbsoluteAddress(runtime()->addressOfZone()), scratch
);
816 Address
needsBarrierAddr(scratch
, Zone::offsetOfNeedsIncrementalBarrier());
817 branchTest32(cond
, needsBarrierAddr
, Imm32(0x1), label
);
821 void MacroAssembler::branchTestMagicValue(Condition cond
,
822 const ValueOperand
& val
,
823 JSWhyMagic why
, Label
* label
) {
824 MOZ_ASSERT(cond
== Equal
|| cond
== NotEqual
);
825 branchTestValue(cond
, val
, MagicValue(why
), label
);
828 void MacroAssembler::branchDoubleNotInInt64Range(Address src
, Register temp
,
830 using mozilla::FloatingPoint
;
832 // Tests if double is in [INT64_MIN; INT64_MAX] range
833 uint32_t EXPONENT_MASK
= 0x7ff00000;
834 uint32_t EXPONENT_SHIFT
= FloatingPoint
<double>::kExponentShift
- 32;
835 uint32_t TOO_BIG_EXPONENT
= (FloatingPoint
<double>::kExponentBias
+ 63)
838 load32(Address(src
.base
, src
.offset
+ sizeof(int32_t)), temp
);
839 and32(Imm32(EXPONENT_MASK
), temp
);
840 branch32(Assembler::GreaterThanOrEqual
, temp
, Imm32(TOO_BIG_EXPONENT
), fail
);
843 void MacroAssembler::branchDoubleNotInUInt64Range(Address src
, Register temp
,
845 using mozilla::FloatingPoint
;
847 // Note: returns failure on -0.0
848 // Tests if double is in [0; UINT64_MAX] range
849 // Take the sign also in the equation. That way we can compare in one test?
850 uint32_t EXPONENT_MASK
= 0xfff00000;
851 uint32_t EXPONENT_SHIFT
= FloatingPoint
<double>::kExponentShift
- 32;
852 uint32_t TOO_BIG_EXPONENT
= (FloatingPoint
<double>::kExponentBias
+ 64)
855 load32(Address(src
.base
, src
.offset
+ sizeof(int32_t)), temp
);
856 and32(Imm32(EXPONENT_MASK
), temp
);
857 branch32(Assembler::AboveOrEqual
, temp
, Imm32(TOO_BIG_EXPONENT
), fail
);
860 void MacroAssembler::branchFloat32NotInInt64Range(Address src
, Register temp
,
862 using mozilla::FloatingPoint
;
864 // Tests if float is in [INT64_MIN; INT64_MAX] range
865 uint32_t EXPONENT_MASK
= 0x7f800000;
866 uint32_t EXPONENT_SHIFT
= FloatingPoint
<float>::kExponentShift
;
867 uint32_t TOO_BIG_EXPONENT
= (FloatingPoint
<float>::kExponentBias
+ 63)
871 and32(Imm32(EXPONENT_MASK
), temp
);
872 branch32(Assembler::GreaterThanOrEqual
, temp
, Imm32(TOO_BIG_EXPONENT
), fail
);
875 void MacroAssembler::branchFloat32NotInUInt64Range(Address src
, Register temp
,
877 using mozilla::FloatingPoint
;
879 // Note: returns failure on -0.0
880 // Tests if float is in [0; UINT64_MAX] range
881 // Take the sign also in the equation. That way we can compare in one test?
882 uint32_t EXPONENT_MASK
= 0xff800000;
883 uint32_t EXPONENT_SHIFT
= FloatingPoint
<float>::kExponentShift
;
884 uint32_t TOO_BIG_EXPONENT
= (FloatingPoint
<float>::kExponentBias
+ 64)
888 and32(Imm32(EXPONENT_MASK
), temp
);
889 branch32(Assembler::AboveOrEqual
, temp
, Imm32(TOO_BIG_EXPONENT
), fail
);
892 // ========================================================================
893 // Canonicalization primitives.
894 void MacroAssembler::canonicalizeFloat(FloatRegister reg
) {
896 branchFloat(DoubleOrdered
, reg
, reg
, ¬NaN
);
897 loadConstantFloat32(float(JS::GenericNaN()), reg
);
901 void MacroAssembler::canonicalizeFloatIfDeterministic(FloatRegister reg
) {
902 // See the comment in TypedArrayObjectTemplate::getElement.
903 if (js::SupportDifferentialTesting()) {
904 canonicalizeFloat(reg
);
908 void MacroAssembler::canonicalizeDouble(FloatRegister reg
) {
910 branchDouble(DoubleOrdered
, reg
, reg
, ¬NaN
);
911 loadConstantDouble(JS::GenericNaN(), reg
);
915 void MacroAssembler::canonicalizeDoubleIfDeterministic(FloatRegister reg
) {
916 // See the comment in TypedArrayObjectTemplate::getElement.
917 if (js::SupportDifferentialTesting()) {
918 canonicalizeDouble(reg
);
922 // ========================================================================
923 // Memory access primitives.
925 FaultingCodeOffset
MacroAssembler::storeDouble(FloatRegister src
,
927 canonicalizeDoubleIfDeterministic(src
);
928 return storeUncanonicalizedDouble(src
, dest
);
931 template FaultingCodeOffset
MacroAssembler::storeDouble(FloatRegister src
,
932 const Address
& dest
);
933 template FaultingCodeOffset
MacroAssembler::storeDouble(FloatRegister src
,
934 const BaseIndex
& dest
);
937 void MacroAssembler::boxDouble(FloatRegister src
, const T
& dest
) {
938 storeDouble(src
, dest
);
942 FaultingCodeOffset
MacroAssembler::storeFloat32(FloatRegister src
,
944 canonicalizeFloatIfDeterministic(src
);
945 return storeUncanonicalizedFloat32(src
, dest
);
948 template FaultingCodeOffset
MacroAssembler::storeFloat32(FloatRegister src
,
949 const Address
& dest
);
950 template FaultingCodeOffset
MacroAssembler::storeFloat32(FloatRegister src
,
951 const BaseIndex
& dest
);
953 template <typename T
>
954 void MacroAssembler::fallibleUnboxInt32(const T
& src
, Register dest
,
956 // Int32Value can be unboxed efficiently with unboxInt32, so use that.
957 branchTestInt32(Assembler::NotEqual
, src
, fail
);
958 unboxInt32(src
, dest
);
961 template <typename T
>
962 void MacroAssembler::fallibleUnboxBoolean(const T
& src
, Register dest
,
964 // BooleanValue can be unboxed efficiently with unboxBoolean, so use that.
965 branchTestBoolean(Assembler::NotEqual
, src
, fail
);
966 unboxBoolean(src
, dest
);
969 template <typename T
>
970 void MacroAssembler::fallibleUnboxObject(const T
& src
, Register dest
,
972 fallibleUnboxPtr(src
, dest
, JSVAL_TYPE_OBJECT
, fail
);
975 template <typename T
>
976 void MacroAssembler::fallibleUnboxString(const T
& src
, Register dest
,
978 fallibleUnboxPtr(src
, dest
, JSVAL_TYPE_STRING
, fail
);
981 template <typename T
>
982 void MacroAssembler::fallibleUnboxSymbol(const T
& src
, Register dest
,
984 fallibleUnboxPtr(src
, dest
, JSVAL_TYPE_SYMBOL
, fail
);
987 template <typename T
>
988 void MacroAssembler::fallibleUnboxBigInt(const T
& src
, Register dest
,
990 fallibleUnboxPtr(src
, dest
, JSVAL_TYPE_BIGINT
, fail
);
993 //}}} check_macroassembler_style
994 // ===============================================================
996 #ifndef JS_CODEGEN_ARM64
998 template <typename T
>
999 void MacroAssembler::branchTestStackPtr(Condition cond
, T t
, Label
* label
) {
1000 branchTestPtr(cond
, getStackPointer(), t
, label
);
1003 template <typename T
>
1004 void MacroAssembler::branchStackPtr(Condition cond
, T rhs
, Label
* label
) {
1005 branchPtr(cond
, getStackPointer(), rhs
, label
);
1008 template <typename T
>
1009 void MacroAssembler::branchStackPtrRhs(Condition cond
, T lhs
, Label
* label
) {
1010 branchPtr(cond
, lhs
, getStackPointer(), label
);
1013 template <typename T
>
1014 void MacroAssembler::addToStackPtr(T t
) {
1015 addPtr(t
, getStackPointer());
1018 template <typename T
>
1019 void MacroAssembler::addStackPtrTo(T t
) {
1020 addPtr(getStackPointer(), t
);
1023 void MacroAssembler::reserveStack(uint32_t amount
) {
1024 subFromStackPtr(Imm32(amount
));
1025 adjustFrame(amount
);
1027 #endif // !JS_CODEGEN_ARM64
1029 void MacroAssembler::loadObjClassUnsafe(Register obj
, Register dest
) {
1030 loadPtr(Address(obj
, JSObject::offsetOfShape()), dest
);
1031 loadPtr(Address(dest
, Shape::offsetOfBaseShape()), dest
);
1032 loadPtr(Address(dest
, BaseShape::offsetOfClasp()), dest
);
1035 template <typename EmitPreBarrier
>
1036 void MacroAssembler::storeObjShape(Register shape
, Register obj
,
1037 EmitPreBarrier emitPreBarrier
) {
1038 MOZ_ASSERT(shape
!= obj
);
1039 Address
shapeAddr(obj
, JSObject::offsetOfShape());
1040 emitPreBarrier(*this, shapeAddr
);
1041 storePtr(shape
, shapeAddr
);
1044 template <typename EmitPreBarrier
>
1045 void MacroAssembler::storeObjShape(Shape
* shape
, Register obj
,
1046 EmitPreBarrier emitPreBarrier
) {
1047 Address
shapeAddr(obj
, JSObject::offsetOfShape());
1048 emitPreBarrier(*this, shapeAddr
);
1049 storePtr(ImmGCPtr(shape
), shapeAddr
);
1052 void MacroAssembler::loadObjProto(Register obj
, Register dest
) {
1053 loadPtr(Address(obj
, JSObject::offsetOfShape()), dest
);
1054 loadPtr(Address(dest
, Shape::offsetOfBaseShape()), dest
);
1055 loadPtr(Address(dest
, BaseShape::offsetOfProto()), dest
);
1058 void MacroAssembler::loadStringLength(Register str
, Register dest
) {
1059 load32(Address(str
, JSString::offsetOfLength()), dest
);
1062 void MacroAssembler::assertStackAlignment(uint32_t alignment
,
1063 int32_t offset
/* = 0 */) {
1066 MOZ_ASSERT(mozilla::IsPowerOfTwo(alignment
));
1068 // Wrap around the offset to be a non-negative number.
1069 offset
%= alignment
;
1071 offset
+= alignment
;
1074 // Test if each bit from offset is set.
1075 uint32_t off
= offset
;
1077 uint32_t lowestBit
= 1 << mozilla::CountTrailingZeroes32(off
);
1078 branchTestStackPtr(Assembler::Zero
, Imm32(lowestBit
), &bad
);
1082 // Check that all remaining bits are zero.
1083 branchTestStackPtr(Assembler::Zero
, Imm32((alignment
- 1) ^ offset
), &ok
);
1091 void MacroAssembler::storeCallBoolResult(Register reg
) {
1092 convertBoolToInt32(ReturnReg
, reg
);
1095 void MacroAssembler::storeCallInt32Result(Register reg
) {
1096 #if JS_BITS_PER_WORD == 32
1097 storeCallPointerResult(reg
);
1099 // Ensure the upper 32 bits are cleared.
1100 move32(ReturnReg
, reg
);
1104 void MacroAssembler::storeCallResultValue(AnyRegister dest
, JSValueType type
) {
1105 unboxValue(JSReturnOperand
, dest
, type
);
1108 void MacroAssembler::storeCallResultValue(TypedOrValueRegister dest
) {
1109 if (dest
.hasValue()) {
1110 storeCallResultValue(dest
.valueReg());
1112 storeCallResultValue(dest
.typedReg(), ValueTypeFromMIRType(dest
.type()));
1119 #endif /* jit_MacroAssembler_inl_h */