1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifndef jit_MacroAssembler_inl_h
8 #define jit_MacroAssembler_inl_h
10 #include "jit/MacroAssembler.h"
12 #include "mozilla/FloatingPoint.h"
13 #include "mozilla/MathAlgorithms.h"
16 #include "jit/CalleeToken.h"
17 #include "jit/CompileWrappers.h"
18 #include "jit/JitFrames.h"
19 #include "jit/JSJitFrameIter.h"
21 #include "util/DifferentialTesting.h"
22 #include "vm/BigIntType.h"
23 #include "vm/JSObject.h"
24 #include "vm/ProxyObject.h"
25 #include "vm/Runtime.h"
26 #include "vm/StringType.h"
28 #include "jit/ABIFunctionList-inl.h"
30 #if defined(JS_CODEGEN_X86)
31 # include "jit/x86/MacroAssembler-x86-inl.h"
32 #elif defined(JS_CODEGEN_X64)
33 # include "jit/x64/MacroAssembler-x64-inl.h"
34 #elif defined(JS_CODEGEN_ARM)
35 # include "jit/arm/MacroAssembler-arm-inl.h"
36 #elif defined(JS_CODEGEN_ARM64)
37 # include "jit/arm64/MacroAssembler-arm64-inl.h"
38 #elif defined(JS_CODEGEN_MIPS32)
39 # include "jit/mips32/MacroAssembler-mips32-inl.h"
40 #elif defined(JS_CODEGEN_MIPS64)
41 # include "jit/mips64/MacroAssembler-mips64-inl.h"
42 #elif defined(JS_CODEGEN_LOONG64)
43 # include "jit/loong64/MacroAssembler-loong64-inl.h"
44 #elif defined(JS_CODEGEN_RISCV64)
45 # include "jit/riscv64/MacroAssembler-riscv64-inl.h"
46 #elif defined(JS_CODEGEN_WASM32)
47 # include "jit/wasm32/MacroAssembler-wasm32-inl.h"
48 #elif !defined(JS_CODEGEN_NONE)
49 # error "Unknown architecture!"
52 #include "wasm/WasmBuiltins.h"
57 template <typename Sig
>
58 DynFn
DynamicFunction(Sig fun
) {
59 ABIFunctionSignature
<Sig
> sig
;
60 return DynFn
{sig
.address(fun
)};
63 // Helper for generatePreBarrier.
64 inline DynFn
JitPreWriteBarrier(MIRType type
) {
66 case MIRType::Value
: {
67 using Fn
= void (*)(JSRuntime
* rt
, Value
* vp
);
68 return DynamicFunction
<Fn
>(JitValuePreWriteBarrier
);
70 case MIRType::String
: {
71 using Fn
= void (*)(JSRuntime
* rt
, JSString
** stringp
);
72 return DynamicFunction
<Fn
>(JitStringPreWriteBarrier
);
74 case MIRType::Object
: {
75 using Fn
= void (*)(JSRuntime
* rt
, JSObject
** objp
);
76 return DynamicFunction
<Fn
>(JitObjectPreWriteBarrier
);
78 case MIRType::Shape
: {
79 using Fn
= void (*)(JSRuntime
* rt
, Shape
** shapep
);
80 return DynamicFunction
<Fn
>(JitShapePreWriteBarrier
);
82 case MIRType::WasmAnyRef
: {
83 using Fn
= void (*)(JSRuntime
* rt
, wasm::AnyRef
* refp
);
84 return DynamicFunction
<Fn
>(JitWasmAnyRefPreWriteBarrier
);
91 //{{{ check_macroassembler_style
92 // ===============================================================
93 // Stack manipulation functions.
95 CodeOffset
MacroAssembler::PushWithPatch(ImmWord word
) {
96 framePushed_
+= sizeof(word
.value
);
97 return pushWithPatch(word
);
100 CodeOffset
MacroAssembler::PushWithPatch(ImmPtr imm
) {
101 return PushWithPatch(ImmWord(uintptr_t(imm
.value
)));
104 // ===============================================================
105 // Simple call functions.
107 void MacroAssembler::call(TrampolinePtr code
) { call(ImmPtr(code
.value
)); }
109 CodeOffset
MacroAssembler::call(const wasm::CallSiteDesc
& desc
,
110 const Register reg
) {
111 CodeOffset l
= call(reg
);
116 CodeOffset
MacroAssembler::call(const wasm::CallSiteDesc
& desc
,
117 uint32_t funcIndex
) {
118 CodeOffset l
= callWithPatch();
119 append(desc
, l
, funcIndex
);
123 void MacroAssembler::call(const wasm::CallSiteDesc
& desc
, wasm::Trap trap
) {
124 CodeOffset l
= callWithPatch();
125 append(desc
, l
, trap
);
128 CodeOffset
MacroAssembler::call(const wasm::CallSiteDesc
& desc
,
129 wasm::SymbolicAddress imm
) {
130 MOZ_ASSERT(wasm::NeedsBuiltinThunk(imm
),
131 "only for functions which may appear in profiler");
132 CodeOffset raOffset
= call(imm
);
133 append(desc
, raOffset
);
137 // ===============================================================
138 // ABI function calls.
140 void MacroAssembler::passABIArg(Register reg
) {
141 passABIArg(MoveOperand(reg
), ABIType::General
);
144 void MacroAssembler::passABIArg(FloatRegister reg
, ABIType type
) {
145 passABIArg(MoveOperand(reg
), type
);
148 void MacroAssembler::callWithABI(DynFn fun
, ABIType result
,
149 CheckUnsafeCallWithABI check
) {
150 AutoProfilerCallInstrumentation
profiler(*this);
151 callWithABINoProfiler(fun
.address
, result
, check
);
154 template <typename Sig
, Sig fun
>
155 void MacroAssembler::callWithABI(ABIType result
, CheckUnsafeCallWithABI check
) {
156 ABIFunction
<Sig
, fun
> abiFun
;
157 AutoProfilerCallInstrumentation
profiler(*this);
158 callWithABINoProfiler(abiFun
.address(), result
, check
);
161 void MacroAssembler::callWithABI(Register fun
, ABIType result
) {
162 AutoProfilerCallInstrumentation
profiler(*this);
163 callWithABINoProfiler(fun
, result
);
166 void MacroAssembler::callWithABI(const Address
& fun
, ABIType result
) {
167 AutoProfilerCallInstrumentation
profiler(*this);
168 callWithABINoProfiler(fun
, result
);
171 void MacroAssembler::appendSignatureType(ABIType type
) {
173 signature_
<<= ABITypeArgShift
;
174 signature_
|= uint32_t(type
);
178 ABIFunctionType
MacroAssembler::signature() const {
181 switch (signature_
) {
191 case Args_Double_None
:
192 case Args_Int_Double
:
193 case Args_Float32_Float32
:
194 case Args_Int_Float32
:
195 case Args_Double_Double
:
196 case Args_Double_Int
:
197 case Args_Double_DoubleInt
:
198 case Args_Double_DoubleDouble
:
199 case Args_Double_IntDouble
:
200 case Args_Int_IntDouble
:
201 case Args_Int_DoubleInt
:
202 case Args_Int_DoubleIntInt
:
203 case Args_Int_IntDoubleIntInt
:
204 case Args_Double_DoubleDoubleDouble
:
205 case Args_Double_DoubleDoubleDoubleDouble
:
206 case Args_Int64_GeneralGeneral
:
209 MOZ_CRASH("Unexpected type");
213 return ABIFunctionType(signature_
);
215 // No simulator enabled.
216 MOZ_CRASH("Only available for making calls within a simulator.");
220 // ===============================================================
223 uint32_t MacroAssembler::callJitNoProfiler(Register callee
) {
224 #ifdef JS_USE_LINK_REGISTER
225 // The return address is pushed by the callee.
228 callAndPushReturnAddress(callee
);
230 return currentOffset();
233 uint32_t MacroAssembler::callJit(Register callee
) {
234 AutoProfilerCallInstrumentation
profiler(*this);
235 uint32_t ret
= callJitNoProfiler(callee
);
239 uint32_t MacroAssembler::callJit(JitCode
* callee
) {
240 AutoProfilerCallInstrumentation
profiler(*this);
242 return currentOffset();
245 uint32_t MacroAssembler::callJit(TrampolinePtr code
) {
246 AutoProfilerCallInstrumentation
profiler(*this);
248 return currentOffset();
251 uint32_t MacroAssembler::callJit(ImmPtr callee
) {
252 AutoProfilerCallInstrumentation
profiler(*this);
254 return currentOffset();
257 void MacroAssembler::pushFrameDescriptor(FrameType type
) {
258 uint32_t descriptor
= MakeFrameDescriptor(type
);
259 push(Imm32(descriptor
));
262 void MacroAssembler::PushFrameDescriptor(FrameType type
) {
263 uint32_t descriptor
= MakeFrameDescriptor(type
);
264 Push(Imm32(descriptor
));
267 void MacroAssembler::pushFrameDescriptorForJitCall(FrameType type
,
269 uint32_t descriptor
= MakeFrameDescriptorForJitCall(type
, argc
);
270 push(Imm32(descriptor
));
273 void MacroAssembler::PushFrameDescriptorForJitCall(FrameType type
,
275 uint32_t descriptor
= MakeFrameDescriptorForJitCall(type
, argc
);
276 Push(Imm32(descriptor
));
279 void MacroAssembler::pushFrameDescriptorForJitCall(FrameType type
,
282 if (argc
!= scratch
) {
285 lshift32(Imm32(NUMACTUALARGS_SHIFT
), scratch
);
286 or32(Imm32(int32_t(type
)), scratch
);
290 void MacroAssembler::PushFrameDescriptorForJitCall(FrameType type
,
293 pushFrameDescriptorForJitCall(type
, argc
, scratch
);
294 framePushed_
+= sizeof(uintptr_t);
297 void MacroAssembler::loadNumActualArgs(Register framePtr
, Register dest
) {
298 loadPtr(Address(framePtr
, JitFrameLayout::offsetOfDescriptor()), dest
);
299 rshift32(Imm32(NUMACTUALARGS_SHIFT
), dest
);
302 void MacroAssembler::PushCalleeToken(Register callee
, bool constructing
) {
304 orPtr(Imm32(CalleeToken_FunctionConstructing
), callee
);
306 andPtr(Imm32(uint32_t(CalleeTokenMask
)), callee
);
308 static_assert(CalleeToken_Function
== 0,
309 "Non-constructing call requires no tagging");
314 void MacroAssembler::loadFunctionFromCalleeToken(Address token
, Register dest
) {
317 loadPtr(token
, dest
);
318 andPtr(Imm32(uint32_t(~CalleeTokenMask
)), dest
);
319 branchPtr(Assembler::Equal
, dest
, Imm32(CalleeToken_Function
), &ok
);
320 branchPtr(Assembler::Equal
, dest
, Imm32(CalleeToken_FunctionConstructing
),
322 assumeUnreachable("Unexpected CalleeToken tag");
325 loadPtr(token
, dest
);
326 andPtr(Imm32(uint32_t(CalleeTokenMask
)), dest
);
329 uint32_t MacroAssembler::buildFakeExitFrame(Register scratch
) {
330 mozilla::DebugOnly
<uint32_t> initialDepth
= framePushed();
332 PushFrameDescriptor(FrameType::IonJS
);
333 uint32_t retAddr
= pushFakeReturnAddress(scratch
);
336 MOZ_ASSERT(framePushed() == initialDepth
+ ExitFrameLayout::Size());
340 // ===============================================================
341 // Exit frame footer.
343 void MacroAssembler::enterExitFrame(Register cxreg
, Register scratch
,
345 linkExitFrame(cxreg
, scratch
);
346 // Push `ExitFrameType::VMFunction + VMFunctionId`, for marking the arguments.
347 // See ExitFooterFrame::data_.
348 uintptr_t type
= uintptr_t(ExitFrameType::VMFunction
) + uintptr_t(f
);
349 MOZ_ASSERT(type
<= INT32_MAX
);
353 void MacroAssembler::enterFakeExitFrame(Register cxreg
, Register scratch
,
354 ExitFrameType type
) {
355 linkExitFrame(cxreg
, scratch
);
356 Push(Imm32(int32_t(type
)));
359 void MacroAssembler::enterFakeExitFrameForNative(Register cxreg
,
361 bool isConstructing
) {
362 enterFakeExitFrame(cxreg
, scratch
,
363 isConstructing
? ExitFrameType::ConstructNative
364 : ExitFrameType::CallNative
);
367 void MacroAssembler::leaveExitFrame(size_t extraFrame
) {
368 freeStack(ExitFooterFrame::Size() + extraFrame
);
371 // ===============================================================
374 void MacroAssembler::moveValue(const ConstantOrRegister
& src
,
375 const ValueOperand
& dest
) {
376 if (src
.constant()) {
377 moveValue(src
.value(), dest
);
381 moveValue(src
.reg(), dest
);
384 // ===============================================================
387 void MacroAssembler::copy64(const Address
& src
, const Address
& dest
,
389 #if JS_BITS_PER_WORD == 32
390 MOZ_RELEASE_ASSERT(src
.base
!= scratch
&& dest
.base
!= scratch
);
391 load32(LowWord(src
), scratch
);
392 store32(scratch
, LowWord(dest
));
393 load32(HighWord(src
), scratch
);
394 store32(scratch
, HighWord(dest
));
396 Register64
scratch64(scratch
);
397 load64(src
, scratch64
);
398 store64(scratch64
, dest
);
402 // ===============================================================
403 // Arithmetic functions
405 void MacroAssembler::addPtr(ImmPtr imm
, Register dest
) {
406 addPtr(ImmWord(uintptr_t(imm
.value
)), dest
);
409 // ===============================================================
413 void MacroAssembler::branchIfFalseBool(Register reg
, L label
) {
414 // Note that C++ bool is only 1 byte, so ignore the higher-order bits.
415 branchTest32(Assembler::Zero
, reg
, Imm32(0xFF), label
);
418 void MacroAssembler::branchIfTrueBool(Register reg
, Label
* label
) {
419 // Note that C++ bool is only 1 byte, so ignore the higher-order bits.
420 branchTest32(Assembler::NonZero
, reg
, Imm32(0xFF), label
);
423 void MacroAssembler::branchIfRope(Register str
, Label
* label
) {
424 Address
flags(str
, JSString::offsetOfFlags());
425 branchTest32(Assembler::Zero
, flags
, Imm32(JSString::LINEAR_BIT
), label
);
428 void MacroAssembler::branchIfNotRope(Register str
, Label
* label
) {
429 Address
flags(str
, JSString::offsetOfFlags());
430 branchTest32(Assembler::NonZero
, flags
, Imm32(JSString::LINEAR_BIT
), label
);
433 void MacroAssembler::branchLatin1String(Register string
, Label
* label
) {
434 branchTest32(Assembler::NonZero
, Address(string
, JSString::offsetOfFlags()),
435 Imm32(JSString::LATIN1_CHARS_BIT
), label
);
438 void MacroAssembler::branchTwoByteString(Register string
, Label
* label
) {
439 branchTest32(Assembler::Zero
, Address(string
, JSString::offsetOfFlags()),
440 Imm32(JSString::LATIN1_CHARS_BIT
), label
);
443 void MacroAssembler::branchIfBigIntIsNegative(Register bigInt
, Label
* label
) {
444 branchTest32(Assembler::NonZero
, Address(bigInt
, BigInt::offsetOfFlags()),
445 Imm32(BigInt::signBitMask()), label
);
448 void MacroAssembler::branchIfBigIntIsNonNegative(Register bigInt
,
450 branchTest32(Assembler::Zero
, Address(bigInt
, BigInt::offsetOfFlags()),
451 Imm32(BigInt::signBitMask()), label
);
454 void MacroAssembler::branchIfBigIntIsZero(Register bigInt
, Label
* label
) {
455 branch32(Assembler::Equal
, Address(bigInt
, BigInt::offsetOfLength()),
459 void MacroAssembler::branchIfBigIntIsNonZero(Register bigInt
, Label
* label
) {
460 branch32(Assembler::NotEqual
, Address(bigInt
, BigInt::offsetOfLength()),
464 void MacroAssembler::branchTestFunctionFlags(Register fun
, uint32_t flags
,
465 Condition cond
, Label
* label
) {
466 Address
address(fun
, JSFunction::offsetOfFlagsAndArgCount());
467 branchTest32(cond
, address
, Imm32(flags
), label
);
470 void MacroAssembler::branchIfNotFunctionIsNonBuiltinCtor(Register fun
,
473 // Guard the function has the BASESCRIPT and CONSTRUCTOR flags and does NOT
474 // have the SELF_HOSTED flag.
475 // This is equivalent to JSFunction::isNonBuiltinConstructor.
476 constexpr int32_t mask
= FunctionFlags::BASESCRIPT
|
477 FunctionFlags::SELF_HOSTED
|
478 FunctionFlags::CONSTRUCTOR
;
479 constexpr int32_t expected
=
480 FunctionFlags::BASESCRIPT
| FunctionFlags::CONSTRUCTOR
;
482 load32(Address(fun
, JSFunction::offsetOfFlagsAndArgCount()), scratch
);
483 and32(Imm32(mask
), scratch
);
484 branch32(Assembler::NotEqual
, scratch
, Imm32(expected
), label
);
487 void MacroAssembler::branchIfFunctionHasNoJitEntry(Register fun
,
490 uint16_t flags
= FunctionFlags::HasJitEntryFlags(isConstructing
);
491 branchTestFunctionFlags(fun
, flags
, Assembler::Zero
, label
);
494 void MacroAssembler::branchIfFunctionHasJitEntry(Register fun
,
497 uint16_t flags
= FunctionFlags::HasJitEntryFlags(isConstructing
);
498 branchTestFunctionFlags(fun
, flags
, Assembler::NonZero
, label
);
501 void MacroAssembler::branchIfScriptHasJitScript(Register script
, Label
* label
) {
502 static_assert(ScriptWarmUpData::JitScriptTag
== 0,
503 "Code below depends on tag value");
504 branchTestPtr(Assembler::Zero
,
505 Address(script
, JSScript::offsetOfWarmUpData()),
506 Imm32(ScriptWarmUpData::TagMask
), label
);
509 void MacroAssembler::branchIfScriptHasNoJitScript(Register script
,
511 static_assert(ScriptWarmUpData::JitScriptTag
== 0,
512 "Code below depends on tag value");
513 static_assert(BaseScript::offsetOfWarmUpData() ==
514 SelfHostedLazyScript::offsetOfWarmUpData(),
515 "SelfHostedLazyScript and BaseScript must use same layout for "
517 branchTestPtr(Assembler::NonZero
,
518 Address(script
, JSScript::offsetOfWarmUpData()),
519 Imm32(ScriptWarmUpData::TagMask
), label
);
522 void MacroAssembler::loadJitScript(Register script
, Register dest
) {
525 branchIfScriptHasJitScript(script
, &ok
);
526 assumeUnreachable("Script has no JitScript!");
530 static_assert(ScriptWarmUpData::JitScriptTag
== 0,
531 "Code below depends on tag value");
532 loadPtr(Address(script
, JSScript::offsetOfWarmUpData()), dest
);
535 void MacroAssembler::loadFunctionArgCount(Register func
, Register output
) {
536 load32(Address(func
, JSFunction::offsetOfFlagsAndArgCount()), output
);
537 rshift32(Imm32(JSFunction::ArgCountShift
), output
);
540 void MacroAssembler::branchIfObjectEmulatesUndefined(Register objReg
,
544 MOZ_ASSERT(objReg
!= scratch
);
547 if (JS::Prefs::use_emulates_undefined_fuse()) {
548 loadPtr(AbsoluteAddress(
549 runtime()->addressOfHasSeenObjectEmulateUndefinedFuse()),
551 branchPtr(Assembler::Equal
, scratch
, ImmPtr(nullptr), &done
);
554 // The branches to out-of-line code here implement a conservative version
555 // of the JSObject::isWrapper test performed in EmulatesUndefined.
556 loadObjClassUnsafe(objReg
, scratch
);
558 branchTestClassIsProxy(true, scratch
, slowCheck
);
560 Address
flags(scratch
, JSClass::offsetOfFlags());
561 branchTest32(Assembler::NonZero
, flags
, Imm32(JSCLASS_EMULATES_UNDEFINED
),
566 void MacroAssembler::branchFunctionKind(Condition cond
,
567 FunctionFlags::FunctionKind kind
,
568 Register fun
, Register scratch
,
570 Address
address(fun
, JSFunction::offsetOfFlagsAndArgCount());
571 load32(address
, scratch
);
572 and32(Imm32(FunctionFlags::FUNCTION_KIND_MASK
), scratch
);
573 branch32(cond
, scratch
, Imm32(kind
), label
);
576 void MacroAssembler::branchTestObjClass(Condition cond
, Register obj
,
577 const JSClass
* clasp
, Register scratch
,
578 Register spectreRegToZero
,
580 MOZ_ASSERT(obj
!= scratch
);
581 MOZ_ASSERT(scratch
!= spectreRegToZero
);
583 loadPtr(Address(obj
, JSObject::offsetOfShape()), scratch
);
584 loadPtr(Address(scratch
, Shape::offsetOfBaseShape()), scratch
);
585 branchPtr(cond
, Address(scratch
, BaseShape::offsetOfClasp()), ImmPtr(clasp
),
588 if (JitOptions
.spectreObjectMitigations
) {
589 spectreZeroRegister(cond
, scratch
, spectreRegToZero
);
593 void MacroAssembler::branchTestObjClassNoSpectreMitigations(
594 Condition cond
, Register obj
, const JSClass
* clasp
, Register scratch
,
596 loadPtr(Address(obj
, JSObject::offsetOfShape()), scratch
);
597 loadPtr(Address(scratch
, Shape::offsetOfBaseShape()), scratch
);
598 branchPtr(cond
, Address(scratch
, BaseShape::offsetOfClasp()), ImmPtr(clasp
),
602 void MacroAssembler::branchTestObjClass(Condition cond
, Register obj
,
603 const Address
& clasp
, Register scratch
,
604 Register spectreRegToZero
,
606 MOZ_ASSERT(obj
!= scratch
);
607 MOZ_ASSERT(scratch
!= spectreRegToZero
);
609 loadObjClassUnsafe(obj
, scratch
);
610 branchPtr(cond
, clasp
, scratch
, label
);
612 if (JitOptions
.spectreObjectMitigations
) {
613 spectreZeroRegister(cond
, scratch
, spectreRegToZero
);
617 void MacroAssembler::branchTestObjClassNoSpectreMitigations(
618 Condition cond
, Register obj
, const Address
& clasp
, Register scratch
,
620 MOZ_ASSERT(obj
!= scratch
);
621 loadObjClassUnsafe(obj
, scratch
);
622 branchPtr(cond
, clasp
, scratch
, label
);
625 void MacroAssembler::branchTestObjClass(Condition cond
, Register obj
,
626 Register clasp
, Register scratch
,
627 Register spectreRegToZero
,
629 MOZ_ASSERT(obj
!= scratch
);
630 MOZ_ASSERT(scratch
!= spectreRegToZero
);
632 loadObjClassUnsafe(obj
, scratch
);
633 branchPtr(cond
, clasp
, scratch
, label
);
635 if (JitOptions
.spectreObjectMitigations
) {
636 spectreZeroRegister(cond
, scratch
, spectreRegToZero
);
640 void MacroAssembler::branchTestClass(
641 Condition cond
, Register clasp
,
642 std::pair
<const JSClass
*, const JSClass
*> classes
, Label
* label
) {
643 MOZ_ASSERT(cond
== Assembler::Equal
|| cond
== Assembler::NotEqual
);
645 if (cond
== Assembler::Equal
) {
646 branchPtr(Assembler::Equal
, clasp
, ImmPtr(classes
.first
), label
);
647 branchPtr(Assembler::Equal
, clasp
, ImmPtr(classes
.second
), label
);
652 branchPtr(Assembler::Equal
, clasp
, ImmPtr(classes
.first
), &isClass
);
653 branchPtr(Assembler::NotEqual
, clasp
, ImmPtr(classes
.second
), label
);
657 void MacroAssembler::branchTestObjClass(
658 Condition cond
, Register obj
,
659 std::pair
<const JSClass
*, const JSClass
*> classes
, Register scratch
,
660 Register spectreRegToZero
, Label
* label
) {
661 MOZ_ASSERT(scratch
!= spectreRegToZero
);
663 branchTestObjClassNoSpectreMitigations(cond
, obj
, classes
, scratch
, label
);
665 if (JitOptions
.spectreObjectMitigations
) {
666 spectreZeroRegister(cond
, scratch
, spectreRegToZero
);
670 void MacroAssembler::branchTestObjClassNoSpectreMitigations(
671 Condition cond
, Register obj
,
672 std::pair
<const JSClass
*, const JSClass
*> classes
, Register scratch
,
674 MOZ_ASSERT(cond
== Assembler::Equal
|| cond
== Assembler::NotEqual
);
675 MOZ_ASSERT(obj
!= scratch
);
677 loadObjClassUnsafe(obj
, scratch
);
678 branchTestClass(cond
, scratch
, classes
, label
);
681 void MacroAssembler::branchTestClassIsFunction(Condition cond
, Register clasp
,
683 return branchTestClass(cond
, clasp
, {&FunctionClass
, &ExtendedFunctionClass
},
687 void MacroAssembler::branchTestObjIsFunction(Condition cond
, Register obj
,
689 Register spectreRegToZero
,
691 MOZ_ASSERT(scratch
!= spectreRegToZero
);
693 branchTestObjIsFunctionNoSpectreMitigations(cond
, obj
, scratch
, label
);
695 if (JitOptions
.spectreObjectMitigations
) {
696 spectreZeroRegister(cond
, scratch
, spectreRegToZero
);
700 void MacroAssembler::branchTestObjIsFunctionNoSpectreMitigations(
701 Condition cond
, Register obj
, Register scratch
, Label
* label
) {
702 MOZ_ASSERT(cond
== Assembler::Equal
|| cond
== Assembler::NotEqual
);
703 MOZ_ASSERT(obj
!= scratch
);
705 loadObjClassUnsafe(obj
, scratch
);
706 branchTestClassIsFunction(cond
, scratch
, label
);
709 void MacroAssembler::branchTestObjShape(Condition cond
, Register obj
,
710 const Shape
* shape
, Register scratch
,
711 Register spectreRegToZero
,
713 MOZ_ASSERT(obj
!= scratch
);
714 MOZ_ASSERT(spectreRegToZero
!= scratch
);
716 if (JitOptions
.spectreObjectMitigations
) {
717 move32(Imm32(0), scratch
);
720 branchPtr(cond
, Address(obj
, JSObject::offsetOfShape()), ImmGCPtr(shape
),
723 if (JitOptions
.spectreObjectMitigations
) {
724 spectreMovePtr(cond
, scratch
, spectreRegToZero
);
728 void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond
,
732 branchPtr(cond
, Address(obj
, JSObject::offsetOfShape()), ImmGCPtr(shape
),
736 void MacroAssembler::branchTestObjShape(Condition cond
, Register obj
,
737 Register shape
, Register scratch
,
738 Register spectreRegToZero
,
740 MOZ_ASSERT(obj
!= scratch
);
741 MOZ_ASSERT(obj
!= shape
);
742 MOZ_ASSERT(spectreRegToZero
!= scratch
);
744 if (JitOptions
.spectreObjectMitigations
) {
745 move32(Imm32(0), scratch
);
748 branchPtr(cond
, Address(obj
, JSObject::offsetOfShape()), shape
, label
);
750 if (JitOptions
.spectreObjectMitigations
) {
751 spectreMovePtr(cond
, scratch
, spectreRegToZero
);
755 void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond
,
759 branchPtr(cond
, Address(obj
, JSObject::offsetOfShape()), shape
, label
);
762 void MacroAssembler::branchTestObjShapeUnsafe(Condition cond
, Register obj
,
763 Register shape
, Label
* label
) {
764 branchTestObjShapeNoSpectreMitigations(cond
, obj
, shape
, label
);
767 void MacroAssembler::branchTestClassIsProxy(bool proxy
, Register clasp
,
769 branchTest32(proxy
? Assembler::NonZero
: Assembler::Zero
,
770 Address(clasp
, JSClass::offsetOfFlags()),
771 Imm32(JSCLASS_IS_PROXY
), label
);
774 void MacroAssembler::branchTestObjectIsProxy(bool proxy
, Register object
,
775 Register scratch
, Label
* label
) {
776 constexpr uint32_t ShiftedMask
= (Shape::kindMask() << Shape::kindShift());
777 static_assert(uint32_t(Shape::Kind::Proxy
) == 0,
778 "branchTest32 below depends on proxy kind being 0");
779 loadPtr(Address(object
, JSObject::offsetOfShape()), scratch
);
780 branchTest32(proxy
? Assembler::Zero
: Assembler::NonZero
,
781 Address(scratch
, Shape::offsetOfImmutableFlags()),
782 Imm32(ShiftedMask
), label
);
785 void MacroAssembler::branchTestProxyHandlerFamily(Condition cond
,
788 const void* handlerp
,
792 branchTestObjectIsProxy(true, proxy
, scratch
, &ok
);
793 assumeUnreachable("Expected ProxyObject in branchTestProxyHandlerFamily");
797 Address
handlerAddr(proxy
, ProxyObject::offsetOfHandler());
798 loadPtr(handlerAddr
, scratch
);
799 Address
familyAddr(scratch
, BaseProxyHandler::offsetOfFamily());
800 branchPtr(cond
, familyAddr
, ImmPtr(handlerp
), label
);
803 void MacroAssembler::branchTestNeedsIncrementalBarrier(Condition cond
,
805 MOZ_ASSERT(cond
== Zero
|| cond
== NonZero
);
806 CompileZone
* zone
= realm()->zone();
807 const uint32_t* needsBarrierAddr
= zone
->addressOfNeedsIncrementalBarrier();
808 branchTest32(cond
, AbsoluteAddress(needsBarrierAddr
), Imm32(0x1), label
);
811 void MacroAssembler::branchTestNeedsIncrementalBarrierAnyZone(
812 Condition cond
, Label
* label
, Register scratch
) {
813 MOZ_ASSERT(cond
== Zero
|| cond
== NonZero
);
815 branchTestNeedsIncrementalBarrier(cond
, label
);
817 // We are compiling the interpreter or another runtime-wide trampoline, so
818 // we have to load cx->zone.
819 loadPtr(AbsoluteAddress(runtime()->addressOfZone()), scratch
);
820 Address
needsBarrierAddr(scratch
, Zone::offsetOfNeedsIncrementalBarrier());
821 branchTest32(cond
, needsBarrierAddr
, Imm32(0x1), label
);
825 void MacroAssembler::branchTestMagicValue(Condition cond
,
826 const ValueOperand
& val
,
827 JSWhyMagic why
, Label
* label
) {
828 MOZ_ASSERT(cond
== Equal
|| cond
== NotEqual
);
829 branchTestValue(cond
, val
, MagicValue(why
), label
);
832 void MacroAssembler::branchDoubleNotInInt64Range(Address src
, Register temp
,
834 using mozilla::FloatingPoint
;
836 // Tests if double is in [INT64_MIN; INT64_MAX] range
837 uint32_t EXPONENT_MASK
= 0x7ff00000;
838 uint32_t EXPONENT_SHIFT
= FloatingPoint
<double>::kExponentShift
- 32;
839 uint32_t TOO_BIG_EXPONENT
= (FloatingPoint
<double>::kExponentBias
+ 63)
842 load32(Address(src
.base
, src
.offset
+ sizeof(int32_t)), temp
);
843 and32(Imm32(EXPONENT_MASK
), temp
);
844 branch32(Assembler::GreaterThanOrEqual
, temp
, Imm32(TOO_BIG_EXPONENT
), fail
);
847 void MacroAssembler::branchDoubleNotInUInt64Range(Address src
, Register temp
,
849 using mozilla::FloatingPoint
;
851 // Note: returns failure on -0.0
852 // Tests if double is in [0; UINT64_MAX] range
853 // Take the sign also in the equation. That way we can compare in one test?
854 uint32_t EXPONENT_MASK
= 0xfff00000;
855 uint32_t EXPONENT_SHIFT
= FloatingPoint
<double>::kExponentShift
- 32;
856 uint32_t TOO_BIG_EXPONENT
= (FloatingPoint
<double>::kExponentBias
+ 64)
859 load32(Address(src
.base
, src
.offset
+ sizeof(int32_t)), temp
);
860 and32(Imm32(EXPONENT_MASK
), temp
);
861 branch32(Assembler::AboveOrEqual
, temp
, Imm32(TOO_BIG_EXPONENT
), fail
);
864 void MacroAssembler::branchFloat32NotInInt64Range(Address src
, Register temp
,
866 using mozilla::FloatingPoint
;
868 // Tests if float is in [INT64_MIN; INT64_MAX] range
869 uint32_t EXPONENT_MASK
= 0x7f800000;
870 uint32_t EXPONENT_SHIFT
= FloatingPoint
<float>::kExponentShift
;
871 uint32_t TOO_BIG_EXPONENT
= (FloatingPoint
<float>::kExponentBias
+ 63)
875 and32(Imm32(EXPONENT_MASK
), temp
);
876 branch32(Assembler::GreaterThanOrEqual
, temp
, Imm32(TOO_BIG_EXPONENT
), fail
);
879 void MacroAssembler::branchFloat32NotInUInt64Range(Address src
, Register temp
,
881 using mozilla::FloatingPoint
;
883 // Note: returns failure on -0.0
884 // Tests if float is in [0; UINT64_MAX] range
885 // Take the sign also in the equation. That way we can compare in one test?
886 uint32_t EXPONENT_MASK
= 0xff800000;
887 uint32_t EXPONENT_SHIFT
= FloatingPoint
<float>::kExponentShift
;
888 uint32_t TOO_BIG_EXPONENT
= (FloatingPoint
<float>::kExponentBias
+ 64)
892 and32(Imm32(EXPONENT_MASK
), temp
);
893 branch32(Assembler::AboveOrEqual
, temp
, Imm32(TOO_BIG_EXPONENT
), fail
);
896 // ========================================================================
897 // Canonicalization primitives.
898 void MacroAssembler::canonicalizeFloat(FloatRegister reg
) {
900 branchFloat(DoubleOrdered
, reg
, reg
, ¬NaN
);
901 loadConstantFloat32(float(JS::GenericNaN()), reg
);
905 void MacroAssembler::canonicalizeFloatIfDeterministic(FloatRegister reg
) {
906 // See the comment in TypedArrayObjectTemplate::getElement.
907 if (js::SupportDifferentialTesting()) {
908 canonicalizeFloat(reg
);
912 void MacroAssembler::canonicalizeDouble(FloatRegister reg
) {
914 branchDouble(DoubleOrdered
, reg
, reg
, ¬NaN
);
915 loadConstantDouble(JS::GenericNaN(), reg
);
919 void MacroAssembler::canonicalizeDoubleIfDeterministic(FloatRegister reg
) {
920 // See the comment in TypedArrayObjectTemplate::getElement.
921 if (js::SupportDifferentialTesting()) {
922 canonicalizeDouble(reg
);
926 // ========================================================================
927 // Memory access primitives.
929 FaultingCodeOffset
MacroAssembler::storeDouble(FloatRegister src
,
931 canonicalizeDoubleIfDeterministic(src
);
932 return storeUncanonicalizedDouble(src
, dest
);
935 template FaultingCodeOffset
MacroAssembler::storeDouble(FloatRegister src
,
936 const Address
& dest
);
937 template FaultingCodeOffset
MacroAssembler::storeDouble(FloatRegister src
,
938 const BaseIndex
& dest
);
941 void MacroAssembler::boxDouble(FloatRegister src
, const T
& dest
) {
942 storeDouble(src
, dest
);
946 FaultingCodeOffset
MacroAssembler::storeFloat32(FloatRegister src
,
948 canonicalizeFloatIfDeterministic(src
);
949 return storeUncanonicalizedFloat32(src
, dest
);
952 template FaultingCodeOffset
MacroAssembler::storeFloat32(FloatRegister src
,
953 const Address
& dest
);
954 template FaultingCodeOffset
MacroAssembler::storeFloat32(FloatRegister src
,
955 const BaseIndex
& dest
);
957 template <typename T
>
958 void MacroAssembler::fallibleUnboxInt32(const T
& src
, Register dest
,
960 // Int32Value can be unboxed efficiently with unboxInt32, so use that.
961 branchTestInt32(Assembler::NotEqual
, src
, fail
);
962 unboxInt32(src
, dest
);
965 template <typename T
>
966 void MacroAssembler::fallibleUnboxBoolean(const T
& src
, Register dest
,
968 // BooleanValue can be unboxed efficiently with unboxBoolean, so use that.
969 branchTestBoolean(Assembler::NotEqual
, src
, fail
);
970 unboxBoolean(src
, dest
);
973 template <typename T
>
974 void MacroAssembler::fallibleUnboxObject(const T
& src
, Register dest
,
976 fallibleUnboxPtr(src
, dest
, JSVAL_TYPE_OBJECT
, fail
);
979 template <typename T
>
980 void MacroAssembler::fallibleUnboxString(const T
& src
, Register dest
,
982 fallibleUnboxPtr(src
, dest
, JSVAL_TYPE_STRING
, fail
);
985 template <typename T
>
986 void MacroAssembler::fallibleUnboxSymbol(const T
& src
, Register dest
,
988 fallibleUnboxPtr(src
, dest
, JSVAL_TYPE_SYMBOL
, fail
);
991 template <typename T
>
992 void MacroAssembler::fallibleUnboxBigInt(const T
& src
, Register dest
,
994 fallibleUnboxPtr(src
, dest
, JSVAL_TYPE_BIGINT
, fail
);
997 //}}} check_macroassembler_style
998 // ===============================================================
1000 #ifndef JS_CODEGEN_ARM64
1002 template <typename T
>
1003 void MacroAssembler::branchTestStackPtr(Condition cond
, T t
, Label
* label
) {
1004 branchTestPtr(cond
, getStackPointer(), t
, label
);
1007 template <typename T
>
1008 void MacroAssembler::branchStackPtr(Condition cond
, T rhs
, Label
* label
) {
1009 branchPtr(cond
, getStackPointer(), rhs
, label
);
1012 template <typename T
>
1013 void MacroAssembler::branchStackPtrRhs(Condition cond
, T lhs
, Label
* label
) {
1014 branchPtr(cond
, lhs
, getStackPointer(), label
);
1017 template <typename T
>
1018 void MacroAssembler::addToStackPtr(T t
) {
1019 addPtr(t
, getStackPointer());
1022 template <typename T
>
1023 void MacroAssembler::addStackPtrTo(T t
) {
1024 addPtr(getStackPointer(), t
);
1027 void MacroAssembler::reserveStack(uint32_t amount
) {
1028 subFromStackPtr(Imm32(amount
));
1029 adjustFrame(amount
);
1031 #endif // !JS_CODEGEN_ARM64
1033 void MacroAssembler::loadObjClassUnsafe(Register obj
, Register dest
) {
1034 loadPtr(Address(obj
, JSObject::offsetOfShape()), dest
);
1035 loadPtr(Address(dest
, Shape::offsetOfBaseShape()), dest
);
1036 loadPtr(Address(dest
, BaseShape::offsetOfClasp()), dest
);
1039 template <typename EmitPreBarrier
>
1040 void MacroAssembler::storeObjShape(Register shape
, Register obj
,
1041 EmitPreBarrier emitPreBarrier
) {
1042 MOZ_ASSERT(shape
!= obj
);
1043 Address
shapeAddr(obj
, JSObject::offsetOfShape());
1044 emitPreBarrier(*this, shapeAddr
);
1045 storePtr(shape
, shapeAddr
);
1048 template <typename EmitPreBarrier
>
1049 void MacroAssembler::storeObjShape(Shape
* shape
, Register obj
,
1050 EmitPreBarrier emitPreBarrier
) {
1051 Address
shapeAddr(obj
, JSObject::offsetOfShape());
1052 emitPreBarrier(*this, shapeAddr
);
1053 storePtr(ImmGCPtr(shape
), shapeAddr
);
1056 void MacroAssembler::loadObjProto(Register obj
, Register dest
) {
1057 loadPtr(Address(obj
, JSObject::offsetOfShape()), dest
);
1058 loadPtr(Address(dest
, Shape::offsetOfBaseShape()), dest
);
1059 loadPtr(Address(dest
, BaseShape::offsetOfProto()), dest
);
1062 void MacroAssembler::loadStringLength(Register str
, Register dest
) {
1063 load32(Address(str
, JSString::offsetOfLength()), dest
);
1066 void MacroAssembler::assertStackAlignment(uint32_t alignment
,
1067 int32_t offset
/* = 0 */) {
1070 MOZ_ASSERT(mozilla::IsPowerOfTwo(alignment
));
1072 // Wrap around the offset to be a non-negative number.
1073 offset
%= alignment
;
1075 offset
+= alignment
;
1078 // Test if each bit from offset is set.
1079 uint32_t off
= offset
;
1081 uint32_t lowestBit
= 1 << mozilla::CountTrailingZeroes32(off
);
1082 branchTestStackPtr(Assembler::Zero
, Imm32(lowestBit
), &bad
);
1086 // Check that all remaining bits are zero.
1087 branchTestStackPtr(Assembler::Zero
, Imm32((alignment
- 1) ^ offset
), &ok
);
1095 void MacroAssembler::storeCallBoolResult(Register reg
) {
1096 convertBoolToInt32(ReturnReg
, reg
);
1099 void MacroAssembler::storeCallInt32Result(Register reg
) {
1100 #if JS_BITS_PER_WORD == 32
1101 storeCallPointerResult(reg
);
1103 // Ensure the upper 32 bits are cleared.
1104 move32(ReturnReg
, reg
);
1108 void MacroAssembler::storeCallResultValue(AnyRegister dest
, JSValueType type
) {
1109 unboxValue(JSReturnOperand
, dest
, type
);
1112 void MacroAssembler::storeCallResultValue(TypedOrValueRegister dest
) {
1113 if (dest
.hasValue()) {
1114 storeCallResultValue(dest
.valueReg());
1116 storeCallResultValue(dest
.typedReg(), ValueTypeFromMIRType(dest
.type()));
1123 #endif /* jit_MacroAssembler_inl_h */