1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifndef jit_MacroAssembler_h
8 #define jit_MacroAssembler_h
10 #include "mozilla/EndianUtils.h"
11 #include "mozilla/MacroForEach.h"
12 #include "mozilla/MathAlgorithms.h"
13 #include "mozilla/Maybe.h"
14 #include "mozilla/Variant.h"
16 #if defined(JS_CODEGEN_X86)
17 # include "jit/x86/MacroAssembler-x86.h"
18 #elif defined(JS_CODEGEN_X64)
19 # include "jit/x64/MacroAssembler-x64.h"
20 #elif defined(JS_CODEGEN_ARM)
21 # include "jit/arm/MacroAssembler-arm.h"
22 #elif defined(JS_CODEGEN_ARM64)
23 # include "jit/arm64/MacroAssembler-arm64.h"
24 #elif defined(JS_CODEGEN_MIPS32)
25 # include "jit/mips32/MacroAssembler-mips32.h"
26 #elif defined(JS_CODEGEN_MIPS64)
27 # include "jit/mips64/MacroAssembler-mips64.h"
28 #elif defined(JS_CODEGEN_LOONG64)
29 # include "jit/loong64/MacroAssembler-loong64.h"
30 #elif defined(JS_CODEGEN_RISCV64)
31 # include "jit/riscv64/MacroAssembler-riscv64.h"
32 #elif defined(JS_CODEGEN_WASM32)
33 # include "jit/wasm32/MacroAssembler-wasm32.h"
34 #elif defined(JS_CODEGEN_NONE)
35 # include "jit/none/MacroAssembler-none.h"
37 # error "Unknown architecture!"
39 #include "jit/ABIArgGenerator.h"
40 #include "jit/ABIFunctions.h"
41 #include "jit/AtomicOp.h"
42 #include "jit/IonTypes.h"
43 #include "jit/MoveResolver.h"
44 #include "jit/VMFunctions.h"
45 #include "js/ScalarType.h" // js::Scalar::Type
46 #include "util/Memory.h"
47 #include "vm/FunctionFlags.h"
48 #include "vm/Opcodes.h"
49 #include "wasm/WasmCodegenTypes.h"
50 #include "wasm/WasmFrame.h"
52 // [SMDOC] MacroAssembler multi-platform overview
54 // * How to read/write MacroAssembler method declarations:
56 // The following macros are made to avoid #ifdef around each method declarations
57 // of the Macro Assembler, and they are also used as an hint on the location of
58 // the implementations of each method. For example, the following declaration
60 // void Pop(FloatRegister t) DEFINED_ON(x86_shared, arm);
62 // suggests the MacroAssembler::Pop(FloatRegister) method is implemented in
63 // x86-shared/MacroAssembler-x86-shared.h, and also in arm/MacroAssembler-arm.h.
65 // - If there is no annotation, then there is only one generic definition in
66 // MacroAssembler.cpp.
68 // - If the declaration is "inline", then the method definition(s) would be in
69 // the "-inl.h" variant of the same file(s).
71 // The script check_macroassembler_style.py (which runs on every build) is
72 // used to verify that method definitions match the annotation on the method
73 // declarations. If there is any difference, then you either forgot to define
74 // the method in one of the macro assembler, or you forgot to update the
75 // annotation of the macro assembler declaration.
77 // Some convenient short-cuts are used to avoid repeating the same list of
78 // architectures on each method declaration, such as PER_ARCH and
81 // Functions that are architecture-agnostic and are the same for all
82 // architectures, that it's necessary to define inline *in this header* to
83 // avoid used-before-defined warnings/errors that would occur if the
84 // definitions were in MacroAssembler-inl.h, should use the OOL_IN_HEADER
85 // marker at end of the declaration:
87 // inline uint32_t framePushed() const OOL_IN_HEADER;
89 // Such functions should then be defined immediately after MacroAssembler's
90 // definition, for example:
92 // //{{{ check_macroassembler_style
94 // MacroAssembler::framePushed() const
96 // return framePushed_;
98 // ////}}} check_macroassembler_style
100 #define ALL_ARCH mips32, mips64, arm, arm64, x86, x64, loong64, riscv64, wasm32
101 #define ALL_SHARED_ARCH \
102 arm, arm64, loong64, riscv64, x86_shared, mips_shared, wasm32
104 // * How this macro works:
106 // DEFINED_ON is a macro which check if, for the current architecture, the
107 // method is defined on the macro assembler or not.
109 // For each architecture, we have a macro named DEFINED_ON_arch. This macro is
110 // empty if this is not the current architecture. Otherwise it must be either
111 // set to "define" or "crash" (only used for the none target so far).
113 // The DEFINED_ON macro maps the list of architecture names given as arguments
114 // to a list of macro names. For example,
116 // DEFINED_ON(arm, x86_shared)
120 // DEFINED_ON_none DEFINED_ON_arm DEFINED_ON_x86_shared
122 // which are later expanded on ARM, x86, x64 by DEFINED_ON_EXPAND_ARCH_RESULTS
127 // or if the JIT is disabled or set to no architecture to
131 // or to nothing, if the current architecture is not listed in the list of
132 // arguments of DEFINED_ON. Note, only one of the DEFINED_ON_arch macro
133 // contributes to the non-empty result, which is the macro of the current
134 // architecture if it is listed in the arguments of DEFINED_ON.
136 // This result is appended to DEFINED_ON_RESULT_ before expanding the macro,
137 // which results in either no annotation, a MOZ_CRASH(), or a "= delete"
138 // annotation on the method declaration.
140 #define DEFINED_ON_x86
141 #define DEFINED_ON_x64
142 #define DEFINED_ON_x86_shared
143 #define DEFINED_ON_arm
144 #define DEFINED_ON_arm64
145 #define DEFINED_ON_mips32
146 #define DEFINED_ON_mips64
147 #define DEFINED_ON_mips_shared
148 #define DEFINED_ON_loong64
149 #define DEFINED_ON_riscv64
150 #define DEFINED_ON_wasm32
151 #define DEFINED_ON_none
153 // Specialize for each architecture.
154 #if defined(JS_CODEGEN_X86)
155 # undef DEFINED_ON_x86
156 # define DEFINED_ON_x86 define
157 # undef DEFINED_ON_x86_shared
158 # define DEFINED_ON_x86_shared define
159 #elif defined(JS_CODEGEN_X64)
160 # undef DEFINED_ON_x64
161 # define DEFINED_ON_x64 define
162 # undef DEFINED_ON_x86_shared
163 # define DEFINED_ON_x86_shared define
164 #elif defined(JS_CODEGEN_ARM)
165 # undef DEFINED_ON_arm
166 # define DEFINED_ON_arm define
167 #elif defined(JS_CODEGEN_ARM64)
168 # undef DEFINED_ON_arm64
169 # define DEFINED_ON_arm64 define
170 #elif defined(JS_CODEGEN_MIPS32)
171 # undef DEFINED_ON_mips32
172 # define DEFINED_ON_mips32 define
173 # undef DEFINED_ON_mips_shared
174 # define DEFINED_ON_mips_shared define
175 #elif defined(JS_CODEGEN_MIPS64)
176 # undef DEFINED_ON_mips64
177 # define DEFINED_ON_mips64 define
178 # undef DEFINED_ON_mips_shared
179 # define DEFINED_ON_mips_shared define
180 #elif defined(JS_CODEGEN_LOONG64)
181 # undef DEFINED_ON_loong64
182 # define DEFINED_ON_loong64 define
183 #elif defined(JS_CODEGEN_RISCV64)
184 # undef DEFINED_ON_riscv64
185 # define DEFINED_ON_riscv64 define
186 #elif defined(JS_CODEGEN_WASM32)
187 # undef DEFINED_ON_wasm32
188 # define DEFINED_ON_wasm32 define
189 #elif defined(JS_CODEGEN_NONE)
190 # undef DEFINED_ON_none
191 # define DEFINED_ON_none crash
193 # error "Unknown architecture!"
196 #define DEFINED_ON_RESULT_crash \
198 #define DEFINED_ON_RESULT_define
199 #define DEFINED_ON_RESULT_ = delete
201 #define DEFINED_ON_DISPATCH_RESULT_2(Macro, Result) Macro##Result
202 #define DEFINED_ON_DISPATCH_RESULT(...) \
203 DEFINED_ON_DISPATCH_RESULT_2(DEFINED_ON_RESULT_, __VA_ARGS__)
205 // We need to let the evaluation of MOZ_FOR_EACH terminates.
206 #define DEFINED_ON_EXPAND_ARCH_RESULTS_3(ParenResult) \
207 DEFINED_ON_DISPATCH_RESULT ParenResult
208 #define DEFINED_ON_EXPAND_ARCH_RESULTS_2(ParenResult) \
209 DEFINED_ON_EXPAND_ARCH_RESULTS_3(ParenResult)
210 #define DEFINED_ON_EXPAND_ARCH_RESULTS(ParenResult) \
211 DEFINED_ON_EXPAND_ARCH_RESULTS_2(ParenResult)
213 #define DEFINED_ON_FWDARCH(Arch) DEFINED_ON_##Arch
214 #define DEFINED_ON_MAP_ON_ARCHS(ArchList) \
215 DEFINED_ON_EXPAND_ARCH_RESULTS( \
216 (MOZ_FOR_EACH(DEFINED_ON_FWDARCH, (), ArchList)))
218 #define DEFINED_ON(...) DEFINED_ON_MAP_ON_ARCHS((none, __VA_ARGS__))
220 #define PER_ARCH DEFINED_ON(ALL_ARCH)
221 #define PER_SHARED_ARCH DEFINED_ON(ALL_SHARED_ARCH)
222 #define OOL_IN_HEADER
225 struct ExpandoAndGeneration
;
231 class TypedArrayObject
;
233 enum class NativeIteratorIndices
: uint32_t;
238 class BytecodeOffset
;
239 class MemoryAccessDesc
;
241 struct ModuleEnvironment
;
243 enum class FailureMode
: uint8_t;
245 enum class SymbolicAddress
;
251 // Defined in JitFrames.h
252 enum class ExitFrameType
: uint8_t;
254 class AutoSaveLiveRegisters
;
256 class TemplateNativeObject
;
257 class TemplateObject
;
259 enum class CheckUnsafeCallWithABI
{
260 // Require the callee to use AutoUnsafeCallWithABI.
263 // We pushed an exit frame so this callWithABI can safely GC and walk the
265 DontCheckHasExitFrame
,
267 // Don't check this callWithABI uses AutoUnsafeCallWithABI, for instance
268 // because we're calling a simple helper function (like malloc or js_free)
269 // that we can't change and/or that we know won't GC.
273 // This is a global function made to create the DynFn type in a controlled
274 // environment which would check if the function signature has been registered
275 // as an ABI function signature.
276 template <typename Sig
>
277 static inline DynFn
DynamicFunction(Sig fun
);
279 enum class CharEncoding
{ Latin1
, TwoByte
};
281 constexpr uint32_t WasmCallerInstanceOffsetBeforeCall
=
282 wasm::FrameWithInstances::callerInstanceOffsetWithoutFrame();
283 constexpr uint32_t WasmCalleeInstanceOffsetBeforeCall
=
284 wasm::FrameWithInstances::calleeInstanceOffsetWithoutFrame();
286 // Allocation sites may be passed to GC thing allocation methods either via a
287 // register (for baseline compilation) or an enum indicating one of the
288 // catch-all allocation sites (for optimized compilation).
289 struct AllocSiteInput
290 : public mozilla::Variant
<Register
, gc::CatchAllAllocSite
> {
291 using Base
= mozilla::Variant
<Register
, gc::CatchAllAllocSite
>;
292 AllocSiteInput() : Base(gc::CatchAllAllocSite::Unknown
) {}
293 explicit AllocSiteInput(gc::CatchAllAllocSite catchAll
) : Base(catchAll
) {}
294 explicit AllocSiteInput(Register reg
) : Base(reg
) {}
297 // [SMDOC] Code generation invariants (incomplete)
299 // ## 64-bit GPRs carrying 32-bit values
301 // At least at the end of every JS or Wasm operation (= SpiderMonkey bytecode or
302 // Wasm bytecode; this is necessarily a little vague), if a 64-bit GPR has a
303 // 32-bit value, then the upper 32 bits of the register may be predictable in
304 // accordance with platform-specific rules, as follows.
306 // - On x64 and arm64, the upper bits are zero
307 // - On mips64 and loongarch64 the upper bits are the sign extension of the
309 // - (On risc-v we have no rule, having no port yet. Sign extension is the most
310 // likely rule, but "unpredictable" is an option.)
312 // In most cases no extra work needs to be done to maintain the invariant:
314 // - 32-bit operations on x64 and arm64 zero-extend the result to 64 bits.
315 // These operations ignore the upper bits of the inputs.
316 // - 32-bit operations on mips64 sign-extend the result to 64 bits (even many
317 // that are labeled as "unsigned", eg ADDU, though not all, eg LU).
318 // Additionally, the inputs to many 32-bit operations must be properly
319 // sign-extended to avoid "unpredictable" behavior, and our simulators check
320 // that inputs conform.
321 // - (32-bit operations on risc-v and loongarch64 sign-extend, much as mips, but
322 // appear to ignore the upper bits of the inputs.)
324 // The upshot of these invariants is, among other things, that:
326 // - No code needs to be generated when a 32-bit value is extended to 64 bits
327 // or a 64-bit value is wrapped to 32 bits, if the upper bits are known to be
328 // correct because they resulted from an operation that produced them
330 // - Literal loads must be careful to avoid instructions that might extend the
331 // literal in the wrong way.
332 // - Code that produces values using intermediate values with non-canonical
333 // extensions must extend according to platform conventions before being
336 // All optimizations are necessarily platform-specific and should only be used
337 // in platform-specific code. We may add architectures in the future that do
338 // not follow the patterns of the few architectures we already have.
340 // Also see MacroAssembler::debugAssertCanonicalInt32().
342 // The public entrypoint for emitting assembly. Note that a MacroAssembler can
343 // use cx->lifoAlloc, so take care not to interleave masm use with other
344 // lifoAlloc use if one will be destroyed before the other.
345 class MacroAssembler
: public MacroAssemblerSpecific
{
347 // Information about the current JSRuntime. This is nullptr only for Wasm
349 CompileRuntime
* maybeRuntime_
= nullptr;
351 // Information about the current Realm. This is nullptr for Wasm compilations
352 // and when compiling JitRuntime trampolines.
353 CompileRealm
* maybeRealm_
= nullptr;
355 // Labels for handling exceptions and failures.
356 NonAssertingLabel failureLabel_
;
359 // Constructor is protected. Use one of the derived classes!
360 explicit MacroAssembler(TempAllocator
& alloc
,
361 CompileRuntime
* maybeRuntime
= nullptr,
362 CompileRealm
* maybeRealm
= nullptr);
365 MoveResolver
& moveResolver() {
366 // As an optimization, the MoveResolver is a persistent data structure
367 // shared between visitors in the CodeGenerator. This assertion
368 // checks that state is not leaking from visitor to visitor
369 // via an unresolved addMove().
370 MOZ_ASSERT(moveResolver_
.hasNoPendingMoves());
371 return moveResolver_
;
374 size_t instructionsSize() const { return size(); }
376 CompileRealm
* realm() const {
377 MOZ_ASSERT(maybeRealm_
);
380 CompileRuntime
* runtime() const {
381 MOZ_ASSERT(maybeRuntime_
);
382 return maybeRuntime_
;
385 #ifdef JS_HAS_HIDDEN_SP
386 void Push(RegisterOrSP reg
);
389 #ifdef ENABLE_WASM_SIMD
390 // `op` should be a shift operation. Return true if a variable-width shift
391 // operation on this architecture should pre-mask the shift count, and if so,
392 // return the mask in `*mask`.
393 static bool MustMaskShiftCountSimd128(wasm::SimdOp op
, int32_t* mask
);
397 // The value returned by GetMaxOffsetGuardLimit() in WasmTypes.h
398 uint32_t wasmMaxOffsetGuardLimit_
;
401 uint32_t wasmMaxOffsetGuardLimit() const { return wasmMaxOffsetGuardLimit_
; }
402 void setWasmMaxOffsetGuardLimit(uint32_t limit
) {
403 wasmMaxOffsetGuardLimit_
= limit
;
406 //{{{ check_macroassembler_decl_style
408 // ===============================================================
409 // MacroAssembler high-level usage.
411 // Flushes the assembly buffer, on platforms that need it.
412 void flush() PER_SHARED_ARCH
;
414 // Add a comment that is visible in the pretty printed assembly code.
415 void comment(const char* msg
) PER_SHARED_ARCH
;
417 // ===============================================================
418 // Frame manipulation functions.
420 inline uint32_t framePushed() const OOL_IN_HEADER
;
421 inline void setFramePushed(uint32_t framePushed
) OOL_IN_HEADER
;
422 inline void adjustFrame(int32_t value
) OOL_IN_HEADER
;
424 // Adjust the frame, to account for implicit modification of the stack
425 // pointer, such that callee can remove arguments on the behalf of the
427 inline void implicitPop(uint32_t bytes
) OOL_IN_HEADER
;
430 // This field is used to statically (at compilation time) emulate a frame
431 // pointer by keeping track of stack manipulations.
433 // It is maintained by all stack manipulation functions below.
434 uint32_t framePushed_
;
437 // ===============================================================
438 // Stack manipulation functions -- sets of registers.
440 // Approximately speaking, the following routines must use the same memory
441 // layout. Any inconsistencies will certainly lead to crashing in generated
444 // MacroAssembler::PushRegsInMaskSizeInBytes
445 // MacroAssembler::PushRegsInMask
446 // MacroAssembler::storeRegsInMask
447 // MacroAssembler::PopRegsInMask
448 // MacroAssembler::PopRegsInMaskIgnore
449 // FloatRegister::getRegisterDumpOffsetInBytes
450 // (no class) PushRegisterDump
451 // (union) RegisterContent
452 // JitRuntime::generateInvalidator
453 // JitRuntime::generateBailoutHandler
454 // JSJitFrameIter::machineState
456 // To be more exact, the invariants are:
458 // * The save area is conceptually viewed as starting at a highest address
459 // (really, at "highest address - 1") and working down to some lower
462 // * PushRegsInMask, storeRegsInMask and PopRegsInMask{Ignore} must use
463 // exactly the same memory layout, when starting from the abovementioned
466 // * PushRegsInMaskSizeInBytes must produce a value which is exactly equal
467 // to the change in the machine's stack pointer register as a result of
468 // calling PushRegsInMask or PopRegsInMask{Ignore}. This value must be at
469 // least uintptr_t-aligned on the target, and may be more aligned than that.
471 // * PushRegsInMaskSizeInBytes must produce a value which is greater than or
472 // equal to the amount of space used by storeRegsInMask.
474 // * Hence, regardless of whether the save area is created with
475 // storeRegsInMask or PushRegsInMask, it is guaranteed to fit inside an
476 // area of size calculated by PushRegsInMaskSizeInBytes.
478 // * For the `ignore` argument of PopRegsInMaskIgnore, equality checking
479 // for the floating point/SIMD registers is done on the basis of the
480 // underlying physical register, regardless of width. For example, if the
481 // to-restore set contains v17 (the SIMD register with encoding 17) and
482 // the ignore set contains d17 (the double register with encoding 17) then
483 // no part of the physical register with encoding 17 will be restored.
484 // (This is probably not true on arm32, since that has aliased float32
485 // registers; but none of our other targets do.)
487 // * {Push,store}RegsInMask/storeRegsInMask are further constrained as
488 // follows: when given the argument AllFloatRegisters, the resulting
489 // memory area must contain exactly all the SIMD/FP registers for the
490 // target at their widest width (that we care about). [We have no targets
491 // where the SIMD registers and FP register sets are disjoint.] They must
492 // be packed end-to-end with no holes, with the register with the lowest
493 // encoding number (0), as returned by FloatRegister::encoding(), at the
494 // abovementioned highest address, register 1 just below that, etc.
496 // Furthermore the sizeof(RegisterContent) must equal the size of a SIMD
497 // register in the abovementioned array.
499 // Furthermore the value returned by
500 // FloatRegister::getRegisterDumpOffsetInBytes must be a correct index
501 // into the abovementioned array. Given the constraints, the only correct
502 // value is `reg.encoding() * sizeof(RegisterContent)`.
504 // Note that some of the routines listed above are JS-only, and do not support
505 // SIMD registers. They are otherwise part of the same equivalence class.
506 // Register spilling for e.g. OOL VM calls is implemented using
507 // PushRegsInMask, and recovered on bailout using machineState. This requires
508 // the same layout to be used in machineState, and therefore in all other code
509 // that can spill registers that are recovered on bailout. Implementations of
510 // JitRuntime::generate{Invalidator,BailoutHandler} should either call
511 // PushRegsInMask, or check carefully to be sure that they generate the same
514 // The size of the area used by PushRegsInMask.
515 size_t PushRegsInMaskSizeInBytes(LiveRegisterSet set
)
516 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
519 void PushRegsInMask(LiveRegisterSet set
)
520 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
522 void PushRegsInMask(LiveGeneralRegisterSet set
);
524 // Like PushRegsInMask, but instead of pushing the registers, store them to
525 // |dest|. |dest| should point to the end of the reserved space, so the
526 // first register will be stored at |dest.offset - sizeof(register)|. It is
527 // required that |dest.offset| is at least as large as the value computed by
528 // PushRegsInMaskSizeInBytes for this |set|. In other words, |dest.base|
529 // must point to either the lowest address in the save area, or some address
531 void storeRegsInMask(LiveRegisterSet set
, Address dest
, Register scratch
)
532 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
535 void PopRegsInMask(LiveRegisterSet set
);
536 void PopRegsInMask(LiveGeneralRegisterSet set
);
537 void PopRegsInMaskIgnore(LiveRegisterSet set
, LiveRegisterSet ignore
)
538 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
541 // ===============================================================
542 // Stack manipulation functions -- single registers/values.
544 void Push(const Operand op
) DEFINED_ON(x86_shared
);
545 void Push(Register reg
) PER_SHARED_ARCH
;
546 void Push(Register reg1
, Register reg2
, Register reg3
, Register reg4
)
548 void Push(const Imm32 imm
) PER_SHARED_ARCH
;
549 void Push(const ImmWord imm
) PER_SHARED_ARCH
;
550 void Push(const ImmPtr imm
) PER_SHARED_ARCH
;
551 void Push(const ImmGCPtr ptr
) PER_SHARED_ARCH
;
552 void Push(FloatRegister reg
) PER_SHARED_ARCH
;
553 void PushBoxed(FloatRegister reg
) PER_ARCH
;
554 void PushFlags() DEFINED_ON(x86_shared
);
555 void Push(PropertyKey key
, Register scratchReg
);
556 void Push(const Address
& addr
);
557 void Push(TypedOrValueRegister v
);
558 void Push(const ConstantOrRegister
& v
);
559 void Push(const ValueOperand
& val
);
560 void Push(const Value
& val
);
561 void Push(JSValueType type
, Register reg
);
562 void Push(const Register64 reg
);
563 void PushEmptyRooted(VMFunctionData::RootType rootType
);
564 inline CodeOffset
PushWithPatch(ImmWord word
);
565 inline CodeOffset
PushWithPatch(ImmPtr imm
);
567 void Pop(const Operand op
) DEFINED_ON(x86_shared
);
568 void Pop(Register reg
) PER_SHARED_ARCH
;
569 void Pop(FloatRegister t
) PER_SHARED_ARCH
;
570 void Pop(const ValueOperand
& val
) PER_SHARED_ARCH
;
571 void PopFlags() DEFINED_ON(x86_shared
);
573 DEFINED_ON(arm
, mips_shared
, x86_shared
, loong64
, riscv64
, wasm32
);
574 void popRooted(VMFunctionData::RootType rootType
, Register cellReg
,
575 const ValueOperand
& valueReg
);
577 // Move the stack pointer based on the requested amount.
578 void adjustStack(int amount
);
579 void freeStack(uint32_t amount
);
581 // Warning: This method does not update the framePushed() counter.
582 void freeStack(Register amount
);
585 // ===============================================================
586 // Register allocation fields.
588 friend AutoRegisterScope
;
589 friend AutoFloatRegisterScope
;
590 // Used to track register scopes for debug builds.
591 // Manipulated by the AutoGenericRegisterScope class.
592 AllocatableRegisterSet debugTrackedRegisters_
;
596 // ===============================================================
597 // Simple call functions.
599 // The returned CodeOffset is the assembler offset for the instruction
600 // immediately following the call; that is, for the return point.
601 CodeOffset
call(Register reg
) PER_SHARED_ARCH
;
602 CodeOffset
call(Label
* label
) PER_SHARED_ARCH
;
604 void call(const Address
& addr
) PER_SHARED_ARCH
;
605 void call(ImmWord imm
) PER_SHARED_ARCH
;
606 // Call a target native function, which is neither traceable nor movable.
607 void call(ImmPtr imm
) PER_SHARED_ARCH
;
608 CodeOffset
call(wasm::SymbolicAddress imm
) PER_SHARED_ARCH
;
609 inline CodeOffset
call(const wasm::CallSiteDesc
& desc
,
610 wasm::SymbolicAddress imm
);
612 // Call a target JitCode, which must be traceable, and may be movable.
613 void call(JitCode
* c
) PER_SHARED_ARCH
;
615 inline void call(TrampolinePtr code
);
617 inline CodeOffset
call(const wasm::CallSiteDesc
& desc
, const Register reg
);
618 inline CodeOffset
call(const wasm::CallSiteDesc
& desc
, uint32_t funcDefIndex
);
619 inline void call(const wasm::CallSiteDesc
& desc
, wasm::Trap trap
);
621 CodeOffset
callWithPatch() PER_SHARED_ARCH
;
622 void patchCall(uint32_t callerOffset
, uint32_t calleeOffset
) PER_SHARED_ARCH
;
624 // Push the return address and make a call. On platforms where this function
625 // is not defined, push the link register (pushReturnAddress) at the entry
626 // point of the callee.
627 void callAndPushReturnAddress(Register reg
) DEFINED_ON(x86_shared
);
628 void callAndPushReturnAddress(Label
* label
) DEFINED_ON(x86_shared
);
630 // These do not adjust framePushed().
631 void pushReturnAddress()
632 DEFINED_ON(mips_shared
, arm
, arm64
, loong64
, riscv64
, wasm32
);
633 void popReturnAddress()
634 DEFINED_ON(mips_shared
, arm
, arm64
, loong64
, riscv64
, wasm32
);
636 // Useful for dealing with two-valued returns.
637 void moveRegPair(Register src0
, Register src1
, Register dst0
, Register dst1
,
638 MoveOp::Type type
= MoveOp::GENERAL
);
641 // ===============================================================
642 // Patchable near/far jumps.
644 // "Far jumps" provide the ability to jump to any uint32_t offset from any
645 // other uint32_t offset without using a constant pool (thus returning a
646 // simple CodeOffset instead of a CodeOffsetJump).
647 CodeOffset
farJumpWithPatch() PER_SHARED_ARCH
;
648 void patchFarJump(CodeOffset farJump
, uint32_t targetOffset
) PER_SHARED_ARCH
;
650 // Emit a nop that can be patched to and from a nop and a call with int32
651 // relative displacement.
652 CodeOffset
nopPatchableToCall() PER_SHARED_ARCH
;
653 void nopPatchableToCall(const wasm::CallSiteDesc
& desc
);
654 static void patchNopToCall(uint8_t* callsite
,
655 uint8_t* target
) PER_SHARED_ARCH
;
656 static void patchCallToNop(uint8_t* callsite
) PER_SHARED_ARCH
;
658 // These methods are like movWithPatch/PatchDataWithValueCheck but allow
659 // using pc-relative addressing on certain platforms (RIP-relative LEA on x64,
660 // ADR instruction on arm64).
662 // Note: "Near" applies to ARM64 where the target must be within 1 MB (this is
663 // release-asserted).
664 CodeOffset
moveNearAddressWithPatch(Register dest
) PER_ARCH
;
665 static void patchNearAddressMove(CodeLocationLabel loc
,
666 CodeLocationLabel target
)
667 DEFINED_ON(x86
, x64
, arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
);
670 // ===============================================================
671 // [SMDOC] JIT-to-C++ Function Calls (callWithABI)
673 // callWithABI is used to make a call using the standard C/C++ system ABI.
675 // callWithABI is a low level interface for making calls, as such every call
676 // made with callWithABI should be organized with 6 steps: spilling live
677 // registers, aligning the stack, listing arguments of the called function,
678 // calling a function pointer, extracting the returned value and restoring
681 // A more detailed example of the six stages:
683 // 1) Saving of registers that are live. This will vary depending on which
684 // SpiderMonkey compiler you are working on. Registers that shouldn't be
685 // restored can be excluded.
687 // LiveRegisterSet volatileRegs(...);
688 // volatileRegs.take(scratch);
689 // masm.PushRegsInMask(volatileRegs);
691 // 2) Align the stack to perform the call with the correct stack alignment.
693 // When the stack pointer alignment is unknown and cannot be corrected
694 // when generating the code, setupUnalignedABICall must be used to
695 // dynamically align the stack pointer to the expectation of the ABI.
696 // When the stack pointer is known at JIT compilation time, the stack can
697 // be fixed manually and setupAlignedABICall and setupWasmABICall can be
700 // setupWasmABICall is a special case of setupAlignedABICall as
701 // SpiderMonkey's WebAssembly implementation mostly follow the system
702 // ABI, except for float/double arguments, which always use floating
703 // point registers, even if this is not supported by the system ABI.
705 // masm.setupUnalignedABICall(scratch);
707 // 3) Passing arguments. Arguments are passed left-to-right.
709 // masm.passABIArg(scratch);
710 // masm.passABIArg(FloatOp0, MoveOp::Double);
712 // Note how float register arguments are annotated with MoveOp::Double.
714 // Concerning stack-relative address, see the note on passABIArg.
718 // using Fn = int32_t (*)(int32_t)
719 // masm.callWithABI<Fn, Callee>();
721 // In the case where the call returns a double, that needs to be
722 // indicated to the callWithABI like this:
724 // using Fn = double (*)(int32_t)
725 // masm.callWithABI<Fn, Callee>(MoveOp::DOUBLE);
727 // There are overloads to allow calls to registers and addresses.
729 // 5) Take care of the result
731 // masm.storeCallPointerResult(scratch1);
732 // masm.storeCallBoolResult(scratch1);
733 // masm.storeCallInt32Result(scratch1);
734 // masm.storeCallFloatResult(scratch1);
736 // 6) Restore the potentially clobbered volatile registers
738 // masm.PopRegsInMask(volatileRegs);
740 // If expecting a returned value, this call should use
741 // PopRegsInMaskIgnore to filter out the registers which are containing
742 // the returned value.
744 // Unless an exit frame is pushed prior to the setupABICall, the callee
745 // should not GC. To ensure this is the case callWithABI is instrumented to
746 // make sure that in the default case callees are annotated with an
747 // AutoUnsafeCallWithABI on the stack.
749 // A callWithABI can opt out of checking, if for example it is known there
750 // is an exit frame, or the callee is known not to GC.
752 // If your callee needs to be able to GC, consider using a VMFunction, or
753 // create a fake exit frame, and instrument the TraceJitExitFrame
756 // Setup a call to C/C++ code, given the assumption that the framePushed
757 // accurately defines the state of the stack, and that the top of the stack
758 // was properly aligned. Note that this only supports cdecl.
760 // As a rule of thumb, this can be used in CodeGenerator but not in CacheIR or
761 // Baseline code (because the stack is not aligned to ABIStackAlignment).
762 void setupAlignedABICall();
764 // As setupAlignedABICall, but for WebAssembly native ABI calls, which pass
765 // through a builtin thunk that uses the wasm ABI. All the wasm ABI calls
766 // can be native, since we always know the stack alignment a priori.
767 void setupWasmABICall();
769 // Setup an ABI call for when the alignment is not known. This may need a
771 void setupUnalignedABICall(Register scratch
) PER_ARCH
;
773 // Arguments must be assigned to a C/C++ call in order. They are moved
774 // in parallel immediately before performing the call. This process may
775 // temporarily use more stack, in which case esp-relative addresses will be
776 // automatically adjusted. It is extremely important that esp-relative
777 // addresses are computed *after* setupABICall(). Furthermore, no
778 // operations should be emitted while setting arguments.
779 void passABIArg(const MoveOperand
& from
, MoveOp::Type type
);
780 inline void passABIArg(Register reg
);
781 inline void passABIArg(FloatRegister reg
, MoveOp::Type type
);
783 inline void callWithABI(
784 DynFn fun
, MoveOp::Type result
= MoveOp::GENERAL
,
785 CheckUnsafeCallWithABI check
= CheckUnsafeCallWithABI::Check
);
786 template <typename Sig
, Sig fun
>
787 inline void callWithABI(
788 MoveOp::Type result
= MoveOp::GENERAL
,
789 CheckUnsafeCallWithABI check
= CheckUnsafeCallWithABI::Check
);
790 inline void callWithABI(Register fun
, MoveOp::Type result
= MoveOp::GENERAL
);
791 inline void callWithABI(const Address
& fun
,
792 MoveOp::Type result
= MoveOp::GENERAL
);
794 CodeOffset
callWithABI(wasm::BytecodeOffset offset
, wasm::SymbolicAddress fun
,
795 mozilla::Maybe
<int32_t> instanceOffset
,
796 MoveOp::Type result
= MoveOp::GENERAL
);
797 void callDebugWithABI(wasm::SymbolicAddress fun
,
798 MoveOp::Type result
= MoveOp::GENERAL
);
801 // Reinitialize the variables which have to be cleared before making a call
803 template <class ABIArgGeneratorT
>
804 void setupABICallHelper();
806 // Reinitialize the variables which have to be cleared before making a call
808 void setupNativeABICall();
810 // Reserve the stack and resolve the arguments move.
811 void callWithABIPre(uint32_t* stackAdjust
,
812 bool callFromWasm
= false) PER_ARCH
;
814 // Emits a call to a C/C++ function, resolving all argument moves.
815 void callWithABINoProfiler(void* fun
, MoveOp::Type result
,
816 CheckUnsafeCallWithABI check
);
817 void callWithABINoProfiler(Register fun
, MoveOp::Type result
) PER_ARCH
;
818 void callWithABINoProfiler(const Address
& fun
, MoveOp::Type result
) PER_ARCH
;
820 // Restore the stack to its state before the setup function call.
821 void callWithABIPost(uint32_t stackAdjust
, MoveOp::Type result
,
822 bool callFromWasm
= false) PER_ARCH
;
824 // Create the signature to be able to decode the arguments of a native
825 // function, when calling a function within the simulator.
826 inline void appendSignatureType(MoveOp::Type type
);
827 inline ABIFunctionType
signature() const;
829 // Private variables used to handle moves between registers given as
830 // arguments to passABIArg and the list of ABI registers expected for the
831 // signature of the function.
832 MoveResolver moveResolver_
;
834 // Architecture specific implementation which specify how registers & stack
835 // offsets are used for calling a function.
836 ABIArgGenerator abiArgs_
;
839 // Flag use to assert that we use ABI function in the right context.
843 // If set by setupUnalignedABICall then callWithABI will pop the stack
844 // register which is on the stack.
845 bool dynamicAlignment_
;
848 // The signature is used to accumulate all types of arguments which are used
849 // by the caller. This is used by the simulators to decode the arguments
850 // properly, and cast the function pointer to the right type.
855 // ===============================================================
858 // These functions are used to build the content of the Jit frames. See
859 // CommonFrameLayout class, and all its derivatives. The content should be
860 // pushed in the opposite order as the fields of the structures, such that
861 // the structures can be used to interpret the content of the stack.
863 // Call the Jit function, and push the return address (or let the callee
864 // push the return address).
866 // These functions return the offset of the return address, in order to use
867 // the return address to index the safepoints, which are used to list all
869 inline uint32_t callJitNoProfiler(Register callee
);
870 inline uint32_t callJit(Register callee
);
871 inline uint32_t callJit(JitCode
* code
);
872 inline uint32_t callJit(TrampolinePtr code
);
873 inline uint32_t callJit(ImmPtr callee
);
875 // The frame descriptor is the second field of all Jit frames, pushed before
876 // calling the Jit function. See CommonFrameLayout::descriptor_.
877 inline void pushFrameDescriptor(FrameType type
);
878 inline void PushFrameDescriptor(FrameType type
);
880 // For JitFrameLayout, the descriptor also stores the number of arguments
881 // passed by the caller. See MakeFrameDescriptorForJitCall.
882 inline void pushFrameDescriptorForJitCall(FrameType type
, uint32_t argc
);
883 inline void pushFrameDescriptorForJitCall(FrameType type
, Register argc
,
885 inline void PushFrameDescriptorForJitCall(FrameType type
, uint32_t argc
);
886 inline void PushFrameDescriptorForJitCall(FrameType type
, Register argc
,
889 // Load the number of actual arguments from the frame's JitFrameLayout.
890 inline void loadNumActualArgs(Register framePtr
, Register dest
);
892 // Push the callee token of a JSFunction which pointer is stored in the
893 // |callee| register. The callee token is packed with a |constructing| flag
894 // which correspond to the fact that the JS function is called with "new" or
896 inline void PushCalleeToken(Register callee
, bool constructing
);
898 // Unpack a callee token located at the |token| address, and return the
899 // JSFunction pointer in the |dest| register.
900 inline void loadFunctionFromCalleeToken(Address token
, Register dest
);
902 // This function emulates a call by pushing an exit frame on the stack,
903 // except that the fake-function is inlined within the body of the caller.
905 // This function assumes that the current frame is an IonJS frame.
907 // This function returns the offset of the /fake/ return address, in order to
908 // use the return address to index the safepoints, which are used to list all
911 // This function should be balanced with a call to adjustStack, to pop the
912 // exit frame and emulate the return statement of the inlined function.
913 inline uint32_t buildFakeExitFrame(Register scratch
);
916 // This function is used by buildFakeExitFrame to push a fake return address
917 // on the stack. This fake return address should never be used for resuming
918 // any execution, and can even be an invalid pointer into the instruction
919 // stream, as long as it does not alias any other.
920 uint32_t pushFakeReturnAddress(Register scratch
) PER_SHARED_ARCH
;
923 // ===============================================================
924 // Exit frame footer.
926 // When calling outside the Jit we push an exit frame. To mark the stack
927 // correctly, we have to push additional information, called the Exit frame
928 // footer, which is used to identify how the stack is marked.
930 // See JitFrames.h, and TraceJitExitFrame in JitFrames.cpp.
932 // Push stub code and the VMFunctionData pointer.
933 inline void enterExitFrame(Register cxreg
, Register scratch
,
934 const VMFunctionData
* f
);
936 // Push an exit frame token to identify which fake exit frame this footer
938 inline void enterFakeExitFrame(Register cxreg
, Register scratch
,
941 // Push an exit frame token for a native call.
942 inline void enterFakeExitFrameForNative(Register cxreg
, Register scratch
,
943 bool isConstructing
);
945 // Pop ExitFrame footer in addition to the extra frame.
946 inline void leaveExitFrame(size_t extraFrame
= 0);
949 // Save the top of the stack into JitActivation::packedExitFP of the
950 // current thread, which should be the location of the latest exit frame.
951 void linkExitFrame(Register cxreg
, Register scratch
);
954 // ===============================================================
957 inline void move64(Imm64 imm
, Register64 dest
) PER_ARCH
;
958 inline void move64(Register64 src
, Register64 dest
) PER_ARCH
;
960 inline void moveFloat32ToGPR(FloatRegister src
,
961 Register dest
) PER_SHARED_ARCH
;
962 inline void moveGPRToFloat32(Register src
,
963 FloatRegister dest
) PER_SHARED_ARCH
;
965 inline void moveDoubleToGPR64(FloatRegister src
, Register64 dest
) PER_ARCH
;
966 inline void moveGPR64ToDouble(Register64 src
, FloatRegister dest
) PER_ARCH
;
968 inline void move8SignExtend(Register src
, Register dest
) PER_SHARED_ARCH
;
969 inline void move16SignExtend(Register src
, Register dest
) PER_SHARED_ARCH
;
971 // move64To32 will clear the high bits of `dest` on 64-bit systems.
972 inline void move64To32(Register64 src
, Register dest
) PER_ARCH
;
974 inline void move32To64ZeroExtend(Register src
, Register64 dest
) PER_ARCH
;
976 inline void move8To64SignExtend(Register src
, Register64 dest
) PER_ARCH
;
977 inline void move16To64SignExtend(Register src
, Register64 dest
) PER_ARCH
;
978 inline void move32To64SignExtend(Register src
, Register64 dest
) PER_ARCH
;
980 inline void move32SignExtendToPtr(Register src
, Register dest
) PER_ARCH
;
981 inline void move32ZeroExtendToPtr(Register src
, Register dest
) PER_ARCH
;
983 // Copy a constant, typed-register, or a ValueOperand into a ValueOperand
985 inline void moveValue(const ConstantOrRegister
& src
,
986 const ValueOperand
& dest
);
987 void moveValue(const TypedOrValueRegister
& src
,
988 const ValueOperand
& dest
) PER_ARCH
;
989 void moveValue(const ValueOperand
& src
, const ValueOperand
& dest
) PER_ARCH
;
990 void moveValue(const Value
& src
, const ValueOperand
& dest
) PER_ARCH
;
992 void movePropertyKey(PropertyKey key
, Register dest
);
994 // ===============================================================
997 inline void load32SignExtendToPtr(const Address
& src
, Register dest
) PER_ARCH
;
999 inline void loadAbiReturnAddress(Register dest
) PER_SHARED_ARCH
;
1002 // ===============================================================
1003 // Logical instructions
1005 inline void not32(Register reg
) PER_SHARED_ARCH
;
1006 inline void notPtr(Register reg
) PER_ARCH
;
1008 inline void and32(Register src
, Register dest
) PER_SHARED_ARCH
;
1009 inline void and32(Imm32 imm
, Register dest
) PER_SHARED_ARCH
;
1010 inline void and32(Imm32 imm
, Register src
, Register dest
) DEFINED_ON(arm64
);
1011 inline void and32(Imm32 imm
, const Address
& dest
) PER_SHARED_ARCH
;
1012 inline void and32(const Address
& src
, Register dest
) PER_SHARED_ARCH
;
1014 inline void andPtr(Register src
, Register dest
) PER_ARCH
;
1015 inline void andPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1017 inline void and64(Imm64 imm
, Register64 dest
) PER_ARCH
;
1018 inline void or64(Imm64 imm
, Register64 dest
) PER_ARCH
;
1019 inline void xor64(Imm64 imm
, Register64 dest
) PER_ARCH
;
1021 inline void or32(Register src
, Register dest
) PER_SHARED_ARCH
;
1022 inline void or32(Imm32 imm
, Register dest
) PER_SHARED_ARCH
;
1023 inline void or32(Imm32 imm
, const Address
& dest
) PER_SHARED_ARCH
;
1025 inline void orPtr(Register src
, Register dest
) PER_ARCH
;
1026 inline void orPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1028 inline void and64(Register64 src
, Register64 dest
) PER_ARCH
;
1029 inline void or64(Register64 src
, Register64 dest
) PER_ARCH
;
1030 inline void xor64(Register64 src
, Register64 dest
) PER_ARCH
;
1032 inline void xor32(Register src
, Register dest
) PER_SHARED_ARCH
;
1033 inline void xor32(Imm32 imm
, Register dest
) PER_SHARED_ARCH
;
1034 inline void xor32(Imm32 imm
, const Address
& dest
) PER_SHARED_ARCH
;
1035 inline void xor32(const Address
& src
, Register dest
) PER_SHARED_ARCH
;
1037 inline void xorPtr(Register src
, Register dest
) PER_ARCH
;
1038 inline void xorPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1040 inline void and64(const Operand
& src
, Register64 dest
)
1041 DEFINED_ON(x64
, mips64
, loong64
, riscv64
);
1042 inline void or64(const Operand
& src
, Register64 dest
)
1043 DEFINED_ON(x64
, mips64
, loong64
, riscv64
);
1044 inline void xor64(const Operand
& src
, Register64 dest
)
1045 DEFINED_ON(x64
, mips64
, loong64
, riscv64
);
1047 // ===============================================================
1048 // Swap instructions
1050 // Swap the two lower bytes and sign extend the result to 32-bit.
1051 inline void byteSwap16SignExtend(Register reg
) PER_SHARED_ARCH
;
1053 // Swap the two lower bytes and zero extend the result to 32-bit.
1054 inline void byteSwap16ZeroExtend(Register reg
) PER_SHARED_ARCH
;
1056 // Swap all four bytes in a 32-bit integer.
1057 inline void byteSwap32(Register reg
) PER_SHARED_ARCH
;
1059 // Swap all eight bytes in a 64-bit integer.
1060 inline void byteSwap64(Register64 reg
) PER_ARCH
;
1062 // ===============================================================
1063 // Arithmetic functions
1065 // Condition flags aren't guaranteed to be set by these functions, for example
1066 // x86 will always set condition flags, but ARM64 won't do it unless
1067 // explicitly requested. Instead use branch(Add|Sub|Mul|Neg) to test for
1068 // condition flags after performing arithmetic operations.
1070 inline void add32(Register src
, Register dest
) PER_SHARED_ARCH
;
1071 inline void add32(Imm32 imm
, Register dest
) PER_SHARED_ARCH
;
1072 inline void add32(Imm32 imm
, const Address
& dest
) PER_SHARED_ARCH
;
1073 inline void add32(Imm32 imm
, const AbsoluteAddress
& dest
)
1074 DEFINED_ON(x86_shared
);
1076 inline void addPtr(Register src
, Register dest
) PER_ARCH
;
1077 inline void addPtr(Register src1
, Register src2
, Register dest
)
1079 inline void addPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1080 inline void addPtr(Imm32 imm
, Register src
, Register dest
) DEFINED_ON(arm64
);
1081 inline void addPtr(ImmWord imm
, Register dest
) PER_ARCH
;
1082 inline void addPtr(ImmPtr imm
, Register dest
);
1083 inline void addPtr(Imm32 imm
, const Address
& dest
)
1084 DEFINED_ON(mips_shared
, arm
, arm64
, x86
, x64
, loong64
, riscv64
, wasm32
);
1085 inline void addPtr(Imm32 imm
, const AbsoluteAddress
& dest
)
1086 DEFINED_ON(x86
, x64
);
1087 inline void addPtr(const Address
& src
, Register dest
)
1088 DEFINED_ON(mips_shared
, arm
, arm64
, x86
, x64
, loong64
, riscv64
, wasm32
);
1090 inline void add64(Register64 src
, Register64 dest
) PER_ARCH
;
1091 inline void add64(Imm32 imm
, Register64 dest
) PER_ARCH
;
1092 inline void add64(Imm64 imm
, Register64 dest
) PER_ARCH
;
1093 inline void add64(const Operand
& src
, Register64 dest
)
1094 DEFINED_ON(x64
, mips64
, loong64
, riscv64
);
1096 inline void addFloat32(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1098 // Compute dest=SP-imm where dest is a pointer registers and not SP. The
1099 // offset returned from sub32FromStackPtrWithPatch() must be passed to
1100 // patchSub32FromStackPtr().
1101 inline CodeOffset
sub32FromStackPtrWithPatch(Register dest
) PER_ARCH
;
1102 inline void patchSub32FromStackPtr(CodeOffset offset
, Imm32 imm
) PER_ARCH
;
1104 inline void addDouble(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1105 inline void addConstantDouble(double d
, FloatRegister dest
) DEFINED_ON(x86
);
1107 inline void sub32(const Address
& src
, Register dest
) PER_SHARED_ARCH
;
1108 inline void sub32(Register src
, Register dest
) PER_SHARED_ARCH
;
1109 inline void sub32(Imm32 imm
, Register dest
) PER_SHARED_ARCH
;
1111 inline void subPtr(Register src
, Register dest
) PER_ARCH
;
1112 inline void subPtr(Register src
, const Address
& dest
)
1113 DEFINED_ON(mips_shared
, arm
, arm64
, x86
, x64
, loong64
, riscv64
, wasm32
);
1114 inline void subPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1115 inline void subPtr(ImmWord imm
, Register dest
) DEFINED_ON(x64
);
1116 inline void subPtr(const Address
& addr
, Register dest
)
1117 DEFINED_ON(mips_shared
, arm
, arm64
, x86
, x64
, loong64
, riscv64
, wasm32
);
1119 inline void sub64(Register64 src
, Register64 dest
) PER_ARCH
;
1120 inline void sub64(Imm64 imm
, Register64 dest
) PER_ARCH
;
1121 inline void sub64(const Operand
& src
, Register64 dest
)
1122 DEFINED_ON(x64
, mips64
, loong64
, riscv64
);
1124 inline void subFloat32(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1126 inline void subDouble(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1128 inline void mul32(Register rhs
, Register srcDest
) PER_SHARED_ARCH
;
1129 inline void mul32(Imm32 imm
, Register srcDest
) PER_SHARED_ARCH
;
1131 inline void mul32(Register src1
, Register src2
, Register dest
, Label
* onOver
)
1134 // Return the high word of the unsigned multiplication into |dest|.
1135 inline void mulHighUnsigned32(Imm32 imm
, Register src
,
1136 Register dest
) PER_ARCH
;
1138 inline void mulPtr(Register rhs
, Register srcDest
) PER_ARCH
;
1140 inline void mul64(const Operand
& src
, const Register64
& dest
) DEFINED_ON(x64
);
1141 inline void mul64(const Operand
& src
, const Register64
& dest
,
1142 const Register temp
)
1143 DEFINED_ON(x64
, mips64
, loong64
, riscv64
);
1144 inline void mul64(Imm64 imm
, const Register64
& dest
) PER_ARCH
;
1145 inline void mul64(Imm64 imm
, const Register64
& dest
, const Register temp
)
1146 DEFINED_ON(x86
, x64
, arm
, mips32
, mips64
, loong64
, riscv64
);
1147 inline void mul64(const Register64
& src
, const Register64
& dest
,
1148 const Register temp
) PER_ARCH
;
1149 inline void mul64(const Register64
& src1
, const Register64
& src2
,
1150 const Register64
& dest
) DEFINED_ON(arm64
);
1151 inline void mul64(Imm64 src1
, const Register64
& src2
, const Register64
& dest
)
1154 inline void mulBy3(Register src
, Register dest
) PER_ARCH
;
1156 inline void mulFloat32(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1157 inline void mulDouble(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1159 inline void mulDoublePtr(ImmPtr imm
, Register temp
, FloatRegister dest
)
1160 DEFINED_ON(mips_shared
, arm
, arm64
, x86
, x64
, loong64
, riscv64
, wasm32
);
1162 // Perform an integer division, returning the integer part rounded toward
1163 // zero. rhs must not be zero, and the division must not overflow.
1165 // On ARM, the chip must have hardware division instructions.
1166 inline void quotient32(Register rhs
, Register srcDest
, bool isUnsigned
)
1167 DEFINED_ON(mips_shared
, arm
, arm64
, loong64
, riscv64
, wasm32
);
1169 // As above, but srcDest must be eax and tempEdx must be edx.
1170 inline void quotient32(Register rhs
, Register srcDest
, Register tempEdx
,
1171 bool isUnsigned
) DEFINED_ON(x86_shared
);
1173 // Perform an integer division, returning the remainder part.
1174 // rhs must not be zero, and the division must not overflow.
1176 // On ARM, the chip must have hardware division instructions.
1177 inline void remainder32(Register rhs
, Register srcDest
, bool isUnsigned
)
1178 DEFINED_ON(mips_shared
, arm
, arm64
, loong64
, riscv64
, wasm32
);
1180 // As above, but srcDest must be eax and tempEdx must be edx.
1181 inline void remainder32(Register rhs
, Register srcDest
, Register tempEdx
,
1182 bool isUnsigned
) DEFINED_ON(x86_shared
);
1184 // Perform an integer division, returning the integer part rounded toward
1185 // zero. rhs must not be zero, and the division must not overflow.
1187 // This variant preserves registers, and doesn't require hardware division
1188 // instructions on ARM (will call out to a runtime routine).
1190 // rhs is preserved, srdDest is clobbered.
1191 void flexibleRemainder32(Register rhs
, Register srcDest
, bool isUnsigned
,
1192 const LiveRegisterSet
& volatileLiveRegs
)
1193 DEFINED_ON(mips_shared
, arm
, arm64
, x86_shared
, loong64
, riscv64
, wasm32
);
1195 // Perform an integer division, returning the integer part rounded toward
1196 // zero. rhs must not be zero, and the division must not overflow.
1198 // This variant preserves registers, and doesn't require hardware division
1199 // instructions on ARM (will call out to a runtime routine).
1201 // rhs is preserved, srdDest is clobbered.
1202 void flexibleQuotient32(Register rhs
, Register srcDest
, bool isUnsigned
,
1203 const LiveRegisterSet
& volatileLiveRegs
)
1204 DEFINED_ON(mips_shared
, arm
, arm64
, x86_shared
, loong64
, riscv64
);
1206 // Perform an integer division, returning the integer part rounded toward
1207 // zero. rhs must not be zero, and the division must not overflow. The
1208 // remainder is stored into the third argument register here.
1210 // This variant preserves registers, and doesn't require hardware division
1211 // instructions on ARM (will call out to a runtime routine).
1213 // rhs is preserved, srdDest and remOutput are clobbered.
1214 void flexibleDivMod32(Register rhs
, Register srcDest
, Register remOutput
,
1216 const LiveRegisterSet
& volatileLiveRegs
)
1217 DEFINED_ON(mips_shared
, arm
, arm64
, x86_shared
, loong64
, riscv64
, wasm32
);
1219 inline void divFloat32(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1220 inline void divDouble(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1222 inline void inc64(AbsoluteAddress dest
) PER_ARCH
;
1224 inline void neg32(Register reg
) PER_SHARED_ARCH
;
1225 inline void neg64(Register64 reg
) PER_ARCH
;
1226 inline void negPtr(Register reg
) PER_ARCH
;
1228 inline void negateFloat(FloatRegister reg
) PER_SHARED_ARCH
;
1230 inline void negateDouble(FloatRegister reg
) PER_SHARED_ARCH
;
1232 inline void abs32(Register src
, Register dest
) PER_SHARED_ARCH
;
1233 inline void absFloat32(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1234 inline void absDouble(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1236 inline void sqrtFloat32(FloatRegister src
,
1237 FloatRegister dest
) PER_SHARED_ARCH
;
1238 inline void sqrtDouble(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1240 void floorFloat32ToInt32(FloatRegister src
, Register dest
,
1241 Label
* fail
) PER_SHARED_ARCH
;
1242 void floorDoubleToInt32(FloatRegister src
, Register dest
,
1243 Label
* fail
) PER_SHARED_ARCH
;
1245 void ceilFloat32ToInt32(FloatRegister src
, Register dest
,
1246 Label
* fail
) PER_SHARED_ARCH
;
1247 void ceilDoubleToInt32(FloatRegister src
, Register dest
,
1248 Label
* fail
) PER_SHARED_ARCH
;
1250 void roundFloat32ToInt32(FloatRegister src
, Register dest
, FloatRegister temp
,
1251 Label
* fail
) PER_SHARED_ARCH
;
1252 void roundDoubleToInt32(FloatRegister src
, Register dest
, FloatRegister temp
,
1253 Label
* fail
) PER_SHARED_ARCH
;
1255 void truncFloat32ToInt32(FloatRegister src
, Register dest
,
1256 Label
* fail
) PER_SHARED_ARCH
;
1257 void truncDoubleToInt32(FloatRegister src
, Register dest
,
1258 Label
* fail
) PER_SHARED_ARCH
;
1260 void nearbyIntDouble(RoundingMode mode
, FloatRegister src
,
1261 FloatRegister dest
) PER_SHARED_ARCH
;
1262 void nearbyIntFloat32(RoundingMode mode
, FloatRegister src
,
1263 FloatRegister dest
) PER_SHARED_ARCH
;
1265 void signInt32(Register input
, Register output
);
1266 void signDouble(FloatRegister input
, FloatRegister output
);
1267 void signDoubleToInt32(FloatRegister input
, Register output
,
1268 FloatRegister temp
, Label
* fail
);
1270 void copySignDouble(FloatRegister lhs
, FloatRegister rhs
,
1271 FloatRegister output
) PER_SHARED_ARCH
;
1272 void copySignFloat32(FloatRegister lhs
, FloatRegister rhs
,
1273 FloatRegister output
) DEFINED_ON(x86_shared
, arm64
);
1275 // Returns a random double in range [0, 1) in |dest|. The |rng| register must
1276 // hold a pointer to a mozilla::non_crypto::XorShift128PlusRNG.
1277 void randomDouble(Register rng
, FloatRegister dest
, Register64 temp0
,
1280 // srcDest = {min,max}{Float32,Double}(srcDest, other)
1281 // For min and max, handle NaN specially if handleNaN is true.
1283 inline void minFloat32(FloatRegister other
, FloatRegister srcDest
,
1284 bool handleNaN
) PER_SHARED_ARCH
;
1285 inline void minDouble(FloatRegister other
, FloatRegister srcDest
,
1286 bool handleNaN
) PER_SHARED_ARCH
;
1288 inline void maxFloat32(FloatRegister other
, FloatRegister srcDest
,
1289 bool handleNaN
) PER_SHARED_ARCH
;
1290 inline void maxDouble(FloatRegister other
, FloatRegister srcDest
,
1291 bool handleNaN
) PER_SHARED_ARCH
;
1293 void minMaxArrayInt32(Register array
, Register result
, Register temp1
,
1294 Register temp2
, Register temp3
, bool isMax
,
1296 void minMaxArrayNumber(Register array
, FloatRegister result
,
1297 FloatRegister floatTemp
, Register temp1
,
1298 Register temp2
, bool isMax
, Label
* fail
);
1300 // Compute |pow(base, power)| and store the result in |dest|. If the result
1301 // exceeds the int32 range, jumps to |onOver|.
1302 // |base| and |power| are preserved, the other input registers are clobbered.
1303 void pow32(Register base
, Register power
, Register dest
, Register temp1
,
1304 Register temp2
, Label
* onOver
);
1306 void sameValueDouble(FloatRegister left
, FloatRegister right
,
1307 FloatRegister temp
, Register dest
);
1309 void branchIfNotRegExpPrototypeOptimizable(Register proto
, Register temp
,
1311 void branchIfNotRegExpInstanceOptimizable(Register regexp
, Register temp
,
1314 void loadRegExpLastIndex(Register regexp
, Register string
, Register lastIndex
,
1315 Label
* notFoundZeroLastIndex
);
1317 // ===============================================================
1320 // For shift-by-register there may be platform-specific variations, for
1321 // example, x86 will perform the shift mod 32 but ARM will perform the shift
1324 // For shift-by-immediate the platform assembler may restrict the immediate,
1325 // for example, the ARM assembler requires the count for 32-bit shifts to be
1326 // in the range [0,31].
1328 inline void lshift32(Imm32 shift
, Register srcDest
) PER_SHARED_ARCH
;
1329 inline void rshift32(Imm32 shift
, Register srcDest
) PER_SHARED_ARCH
;
1330 inline void rshift32Arithmetic(Imm32 shift
, Register srcDest
) PER_SHARED_ARCH
;
1332 inline void lshiftPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1333 inline void rshiftPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1334 inline void rshiftPtr(Imm32 imm
, Register src
, Register dest
)
1336 inline void rshiftPtrArithmetic(Imm32 imm
, Register dest
) PER_ARCH
;
1338 inline void lshift64(Imm32 imm
, Register64 dest
) PER_ARCH
;
1339 inline void rshift64(Imm32 imm
, Register64 dest
) PER_ARCH
;
1340 inline void rshift64Arithmetic(Imm32 imm
, Register64 dest
) PER_ARCH
;
1342 // On x86_shared these have the constraint that shift must be in CL.
1343 inline void lshift32(Register shift
, Register srcDest
) PER_SHARED_ARCH
;
1344 inline void rshift32(Register shift
, Register srcDest
) PER_SHARED_ARCH
;
1345 inline void rshift32Arithmetic(Register shift
,
1346 Register srcDest
) PER_SHARED_ARCH
;
1347 inline void lshiftPtr(Register shift
, Register srcDest
) PER_ARCH
;
1348 inline void rshiftPtr(Register shift
, Register srcDest
) PER_ARCH
;
1350 // These variants do not have the above constraint, but may emit some extra
1351 // instructions on x86_shared. They also handle shift >= 32 consistently by
1352 // masking with 0x1F (either explicitly or relying on the hardware to do
1354 inline void flexibleLshift32(Register shift
,
1355 Register srcDest
) PER_SHARED_ARCH
;
1356 inline void flexibleRshift32(Register shift
,
1357 Register srcDest
) PER_SHARED_ARCH
;
1358 inline void flexibleRshift32Arithmetic(Register shift
,
1359 Register srcDest
) PER_SHARED_ARCH
;
1361 inline void lshift64(Register shift
, Register64 srcDest
) PER_ARCH
;
1362 inline void rshift64(Register shift
, Register64 srcDest
) PER_ARCH
;
1363 inline void rshift64Arithmetic(Register shift
, Register64 srcDest
) PER_ARCH
;
1365 // ===============================================================
1366 // Rotation functions
1367 // Note: - on x86 and x64 the count register must be in CL.
1368 // - on x64 the temp register should be InvalidReg.
1370 inline void rotateLeft(Imm32 count
, Register input
,
1371 Register dest
) PER_SHARED_ARCH
;
1372 inline void rotateLeft(Register count
, Register input
,
1373 Register dest
) PER_SHARED_ARCH
;
1374 inline void rotateLeft64(Imm32 count
, Register64 input
, Register64 dest
)
1376 inline void rotateLeft64(Register count
, Register64 input
, Register64 dest
)
1378 inline void rotateLeft64(Imm32 count
, Register64 input
, Register64 dest
,
1379 Register temp
) PER_ARCH
;
1380 inline void rotateLeft64(Register count
, Register64 input
, Register64 dest
,
1381 Register temp
) PER_ARCH
;
1383 inline void rotateRight(Imm32 count
, Register input
,
1384 Register dest
) PER_SHARED_ARCH
;
1385 inline void rotateRight(Register count
, Register input
,
1386 Register dest
) PER_SHARED_ARCH
;
1387 inline void rotateRight64(Imm32 count
, Register64 input
, Register64 dest
)
1389 inline void rotateRight64(Register count
, Register64 input
, Register64 dest
)
1391 inline void rotateRight64(Imm32 count
, Register64 input
, Register64 dest
,
1392 Register temp
) PER_ARCH
;
1393 inline void rotateRight64(Register count
, Register64 input
, Register64 dest
,
1394 Register temp
) PER_ARCH
;
1396 // ===============================================================
1397 // Bit counting functions
1399 // knownNotZero may be true only if the src is known not to be zero.
1400 inline void clz32(Register src
, Register dest
,
1401 bool knownNotZero
) PER_SHARED_ARCH
;
1402 inline void ctz32(Register src
, Register dest
,
1403 bool knownNotZero
) PER_SHARED_ARCH
;
1405 inline void clz64(Register64 src
, Register dest
) PER_ARCH
;
1406 inline void ctz64(Register64 src
, Register dest
) PER_ARCH
;
1408 // On x86_shared, temp may be Invalid only if the chip has the POPCNT
1409 // instruction. On ARM, temp may never be Invalid.
1410 inline void popcnt32(Register src
, Register dest
,
1411 Register temp
) PER_SHARED_ARCH
;
1413 // temp may be invalid only if the chip has the POPCNT instruction.
1414 inline void popcnt64(Register64 src
, Register64 dest
, Register temp
) PER_ARCH
;
1416 // ===============================================================
1417 // Condition functions
1419 inline void cmp8Set(Condition cond
, Address lhs
, Imm32 rhs
,
1420 Register dest
) PER_SHARED_ARCH
;
1422 inline void cmp16Set(Condition cond
, Address lhs
, Imm32 rhs
,
1423 Register dest
) PER_SHARED_ARCH
;
1425 template <typename T1
, typename T2
>
1426 inline void cmp32Set(Condition cond
, T1 lhs
, T2 rhs
, Register dest
)
1427 DEFINED_ON(x86_shared
, arm
, arm64
, mips32
, mips64
, loong64
, riscv64
,
1430 // Only the NotEqual and Equal conditions are allowed.
1431 inline void cmp64Set(Condition cond
, Address lhs
, Imm64 rhs
,
1432 Register dest
) PER_ARCH
;
1434 template <typename T1
, typename T2
>
1435 inline void cmpPtrSet(Condition cond
, T1 lhs
, T2 rhs
, Register dest
) PER_ARCH
;
1437 // ===============================================================
1440 inline void branch8(Condition cond
, const Address
& lhs
, Imm32 rhs
,
1441 Label
* label
) PER_SHARED_ARCH
;
1443 // Compares the byte in |lhs| against |rhs| using a 8-bit comparison on
1444 // x86/x64 or a 32-bit comparison (all other platforms). The caller should
1445 // ensure |rhs| is a zero- resp. sign-extended byte value for cross-platform
1447 inline void branch8(Condition cond
, const BaseIndex
& lhs
, Register rhs
,
1448 Label
* label
) PER_SHARED_ARCH
;
1450 inline void branch16(Condition cond
, const Address
& lhs
, Imm32 rhs
,
1451 Label
* label
) PER_SHARED_ARCH
;
1454 inline void branch32(Condition cond
, Register lhs
, Register rhs
,
1455 L label
) PER_SHARED_ARCH
;
1457 inline void branch32(Condition cond
, Register lhs
, Imm32 rhs
,
1458 L label
) PER_SHARED_ARCH
;
1460 inline void branch32(Condition cond
, Register lhs
, const Address
& rhs
,
1461 Label
* label
) DEFINED_ON(arm64
);
1463 inline void branch32(Condition cond
, const Address
& lhs
, Register rhs
,
1464 Label
* label
) PER_SHARED_ARCH
;
1465 inline void branch32(Condition cond
, const Address
& lhs
, Imm32 rhs
,
1466 Label
* label
) PER_SHARED_ARCH
;
1468 inline void branch32(Condition cond
, const AbsoluteAddress
& lhs
, Register rhs
,
1470 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1471 inline void branch32(Condition cond
, const AbsoluteAddress
& lhs
, Imm32 rhs
,
1473 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1475 inline void branch32(Condition cond
, const BaseIndex
& lhs
, Register rhs
,
1476 Label
* label
) DEFINED_ON(arm
, x86_shared
);
1477 inline void branch32(Condition cond
, const BaseIndex
& lhs
, Imm32 rhs
,
1478 Label
* label
) PER_SHARED_ARCH
;
1480 inline void branch32(Condition cond
, const Operand
& lhs
, Register rhs
,
1481 Label
* label
) DEFINED_ON(x86_shared
);
1482 inline void branch32(Condition cond
, const Operand
& lhs
, Imm32 rhs
,
1483 Label
* label
) DEFINED_ON(x86_shared
);
1485 inline void branch32(Condition cond
, wasm::SymbolicAddress lhs
, Imm32 rhs
,
1487 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1489 // The supported condition are Equal, NotEqual, LessThan(orEqual),
1490 // GreaterThan(orEqual), Below(orEqual) and Above(orEqual). When a fail label
1491 // is not defined it will fall through to next instruction, else jump to the
1493 inline void branch64(Condition cond
, Register64 lhs
, Imm64 val
,
1494 Label
* success
, Label
* fail
= nullptr) PER_ARCH
;
1495 inline void branch64(Condition cond
, Register64 lhs
, Register64 rhs
,
1496 Label
* success
, Label
* fail
= nullptr) PER_ARCH
;
1497 // Only the NotEqual and Equal conditions are allowed for the branch64
1498 // variants with Address as lhs.
1499 inline void branch64(Condition cond
, const Address
& lhs
, Imm64 val
,
1500 Label
* label
) PER_ARCH
;
1501 inline void branch64(Condition cond
, const Address
& lhs
, Register64 rhs
,
1502 Label
* label
) PER_ARCH
;
1504 // Compare the value at |lhs| with the value at |rhs|. The scratch
1505 // register *must not* be the base of |lhs| or |rhs|.
1506 inline void branch64(Condition cond
, const Address
& lhs
, const Address
& rhs
,
1507 Register scratch
, Label
* label
) PER_ARCH
;
1510 inline void branchPtr(Condition cond
, Register lhs
, Register rhs
,
1511 L label
) PER_SHARED_ARCH
;
1512 inline void branchPtr(Condition cond
, Register lhs
, Imm32 rhs
,
1513 Label
* label
) PER_SHARED_ARCH
;
1514 inline void branchPtr(Condition cond
, Register lhs
, ImmPtr rhs
,
1515 Label
* label
) PER_SHARED_ARCH
;
1516 inline void branchPtr(Condition cond
, Register lhs
, ImmGCPtr rhs
,
1517 Label
* label
) PER_SHARED_ARCH
;
1518 inline void branchPtr(Condition cond
, Register lhs
, ImmWord rhs
,
1519 Label
* label
) PER_SHARED_ARCH
;
1522 inline void branchPtr(Condition cond
, const Address
& lhs
, Register rhs
,
1523 L label
) PER_SHARED_ARCH
;
1524 inline void branchPtr(Condition cond
, const Address
& lhs
, ImmPtr rhs
,
1525 Label
* label
) PER_SHARED_ARCH
;
1526 inline void branchPtr(Condition cond
, const Address
& lhs
, ImmGCPtr rhs
,
1527 Label
* label
) PER_SHARED_ARCH
;
1528 inline void branchPtr(Condition cond
, const Address
& lhs
, ImmWord rhs
,
1529 Label
* label
) PER_SHARED_ARCH
;
1531 inline void branchPtr(Condition cond
, const BaseIndex
& lhs
, ImmWord rhs
,
1532 Label
* label
) PER_SHARED_ARCH
;
1533 inline void branchPtr(Condition cond
, const BaseIndex
& lhs
, Register rhs
,
1534 Label
* label
) PER_SHARED_ARCH
;
1536 inline void branchPtr(Condition cond
, const AbsoluteAddress
& lhs
,
1537 Register rhs
, Label
* label
)
1538 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1539 inline void branchPtr(Condition cond
, const AbsoluteAddress
& lhs
, ImmWord rhs
,
1541 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1543 inline void branchPtr(Condition cond
, wasm::SymbolicAddress lhs
, Register rhs
,
1545 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1547 // Given a pointer to a GC Cell, retrieve the StoreBuffer pointer from its
1548 // chunk header, or nullptr if it is in the tenured heap.
1549 void loadStoreBuffer(Register ptr
, Register buffer
) PER_ARCH
;
1551 void branchPtrInNurseryChunk(Condition cond
, Register ptr
, Register temp
,
1553 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1554 void branchPtrInNurseryChunk(Condition cond
, const Address
& address
,
1555 Register temp
, Label
* label
) DEFINED_ON(x86
);
1556 void branchValueIsNurseryCell(Condition cond
, const Address
& address
,
1557 Register temp
, Label
* label
) PER_ARCH
;
1558 void branchValueIsNurseryCell(Condition cond
, ValueOperand value
,
1559 Register temp
, Label
* label
) PER_ARCH
;
1561 // This function compares a Value (lhs) which is having a private pointer
1562 // boxed inside a js::Value, with a raw pointer (rhs).
1563 inline void branchPrivatePtr(Condition cond
, const Address
& lhs
, Register rhs
,
1564 Label
* label
) PER_ARCH
;
1566 inline void branchFloat(DoubleCondition cond
, FloatRegister lhs
,
1567 FloatRegister rhs
, Label
* label
) PER_SHARED_ARCH
;
1569 // Truncate a double/float32 to int32 and when it doesn't fit an int32 it will
1570 // jump to the failure label. This particular variant is allowed to return the
1571 // value module 2**32, which isn't implemented on all architectures. E.g. the
1572 // x64 variants will do this only in the int64_t range.
1573 inline void branchTruncateFloat32MaybeModUint32(FloatRegister src
,
1574 Register dest
, Label
* fail
)
1575 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1576 inline void branchTruncateDoubleMaybeModUint32(FloatRegister src
,
1577 Register dest
, Label
* fail
)
1578 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1580 // Truncate a double/float32 to intptr and when it doesn't fit jump to the
1582 inline void branchTruncateFloat32ToPtr(FloatRegister src
, Register dest
,
1583 Label
* fail
) DEFINED_ON(x86
, x64
);
1584 inline void branchTruncateDoubleToPtr(FloatRegister src
, Register dest
,
1585 Label
* fail
) DEFINED_ON(x86
, x64
);
1587 // Truncate a double/float32 to int32 and when it doesn't fit jump to the
1589 inline void branchTruncateFloat32ToInt32(FloatRegister src
, Register dest
,
1591 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1592 inline void branchTruncateDoubleToInt32(FloatRegister src
, Register dest
,
1593 Label
* fail
) PER_ARCH
;
1595 inline void branchDouble(DoubleCondition cond
, FloatRegister lhs
,
1596 FloatRegister rhs
, Label
* label
) PER_SHARED_ARCH
;
1598 inline void branchDoubleNotInInt64Range(Address src
, Register temp
,
1600 inline void branchDoubleNotInUInt64Range(Address src
, Register temp
,
1602 inline void branchFloat32NotInInt64Range(Address src
, Register temp
,
1604 inline void branchFloat32NotInUInt64Range(Address src
, Register temp
,
1607 template <typename T
>
1608 inline void branchAdd32(Condition cond
, T src
, Register dest
,
1609 Label
* label
) PER_SHARED_ARCH
;
1610 template <typename T
>
1611 inline void branchSub32(Condition cond
, T src
, Register dest
,
1612 Label
* label
) PER_SHARED_ARCH
;
1613 template <typename T
>
1614 inline void branchMul32(Condition cond
, T src
, Register dest
,
1615 Label
* label
) PER_SHARED_ARCH
;
1616 template <typename T
>
1617 inline void branchRshift32(Condition cond
, T src
, Register dest
,
1618 Label
* label
) PER_SHARED_ARCH
;
1620 inline void branchNeg32(Condition cond
, Register reg
,
1621 Label
* label
) PER_SHARED_ARCH
;
1623 inline void branchAdd64(Condition cond
, Imm64 imm
, Register64 dest
,
1624 Label
* label
) DEFINED_ON(x86
, arm
, wasm32
);
1626 template <typename T
>
1627 inline void branchAddPtr(Condition cond
, T src
, Register dest
,
1628 Label
* label
) PER_SHARED_ARCH
;
1630 template <typename T
>
1631 inline void branchSubPtr(Condition cond
, T src
, Register dest
,
1632 Label
* label
) PER_SHARED_ARCH
;
1634 inline void branchMulPtr(Condition cond
, Register src
, Register dest
,
1635 Label
* label
) PER_SHARED_ARCH
;
1637 inline void decBranchPtr(Condition cond
, Register lhs
, Imm32 rhs
,
1638 Label
* label
) PER_SHARED_ARCH
;
1641 inline void branchTest32(Condition cond
, Register lhs
, Register rhs
,
1642 L label
) PER_SHARED_ARCH
;
1644 inline void branchTest32(Condition cond
, Register lhs
, Imm32 rhs
,
1645 L label
) PER_SHARED_ARCH
;
1646 inline void branchTest32(Condition cond
, const Address
& lhs
, Imm32 rhh
,
1647 Label
* label
) PER_SHARED_ARCH
;
1648 inline void branchTest32(Condition cond
, const AbsoluteAddress
& lhs
,
1649 Imm32 rhs
, Label
* label
)
1650 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1653 inline void branchTestPtr(Condition cond
, Register lhs
, Register rhs
,
1654 L label
) PER_SHARED_ARCH
;
1655 inline void branchTestPtr(Condition cond
, Register lhs
, Imm32 rhs
,
1656 Label
* label
) PER_SHARED_ARCH
;
1657 inline void branchTestPtr(Condition cond
, const Address
& lhs
, Imm32 rhs
,
1658 Label
* label
) PER_SHARED_ARCH
;
1661 inline void branchTest64(Condition cond
, Register64 lhs
, Register64 rhs
,
1662 Register temp
, L label
) PER_ARCH
;
1664 // Branches to |label| if |reg| is false. |reg| should be a C++ bool.
1666 inline void branchIfFalseBool(Register reg
, L label
);
1668 // Branches to |label| if |reg| is true. |reg| should be a C++ bool.
1669 inline void branchIfTrueBool(Register reg
, Label
* label
);
1671 inline void branchIfRope(Register str
, Label
* label
);
1672 inline void branchIfNotRope(Register str
, Label
* label
);
1674 inline void branchLatin1String(Register string
, Label
* label
);
1675 inline void branchTwoByteString(Register string
, Label
* label
);
1677 inline void branchIfBigIntIsNegative(Register bigInt
, Label
* label
);
1678 inline void branchIfBigIntIsNonNegative(Register bigInt
, Label
* label
);
1679 inline void branchIfBigIntIsZero(Register bigInt
, Label
* label
);
1680 inline void branchIfBigIntIsNonZero(Register bigInt
, Label
* label
);
1682 inline void branchTestFunctionFlags(Register fun
, uint32_t flags
,
1683 Condition cond
, Label
* label
);
1685 inline void branchIfNotFunctionIsNonBuiltinCtor(Register fun
,
1689 inline void branchIfFunctionHasNoJitEntry(Register fun
, bool isConstructing
,
1691 inline void branchIfFunctionHasJitEntry(Register fun
, bool isConstructing
,
1694 inline void branchIfScriptHasJitScript(Register script
, Label
* label
);
1695 inline void branchIfScriptHasNoJitScript(Register script
, Label
* label
);
1696 inline void loadJitScript(Register script
, Register dest
);
1698 // Loads the function's argument count.
1699 inline void loadFunctionArgCount(Register func
, Register output
);
1701 // Loads the function length. This handles interpreted, native, and bound
1702 // functions. The caller is responsible for checking that INTERPRETED_LAZY and
1703 // RESOLVED_LENGTH flags are not set.
1704 void loadFunctionLength(Register func
, Register funFlagsAndArgCount
,
1705 Register output
, Label
* slowPath
);
1707 // Loads the function name. This handles interpreted, native, and bound
1709 void loadFunctionName(Register func
, Register output
, ImmGCPtr emptyString
,
1712 void assertFunctionIsExtended(Register func
);
1714 inline void branchFunctionKind(Condition cond
,
1715 FunctionFlags::FunctionKind kind
, Register fun
,
1716 Register scratch
, Label
* label
);
1718 inline void branchIfObjectEmulatesUndefined(Register objReg
, Register scratch
,
1719 Label
* slowCheck
, Label
* label
);
1721 // For all methods below: spectreRegToZero is a register that will be zeroed
1722 // on speculatively executed code paths (when the branch should be taken but
1723 // branch prediction speculates it isn't). Usually this will be the object
1724 // register but the caller may pass a different register.
1726 inline void branchTestObjClass(Condition cond
, Register obj
,
1727 const JSClass
* clasp
, Register scratch
,
1728 Register spectreRegToZero
, Label
* label
);
1729 inline void branchTestObjClassNoSpectreMitigations(Condition cond
,
1731 const JSClass
* clasp
,
1735 inline void branchTestObjClass(Condition cond
, Register obj
,
1736 const Address
& clasp
, Register scratch
,
1737 Register spectreRegToZero
, Label
* label
);
1738 inline void branchTestObjClassNoSpectreMitigations(Condition cond
,
1740 const Address
& clasp
,
1744 inline void branchTestObjClass(Condition cond
, Register obj
, Register clasp
,
1745 Register scratch
, Register spectreRegToZero
,
1748 inline void branchTestObjShape(Condition cond
, Register obj
,
1749 const Shape
* shape
, Register scratch
,
1750 Register spectreRegToZero
, Label
* label
);
1751 inline void branchTestObjShapeNoSpectreMitigations(Condition cond
,
1756 void branchTestObjShapeList(Condition cond
, Register obj
,
1757 Register shapeElements
, Register shapeScratch
,
1758 Register endScratch
, Register spectreScratch
,
1761 inline void branchTestClassIsFunction(Condition cond
, Register clasp
,
1763 inline void branchTestObjIsFunction(Condition cond
, Register obj
,
1765 Register spectreRegToZero
, Label
* label
);
1766 inline void branchTestObjIsFunctionNoSpectreMitigations(Condition cond
,
1771 inline void branchTestObjShape(Condition cond
, Register obj
, Register shape
,
1772 Register scratch
, Register spectreRegToZero
,
1774 inline void branchTestObjShapeNoSpectreMitigations(Condition cond
,
1779 // TODO: audit/fix callers to be Spectre safe.
1780 inline void branchTestObjShapeUnsafe(Condition cond
, Register obj
,
1781 Register shape
, Label
* label
);
1783 void branchTestObjCompartment(Condition cond
, Register obj
,
1784 const Address
& compartment
, Register scratch
,
1786 void branchTestObjCompartment(Condition cond
, Register obj
,
1787 const JS::Compartment
* compartment
,
1788 Register scratch
, Label
* label
);
1790 void branchIfNonNativeObj(Register obj
, Register scratch
, Label
* label
);
1792 void branchIfObjectNotExtensible(Register obj
, Register scratch
,
1795 inline void branchTestClassIsProxy(bool proxy
, Register clasp
, Label
* label
);
1797 inline void branchTestObjectIsProxy(bool proxy
, Register object
,
1798 Register scratch
, Label
* label
);
1800 inline void branchTestProxyHandlerFamily(Condition cond
, Register proxy
,
1802 const void* handlerp
, Label
* label
);
1804 inline void branchTestObjectIsWasmGcObject(bool isGcObject
, Register obj
,
1805 Register scratch
, Label
* label
);
1807 inline void branchTestNeedsIncrementalBarrier(Condition cond
, Label
* label
);
1808 inline void branchTestNeedsIncrementalBarrierAnyZone(Condition cond
,
1812 // Perform a type-test on a tag of a Value (32bits boxing), or the tagged
1813 // value (64bits boxing).
1814 inline void branchTestUndefined(Condition cond
, Register tag
,
1815 Label
* label
) PER_SHARED_ARCH
;
1816 inline void branchTestInt32(Condition cond
, Register tag
,
1817 Label
* label
) PER_SHARED_ARCH
;
1818 inline void branchTestDouble(Condition cond
, Register tag
, Label
* label
)
1819 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1821 inline void branchTestNumber(Condition cond
, Register tag
,
1822 Label
* label
) PER_SHARED_ARCH
;
1823 inline void branchTestBoolean(Condition cond
, Register tag
,
1824 Label
* label
) PER_SHARED_ARCH
;
1825 inline void branchTestString(Condition cond
, Register tag
,
1826 Label
* label
) PER_SHARED_ARCH
;
1827 inline void branchTestSymbol(Condition cond
, Register tag
,
1828 Label
* label
) PER_SHARED_ARCH
;
1829 inline void branchTestBigInt(Condition cond
, Register tag
,
1830 Label
* label
) PER_SHARED_ARCH
;
1831 inline void branchTestNull(Condition cond
, Register tag
,
1832 Label
* label
) PER_SHARED_ARCH
;
1833 inline void branchTestObject(Condition cond
, Register tag
,
1834 Label
* label
) PER_SHARED_ARCH
;
1835 inline void branchTestPrimitive(Condition cond
, Register tag
,
1836 Label
* label
) PER_SHARED_ARCH
;
1837 inline void branchTestMagic(Condition cond
, Register tag
,
1838 Label
* label
) PER_SHARED_ARCH
;
1839 void branchTestType(Condition cond
, Register tag
, JSValueType type
,
1842 // Perform a type-test on a Value, addressed by Address or BaseIndex, or
1843 // loaded into ValueOperand.
1844 // BaseIndex and ValueOperand variants clobber the ScratchReg on x64.
1845 // All Variants clobber the ScratchReg on arm64.
1846 inline void branchTestUndefined(Condition cond
, const Address
& address
,
1847 Label
* label
) PER_SHARED_ARCH
;
1848 inline void branchTestUndefined(Condition cond
, const BaseIndex
& address
,
1849 Label
* label
) PER_SHARED_ARCH
;
1850 inline void branchTestUndefined(Condition cond
, const ValueOperand
& value
,
1852 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1855 inline void branchTestInt32(Condition cond
, const Address
& address
,
1856 Label
* label
) PER_SHARED_ARCH
;
1857 inline void branchTestInt32(Condition cond
, const BaseIndex
& address
,
1858 Label
* label
) PER_SHARED_ARCH
;
1859 inline void branchTestInt32(Condition cond
, const ValueOperand
& value
,
1861 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1864 inline void branchTestDouble(Condition cond
, const Address
& address
,
1865 Label
* label
) PER_SHARED_ARCH
;
1866 inline void branchTestDouble(Condition cond
, const BaseIndex
& address
,
1867 Label
* label
) PER_SHARED_ARCH
;
1868 inline void branchTestDouble(Condition cond
, const ValueOperand
& value
,
1870 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1873 inline void branchTestNumber(Condition cond
, const ValueOperand
& value
,
1875 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1878 inline void branchTestBoolean(Condition cond
, const Address
& address
,
1879 Label
* label
) PER_SHARED_ARCH
;
1880 inline void branchTestBoolean(Condition cond
, const BaseIndex
& address
,
1881 Label
* label
) PER_SHARED_ARCH
;
1882 inline void branchTestBoolean(Condition cond
, const ValueOperand
& value
,
1884 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1887 inline void branchTestString(Condition cond
, const Address
& address
,
1888 Label
* label
) PER_SHARED_ARCH
;
1889 inline void branchTestString(Condition cond
, const BaseIndex
& address
,
1890 Label
* label
) PER_SHARED_ARCH
;
1891 inline void branchTestString(Condition cond
, const ValueOperand
& value
,
1893 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1896 inline void branchTestSymbol(Condition cond
, const Address
& address
,
1897 Label
* label
) PER_SHARED_ARCH
;
1898 inline void branchTestSymbol(Condition cond
, const BaseIndex
& address
,
1899 Label
* label
) PER_SHARED_ARCH
;
1900 inline void branchTestSymbol(Condition cond
, const ValueOperand
& value
,
1902 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1905 inline void branchTestBigInt(Condition cond
, const Address
& address
,
1906 Label
* label
) PER_SHARED_ARCH
;
1907 inline void branchTestBigInt(Condition cond
, const BaseIndex
& address
,
1908 Label
* label
) PER_SHARED_ARCH
;
1909 inline void branchTestBigInt(Condition cond
, const ValueOperand
& value
,
1911 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1914 inline void branchTestNull(Condition cond
, const Address
& address
,
1915 Label
* label
) PER_SHARED_ARCH
;
1916 inline void branchTestNull(Condition cond
, const BaseIndex
& address
,
1917 Label
* label
) PER_SHARED_ARCH
;
1918 inline void branchTestNull(Condition cond
, const ValueOperand
& value
,
1920 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1923 // Clobbers the ScratchReg on x64.
1924 inline void branchTestObject(Condition cond
, const Address
& address
,
1925 Label
* label
) PER_SHARED_ARCH
;
1926 inline void branchTestObject(Condition cond
, const BaseIndex
& address
,
1927 Label
* label
) PER_SHARED_ARCH
;
1928 inline void branchTestObject(Condition cond
, const ValueOperand
& value
,
1930 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1933 inline void branchTestGCThing(Condition cond
, const Address
& address
,
1934 Label
* label
) PER_SHARED_ARCH
;
1935 inline void branchTestGCThing(Condition cond
, const BaseIndex
& address
,
1936 Label
* label
) PER_SHARED_ARCH
;
1937 inline void branchTestGCThing(Condition cond
, const ValueOperand
& value
,
1938 Label
* label
) PER_SHARED_ARCH
;
1940 inline void branchTestPrimitive(Condition cond
, const ValueOperand
& value
,
1942 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1945 inline void branchTestMagic(Condition cond
, const Address
& address
,
1946 Label
* label
) PER_SHARED_ARCH
;
1947 inline void branchTestMagic(Condition cond
, const BaseIndex
& address
,
1948 Label
* label
) PER_SHARED_ARCH
;
1950 inline void branchTestMagic(Condition cond
, const ValueOperand
& value
,
1952 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1955 inline void branchTestMagic(Condition cond
, const Address
& valaddr
,
1956 JSWhyMagic why
, Label
* label
) PER_ARCH
;
1958 inline void branchTestMagicValue(Condition cond
, const ValueOperand
& val
,
1959 JSWhyMagic why
, Label
* label
);
1961 void branchTestValue(Condition cond
, const ValueOperand
& lhs
,
1962 const Value
& rhs
, Label
* label
) PER_ARCH
;
1964 inline void branchTestValue(Condition cond
, const BaseIndex
& lhs
,
1965 const ValueOperand
& rhs
, Label
* label
) PER_ARCH
;
1967 // Checks if given Value is evaluated to true or false in a condition.
1968 // The type of the value should match the type of the method.
1969 inline void branchTestInt32Truthy(bool truthy
, const ValueOperand
& value
,
1971 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, x86_shared
,
1973 inline void branchTestDoubleTruthy(bool truthy
, FloatRegister reg
,
1974 Label
* label
) PER_SHARED_ARCH
;
1975 inline void branchTestBooleanTruthy(bool truthy
, const ValueOperand
& value
,
1976 Label
* label
) PER_ARCH
;
1977 inline void branchTestStringTruthy(bool truthy
, const ValueOperand
& value
,
1979 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1981 inline void branchTestBigIntTruthy(bool truthy
, const ValueOperand
& value
,
1983 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1986 // Create an unconditional branch to the address given as argument.
1987 inline void branchToComputedAddress(const BaseIndex
& address
) PER_ARCH
;
1990 template <typename T
, typename S
, typename L
>
1991 inline void branchPtrImpl(Condition cond
, const T
& lhs
, const S
& rhs
, L label
)
1992 DEFINED_ON(x86_shared
);
1994 void branchPtrInNurseryChunkImpl(Condition cond
, Register ptr
, Label
* label
)
1996 template <typename T
>
1997 void branchValueIsNurseryCellImpl(Condition cond
, const T
& value
,
1998 Register temp
, Label
* label
)
1999 DEFINED_ON(arm64
, x64
, mips64
, loong64
, riscv64
);
2001 template <typename T
>
2002 inline void branchTestUndefinedImpl(Condition cond
, const T
& t
, Label
* label
)
2003 DEFINED_ON(arm
, arm64
, x86_shared
);
2004 template <typename T
>
2005 inline void branchTestInt32Impl(Condition cond
, const T
& t
, Label
* label
)
2006 DEFINED_ON(arm
, arm64
, x86_shared
);
2007 template <typename T
>
2008 inline void branchTestDoubleImpl(Condition cond
, const T
& t
, Label
* label
)
2009 DEFINED_ON(arm
, arm64
, x86_shared
);
2010 template <typename T
>
2011 inline void branchTestNumberImpl(Condition cond
, const T
& t
, Label
* label
)
2012 DEFINED_ON(arm
, arm64
, x86_shared
);
2013 template <typename T
>
2014 inline void branchTestBooleanImpl(Condition cond
, const T
& t
, Label
* label
)
2015 DEFINED_ON(arm
, arm64
, x86_shared
);
2016 template <typename T
>
2017 inline void branchTestStringImpl(Condition cond
, const T
& t
, Label
* label
)
2018 DEFINED_ON(arm
, arm64
, x86_shared
);
2019 template <typename T
>
2020 inline void branchTestSymbolImpl(Condition cond
, const T
& t
, Label
* label
)
2021 DEFINED_ON(arm
, arm64
, x86_shared
);
2022 template <typename T
>
2023 inline void branchTestBigIntImpl(Condition cond
, const T
& t
, Label
* label
)
2024 DEFINED_ON(arm
, arm64
, x86_shared
);
2025 template <typename T
>
2026 inline void branchTestNullImpl(Condition cond
, const T
& t
, Label
* label
)
2027 DEFINED_ON(arm
, arm64
, x86_shared
);
2028 template <typename T
>
2029 inline void branchTestObjectImpl(Condition cond
, const T
& t
, Label
* label
)
2030 DEFINED_ON(arm
, arm64
, x86_shared
);
2031 template <typename T
>
2032 inline void branchTestGCThingImpl(Condition cond
, const T
& t
,
2033 Label
* label
) PER_SHARED_ARCH
;
2034 template <typename T
>
2035 inline void branchTestPrimitiveImpl(Condition cond
, const T
& t
, Label
* label
)
2036 DEFINED_ON(arm
, arm64
, x86_shared
);
2037 template <typename T
, class L
>
2038 inline void branchTestMagicImpl(Condition cond
, const T
& t
, L label
)
2039 DEFINED_ON(arm
, arm64
, x86_shared
);
2042 template <typename T
>
2043 inline void testNumberSet(Condition cond
, const T
& src
,
2044 Register dest
) PER_SHARED_ARCH
;
2045 template <typename T
>
2046 inline void testBooleanSet(Condition cond
, const T
& src
,
2047 Register dest
) PER_SHARED_ARCH
;
2048 template <typename T
>
2049 inline void testStringSet(Condition cond
, const T
& src
,
2050 Register dest
) PER_SHARED_ARCH
;
2051 template <typename T
>
2052 inline void testSymbolSet(Condition cond
, const T
& src
,
2053 Register dest
) PER_SHARED_ARCH
;
2054 template <typename T
>
2055 inline void testBigIntSet(Condition cond
, const T
& src
,
2056 Register dest
) PER_SHARED_ARCH
;
2059 // The fallibleUnbox* methods below combine a Value type check with an unbox.
2060 // Especially on 64-bit platforms this can be implemented more efficiently
2061 // than a separate branch + unbox.
2063 // |src| and |dest| can be the same register, but |dest| may hold garbage on
2065 inline void fallibleUnboxPtr(const ValueOperand
& src
, Register dest
,
2066 JSValueType type
, Label
* fail
) PER_ARCH
;
2067 inline void fallibleUnboxPtr(const Address
& src
, Register dest
,
2068 JSValueType type
, Label
* fail
) PER_ARCH
;
2069 inline void fallibleUnboxPtr(const BaseIndex
& src
, Register dest
,
2070 JSValueType type
, Label
* fail
) PER_ARCH
;
2071 template <typename T
>
2072 inline void fallibleUnboxInt32(const T
& src
, Register dest
, Label
* fail
);
2073 template <typename T
>
2074 inline void fallibleUnboxBoolean(const T
& src
, Register dest
, Label
* fail
);
2075 template <typename T
>
2076 inline void fallibleUnboxObject(const T
& src
, Register dest
, Label
* fail
);
2077 template <typename T
>
2078 inline void fallibleUnboxString(const T
& src
, Register dest
, Label
* fail
);
2079 template <typename T
>
2080 inline void fallibleUnboxSymbol(const T
& src
, Register dest
, Label
* fail
);
2081 template <typename T
>
2082 inline void fallibleUnboxBigInt(const T
& src
, Register dest
, Label
* fail
);
2084 inline void cmp32Move32(Condition cond
, Register lhs
, Register rhs
,
2085 Register src
, Register dest
)
2086 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86_shared
);
2088 inline void cmp32Move32(Condition cond
, Register lhs
, const Address
& rhs
,
2089 Register src
, Register dest
)
2090 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86_shared
);
2092 inline void cmpPtrMovePtr(Condition cond
, Register lhs
, Register rhs
,
2093 Register src
, Register dest
) PER_ARCH
;
2095 inline void cmpPtrMovePtr(Condition cond
, Register lhs
, const Address
& rhs
,
2096 Register src
, Register dest
) PER_ARCH
;
2098 inline void cmp32Load32(Condition cond
, Register lhs
, const Address
& rhs
,
2099 const Address
& src
, Register dest
)
2100 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, mips_shared
, x86_shared
);
2102 inline void cmp32Load32(Condition cond
, Register lhs
, Register rhs
,
2103 const Address
& src
, Register dest
)
2104 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, mips_shared
, x86_shared
);
2106 inline void cmp32LoadPtr(Condition cond
, const Address
& lhs
, Imm32 rhs
,
2107 const Address
& src
, Register dest
)
2108 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86
, x64
);
2110 inline void cmp32MovePtr(Condition cond
, Register lhs
, Imm32 rhs
,
2111 Register src
, Register dest
)
2112 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86
, x64
);
2114 inline void test32LoadPtr(Condition cond
, const Address
& addr
, Imm32 mask
,
2115 const Address
& src
, Register dest
)
2116 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86
, x64
);
2118 inline void test32MovePtr(Condition cond
, const Address
& addr
, Imm32 mask
,
2119 Register src
, Register dest
)
2120 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86
, x64
);
2122 // Conditional move for Spectre mitigations.
2123 inline void spectreMovePtr(Condition cond
, Register src
, Register dest
)
2124 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
2126 // Zeroes dest if the condition is true.
2127 inline void spectreZeroRegister(Condition cond
, Register scratch
,
2129 DEFINED_ON(arm
, arm64
, mips_shared
, x86_shared
, loong64
, riscv64
, wasm32
);
2131 // Performs a bounds check and zeroes the index register if out-of-bounds
2132 // (to mitigate Spectre).
2134 inline void spectreBoundsCheck32(Register index
, const Operand
& length
,
2135 Register maybeScratch
, Label
* failure
)
2139 inline void spectreBoundsCheck32(Register index
, Register length
,
2140 Register maybeScratch
, Label
* failure
)
2141 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
2142 inline void spectreBoundsCheck32(Register index
, const Address
& length
,
2143 Register maybeScratch
, Label
* failure
)
2144 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
2146 inline void spectreBoundsCheckPtr(Register index
, Register length
,
2147 Register maybeScratch
, Label
* failure
)
2148 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
2149 inline void spectreBoundsCheckPtr(Register index
, const Address
& length
,
2150 Register maybeScratch
, Label
* failure
)
2151 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
2153 // ========================================================================
2154 // Canonicalization primitives.
2155 inline void canonicalizeDouble(FloatRegister reg
);
2156 inline void canonicalizeDoubleIfDeterministic(FloatRegister reg
);
2158 inline void canonicalizeFloat(FloatRegister reg
);
2159 inline void canonicalizeFloatIfDeterministic(FloatRegister reg
);
2162 // ========================================================================
2163 // Memory access primitives.
2164 inline void storeUncanonicalizedDouble(FloatRegister src
, const Address
& dest
)
2165 DEFINED_ON(x86_shared
, arm
, arm64
, mips32
, mips64
, loong64
, riscv64
,
2167 inline void storeUncanonicalizedDouble(FloatRegister src
,
2168 const BaseIndex
& dest
)
2169 DEFINED_ON(x86_shared
, arm
, arm64
, mips32
, mips64
, loong64
, riscv64
,
2171 inline void storeUncanonicalizedDouble(FloatRegister src
, const Operand
& dest
)
2172 DEFINED_ON(x86_shared
);
2175 inline void storeDouble(FloatRegister src
, const T
& dest
);
2178 inline void boxDouble(FloatRegister src
, const T
& dest
);
2180 using MacroAssemblerSpecific::boxDouble
;
2182 inline void storeUncanonicalizedFloat32(FloatRegister src
,
2183 const Address
& dest
)
2184 DEFINED_ON(x86_shared
, arm
, arm64
, mips32
, mips64
, loong64
, riscv64
,
2186 inline void storeUncanonicalizedFloat32(FloatRegister src
,
2187 const BaseIndex
& dest
)
2188 DEFINED_ON(x86_shared
, arm
, arm64
, mips32
, mips64
, loong64
, riscv64
,
2190 inline void storeUncanonicalizedFloat32(FloatRegister src
,
2191 const Operand
& dest
)
2192 DEFINED_ON(x86_shared
);
2195 inline void storeFloat32(FloatRegister src
, const T
& dest
);
2197 template <typename T
>
2198 void storeUnboxedValue(const ConstantOrRegister
& value
, MIRType valueType
,
2199 const T
& dest
) PER_ARCH
;
2201 inline void memoryBarrier(MemoryBarrierBits barrier
) PER_SHARED_ARCH
;
2204 // ========================================================================
2207 // Naming is "operationSimd128" when operate on the whole vector, otherwise
2208 // it's "operation<Type><Size>x<Lanes>".
2210 // For microarchitectural reasons we can in principle get a performance win by
2211 // using int or float specific instructions in the operationSimd128 case when
2212 // we know that subsequent operations on the result are int or float oriented.
2213 // In practice, we don't care about that yet.
2215 // The order of operations here follows those in the SIMD overview document,
2216 // https://github.com/WebAssembly/simd/blob/master/proposals/simd/SIMD.md.
2218 // Since we must target Intel SSE indefinitely and SSE is one-address or
2219 // two-address, the x86 porting interfaces are nearly all one-address or
2220 // two-address. Likewise there are two-address ARM64 interfaces to support
2221 // the baseline compiler. But there are also three-address ARM64 interfaces
2222 // as the ARM64 Ion back-end can use those. In the future, they may support
2223 // AVX2 or similar for x86.
2225 // Conventions for argument order and naming and semantics:
2226 // - Condition codes come first.
2227 // - Other immediates (masks, shift counts) come next.
2228 // - Operands come next:
2229 // - For a binary two-address operator where the left-hand-side has the
2230 // same type as the result, one register parameter is normally named
2231 // `lhsDest` and is both the left-hand side and destination; the other
2232 // parameter is named `rhs` and is the right-hand side. `rhs` comes
2233 // first, `lhsDest` second. `rhs` and `lhsDest` may be the same register
2234 // (if rhs is a register).
2235 // - For a binary three-address operator the order is `lhs`, `rhs`, `dest`,
2236 // and generally these registers may be the same.
2237 // - For a unary operator, the input is named `src` and the output is named
2238 // `dest`. `src` comes first, `dest` second. `src` and `dest` may be
2239 // the same register (if `src` is a register).
2240 // - Temp registers follow operands and are named `temp` if there's only one,
2241 // otherwise `temp1`, `temp2`, etc regardless of type. GPR temps precede
2242 // FPU temps. If there are several temps then they must be distinct
2243 // registers, and they must be distinct from the operand registers unless
2248 inline void moveSimd128(FloatRegister src
, FloatRegister dest
)
2249 DEFINED_ON(x86_shared
, arm64
);
2253 inline void loadConstantSimd128(const SimdConstant
& v
, FloatRegister dest
)
2254 DEFINED_ON(x86_shared
, arm64
);
2258 inline void splatX16(Register src
, FloatRegister dest
)
2259 DEFINED_ON(x86_shared
, arm64
);
2261 inline void splatX16(uint32_t srcLane
, FloatRegister src
, FloatRegister dest
)
2264 inline void splatX8(Register src
, FloatRegister dest
)
2265 DEFINED_ON(x86_shared
, arm64
);
2267 inline void splatX8(uint32_t srcLane
, FloatRegister src
, FloatRegister dest
)
2270 inline void splatX4(Register src
, FloatRegister dest
)
2271 DEFINED_ON(x86_shared
, arm64
);
2273 inline void splatX4(FloatRegister src
, FloatRegister dest
)
2274 DEFINED_ON(x86_shared
, arm64
);
2276 inline void splatX2(Register64 src
, FloatRegister dest
)
2277 DEFINED_ON(x86
, x64
, arm64
);
2279 inline void splatX2(FloatRegister src
, FloatRegister dest
)
2280 DEFINED_ON(x86_shared
, arm64
);
2282 // Extract lane as scalar. Float extraction does not canonicalize the value.
2284 inline void extractLaneInt8x16(uint32_t lane
, FloatRegister src
,
2285 Register dest
) DEFINED_ON(x86_shared
, arm64
);
2287 inline void unsignedExtractLaneInt8x16(uint32_t lane
, FloatRegister src
,
2289 DEFINED_ON(x86_shared
, arm64
);
2291 inline void extractLaneInt16x8(uint32_t lane
, FloatRegister src
,
2292 Register dest
) DEFINED_ON(x86_shared
, arm64
);
2294 inline void unsignedExtractLaneInt16x8(uint32_t lane
, FloatRegister src
,
2296 DEFINED_ON(x86_shared
, arm64
);
2298 inline void extractLaneInt32x4(uint32_t lane
, FloatRegister src
,
2299 Register dest
) DEFINED_ON(x86_shared
, arm64
);
2301 inline void extractLaneInt64x2(uint32_t lane
, FloatRegister src
,
2302 Register64 dest
) DEFINED_ON(x86
, x64
, arm64
);
2304 inline void extractLaneFloat32x4(uint32_t lane
, FloatRegister src
,
2306 DEFINED_ON(x86_shared
, arm64
);
2308 inline void extractLaneFloat64x2(uint32_t lane
, FloatRegister src
,
2310 DEFINED_ON(x86_shared
, arm64
);
2312 // Replace lane value
2314 inline void replaceLaneInt8x16(unsigned lane
, FloatRegister lhs
, Register rhs
,
2315 FloatRegister dest
) DEFINED_ON(x86_shared
);
2317 inline void replaceLaneInt8x16(unsigned lane
, Register rhs
,
2318 FloatRegister lhsDest
)
2319 DEFINED_ON(x86_shared
, arm64
);
2321 inline void replaceLaneInt16x8(unsigned lane
, FloatRegister lhs
, Register rhs
,
2322 FloatRegister dest
) DEFINED_ON(x86_shared
);
2324 inline void replaceLaneInt16x8(unsigned lane
, Register rhs
,
2325 FloatRegister lhsDest
)
2326 DEFINED_ON(x86_shared
, arm64
);
2328 inline void replaceLaneInt32x4(unsigned lane
, FloatRegister lhs
, Register rhs
,
2329 FloatRegister dest
) DEFINED_ON(x86_shared
);
2331 inline void replaceLaneInt32x4(unsigned lane
, Register rhs
,
2332 FloatRegister lhsDest
)
2333 DEFINED_ON(x86_shared
, arm64
);
2335 inline void replaceLaneInt64x2(unsigned lane
, FloatRegister lhs
,
2336 Register64 rhs
, FloatRegister dest
)
2337 DEFINED_ON(x86
, x64
);
2339 inline void replaceLaneInt64x2(unsigned lane
, Register64 rhs
,
2340 FloatRegister lhsDest
)
2341 DEFINED_ON(x86
, x64
, arm64
);
2343 inline void replaceLaneFloat32x4(unsigned lane
, FloatRegister lhs
,
2344 FloatRegister rhs
, FloatRegister dest
)
2345 DEFINED_ON(x86_shared
);
2347 inline void replaceLaneFloat32x4(unsigned lane
, FloatRegister rhs
,
2348 FloatRegister lhsDest
)
2349 DEFINED_ON(x86_shared
, arm64
);
2351 inline void replaceLaneFloat64x2(unsigned lane
, FloatRegister lhs
,
2352 FloatRegister rhs
, FloatRegister dest
)
2353 DEFINED_ON(x86_shared
);
2355 inline void replaceLaneFloat64x2(unsigned lane
, FloatRegister rhs
,
2356 FloatRegister lhsDest
)
2357 DEFINED_ON(x86_shared
, arm64
);
2359 // Shuffle - blend and permute with immediate indices, and its many
2360 // specializations. Lane values other than those mentioned are illegal.
2362 // lane values 0..31
2363 inline void shuffleInt8x16(const uint8_t lanes
[16], FloatRegister rhs
,
2364 FloatRegister lhsDest
)
2365 DEFINED_ON(x86_shared
, arm64
);
2367 inline void shuffleInt8x16(const uint8_t lanes
[16], FloatRegister lhs
,
2368 FloatRegister rhs
, FloatRegister dest
)
2369 DEFINED_ON(x86_shared
, arm64
);
2371 // Lane values must be 0 (select from lhs) or FF (select from rhs).
2372 // The behavior is undefined for lane values that are neither 0 nor FF.
2373 // on x86_shared: it is required that lhs == dest.
2374 inline void blendInt8x16(const uint8_t lanes
[16], FloatRegister lhs
,
2375 FloatRegister rhs
, FloatRegister dest
,
2376 FloatRegister temp
) DEFINED_ON(x86_shared
);
2378 // Lane values must be 0 (select from lhs) or FF (select from rhs).
2379 // The behavior is undefined for lane values that are neither 0 nor FF.
2380 inline void blendInt8x16(const uint8_t lanes
[16], FloatRegister lhs
,
2381 FloatRegister rhs
, FloatRegister dest
)
2384 // Lane values must be 0 (select from lhs) or FFFF (select from rhs).
2385 // The behavior is undefined for lane values that are neither 0 nor FFFF.
2386 // on x86_shared: it is required that lhs == dest.
2387 inline void blendInt16x8(const uint16_t lanes
[8], FloatRegister lhs
,
2388 FloatRegister rhs
, FloatRegister dest
)
2389 DEFINED_ON(x86_shared
, arm64
);
2391 // Mask lane values must be ~0 or 0. The former selects from lhs and the
2393 // The implementation works effectively for I8x16, I16x8, I32x4, and I64x2.
2394 inline void laneSelectSimd128(FloatRegister mask
, FloatRegister lhs
,
2395 FloatRegister rhs
, FloatRegister dest
)
2396 DEFINED_ON(x86_shared
, arm64
);
2398 inline void interleaveHighInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2400 DEFINED_ON(x86_shared
, arm64
);
2402 inline void interleaveHighInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2404 DEFINED_ON(x86_shared
, arm64
);
2406 inline void interleaveHighInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2408 DEFINED_ON(x86_shared
, arm64
);
2410 inline void interleaveHighInt64x2(FloatRegister lhs
, FloatRegister rhs
,
2412 DEFINED_ON(x86_shared
, arm64
);
2414 inline void interleaveLowInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2416 DEFINED_ON(x86_shared
, arm64
);
2418 inline void interleaveLowInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2420 DEFINED_ON(x86_shared
, arm64
);
2422 inline void interleaveLowInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2424 DEFINED_ON(x86_shared
, arm64
);
2426 inline void interleaveLowInt64x2(FloatRegister lhs
, FloatRegister rhs
,
2428 DEFINED_ON(x86_shared
, arm64
);
2430 // Permute - permute with immediate indices.
2432 // lane values 0..15
2433 inline void permuteInt8x16(const uint8_t lanes
[16], FloatRegister src
,
2434 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2437 inline void permuteInt16x8(const uint16_t lanes
[8], FloatRegister src
,
2438 FloatRegister dest
) DEFINED_ON(arm64
);
2440 // lane values 0..3 [sic].
2441 inline void permuteHighInt16x8(const uint16_t lanes
[4], FloatRegister src
,
2442 FloatRegister dest
) DEFINED_ON(x86_shared
);
2444 // lane values 0..3.
2445 inline void permuteLowInt16x8(const uint16_t lanes
[4], FloatRegister src
,
2446 FloatRegister dest
) DEFINED_ON(x86_shared
);
2449 inline void permuteInt32x4(const uint32_t lanes
[4], FloatRegister src
,
2450 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2452 // Funnel shift by immediate count:
2453 // low_16_bytes_of((lhs ++ rhs) >> shift*8), shift must be < 16
2454 inline void concatAndRightShiftSimd128(FloatRegister lhs
, FloatRegister rhs
,
2455 FloatRegister dest
, uint32_t shift
)
2456 DEFINED_ON(x86_shared
, arm64
);
2458 // Rotate right by immediate count:
2459 // low_16_bytes_of((src ++ src) >> shift*8), shift must be < 16
2460 inline void rotateRightSimd128(FloatRegister src
, FloatRegister dest
,
2461 uint32_t shift
) DEFINED_ON(arm64
);
2463 // Shift bytes with immediate count, shifting in zeroes. Shift count 0..15.
2465 inline void leftShiftSimd128(Imm32 count
, FloatRegister src
,
2467 DEFINED_ON(x86_shared
, arm64
);
2469 inline void rightShiftSimd128(Imm32 count
, FloatRegister src
,
2471 DEFINED_ON(x86_shared
, arm64
);
2473 // Reverse bytes in lanes.
2475 inline void reverseInt16x8(FloatRegister src
, FloatRegister dest
)
2476 DEFINED_ON(x86_shared
, arm64
);
2478 inline void reverseInt32x4(FloatRegister src
, FloatRegister dest
)
2479 DEFINED_ON(x86_shared
, arm64
);
2481 inline void reverseInt64x2(FloatRegister src
, FloatRegister dest
)
2482 DEFINED_ON(x86_shared
, arm64
);
2484 // Swizzle - permute with variable indices. `rhs` holds the lanes parameter.
2486 inline void swizzleInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2487 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2489 inline void swizzleInt8x16Relaxed(FloatRegister lhs
, FloatRegister rhs
,
2491 DEFINED_ON(x86_shared
, arm64
);
2495 inline void addInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2496 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2498 inline void addInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2499 FloatRegister dest
) DEFINED_ON(x86_shared
);
2501 inline void addInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2502 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2504 inline void addInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2505 FloatRegister dest
) DEFINED_ON(x86_shared
);
2507 inline void addInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2508 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2510 inline void addInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2511 FloatRegister dest
) DEFINED_ON(x86_shared
);
2513 inline void addInt64x2(FloatRegister lhs
, FloatRegister rhs
,
2514 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2516 inline void addInt64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
2517 FloatRegister dest
) DEFINED_ON(x86_shared
);
2521 inline void subInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2522 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2524 inline void subInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2525 FloatRegister dest
) DEFINED_ON(x86_shared
);
2527 inline void subInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2528 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2530 inline void subInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2531 FloatRegister dest
) DEFINED_ON(x86_shared
);
2533 inline void subInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2534 FloatRegister dest
) DEFINED_ON(x86_shared
);
2536 inline void subInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2537 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2539 inline void subInt64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
2540 FloatRegister dest
) DEFINED_ON(x86_shared
);
2542 inline void subInt64x2(FloatRegister lhs
, FloatRegister rhs
,
2543 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2547 inline void mulInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2548 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2550 inline void mulInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2551 FloatRegister dest
) DEFINED_ON(x86_shared
);
2553 inline void mulInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2554 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2556 inline void mulInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2557 FloatRegister dest
) DEFINED_ON(x86_shared
);
2559 // On x86_shared, it is required lhs == dest
2560 inline void mulInt64x2(FloatRegister lhs
, FloatRegister rhs
,
2561 FloatRegister dest
, FloatRegister temp
)
2562 DEFINED_ON(x86_shared
);
2564 inline void mulInt64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
2565 FloatRegister dest
, FloatRegister temp
)
2566 DEFINED_ON(x86_shared
);
2568 inline void mulInt64x2(FloatRegister lhs
, FloatRegister rhs
,
2569 FloatRegister dest
, FloatRegister temp1
,
2570 FloatRegister temp2
) DEFINED_ON(arm64
);
2572 // Note for the extMul opcodes, the NxM designation is for the input lanes;
2573 // the output lanes are twice as wide.
2574 inline void extMulLowInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2576 DEFINED_ON(x86_shared
, arm64
);
2578 inline void extMulHighInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2580 DEFINED_ON(x86_shared
, arm64
);
2582 inline void unsignedExtMulLowInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2584 DEFINED_ON(x86_shared
, arm64
);
2586 inline void unsignedExtMulHighInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2588 DEFINED_ON(x86_shared
, arm64
);
2590 inline void extMulLowInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2592 DEFINED_ON(x86_shared
, arm64
);
2594 inline void extMulHighInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2596 DEFINED_ON(x86_shared
, arm64
);
2598 inline void unsignedExtMulLowInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2600 DEFINED_ON(x86_shared
, arm64
);
2602 inline void unsignedExtMulHighInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2604 DEFINED_ON(x86_shared
, arm64
);
2606 inline void extMulLowInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2608 DEFINED_ON(x86_shared
, arm64
);
2610 inline void extMulHighInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2612 DEFINED_ON(x86_shared
, arm64
);
2614 inline void unsignedExtMulLowInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2616 DEFINED_ON(x86_shared
, arm64
);
2618 inline void unsignedExtMulHighInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2620 DEFINED_ON(x86_shared
, arm64
);
2622 inline void q15MulrSatInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2624 DEFINED_ON(x86_shared
, arm64
);
2628 inline void negInt8x16(FloatRegister src
, FloatRegister dest
)
2629 DEFINED_ON(x86_shared
, arm64
);
2631 inline void negInt16x8(FloatRegister src
, FloatRegister dest
)
2632 DEFINED_ON(x86_shared
, arm64
);
2634 inline void negInt32x4(FloatRegister src
, FloatRegister dest
)
2635 DEFINED_ON(x86_shared
, arm64
);
2637 inline void negInt64x2(FloatRegister src
, FloatRegister dest
)
2638 DEFINED_ON(x86_shared
, arm64
);
2640 // Saturating integer add
2642 inline void addSatInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2643 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2645 inline void addSatInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2646 FloatRegister dest
) DEFINED_ON(x86_shared
);
2648 inline void unsignedAddSatInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2650 DEFINED_ON(x86_shared
, arm64
);
2652 inline void unsignedAddSatInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2653 FloatRegister dest
) DEFINED_ON(x86_shared
);
2655 inline void addSatInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2656 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2658 inline void addSatInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2659 FloatRegister dest
) DEFINED_ON(x86_shared
);
2661 inline void unsignedAddSatInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2663 DEFINED_ON(x86_shared
, arm64
);
2665 inline void unsignedAddSatInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2666 FloatRegister dest
) DEFINED_ON(x86_shared
);
2668 // Saturating integer subtract
2670 inline void subSatInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2671 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2673 inline void subSatInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2674 FloatRegister dest
) DEFINED_ON(x86_shared
);
2676 inline void unsignedSubSatInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2678 DEFINED_ON(x86_shared
, arm64
);
2680 inline void unsignedSubSatInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2681 FloatRegister dest
) DEFINED_ON(x86_shared
);
2683 inline void subSatInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2684 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2686 inline void subSatInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2687 FloatRegister dest
) DEFINED_ON(x86_shared
);
2689 inline void unsignedSubSatInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2691 DEFINED_ON(x86_shared
, arm64
);
2693 inline void unsignedSubSatInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2694 FloatRegister dest
) DEFINED_ON(x86_shared
);
2696 // Lane-wise integer minimum
2698 inline void minInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2699 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2701 inline void minInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2702 FloatRegister dest
) DEFINED_ON(x86_shared
);
2704 inline void unsignedMinInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2706 DEFINED_ON(x86_shared
, arm64
);
2708 inline void unsignedMinInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2709 FloatRegister dest
) DEFINED_ON(x86_shared
);
2711 inline void minInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2712 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2714 inline void minInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2715 FloatRegister dest
) DEFINED_ON(x86_shared
);
2717 inline void unsignedMinInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2719 DEFINED_ON(x86_shared
, arm64
);
2721 inline void unsignedMinInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2722 FloatRegister dest
) DEFINED_ON(x86_shared
);
2724 inline void minInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2725 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2727 inline void minInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2728 FloatRegister dest
) DEFINED_ON(x86_shared
);
2730 inline void unsignedMinInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2732 DEFINED_ON(x86_shared
, arm64
);
2734 inline void unsignedMinInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2735 FloatRegister dest
) DEFINED_ON(x86_shared
);
2737 // Lane-wise integer maximum
2739 inline void maxInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2740 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2742 inline void maxInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2743 FloatRegister dest
) DEFINED_ON(x86_shared
);
2745 inline void unsignedMaxInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2747 DEFINED_ON(x86_shared
, arm64
);
2749 inline void unsignedMaxInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2750 FloatRegister dest
) DEFINED_ON(x86_shared
);
2752 inline void maxInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2753 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2755 inline void maxInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2756 FloatRegister dest
) DEFINED_ON(x86_shared
);
2758 inline void unsignedMaxInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2760 DEFINED_ON(x86_shared
, arm64
);
2762 inline void unsignedMaxInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2763 FloatRegister dest
) DEFINED_ON(x86_shared
);
2765 inline void maxInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2766 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2768 inline void maxInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2769 FloatRegister dest
) DEFINED_ON(x86_shared
);
2771 inline void unsignedMaxInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2773 DEFINED_ON(x86_shared
, arm64
);
2775 inline void unsignedMaxInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2776 FloatRegister dest
) DEFINED_ON(x86_shared
);
2778 // Lane-wise integer rounding average
2780 inline void unsignedAverageInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2782 DEFINED_ON(x86_shared
, arm64
);
2784 inline void unsignedAverageInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2786 DEFINED_ON(x86_shared
, arm64
);
2788 // Lane-wise integer absolute value
2790 inline void absInt8x16(FloatRegister src
, FloatRegister dest
)
2791 DEFINED_ON(x86_shared
, arm64
);
2793 inline void absInt16x8(FloatRegister src
, FloatRegister dest
)
2794 DEFINED_ON(x86_shared
, arm64
);
2796 inline void absInt32x4(FloatRegister src
, FloatRegister dest
)
2797 DEFINED_ON(x86_shared
, arm64
);
2799 inline void absInt64x2(FloatRegister src
, FloatRegister dest
)
2800 DEFINED_ON(x86_shared
, arm64
);
2802 // Left shift by scalar. Immediates and variable shifts must have been
2803 // masked; shifts of zero will work but may or may not generate code.
2805 inline void leftShiftInt8x16(Register rhs
, FloatRegister lhsDest
,
2806 FloatRegister temp
) DEFINED_ON(x86_shared
);
2808 inline void leftShiftInt8x16(FloatRegister lhs
, Register rhs
,
2809 FloatRegister dest
) DEFINED_ON(arm64
);
2811 inline void leftShiftInt8x16(Imm32 count
, FloatRegister src
,
2813 DEFINED_ON(x86_shared
, arm64
);
2815 inline void leftShiftInt16x8(Register rhs
, FloatRegister lhsDest
)
2816 DEFINED_ON(x86_shared
);
2818 inline void leftShiftInt16x8(FloatRegister lhs
, Register rhs
,
2819 FloatRegister dest
) DEFINED_ON(arm64
);
2821 inline void leftShiftInt16x8(Imm32 count
, FloatRegister src
,
2823 DEFINED_ON(x86_shared
, arm64
);
2825 inline void leftShiftInt32x4(Register rhs
, FloatRegister lhsDest
)
2826 DEFINED_ON(x86_shared
);
2828 inline void leftShiftInt32x4(FloatRegister lhs
, Register rhs
,
2829 FloatRegister dest
) DEFINED_ON(arm64
);
2831 inline void leftShiftInt32x4(Imm32 count
, FloatRegister src
,
2833 DEFINED_ON(x86_shared
, arm64
);
2835 inline void leftShiftInt64x2(Register rhs
, FloatRegister lhsDest
)
2836 DEFINED_ON(x86_shared
);
2838 inline void leftShiftInt64x2(FloatRegister lhs
, Register rhs
,
2839 FloatRegister dest
) DEFINED_ON(arm64
);
2841 inline void leftShiftInt64x2(Imm32 count
, FloatRegister src
,
2843 DEFINED_ON(x86_shared
, arm64
);
2845 // Right shift by scalar. Immediates and variable shifts must have been
2846 // masked; shifts of zero will work but may or may not generate code.
2848 inline void rightShiftInt8x16(Register rhs
, FloatRegister lhsDest
,
2849 FloatRegister temp
) DEFINED_ON(x86_shared
);
2851 inline void rightShiftInt8x16(FloatRegister lhs
, Register rhs
,
2852 FloatRegister dest
) DEFINED_ON(arm64
);
2854 inline void rightShiftInt8x16(Imm32 count
, FloatRegister src
,
2856 DEFINED_ON(x86_shared
, arm64
);
2858 inline void unsignedRightShiftInt8x16(Register rhs
, FloatRegister lhsDest
,
2860 DEFINED_ON(x86_shared
);
2862 inline void unsignedRightShiftInt8x16(FloatRegister lhs
, Register rhs
,
2863 FloatRegister dest
) DEFINED_ON(arm64
);
2865 inline void unsignedRightShiftInt8x16(Imm32 count
, FloatRegister src
,
2867 DEFINED_ON(x86_shared
, arm64
);
2869 inline void rightShiftInt16x8(Register rhs
, FloatRegister lhsDest
)
2870 DEFINED_ON(x86_shared
);
2872 inline void rightShiftInt16x8(FloatRegister lhs
, Register rhs
,
2873 FloatRegister dest
) DEFINED_ON(arm64
);
2875 inline void rightShiftInt16x8(Imm32 count
, FloatRegister src
,
2877 DEFINED_ON(x86_shared
, arm64
);
2879 inline void unsignedRightShiftInt16x8(Register rhs
, FloatRegister lhsDest
)
2880 DEFINED_ON(x86_shared
);
2882 inline void unsignedRightShiftInt16x8(FloatRegister lhs
, Register rhs
,
2883 FloatRegister dest
) DEFINED_ON(arm64
);
2885 inline void unsignedRightShiftInt16x8(Imm32 count
, FloatRegister src
,
2887 DEFINED_ON(x86_shared
, arm64
);
2889 inline void rightShiftInt32x4(Register rhs
, FloatRegister lhsDest
)
2890 DEFINED_ON(x86_shared
);
2892 inline void rightShiftInt32x4(FloatRegister lhs
, Register rhs
,
2893 FloatRegister dest
) DEFINED_ON(arm64
);
2895 inline void rightShiftInt32x4(Imm32 count
, FloatRegister src
,
2897 DEFINED_ON(x86_shared
, arm64
);
2899 inline void unsignedRightShiftInt32x4(Register rhs
, FloatRegister lhsDest
)
2900 DEFINED_ON(x86_shared
);
2902 inline void unsignedRightShiftInt32x4(FloatRegister lhs
, Register rhs
,
2903 FloatRegister dest
) DEFINED_ON(arm64
);
2905 inline void unsignedRightShiftInt32x4(Imm32 count
, FloatRegister src
,
2907 DEFINED_ON(x86_shared
, arm64
);
2909 inline void rightShiftInt64x2(Register rhs
, FloatRegister lhsDest
,
2910 FloatRegister temp
) DEFINED_ON(x86_shared
);
2912 inline void rightShiftInt64x2(Imm32 count
, FloatRegister src
,
2914 DEFINED_ON(x86_shared
, arm64
);
2916 inline void rightShiftInt64x2(FloatRegister lhs
, Register rhs
,
2917 FloatRegister dest
) DEFINED_ON(arm64
);
2919 inline void unsignedRightShiftInt64x2(Register rhs
, FloatRegister lhsDest
)
2920 DEFINED_ON(x86_shared
);
2922 inline void unsignedRightShiftInt64x2(FloatRegister lhs
, Register rhs
,
2923 FloatRegister dest
) DEFINED_ON(arm64
);
2925 inline void unsignedRightShiftInt64x2(Imm32 count
, FloatRegister src
,
2927 DEFINED_ON(x86_shared
, arm64
);
2929 // Sign replication operation
2931 inline void signReplicationInt8x16(FloatRegister src
, FloatRegister dest
)
2932 DEFINED_ON(x86_shared
);
2934 inline void signReplicationInt16x8(FloatRegister src
, FloatRegister dest
)
2935 DEFINED_ON(x86_shared
);
2937 inline void signReplicationInt32x4(FloatRegister src
, FloatRegister dest
)
2938 DEFINED_ON(x86_shared
);
2940 inline void signReplicationInt64x2(FloatRegister src
, FloatRegister dest
)
2941 DEFINED_ON(x86_shared
);
2943 // Bitwise and, or, xor, not
2945 inline void bitwiseAndSimd128(FloatRegister rhs
, FloatRegister lhsDest
)
2946 DEFINED_ON(x86_shared
, arm64
);
2948 inline void bitwiseAndSimd128(FloatRegister lhs
, FloatRegister rhs
,
2950 DEFINED_ON(x86_shared
, arm64
);
2952 inline void bitwiseAndSimd128(FloatRegister lhs
, const SimdConstant
& rhs
,
2953 FloatRegister dest
) DEFINED_ON(x86_shared
);
2955 inline void bitwiseOrSimd128(FloatRegister rhs
, FloatRegister lhsDest
)
2956 DEFINED_ON(x86_shared
, arm64
);
2958 inline void bitwiseOrSimd128(FloatRegister lhs
, FloatRegister rhs
,
2960 DEFINED_ON(x86_shared
, arm64
);
2962 inline void bitwiseOrSimd128(FloatRegister lhs
, const SimdConstant
& rhs
,
2963 FloatRegister dest
) DEFINED_ON(x86_shared
);
2965 inline void bitwiseXorSimd128(FloatRegister rhs
, FloatRegister lhsDest
)
2966 DEFINED_ON(x86_shared
, arm64
);
2968 inline void bitwiseXorSimd128(FloatRegister lhs
, FloatRegister rhs
,
2970 DEFINED_ON(x86_shared
, arm64
);
2972 inline void bitwiseXorSimd128(FloatRegister lhs
, const SimdConstant
& rhs
,
2973 FloatRegister dest
) DEFINED_ON(x86_shared
);
2975 inline void bitwiseNotSimd128(FloatRegister src
, FloatRegister dest
)
2976 DEFINED_ON(x86_shared
, arm64
);
2978 // Bitwise AND with compliment: dest = lhs & ~rhs, note only arm64 can do it.
2979 inline void bitwiseAndNotSimd128(FloatRegister lhs
, FloatRegister rhs
,
2980 FloatRegister lhsDest
) DEFINED_ON(arm64
);
2982 // Bitwise AND with complement: dest = ~lhs & rhs, note this is not what Wasm
2983 // wants but what the x86 hardware offers. Hence the name.
2985 inline void bitwiseNotAndSimd128(FloatRegister rhs
, FloatRegister lhsDest
)
2986 DEFINED_ON(x86_shared
, arm64
);
2988 inline void bitwiseNotAndSimd128(FloatRegister lhs
, FloatRegister rhs
,
2989 FloatRegister lhsDest
)
2990 DEFINED_ON(x86_shared
);
2994 inline void bitwiseSelectSimd128(FloatRegister mask
, FloatRegister onTrue
,
2995 FloatRegister onFalse
, FloatRegister dest
,
2996 FloatRegister temp
) DEFINED_ON(x86_shared
);
2998 inline void bitwiseSelectSimd128(FloatRegister onTrue
, FloatRegister onFalse
,
2999 FloatRegister maskDest
) DEFINED_ON(arm64
);
3003 inline void popcntInt8x16(FloatRegister src
, FloatRegister dest
,
3004 FloatRegister temp
) DEFINED_ON(x86_shared
);
3006 inline void popcntInt8x16(FloatRegister src
, FloatRegister dest
)
3009 // Any lane true, ie, any bit set
3011 inline void anyTrueSimd128(FloatRegister src
, Register dest
)
3012 DEFINED_ON(x86_shared
, arm64
);
3016 inline void allTrueInt8x16(FloatRegister src
, Register dest
)
3017 DEFINED_ON(x86_shared
, arm64
);
3019 inline void allTrueInt16x8(FloatRegister src
, Register dest
)
3020 DEFINED_ON(x86_shared
, arm64
);
3022 inline void allTrueInt32x4(FloatRegister src
, Register dest
)
3023 DEFINED_ON(x86_shared
, arm64
);
3025 inline void allTrueInt64x2(FloatRegister src
, Register dest
)
3026 DEFINED_ON(x86_shared
, arm64
);
3028 // Bitmask, ie extract and compress high bits of all lanes
3030 inline void bitmaskInt8x16(FloatRegister src
, Register dest
)
3031 DEFINED_ON(x86_shared
);
3033 inline void bitmaskInt8x16(FloatRegister src
, Register dest
,
3034 FloatRegister temp
) DEFINED_ON(arm64
);
3036 inline void bitmaskInt16x8(FloatRegister src
, Register dest
)
3037 DEFINED_ON(x86_shared
);
3039 inline void bitmaskInt16x8(FloatRegister src
, Register dest
,
3040 FloatRegister temp
) DEFINED_ON(arm64
);
3042 inline void bitmaskInt32x4(FloatRegister src
, Register dest
)
3043 DEFINED_ON(x86_shared
);
3045 inline void bitmaskInt32x4(FloatRegister src
, Register dest
,
3046 FloatRegister temp
) DEFINED_ON(arm64
);
3048 inline void bitmaskInt64x2(FloatRegister src
, Register dest
)
3049 DEFINED_ON(x86_shared
);
3051 inline void bitmaskInt64x2(FloatRegister src
, Register dest
,
3052 FloatRegister temp
) DEFINED_ON(arm64
);
3054 // Comparisons (integer and floating-point)
3056 inline void compareInt8x16(Assembler::Condition cond
, FloatRegister rhs
,
3057 FloatRegister lhsDest
)
3058 DEFINED_ON(x86_shared
, arm64
);
3060 // On x86_shared, limited to !=, ==, <=, >
3061 inline void compareInt8x16(Assembler::Condition cond
, FloatRegister lhs
,
3062 const SimdConstant
& rhs
, FloatRegister dest
)
3063 DEFINED_ON(x86_shared
);
3065 // On arm64, use any integer comparison condition.
3066 inline void compareInt8x16(Assembler::Condition cond
, FloatRegister lhs
,
3067 FloatRegister rhs
, FloatRegister dest
)
3068 DEFINED_ON(x86_shared
, arm64
);
3070 inline void compareInt16x8(Assembler::Condition cond
, FloatRegister rhs
,
3071 FloatRegister lhsDest
)
3072 DEFINED_ON(x86_shared
, arm64
);
3074 inline void compareInt16x8(Assembler::Condition cond
, FloatRegister lhs
,
3075 FloatRegister rhs
, FloatRegister dest
)
3076 DEFINED_ON(x86_shared
, arm64
);
3078 // On x86_shared, limited to !=, ==, <=, >
3079 inline void compareInt16x8(Assembler::Condition cond
, FloatRegister lhs
,
3080 const SimdConstant
& rhs
, FloatRegister dest
)
3081 DEFINED_ON(x86_shared
);
3083 // On x86_shared, limited to !=, ==, <=, >
3084 inline void compareInt32x4(Assembler::Condition cond
, FloatRegister rhs
,
3085 FloatRegister lhsDest
)
3086 DEFINED_ON(x86_shared
, arm64
);
3088 inline void compareInt32x4(Assembler::Condition cond
, FloatRegister lhs
,
3089 const SimdConstant
& rhs
, FloatRegister dest
)
3090 DEFINED_ON(x86_shared
);
3092 // On arm64, use any integer comparison condition.
3093 inline void compareInt32x4(Assembler::Condition cond
, FloatRegister lhs
,
3094 FloatRegister rhs
, FloatRegister dest
)
3095 DEFINED_ON(x86_shared
, arm64
);
3097 inline void compareForEqualityInt64x2(Assembler::Condition cond
,
3098 FloatRegister lhs
, FloatRegister rhs
,
3100 DEFINED_ON(x86_shared
);
3102 inline void compareForOrderingInt64x2(Assembler::Condition cond
,
3103 FloatRegister lhs
, FloatRegister rhs
,
3104 FloatRegister dest
, FloatRegister temp1
,
3105 FloatRegister temp2
)
3106 DEFINED_ON(x86_shared
);
3108 inline void compareInt64x2(Assembler::Condition cond
, FloatRegister rhs
,
3109 FloatRegister lhsDest
) DEFINED_ON(arm64
);
3111 inline void compareInt64x2(Assembler::Condition cond
, FloatRegister lhs
,
3112 FloatRegister rhs
, FloatRegister dest
)
3115 inline void compareFloat32x4(Assembler::Condition cond
, FloatRegister rhs
,
3116 FloatRegister lhsDest
)
3117 DEFINED_ON(x86_shared
, arm64
);
3119 // On x86_shared, limited to ==, !=, <, <=
3120 inline void compareFloat32x4(Assembler::Condition cond
, FloatRegister lhs
,
3121 const SimdConstant
& rhs
, FloatRegister dest
)
3122 DEFINED_ON(x86_shared
);
3124 // On x86_shared, limited to ==, !=, <, <=
3125 // On arm64, use any float-point comparison condition.
3126 inline void compareFloat32x4(Assembler::Condition cond
, FloatRegister lhs
,
3127 FloatRegister rhs
, FloatRegister dest
)
3128 DEFINED_ON(x86_shared
, arm64
);
3130 inline void compareFloat64x2(Assembler::Condition cond
, FloatRegister rhs
,
3131 FloatRegister lhsDest
)
3132 DEFINED_ON(x86_shared
, arm64
);
3134 // On x86_shared, limited to ==, !=, <, <=
3135 inline void compareFloat64x2(Assembler::Condition cond
, FloatRegister lhs
,
3136 const SimdConstant
& rhs
, FloatRegister dest
)
3137 DEFINED_ON(x86_shared
);
3139 // On x86_shared, limited to ==, !=, <, <=
3140 // On arm64, use any float-point comparison condition.
3141 inline void compareFloat64x2(Assembler::Condition cond
, FloatRegister lhs
,
3142 FloatRegister rhs
, FloatRegister dest
)
3143 DEFINED_ON(x86_shared
, arm64
);
3147 inline void loadUnalignedSimd128(const Operand
& src
, FloatRegister dest
)
3148 DEFINED_ON(x86_shared
);
3150 inline void loadUnalignedSimd128(const Address
& src
, FloatRegister dest
)
3151 DEFINED_ON(x86_shared
, arm64
);
3153 inline void loadUnalignedSimd128(const BaseIndex
& src
, FloatRegister dest
)
3154 DEFINED_ON(x86_shared
, arm64
);
3158 inline void storeUnalignedSimd128(FloatRegister src
, const Address
& dest
)
3159 DEFINED_ON(x86_shared
, arm64
);
3161 inline void storeUnalignedSimd128(FloatRegister src
, const BaseIndex
& dest
)
3162 DEFINED_ON(x86_shared
, arm64
);
3164 // Floating point negation
3166 inline void negFloat32x4(FloatRegister src
, FloatRegister dest
)
3167 DEFINED_ON(x86_shared
, arm64
);
3169 inline void negFloat64x2(FloatRegister src
, FloatRegister dest
)
3170 DEFINED_ON(x86_shared
, arm64
);
3172 // Floating point absolute value
3174 inline void absFloat32x4(FloatRegister src
, FloatRegister dest
)
3175 DEFINED_ON(x86_shared
, arm64
);
3177 inline void absFloat64x2(FloatRegister src
, FloatRegister dest
)
3178 DEFINED_ON(x86_shared
, arm64
);
3180 // NaN-propagating minimum
3182 inline void minFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3183 FloatRegister dest
, FloatRegister temp1
,
3184 FloatRegister temp2
) DEFINED_ON(x86_shared
);
3186 inline void minFloat32x4(FloatRegister rhs
, FloatRegister lhsDest
)
3189 inline void minFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3190 FloatRegister dest
) DEFINED_ON(arm64
);
3192 inline void minFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3193 FloatRegister dest
, FloatRegister temp1
,
3194 FloatRegister temp2
) DEFINED_ON(x86_shared
);
3196 inline void minFloat64x2(FloatRegister rhs
, FloatRegister lhsDest
)
3199 inline void minFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3200 FloatRegister dest
) DEFINED_ON(arm64
);
3202 // NaN-propagating maximum
3204 inline void maxFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3205 FloatRegister dest
, FloatRegister temp1
,
3206 FloatRegister temp2
) DEFINED_ON(x86_shared
);
3208 inline void maxFloat32x4(FloatRegister rhs
, FloatRegister lhsDest
)
3211 inline void maxFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3212 FloatRegister dest
) DEFINED_ON(arm64
);
3214 inline void maxFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3215 FloatRegister dest
, FloatRegister temp1
,
3216 FloatRegister temp2
) DEFINED_ON(x86_shared
);
3218 inline void maxFloat64x2(FloatRegister rhs
, FloatRegister lhsDest
)
3221 inline void maxFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3222 FloatRegister dest
) DEFINED_ON(arm64
);
3226 inline void addFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3227 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3229 inline void addFloat32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
3230 FloatRegister dest
) DEFINED_ON(x86_shared
);
3232 inline void addFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3233 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3235 inline void addFloat64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
3236 FloatRegister dest
) DEFINED_ON(x86_shared
);
3238 // Floating subtract
3240 inline void subFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3241 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3243 inline void subFloat32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
3244 FloatRegister dest
) DEFINED_ON(x86_shared
);
3246 inline void subFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3247 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3249 inline void subFloat64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
3250 FloatRegister dest
) DEFINED_ON(x86_shared
);
3252 // Floating division
3254 inline void divFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3255 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3257 inline void divFloat32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
3258 FloatRegister dest
) DEFINED_ON(x86_shared
);
3260 inline void divFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3261 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3263 inline void divFloat64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
3264 FloatRegister dest
) DEFINED_ON(x86_shared
);
3266 // Floating Multiply
3268 inline void mulFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3269 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3271 inline void mulFloat32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
3272 FloatRegister dest
) DEFINED_ON(x86_shared
);
3274 inline void mulFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3275 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3277 inline void mulFloat64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
3278 FloatRegister dest
) DEFINED_ON(x86_shared
);
3282 inline void extAddPairwiseInt8x16(FloatRegister src
, FloatRegister dest
)
3283 DEFINED_ON(x86_shared
, arm64
);
3285 inline void unsignedExtAddPairwiseInt8x16(FloatRegister src
,
3287 DEFINED_ON(x86_shared
, arm64
);
3289 inline void extAddPairwiseInt16x8(FloatRegister src
, FloatRegister dest
)
3290 DEFINED_ON(x86_shared
, arm64
);
3292 inline void unsignedExtAddPairwiseInt16x8(FloatRegister src
,
3294 DEFINED_ON(x86_shared
, arm64
);
3296 // Floating square root
3298 inline void sqrtFloat32x4(FloatRegister src
, FloatRegister dest
)
3299 DEFINED_ON(x86_shared
, arm64
);
3301 inline void sqrtFloat64x2(FloatRegister src
, FloatRegister dest
)
3302 DEFINED_ON(x86_shared
, arm64
);
3304 // Integer to floating point with rounding
3306 inline void convertInt32x4ToFloat32x4(FloatRegister src
, FloatRegister dest
)
3307 DEFINED_ON(x86_shared
, arm64
);
3309 inline void unsignedConvertInt32x4ToFloat32x4(FloatRegister src
,
3311 DEFINED_ON(x86_shared
, arm64
);
3313 inline void convertInt32x4ToFloat64x2(FloatRegister src
, FloatRegister dest
)
3314 DEFINED_ON(x86_shared
, arm64
);
3316 inline void unsignedConvertInt32x4ToFloat64x2(FloatRegister src
,
3318 DEFINED_ON(x86_shared
, arm64
);
3320 // Floating point to integer with saturation
3322 inline void truncSatFloat32x4ToInt32x4(FloatRegister src
, FloatRegister dest
)
3323 DEFINED_ON(x86_shared
, arm64
);
3325 inline void unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src
,
3328 DEFINED_ON(x86_shared
);
3330 inline void unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src
,
3334 inline void truncSatFloat64x2ToInt32x4(FloatRegister src
, FloatRegister dest
,
3336 DEFINED_ON(x86_shared
, arm64
);
3338 inline void unsignedTruncSatFloat64x2ToInt32x4(FloatRegister src
,
3341 DEFINED_ON(x86_shared
, arm64
);
3343 inline void truncFloat32x4ToInt32x4Relaxed(FloatRegister src
,
3345 DEFINED_ON(x86_shared
, arm64
);
3347 inline void unsignedTruncFloat32x4ToInt32x4Relaxed(FloatRegister src
,
3349 DEFINED_ON(x86_shared
, arm64
);
3351 inline void truncFloat64x2ToInt32x4Relaxed(FloatRegister src
,
3353 DEFINED_ON(x86_shared
, arm64
);
3355 inline void unsignedTruncFloat64x2ToInt32x4Relaxed(FloatRegister src
,
3357 DEFINED_ON(x86_shared
, arm64
);
3359 // Floating point narrowing
3361 inline void convertFloat64x2ToFloat32x4(FloatRegister src
, FloatRegister dest
)
3362 DEFINED_ON(x86_shared
, arm64
);
3364 // Floating point widening
3366 inline void convertFloat32x4ToFloat64x2(FloatRegister src
, FloatRegister dest
)
3367 DEFINED_ON(x86_shared
, arm64
);
3369 // Integer to integer narrowing
3371 inline void narrowInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
3372 FloatRegister dest
) DEFINED_ON(x86_shared
);
3374 inline void narrowInt16x8(FloatRegister lhs
, FloatRegister rhs
,
3375 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3377 inline void unsignedNarrowInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
3378 FloatRegister dest
) DEFINED_ON(x86_shared
);
3380 inline void unsignedNarrowInt16x8(FloatRegister lhs
, FloatRegister rhs
,
3382 DEFINED_ON(x86_shared
, arm64
);
3384 inline void narrowInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
3385 FloatRegister dest
) DEFINED_ON(x86_shared
);
3387 inline void narrowInt32x4(FloatRegister lhs
, FloatRegister rhs
,
3388 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3390 inline void unsignedNarrowInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
3391 FloatRegister dest
) DEFINED_ON(x86_shared
);
3393 inline void unsignedNarrowInt32x4(FloatRegister lhs
, FloatRegister rhs
,
3395 DEFINED_ON(x86_shared
, arm64
);
3397 // Integer to integer widening
3399 inline void widenLowInt8x16(FloatRegister src
, FloatRegister dest
)
3400 DEFINED_ON(x86_shared
, arm64
);
3402 inline void widenHighInt8x16(FloatRegister src
, FloatRegister dest
)
3403 DEFINED_ON(x86_shared
, arm64
);
3405 inline void unsignedWidenLowInt8x16(FloatRegister src
, FloatRegister dest
)
3406 DEFINED_ON(x86_shared
, arm64
);
3408 inline void unsignedWidenHighInt8x16(FloatRegister src
, FloatRegister dest
)
3409 DEFINED_ON(x86_shared
, arm64
);
3411 inline void widenLowInt16x8(FloatRegister src
, FloatRegister dest
)
3412 DEFINED_ON(x86_shared
, arm64
);
3414 inline void widenHighInt16x8(FloatRegister src
, FloatRegister dest
)
3415 DEFINED_ON(x86_shared
, arm64
);
3417 inline void unsignedWidenLowInt16x8(FloatRegister src
, FloatRegister dest
)
3418 DEFINED_ON(x86_shared
, arm64
);
3420 inline void unsignedWidenHighInt16x8(FloatRegister src
, FloatRegister dest
)
3421 DEFINED_ON(x86_shared
, arm64
);
3423 inline void widenLowInt32x4(FloatRegister src
, FloatRegister dest
)
3424 DEFINED_ON(x86_shared
, arm64
);
3426 inline void unsignedWidenLowInt32x4(FloatRegister src
, FloatRegister dest
)
3427 DEFINED_ON(x86_shared
, arm64
);
3429 inline void widenHighInt32x4(FloatRegister src
, FloatRegister dest
)
3430 DEFINED_ON(x86_shared
, arm64
);
3432 inline void unsignedWidenHighInt32x4(FloatRegister src
, FloatRegister dest
)
3433 DEFINED_ON(x86_shared
, arm64
);
3435 // Compare-based minimum/maximum
3437 // On x86, the signature is (rhsDest, lhs); on arm64 it is (rhs, lhsDest).
3439 // The masm preprocessor can't deal with multiple declarations with identical
3440 // signatures even if they are on different platforms, hence the weird
3443 inline void pseudoMinFloat32x4(FloatRegister rhsOrRhsDest
,
3444 FloatRegister lhsOrLhsDest
)
3445 DEFINED_ON(x86_shared
, arm64
);
3447 inline void pseudoMinFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3449 DEFINED_ON(x86_shared
, arm64
);
3451 inline void pseudoMinFloat64x2(FloatRegister rhsOrRhsDest
,
3452 FloatRegister lhsOrLhsDest
)
3453 DEFINED_ON(x86_shared
, arm64
);
3455 inline void pseudoMinFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3457 DEFINED_ON(x86_shared
, arm64
);
3459 inline void pseudoMaxFloat32x4(FloatRegister rhsOrRhsDest
,
3460 FloatRegister lhsOrLhsDest
)
3461 DEFINED_ON(x86_shared
, arm64
);
3463 inline void pseudoMaxFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3465 DEFINED_ON(x86_shared
, arm64
);
3467 inline void pseudoMaxFloat64x2(FloatRegister rhsOrRhsDest
,
3468 FloatRegister lhsOrLhsDest
)
3469 DEFINED_ON(x86_shared
, arm64
);
3471 inline void pseudoMaxFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3473 DEFINED_ON(x86_shared
, arm64
);
3475 // Widening/pairwise integer dot product
3477 inline void widenDotInt16x8(FloatRegister lhs
, FloatRegister rhs
,
3478 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3480 inline void widenDotInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
3481 FloatRegister dest
) DEFINED_ON(x86_shared
);
3483 inline void dotInt8x16Int7x16(FloatRegister lhs
, FloatRegister rhs
,
3485 DEFINED_ON(x86_shared
, arm64
);
3487 inline void dotInt8x16Int7x16ThenAdd(FloatRegister lhs
, FloatRegister rhs
,
3489 DEFINED_ON(x86_shared
);
3491 inline void dotInt8x16Int7x16ThenAdd(FloatRegister lhs
, FloatRegister rhs
,
3492 FloatRegister dest
, FloatRegister temp
)
3495 // Floating point rounding
3497 inline void ceilFloat32x4(FloatRegister src
, FloatRegister dest
)
3498 DEFINED_ON(x86_shared
, arm64
);
3500 inline void ceilFloat64x2(FloatRegister src
, FloatRegister dest
)
3501 DEFINED_ON(x86_shared
, arm64
);
3503 inline void floorFloat32x4(FloatRegister src
, FloatRegister dest
)
3504 DEFINED_ON(x86_shared
, arm64
);
3506 inline void floorFloat64x2(FloatRegister src
, FloatRegister dest
)
3507 DEFINED_ON(x86_shared
, arm64
);
3509 inline void truncFloat32x4(FloatRegister src
, FloatRegister dest
)
3510 DEFINED_ON(x86_shared
, arm64
);
3512 inline void truncFloat64x2(FloatRegister src
, FloatRegister dest
)
3513 DEFINED_ON(x86_shared
, arm64
);
3515 inline void nearestFloat32x4(FloatRegister src
, FloatRegister dest
)
3516 DEFINED_ON(x86_shared
, arm64
);
3518 inline void nearestFloat64x2(FloatRegister src
, FloatRegister dest
)
3519 DEFINED_ON(x86_shared
, arm64
);
3521 // Floating multiply-accumulate: srcDest [+-]= src1 * src2
3523 inline void fmaFloat32x4(FloatRegister src1
, FloatRegister src2
,
3524 FloatRegister srcDest
) DEFINED_ON(x86_shared
, arm64
);
3526 inline void fnmaFloat32x4(FloatRegister src1
, FloatRegister src2
,
3527 FloatRegister srcDest
)
3528 DEFINED_ON(x86_shared
, arm64
);
3530 inline void fmaFloat64x2(FloatRegister src1
, FloatRegister src2
,
3531 FloatRegister srcDest
) DEFINED_ON(x86_shared
, arm64
);
3533 inline void fnmaFloat64x2(FloatRegister src1
, FloatRegister src2
,
3534 FloatRegister srcDest
)
3535 DEFINED_ON(x86_shared
, arm64
);
3537 inline void minFloat32x4Relaxed(FloatRegister src
, FloatRegister srcDest
)
3538 DEFINED_ON(x86_shared
, arm64
);
3540 inline void minFloat32x4Relaxed(FloatRegister lhs
, FloatRegister rhs
,
3542 DEFINED_ON(x86_shared
, arm64
);
3544 inline void maxFloat32x4Relaxed(FloatRegister src
, FloatRegister srcDest
)
3545 DEFINED_ON(x86_shared
, arm64
);
3547 inline void maxFloat32x4Relaxed(FloatRegister lhs
, FloatRegister rhs
,
3549 DEFINED_ON(x86_shared
, arm64
);
3551 inline void minFloat64x2Relaxed(FloatRegister src
, FloatRegister srcDest
)
3552 DEFINED_ON(x86_shared
, arm64
);
3554 inline void minFloat64x2Relaxed(FloatRegister lhs
, FloatRegister rhs
,
3556 DEFINED_ON(x86_shared
, arm64
);
3558 inline void maxFloat64x2Relaxed(FloatRegister src
, FloatRegister srcDest
)
3559 DEFINED_ON(x86_shared
, arm64
);
3561 inline void maxFloat64x2Relaxed(FloatRegister lhs
, FloatRegister rhs
,
3563 DEFINED_ON(x86_shared
, arm64
);
3565 inline void q15MulrInt16x8Relaxed(FloatRegister lhs
, FloatRegister rhs
,
3567 DEFINED_ON(x86_shared
, arm64
);
3570 // ========================================================================
3571 // Truncate floating point.
3573 // Undefined behaviour when truncation is outside Int64 range.
3574 // Needs a temp register if SSE3 is not present.
3575 inline void truncateFloat32ToInt64(Address src
, Address dest
, Register temp
)
3576 DEFINED_ON(x86_shared
);
3577 inline void truncateFloat32ToUInt64(Address src
, Address dest
, Register temp
,
3578 FloatRegister floatTemp
)
3579 DEFINED_ON(x86
, x64
);
3580 inline void truncateDoubleToInt64(Address src
, Address dest
, Register temp
)
3581 DEFINED_ON(x86_shared
);
3582 inline void truncateDoubleToUInt64(Address src
, Address dest
, Register temp
,
3583 FloatRegister floatTemp
)
3584 DEFINED_ON(x86
, x64
);
3587 // ========================================================================
3588 // Convert floating point.
3590 // temp required on x86 and x64; must be undefined on mips64 and loong64.
3591 void convertUInt64ToFloat32(Register64 src
, FloatRegister dest
, Register temp
)
3592 DEFINED_ON(arm64
, mips64
, loong64
, riscv64
, wasm32
, x64
, x86
);
3594 void convertInt64ToFloat32(Register64 src
, FloatRegister dest
)
3595 DEFINED_ON(arm64
, mips64
, loong64
, riscv64
, wasm32
, x64
, x86
);
3597 bool convertUInt64ToDoubleNeedsTemp() PER_ARCH
;
3599 // temp required when convertUInt64ToDoubleNeedsTemp() returns true.
3600 void convertUInt64ToDouble(Register64 src
, FloatRegister dest
,
3601 Register temp
) PER_ARCH
;
3603 void convertInt64ToDouble(Register64 src
, FloatRegister dest
) PER_ARCH
;
3605 void convertIntPtrToDouble(Register src
, FloatRegister dest
) PER_ARCH
;
3608 // ========================================================================
3611 CodeOffset
wasmTrapInstruction() PER_SHARED_ARCH
;
3613 void wasmTrap(wasm::Trap trap
, wasm::BytecodeOffset bytecodeOffset
);
3615 // Load all pinned regs via InstanceReg. If the trapOffset is something,
3616 // give the first load a trap descriptor with type IndirectCallToNull, so that
3617 // a null instance will cause a trap.
3618 void loadWasmPinnedRegsFromInstance(
3619 mozilla::Maybe
<wasm::BytecodeOffset
> trapOffset
= mozilla::Nothing());
3621 // Returns a pair: the offset of the undefined (trapping) instruction, and
3622 // the number of extra bytes of stack allocated prior to the trap
3623 // instruction proper.
3624 std::pair
<CodeOffset
, uint32_t> wasmReserveStackChecked(
3625 uint32_t amount
, wasm::BytecodeOffset trapOffset
);
3627 // Emit a bounds check against the wasm heap limit, jumping to 'ok' if 'cond'
3628 // holds; this can be the label either of the access or of the trap. The
3629 // label should name a code position greater than the position of the bounds
3632 // If JitOptions.spectreMaskIndex is true, a no-op speculation barrier is
3633 // emitted in the code stream after the check to prevent an OOB access from
3634 // being executed speculatively. (On current tier-1 platforms the barrier is
3635 // a conditional saturation of 'index' to 'boundsCheckLimit', using the same
3636 // condition as the check.) If the condition is such that the bounds check
3637 // branches out of line to the trap, the barrier will actually be executed
3638 // when the bounds check passes.
3640 // On 32-bit systems for both wasm and asm.js, and on 64-bit systems for
3641 // asm.js, heap lengths are limited to 2GB. On 64-bit systems for wasm,
3642 // 32-bit heap lengths are limited to 4GB, and 64-bit heap lengths will be
3643 // limited to something much larger.
3645 void wasmBoundsCheck32(Condition cond
, Register index
,
3646 Register boundsCheckLimit
, Label
* ok
)
3647 DEFINED_ON(arm
, arm64
, mips32
, mips64
, x86_shared
, loong64
, riscv64
,
3650 void wasmBoundsCheck32(Condition cond
, Register index
,
3651 Address boundsCheckLimit
, Label
* ok
)
3652 DEFINED_ON(arm
, arm64
, mips32
, mips64
, x86_shared
, loong64
, riscv64
,
3655 void wasmBoundsCheck64(Condition cond
, Register64 index
,
3656 Register64 boundsCheckLimit
, Label
* ok
)
3657 DEFINED_ON(arm64
, mips64
, x64
, x86
, arm
, loong64
, riscv64
, wasm32
);
3659 void wasmBoundsCheck64(Condition cond
, Register64 index
,
3660 Address boundsCheckLimit
, Label
* ok
)
3661 DEFINED_ON(arm64
, mips64
, x64
, x86
, arm
, loong64
, riscv64
, wasm32
);
3663 // Each wasm load/store instruction appends its own wasm::Trap::OutOfBounds.
3664 void wasmLoad(const wasm::MemoryAccessDesc
& access
, Operand srcAddr
,
3665 AnyRegister out
) DEFINED_ON(x86
, x64
);
3666 void wasmLoadI64(const wasm::MemoryAccessDesc
& access
, Operand srcAddr
,
3667 Register64 out
) DEFINED_ON(x86
, x64
);
3668 void wasmStore(const wasm::MemoryAccessDesc
& access
, AnyRegister value
,
3669 Operand dstAddr
) DEFINED_ON(x86
, x64
);
3670 void wasmStoreI64(const wasm::MemoryAccessDesc
& access
, Register64 value
,
3671 Operand dstAddr
) DEFINED_ON(x86
);
3673 // For all the ARM/MIPS/LOONG64 wasmLoad and wasmStore functions below, `ptr`
3674 // MUST equal `ptrScratch`, and that register will be updated based on
3675 // conditions listed below (where it is only mentioned as `ptr`).
3677 // `ptr` will be updated if access.offset() != 0 or access.type() ==
3679 void wasmLoad(const wasm::MemoryAccessDesc
& access
, Register memoryBase
,
3680 Register ptr
, Register ptrScratch
, AnyRegister output
)
3681 DEFINED_ON(arm
, loong64
, riscv64
, mips_shared
);
3682 void wasmLoadI64(const wasm::MemoryAccessDesc
& access
, Register memoryBase
,
3683 Register ptr
, Register ptrScratch
, Register64 output
)
3684 DEFINED_ON(arm
, mips32
, mips64
, loong64
, riscv64
);
3685 void wasmStore(const wasm::MemoryAccessDesc
& access
, AnyRegister value
,
3686 Register memoryBase
, Register ptr
, Register ptrScratch
)
3687 DEFINED_ON(arm
, loong64
, riscv64
, mips_shared
);
3688 void wasmStoreI64(const wasm::MemoryAccessDesc
& access
, Register64 value
,
3689 Register memoryBase
, Register ptr
, Register ptrScratch
)
3690 DEFINED_ON(arm
, mips32
, mips64
, loong64
, riscv64
);
3692 // These accept general memoryBase + ptr + offset (in `access`); the offset is
3693 // always smaller than the guard region. They will insert an additional add
3694 // if the offset is nonzero, and of course that add may require a temporary
3695 // register for the offset if the offset is large, and instructions to set it
3697 void wasmLoad(const wasm::MemoryAccessDesc
& access
, Register memoryBase
,
3698 Register ptr
, AnyRegister output
) DEFINED_ON(arm64
);
3699 void wasmLoadI64(const wasm::MemoryAccessDesc
& access
, Register memoryBase
,
3700 Register ptr
, Register64 output
) DEFINED_ON(arm64
);
3701 void wasmStore(const wasm::MemoryAccessDesc
& access
, AnyRegister value
,
3702 Register memoryBase
, Register ptr
) DEFINED_ON(arm64
);
3703 void wasmStoreI64(const wasm::MemoryAccessDesc
& access
, Register64 value
,
3704 Register memoryBase
, Register ptr
) DEFINED_ON(arm64
);
3706 // `ptr` will always be updated.
3707 void wasmUnalignedLoad(const wasm::MemoryAccessDesc
& access
,
3708 Register memoryBase
, Register ptr
, Register ptrScratch
,
3709 Register output
, Register tmp
)
3710 DEFINED_ON(mips32
, mips64
);
3712 // MIPS: `ptr` will always be updated.
3713 void wasmUnalignedLoadFP(const wasm::MemoryAccessDesc
& access
,
3714 Register memoryBase
, Register ptr
,
3715 Register ptrScratch
, FloatRegister output
,
3716 Register tmp1
) DEFINED_ON(mips32
, mips64
);
3718 // `ptr` will always be updated.
3719 void wasmUnalignedLoadI64(const wasm::MemoryAccessDesc
& access
,
3720 Register memoryBase
, Register ptr
,
3721 Register ptrScratch
, Register64 output
,
3722 Register tmp
) DEFINED_ON(mips32
, mips64
);
3724 // MIPS: `ptr` will always be updated.
3725 void wasmUnalignedStore(const wasm::MemoryAccessDesc
& access
, Register value
,
3726 Register memoryBase
, Register ptr
,
3727 Register ptrScratch
, Register tmp
)
3728 DEFINED_ON(mips32
, mips64
);
3730 // `ptr` will always be updated.
3731 void wasmUnalignedStoreFP(const wasm::MemoryAccessDesc
& access
,
3732 FloatRegister floatValue
, Register memoryBase
,
3733 Register ptr
, Register ptrScratch
, Register tmp
)
3734 DEFINED_ON(mips32
, mips64
);
3736 // `ptr` will always be updated.
3737 void wasmUnalignedStoreI64(const wasm::MemoryAccessDesc
& access
,
3738 Register64 value
, Register memoryBase
,
3739 Register ptr
, Register ptrScratch
, Register tmp
)
3740 DEFINED_ON(mips32
, mips64
);
3742 // wasm specific methods, used in both the wasm baseline compiler and ion.
3744 // The truncate-to-int32 methods do not bind the rejoin label; clients must
3745 // do so if oolWasmTruncateCheckF64ToI32() can jump to it.
3746 void wasmTruncateDoubleToUInt32(FloatRegister input
, Register output
,
3747 bool isSaturating
, Label
* oolEntry
) PER_ARCH
;
3748 void wasmTruncateDoubleToInt32(FloatRegister input
, Register output
,
3750 Label
* oolEntry
) PER_SHARED_ARCH
;
3751 void oolWasmTruncateCheckF64ToI32(FloatRegister input
, Register output
,
3752 TruncFlags flags
, wasm::BytecodeOffset off
,
3754 DEFINED_ON(arm
, arm64
, x86_shared
, mips_shared
, loong64
, riscv64
, wasm32
);
3756 void wasmTruncateFloat32ToUInt32(FloatRegister input
, Register output
,
3757 bool isSaturating
, Label
* oolEntry
) PER_ARCH
;
3758 void wasmTruncateFloat32ToInt32(FloatRegister input
, Register output
,
3760 Label
* oolEntry
) PER_SHARED_ARCH
;
3761 void oolWasmTruncateCheckF32ToI32(FloatRegister input
, Register output
,
3762 TruncFlags flags
, wasm::BytecodeOffset off
,
3764 DEFINED_ON(arm
, arm64
, x86_shared
, mips_shared
, loong64
, riscv64
, wasm32
);
3766 // The truncate-to-int64 methods will always bind the `oolRejoin` label
3767 // after the last emitted instruction.
3768 void wasmTruncateDoubleToInt64(FloatRegister input
, Register64 output
,
3769 bool isSaturating
, Label
* oolEntry
,
3770 Label
* oolRejoin
, FloatRegister tempDouble
)
3771 DEFINED_ON(arm64
, x86
, x64
, mips64
, loong64
, riscv64
, wasm32
);
3772 void wasmTruncateDoubleToUInt64(FloatRegister input
, Register64 output
,
3773 bool isSaturating
, Label
* oolEntry
,
3774 Label
* oolRejoin
, FloatRegister tempDouble
)
3775 DEFINED_ON(arm64
, x86
, x64
, mips64
, loong64
, riscv64
, wasm32
);
3776 void oolWasmTruncateCheckF64ToI64(FloatRegister input
, Register64 output
,
3777 TruncFlags flags
, wasm::BytecodeOffset off
,
3779 DEFINED_ON(arm
, arm64
, x86_shared
, mips_shared
, loong64
, riscv64
, wasm32
);
3781 void wasmTruncateFloat32ToInt64(FloatRegister input
, Register64 output
,
3782 bool isSaturating
, Label
* oolEntry
,
3783 Label
* oolRejoin
, FloatRegister tempDouble
)
3784 DEFINED_ON(arm64
, x86
, x64
, mips64
, loong64
, riscv64
, wasm32
);
3785 void wasmTruncateFloat32ToUInt64(FloatRegister input
, Register64 output
,
3786 bool isSaturating
, Label
* oolEntry
,
3787 Label
* oolRejoin
, FloatRegister tempDouble
)
3788 DEFINED_ON(arm64
, x86
, x64
, mips64
, loong64
, riscv64
, wasm32
);
3789 void oolWasmTruncateCheckF32ToI64(FloatRegister input
, Register64 output
,
3790 TruncFlags flags
, wasm::BytecodeOffset off
,
3792 DEFINED_ON(arm
, arm64
, x86_shared
, mips_shared
, loong64
, riscv64
, wasm32
);
3794 // This function takes care of loading the callee's instance and pinned regs
3795 // but it is the caller's responsibility to save/restore instance or pinned
3797 CodeOffset
wasmCallImport(const wasm::CallSiteDesc
& desc
,
3798 const wasm::CalleeDesc
& callee
);
3800 // WasmTableCallIndexReg must contain the index of the indirect call. This is
3801 // for wasm calls only.
3803 // Indirect calls use a dual-path mechanism where a run-time test determines
3804 // whether a context switch is needed (slow path) or not (fast path). This
3805 // gives rise to two call instructions, both of which need safe points. As
3806 // per normal, the call offsets are the code offsets at the end of the call
3807 // instructions (the return points).
3809 // `boundsCheckFailedLabel` is non-null iff a bounds check is required.
3810 // `nullCheckFailedLabel` is non-null only on platforms that can't fold the
3811 // null check into the rest of the call instructions.
3812 void wasmCallIndirect(const wasm::CallSiteDesc
& desc
,
3813 const wasm::CalleeDesc
& callee
,
3814 Label
* boundsCheckFailedLabel
,
3815 Label
* nullCheckFailedLabel
,
3816 mozilla::Maybe
<uint32_t> tableSize
,
3817 CodeOffset
* fastCallOffset
, CodeOffset
* slowCallOffset
);
3819 // This function takes care of loading the callee's instance and address from
3821 void wasmCallRef(const wasm::CallSiteDesc
& desc
,
3822 const wasm::CalleeDesc
& callee
, CodeOffset
* fastCallOffset
,
3823 CodeOffset
* slowCallOffset
);
3825 // WasmTableCallIndexReg must contain the index of the indirect call.
3826 // This is for asm.js calls only.
3827 CodeOffset
asmCallIndirect(const wasm::CallSiteDesc
& desc
,
3828 const wasm::CalleeDesc
& callee
);
3830 // This function takes care of loading the pointer to the current instance
3831 // as the implicit first argument. It preserves instance and pinned registers.
3832 // (instance & pinned regs are non-volatile registers in the system ABI).
3833 CodeOffset
wasmCallBuiltinInstanceMethod(const wasm::CallSiteDesc
& desc
,
3834 const ABIArg
& instanceArg
,
3835 wasm::SymbolicAddress builtin
,
3836 wasm::FailureMode failureMode
);
3838 // Perform a subtype check that `object` is a subtype of `type`, branching to
3839 // `label` depending on `onSuccess`. `type` must be in the `any` hierarchy.
3841 // `superSuperTypeVector` is required iff the destination type is a concrete
3842 // type. `scratch1` is required iff the destination type is eq or lower and
3843 // not none. `scratch2` is required iff the destination type is a concrete
3844 // type and its `subTypingDepth` is >= wasm::MinSuperTypeVectorLength.
3846 // `object` and `superSuperTypeVector` are preserved. Scratch registers are
3848 void branchWasmGcObjectIsRefType(Register object
, wasm::RefType sourceType
,
3849 wasm::RefType destType
, Label
* label
,
3851 Register superSuperTypeVector
,
3852 Register scratch1
, Register scratch2
);
3853 static bool needScratch1ForBranchWasmGcRefType(wasm::RefType type
);
3854 static bool needScratch2ForBranchWasmGcRefType(wasm::RefType type
);
3855 static bool needSuperSuperTypeVectorForBranchWasmGcRefType(
3856 wasm::RefType type
);
3858 // Perform a subtype check that `subSuperTypeVector` is a subtype of
3859 // `superSuperTypeVector`, branching to `label` depending on `onSuccess`.
3860 // This method is a specialization of the general
3861 // `wasm::TypeDef::isSubTypeOf` method for the case where the
3862 // `superSuperTypeVector` is statically known, which is the case for all
3863 // wasm instructions.
3865 // `scratch` is required iff the `subTypeDepth` is >=
3866 // wasm::MinSuperTypeVectorLength. `subSuperTypeVector` is clobbered by this
3867 // method. `superSuperTypeVector` is preserved.
3868 void branchWasmSuperTypeVectorIsSubtype(Register subSuperTypeVector
,
3869 Register superSuperTypeVector
,
3871 uint32_t superTypeDepth
, Label
* label
,
3874 // Compute ptr += (indexTemp32 << shift) where shift can be any value < 32.
3875 // May destroy indexTemp32. The value of indexTemp32 must be positive, and it
3876 // is implementation-defined what happens if bits are lost or the value
3877 // becomes negative through the shift. On 64-bit systems, the high 32 bits of
3878 // indexTemp32 must be zero, not garbage.
3879 void shiftIndex32AndAdd(Register indexTemp32
, int shift
,
3880 Register pointer
) PER_SHARED_ARCH
;
3882 // The System ABI frequently states that the high bits of a 64-bit register
3883 // that holds a 32-bit return value are unpredictable, and C++ compilers will
3884 // indeed generate code that leaves garbage in the upper bits.
3886 // Adjust the contents of the 64-bit register `r` to conform to our internal
3887 // convention, which requires predictable high bits. In practice, this means
3888 // that the 32-bit value will be zero-extended or sign-extended to 64 bits as
3889 // appropriate for the platform.
3890 void widenInt32(Register r
) DEFINED_ON(arm64
, x64
, mips64
, loong64
, riscv64
);
3892 // As enterFakeExitFrame(), but using register conventions appropriate for
3894 void enterFakeExitFrameForWasm(Register cxreg
, Register scratch
,
3895 ExitFrameType type
) PER_SHARED_ARCH
;
3898 // ========================================================================
3899 // Barrier functions.
3901 void emitPreBarrierFastPath(JSRuntime
* rt
, MIRType type
, Register temp1
,
3902 Register temp2
, Register temp3
, Label
* noBarrier
);
3905 // ========================================================================
3906 // Clamping functions.
3908 inline void clampIntToUint8(Register reg
) PER_SHARED_ARCH
;
3911 // ========================================================================
3912 // Primitive atomic operations.
3914 // If the access is from JS and the eventual destination of the result is a
3915 // js::Value, it's probably best to use the JS-specific versions of these,
3916 // see further below.
3918 // Temp registers must be defined unless otherwise noted in the per-function
3921 // 8-bit, 16-bit, and 32-bit wide operations.
3923 // The 8-bit and 16-bit operations zero-extend or sign-extend the result to
3924 // 32 bits, according to `type`. On 64-bit systems, the upper 32 bits of the
3925 // result will be zero on some platforms (eg, on x64) and will be the sign
3926 // extension of the lower bits on other platforms (eg, MIPS).
3928 // CompareExchange with memory. Return the value that was in memory,
3929 // whether we wrote or not.
3931 // x86-shared: `output` must be eax.
3932 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
3933 // and 16-bit wide operations.
3935 void compareExchange(Scalar::Type type
, const Synchronization
& sync
,
3936 const Address
& mem
, Register expected
,
3937 Register replacement
, Register output
)
3938 DEFINED_ON(arm
, arm64
, x86_shared
);
3940 void compareExchange(Scalar::Type type
, const Synchronization
& sync
,
3941 const BaseIndex
& mem
, Register expected
,
3942 Register replacement
, Register output
)
3943 DEFINED_ON(arm
, arm64
, x86_shared
);
3945 void compareExchange(Scalar::Type type
, const Synchronization
& sync
,
3946 const Address
& mem
, Register expected
,
3947 Register replacement
, Register valueTemp
,
3948 Register offsetTemp
, Register maskTemp
, Register output
)
3949 DEFINED_ON(mips_shared
, loong64
, riscv64
);
3951 void compareExchange(Scalar::Type type
, const Synchronization
& sync
,
3952 const BaseIndex
& mem
, Register expected
,
3953 Register replacement
, Register valueTemp
,
3954 Register offsetTemp
, Register maskTemp
, Register output
)
3955 DEFINED_ON(mips_shared
, loong64
, riscv64
);
3957 // x86: `expected` and `output` must be edx:eax; `replacement` is ecx:ebx.
3958 // x64: `output` must be rax.
3959 // ARM: Registers must be distinct; `replacement` and `output` must be
3960 // (even,odd) pairs.
3962 void compareExchange64(const Synchronization
& sync
, const Address
& mem
,
3963 Register64 expected
, Register64 replacement
,
3965 DEFINED_ON(arm
, arm64
, x64
, x86
, mips64
, loong64
, riscv64
);
3967 void compareExchange64(const Synchronization
& sync
, const BaseIndex
& mem
,
3968 Register64 expected
, Register64 replacement
,
3970 DEFINED_ON(arm
, arm64
, x64
, x86
, mips64
, loong64
, riscv64
);
3972 // Exchange with memory. Return the value initially in memory.
3973 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
3974 // and 16-bit wide operations.
3976 void atomicExchange(Scalar::Type type
, const Synchronization
& sync
,
3977 const Address
& mem
, Register value
, Register output
)
3978 DEFINED_ON(arm
, arm64
, x86_shared
);
3980 void atomicExchange(Scalar::Type type
, const Synchronization
& sync
,
3981 const BaseIndex
& mem
, Register value
, Register output
)
3982 DEFINED_ON(arm
, arm64
, x86_shared
);
3984 void atomicExchange(Scalar::Type type
, const Synchronization
& sync
,
3985 const Address
& mem
, Register value
, Register valueTemp
,
3986 Register offsetTemp
, Register maskTemp
, Register output
)
3987 DEFINED_ON(mips_shared
, loong64
, riscv64
);
3989 void atomicExchange(Scalar::Type type
, const Synchronization
& sync
,
3990 const BaseIndex
& mem
, Register value
, Register valueTemp
,
3991 Register offsetTemp
, Register maskTemp
, Register output
)
3992 DEFINED_ON(mips_shared
, loong64
, riscv64
);
3994 // x86: `value` must be ecx:ebx; `output` must be edx:eax.
3995 // ARM: `value` and `output` must be distinct and (even,odd) pairs.
3996 // ARM64: `value` and `output` must be distinct.
3998 void atomicExchange64(const Synchronization
& sync
, const Address
& mem
,
3999 Register64 value
, Register64 output
)
4000 DEFINED_ON(arm
, arm64
, x64
, x86
, mips64
, loong64
, riscv64
);
4002 void atomicExchange64(const Synchronization
& sync
, const BaseIndex
& mem
,
4003 Register64 value
, Register64 output
)
4004 DEFINED_ON(arm
, arm64
, x64
, x86
, mips64
, loong64
, riscv64
);
4006 // Read-modify-write with memory. Return the value in memory before the
4010 // For 8-bit operations, `value` and `output` must have a byte subregister.
4011 // For Add and Sub, `temp` must be invalid.
4012 // For And, Or, and Xor, `output` must be eax and `temp` must have a byte
4015 // ARM: Registers `value` and `output` must differ.
4016 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
4017 // and 16-bit wide operations; `value` and `output` must differ.
4019 void atomicFetchOp(Scalar::Type type
, const Synchronization
& sync
,
4020 AtomicOp op
, Register value
, const Address
& mem
,
4021 Register temp
, Register output
)
4022 DEFINED_ON(arm
, arm64
, x86_shared
);
4024 void atomicFetchOp(Scalar::Type type
, const Synchronization
& sync
,
4025 AtomicOp op
, Imm32 value
, const Address
& mem
,
4026 Register temp
, Register output
) DEFINED_ON(x86_shared
);
4028 void atomicFetchOp(Scalar::Type type
, const Synchronization
& sync
,
4029 AtomicOp op
, Register value
, const BaseIndex
& mem
,
4030 Register temp
, Register output
)
4031 DEFINED_ON(arm
, arm64
, x86_shared
);
4033 void atomicFetchOp(Scalar::Type type
, const Synchronization
& sync
,
4034 AtomicOp op
, Imm32 value
, const BaseIndex
& mem
,
4035 Register temp
, Register output
) DEFINED_ON(x86_shared
);
4037 void atomicFetchOp(Scalar::Type type
, const Synchronization
& sync
,
4038 AtomicOp op
, Register value
, const Address
& mem
,
4039 Register valueTemp
, Register offsetTemp
, Register maskTemp
,
4040 Register output
) DEFINED_ON(mips_shared
, loong64
, riscv64
);
4042 void atomicFetchOp(Scalar::Type type
, const Synchronization
& sync
,
4043 AtomicOp op
, Register value
, const BaseIndex
& mem
,
4044 Register valueTemp
, Register offsetTemp
, Register maskTemp
,
4045 Register output
) DEFINED_ON(mips_shared
, loong64
, riscv64
);
4048 // `temp` must be ecx:ebx; `output` must be edx:eax.
4050 // For Add and Sub, `temp` is ignored.
4051 // For And, Or, and Xor, `output` must be rax.
4053 // `temp` and `output` must be (even,odd) pairs and distinct from `value`.
4055 // Registers `value`, `temp`, and `output` must all differ.
4057 void atomicFetchOp64(const Synchronization
& sync
, AtomicOp op
,
4058 Register64 value
, const Address
& mem
, Register64 temp
,
4060 DEFINED_ON(arm
, arm64
, x64
, mips64
, loong64
, riscv64
);
4062 void atomicFetchOp64(const Synchronization
& sync
, AtomicOp op
,
4063 const Address
& value
, const Address
& mem
,
4064 Register64 temp
, Register64 output
) DEFINED_ON(x86
);
4066 void atomicFetchOp64(const Synchronization
& sync
, AtomicOp op
,
4067 Register64 value
, const BaseIndex
& mem
, Register64 temp
,
4069 DEFINED_ON(arm
, arm64
, x64
, mips64
, loong64
, riscv64
);
4071 void atomicFetchOp64(const Synchronization
& sync
, AtomicOp op
,
4072 const Address
& value
, const BaseIndex
& mem
,
4073 Register64 temp
, Register64 output
) DEFINED_ON(x86
);
4076 // `value` can be any register.
4078 // `temp` must be an (even,odd) pair and distinct from `value`.
4080 // Registers `value` and `temp` must differ.
4082 void atomicEffectOp64(const Synchronization
& sync
, AtomicOp op
,
4083 Register64 value
, const Address
& mem
) DEFINED_ON(x64
);
4085 void atomicEffectOp64(const Synchronization
& sync
, AtomicOp op
,
4086 Register64 value
, const Address
& mem
, Register64 temp
)
4087 DEFINED_ON(arm
, arm64
, mips64
, loong64
, riscv64
);
4089 void atomicEffectOp64(const Synchronization
& sync
, AtomicOp op
,
4090 Register64 value
, const BaseIndex
& mem
) DEFINED_ON(x64
);
4092 void atomicEffectOp64(const Synchronization
& sync
, AtomicOp op
,
4093 Register64 value
, const BaseIndex
& mem
, Register64 temp
)
4094 DEFINED_ON(arm
, arm64
, mips64
, loong64
, riscv64
);
4096 // 64-bit atomic load. On 64-bit systems, use regular load with
4097 // Synchronization::Load, not this method.
4099 // x86: `temp` must be ecx:ebx; `output` must be edx:eax.
4100 // ARM: `output` must be (even,odd) pair.
4102 void atomicLoad64(const Synchronization
& sync
, const Address
& mem
,
4103 Register64 temp
, Register64 output
) DEFINED_ON(x86
);
4105 void atomicLoad64(const Synchronization
& sync
, const BaseIndex
& mem
,
4106 Register64 temp
, Register64 output
) DEFINED_ON(x86
);
4108 void atomicLoad64(const Synchronization
& sync
, const Address
& mem
,
4109 Register64 output
) DEFINED_ON(arm
);
4111 void atomicLoad64(const Synchronization
& sync
, const BaseIndex
& mem
,
4112 Register64 output
) DEFINED_ON(arm
);
4114 // 64-bit atomic store. On 64-bit systems, use regular store with
4115 // Synchronization::Store, not this method.
4117 // x86: `value` must be ecx:ebx; `temp` must be edx:eax.
4118 // ARM: `value` and `temp` must be (even,odd) pairs.
4120 void atomicStore64(const Synchronization
& sync
, const Address
& mem
,
4121 Register64 value
, Register64 temp
) DEFINED_ON(x86
, arm
);
4123 void atomicStore64(const Synchronization
& sync
, const BaseIndex
& mem
,
4124 Register64 value
, Register64 temp
) DEFINED_ON(x86
, arm
);
4126 // ========================================================================
4127 // Wasm atomic operations.
4129 // Constraints, when omitted, are exactly as for the primitive operations
4132 void wasmCompareExchange(const wasm::MemoryAccessDesc
& access
,
4133 const Address
& mem
, Register expected
,
4134 Register replacement
, Register output
)
4135 DEFINED_ON(arm
, arm64
, x86_shared
);
4137 void wasmCompareExchange(const wasm::MemoryAccessDesc
& access
,
4138 const BaseIndex
& mem
, Register expected
,
4139 Register replacement
, Register output
)
4140 DEFINED_ON(arm
, arm64
, x86_shared
);
4142 void wasmCompareExchange(const wasm::MemoryAccessDesc
& access
,
4143 const Address
& mem
, Register expected
,
4144 Register replacement
, Register valueTemp
,
4145 Register offsetTemp
, Register maskTemp
,
4147 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4149 void wasmCompareExchange(const wasm::MemoryAccessDesc
& access
,
4150 const BaseIndex
& mem
, Register expected
,
4151 Register replacement
, Register valueTemp
,
4152 Register offsetTemp
, Register maskTemp
,
4154 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4156 void wasmAtomicExchange(const wasm::MemoryAccessDesc
& access
,
4157 const Address
& mem
, Register value
, Register output
)
4158 DEFINED_ON(arm
, arm64
, x86_shared
);
4160 void wasmAtomicExchange(const wasm::MemoryAccessDesc
& access
,
4161 const BaseIndex
& mem
, Register value
, Register output
)
4162 DEFINED_ON(arm
, arm64
, x86_shared
);
4164 void wasmAtomicExchange(const wasm::MemoryAccessDesc
& access
,
4165 const Address
& mem
, Register value
,
4166 Register valueTemp
, Register offsetTemp
,
4167 Register maskTemp
, Register output
)
4168 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4170 void wasmAtomicExchange(const wasm::MemoryAccessDesc
& access
,
4171 const BaseIndex
& mem
, Register value
,
4172 Register valueTemp
, Register offsetTemp
,
4173 Register maskTemp
, Register output
)
4174 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4176 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4177 Register value
, const Address
& mem
, Register temp
,
4178 Register output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4180 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4181 Imm32 value
, const Address
& mem
, Register temp
,
4182 Register output
) DEFINED_ON(x86_shared
);
4184 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4185 Register value
, const BaseIndex
& mem
, Register temp
,
4186 Register output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4188 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4189 Imm32 value
, const BaseIndex
& mem
, Register temp
,
4190 Register output
) DEFINED_ON(x86_shared
);
4192 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4193 Register value
, const Address
& mem
, Register valueTemp
,
4194 Register offsetTemp
, Register maskTemp
,
4196 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4198 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4199 Register value
, const BaseIndex
& mem
,
4200 Register valueTemp
, Register offsetTemp
,
4201 Register maskTemp
, Register output
)
4202 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4204 // Read-modify-write with memory. Return no value.
4206 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
4207 // and 16-bit wide operations.
4209 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4210 Register value
, const Address
& mem
, Register temp
)
4211 DEFINED_ON(arm
, arm64
, x86_shared
);
4213 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4214 Imm32 value
, const Address
& mem
, Register temp
)
4215 DEFINED_ON(x86_shared
);
4217 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4218 Register value
, const BaseIndex
& mem
, Register temp
)
4219 DEFINED_ON(arm
, arm64
, x86_shared
);
4221 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4222 Imm32 value
, const BaseIndex
& mem
, Register temp
)
4223 DEFINED_ON(x86_shared
);
4225 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4226 Register value
, const Address
& mem
,
4227 Register valueTemp
, Register offsetTemp
,
4229 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4231 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4232 Register value
, const BaseIndex
& mem
,
4233 Register valueTemp
, Register offsetTemp
,
4235 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4237 // 64-bit wide operations.
4239 // 64-bit atomic load. On 64-bit systems, use regular wasm load with
4240 // Synchronization::Load, not this method.
4242 // x86: `temp` must be ecx:ebx; `output` must be edx:eax.
4243 // ARM: `temp` should be invalid; `output` must be (even,odd) pair.
4244 // MIPS32: `temp` should be invalid.
4246 void wasmAtomicLoad64(const wasm::MemoryAccessDesc
& access
,
4247 const Address
& mem
, Register64 temp
, Register64 output
)
4248 DEFINED_ON(arm
, mips32
, x86
, wasm32
);
4250 void wasmAtomicLoad64(const wasm::MemoryAccessDesc
& access
,
4251 const BaseIndex
& mem
, Register64 temp
,
4252 Register64 output
) DEFINED_ON(arm
, mips32
, x86
, wasm32
);
4254 // x86: `expected` must be the same as `output`, and must be edx:eax.
4255 // x86: `replacement` must be ecx:ebx.
4256 // x64: `output` must be rax.
4257 // ARM: Registers must be distinct; `replacement` and `output` must be
4258 // (even,odd) pairs.
4259 // ARM64: The base register in `mem` must not overlap `output`.
4260 // MIPS: Registers must be distinct.
4262 void wasmCompareExchange64(const wasm::MemoryAccessDesc
& access
,
4263 const Address
& mem
, Register64 expected
,
4264 Register64 replacement
,
4265 Register64 output
) PER_ARCH
;
4267 void wasmCompareExchange64(const wasm::MemoryAccessDesc
& access
,
4268 const BaseIndex
& mem
, Register64 expected
,
4269 Register64 replacement
,
4270 Register64 output
) PER_ARCH
;
4272 // x86: `value` must be ecx:ebx; `output` must be edx:eax.
4273 // ARM: Registers must be distinct; `value` and `output` must be (even,odd)
4275 // MIPS: Registers must be distinct.
4277 void wasmAtomicExchange64(const wasm::MemoryAccessDesc
& access
,
4278 const Address
& mem
, Register64 value
,
4279 Register64 output
) PER_ARCH
;
4281 void wasmAtomicExchange64(const wasm::MemoryAccessDesc
& access
,
4282 const BaseIndex
& mem
, Register64 value
,
4283 Register64 output
) PER_ARCH
;
4285 // x86: `output` must be edx:eax, `temp` must be ecx:ebx.
4286 // x64: For And, Or, and Xor `output` must be rax.
4287 // ARM: Registers must be distinct; `temp` and `output` must be (even,odd)
4289 // MIPS: Registers must be distinct.
4290 // MIPS32: `temp` should be invalid.
4292 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4293 Register64 value
, const Address
& mem
,
4294 Register64 temp
, Register64 output
)
4295 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, x64
);
4297 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4298 Register64 value
, const BaseIndex
& mem
,
4299 Register64 temp
, Register64 output
)
4300 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, x64
);
4302 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4303 const Address
& value
, const Address
& mem
,
4304 Register64 temp
, Register64 output
) DEFINED_ON(x86
);
4306 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4307 const Address
& value
, const BaseIndex
& mem
,
4308 Register64 temp
, Register64 output
) DEFINED_ON(x86
);
4310 // Here `value` can be any register.
4312 void wasmAtomicEffectOp64(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4313 Register64 value
, const BaseIndex
& mem
)
4316 void wasmAtomicEffectOp64(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4317 Register64 value
, const BaseIndex
& mem
,
4318 Register64 temp
) DEFINED_ON(arm64
);
4320 // ========================================================================
4321 // JS atomic operations.
4323 // Here the arrayType must be a type that is valid for JS. As of 2017 that
4324 // is an 8-bit, 16-bit, or 32-bit integer type.
4326 // If arrayType is Scalar::Uint32 then:
4328 // - `output` must be a float register
4329 // - if the operation takes one temp register then `temp` must be defined
4330 // - if the operation takes two temp registers then `temp2` must be defined.
4332 // Otherwise `output` must be a GPR and `temp`/`temp2` should be InvalidReg.
4333 // (`temp1` must always be valid.)
4335 // For additional register constraints, see the primitive 32-bit operations
4336 // and/or wasm operations above.
4338 void compareExchangeJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4339 const Address
& mem
, Register expected
,
4340 Register replacement
, Register temp
,
4341 AnyRegister output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4343 void compareExchangeJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4344 const BaseIndex
& mem
, Register expected
,
4345 Register replacement
, Register temp
,
4346 AnyRegister output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4348 void compareExchangeJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4349 const Address
& mem
, Register expected
,
4350 Register replacement
, Register valueTemp
,
4351 Register offsetTemp
, Register maskTemp
, Register temp
,
4353 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4355 void compareExchangeJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4356 const BaseIndex
& mem
, Register expected
,
4357 Register replacement
, Register valueTemp
,
4358 Register offsetTemp
, Register maskTemp
, Register temp
,
4360 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4362 void atomicExchangeJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4363 const Address
& mem
, Register value
, Register temp
,
4364 AnyRegister output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4366 void atomicExchangeJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4367 const BaseIndex
& mem
, Register value
, Register temp
,
4368 AnyRegister output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4370 void atomicExchangeJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4371 const Address
& mem
, Register value
, Register valueTemp
,
4372 Register offsetTemp
, Register maskTemp
, Register temp
,
4374 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4376 void atomicExchangeJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4377 const BaseIndex
& mem
, Register value
,
4378 Register valueTemp
, Register offsetTemp
,
4379 Register maskTemp
, Register temp
, AnyRegister output
)
4380 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4382 void atomicFetchOpJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4383 AtomicOp op
, Register value
, const Address
& mem
,
4384 Register temp1
, Register temp2
, AnyRegister output
)
4385 DEFINED_ON(arm
, arm64
, x86_shared
);
4387 void atomicFetchOpJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4388 AtomicOp op
, Register value
, const BaseIndex
& mem
,
4389 Register temp1
, Register temp2
, AnyRegister output
)
4390 DEFINED_ON(arm
, arm64
, x86_shared
);
4392 void atomicFetchOpJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4393 AtomicOp op
, Imm32 value
, const Address
& mem
,
4394 Register temp1
, Register temp2
, AnyRegister output
)
4395 DEFINED_ON(x86_shared
);
4397 void atomicFetchOpJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4398 AtomicOp op
, Imm32 value
, const BaseIndex
& mem
,
4399 Register temp1
, Register temp2
, AnyRegister output
)
4400 DEFINED_ON(x86_shared
);
4402 void atomicFetchOpJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4403 AtomicOp op
, Register value
, const Address
& mem
,
4404 Register valueTemp
, Register offsetTemp
,
4405 Register maskTemp
, Register temp
, AnyRegister output
)
4406 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4408 void atomicFetchOpJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4409 AtomicOp op
, Register value
, const BaseIndex
& mem
,
4410 Register valueTemp
, Register offsetTemp
,
4411 Register maskTemp
, Register temp
, AnyRegister output
)
4412 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4414 void atomicEffectOpJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4415 AtomicOp op
, Register value
, const Address
& mem
,
4416 Register temp
) DEFINED_ON(arm
, arm64
, x86_shared
);
4418 void atomicEffectOpJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4419 AtomicOp op
, Register value
, const BaseIndex
& mem
,
4420 Register temp
) DEFINED_ON(arm
, arm64
, x86_shared
);
4422 void atomicEffectOpJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4423 AtomicOp op
, Imm32 value
, const Address
& mem
,
4424 Register temp
) DEFINED_ON(x86_shared
);
4426 void atomicEffectOpJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4427 AtomicOp op
, Imm32 value
, const BaseIndex
& mem
,
4428 Register temp
) DEFINED_ON(x86_shared
);
4430 void atomicEffectOpJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4431 AtomicOp op
, Register value
, const Address
& mem
,
4432 Register valueTemp
, Register offsetTemp
,
4434 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4436 void atomicEffectOpJS(Scalar::Type arrayType
, const Synchronization
& sync
,
4437 AtomicOp op
, Register value
, const BaseIndex
& mem
,
4438 Register valueTemp
, Register offsetTemp
,
4440 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4442 void atomicIsLockFreeJS(Register value
, Register output
);
4444 // ========================================================================
4445 // Spectre Mitigations.
4447 // Spectre attacks are side-channel attacks based on cache pollution or
4448 // slow-execution of some instructions. We have multiple spectre mitigations
4451 // - Stop speculative executions, with memory barriers. Memory barriers
4452 // force all branches depending on loads to be resolved, and thus
4453 // resolve all miss-speculated paths.
4455 // - Use conditional move instructions. Some CPUs have a branch predictor,
4456 // and not a flag predictor. In such cases, using a conditional move
4457 // instruction to zero some pointer/index is enough to add a
4458 // data-dependency which prevents any futher executions until the load is
4461 void spectreMaskIndex32(Register index
, Register length
, Register output
);
4462 void spectreMaskIndex32(Register index
, const Address
& length
,
4464 void spectreMaskIndexPtr(Register index
, Register length
, Register output
);
4465 void spectreMaskIndexPtr(Register index
, const Address
& length
,
4468 // The length must be a power of two. Performs a bounds check and Spectre
4470 void boundsCheck32PowerOfTwo(Register index
, uint32_t length
, Label
* failure
);
4472 void speculationBarrier() PER_SHARED_ARCH
;
4474 //}}} check_macroassembler_decl_style
4476 // Unsafe here means the caller is responsible for Spectre mitigations if
4477 // needed. Prefer branchTestObjClass or one of the other masm helpers!
4478 inline void loadObjClassUnsafe(Register obj
, Register dest
);
4480 template <typename EmitPreBarrier
>
4481 inline void storeObjShape(Register shape
, Register obj
,
4482 EmitPreBarrier emitPreBarrier
);
4483 template <typename EmitPreBarrier
>
4484 inline void storeObjShape(Shape
* shape
, Register obj
,
4485 EmitPreBarrier emitPreBarrier
);
4487 inline void loadObjProto(Register obj
, Register dest
);
4489 inline void loadStringLength(Register str
, Register dest
);
4491 void loadStringChars(Register str
, Register dest
, CharEncoding encoding
);
4493 void loadNonInlineStringChars(Register str
, Register dest
,
4494 CharEncoding encoding
);
4495 void loadNonInlineStringCharsForStore(Register str
, Register dest
);
4496 void storeNonInlineStringChars(Register chars
, Register str
);
4498 void loadInlineStringChars(Register str
, Register dest
,
4499 CharEncoding encoding
);
4500 void loadInlineStringCharsForStore(Register str
, Register dest
);
4503 void loadRopeChild(Register str
, Register index
, Register output
,
4507 void branchIfCanLoadStringChar(Register str
, Register index
, Register scratch
,
4509 void branchIfNotCanLoadStringChar(Register str
, Register index
,
4510 Register scratch
, Label
* label
);
4512 void loadStringChar(Register str
, Register index
, Register output
,
4513 Register scratch1
, Register scratch2
, Label
* fail
);
4515 void loadRopeLeftChild(Register str
, Register dest
);
4516 void loadRopeRightChild(Register str
, Register dest
);
4517 void storeRopeChildren(Register left
, Register right
, Register str
);
4519 void loadDependentStringBase(Register str
, Register dest
);
4520 void storeDependentStringBase(Register base
, Register str
);
4522 void loadStringIndexValue(Register str
, Register dest
, Label
* fail
);
4525 * Store the character in |src| to |dest|.
4527 template <typename T
>
4528 void storeChar(const T
& src
, Address dest
, CharEncoding encoding
) {
4529 if (encoding
== CharEncoding::Latin1
) {
4537 * Load the character at |src| into |dest|.
4539 template <typename T
>
4540 void loadChar(const T
& src
, Register dest
, CharEncoding encoding
) {
4541 if (encoding
== CharEncoding::Latin1
) {
4542 load8ZeroExtend(src
, dest
);
4544 load16ZeroExtend(src
, dest
);
4549 * Load the character at |chars[index + offset]| into |dest|. The optional
4550 * offset argument is not scaled to the character encoding.
4552 void loadChar(Register chars
, Register index
, Register dest
,
4553 CharEncoding encoding
, int32_t offset
= 0);
4556 * Add |index| to |chars| so that |chars| now points at |chars[index]|.
4558 void addToCharPtr(Register chars
, Register index
, CharEncoding encoding
);
4561 void loadStringFromUnit(Register unit
, Register dest
,
4562 const StaticStrings
& staticStrings
);
4563 void loadLengthTwoString(Register c1
, Register c2
, Register dest
,
4564 const StaticStrings
& staticStrings
);
4568 * Load the string representation of |input| in base |base|. Jumps to |fail|
4569 * when the string representation needs to be allocated dynamically.
4571 void loadInt32ToStringWithBase(Register input
, Register base
, Register dest
,
4572 Register scratch1
, Register scratch2
,
4573 const StaticStrings
& staticStrings
,
4574 const LiveRegisterSet
& volatileRegs
,
4576 void loadInt32ToStringWithBase(Register input
, int32_t base
, Register dest
,
4577 Register scratch1
, Register scratch2
,
4578 const StaticStrings
& staticStrings
,
4582 * Load the BigInt digits from |bigInt| into |digits|.
4584 void loadBigIntDigits(Register bigInt
, Register digits
);
4587 * Load the first [u]int64 value from |bigInt| into |dest|.
4589 void loadBigInt64(Register bigInt
, Register64 dest
);
4592 * Load the first digit from |bigInt| into |dest|. Handles the case when the
4593 * BigInt digits length is zero.
4595 * Note: A BigInt digit is a pointer-sized value.
4597 void loadFirstBigIntDigitOrZero(Register bigInt
, Register dest
);
4600 * Load the number stored in |bigInt| into |dest|. Handles the case when the
4601 * BigInt digits length is zero. Jumps to |fail| when the number can't be
4602 * saved into a single pointer-sized register.
4604 void loadBigInt(Register bigInt
, Register dest
, Label
* fail
);
4607 * Load the number stored in |bigInt| into |dest|. Doesn't handle the case
4608 * when the BigInt digits length is zero. Jumps to |fail| when the number
4609 * can't be saved into a single pointer-sized register.
4611 void loadBigIntNonZero(Register bigInt
, Register dest
, Label
* fail
);
4614 * Load the absolute number stored in |bigInt| into |dest|. Handles the case
4615 * when the BigInt digits length is zero. Jumps to |fail| when the number
4616 * can't be saved into a single pointer-sized register.
4618 void loadBigIntAbsolute(Register bigInt
, Register dest
, Label
* fail
);
4621 * In-place modifies the BigInt digit to a signed pointer-sized value. Jumps
4622 * to |fail| when the digit exceeds the representable range.
4624 void bigIntDigitToSignedPtr(Register bigInt
, Register digit
, Label
* fail
);
4627 * Initialize a BigInt from |val|. Clobbers |val|!
4629 void initializeBigInt64(Scalar::Type type
, Register bigInt
, Register64 val
);
4632 * Initialize a BigInt from the signed, pointer-sized register |val|.
4635 void initializeBigInt(Register bigInt
, Register val
);
4638 * Initialize a BigInt from the pointer-sized register |val|.
4640 void initializeBigIntAbsolute(Register bigInt
, Register val
);
4643 * Copy a BigInt. Jumps to |fail| on allocation failure or when the BigInt
4644 * digits need to be heap allocated.
4646 void copyBigIntWithInlineDigits(Register src
, Register dest
, Register temp
,
4647 gc::InitialHeap initialHeap
, Label
* fail
);
4650 * Compare a BigInt and an Int32 value. Falls through to the false case.
4652 void compareBigIntAndInt32(JSOp op
, Register bigInt
, Register int32
,
4653 Register scratch1
, Register scratch2
,
4654 Label
* ifTrue
, Label
* ifFalse
);
4657 * Compare two BigInts for equality. Falls through if both BigInts are equal
4660 * - When we jump to |notSameLength|, |temp1| holds the length of the right
4662 * - When we jump to |notSameDigit|, |temp2| points to the current digit of
4663 * the left operand and |temp4| holds the current digit of the right
4666 void equalBigInts(Register left
, Register right
, Register temp1
,
4667 Register temp2
, Register temp3
, Register temp4
,
4668 Label
* notSameSign
, Label
* notSameLength
,
4669 Label
* notSameDigit
);
4671 void loadJSContext(Register dest
);
4673 void switchToRealm(Register realm
);
4674 void switchToRealm(const void* realm
, Register scratch
);
4675 void switchToObjectRealm(Register obj
, Register scratch
);
4676 void switchToBaselineFrameRealm(Register scratch
);
4677 void switchToWasmInstanceRealm(Register scratch1
, Register scratch2
);
4678 void debugAssertContextRealm(const void* realm
, Register scratch
);
4680 void loadJitActivation(Register dest
);
4682 void guardSpecificAtom(Register str
, JSAtom
* atom
, Register scratch
,
4683 const LiveRegisterSet
& volatileRegs
, Label
* fail
);
4685 void guardStringToInt32(Register str
, Register output
, Register scratch
,
4686 LiveRegisterSet volatileRegs
, Label
* fail
);
4688 template <typename T
>
4689 void loadTypedOrValue(const T
& src
, TypedOrValueRegister dest
) {
4690 if (dest
.hasValue()) {
4691 loadValue(src
, dest
.valueReg());
4693 loadUnboxedValue(src
, dest
.type(), dest
.typedReg());
4697 template <typename T
>
4698 void storeTypedOrValue(TypedOrValueRegister src
, const T
& dest
) {
4699 if (src
.hasValue()) {
4700 storeValue(src
.valueReg(), dest
);
4701 } else if (IsFloatingPointType(src
.type())) {
4702 FloatRegister reg
= src
.typedReg().fpu();
4703 if (src
.type() == MIRType::Float32
) {
4704 ScratchDoubleScope
fpscratch(*this);
4705 convertFloat32ToDouble(reg
, fpscratch
);
4706 boxDouble(fpscratch
, dest
);
4708 boxDouble(reg
, dest
);
4711 storeValue(ValueTypeFromMIRType(src
.type()), src
.typedReg().gpr(), dest
);
4715 template <typename T
>
4716 void storeConstantOrRegister(const ConstantOrRegister
& src
, const T
& dest
) {
4717 if (src
.constant()) {
4718 storeValue(src
.value(), dest
);
4720 storeTypedOrValue(src
.reg(), dest
);
4724 void storeCallPointerResult(Register reg
) {
4725 if (reg
!= ReturnReg
) {
4726 mov(ReturnReg
, reg
);
4730 inline void storeCallBoolResult(Register reg
);
4731 inline void storeCallInt32Result(Register reg
);
4733 void storeCallFloatResult(FloatRegister reg
) {
4734 if (reg
!= ReturnDoubleReg
) {
4735 moveDouble(ReturnDoubleReg
, reg
);
4739 inline void storeCallResultValue(AnyRegister dest
, JSValueType type
);
4741 void storeCallResultValue(ValueOperand dest
) {
4742 #if defined(JS_NUNBOX32)
4743 // reshuffle the return registers used for a call result to store into
4744 // dest, using ReturnReg as a scratch register if necessary. This must
4745 // only be called after returning from a call, at a point when the
4746 // return register is not live. XXX would be better to allow wrappers
4747 // to store the return value to different places.
4748 if (dest
.typeReg() == JSReturnReg_Data
) {
4749 if (dest
.payloadReg() == JSReturnReg_Type
) {
4750 // swap the two registers.
4751 mov(JSReturnReg_Type
, ReturnReg
);
4752 mov(JSReturnReg_Data
, JSReturnReg_Type
);
4753 mov(ReturnReg
, JSReturnReg_Data
);
4755 mov(JSReturnReg_Data
, dest
.payloadReg());
4756 mov(JSReturnReg_Type
, dest
.typeReg());
4759 mov(JSReturnReg_Type
, dest
.typeReg());
4760 mov(JSReturnReg_Data
, dest
.payloadReg());
4762 #elif defined(JS_PUNBOX64)
4763 if (dest
.valueReg() != JSReturnReg
) {
4764 mov(JSReturnReg
, dest
.valueReg());
4767 # error "Bad architecture"
4771 inline void storeCallResultValue(TypedOrValueRegister dest
);
4774 TrampolinePtr
preBarrierTrampoline(MIRType type
);
4776 template <typename T
>
4777 void unguardedCallPreBarrier(const T
& address
, MIRType type
) {
4779 if (type
== MIRType::Value
) {
4780 branchTestGCThing(Assembler::NotEqual
, address
, &done
);
4781 } else if (type
== MIRType::Object
|| type
== MIRType::String
) {
4782 branchPtr(Assembler::Equal
, address
, ImmWord(0), &done
);
4785 Push(PreBarrierReg
);
4786 computeEffectiveAddress(address
, PreBarrierReg
);
4788 TrampolinePtr preBarrier
= preBarrierTrampoline(type
);
4792 // On arm64, SP may be < PSP now (that's OK).
4793 // eg testcase: tests/auto-regress/bug702915.js
4798 template <typename T
>
4799 void guardedCallPreBarrier(const T
& address
, MIRType type
) {
4801 branchTestNeedsIncrementalBarrier(Assembler::Zero
, &done
);
4802 unguardedCallPreBarrier(address
, type
);
4806 // Like guardedCallPreBarrier, but unlike guardedCallPreBarrier this can be
4807 // called from runtime-wide trampolines because it loads cx->zone (instead of
4808 // baking in the current Zone) if JitContext::realm is nullptr.
4809 template <typename T
>
4810 void guardedCallPreBarrierAnyZone(const T
& address
, MIRType type
,
4813 branchTestNeedsIncrementalBarrierAnyZone(Assembler::Zero
, &done
, scratch
);
4814 unguardedCallPreBarrier(address
, type
);
4818 enum class Uint32Mode
{ FailOnDouble
, ForceDouble
};
4820 void boxUint32(Register source
, ValueOperand dest
, Uint32Mode uint32Mode
,
4823 template <typename T
>
4824 void loadFromTypedArray(Scalar::Type arrayType
, const T
& src
,
4825 AnyRegister dest
, Register temp
, Label
* fail
);
4827 template <typename T
>
4828 void loadFromTypedArray(Scalar::Type arrayType
, const T
& src
,
4829 const ValueOperand
& dest
, Uint32Mode uint32Mode
,
4830 Register temp
, Label
* fail
);
4832 template <typename T
>
4833 void loadFromTypedBigIntArray(Scalar::Type arrayType
, const T
& src
,
4834 Register bigInt
, Register64 temp
);
4836 template <typename S
, typename T
>
4837 void storeToTypedIntArray(Scalar::Type arrayType
, const S
& value
,
4839 switch (arrayType
) {
4842 case Scalar::Uint8Clamped
:
4843 store8(value
, dest
);
4846 case Scalar::Uint16
:
4847 store16(value
, dest
);
4850 case Scalar::Uint32
:
4851 store32(value
, dest
);
4854 MOZ_CRASH("Invalid typed array type");
4858 void storeToTypedFloatArray(Scalar::Type arrayType
, FloatRegister value
,
4859 const BaseIndex
& dest
);
4860 void storeToTypedFloatArray(Scalar::Type arrayType
, FloatRegister value
,
4861 const Address
& dest
);
4863 void storeToTypedBigIntArray(Scalar::Type arrayType
, Register64 value
,
4864 const BaseIndex
& dest
);
4865 void storeToTypedBigIntArray(Scalar::Type arrayType
, Register64 value
,
4866 const Address
& dest
);
4868 void memoryBarrierBefore(const Synchronization
& sync
);
4869 void memoryBarrierAfter(const Synchronization
& sync
);
4871 void debugAssertIsObject(const ValueOperand
& val
);
4872 void debugAssertObjHasFixedSlots(Register obj
, Register scratch
);
4874 void debugAssertObjectHasClass(Register obj
, Register scratch
,
4875 const JSClass
* clasp
);
4877 void branchArrayIsNotPacked(Register array
, Register temp1
, Register temp2
,
4880 void setIsPackedArray(Register obj
, Register output
, Register temp
);
4882 void packedArrayPop(Register array
, ValueOperand output
, Register temp1
,
4883 Register temp2
, Label
* fail
);
4884 void packedArrayShift(Register array
, ValueOperand output
, Register temp1
,
4885 Register temp2
, LiveRegisterSet volatileRegs
,
4888 void loadArgumentsObjectElement(Register obj
, Register index
,
4889 ValueOperand output
, Register temp
,
4891 void loadArgumentsObjectElementHole(Register obj
, Register index
,
4892 ValueOperand output
, Register temp
,
4894 void loadArgumentsObjectElementExists(Register obj
, Register index
,
4895 Register output
, Register temp
,
4898 void loadArgumentsObjectLength(Register obj
, Register output
, Label
* fail
);
4900 void branchTestArgumentsObjectFlags(Register obj
, Register temp
,
4901 uint32_t flags
, Condition cond
,
4904 void typedArrayElementSize(Register obj
, Register output
);
4905 void branchIfClassIsNotTypedArray(Register clasp
, Label
* notTypedArray
);
4907 void branchIfHasDetachedArrayBuffer(Register obj
, Register temp
,
4910 void branchIfNativeIteratorNotReusable(Register ni
, Label
* notReusable
);
4911 void branchNativeIteratorIndices(Condition cond
, Register ni
, Register temp
,
4912 NativeIteratorIndices kind
, Label
* label
);
4914 void maybeLoadIteratorFromShape(Register obj
, Register dest
, Register temp
,
4915 Register temp2
, Register temp3
,
4918 void iteratorMore(Register obj
, ValueOperand output
, Register temp
);
4919 void iteratorClose(Register obj
, Register temp1
, Register temp2
,
4921 void registerIterator(Register enumeratorsList
, Register iter
, Register temp
);
4923 void toHashableNonGCThing(ValueOperand value
, ValueOperand result
,
4924 FloatRegister tempFloat
);
4926 void toHashableValue(ValueOperand value
, ValueOperand result
,
4927 FloatRegister tempFloat
, Label
* atomizeString
,
4931 void scrambleHashCode(Register result
);
4934 void prepareHashNonGCThing(ValueOperand value
, Register result
,
4936 void prepareHashString(Register str
, Register result
, Register temp
);
4937 void prepareHashSymbol(Register sym
, Register result
);
4938 void prepareHashBigInt(Register bigInt
, Register result
, Register temp1
,
4939 Register temp2
, Register temp3
);
4940 void prepareHashObject(Register setObj
, ValueOperand value
, Register result
,
4941 Register temp1
, Register temp2
, Register temp3
,
4943 void prepareHashValue(Register setObj
, ValueOperand value
, Register result
,
4944 Register temp1
, Register temp2
, Register temp3
,
4948 enum class IsBigInt
{ No
, Yes
, Maybe
};
4951 * Search for a value in a OrderedHashTable.
4953 * When we jump to |found|, |entryTemp| holds the found hashtable entry.
4955 template <typename OrderedHashTable
>
4956 void orderedHashTableLookup(Register setOrMapObj
, ValueOperand value
,
4957 Register hash
, Register entryTemp
, Register temp1
,
4958 Register temp3
, Register temp4
, Register temp5
,
4959 Label
* found
, IsBigInt isBigInt
);
4961 void setObjectHas(Register setObj
, ValueOperand value
, Register hash
,
4962 Register result
, Register temp1
, Register temp2
,
4963 Register temp3
, Register temp4
, IsBigInt isBigInt
);
4965 void mapObjectHas(Register mapObj
, ValueOperand value
, Register hash
,
4966 Register result
, Register temp1
, Register temp2
,
4967 Register temp3
, Register temp4
, IsBigInt isBigInt
);
4969 void mapObjectGet(Register mapObj
, ValueOperand value
, Register hash
,
4970 ValueOperand result
, Register temp1
, Register temp2
,
4971 Register temp3
, Register temp4
, Register temp5
,
4975 void setObjectHasNonBigInt(Register setObj
, ValueOperand value
, Register hash
,
4976 Register result
, Register temp1
, Register temp2
) {
4977 return setObjectHas(setObj
, value
, hash
, result
, temp1
, temp2
, InvalidReg
,
4978 InvalidReg
, IsBigInt::No
);
4980 void setObjectHasBigInt(Register setObj
, ValueOperand value
, Register hash
,
4981 Register result
, Register temp1
, Register temp2
,
4982 Register temp3
, Register temp4
) {
4983 return setObjectHas(setObj
, value
, hash
, result
, temp1
, temp2
, temp3
, temp4
,
4986 void setObjectHasValue(Register setObj
, ValueOperand value
, Register hash
,
4987 Register result
, Register temp1
, Register temp2
,
4988 Register temp3
, Register temp4
) {
4989 return setObjectHas(setObj
, value
, hash
, result
, temp1
, temp2
, temp3
, temp4
,
4993 void mapObjectHasNonBigInt(Register mapObj
, ValueOperand value
, Register hash
,
4994 Register result
, Register temp1
, Register temp2
) {
4995 return mapObjectHas(mapObj
, value
, hash
, result
, temp1
, temp2
, InvalidReg
,
4996 InvalidReg
, IsBigInt::No
);
4998 void mapObjectHasBigInt(Register mapObj
, ValueOperand value
, Register hash
,
4999 Register result
, Register temp1
, Register temp2
,
5000 Register temp3
, Register temp4
) {
5001 return mapObjectHas(mapObj
, value
, hash
, result
, temp1
, temp2
, temp3
, temp4
,
5004 void mapObjectHasValue(Register mapObj
, ValueOperand value
, Register hash
,
5005 Register result
, Register temp1
, Register temp2
,
5006 Register temp3
, Register temp4
) {
5007 return mapObjectHas(mapObj
, value
, hash
, result
, temp1
, temp2
, temp3
, temp4
,
5011 void mapObjectGetNonBigInt(Register mapObj
, ValueOperand value
, Register hash
,
5012 ValueOperand result
, Register temp1
,
5013 Register temp2
, Register temp3
) {
5014 return mapObjectGet(mapObj
, value
, hash
, result
, temp1
, temp2
, temp3
,
5015 InvalidReg
, InvalidReg
, IsBigInt::No
);
5017 void mapObjectGetBigInt(Register mapObj
, ValueOperand value
, Register hash
,
5018 ValueOperand result
, Register temp1
, Register temp2
,
5019 Register temp3
, Register temp4
, Register temp5
) {
5020 return mapObjectGet(mapObj
, value
, hash
, result
, temp1
, temp2
, temp3
, temp4
,
5021 temp5
, IsBigInt::Yes
);
5023 void mapObjectGetValue(Register mapObj
, ValueOperand value
, Register hash
,
5024 ValueOperand result
, Register temp1
, Register temp2
,
5025 Register temp3
, Register temp4
, Register temp5
) {
5026 return mapObjectGet(mapObj
, value
, hash
, result
, temp1
, temp2
, temp3
, temp4
,
5027 temp5
, IsBigInt::Maybe
);
5031 template <typename OrderedHashTable
>
5032 void loadOrderedHashTableCount(Register setOrMapObj
, Register result
);
5035 void loadSetObjectSize(Register setObj
, Register result
);
5036 void loadMapObjectSize(Register mapObj
, Register result
);
5038 // Inline version of js_TypedArray_uint8_clamp_double.
5039 // This function clobbers the input register.
5040 void clampDoubleToUint8(FloatRegister input
, Register output
) PER_ARCH
;
5042 using MacroAssemblerSpecific::ensureDouble
;
5044 template <typename S
>
5045 void ensureDouble(const S
& source
, FloatRegister dest
, Label
* failure
) {
5046 Label isDouble
, done
;
5047 branchTestDouble(Assembler::Equal
, source
, &isDouble
);
5048 branchTestInt32(Assembler::NotEqual
, source
, failure
);
5050 convertInt32ToDouble(source
, dest
);
5054 unboxDouble(source
, dest
);
5059 // Inline allocation.
5061 void checkAllocatorState(Label
* fail
);
5062 bool shouldNurseryAllocate(gc::AllocKind allocKind
,
5063 gc::InitialHeap initialHeap
);
5064 void nurseryAllocateObject(
5065 Register result
, Register temp
, gc::AllocKind allocKind
,
5066 size_t nDynamicSlots
, Label
* fail
,
5067 const AllocSiteInput
& allocSite
= AllocSiteInput());
5068 void bumpPointerAllocate(Register result
, Register temp
, Label
* fail
,
5069 CompileZone
* zone
, JS::TraceKind traceKind
,
5071 const AllocSiteInput
& allocSite
= AllocSiteInput());
5072 void updateAllocSite(Register temp
, Register result
, CompileZone
* zone
,
5075 void freeListAllocate(Register result
, Register temp
, gc::AllocKind allocKind
,
5077 void allocateObject(Register result
, Register temp
, gc::AllocKind allocKind
,
5078 uint32_t nDynamicSlots
, gc::InitialHeap initialHeap
,
5080 const AllocSiteInput
& allocSite
= AllocSiteInput());
5081 void nurseryAllocateString(Register result
, Register temp
,
5082 gc::AllocKind allocKind
, Label
* fail
);
5083 void allocateString(Register result
, Register temp
, gc::AllocKind allocKind
,
5084 gc::InitialHeap initialHeap
, Label
* fail
);
5085 void nurseryAllocateBigInt(Register result
, Register temp
, Label
* fail
);
5086 void copySlotsFromTemplate(Register obj
,
5087 const TemplateNativeObject
& templateObj
,
5088 uint32_t start
, uint32_t end
);
5089 void fillSlotsWithConstantValue(Address addr
, Register temp
, uint32_t start
,
5090 uint32_t end
, const Value
& v
);
5091 void fillSlotsWithUndefined(Address addr
, Register temp
, uint32_t start
,
5093 void fillSlotsWithUninitialized(Address addr
, Register temp
, uint32_t start
,
5096 void initGCSlots(Register obj
, Register temp
,
5097 const TemplateNativeObject
& templateObj
);
5100 void callFreeStub(Register slots
);
5101 void createGCObject(Register result
, Register temp
,
5102 const TemplateObject
& templateObj
,
5103 gc::InitialHeap initialHeap
, Label
* fail
,
5104 bool initContents
= true);
5106 void createPlainGCObject(Register result
, Register shape
, Register temp
,
5107 Register temp2
, uint32_t numFixedSlots
,
5108 uint32_t numDynamicSlots
, gc::AllocKind allocKind
,
5109 gc::InitialHeap initialHeap
, Label
* fail
,
5110 const AllocSiteInput
& allocSite
,
5111 bool initContents
= true);
5113 void createArrayWithFixedElements(
5114 Register result
, Register shape
, Register temp
, uint32_t arrayLength
,
5115 uint32_t arrayCapacity
, gc::AllocKind allocKind
,
5116 gc::InitialHeap initialHeap
, Label
* fail
,
5117 const AllocSiteInput
& allocSite
= AllocSiteInput());
5119 void initGCThing(Register obj
, Register temp
,
5120 const TemplateObject
& templateObj
, bool initContents
= true);
5122 enum class TypedArrayLength
{ Fixed
, Dynamic
};
5124 void initTypedArraySlots(Register obj
, Register temp
, Register lengthReg
,
5125 LiveRegisterSet liveRegs
, Label
* fail
,
5126 TypedArrayObject
* templateObj
,
5127 TypedArrayLength lengthKind
);
5129 void newGCString(Register result
, Register temp
, gc::InitialHeap initialHeap
,
5131 void newGCFatInlineString(Register result
, Register temp
,
5132 gc::InitialHeap initialHeap
, Label
* fail
);
5134 void newGCBigInt(Register result
, Register temp
, gc::InitialHeap initialHeap
,
5137 // Compares two strings for equality based on the JSOP.
5138 // This checks for identical pointers, atoms and length and fails for
5140 void compareStrings(JSOp op
, Register left
, Register right
, Register result
,
5143 // Result of the typeof operation. Falls back to slow-path for proxies.
5144 void typeOfObject(Register objReg
, Register scratch
, Label
* slow
,
5145 Label
* isObject
, Label
* isCallable
, Label
* isUndefined
);
5147 // Implementation of IsCallable. Doesn't handle proxies.
5148 void isCallable(Register obj
, Register output
, Label
* isProxy
) {
5149 isCallableOrConstructor(true, obj
, output
, isProxy
);
5151 void isConstructor(Register obj
, Register output
, Label
* isProxy
) {
5152 isCallableOrConstructor(false, obj
, output
, isProxy
);
5155 void setIsCrossRealmArrayConstructor(Register obj
, Register output
);
5157 void setIsDefinitelyTypedArrayConstructor(Register obj
, Register output
);
5159 void loadMegamorphicCache(Register dest
);
5160 void loadStringToAtomCacheLastLookups(Register dest
);
5161 void loadMegamorphicSetPropCache(Register dest
);
5163 void loadAtomOrSymbolAndHash(ValueOperand value
, Register outId
,
5164 Register outHash
, Label
* cacheMiss
);
5166 void loadAtomHash(Register id
, Register hash
, Label
* done
);
5168 void emitExtractValueFromMegamorphicCacheEntry(
5169 Register obj
, Register entry
, Register scratch1
, Register scratch2
,
5170 ValueOperand output
, Label
* cacheHit
, Label
* cacheMiss
);
5172 template <typename IdOperandType
>
5173 void emitMegamorphicCacheLookupByValueCommon(
5174 IdOperandType id
, Register obj
, Register scratch1
, Register scratch2
,
5175 Register outEntryPtr
, Label
* cacheMiss
, Label
* cacheMissWithEntry
);
5177 void emitMegamorphicCacheLookup(PropertyKey id
, Register obj
,
5178 Register scratch1
, Register scratch2
,
5179 Register outEntryPtr
, ValueOperand output
,
5182 // NOTE: |id| must either be a ValueOperand or a Register. If it is a
5183 // Register, we assume that it is an atom.
5184 template <typename IdOperandType
>
5185 void emitMegamorphicCacheLookupByValue(IdOperandType id
, Register obj
,
5186 Register scratch1
, Register scratch2
,
5187 Register outEntryPtr
,
5188 ValueOperand output
, Label
* cacheHit
);
5190 void emitMegamorphicCacheLookupExists(ValueOperand id
, Register obj
,
5191 Register scratch1
, Register scratch2
,
5192 Register outEntryPtr
, Register output
,
5193 Label
* cacheHit
, bool hasOwn
);
5195 // Given a PropertyIteratorObject with valid indices, extract the current
5196 // PropertyIndex, storing the index in |outIndex| and the kind in |outKind|
5197 void extractCurrentIndexAndKindFromIterator(Register iterator
,
5201 template <typename IdType
>
5202 #ifdef JS_CODEGEN_X86
5203 // See MegamorphicSetElement in LIROps.yaml
5204 void emitMegamorphicCachedSetSlot(IdType id
, Register obj
, Register scratch1
,
5205 ValueOperand value
, Label
* cacheHit
,
5206 void (*emitPreBarrier
)(MacroAssembler
&,
5210 void emitMegamorphicCachedSetSlot(
5211 IdType id
, Register obj
, Register scratch1
, Register scratch2
,
5212 Register scratch3
, ValueOperand value
, Label
* cacheHit
,
5213 void (*emitPreBarrier
)(MacroAssembler
&, const Address
&, MIRType
));
5216 void loadDOMExpandoValueGuardGeneration(
5217 Register obj
, ValueOperand output
,
5218 JS::ExpandoAndGeneration
* expandoAndGeneration
, uint64_t generation
,
5221 void guardNonNegativeIntPtrToInt32(Register reg
, Label
* fail
);
5223 void loadArrayBufferByteLengthIntPtr(Register obj
, Register output
);
5224 void loadArrayBufferViewByteOffsetIntPtr(Register obj
, Register output
);
5225 void loadArrayBufferViewLengthIntPtr(Register obj
, Register output
);
5228 void isCallableOrConstructor(bool isCallable
, Register obj
, Register output
,
5232 // Generates code used to complete a bailout.
5233 void generateBailoutTail(Register scratch
, Register bailoutInfo
);
5236 #ifndef JS_CODEGEN_ARM64
5237 // StackPointer manipulation functions.
5238 // On ARM64, the StackPointer is implemented as two synchronized registers.
5239 // Code shared across platforms must use these functions to be valid.
5240 template <typename T
>
5241 inline void addToStackPtr(T t
);
5242 template <typename T
>
5243 inline void addStackPtrTo(T t
);
5245 void subFromStackPtr(Imm32 imm32
)
5246 DEFINED_ON(mips32
, mips64
, loong64
, riscv64
, wasm32
, arm
, x86
, x64
);
5247 void subFromStackPtr(Register reg
);
5249 template <typename T
>
5250 void subStackPtrFrom(T t
) {
5251 subPtr(getStackPointer(), t
);
5254 template <typename T
>
5255 void andToStackPtr(T t
) {
5256 andPtr(t
, getStackPointer());
5259 template <typename T
>
5260 void moveToStackPtr(T t
) {
5261 movePtr(t
, getStackPointer());
5263 template <typename T
>
5264 void moveStackPtrTo(T t
) {
5265 movePtr(getStackPointer(), t
);
5268 template <typename T
>
5269 void loadStackPtr(T t
) {
5270 loadPtr(t
, getStackPointer());
5272 template <typename T
>
5273 void storeStackPtr(T t
) {
5274 storePtr(getStackPointer(), t
);
5277 // StackPointer testing functions.
5278 // On ARM64, sp can function as the zero register depending on context.
5279 // Code shared across platforms must use these functions to be valid.
5280 template <typename T
>
5281 inline void branchTestStackPtr(Condition cond
, T t
, Label
* label
);
5282 template <typename T
>
5283 inline void branchStackPtr(Condition cond
, T rhs
, Label
* label
);
5284 template <typename T
>
5285 inline void branchStackPtrRhs(Condition cond
, T lhs
, Label
* label
);
5287 // Move the stack pointer based on the requested amount.
5288 inline void reserveStack(uint32_t amount
);
5289 #else // !JS_CODEGEN_ARM64
5290 void reserveStack(uint32_t amount
);
5294 void enableProfilingInstrumentation() {
5295 emitProfilingInstrumentation_
= true;
5299 // This class is used to surround call sites throughout the assembler. This
5300 // is used by callWithABI, and callJit functions, except if suffixed by
5302 class MOZ_RAII AutoProfilerCallInstrumentation
{
5304 explicit AutoProfilerCallInstrumentation(MacroAssembler
& masm
);
5305 ~AutoProfilerCallInstrumentation() = default;
5307 friend class AutoProfilerCallInstrumentation
;
5309 void appendProfilerCallSite(CodeOffset label
) {
5310 propagateOOM(profilerCallSites_
.append(label
));
5313 // Fix up the code pointers to be written for locations where profilerCallSite
5314 // emitted moves of RIP to a register.
5315 void linkProfilerCallSites(JitCode
* code
);
5317 // This field is used to manage profiling instrumentation output. If
5318 // provided and enabled, then instrumentation will be emitted around call
5320 bool emitProfilingInstrumentation_
;
5322 // Record locations of the call sites.
5323 Vector
<CodeOffset
, 0, SystemAllocPolicy
> profilerCallSites_
;
5326 void loadJitCodeRaw(Register func
, Register dest
);
5327 void loadBaselineJitCodeRaw(Register func
, Register dest
,
5328 Label
* failure
= nullptr);
5329 void storeICScriptInJSContext(Register icScript
);
5331 void loadBaselineFramePtr(Register framePtr
, Register dest
);
5333 void pushBaselineFramePtr(Register framePtr
, Register scratch
) {
5334 loadBaselineFramePtr(framePtr
, scratch
);
5338 void PushBaselineFramePtr(Register framePtr
, Register scratch
) {
5339 loadBaselineFramePtr(framePtr
, scratch
);
5343 using MacroAssemblerSpecific::movePtr
;
5345 void movePtr(TrampolinePtr ptr
, Register dest
) {
5346 movePtr(ImmPtr(ptr
.value
), dest
);
5350 void handleFailure();
5353 Label
* exceptionLabel() {
5354 // Exceptions are currently handled the same way as sequential failures.
5355 return &failureLabel_
;
5358 Label
* failureLabel() { return &failureLabel_
; }
5361 void link(JitCode
* code
);
5363 void assumeUnreachable(const char* output
);
5365 void printf(const char* output
);
5366 void printf(const char* output
, Register value
);
5368 #define DISPATCH_FLOATING_POINT_OP(method, type, arg1d, arg1f, arg2) \
5369 MOZ_ASSERT(IsFloatingPointType(type)); \
5370 if (type == MIRType::Double) \
5371 method##Double(arg1d, arg2); \
5373 method##Float32(arg1f, arg2);
5375 void loadConstantFloatingPoint(double d
, float f
, FloatRegister dest
,
5377 DISPATCH_FLOATING_POINT_OP(loadConstant
, destType
, d
, f
, dest
);
5379 void boolValueToFloatingPoint(ValueOperand value
, FloatRegister dest
,
5381 DISPATCH_FLOATING_POINT_OP(boolValueTo
, destType
, value
, value
, dest
);
5383 void int32ValueToFloatingPoint(ValueOperand value
, FloatRegister dest
,
5385 DISPATCH_FLOATING_POINT_OP(int32ValueTo
, destType
, value
, value
, dest
);
5387 void convertInt32ToFloatingPoint(Register src
, FloatRegister dest
,
5389 DISPATCH_FLOATING_POINT_OP(convertInt32To
, destType
, src
, src
, dest
);
5392 #undef DISPATCH_FLOATING_POINT_OP
5394 void convertValueToFloatingPoint(ValueOperand value
, FloatRegister output
,
5395 Label
* fail
, MIRType outputType
);
5397 void outOfLineTruncateSlow(FloatRegister src
, Register dest
,
5398 bool widenFloatToDouble
, bool compilingWasm
,
5399 wasm::BytecodeOffset callOffset
);
5401 void convertInt32ValueToDouble(ValueOperand val
);
5403 void convertValueToDouble(ValueOperand value
, FloatRegister output
,
5405 convertValueToFloatingPoint(value
, output
, fail
, MIRType::Double
);
5408 void convertValueToFloat(ValueOperand value
, FloatRegister output
,
5410 convertValueToFloatingPoint(value
, output
, fail
, MIRType::Float32
);
5414 // Functions for converting values to int.
5416 void convertDoubleToInt(FloatRegister src
, Register output
,
5417 FloatRegister temp
, Label
* truncateFail
, Label
* fail
,
5418 IntConversionBehavior behavior
);
5420 // Strings may be handled by providing labels to jump to when the behavior
5421 // is truncation or clamping. The subroutine, usually an OOL call, is
5422 // passed the unboxed string in |stringReg| and should convert it to a
5423 // double store into |temp|.
5424 void convertValueToInt(
5425 ValueOperand value
, Label
* handleStringEntry
, Label
* handleStringRejoin
,
5426 Label
* truncateDoubleSlow
, Register stringReg
, FloatRegister temp
,
5427 Register output
, Label
* fail
, IntConversionBehavior behavior
,
5428 IntConversionInputKind conversion
= IntConversionInputKind::Any
);
5430 // This carries over the MToNumberInt32 operation on the ValueOperand
5431 // input; see comment at the top of this class.
5432 void convertValueToInt32(
5433 ValueOperand value
, FloatRegister temp
, Register output
, Label
* fail
,
5434 bool negativeZeroCheck
,
5435 IntConversionInputKind conversion
= IntConversionInputKind::Any
) {
5437 value
, nullptr, nullptr, nullptr, InvalidReg
, temp
, output
, fail
,
5438 negativeZeroCheck
? IntConversionBehavior::NegativeZeroCheck
5439 : IntConversionBehavior::Normal
,
5443 // This carries over the MTruncateToInt32 operation on the ValueOperand
5444 // input; see the comment at the top of this class.
5445 void truncateValueToInt32(ValueOperand value
, Label
* handleStringEntry
,
5446 Label
* handleStringRejoin
,
5447 Label
* truncateDoubleSlow
, Register stringReg
,
5448 FloatRegister temp
, Register output
, Label
* fail
) {
5449 convertValueToInt(value
, handleStringEntry
, handleStringRejoin
,
5450 truncateDoubleSlow
, stringReg
, temp
, output
, fail
,
5451 IntConversionBehavior::Truncate
);
5454 void truncateValueToInt32(ValueOperand value
, FloatRegister temp
,
5455 Register output
, Label
* fail
) {
5456 truncateValueToInt32(value
, nullptr, nullptr, nullptr, InvalidReg
, temp
,
5460 // Convenience functions for clamping values to uint8.
5461 void clampValueToUint8(ValueOperand value
, Label
* handleStringEntry
,
5462 Label
* handleStringRejoin
, Register stringReg
,
5463 FloatRegister temp
, Register output
, Label
* fail
) {
5464 convertValueToInt(value
, handleStringEntry
, handleStringRejoin
, nullptr,
5465 stringReg
, temp
, output
, fail
,
5466 IntConversionBehavior::ClampToUint8
);
5469 [[nodiscard
]] bool icBuildOOLFakeExitFrame(void* fakeReturnAddr
,
5470 AutoSaveLiveRegisters
& save
);
5472 // Align the stack pointer based on the number of arguments which are pushed
5473 // on the stack, such that the JitFrameLayout would be correctly aligned on
5474 // the JitStackAlignment.
5475 void alignJitStackBasedOnNArgs(Register nargs
, bool countIncludesThis
);
5476 void alignJitStackBasedOnNArgs(uint32_t argc
, bool countIncludesThis
);
5478 inline void assertStackAlignment(uint32_t alignment
, int32_t offset
= 0);
5480 void touchFrameValues(Register numStackValues
, Register scratch1
,
5484 // See comment block "64-bit GPRs carrying 32-bit values" above. This asserts
5485 // that the high bits of the register are appropriate for the architecture and
5486 // the value in the low bits.
5487 void debugAssertCanonicalInt32(Register r
);
5491 // StackMacroAssembler checks no GC will happen while it's on the stack.
5492 class MOZ_RAII StackMacroAssembler
: public MacroAssembler
{
5493 JS::AutoCheckCannotGC nogc
;
5496 StackMacroAssembler(JSContext
* cx
, TempAllocator
& alloc
);
5499 // WasmMacroAssembler does not contain GC pointers, so it doesn't need the no-GC
5500 // checking StackMacroAssembler has.
5501 class MOZ_RAII WasmMacroAssembler
: public MacroAssembler
{
5503 explicit WasmMacroAssembler(TempAllocator
& alloc
, bool limitedSize
= true);
5504 explicit WasmMacroAssembler(TempAllocator
& alloc
,
5505 const wasm::ModuleEnvironment
& env
,
5506 bool limitedSize
= true);
5507 ~WasmMacroAssembler() { assertNoGCThings(); }
5510 // Heap-allocated MacroAssembler used for Ion off-thread code generation.
5511 // GC cancels off-thread compilations.
5512 class IonHeapMacroAssembler
: public MacroAssembler
{
5514 IonHeapMacroAssembler(TempAllocator
& alloc
, CompileRealm
* realm
);
5517 //{{{ check_macroassembler_style
5518 inline uint32_t MacroAssembler::framePushed() const { return framePushed_
; }
5520 inline void MacroAssembler::setFramePushed(uint32_t framePushed
) {
5521 framePushed_
= framePushed
;
5524 inline void MacroAssembler::adjustFrame(int32_t value
) {
5525 MOZ_ASSERT_IF(value
< 0, framePushed_
>= uint32_t(-value
));
5526 setFramePushed(framePushed_
+ value
);
5529 inline void MacroAssembler::implicitPop(uint32_t bytes
) {
5530 MOZ_ASSERT(bytes
% sizeof(intptr_t) == 0);
5531 MOZ_ASSERT(bytes
<= INT32_MAX
);
5532 adjustFrame(-int32_t(bytes
));
5534 //}}} check_macroassembler_style
5536 static inline Assembler::DoubleCondition
JSOpToDoubleCondition(JSOp op
) {
5539 case JSOp::StrictEq
:
5540 return Assembler::DoubleEqual
;
5542 case JSOp::StrictNe
:
5543 return Assembler::DoubleNotEqualOrUnordered
;
5545 return Assembler::DoubleLessThan
;
5547 return Assembler::DoubleLessThanOrEqual
;
5549 return Assembler::DoubleGreaterThan
;
5551 return Assembler::DoubleGreaterThanOrEqual
;
5553 MOZ_CRASH("Unexpected comparison operation");
5557 // Note: the op may have been inverted during lowering (to put constants in a
5558 // position where they can be immediates), so it is important to use the
5559 // lir->jsop() instead of the mir->jsop() when it is present.
5560 static inline Assembler::Condition
JSOpToCondition(JSOp op
, bool isSigned
) {
5564 case JSOp::StrictEq
:
5565 return Assembler::Equal
;
5567 case JSOp::StrictNe
:
5568 return Assembler::NotEqual
;
5570 return Assembler::LessThan
;
5572 return Assembler::LessThanOrEqual
;
5574 return Assembler::GreaterThan
;
5576 return Assembler::GreaterThanOrEqual
;
5578 MOZ_CRASH("Unrecognized comparison operation");
5583 case JSOp::StrictEq
:
5584 return Assembler::Equal
;
5586 case JSOp::StrictNe
:
5587 return Assembler::NotEqual
;
5589 return Assembler::Below
;
5591 return Assembler::BelowOrEqual
;
5593 return Assembler::Above
;
5595 return Assembler::AboveOrEqual
;
5597 MOZ_CRASH("Unrecognized comparison operation");
5602 static inline size_t StackDecrementForCall(uint32_t alignment
,
5603 size_t bytesAlreadyPushed
,
5604 size_t bytesToPush
) {
5605 return bytesToPush
+
5606 ComputeByteAlignment(bytesAlreadyPushed
+ bytesToPush
, alignment
);
5609 // Helper for generatePreBarrier.
5610 inline DynFn
JitPreWriteBarrier(MIRType type
);
5615 #endif /* jit_MacroAssembler_h */