1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
2 * vim: set ts=8 sts=2 et sw=2 tw=80:
3 * This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
7 #ifndef jit_MacroAssembler_h
8 #define jit_MacroAssembler_h
10 #include "mozilla/EndianUtils.h"
11 #include "mozilla/MacroForEach.h"
12 #include "mozilla/MathAlgorithms.h"
13 #include "mozilla/Maybe.h"
14 #include "mozilla/Variant.h"
18 #if defined(JS_CODEGEN_X86)
19 # include "jit/x86/MacroAssembler-x86.h"
20 #elif defined(JS_CODEGEN_X64)
21 # include "jit/x64/MacroAssembler-x64.h"
22 #elif defined(JS_CODEGEN_ARM)
23 # include "jit/arm/MacroAssembler-arm.h"
24 #elif defined(JS_CODEGEN_ARM64)
25 # include "jit/arm64/MacroAssembler-arm64.h"
26 #elif defined(JS_CODEGEN_MIPS32)
27 # include "jit/mips32/MacroAssembler-mips32.h"
28 #elif defined(JS_CODEGEN_MIPS64)
29 # include "jit/mips64/MacroAssembler-mips64.h"
30 #elif defined(JS_CODEGEN_LOONG64)
31 # include "jit/loong64/MacroAssembler-loong64.h"
32 #elif defined(JS_CODEGEN_RISCV64)
33 # include "jit/riscv64/MacroAssembler-riscv64.h"
34 #elif defined(JS_CODEGEN_WASM32)
35 # include "jit/wasm32/MacroAssembler-wasm32.h"
36 #elif defined(JS_CODEGEN_NONE)
37 # include "jit/none/MacroAssembler-none.h"
39 # error "Unknown architecture!"
41 #include "jit/ABIArgGenerator.h"
42 #include "jit/ABIFunctions.h"
43 #include "jit/AtomicOp.h"
44 #include "jit/IonTypes.h"
45 #include "jit/MoveResolver.h"
46 #include "jit/VMFunctions.h"
47 #include "js/ScalarType.h" // js::Scalar::Type
48 #include "util/Memory.h"
49 #include "vm/FunctionFlags.h"
50 #include "vm/Opcodes.h"
51 #include "vm/RealmFuses.h"
52 #include "wasm/WasmCodegenTypes.h"
53 #include "wasm/WasmFrame.h"
55 // [SMDOC] MacroAssembler multi-platform overview
57 // * How to read/write MacroAssembler method declarations:
59 // The following macros are made to avoid #ifdef around each method declarations
60 // of the Macro Assembler, and they are also used as an hint on the location of
61 // the implementations of each method. For example, the following declaration
63 // void Pop(FloatRegister t) DEFINED_ON(x86_shared, arm);
65 // suggests the MacroAssembler::Pop(FloatRegister) method is implemented in
66 // x86-shared/MacroAssembler-x86-shared.h, and also in arm/MacroAssembler-arm.h.
68 // - If there is no annotation, then there is only one generic definition in
69 // MacroAssembler.cpp.
71 // - If the declaration is "inline", then the method definition(s) would be in
72 // the "-inl.h" variant of the same file(s).
74 // The script check_macroassembler_style.py (which runs on every build) is
75 // used to verify that method definitions match the annotation on the method
76 // declarations. If there is any difference, then you either forgot to define
77 // the method in one of the macro assembler, or you forgot to update the
78 // annotation of the macro assembler declaration.
80 // Some convenient short-cuts are used to avoid repeating the same list of
81 // architectures on each method declaration, such as PER_ARCH and
84 // Functions that are architecture-agnostic and are the same for all
85 // architectures, that it's necessary to define inline *in this header* to
86 // avoid used-before-defined warnings/errors that would occur if the
87 // definitions were in MacroAssembler-inl.h, should use the OOL_IN_HEADER
88 // marker at end of the declaration:
90 // inline uint32_t framePushed() const OOL_IN_HEADER;
92 // Such functions should then be defined immediately after MacroAssembler's
93 // definition, for example:
95 // //{{{ check_macroassembler_style
97 // MacroAssembler::framePushed() const
99 // return framePushed_;
101 // ////}}} check_macroassembler_style
103 #define ALL_ARCH mips32, mips64, arm, arm64, x86, x64, loong64, riscv64, wasm32
104 #define ALL_SHARED_ARCH \
105 arm, arm64, loong64, riscv64, x86_shared, mips_shared, wasm32
107 // * How this macro works:
109 // DEFINED_ON is a macro which check if, for the current architecture, the
110 // method is defined on the macro assembler or not.
112 // For each architecture, we have a macro named DEFINED_ON_arch. This macro is
113 // empty if this is not the current architecture. Otherwise it must be either
114 // set to "define" or "crash" (only used for the none target so far).
116 // The DEFINED_ON macro maps the list of architecture names given as arguments
117 // to a list of macro names. For example,
119 // DEFINED_ON(arm, x86_shared)
123 // DEFINED_ON_none DEFINED_ON_arm DEFINED_ON_x86_shared
125 // which are later expanded on ARM, x86, x64 by DEFINED_ON_EXPAND_ARCH_RESULTS
130 // or if the JIT is disabled or set to no architecture to
134 // or to nothing, if the current architecture is not listed in the list of
135 // arguments of DEFINED_ON. Note, only one of the DEFINED_ON_arch macro
136 // contributes to the non-empty result, which is the macro of the current
137 // architecture if it is listed in the arguments of DEFINED_ON.
139 // This result is appended to DEFINED_ON_RESULT_ before expanding the macro,
140 // which results in either no annotation, a MOZ_CRASH(), or a "= delete"
141 // annotation on the method declaration.
143 #define DEFINED_ON_x86
144 #define DEFINED_ON_x64
145 #define DEFINED_ON_x86_shared
146 #define DEFINED_ON_arm
147 #define DEFINED_ON_arm64
148 #define DEFINED_ON_mips32
149 #define DEFINED_ON_mips64
150 #define DEFINED_ON_mips_shared
151 #define DEFINED_ON_loong64
152 #define DEFINED_ON_riscv64
153 #define DEFINED_ON_wasm32
154 #define DEFINED_ON_none
156 // Specialize for each architecture.
157 #if defined(JS_CODEGEN_X86)
158 # undef DEFINED_ON_x86
159 # define DEFINED_ON_x86 define
160 # undef DEFINED_ON_x86_shared
161 # define DEFINED_ON_x86_shared define
162 #elif defined(JS_CODEGEN_X64)
163 # undef DEFINED_ON_x64
164 # define DEFINED_ON_x64 define
165 # undef DEFINED_ON_x86_shared
166 # define DEFINED_ON_x86_shared define
167 #elif defined(JS_CODEGEN_ARM)
168 # undef DEFINED_ON_arm
169 # define DEFINED_ON_arm define
170 #elif defined(JS_CODEGEN_ARM64)
171 # undef DEFINED_ON_arm64
172 # define DEFINED_ON_arm64 define
173 #elif defined(JS_CODEGEN_MIPS32)
174 # undef DEFINED_ON_mips32
175 # define DEFINED_ON_mips32 define
176 # undef DEFINED_ON_mips_shared
177 # define DEFINED_ON_mips_shared define
178 #elif defined(JS_CODEGEN_MIPS64)
179 # undef DEFINED_ON_mips64
180 # define DEFINED_ON_mips64 define
181 # undef DEFINED_ON_mips_shared
182 # define DEFINED_ON_mips_shared define
183 #elif defined(JS_CODEGEN_LOONG64)
184 # undef DEFINED_ON_loong64
185 # define DEFINED_ON_loong64 define
186 #elif defined(JS_CODEGEN_RISCV64)
187 # undef DEFINED_ON_riscv64
188 # define DEFINED_ON_riscv64 define
189 #elif defined(JS_CODEGEN_WASM32)
190 # undef DEFINED_ON_wasm32
191 # define DEFINED_ON_wasm32 define
192 #elif defined(JS_CODEGEN_NONE)
193 # undef DEFINED_ON_none
194 # define DEFINED_ON_none crash
196 # error "Unknown architecture!"
199 #define DEFINED_ON_RESULT_crash \
201 #define DEFINED_ON_RESULT_define
202 #define DEFINED_ON_RESULT_ = delete
204 #define DEFINED_ON_DISPATCH_RESULT_2(Macro, Result) Macro##Result
205 #define DEFINED_ON_DISPATCH_RESULT(...) \
206 DEFINED_ON_DISPATCH_RESULT_2(DEFINED_ON_RESULT_, __VA_ARGS__)
208 // We need to let the evaluation of MOZ_FOR_EACH terminates.
209 #define DEFINED_ON_EXPAND_ARCH_RESULTS_3(ParenResult) \
210 DEFINED_ON_DISPATCH_RESULT ParenResult
211 #define DEFINED_ON_EXPAND_ARCH_RESULTS_2(ParenResult) \
212 DEFINED_ON_EXPAND_ARCH_RESULTS_3(ParenResult)
213 #define DEFINED_ON_EXPAND_ARCH_RESULTS(ParenResult) \
214 DEFINED_ON_EXPAND_ARCH_RESULTS_2(ParenResult)
216 #define DEFINED_ON_FWDARCH(Arch) DEFINED_ON_##Arch
217 #define DEFINED_ON_MAP_ON_ARCHS(ArchList) \
218 DEFINED_ON_EXPAND_ARCH_RESULTS( \
219 (MOZ_FOR_EACH(DEFINED_ON_FWDARCH, (), ArchList)))
221 #define DEFINED_ON(...) DEFINED_ON_MAP_ON_ARCHS((none, __VA_ARGS__))
223 #define PER_ARCH DEFINED_ON(ALL_ARCH)
224 #define PER_SHARED_ARCH DEFINED_ON(ALL_SHARED_ARCH)
225 #define OOL_IN_HEADER
227 class JSLinearString
;
230 struct ExpandoAndGeneration
;
236 class FixedLengthTypedArrayObject
;
238 enum class NativeIteratorIndices
: uint32_t;
243 class BytecodeOffset
;
244 class MemoryAccessDesc
;
246 struct ModuleEnvironment
;
248 enum class FailureMode
: uint8_t;
250 enum class SymbolicAddress
;
256 // Defined in JitFrames.h
257 enum class ExitFrameType
: uint8_t;
259 class AutoSaveLiveRegisters
;
261 class TemplateNativeObject
;
262 class TemplateObject
;
264 enum class CheckUnsafeCallWithABI
{
265 // Require the callee to use AutoUnsafeCallWithABI.
268 // We pushed an exit frame so this callWithABI can safely GC and walk the
270 DontCheckHasExitFrame
,
272 // Don't check this callWithABI uses AutoUnsafeCallWithABI, for instance
273 // because we're calling a simple helper function (like malloc or js_free)
274 // that we can't change and/or that we know won't GC.
278 // This is a global function made to create the DynFn type in a controlled
279 // environment which would check if the function signature has been registered
280 // as an ABI function signature.
281 template <typename Sig
>
282 static inline DynFn
DynamicFunction(Sig fun
);
284 enum class CharEncoding
{ Latin1
, TwoByte
};
286 constexpr uint32_t WasmCallerInstanceOffsetBeforeCall
=
287 wasm::FrameWithInstances::callerInstanceOffsetWithoutFrame();
288 constexpr uint32_t WasmCalleeInstanceOffsetBeforeCall
=
289 wasm::FrameWithInstances::calleeInstanceOffsetWithoutFrame();
291 // Allocation sites may be passed to GC thing allocation methods either via a
292 // register (for baseline compilation) or an enum indicating one of the
293 // catch-all allocation sites (for optimized compilation).
294 struct AllocSiteInput
295 : public mozilla::Variant
<Register
, gc::CatchAllAllocSite
> {
296 using Base
= mozilla::Variant
<Register
, gc::CatchAllAllocSite
>;
297 AllocSiteInput() : Base(gc::CatchAllAllocSite::Unknown
) {}
298 explicit AllocSiteInput(gc::CatchAllAllocSite catchAll
) : Base(catchAll
) {}
299 explicit AllocSiteInput(Register reg
) : Base(reg
) {}
302 #ifdef ENABLE_WASM_TAIL_CALLS
303 // Instance slots (including ShadowStackArea) and arguments size information
304 // from two neighboring frames.
305 // Used in Wasm tail calls to remove frame.
306 struct ReturnCallAdjustmentInfo
{
307 uint32_t newSlotsAndStackArgBytes
;
308 uint32_t oldSlotsAndStackArgBytes
;
310 ReturnCallAdjustmentInfo(uint32_t newSlotsAndStackArgBytes
,
311 uint32_t oldSlotsAndStackArgBytes
)
312 : newSlotsAndStackArgBytes(newSlotsAndStackArgBytes
),
313 oldSlotsAndStackArgBytes(oldSlotsAndStackArgBytes
) {}
315 #endif // ENABLE_WASM_TAIL_CALLS
317 struct BranchWasmRefIsSubtypeRegisters
{
323 // [SMDOC] Code generation invariants (incomplete)
325 // ## 64-bit GPRs carrying 32-bit values
327 // At least at the end of every JS or Wasm operation (= SpiderMonkey bytecode or
328 // Wasm bytecode; this is necessarily a little vague), if a 64-bit GPR has a
329 // 32-bit value, then the upper 32 bits of the register may be predictable in
330 // accordance with platform-specific rules, as follows.
332 // - On x64 and arm64, the upper bits are zero
333 // - On mips64 and loongarch64 the upper bits are the sign extension of the
335 // - (On risc-v we have no rule, having no port yet. Sign extension is the most
336 // likely rule, but "unpredictable" is an option.)
338 // In most cases no extra work needs to be done to maintain the invariant:
340 // - 32-bit operations on x64 and arm64 zero-extend the result to 64 bits.
341 // These operations ignore the upper bits of the inputs.
342 // - 32-bit operations on mips64 sign-extend the result to 64 bits (even many
343 // that are labeled as "unsigned", eg ADDU, though not all, eg LU).
344 // Additionally, the inputs to many 32-bit operations must be properly
345 // sign-extended to avoid "unpredictable" behavior, and our simulators check
346 // that inputs conform.
347 // - (32-bit operations on risc-v and loongarch64 sign-extend, much as mips, but
348 // appear to ignore the upper bits of the inputs.)
350 // The upshot of these invariants is, among other things, that:
352 // - No code needs to be generated when a 32-bit value is extended to 64 bits
353 // or a 64-bit value is wrapped to 32 bits, if the upper bits are known to be
354 // correct because they resulted from an operation that produced them
356 // - Literal loads must be careful to avoid instructions that might extend the
357 // literal in the wrong way.
358 // - Code that produces values using intermediate values with non-canonical
359 // extensions must extend according to platform conventions before being
362 // All optimizations are necessarily platform-specific and should only be used
363 // in platform-specific code. We may add architectures in the future that do
364 // not follow the patterns of the few architectures we already have.
366 // Also see MacroAssembler::debugAssertCanonicalInt32().
368 // The public entrypoint for emitting assembly. Note that a MacroAssembler can
369 // use cx->lifoAlloc, so take care not to interleave masm use with other
370 // lifoAlloc use if one will be destroyed before the other.
371 class MacroAssembler
: public MacroAssemblerSpecific
{
373 // Information about the current JSRuntime. This is nullptr only for Wasm
375 CompileRuntime
* maybeRuntime_
= nullptr;
377 // Information about the current Realm. This is nullptr for Wasm compilations
378 // and when compiling JitRuntime trampolines.
379 CompileRealm
* maybeRealm_
= nullptr;
381 // Labels for handling exceptions and failures.
382 NonAssertingLabel failureLabel_
;
385 // Constructor is protected. Use one of the derived classes!
386 explicit MacroAssembler(TempAllocator
& alloc
,
387 CompileRuntime
* maybeRuntime
= nullptr,
388 CompileRealm
* maybeRealm
= nullptr);
391 MoveResolver
& moveResolver() {
392 // As an optimization, the MoveResolver is a persistent data structure
393 // shared between visitors in the CodeGenerator. This assertion
394 // checks that state is not leaking from visitor to visitor
395 // via an unresolved addMove().
396 MOZ_ASSERT(moveResolver_
.hasNoPendingMoves());
397 return moveResolver_
;
400 size_t instructionsSize() const { return size(); }
402 CompileRealm
* realm() const {
403 MOZ_ASSERT(maybeRealm_
);
406 CompileRuntime
* runtime() const {
407 MOZ_ASSERT(maybeRuntime_
);
408 return maybeRuntime_
;
411 #ifdef JS_HAS_HIDDEN_SP
412 void Push(RegisterOrSP reg
);
415 #ifdef ENABLE_WASM_SIMD
416 // `op` should be a shift operation. Return true if a variable-width shift
417 // operation on this architecture should pre-mask the shift count, and if so,
418 // return the mask in `*mask`.
419 static bool MustMaskShiftCountSimd128(wasm::SimdOp op
, int32_t* mask
);
422 //{{{ check_macroassembler_decl_style
424 // ===============================================================
425 // MacroAssembler high-level usage.
427 // Flushes the assembly buffer, on platforms that need it.
428 void flush() PER_SHARED_ARCH
;
430 // Add a comment that is visible in the pretty printed assembly code.
431 void comment(const char* msg
) PER_SHARED_ARCH
;
433 // ===============================================================
434 // Frame manipulation functions.
436 inline uint32_t framePushed() const OOL_IN_HEADER
;
437 inline void setFramePushed(uint32_t framePushed
) OOL_IN_HEADER
;
438 inline void adjustFrame(int32_t value
) OOL_IN_HEADER
;
440 // Adjust the frame, to account for implicit modification of the stack
441 // pointer, such that callee can remove arguments on the behalf of the
443 inline void implicitPop(uint32_t bytes
) OOL_IN_HEADER
;
446 // This field is used to statically (at compilation time) emulate a frame
447 // pointer by keeping track of stack manipulations.
449 // It is maintained by all stack manipulation functions below.
450 uint32_t framePushed_
;
453 // ===============================================================
454 // Stack manipulation functions -- sets of registers.
456 // Approximately speaking, the following routines must use the same memory
457 // layout. Any inconsistencies will certainly lead to crashing in generated
460 // MacroAssembler::PushRegsInMaskSizeInBytes
461 // MacroAssembler::PushRegsInMask
462 // MacroAssembler::storeRegsInMask
463 // MacroAssembler::PopRegsInMask
464 // MacroAssembler::PopRegsInMaskIgnore
465 // FloatRegister::getRegisterDumpOffsetInBytes
466 // (no class) PushRegisterDump
467 // (union) RegisterContent
468 // JitRuntime::generateInvalidator
469 // JitRuntime::generateBailoutHandler
470 // JSJitFrameIter::machineState
472 // To be more exact, the invariants are:
474 // * The save area is conceptually viewed as starting at a highest address
475 // (really, at "highest address - 1") and working down to some lower
478 // * PushRegsInMask, storeRegsInMask and PopRegsInMask{Ignore} must use
479 // exactly the same memory layout, when starting from the abovementioned
482 // * PushRegsInMaskSizeInBytes must produce a value which is exactly equal
483 // to the change in the machine's stack pointer register as a result of
484 // calling PushRegsInMask or PopRegsInMask{Ignore}. This value must be at
485 // least uintptr_t-aligned on the target, and may be more aligned than that.
487 // * PushRegsInMaskSizeInBytes must produce a value which is greater than or
488 // equal to the amount of space used by storeRegsInMask.
490 // * Hence, regardless of whether the save area is created with
491 // storeRegsInMask or PushRegsInMask, it is guaranteed to fit inside an
492 // area of size calculated by PushRegsInMaskSizeInBytes.
494 // * For the `ignore` argument of PopRegsInMaskIgnore, equality checking
495 // for the floating point/SIMD registers is done on the basis of the
496 // underlying physical register, regardless of width. For example, if the
497 // to-restore set contains v17 (the SIMD register with encoding 17) and
498 // the ignore set contains d17 (the double register with encoding 17) then
499 // no part of the physical register with encoding 17 will be restored.
500 // (This is probably not true on arm32, since that has aliased float32
501 // registers; but none of our other targets do.)
503 // * {Push,store}RegsInMask/storeRegsInMask are further constrained as
504 // follows: when given the argument AllFloatRegisters, the resulting
505 // memory area must contain exactly all the SIMD/FP registers for the
506 // target at their widest width (that we care about). [We have no targets
507 // where the SIMD registers and FP register sets are disjoint.] They must
508 // be packed end-to-end with no holes, with the register with the lowest
509 // encoding number (0), as returned by FloatRegister::encoding(), at the
510 // abovementioned highest address, register 1 just below that, etc.
512 // Furthermore the sizeof(RegisterContent) must equal the size of a SIMD
513 // register in the abovementioned array.
515 // Furthermore the value returned by
516 // FloatRegister::getRegisterDumpOffsetInBytes must be a correct index
517 // into the abovementioned array. Given the constraints, the only correct
518 // value is `reg.encoding() * sizeof(RegisterContent)`.
520 // Note that some of the routines listed above are JS-only, and do not support
521 // SIMD registers. They are otherwise part of the same equivalence class.
522 // Register spilling for e.g. OOL VM calls is implemented using
523 // PushRegsInMask, and recovered on bailout using machineState. This requires
524 // the same layout to be used in machineState, and therefore in all other code
525 // that can spill registers that are recovered on bailout. Implementations of
526 // JitRuntime::generate{Invalidator,BailoutHandler} should either call
527 // PushRegsInMask, or check carefully to be sure that they generate the same
530 // The size of the area used by PushRegsInMask.
531 static size_t PushRegsInMaskSizeInBytes(LiveRegisterSet set
)
532 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
535 void PushRegsInMask(LiveRegisterSet set
)
536 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
538 void PushRegsInMask(LiveGeneralRegisterSet set
);
540 // Like PushRegsInMask, but instead of pushing the registers, store them to
541 // |dest|. |dest| should point to the end of the reserved space, so the
542 // first register will be stored at |dest.offset - sizeof(register)|. It is
543 // required that |dest.offset| is at least as large as the value computed by
544 // PushRegsInMaskSizeInBytes for this |set|. In other words, |dest.base|
545 // must point to either the lowest address in the save area, or some address
547 void storeRegsInMask(LiveRegisterSet set
, Address dest
, Register scratch
)
548 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
551 void PopRegsInMask(LiveRegisterSet set
);
552 void PopRegsInMask(LiveGeneralRegisterSet set
);
553 void PopRegsInMaskIgnore(LiveRegisterSet set
, LiveRegisterSet ignore
)
554 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
557 // ===============================================================
558 // Stack manipulation functions -- single registers/values.
560 void Push(const Operand op
) DEFINED_ON(x86_shared
);
561 void Push(Register reg
) PER_SHARED_ARCH
;
562 void Push(Register reg1
, Register reg2
, Register reg3
, Register reg4
)
564 void Push(const Imm32 imm
) PER_SHARED_ARCH
;
565 void Push(const ImmWord imm
) PER_SHARED_ARCH
;
566 void Push(const ImmPtr imm
) PER_SHARED_ARCH
;
567 void Push(const ImmGCPtr ptr
) PER_SHARED_ARCH
;
568 void Push(FloatRegister reg
) PER_SHARED_ARCH
;
569 void PushBoxed(FloatRegister reg
) PER_ARCH
;
570 void PushFlags() DEFINED_ON(x86_shared
);
571 void Push(PropertyKey key
, Register scratchReg
);
572 void Push(const Address
& addr
);
573 void Push(TypedOrValueRegister v
);
574 void Push(const ConstantOrRegister
& v
);
575 void Push(const ValueOperand
& val
);
576 void Push(const Value
& val
);
577 void Push(JSValueType type
, Register reg
);
578 void Push(const Register64 reg
);
579 void PushEmptyRooted(VMFunctionData::RootType rootType
);
580 inline CodeOffset
PushWithPatch(ImmWord word
);
581 inline CodeOffset
PushWithPatch(ImmPtr imm
);
583 void Pop(const Operand op
) DEFINED_ON(x86_shared
);
584 void Pop(Register reg
) PER_SHARED_ARCH
;
585 void Pop(FloatRegister t
) PER_SHARED_ARCH
;
586 void Pop(const ValueOperand
& val
) PER_SHARED_ARCH
;
587 void PopFlags() DEFINED_ON(x86_shared
);
589 DEFINED_ON(arm
, mips_shared
, x86_shared
, loong64
, riscv64
, wasm32
);
591 // Move the stack pointer based on the requested amount.
592 void adjustStack(int amount
);
593 void freeStack(uint32_t amount
);
595 // Move the stack pointer to the specified position. It assumes the SP
596 // register is not valid -- it uses FP to set the position.
597 void freeStackTo(uint32_t framePushed
)
598 DEFINED_ON(x86_shared
, arm
, arm64
, loong64
, mips64
, riscv64
);
600 // Warning: This method does not update the framePushed() counter.
601 void freeStack(Register amount
);
604 // ===============================================================
605 // Register allocation fields.
607 friend AutoRegisterScope
;
608 friend AutoFloatRegisterScope
;
609 // Used to track register scopes for debug builds.
610 // Manipulated by the AutoGenericRegisterScope class.
611 AllocatableRegisterSet debugTrackedRegisters_
;
615 // ===============================================================
616 // Simple call functions.
618 // The returned CodeOffset is the assembler offset for the instruction
619 // immediately following the call; that is, for the return point.
620 CodeOffset
call(Register reg
) PER_SHARED_ARCH
;
621 CodeOffset
call(Label
* label
) PER_SHARED_ARCH
;
623 void call(const Address
& addr
) PER_SHARED_ARCH
;
624 void call(ImmWord imm
) PER_SHARED_ARCH
;
625 // Call a target native function, which is neither traceable nor movable.
626 void call(ImmPtr imm
) PER_SHARED_ARCH
;
627 CodeOffset
call(wasm::SymbolicAddress imm
) PER_SHARED_ARCH
;
628 inline CodeOffset
call(const wasm::CallSiteDesc
& desc
,
629 wasm::SymbolicAddress imm
);
631 // Call a target JitCode, which must be traceable, and may be movable.
632 void call(JitCode
* c
) PER_SHARED_ARCH
;
634 inline void call(TrampolinePtr code
);
636 inline CodeOffset
call(const wasm::CallSiteDesc
& desc
, const Register reg
);
637 inline CodeOffset
call(const wasm::CallSiteDesc
& desc
, uint32_t funcDefIndex
);
638 inline void call(const wasm::CallSiteDesc
& desc
, wasm::Trap trap
);
640 CodeOffset
callWithPatch() PER_SHARED_ARCH
;
641 void patchCall(uint32_t callerOffset
, uint32_t calleeOffset
) PER_SHARED_ARCH
;
643 // Push the return address and make a call. On platforms where this function
644 // is not defined, push the link register (pushReturnAddress) at the entry
645 // point of the callee.
646 void callAndPushReturnAddress(Register reg
) DEFINED_ON(x86_shared
);
647 void callAndPushReturnAddress(Label
* label
) DEFINED_ON(x86_shared
);
649 // These do not adjust framePushed().
650 void pushReturnAddress()
651 DEFINED_ON(mips_shared
, arm
, arm64
, loong64
, riscv64
, wasm32
);
652 void popReturnAddress()
653 DEFINED_ON(mips_shared
, arm
, arm64
, loong64
, riscv64
, wasm32
);
655 // Useful for dealing with two-valued returns.
656 void moveRegPair(Register src0
, Register src1
, Register dst0
, Register dst1
,
657 MoveOp::Type type
= MoveOp::GENERAL
);
659 void reserveVMFunctionOutParamSpace(const VMFunctionData
& f
);
660 void loadVMFunctionOutParam(const VMFunctionData
& f
, const Address
& addr
);
663 // ===============================================================
664 // Patchable near/far jumps.
666 // "Far jumps" provide the ability to jump to any uint32_t offset from any
667 // other uint32_t offset without using a constant pool (thus returning a
668 // simple CodeOffset instead of a CodeOffsetJump).
669 CodeOffset
farJumpWithPatch() PER_SHARED_ARCH
;
670 void patchFarJump(CodeOffset farJump
, uint32_t targetOffset
) PER_SHARED_ARCH
;
672 // Emit a nop that can be patched to and from a nop and a call with int32
673 // relative displacement.
674 CodeOffset
nopPatchableToCall() PER_SHARED_ARCH
;
675 void nopPatchableToCall(const wasm::CallSiteDesc
& desc
);
676 static void patchNopToCall(uint8_t* callsite
,
677 uint8_t* target
) PER_SHARED_ARCH
;
678 static void patchCallToNop(uint8_t* callsite
) PER_SHARED_ARCH
;
680 // These methods are like movWithPatch/PatchDataWithValueCheck but allow
681 // using pc-relative addressing on certain platforms (RIP-relative LEA on x64,
682 // ADR instruction on arm64).
684 // Note: "Near" applies to ARM64 where the target must be within 1 MB (this is
685 // release-asserted).
686 CodeOffset
moveNearAddressWithPatch(Register dest
) PER_ARCH
;
687 static void patchNearAddressMove(CodeLocationLabel loc
,
688 CodeLocationLabel target
)
689 DEFINED_ON(x86
, x64
, arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
);
692 // ===============================================================
693 // [SMDOC] JIT-to-C++ Function Calls (callWithABI)
695 // callWithABI is used to make a call using the standard C/C++ system ABI.
697 // callWithABI is a low level interface for making calls, as such every call
698 // made with callWithABI should be organized with 6 steps: spilling live
699 // registers, aligning the stack, listing arguments of the called function,
700 // calling a function pointer, extracting the returned value and restoring
703 // A more detailed example of the six stages:
705 // 1) Saving of registers that are live. This will vary depending on which
706 // SpiderMonkey compiler you are working on. Registers that shouldn't be
707 // restored can be excluded.
709 // LiveRegisterSet volatileRegs(...);
710 // volatileRegs.take(scratch);
711 // masm.PushRegsInMask(volatileRegs);
713 // 2) Align the stack to perform the call with the correct stack alignment.
715 // When the stack pointer alignment is unknown and cannot be corrected
716 // when generating the code, setupUnalignedABICall must be used to
717 // dynamically align the stack pointer to the expectation of the ABI.
718 // When the stack pointer is known at JIT compilation time, the stack can
719 // be fixed manually and setupAlignedABICall and setupWasmABICall can be
722 // setupWasmABICall is a special case of setupAlignedABICall as
723 // SpiderMonkey's WebAssembly implementation mostly follow the system
724 // ABI, except for float/double arguments, which always use floating
725 // point registers, even if this is not supported by the system ABI.
727 // masm.setupUnalignedABICall(scratch);
729 // 3) Passing arguments. Arguments are passed left-to-right.
731 // masm.passABIArg(scratch);
732 // masm.passABIArg(FloatOp0, ABIType::Float64);
734 // Note how float register arguments are annotated with ABIType::Float64.
736 // Concerning stack-relative address, see the note on passABIArg.
740 // using Fn = int32_t (*)(int32_t)
741 // masm.callWithABI<Fn, Callee>();
743 // In the case where the call returns a double, that needs to be
744 // indicated to the callWithABI like this:
746 // using Fn = double (*)(int32_t)
747 // masm.callWithABI<Fn, Callee>(ABIType::Float64);
749 // There are overloads to allow calls to registers and addresses.
751 // 5) Take care of the result
753 // masm.storeCallPointerResult(scratch1);
754 // masm.storeCallBoolResult(scratch1);
755 // masm.storeCallInt32Result(scratch1);
756 // masm.storeCallFloatResult(scratch1);
758 // 6) Restore the potentially clobbered volatile registers
760 // masm.PopRegsInMask(volatileRegs);
762 // If expecting a returned value, this call should use
763 // PopRegsInMaskIgnore to filter out the registers which are containing
764 // the returned value.
766 // Unless an exit frame is pushed prior to the setupABICall, the callee
767 // should not GC. To ensure this is the case callWithABI is instrumented to
768 // make sure that in the default case callees are annotated with an
769 // AutoUnsafeCallWithABI on the stack.
771 // A callWithABI can opt out of checking, if for example it is known there
772 // is an exit frame, or the callee is known not to GC.
774 // If your callee needs to be able to GC, consider using a VMFunction, or
775 // create a fake exit frame, and instrument the TraceJitExitFrame
778 // Setup a call to C/C++ code, given the assumption that the framePushed
779 // accurately defines the state of the stack, and that the top of the stack
780 // was properly aligned. Note that this only supports cdecl.
782 // As a rule of thumb, this can be used in CodeGenerator but not in CacheIR or
783 // Baseline code (because the stack is not aligned to ABIStackAlignment).
784 void setupAlignedABICall();
786 // As setupAlignedABICall, but for WebAssembly native ABI calls, which pass
787 // through a builtin thunk that uses the wasm ABI. All the wasm ABI calls
788 // can be native, since we always know the stack alignment a priori.
789 void setupWasmABICall();
791 // Setup an ABI call for when the alignment is not known. This may need a
793 void setupUnalignedABICall(Register scratch
) PER_ARCH
;
795 // Like setupUnalignedABICall, but more efficient because it doesn't push/pop
796 // the unaligned stack pointer. The caller is responsible for restoring SP
797 // after the callWithABI, for example using the frame pointer register.
798 void setupUnalignedABICallDontSaveRestoreSP();
800 // Arguments must be assigned to a C/C++ call in order. They are moved
801 // in parallel immediately before performing the call. This process may
802 // temporarily use more stack, in which case esp-relative addresses will be
803 // automatically adjusted. It is extremely important that esp-relative
804 // addresses are computed *after* setupABICall(). Furthermore, no
805 // operations should be emitted while setting arguments.
806 void passABIArg(const MoveOperand
& from
, ABIType type
);
807 inline void passABIArg(Register reg
);
808 inline void passABIArg(FloatRegister reg
, ABIType type
);
810 inline void callWithABI(
811 DynFn fun
, ABIType result
= ABIType::General
,
812 CheckUnsafeCallWithABI check
= CheckUnsafeCallWithABI::Check
);
813 template <typename Sig
, Sig fun
>
814 inline void callWithABI(
815 ABIType result
= ABIType::General
,
816 CheckUnsafeCallWithABI check
= CheckUnsafeCallWithABI::Check
);
817 inline void callWithABI(Register fun
, ABIType result
= ABIType::General
);
818 inline void callWithABI(const Address
& fun
,
819 ABIType result
= ABIType::General
);
821 CodeOffset
callWithABI(wasm::BytecodeOffset offset
, wasm::SymbolicAddress fun
,
822 mozilla::Maybe
<int32_t> instanceOffset
,
823 ABIType result
= ABIType::General
);
824 void callDebugWithABI(wasm::SymbolicAddress fun
,
825 ABIType result
= ABIType::General
);
828 // Reinitialize the variables which have to be cleared before making a call
830 template <class ABIArgGeneratorT
>
831 void setupABICallHelper();
833 // Reinitialize the variables which have to be cleared before making a call
835 void setupNativeABICall();
837 // Reserve the stack and resolve the arguments move.
838 void callWithABIPre(uint32_t* stackAdjust
,
839 bool callFromWasm
= false) PER_ARCH
;
841 // Emits a call to a C/C++ function, resolving all argument moves.
842 void callWithABINoProfiler(void* fun
, ABIType result
,
843 CheckUnsafeCallWithABI check
);
844 void callWithABINoProfiler(Register fun
, ABIType result
) PER_ARCH
;
845 void callWithABINoProfiler(const Address
& fun
, ABIType result
) PER_ARCH
;
847 // Restore the stack to its state before the setup function call.
848 void callWithABIPost(uint32_t stackAdjust
, ABIType result
,
849 bool callFromWasm
= false) PER_ARCH
;
851 // Create the signature to be able to decode the arguments of a native
852 // function, when calling a function within the simulator.
853 inline void appendSignatureType(ABIType type
);
854 inline ABIFunctionType
signature() const;
856 // Private variables used to handle moves between registers given as
857 // arguments to passABIArg and the list of ABI registers expected for the
858 // signature of the function.
859 MoveResolver moveResolver_
;
861 // Architecture specific implementation which specify how registers & stack
862 // offsets are used for calling a function.
863 ABIArgGenerator abiArgs_
;
866 // Flag use to assert that we use ABI function in the right context.
870 // If set by setupUnalignedABICall then callWithABI will pop the stack
871 // register which is on the stack.
872 bool dynamicAlignment_
;
875 // The signature is used to accumulate all types of arguments which are used
876 // by the caller. This is used by the simulators to decode the arguments
877 // properly, and cast the function pointer to the right type.
882 // ===============================================================
885 // These functions are used to build the content of the Jit frames. See
886 // CommonFrameLayout class, and all its derivatives. The content should be
887 // pushed in the opposite order as the fields of the structures, such that
888 // the structures can be used to interpret the content of the stack.
890 // Call the Jit function, and push the return address (or let the callee
891 // push the return address).
893 // These functions return the offset of the return address, in order to use
894 // the return address to index the safepoints, which are used to list all
896 inline uint32_t callJitNoProfiler(Register callee
);
897 inline uint32_t callJit(Register callee
);
898 inline uint32_t callJit(JitCode
* code
);
899 inline uint32_t callJit(TrampolinePtr code
);
900 inline uint32_t callJit(ImmPtr callee
);
902 // The frame descriptor is the second field of all Jit frames, pushed before
903 // calling the Jit function. See CommonFrameLayout::descriptor_.
904 inline void pushFrameDescriptor(FrameType type
);
905 inline void PushFrameDescriptor(FrameType type
);
907 // For JitFrameLayout, the descriptor also stores the number of arguments
908 // passed by the caller. See MakeFrameDescriptorForJitCall.
909 inline void pushFrameDescriptorForJitCall(FrameType type
, uint32_t argc
);
910 inline void pushFrameDescriptorForJitCall(FrameType type
, Register argc
,
912 inline void PushFrameDescriptorForJitCall(FrameType type
, uint32_t argc
);
913 inline void PushFrameDescriptorForJitCall(FrameType type
, Register argc
,
916 // Load the number of actual arguments from the frame's JitFrameLayout.
917 inline void loadNumActualArgs(Register framePtr
, Register dest
);
919 // Push the callee token of a JSFunction which pointer is stored in the
920 // |callee| register. The callee token is packed with a |constructing| flag
921 // which correspond to the fact that the JS function is called with "new" or
923 inline void PushCalleeToken(Register callee
, bool constructing
);
925 // Unpack a callee token located at the |token| address, and return the
926 // JSFunction pointer in the |dest| register.
927 inline void loadFunctionFromCalleeToken(Address token
, Register dest
);
929 // This function emulates a call by pushing an exit frame on the stack,
930 // except that the fake-function is inlined within the body of the caller.
932 // This function assumes that the current frame is an IonJS frame.
934 // This function returns the offset of the /fake/ return address, in order to
935 // use the return address to index the safepoints, which are used to list all
938 // This function should be balanced with a call to adjustStack, to pop the
939 // exit frame and emulate the return statement of the inlined function.
940 inline uint32_t buildFakeExitFrame(Register scratch
);
943 // This function is used by buildFakeExitFrame to push a fake return address
944 // on the stack. This fake return address should never be used for resuming
945 // any execution, and can even be an invalid pointer into the instruction
946 // stream, as long as it does not alias any other.
947 uint32_t pushFakeReturnAddress(Register scratch
) PER_SHARED_ARCH
;
950 // ===============================================================
951 // Exit frame footer.
953 // When calling outside the Jit we push an exit frame. To mark the stack
954 // correctly, we have to push additional information, called the Exit frame
955 // footer, which is used to identify how the stack is marked.
957 // See JitFrames.h, and TraceJitExitFrame in JitFrames.cpp.
959 // Links the exit frame and pushes the ExitFooterFrame.
960 inline void enterExitFrame(Register cxreg
, Register scratch
, VMFunctionId f
);
962 // Push an exit frame token to identify which fake exit frame this footer
964 inline void enterFakeExitFrame(Register cxreg
, Register scratch
,
967 // Push an exit frame token for a native call.
968 inline void enterFakeExitFrameForNative(Register cxreg
, Register scratch
,
969 bool isConstructing
);
971 // Pop ExitFrame footer in addition to the extra frame.
972 inline void leaveExitFrame(size_t extraFrame
= 0);
975 // Save the top of the stack into JitActivation::packedExitFP of the
976 // current thread, which should be the location of the latest exit frame.
977 void linkExitFrame(Register cxreg
, Register scratch
);
980 // ===============================================================
983 inline void move64(Imm64 imm
, Register64 dest
) PER_ARCH
;
984 inline void move64(Register64 src
, Register64 dest
) PER_ARCH
;
986 inline void moveFloat32ToGPR(FloatRegister src
,
987 Register dest
) PER_SHARED_ARCH
;
988 inline void moveGPRToFloat32(Register src
,
989 FloatRegister dest
) PER_SHARED_ARCH
;
991 inline void moveDoubleToGPR64(FloatRegister src
, Register64 dest
) PER_ARCH
;
992 inline void moveGPR64ToDouble(Register64 src
, FloatRegister dest
) PER_ARCH
;
994 inline void move8ZeroExtend(Register src
, Register dest
) PER_SHARED_ARCH
;
996 inline void move8SignExtend(Register src
, Register dest
) PER_SHARED_ARCH
;
997 inline void move16SignExtend(Register src
, Register dest
) PER_SHARED_ARCH
;
999 // move64To32 will clear the high bits of `dest` on 64-bit systems.
1000 inline void move64To32(Register64 src
, Register dest
) PER_ARCH
;
1002 inline void move32To64ZeroExtend(Register src
, Register64 dest
) PER_ARCH
;
1004 inline void move8To64SignExtend(Register src
, Register64 dest
) PER_ARCH
;
1005 inline void move16To64SignExtend(Register src
, Register64 dest
) PER_ARCH
;
1006 inline void move32To64SignExtend(Register src
, Register64 dest
) PER_ARCH
;
1008 inline void move32SignExtendToPtr(Register src
, Register dest
) PER_ARCH
;
1009 inline void move32ZeroExtendToPtr(Register src
, Register dest
) PER_ARCH
;
1011 // Copy a constant, typed-register, or a ValueOperand into a ValueOperand
1013 inline void moveValue(const ConstantOrRegister
& src
,
1014 const ValueOperand
& dest
);
1015 void moveValue(const TypedOrValueRegister
& src
,
1016 const ValueOperand
& dest
) PER_ARCH
;
1017 void moveValue(const ValueOperand
& src
, const ValueOperand
& dest
) PER_ARCH
;
1018 void moveValue(const Value
& src
, const ValueOperand
& dest
) PER_ARCH
;
1020 void movePropertyKey(PropertyKey key
, Register dest
);
1022 // ===============================================================
1023 // Load instructions
1025 inline void load32SignExtendToPtr(const Address
& src
, Register dest
) PER_ARCH
;
1027 inline void loadAbiReturnAddress(Register dest
) PER_SHARED_ARCH
;
1029 // ===============================================================
1030 // Copy instructions
1032 inline void copy64(const Address
& src
, const Address
& dest
, Register scratch
);
1035 // ===============================================================
1036 // Logical instructions
1038 inline void not32(Register reg
) PER_SHARED_ARCH
;
1039 inline void notPtr(Register reg
) PER_ARCH
;
1041 inline void and32(Register src
, Register dest
) PER_SHARED_ARCH
;
1042 inline void and32(Imm32 imm
, Register dest
) PER_SHARED_ARCH
;
1043 inline void and32(Imm32 imm
, Register src
, Register dest
) DEFINED_ON(arm64
);
1044 inline void and32(Imm32 imm
, const Address
& dest
) PER_SHARED_ARCH
;
1045 inline void and32(const Address
& src
, Register dest
) PER_SHARED_ARCH
;
1047 inline void andPtr(Register src
, Register dest
) PER_ARCH
;
1048 inline void andPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1050 inline void and64(Imm64 imm
, Register64 dest
) PER_ARCH
;
1051 inline void or64(Imm64 imm
, Register64 dest
) PER_ARCH
;
1052 inline void xor64(Imm64 imm
, Register64 dest
) PER_ARCH
;
1054 inline void or32(Register src
, Register dest
) PER_SHARED_ARCH
;
1055 inline void or32(Imm32 imm
, Register dest
) PER_SHARED_ARCH
;
1056 inline void or32(Imm32 imm
, const Address
& dest
) PER_SHARED_ARCH
;
1058 inline void orPtr(Register src
, Register dest
) PER_ARCH
;
1059 inline void orPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1061 inline void and64(Register64 src
, Register64 dest
) PER_ARCH
;
1062 inline void or64(Register64 src
, Register64 dest
) PER_ARCH
;
1063 inline void xor64(Register64 src
, Register64 dest
) PER_ARCH
;
1065 inline void xor32(Register src
, Register dest
) PER_SHARED_ARCH
;
1066 inline void xor32(Imm32 imm
, Register dest
) PER_SHARED_ARCH
;
1067 inline void xor32(Imm32 imm
, const Address
& dest
) PER_SHARED_ARCH
;
1068 inline void xor32(const Address
& src
, Register dest
) PER_SHARED_ARCH
;
1070 inline void xorPtr(Register src
, Register dest
) PER_ARCH
;
1071 inline void xorPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1073 inline void and64(const Operand
& src
, Register64 dest
)
1074 DEFINED_ON(x64
, mips64
, loong64
, riscv64
);
1075 inline void or64(const Operand
& src
, Register64 dest
)
1076 DEFINED_ON(x64
, mips64
, loong64
, riscv64
);
1077 inline void xor64(const Operand
& src
, Register64 dest
)
1078 DEFINED_ON(x64
, mips64
, loong64
, riscv64
);
1080 // ===============================================================
1081 // Swap instructions
1083 // Swap the two lower bytes and sign extend the result to 32-bit.
1084 inline void byteSwap16SignExtend(Register reg
) PER_SHARED_ARCH
;
1086 // Swap the two lower bytes and zero extend the result to 32-bit.
1087 inline void byteSwap16ZeroExtend(Register reg
) PER_SHARED_ARCH
;
1089 // Swap all four bytes in a 32-bit integer.
1090 inline void byteSwap32(Register reg
) PER_SHARED_ARCH
;
1092 // Swap all eight bytes in a 64-bit integer.
1093 inline void byteSwap64(Register64 reg
) PER_ARCH
;
1095 // ===============================================================
1096 // Arithmetic functions
1098 // Condition flags aren't guaranteed to be set by these functions, for example
1099 // x86 will always set condition flags, but ARM64 won't do it unless
1100 // explicitly requested. Instead use branch(Add|Sub|Mul|Neg) to test for
1101 // condition flags after performing arithmetic operations.
1103 inline void add32(Register src
, Register dest
) PER_SHARED_ARCH
;
1104 inline void add32(Imm32 imm
, Register dest
) PER_SHARED_ARCH
;
1105 inline void add32(Imm32 imm
, Register src
, Register dest
) PER_SHARED_ARCH
;
1106 inline void add32(Imm32 imm
, const Address
& dest
) PER_SHARED_ARCH
;
1107 inline void add32(Imm32 imm
, const AbsoluteAddress
& dest
)
1108 DEFINED_ON(x86_shared
);
1110 inline void addPtr(Register src
, Register dest
) PER_ARCH
;
1111 inline void addPtr(Register src1
, Register src2
, Register dest
)
1113 inline void addPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1114 inline void addPtr(Imm32 imm
, Register src
, Register dest
) DEFINED_ON(arm64
);
1115 inline void addPtr(ImmWord imm
, Register dest
) PER_ARCH
;
1116 inline void addPtr(ImmPtr imm
, Register dest
);
1117 inline void addPtr(Imm32 imm
, const Address
& dest
)
1118 DEFINED_ON(mips_shared
, arm
, arm64
, x86
, x64
, loong64
, riscv64
, wasm32
);
1119 inline void addPtr(Imm32 imm
, const AbsoluteAddress
& dest
)
1120 DEFINED_ON(x86
, x64
);
1121 inline void addPtr(const Address
& src
, Register dest
)
1122 DEFINED_ON(mips_shared
, arm
, arm64
, x86
, x64
, loong64
, riscv64
, wasm32
);
1124 inline void add64(Register64 src
, Register64 dest
) PER_ARCH
;
1125 inline void add64(Imm32 imm
, Register64 dest
) PER_ARCH
;
1126 inline void add64(Imm64 imm
, Register64 dest
) PER_ARCH
;
1127 inline void add64(const Operand
& src
, Register64 dest
)
1128 DEFINED_ON(x64
, mips64
, loong64
, riscv64
);
1130 inline void addFloat32(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1132 // Compute dest=SP-imm where dest is a pointer registers and not SP. The
1133 // offset returned from sub32FromStackPtrWithPatch() must be passed to
1134 // patchSub32FromStackPtr().
1135 inline CodeOffset
sub32FromStackPtrWithPatch(Register dest
) PER_ARCH
;
1136 inline void patchSub32FromStackPtr(CodeOffset offset
, Imm32 imm
) PER_ARCH
;
1138 inline void addDouble(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1139 inline void addConstantDouble(double d
, FloatRegister dest
) DEFINED_ON(x86
);
1141 inline void sub32(const Address
& src
, Register dest
) PER_SHARED_ARCH
;
1142 inline void sub32(Register src
, Register dest
) PER_SHARED_ARCH
;
1143 inline void sub32(Imm32 imm
, Register dest
) PER_SHARED_ARCH
;
1145 inline void subPtr(Register src
, Register dest
) PER_ARCH
;
1146 inline void subPtr(Register src
, const Address
& dest
)
1147 DEFINED_ON(mips_shared
, arm
, arm64
, x86
, x64
, loong64
, riscv64
, wasm32
);
1148 inline void subPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1149 inline void subPtr(ImmWord imm
, Register dest
) DEFINED_ON(x64
);
1150 inline void subPtr(const Address
& addr
, Register dest
)
1151 DEFINED_ON(mips_shared
, arm
, arm64
, x86
, x64
, loong64
, riscv64
, wasm32
);
1153 inline void sub64(Register64 src
, Register64 dest
) PER_ARCH
;
1154 inline void sub64(Imm64 imm
, Register64 dest
) PER_ARCH
;
1155 inline void sub64(const Operand
& src
, Register64 dest
)
1156 DEFINED_ON(x64
, mips64
, loong64
, riscv64
);
1158 inline void subFloat32(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1160 inline void subDouble(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1162 inline void mul32(Register rhs
, Register srcDest
) PER_SHARED_ARCH
;
1163 inline void mul32(Imm32 imm
, Register srcDest
) PER_SHARED_ARCH
;
1165 inline void mul32(Register src1
, Register src2
, Register dest
, Label
* onOver
)
1168 // Return the high word of the unsigned multiplication into |dest|.
1169 inline void mulHighUnsigned32(Imm32 imm
, Register src
,
1170 Register dest
) PER_ARCH
;
1172 inline void mulPtr(Register rhs
, Register srcDest
) PER_ARCH
;
1174 inline void mul64(const Operand
& src
, const Register64
& dest
) DEFINED_ON(x64
);
1175 inline void mul64(const Operand
& src
, const Register64
& dest
,
1176 const Register temp
)
1177 DEFINED_ON(x64
, mips64
, loong64
, riscv64
);
1178 inline void mul64(Imm64 imm
, const Register64
& dest
) PER_ARCH
;
1179 inline void mul64(Imm64 imm
, const Register64
& dest
, const Register temp
)
1180 DEFINED_ON(x86
, x64
, arm
, mips32
, mips64
, loong64
, riscv64
);
1181 inline void mul64(const Register64
& src
, const Register64
& dest
,
1182 const Register temp
) PER_ARCH
;
1183 inline void mul64(const Register64
& src1
, const Register64
& src2
,
1184 const Register64
& dest
) DEFINED_ON(arm64
);
1185 inline void mul64(Imm64 src1
, const Register64
& src2
, const Register64
& dest
)
1188 inline void mulBy3(Register src
, Register dest
) PER_ARCH
;
1190 inline void mulFloat32(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1191 inline void mulDouble(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1193 inline void mulDoublePtr(ImmPtr imm
, Register temp
, FloatRegister dest
)
1194 DEFINED_ON(mips_shared
, arm
, arm64
, x86
, x64
, loong64
, riscv64
, wasm32
);
1196 // Perform an integer division, returning the integer part rounded toward
1197 // zero. rhs must not be zero, and the division must not overflow.
1199 // On ARM, the chip must have hardware division instructions.
1200 inline void quotient32(Register rhs
, Register srcDest
, bool isUnsigned
)
1201 DEFINED_ON(mips_shared
, arm
, arm64
, loong64
, riscv64
, wasm32
);
1203 // As above, but srcDest must be eax and tempEdx must be edx.
1204 inline void quotient32(Register rhs
, Register srcDest
, Register tempEdx
,
1205 bool isUnsigned
) DEFINED_ON(x86_shared
);
1207 // Perform an integer division, returning the remainder part.
1208 // rhs must not be zero, and the division must not overflow.
1210 // On ARM, the chip must have hardware division instructions.
1211 inline void remainder32(Register rhs
, Register srcDest
, bool isUnsigned
)
1212 DEFINED_ON(mips_shared
, arm
, arm64
, loong64
, riscv64
, wasm32
);
1214 // As above, but srcDest must be eax and tempEdx must be edx.
1215 inline void remainder32(Register rhs
, Register srcDest
, Register tempEdx
,
1216 bool isUnsigned
) DEFINED_ON(x86_shared
);
1218 // Perform an integer division, returning the integer part rounded toward
1219 // zero. rhs must not be zero, and the division must not overflow.
1221 // This variant preserves registers, and doesn't require hardware division
1222 // instructions on ARM (will call out to a runtime routine).
1224 // rhs is preserved, srdDest is clobbered.
1225 void flexibleRemainder32(Register rhs
, Register srcDest
, bool isUnsigned
,
1226 const LiveRegisterSet
& volatileLiveRegs
)
1227 DEFINED_ON(mips_shared
, arm
, arm64
, x86_shared
, loong64
, riscv64
, wasm32
);
1229 // Perform an integer division, returning the integer part rounded toward
1230 // zero. rhs must not be zero, and the division must not overflow.
1232 // This variant preserves registers, and doesn't require hardware division
1233 // instructions on ARM (will call out to a runtime routine).
1235 // rhs is preserved, srdDest is clobbered.
1236 void flexibleQuotient32(Register rhs
, Register srcDest
, bool isUnsigned
,
1237 const LiveRegisterSet
& volatileLiveRegs
)
1238 DEFINED_ON(mips_shared
, arm
, arm64
, x86_shared
, loong64
, riscv64
);
1240 // Perform an integer division, returning the integer part rounded toward
1241 // zero. rhs must not be zero, and the division must not overflow. The
1242 // remainder is stored into the third argument register here.
1244 // This variant preserves registers, and doesn't require hardware division
1245 // instructions on ARM (will call out to a runtime routine).
1247 // rhs is preserved, srdDest and remOutput are clobbered.
1248 void flexibleDivMod32(Register rhs
, Register srcDest
, Register remOutput
,
1250 const LiveRegisterSet
& volatileLiveRegs
)
1251 DEFINED_ON(mips_shared
, arm
, arm64
, x86_shared
, loong64
, riscv64
, wasm32
);
1253 inline void divFloat32(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1254 inline void divDouble(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1256 inline void inc64(AbsoluteAddress dest
) PER_ARCH
;
1258 inline void neg32(Register reg
) PER_SHARED_ARCH
;
1259 inline void neg64(Register64 reg
) PER_ARCH
;
1260 inline void negPtr(Register reg
) PER_ARCH
;
1262 inline void negateFloat(FloatRegister reg
) PER_SHARED_ARCH
;
1264 inline void negateDouble(FloatRegister reg
) PER_SHARED_ARCH
;
1266 inline void abs32(Register src
, Register dest
) PER_SHARED_ARCH
;
1267 inline void absFloat32(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1268 inline void absDouble(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1270 inline void sqrtFloat32(FloatRegister src
,
1271 FloatRegister dest
) PER_SHARED_ARCH
;
1272 inline void sqrtDouble(FloatRegister src
, FloatRegister dest
) PER_SHARED_ARCH
;
1274 void floorFloat32ToInt32(FloatRegister src
, Register dest
,
1275 Label
* fail
) PER_SHARED_ARCH
;
1276 void floorDoubleToInt32(FloatRegister src
, Register dest
,
1277 Label
* fail
) PER_SHARED_ARCH
;
1279 void ceilFloat32ToInt32(FloatRegister src
, Register dest
,
1280 Label
* fail
) PER_SHARED_ARCH
;
1281 void ceilDoubleToInt32(FloatRegister src
, Register dest
,
1282 Label
* fail
) PER_SHARED_ARCH
;
1284 void roundFloat32ToInt32(FloatRegister src
, Register dest
, FloatRegister temp
,
1285 Label
* fail
) PER_SHARED_ARCH
;
1286 void roundDoubleToInt32(FloatRegister src
, Register dest
, FloatRegister temp
,
1287 Label
* fail
) PER_SHARED_ARCH
;
1289 void truncFloat32ToInt32(FloatRegister src
, Register dest
,
1290 Label
* fail
) PER_SHARED_ARCH
;
1291 void truncDoubleToInt32(FloatRegister src
, Register dest
,
1292 Label
* fail
) PER_SHARED_ARCH
;
1294 void nearbyIntDouble(RoundingMode mode
, FloatRegister src
,
1295 FloatRegister dest
) PER_SHARED_ARCH
;
1296 void nearbyIntFloat32(RoundingMode mode
, FloatRegister src
,
1297 FloatRegister dest
) PER_SHARED_ARCH
;
1299 void signInt32(Register input
, Register output
);
1300 void signDouble(FloatRegister input
, FloatRegister output
);
1301 void signDoubleToInt32(FloatRegister input
, Register output
,
1302 FloatRegister temp
, Label
* fail
);
1304 void copySignDouble(FloatRegister lhs
, FloatRegister rhs
,
1305 FloatRegister output
) PER_SHARED_ARCH
;
1306 void copySignFloat32(FloatRegister lhs
, FloatRegister rhs
,
1307 FloatRegister output
) DEFINED_ON(x86_shared
, arm64
);
1309 // Returns a random double in range [0, 1) in |dest|. The |rng| register must
1310 // hold a pointer to a mozilla::non_crypto::XorShift128PlusRNG.
1311 void randomDouble(Register rng
, FloatRegister dest
, Register64 temp0
,
1314 // srcDest = {min,max}{Float32,Double}(srcDest, other)
1315 // For min and max, handle NaN specially if handleNaN is true.
1317 inline void minFloat32(FloatRegister other
, FloatRegister srcDest
,
1318 bool handleNaN
) PER_SHARED_ARCH
;
1319 inline void minDouble(FloatRegister other
, FloatRegister srcDest
,
1320 bool handleNaN
) PER_SHARED_ARCH
;
1322 inline void maxFloat32(FloatRegister other
, FloatRegister srcDest
,
1323 bool handleNaN
) PER_SHARED_ARCH
;
1324 inline void maxDouble(FloatRegister other
, FloatRegister srcDest
,
1325 bool handleNaN
) PER_SHARED_ARCH
;
1327 void minMaxArrayInt32(Register array
, Register result
, Register temp1
,
1328 Register temp2
, Register temp3
, bool isMax
,
1330 void minMaxArrayNumber(Register array
, FloatRegister result
,
1331 FloatRegister floatTemp
, Register temp1
,
1332 Register temp2
, bool isMax
, Label
* fail
);
1334 // Compute |pow(base, power)| and store the result in |dest|. If the result
1335 // exceeds the int32 range, jumps to |onOver|.
1336 // |base| and |power| are preserved, the other input registers are clobbered.
1337 void pow32(Register base
, Register power
, Register dest
, Register temp1
,
1338 Register temp2
, Label
* onOver
);
1340 void sameValueDouble(FloatRegister left
, FloatRegister right
,
1341 FloatRegister temp
, Register dest
);
1343 void branchIfNotRegExpPrototypeOptimizable(Register proto
, Register temp
,
1344 const GlobalObject
* maybeGlobal
,
1346 void branchIfNotRegExpInstanceOptimizable(Register regexp
, Register temp
,
1347 const GlobalObject
* maybeGlobal
,
1350 void loadRegExpLastIndex(Register regexp
, Register string
, Register lastIndex
,
1351 Label
* notFoundZeroLastIndex
);
1353 void loadAndClearRegExpSearcherLastLimit(Register result
, Register scratch
);
1355 void loadParsedRegExpShared(Register regexp
, Register result
,
1358 // ===============================================================
1361 // For shift-by-register there may be platform-specific variations, for
1362 // example, x86 will perform the shift mod 32 but ARM will perform the shift
1365 // For shift-by-immediate the platform assembler may restrict the immediate,
1366 // for example, the ARM assembler requires the count for 32-bit shifts to be
1367 // in the range [0,31].
1369 inline void lshift32(Imm32 shift
, Register srcDest
) PER_SHARED_ARCH
;
1370 inline void rshift32(Imm32 shift
, Register srcDest
) PER_SHARED_ARCH
;
1371 inline void rshift32Arithmetic(Imm32 shift
, Register srcDest
) PER_SHARED_ARCH
;
1373 inline void lshiftPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1374 inline void rshiftPtr(Imm32 imm
, Register dest
) PER_ARCH
;
1375 inline void rshiftPtr(Imm32 imm
, Register src
, Register dest
)
1377 inline void rshiftPtrArithmetic(Imm32 imm
, Register dest
) PER_ARCH
;
1379 inline void lshift64(Imm32 imm
, Register64 dest
) PER_ARCH
;
1380 inline void rshift64(Imm32 imm
, Register64 dest
) PER_ARCH
;
1381 inline void rshift64Arithmetic(Imm32 imm
, Register64 dest
) PER_ARCH
;
1383 // On x86_shared these have the constraint that shift must be in CL.
1384 inline void lshift32(Register shift
, Register srcDest
) PER_SHARED_ARCH
;
1385 inline void rshift32(Register shift
, Register srcDest
) PER_SHARED_ARCH
;
1386 inline void rshift32Arithmetic(Register shift
,
1387 Register srcDest
) PER_SHARED_ARCH
;
1388 inline void lshiftPtr(Register shift
, Register srcDest
) PER_ARCH
;
1389 inline void rshiftPtr(Register shift
, Register srcDest
) PER_ARCH
;
1391 // These variants do not have the above constraint, but may emit some extra
1392 // instructions on x86_shared. They also handle shift >= 32 consistently by
1393 // masking with 0x1F (either explicitly or relying on the hardware to do
1395 inline void flexibleLshift32(Register shift
,
1396 Register srcDest
) PER_SHARED_ARCH
;
1397 inline void flexibleRshift32(Register shift
,
1398 Register srcDest
) PER_SHARED_ARCH
;
1399 inline void flexibleRshift32Arithmetic(Register shift
,
1400 Register srcDest
) PER_SHARED_ARCH
;
1402 inline void lshift64(Register shift
, Register64 srcDest
) PER_ARCH
;
1403 inline void rshift64(Register shift
, Register64 srcDest
) PER_ARCH
;
1404 inline void rshift64Arithmetic(Register shift
, Register64 srcDest
) PER_ARCH
;
1406 // ===============================================================
1407 // Rotation functions
1408 // Note: - on x86 and x64 the count register must be in CL.
1409 // - on x64 the temp register should be InvalidReg.
1411 inline void rotateLeft(Imm32 count
, Register input
,
1412 Register dest
) PER_SHARED_ARCH
;
1413 inline void rotateLeft(Register count
, Register input
,
1414 Register dest
) PER_SHARED_ARCH
;
1415 inline void rotateLeft64(Imm32 count
, Register64 input
, Register64 dest
)
1417 inline void rotateLeft64(Register count
, Register64 input
, Register64 dest
)
1419 inline void rotateLeft64(Imm32 count
, Register64 input
, Register64 dest
,
1420 Register temp
) PER_ARCH
;
1421 inline void rotateLeft64(Register count
, Register64 input
, Register64 dest
,
1422 Register temp
) PER_ARCH
;
1424 inline void rotateRight(Imm32 count
, Register input
,
1425 Register dest
) PER_SHARED_ARCH
;
1426 inline void rotateRight(Register count
, Register input
,
1427 Register dest
) PER_SHARED_ARCH
;
1428 inline void rotateRight64(Imm32 count
, Register64 input
, Register64 dest
)
1430 inline void rotateRight64(Register count
, Register64 input
, Register64 dest
)
1432 inline void rotateRight64(Imm32 count
, Register64 input
, Register64 dest
,
1433 Register temp
) PER_ARCH
;
1434 inline void rotateRight64(Register count
, Register64 input
, Register64 dest
,
1435 Register temp
) PER_ARCH
;
1437 // ===============================================================
1438 // Bit counting functions
1440 // knownNotZero may be true only if the src is known not to be zero.
1441 inline void clz32(Register src
, Register dest
,
1442 bool knownNotZero
) PER_SHARED_ARCH
;
1443 inline void ctz32(Register src
, Register dest
,
1444 bool knownNotZero
) PER_SHARED_ARCH
;
1446 inline void clz64(Register64 src
, Register dest
) PER_ARCH
;
1447 inline void ctz64(Register64 src
, Register dest
) PER_ARCH
;
1449 // On x86_shared, temp may be Invalid only if the chip has the POPCNT
1450 // instruction. On ARM, temp may never be Invalid.
1451 inline void popcnt32(Register src
, Register dest
,
1452 Register temp
) PER_SHARED_ARCH
;
1454 // temp may be invalid only if the chip has the POPCNT instruction.
1455 inline void popcnt64(Register64 src
, Register64 dest
, Register temp
) PER_ARCH
;
1457 // ===============================================================
1458 // Condition functions
1460 inline void cmp8Set(Condition cond
, Address lhs
, Imm32 rhs
,
1461 Register dest
) PER_SHARED_ARCH
;
1463 inline void cmp16Set(Condition cond
, Address lhs
, Imm32 rhs
,
1464 Register dest
) PER_SHARED_ARCH
;
1466 template <typename T1
, typename T2
>
1467 inline void cmp32Set(Condition cond
, T1 lhs
, T2 rhs
, Register dest
)
1468 DEFINED_ON(x86_shared
, arm
, arm64
, mips32
, mips64
, loong64
, riscv64
,
1471 // Only the NotEqual and Equal conditions are allowed.
1472 inline void cmp64Set(Condition cond
, Address lhs
, Imm64 rhs
,
1473 Register dest
) PER_ARCH
;
1475 template <typename T1
, typename T2
>
1476 inline void cmpPtrSet(Condition cond
, T1 lhs
, T2 rhs
, Register dest
) PER_ARCH
;
1478 // ===============================================================
1481 inline void branch8(Condition cond
, const Address
& lhs
, Imm32 rhs
,
1482 Label
* label
) PER_SHARED_ARCH
;
1484 // Compares the byte in |lhs| against |rhs| using a 8-bit comparison on
1485 // x86/x64 or a 32-bit comparison (all other platforms). The caller should
1486 // ensure |rhs| is a zero- resp. sign-extended byte value for cross-platform
1488 inline void branch8(Condition cond
, const BaseIndex
& lhs
, Register rhs
,
1489 Label
* label
) PER_SHARED_ARCH
;
1491 inline void branch16(Condition cond
, const Address
& lhs
, Imm32 rhs
,
1492 Label
* label
) PER_SHARED_ARCH
;
1495 inline void branch32(Condition cond
, Register lhs
, Register rhs
,
1496 L label
) PER_SHARED_ARCH
;
1498 inline void branch32(Condition cond
, Register lhs
, Imm32 rhs
,
1499 L label
) PER_SHARED_ARCH
;
1501 inline void branch32(Condition cond
, Register lhs
, const Address
& rhs
,
1502 Label
* label
) DEFINED_ON(arm64
);
1504 inline void branch32(Condition cond
, const Address
& lhs
, Register rhs
,
1505 Label
* label
) PER_SHARED_ARCH
;
1506 inline void branch32(Condition cond
, const Address
& lhs
, Imm32 rhs
,
1507 Label
* label
) PER_SHARED_ARCH
;
1509 inline void branch32(Condition cond
, const AbsoluteAddress
& lhs
, Register rhs
,
1511 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1512 inline void branch32(Condition cond
, const AbsoluteAddress
& lhs
, Imm32 rhs
,
1514 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1516 inline void branch32(Condition cond
, const BaseIndex
& lhs
, Register rhs
,
1517 Label
* label
) DEFINED_ON(arm
, x86_shared
);
1518 inline void branch32(Condition cond
, const BaseIndex
& lhs
, Imm32 rhs
,
1519 Label
* label
) PER_SHARED_ARCH
;
1521 inline void branch32(Condition cond
, const Operand
& lhs
, Register rhs
,
1522 Label
* label
) DEFINED_ON(x86_shared
);
1523 inline void branch32(Condition cond
, const Operand
& lhs
, Imm32 rhs
,
1524 Label
* label
) DEFINED_ON(x86_shared
);
1526 inline void branch32(Condition cond
, wasm::SymbolicAddress lhs
, Imm32 rhs
,
1528 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1530 // The supported condition are Equal, NotEqual, LessThan(orEqual),
1531 // GreaterThan(orEqual), Below(orEqual) and Above(orEqual). When a fail label
1532 // is not defined it will fall through to next instruction, else jump to the
1534 inline void branch64(Condition cond
, Register64 lhs
, Imm64 val
,
1535 Label
* success
, Label
* fail
= nullptr) PER_ARCH
;
1536 inline void branch64(Condition cond
, Register64 lhs
, Register64 rhs
,
1537 Label
* success
, Label
* fail
= nullptr) PER_ARCH
;
1538 // Only the NotEqual and Equal conditions are allowed for the branch64
1539 // variants with Address as lhs.
1540 inline void branch64(Condition cond
, const Address
& lhs
, Imm64 val
,
1541 Label
* label
) PER_ARCH
;
1542 inline void branch64(Condition cond
, const Address
& lhs
, Register64 rhs
,
1543 Label
* label
) PER_ARCH
;
1545 // Compare the value at |lhs| with the value at |rhs|. The scratch
1546 // register *must not* be the base of |lhs| or |rhs|.
1547 inline void branch64(Condition cond
, const Address
& lhs
, const Address
& rhs
,
1548 Register scratch
, Label
* label
) PER_ARCH
;
1551 inline void branchPtr(Condition cond
, Register lhs
, Register rhs
,
1552 L label
) PER_SHARED_ARCH
;
1553 inline void branchPtr(Condition cond
, Register lhs
, Imm32 rhs
,
1554 Label
* label
) PER_SHARED_ARCH
;
1555 inline void branchPtr(Condition cond
, Register lhs
, ImmPtr rhs
,
1556 Label
* label
) PER_SHARED_ARCH
;
1557 inline void branchPtr(Condition cond
, Register lhs
, ImmGCPtr rhs
,
1558 Label
* label
) PER_SHARED_ARCH
;
1559 inline void branchPtr(Condition cond
, Register lhs
, ImmWord rhs
,
1560 Label
* label
) PER_SHARED_ARCH
;
1563 inline void branchPtr(Condition cond
, const Address
& lhs
, Register rhs
,
1564 L label
) PER_SHARED_ARCH
;
1565 inline void branchPtr(Condition cond
, const Address
& lhs
, ImmPtr rhs
,
1566 Label
* label
) PER_SHARED_ARCH
;
1567 inline void branchPtr(Condition cond
, const Address
& lhs
, ImmGCPtr rhs
,
1568 Label
* label
) PER_SHARED_ARCH
;
1569 inline void branchPtr(Condition cond
, const Address
& lhs
, ImmWord rhs
,
1570 Label
* label
) PER_SHARED_ARCH
;
1572 inline void branchPtr(Condition cond
, const BaseIndex
& lhs
, ImmWord rhs
,
1573 Label
* label
) PER_SHARED_ARCH
;
1574 inline void branchPtr(Condition cond
, const BaseIndex
& lhs
, Register rhs
,
1575 Label
* label
) PER_SHARED_ARCH
;
1577 inline void branchPtr(Condition cond
, const AbsoluteAddress
& lhs
,
1578 Register rhs
, Label
* label
)
1579 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1580 inline void branchPtr(Condition cond
, const AbsoluteAddress
& lhs
, ImmWord rhs
,
1582 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1584 inline void branchPtr(Condition cond
, wasm::SymbolicAddress lhs
, Register rhs
,
1586 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1588 // Given a pointer to a GC Cell, retrieve the StoreBuffer pointer from its
1589 // chunk header, or nullptr if it is in the tenured heap.
1590 void loadStoreBuffer(Register ptr
, Register buffer
) PER_ARCH
;
1592 void branchPtrInNurseryChunk(Condition cond
, Register ptr
, Register temp
,
1594 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1595 void branchPtrInNurseryChunk(Condition cond
, const Address
& address
,
1596 Register temp
, Label
* label
) DEFINED_ON(x86
);
1597 void branchValueIsNurseryCell(Condition cond
, const Address
& address
,
1598 Register temp
, Label
* label
) PER_ARCH
;
1599 void branchValueIsNurseryCell(Condition cond
, ValueOperand value
,
1600 Register temp
, Label
* label
) PER_ARCH
;
1602 // This function compares a Value (lhs) which is having a private pointer
1603 // boxed inside a js::Value, with a raw pointer (rhs).
1604 inline void branchPrivatePtr(Condition cond
, const Address
& lhs
, Register rhs
,
1605 Label
* label
) PER_ARCH
;
1607 inline void branchFloat(DoubleCondition cond
, FloatRegister lhs
,
1608 FloatRegister rhs
, Label
* label
) PER_SHARED_ARCH
;
1610 // Truncate a double/float32 to int32 and when it doesn't fit an int32 it will
1611 // jump to the failure label. This particular variant is allowed to return the
1612 // value module 2**32, which isn't implemented on all architectures. E.g. the
1613 // x64 variants will do this only in the int64_t range.
1614 inline void branchTruncateFloat32MaybeModUint32(FloatRegister src
,
1615 Register dest
, Label
* fail
)
1616 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1617 inline void branchTruncateDoubleMaybeModUint32(FloatRegister src
,
1618 Register dest
, Label
* fail
)
1619 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1621 // Truncate a double/float32 to intptr and when it doesn't fit jump to the
1623 inline void branchTruncateFloat32ToPtr(FloatRegister src
, Register dest
,
1624 Label
* fail
) DEFINED_ON(x86
, x64
);
1625 inline void branchTruncateDoubleToPtr(FloatRegister src
, Register dest
,
1626 Label
* fail
) DEFINED_ON(x86
, x64
);
1628 // Truncate a double/float32 to int32 and when it doesn't fit jump to the
1630 inline void branchTruncateFloat32ToInt32(FloatRegister src
, Register dest
,
1632 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1633 inline void branchTruncateDoubleToInt32(FloatRegister src
, Register dest
,
1634 Label
* fail
) PER_ARCH
;
1636 inline void branchDouble(DoubleCondition cond
, FloatRegister lhs
,
1637 FloatRegister rhs
, Label
* label
) PER_SHARED_ARCH
;
1639 inline void branchDoubleNotInInt64Range(Address src
, Register temp
,
1641 inline void branchDoubleNotInUInt64Range(Address src
, Register temp
,
1643 inline void branchFloat32NotInInt64Range(Address src
, Register temp
,
1645 inline void branchFloat32NotInUInt64Range(Address src
, Register temp
,
1648 template <typename T
>
1649 inline void branchAdd32(Condition cond
, T src
, Register dest
,
1650 Label
* label
) PER_SHARED_ARCH
;
1651 template <typename T
>
1652 inline void branchSub32(Condition cond
, T src
, Register dest
,
1653 Label
* label
) PER_SHARED_ARCH
;
1654 template <typename T
>
1655 inline void branchMul32(Condition cond
, T src
, Register dest
,
1656 Label
* label
) PER_SHARED_ARCH
;
1657 template <typename T
>
1658 inline void branchRshift32(Condition cond
, T src
, Register dest
,
1659 Label
* label
) PER_SHARED_ARCH
;
1661 inline void branchNeg32(Condition cond
, Register reg
,
1662 Label
* label
) PER_SHARED_ARCH
;
1664 inline void branchAdd64(Condition cond
, Imm64 imm
, Register64 dest
,
1665 Label
* label
) DEFINED_ON(x86
, arm
, wasm32
);
1667 template <typename T
>
1668 inline void branchAddPtr(Condition cond
, T src
, Register dest
,
1669 Label
* label
) PER_SHARED_ARCH
;
1671 template <typename T
>
1672 inline void branchSubPtr(Condition cond
, T src
, Register dest
,
1673 Label
* label
) PER_SHARED_ARCH
;
1675 inline void branchMulPtr(Condition cond
, Register src
, Register dest
,
1676 Label
* label
) PER_SHARED_ARCH
;
1678 inline void decBranchPtr(Condition cond
, Register lhs
, Imm32 rhs
,
1679 Label
* label
) PER_SHARED_ARCH
;
1682 inline void branchTest32(Condition cond
, Register lhs
, Register rhs
,
1683 L label
) PER_SHARED_ARCH
;
1685 inline void branchTest32(Condition cond
, Register lhs
, Imm32 rhs
,
1686 L label
) PER_SHARED_ARCH
;
1687 inline void branchTest32(Condition cond
, const Address
& lhs
, Imm32 rhh
,
1688 Label
* label
) PER_SHARED_ARCH
;
1689 inline void branchTest32(Condition cond
, const AbsoluteAddress
& lhs
,
1690 Imm32 rhs
, Label
* label
)
1691 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
1694 inline void branchTestPtr(Condition cond
, Register lhs
, Register rhs
,
1695 L label
) PER_SHARED_ARCH
;
1696 inline void branchTestPtr(Condition cond
, Register lhs
, Imm32 rhs
,
1697 Label
* label
) PER_SHARED_ARCH
;
1698 inline void branchTestPtr(Condition cond
, const Address
& lhs
, Imm32 rhs
,
1699 Label
* label
) PER_SHARED_ARCH
;
1702 inline void branchTest64(Condition cond
, Register64 lhs
, Register64 rhs
,
1703 Register temp
, L label
) PER_ARCH
;
1705 // Branches to |label| if |reg| is false. |reg| should be a C++ bool.
1707 inline void branchIfFalseBool(Register reg
, L label
);
1709 // Branches to |label| if |reg| is true. |reg| should be a C++ bool.
1710 inline void branchIfTrueBool(Register reg
, Label
* label
);
1712 inline void branchIfRope(Register str
, Label
* label
);
1713 inline void branchIfNotRope(Register str
, Label
* label
);
1715 inline void branchLatin1String(Register string
, Label
* label
);
1716 inline void branchTwoByteString(Register string
, Label
* label
);
1718 inline void branchIfBigIntIsNegative(Register bigInt
, Label
* label
);
1719 inline void branchIfBigIntIsNonNegative(Register bigInt
, Label
* label
);
1720 inline void branchIfBigIntIsZero(Register bigInt
, Label
* label
);
1721 inline void branchIfBigIntIsNonZero(Register bigInt
, Label
* label
);
1723 inline void branchTestFunctionFlags(Register fun
, uint32_t flags
,
1724 Condition cond
, Label
* label
);
1726 inline void branchIfNotFunctionIsNonBuiltinCtor(Register fun
,
1730 inline void branchIfFunctionHasNoJitEntry(Register fun
, Label
* label
);
1731 inline void branchIfFunctionHasJitEntry(Register fun
, Label
* label
);
1733 inline void branchIfScriptHasJitScript(Register script
, Label
* label
);
1734 inline void branchIfScriptHasNoJitScript(Register script
, Label
* label
);
1735 inline void loadJitScript(Register script
, Register dest
);
1737 // Loads the function's argument count.
1738 inline void loadFunctionArgCount(Register func
, Register output
);
1740 // Loads the function length. This handles interpreted, native, and bound
1741 // functions. The caller is responsible for checking that INTERPRETED_LAZY and
1742 // RESOLVED_LENGTH flags are not set.
1743 void loadFunctionLength(Register func
, Register funFlagsAndArgCount
,
1744 Register output
, Label
* slowPath
);
1746 // Loads the function name. This handles interpreted, native, and bound
1748 void loadFunctionName(Register func
, Register output
, ImmGCPtr emptyString
,
1751 void assertFunctionIsExtended(Register func
);
1753 inline void branchFunctionKind(Condition cond
,
1754 FunctionFlags::FunctionKind kind
, Register fun
,
1755 Register scratch
, Label
* label
);
1757 inline void branchIfObjectEmulatesUndefined(Register objReg
, Register scratch
,
1758 Label
* slowCheck
, Label
* label
);
1760 // For all methods below: spectreRegToZero is a register that will be zeroed
1761 // on speculatively executed code paths (when the branch should be taken but
1762 // branch prediction speculates it isn't). Usually this will be the object
1763 // register but the caller may pass a different register.
1765 inline void branchTestObjClass(Condition cond
, Register obj
,
1766 const JSClass
* clasp
, Register scratch
,
1767 Register spectreRegToZero
, Label
* label
);
1768 inline void branchTestObjClassNoSpectreMitigations(Condition cond
,
1770 const JSClass
* clasp
,
1774 inline void branchTestObjClass(Condition cond
, Register obj
,
1775 const Address
& clasp
, Register scratch
,
1776 Register spectreRegToZero
, Label
* label
);
1777 inline void branchTestObjClassNoSpectreMitigations(Condition cond
,
1779 const Address
& clasp
,
1783 inline void branchTestObjClass(Condition cond
, Register obj
, Register clasp
,
1784 Register scratch
, Register spectreRegToZero
,
1788 inline void branchTestClass(Condition cond
, Register clasp
,
1789 std::pair
<const JSClass
*, const JSClass
*> classes
,
1793 inline void branchTestObjClass(
1794 Condition cond
, Register obj
,
1795 std::pair
<const JSClass
*, const JSClass
*> classes
, Register scratch
,
1796 Register spectreRegToZero
, Label
* label
);
1797 inline void branchTestObjClassNoSpectreMitigations(
1798 Condition cond
, Register obj
,
1799 std::pair
<const JSClass
*, const JSClass
*> classes
, Register scratch
,
1802 inline void branchTestObjShape(Condition cond
, Register obj
,
1803 const Shape
* shape
, Register scratch
,
1804 Register spectreRegToZero
, Label
* label
);
1805 inline void branchTestObjShapeNoSpectreMitigations(Condition cond
,
1810 void branchTestObjShapeList(Condition cond
, Register obj
,
1811 Register shapeElements
, Register shapeScratch
,
1812 Register endScratch
, Register spectreScratch
,
1815 inline void branchTestClassIsFunction(Condition cond
, Register clasp
,
1817 inline void branchTestObjIsFunction(Condition cond
, Register obj
,
1819 Register spectreRegToZero
, Label
* label
);
1820 inline void branchTestObjIsFunctionNoSpectreMitigations(Condition cond
,
1825 inline void branchTestObjShape(Condition cond
, Register obj
, Register shape
,
1826 Register scratch
, Register spectreRegToZero
,
1828 inline void branchTestObjShapeNoSpectreMitigations(Condition cond
,
1833 // TODO: audit/fix callers to be Spectre safe.
1834 inline void branchTestObjShapeUnsafe(Condition cond
, Register obj
,
1835 Register shape
, Label
* label
);
1837 void branchTestObjCompartment(Condition cond
, Register obj
,
1838 const Address
& compartment
, Register scratch
,
1840 void branchTestObjCompartment(Condition cond
, Register obj
,
1841 const JS::Compartment
* compartment
,
1842 Register scratch
, Label
* label
);
1844 void branchIfNonNativeObj(Register obj
, Register scratch
, Label
* label
);
1846 void branchIfObjectNotExtensible(Register obj
, Register scratch
,
1849 void branchTestObjectNeedsProxyResultValidation(Condition condition
,
1854 inline void branchTestClassIsProxy(bool proxy
, Register clasp
, Label
* label
);
1856 inline void branchTestObjectIsProxy(bool proxy
, Register object
,
1857 Register scratch
, Label
* label
);
1859 inline void branchTestProxyHandlerFamily(Condition cond
, Register proxy
,
1861 const void* handlerp
, Label
* label
);
1863 inline void branchTestNeedsIncrementalBarrier(Condition cond
, Label
* label
);
1864 inline void branchTestNeedsIncrementalBarrierAnyZone(Condition cond
,
1868 // Perform a type-test on a tag of a Value (32bits boxing), or the tagged
1869 // value (64bits boxing).
1870 inline void branchTestUndefined(Condition cond
, Register tag
,
1871 Label
* label
) PER_SHARED_ARCH
;
1872 inline void branchTestInt32(Condition cond
, Register tag
,
1873 Label
* label
) PER_SHARED_ARCH
;
1874 inline void branchTestDouble(Condition cond
, Register tag
, Label
* label
)
1875 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1877 inline void branchTestNumber(Condition cond
, Register tag
,
1878 Label
* label
) PER_SHARED_ARCH
;
1879 inline void branchTestBoolean(Condition cond
, Register tag
,
1880 Label
* label
) PER_SHARED_ARCH
;
1881 inline void branchTestString(Condition cond
, Register tag
,
1882 Label
* label
) PER_SHARED_ARCH
;
1883 inline void branchTestSymbol(Condition cond
, Register tag
,
1884 Label
* label
) PER_SHARED_ARCH
;
1885 inline void branchTestBigInt(Condition cond
, Register tag
,
1886 Label
* label
) PER_SHARED_ARCH
;
1887 inline void branchTestNull(Condition cond
, Register tag
,
1888 Label
* label
) PER_SHARED_ARCH
;
1889 inline void branchTestObject(Condition cond
, Register tag
,
1890 Label
* label
) PER_SHARED_ARCH
;
1891 inline void branchTestPrimitive(Condition cond
, Register tag
,
1892 Label
* label
) PER_SHARED_ARCH
;
1893 inline void branchTestMagic(Condition cond
, Register tag
,
1894 Label
* label
) PER_SHARED_ARCH
;
1895 void branchTestType(Condition cond
, Register tag
, JSValueType type
,
1898 // Perform a type-test on a Value, addressed by Address or BaseIndex, or
1899 // loaded into ValueOperand.
1900 // BaseIndex and ValueOperand variants clobber the ScratchReg on x64.
1901 // All Variants clobber the ScratchReg on arm64.
1902 inline void branchTestUndefined(Condition cond
, const Address
& address
,
1903 Label
* label
) PER_SHARED_ARCH
;
1904 inline void branchTestUndefined(Condition cond
, const BaseIndex
& address
,
1905 Label
* label
) PER_SHARED_ARCH
;
1906 inline void branchTestUndefined(Condition cond
, const ValueOperand
& value
,
1908 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1911 inline void branchTestInt32(Condition cond
, const Address
& address
,
1912 Label
* label
) PER_SHARED_ARCH
;
1913 inline void branchTestInt32(Condition cond
, const BaseIndex
& address
,
1914 Label
* label
) PER_SHARED_ARCH
;
1915 inline void branchTestInt32(Condition cond
, const ValueOperand
& value
,
1917 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1920 inline void branchTestDouble(Condition cond
, const Address
& address
,
1921 Label
* label
) PER_SHARED_ARCH
;
1922 inline void branchTestDouble(Condition cond
, const BaseIndex
& address
,
1923 Label
* label
) PER_SHARED_ARCH
;
1924 inline void branchTestDouble(Condition cond
, const ValueOperand
& value
,
1926 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1929 inline void branchTestNumber(Condition cond
, const ValueOperand
& value
,
1931 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1934 inline void branchTestBoolean(Condition cond
, const Address
& address
,
1935 Label
* label
) PER_SHARED_ARCH
;
1936 inline void branchTestBoolean(Condition cond
, const BaseIndex
& address
,
1937 Label
* label
) PER_SHARED_ARCH
;
1938 inline void branchTestBoolean(Condition cond
, const ValueOperand
& value
,
1940 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1943 inline void branchTestString(Condition cond
, const Address
& address
,
1944 Label
* label
) PER_SHARED_ARCH
;
1945 inline void branchTestString(Condition cond
, const BaseIndex
& address
,
1946 Label
* label
) PER_SHARED_ARCH
;
1947 inline void branchTestString(Condition cond
, const ValueOperand
& value
,
1949 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1952 inline void branchTestSymbol(Condition cond
, const Address
& address
,
1953 Label
* label
) PER_SHARED_ARCH
;
1954 inline void branchTestSymbol(Condition cond
, const BaseIndex
& address
,
1955 Label
* label
) PER_SHARED_ARCH
;
1956 inline void branchTestSymbol(Condition cond
, const ValueOperand
& value
,
1958 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1961 inline void branchTestBigInt(Condition cond
, const Address
& address
,
1962 Label
* label
) PER_SHARED_ARCH
;
1963 inline void branchTestBigInt(Condition cond
, const BaseIndex
& address
,
1964 Label
* label
) PER_SHARED_ARCH
;
1965 inline void branchTestBigInt(Condition cond
, const ValueOperand
& value
,
1967 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1970 inline void branchTestNull(Condition cond
, const Address
& address
,
1971 Label
* label
) PER_SHARED_ARCH
;
1972 inline void branchTestNull(Condition cond
, const BaseIndex
& address
,
1973 Label
* label
) PER_SHARED_ARCH
;
1974 inline void branchTestNull(Condition cond
, const ValueOperand
& value
,
1976 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1979 // Clobbers the ScratchReg on x64.
1980 inline void branchTestObject(Condition cond
, const Address
& address
,
1981 Label
* label
) PER_SHARED_ARCH
;
1982 inline void branchTestObject(Condition cond
, const BaseIndex
& address
,
1983 Label
* label
) PER_SHARED_ARCH
;
1984 inline void branchTestObject(Condition cond
, const ValueOperand
& value
,
1986 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
1989 inline void branchTestGCThing(Condition cond
, const Address
& address
,
1990 Label
* label
) PER_SHARED_ARCH
;
1991 inline void branchTestGCThing(Condition cond
, const BaseIndex
& address
,
1992 Label
* label
) PER_SHARED_ARCH
;
1993 inline void branchTestGCThing(Condition cond
, const ValueOperand
& value
,
1994 Label
* label
) PER_SHARED_ARCH
;
1996 inline void branchTestPrimitive(Condition cond
, const ValueOperand
& value
,
1998 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
2001 inline void branchTestMagic(Condition cond
, const Address
& address
,
2002 Label
* label
) PER_SHARED_ARCH
;
2003 inline void branchTestMagic(Condition cond
, const BaseIndex
& address
,
2004 Label
* label
) PER_SHARED_ARCH
;
2006 inline void branchTestMagic(Condition cond
, const ValueOperand
& value
,
2008 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
2011 inline void branchTestMagic(Condition cond
, const Address
& valaddr
,
2012 JSWhyMagic why
, Label
* label
) PER_ARCH
;
2014 inline void branchTestMagicValue(Condition cond
, const ValueOperand
& val
,
2015 JSWhyMagic why
, Label
* label
);
2017 void branchTestValue(Condition cond
, const ValueOperand
& lhs
,
2018 const Value
& rhs
, Label
* label
) PER_ARCH
;
2020 inline void branchTestValue(Condition cond
, const BaseIndex
& lhs
,
2021 const ValueOperand
& rhs
, Label
* label
) PER_ARCH
;
2023 // Checks if given Value is evaluated to true or false in a condition.
2024 // The type of the value should match the type of the method.
2025 inline void branchTestInt32Truthy(bool truthy
, const ValueOperand
& value
,
2027 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, x86_shared
,
2029 inline void branchTestDoubleTruthy(bool truthy
, FloatRegister reg
,
2030 Label
* label
) PER_SHARED_ARCH
;
2031 inline void branchTestBooleanTruthy(bool truthy
, const ValueOperand
& value
,
2032 Label
* label
) PER_ARCH
;
2033 inline void branchTestStringTruthy(bool truthy
, const ValueOperand
& value
,
2035 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
2037 inline void branchTestBigIntTruthy(bool truthy
, const ValueOperand
& value
,
2039 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, wasm32
,
2042 // Create an unconditional branch to the address given as argument.
2043 inline void branchToComputedAddress(const BaseIndex
& address
) PER_ARCH
;
2046 template <typename T
, typename S
, typename L
>
2047 inline void branchPtrImpl(Condition cond
, const T
& lhs
, const S
& rhs
, L label
)
2048 DEFINED_ON(x86_shared
);
2050 void branchPtrInNurseryChunkImpl(Condition cond
, Register ptr
, Label
* label
)
2052 template <typename T
>
2053 void branchValueIsNurseryCellImpl(Condition cond
, const T
& value
,
2054 Register temp
, Label
* label
)
2055 DEFINED_ON(arm64
, x64
, mips64
, loong64
, riscv64
);
2057 template <typename T
>
2058 inline void branchTestUndefinedImpl(Condition cond
, const T
& t
, Label
* label
)
2059 DEFINED_ON(arm
, arm64
, x86_shared
);
2060 template <typename T
>
2061 inline void branchTestInt32Impl(Condition cond
, const T
& t
, Label
* label
)
2062 DEFINED_ON(arm
, arm64
, x86_shared
);
2063 template <typename T
>
2064 inline void branchTestDoubleImpl(Condition cond
, const T
& t
, Label
* label
)
2065 DEFINED_ON(arm
, arm64
, x86_shared
);
2066 template <typename T
>
2067 inline void branchTestNumberImpl(Condition cond
, const T
& t
, Label
* label
)
2068 DEFINED_ON(arm
, arm64
, x86_shared
);
2069 template <typename T
>
2070 inline void branchTestBooleanImpl(Condition cond
, const T
& t
, Label
* label
)
2071 DEFINED_ON(arm
, arm64
, x86_shared
);
2072 template <typename T
>
2073 inline void branchTestStringImpl(Condition cond
, const T
& t
, Label
* label
)
2074 DEFINED_ON(arm
, arm64
, x86_shared
);
2075 template <typename T
>
2076 inline void branchTestSymbolImpl(Condition cond
, const T
& t
, Label
* label
)
2077 DEFINED_ON(arm
, arm64
, x86_shared
);
2078 template <typename T
>
2079 inline void branchTestBigIntImpl(Condition cond
, const T
& t
, Label
* label
)
2080 DEFINED_ON(arm
, arm64
, x86_shared
);
2081 template <typename T
>
2082 inline void branchTestNullImpl(Condition cond
, const T
& t
, Label
* label
)
2083 DEFINED_ON(arm
, arm64
, x86_shared
);
2084 template <typename T
>
2085 inline void branchTestObjectImpl(Condition cond
, const T
& t
, Label
* label
)
2086 DEFINED_ON(arm
, arm64
, x86_shared
);
2087 template <typename T
>
2088 inline void branchTestGCThingImpl(Condition cond
, const T
& t
,
2089 Label
* label
) PER_SHARED_ARCH
;
2090 template <typename T
>
2091 inline void branchTestPrimitiveImpl(Condition cond
, const T
& t
, Label
* label
)
2092 DEFINED_ON(arm
, arm64
, x86_shared
);
2093 template <typename T
, class L
>
2094 inline void branchTestMagicImpl(Condition cond
, const T
& t
, L label
)
2095 DEFINED_ON(arm
, arm64
, x86_shared
);
2098 template <typename T
>
2099 inline void testNumberSet(Condition cond
, const T
& src
,
2100 Register dest
) PER_SHARED_ARCH
;
2101 template <typename T
>
2102 inline void testBooleanSet(Condition cond
, const T
& src
,
2103 Register dest
) PER_SHARED_ARCH
;
2104 template <typename T
>
2105 inline void testStringSet(Condition cond
, const T
& src
,
2106 Register dest
) PER_SHARED_ARCH
;
2107 template <typename T
>
2108 inline void testSymbolSet(Condition cond
, const T
& src
,
2109 Register dest
) PER_SHARED_ARCH
;
2110 template <typename T
>
2111 inline void testBigIntSet(Condition cond
, const T
& src
,
2112 Register dest
) PER_SHARED_ARCH
;
2115 // The fallibleUnbox* methods below combine a Value type check with an unbox.
2116 // Especially on 64-bit platforms this can be implemented more efficiently
2117 // than a separate branch + unbox.
2119 // |src| and |dest| can be the same register, but |dest| may hold garbage on
2121 inline void fallibleUnboxPtr(const ValueOperand
& src
, Register dest
,
2122 JSValueType type
, Label
* fail
) PER_ARCH
;
2123 inline void fallibleUnboxPtr(const Address
& src
, Register dest
,
2124 JSValueType type
, Label
* fail
) PER_ARCH
;
2125 inline void fallibleUnboxPtr(const BaseIndex
& src
, Register dest
,
2126 JSValueType type
, Label
* fail
) PER_ARCH
;
2127 template <typename T
>
2128 inline void fallibleUnboxInt32(const T
& src
, Register dest
, Label
* fail
);
2129 template <typename T
>
2130 inline void fallibleUnboxBoolean(const T
& src
, Register dest
, Label
* fail
);
2131 template <typename T
>
2132 inline void fallibleUnboxObject(const T
& src
, Register dest
, Label
* fail
);
2133 template <typename T
>
2134 inline void fallibleUnboxString(const T
& src
, Register dest
, Label
* fail
);
2135 template <typename T
>
2136 inline void fallibleUnboxSymbol(const T
& src
, Register dest
, Label
* fail
);
2137 template <typename T
>
2138 inline void fallibleUnboxBigInt(const T
& src
, Register dest
, Label
* fail
);
2140 inline void cmp32Move32(Condition cond
, Register lhs
, Imm32 rhs
, Register src
,
2142 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86_shared
);
2144 inline void cmp32Move32(Condition cond
, Register lhs
, Register rhs
,
2145 Register src
, Register dest
)
2146 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86_shared
);
2148 inline void cmp32Move32(Condition cond
, Register lhs
, const Address
& rhs
,
2149 Register src
, Register dest
)
2150 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86_shared
);
2152 inline void cmpPtrMovePtr(Condition cond
, Register lhs
, Register rhs
,
2153 Register src
, Register dest
) PER_ARCH
;
2155 inline void cmpPtrMovePtr(Condition cond
, Register lhs
, const Address
& rhs
,
2156 Register src
, Register dest
) PER_ARCH
;
2158 inline void cmp32Load32(Condition cond
, Register lhs
, const Address
& rhs
,
2159 const Address
& src
, Register dest
)
2160 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, mips_shared
, x86_shared
);
2162 inline void cmp32Load32(Condition cond
, Register lhs
, Register rhs
,
2163 const Address
& src
, Register dest
)
2164 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, mips_shared
, x86_shared
);
2166 inline void cmp32Load32(Condition cond
, Register lhs
, Imm32 rhs
,
2167 const Address
& src
, Register dest
)
2168 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86_shared
);
2170 inline void cmp32LoadPtr(Condition cond
, const Address
& lhs
, Imm32 rhs
,
2171 const Address
& src
, Register dest
)
2172 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86
, x64
);
2174 inline void cmp32MovePtr(Condition cond
, Register lhs
, Imm32 rhs
,
2175 Register src
, Register dest
)
2176 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86
, x64
);
2178 inline void test32LoadPtr(Condition cond
, const Address
& addr
, Imm32 mask
,
2179 const Address
& src
, Register dest
)
2180 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86
, x64
);
2182 inline void test32MovePtr(Condition cond
, const Address
& addr
, Imm32 mask
,
2183 Register src
, Register dest
)
2184 DEFINED_ON(arm
, arm64
, loong64
, riscv64
, wasm32
, mips_shared
, x86
, x64
);
2186 // Conditional move for Spectre mitigations.
2187 inline void spectreMovePtr(Condition cond
, Register src
, Register dest
)
2188 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
2190 // Zeroes dest if the condition is true.
2191 inline void spectreZeroRegister(Condition cond
, Register scratch
,
2193 DEFINED_ON(arm
, arm64
, mips_shared
, x86_shared
, loong64
, riscv64
, wasm32
);
2195 // Performs a bounds check and zeroes the index register if out-of-bounds
2196 // (to mitigate Spectre).
2198 inline void spectreBoundsCheck32(Register index
, const Operand
& length
,
2199 Register maybeScratch
, Label
* failure
)
2203 inline void spectreBoundsCheck32(Register index
, Register length
,
2204 Register maybeScratch
, Label
* failure
)
2205 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
2206 inline void spectreBoundsCheck32(Register index
, const Address
& length
,
2207 Register maybeScratch
, Label
* failure
)
2208 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
2210 inline void spectreBoundsCheckPtr(Register index
, Register length
,
2211 Register maybeScratch
, Label
* failure
)
2212 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
2213 inline void spectreBoundsCheckPtr(Register index
, const Address
& length
,
2214 Register maybeScratch
, Label
* failure
)
2215 DEFINED_ON(arm
, arm64
, mips_shared
, x86
, x64
, loong64
, riscv64
, wasm32
);
2217 // ========================================================================
2218 // Canonicalization primitives.
2219 inline void canonicalizeDouble(FloatRegister reg
);
2220 inline void canonicalizeDoubleIfDeterministic(FloatRegister reg
);
2222 inline void canonicalizeFloat(FloatRegister reg
);
2223 inline void canonicalizeFloatIfDeterministic(FloatRegister reg
);
2226 // ========================================================================
2227 // Memory access primitives.
2228 inline FaultingCodeOffset
storeUncanonicalizedDouble(FloatRegister src
,
2229 const Address
& dest
)
2230 DEFINED_ON(x86_shared
, arm
, arm64
, mips32
, mips64
, loong64
, riscv64
,
2232 inline FaultingCodeOffset
storeUncanonicalizedDouble(FloatRegister src
,
2233 const BaseIndex
& dest
)
2234 DEFINED_ON(x86_shared
, arm
, arm64
, mips32
, mips64
, loong64
, riscv64
,
2236 inline FaultingCodeOffset
storeUncanonicalizedDouble(FloatRegister src
,
2237 const Operand
& dest
)
2238 DEFINED_ON(x86_shared
);
2241 inline FaultingCodeOffset
storeDouble(FloatRegister src
, const T
& dest
);
2244 inline void boxDouble(FloatRegister src
, const T
& dest
);
2246 using MacroAssemblerSpecific::boxDouble
;
2248 inline FaultingCodeOffset
storeUncanonicalizedFloat32(FloatRegister src
,
2249 const Address
& dest
)
2250 DEFINED_ON(x86_shared
, arm
, arm64
, mips32
, mips64
, loong64
, riscv64
,
2252 inline FaultingCodeOffset
storeUncanonicalizedFloat32(FloatRegister src
,
2253 const BaseIndex
& dest
)
2254 DEFINED_ON(x86_shared
, arm
, arm64
, mips32
, mips64
, loong64
, riscv64
,
2256 inline FaultingCodeOffset
storeUncanonicalizedFloat32(FloatRegister src
,
2257 const Operand
& dest
)
2258 DEFINED_ON(x86_shared
);
2261 inline FaultingCodeOffset
storeFloat32(FloatRegister src
, const T
& dest
);
2263 template <typename T
>
2264 void storeUnboxedValue(const ConstantOrRegister
& value
, MIRType valueType
,
2265 const T
& dest
) PER_ARCH
;
2267 inline void memoryBarrier(MemoryBarrierBits barrier
) PER_SHARED_ARCH
;
2270 // ========================================================================
2273 // Naming is "operationSimd128" when operate on the whole vector, otherwise
2274 // it's "operation<Type><Size>x<Lanes>".
2276 // For microarchitectural reasons we can in principle get a performance win by
2277 // using int or float specific instructions in the operationSimd128 case when
2278 // we know that subsequent operations on the result are int or float oriented.
2279 // In practice, we don't care about that yet.
2281 // The order of operations here follows those in the SIMD overview document,
2282 // https://github.com/WebAssembly/simd/blob/master/proposals/simd/SIMD.md.
2284 // Since we must target Intel SSE indefinitely and SSE is one-address or
2285 // two-address, the x86 porting interfaces are nearly all one-address or
2286 // two-address. Likewise there are two-address ARM64 interfaces to support
2287 // the baseline compiler. But there are also three-address ARM64 interfaces
2288 // as the ARM64 Ion back-end can use those. In the future, they may support
2289 // AVX2 or similar for x86.
2291 // Conventions for argument order and naming and semantics:
2292 // - Condition codes come first.
2293 // - Other immediates (masks, shift counts) come next.
2294 // - Operands come next:
2295 // - For a binary two-address operator where the left-hand-side has the
2296 // same type as the result, one register parameter is normally named
2297 // `lhsDest` and is both the left-hand side and destination; the other
2298 // parameter is named `rhs` and is the right-hand side. `rhs` comes
2299 // first, `lhsDest` second. `rhs` and `lhsDest` may be the same register
2300 // (if rhs is a register).
2301 // - For a binary three-address operator the order is `lhs`, `rhs`, `dest`,
2302 // and generally these registers may be the same.
2303 // - For a unary operator, the input is named `src` and the output is named
2304 // `dest`. `src` comes first, `dest` second. `src` and `dest` may be
2305 // the same register (if `src` is a register).
2306 // - Temp registers follow operands and are named `temp` if there's only one,
2307 // otherwise `temp1`, `temp2`, etc regardless of type. GPR temps precede
2308 // FPU temps. If there are several temps then they must be distinct
2309 // registers, and they must be distinct from the operand registers unless
2314 inline void moveSimd128(FloatRegister src
, FloatRegister dest
)
2315 DEFINED_ON(x86_shared
, arm64
);
2319 inline void loadConstantSimd128(const SimdConstant
& v
, FloatRegister dest
)
2320 DEFINED_ON(x86_shared
, arm64
);
2324 inline void splatX16(Register src
, FloatRegister dest
)
2325 DEFINED_ON(x86_shared
, arm64
);
2327 inline void splatX16(uint32_t srcLane
, FloatRegister src
, FloatRegister dest
)
2330 inline void splatX8(Register src
, FloatRegister dest
)
2331 DEFINED_ON(x86_shared
, arm64
);
2333 inline void splatX8(uint32_t srcLane
, FloatRegister src
, FloatRegister dest
)
2336 inline void splatX4(Register src
, FloatRegister dest
)
2337 DEFINED_ON(x86_shared
, arm64
);
2339 inline void splatX4(FloatRegister src
, FloatRegister dest
)
2340 DEFINED_ON(x86_shared
, arm64
);
2342 inline void splatX2(Register64 src
, FloatRegister dest
)
2343 DEFINED_ON(x86
, x64
, arm64
);
2345 inline void splatX2(FloatRegister src
, FloatRegister dest
)
2346 DEFINED_ON(x86_shared
, arm64
);
2348 // Extract lane as scalar. Float extraction does not canonicalize the value.
2350 inline void extractLaneInt8x16(uint32_t lane
, FloatRegister src
,
2351 Register dest
) DEFINED_ON(x86_shared
, arm64
);
2353 inline void unsignedExtractLaneInt8x16(uint32_t lane
, FloatRegister src
,
2355 DEFINED_ON(x86_shared
, arm64
);
2357 inline void extractLaneInt16x8(uint32_t lane
, FloatRegister src
,
2358 Register dest
) DEFINED_ON(x86_shared
, arm64
);
2360 inline void unsignedExtractLaneInt16x8(uint32_t lane
, FloatRegister src
,
2362 DEFINED_ON(x86_shared
, arm64
);
2364 inline void extractLaneInt32x4(uint32_t lane
, FloatRegister src
,
2365 Register dest
) DEFINED_ON(x86_shared
, arm64
);
2367 inline void extractLaneInt64x2(uint32_t lane
, FloatRegister src
,
2368 Register64 dest
) DEFINED_ON(x86
, x64
, arm64
);
2370 inline void extractLaneFloat32x4(uint32_t lane
, FloatRegister src
,
2372 DEFINED_ON(x86_shared
, arm64
);
2374 inline void extractLaneFloat64x2(uint32_t lane
, FloatRegister src
,
2376 DEFINED_ON(x86_shared
, arm64
);
2378 // Replace lane value
2380 inline void replaceLaneInt8x16(unsigned lane
, FloatRegister lhs
, Register rhs
,
2381 FloatRegister dest
) DEFINED_ON(x86_shared
);
2383 inline void replaceLaneInt8x16(unsigned lane
, Register rhs
,
2384 FloatRegister lhsDest
)
2385 DEFINED_ON(x86_shared
, arm64
);
2387 inline void replaceLaneInt16x8(unsigned lane
, FloatRegister lhs
, Register rhs
,
2388 FloatRegister dest
) DEFINED_ON(x86_shared
);
2390 inline void replaceLaneInt16x8(unsigned lane
, Register rhs
,
2391 FloatRegister lhsDest
)
2392 DEFINED_ON(x86_shared
, arm64
);
2394 inline void replaceLaneInt32x4(unsigned lane
, FloatRegister lhs
, Register rhs
,
2395 FloatRegister dest
) DEFINED_ON(x86_shared
);
2397 inline void replaceLaneInt32x4(unsigned lane
, Register rhs
,
2398 FloatRegister lhsDest
)
2399 DEFINED_ON(x86_shared
, arm64
);
2401 inline void replaceLaneInt64x2(unsigned lane
, FloatRegister lhs
,
2402 Register64 rhs
, FloatRegister dest
)
2403 DEFINED_ON(x86
, x64
);
2405 inline void replaceLaneInt64x2(unsigned lane
, Register64 rhs
,
2406 FloatRegister lhsDest
)
2407 DEFINED_ON(x86
, x64
, arm64
);
2409 inline void replaceLaneFloat32x4(unsigned lane
, FloatRegister lhs
,
2410 FloatRegister rhs
, FloatRegister dest
)
2411 DEFINED_ON(x86_shared
);
2413 inline void replaceLaneFloat32x4(unsigned lane
, FloatRegister rhs
,
2414 FloatRegister lhsDest
)
2415 DEFINED_ON(x86_shared
, arm64
);
2417 inline void replaceLaneFloat64x2(unsigned lane
, FloatRegister lhs
,
2418 FloatRegister rhs
, FloatRegister dest
)
2419 DEFINED_ON(x86_shared
);
2421 inline void replaceLaneFloat64x2(unsigned lane
, FloatRegister rhs
,
2422 FloatRegister lhsDest
)
2423 DEFINED_ON(x86_shared
, arm64
);
2425 // Shuffle - blend and permute with immediate indices, and its many
2426 // specializations. Lane values other than those mentioned are illegal.
2428 // lane values 0..31
2429 inline void shuffleInt8x16(const uint8_t lanes
[16], FloatRegister rhs
,
2430 FloatRegister lhsDest
)
2431 DEFINED_ON(x86_shared
, arm64
);
2433 inline void shuffleInt8x16(const uint8_t lanes
[16], FloatRegister lhs
,
2434 FloatRegister rhs
, FloatRegister dest
)
2435 DEFINED_ON(x86_shared
, arm64
);
2437 // Lane values must be 0 (select from lhs) or FF (select from rhs).
2438 // The behavior is undefined for lane values that are neither 0 nor FF.
2439 // on x86_shared: it is required that lhs == dest.
2440 inline void blendInt8x16(const uint8_t lanes
[16], FloatRegister lhs
,
2441 FloatRegister rhs
, FloatRegister dest
,
2442 FloatRegister temp
) DEFINED_ON(x86_shared
);
2444 // Lane values must be 0 (select from lhs) or FF (select from rhs).
2445 // The behavior is undefined for lane values that are neither 0 nor FF.
2446 inline void blendInt8x16(const uint8_t lanes
[16], FloatRegister lhs
,
2447 FloatRegister rhs
, FloatRegister dest
)
2450 // Lane values must be 0 (select from lhs) or FFFF (select from rhs).
2451 // The behavior is undefined for lane values that are neither 0 nor FFFF.
2452 // on x86_shared: it is required that lhs == dest.
2453 inline void blendInt16x8(const uint16_t lanes
[8], FloatRegister lhs
,
2454 FloatRegister rhs
, FloatRegister dest
)
2455 DEFINED_ON(x86_shared
, arm64
);
2457 // Mask lane values must be ~0 or 0. The former selects from lhs and the
2459 // The implementation works effectively for I8x16, I16x8, I32x4, and I64x2.
2460 inline void laneSelectSimd128(FloatRegister mask
, FloatRegister lhs
,
2461 FloatRegister rhs
, FloatRegister dest
)
2462 DEFINED_ON(x86_shared
, arm64
);
2464 inline void interleaveHighInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2466 DEFINED_ON(x86_shared
, arm64
);
2468 inline void interleaveHighInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2470 DEFINED_ON(x86_shared
, arm64
);
2472 inline void interleaveHighInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2474 DEFINED_ON(x86_shared
, arm64
);
2476 inline void interleaveHighInt64x2(FloatRegister lhs
, FloatRegister rhs
,
2478 DEFINED_ON(x86_shared
, arm64
);
2480 inline void interleaveLowInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2482 DEFINED_ON(x86_shared
, arm64
);
2484 inline void interleaveLowInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2486 DEFINED_ON(x86_shared
, arm64
);
2488 inline void interleaveLowInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2490 DEFINED_ON(x86_shared
, arm64
);
2492 inline void interleaveLowInt64x2(FloatRegister lhs
, FloatRegister rhs
,
2494 DEFINED_ON(x86_shared
, arm64
);
2496 // Permute - permute with immediate indices.
2498 // lane values 0..15
2499 inline void permuteInt8x16(const uint8_t lanes
[16], FloatRegister src
,
2500 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2503 inline void permuteInt16x8(const uint16_t lanes
[8], FloatRegister src
,
2504 FloatRegister dest
) DEFINED_ON(arm64
);
2506 // lane values 0..3 [sic].
2507 inline void permuteHighInt16x8(const uint16_t lanes
[4], FloatRegister src
,
2508 FloatRegister dest
) DEFINED_ON(x86_shared
);
2510 // lane values 0..3.
2511 inline void permuteLowInt16x8(const uint16_t lanes
[4], FloatRegister src
,
2512 FloatRegister dest
) DEFINED_ON(x86_shared
);
2515 inline void permuteInt32x4(const uint32_t lanes
[4], FloatRegister src
,
2516 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2518 // Funnel shift by immediate count:
2519 // low_16_bytes_of((lhs ++ rhs) >> shift*8), shift must be < 16
2520 inline void concatAndRightShiftSimd128(FloatRegister lhs
, FloatRegister rhs
,
2521 FloatRegister dest
, uint32_t shift
)
2522 DEFINED_ON(x86_shared
, arm64
);
2524 // Rotate right by immediate count:
2525 // low_16_bytes_of((src ++ src) >> shift*8), shift must be < 16
2526 inline void rotateRightSimd128(FloatRegister src
, FloatRegister dest
,
2527 uint32_t shift
) DEFINED_ON(arm64
);
2529 // Shift bytes with immediate count, shifting in zeroes. Shift count 0..15.
2531 inline void leftShiftSimd128(Imm32 count
, FloatRegister src
,
2533 DEFINED_ON(x86_shared
, arm64
);
2535 inline void rightShiftSimd128(Imm32 count
, FloatRegister src
,
2537 DEFINED_ON(x86_shared
, arm64
);
2539 // Zero extend int values.
2541 inline void zeroExtend8x16To16x8(FloatRegister src
, FloatRegister dest
)
2542 DEFINED_ON(x86_shared
, arm64
);
2543 inline void zeroExtend8x16To32x4(FloatRegister src
, FloatRegister dest
)
2544 DEFINED_ON(x86_shared
, arm64
);
2545 inline void zeroExtend8x16To64x2(FloatRegister src
, FloatRegister dest
)
2546 DEFINED_ON(x86_shared
, arm64
);
2547 inline void zeroExtend16x8To32x4(FloatRegister src
, FloatRegister dest
)
2548 DEFINED_ON(x86_shared
, arm64
);
2549 inline void zeroExtend16x8To64x2(FloatRegister src
, FloatRegister dest
)
2550 DEFINED_ON(x86_shared
, arm64
);
2551 inline void zeroExtend32x4To64x2(FloatRegister src
, FloatRegister dest
)
2552 DEFINED_ON(x86_shared
, arm64
);
2554 // Reverse bytes in lanes.
2556 inline void reverseInt16x8(FloatRegister src
, FloatRegister dest
)
2557 DEFINED_ON(x86_shared
, arm64
);
2559 inline void reverseInt32x4(FloatRegister src
, FloatRegister dest
)
2560 DEFINED_ON(x86_shared
, arm64
);
2562 inline void reverseInt64x2(FloatRegister src
, FloatRegister dest
)
2563 DEFINED_ON(x86_shared
, arm64
);
2565 // Swizzle - permute with variable indices. `rhs` holds the lanes parameter.
2567 inline void swizzleInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2568 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2570 inline void swizzleInt8x16Relaxed(FloatRegister lhs
, FloatRegister rhs
,
2572 DEFINED_ON(x86_shared
, arm64
);
2576 inline void addInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2577 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2579 inline void addInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2580 FloatRegister dest
) DEFINED_ON(x86_shared
);
2582 inline void addInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2583 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2585 inline void addInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2586 FloatRegister dest
) DEFINED_ON(x86_shared
);
2588 inline void addInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2589 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2591 inline void addInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2592 FloatRegister dest
) DEFINED_ON(x86_shared
);
2594 inline void addInt64x2(FloatRegister lhs
, FloatRegister rhs
,
2595 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2597 inline void addInt64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
2598 FloatRegister dest
) DEFINED_ON(x86_shared
);
2602 inline void subInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2603 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2605 inline void subInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2606 FloatRegister dest
) DEFINED_ON(x86_shared
);
2608 inline void subInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2609 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2611 inline void subInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2612 FloatRegister dest
) DEFINED_ON(x86_shared
);
2614 inline void subInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2615 FloatRegister dest
) DEFINED_ON(x86_shared
);
2617 inline void subInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2618 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2620 inline void subInt64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
2621 FloatRegister dest
) DEFINED_ON(x86_shared
);
2623 inline void subInt64x2(FloatRegister lhs
, FloatRegister rhs
,
2624 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2628 inline void mulInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2629 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2631 inline void mulInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2632 FloatRegister dest
) DEFINED_ON(x86_shared
);
2634 inline void mulInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2635 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2637 inline void mulInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2638 FloatRegister dest
) DEFINED_ON(x86_shared
);
2640 // On x86_shared, it is required lhs == dest
2641 inline void mulInt64x2(FloatRegister lhs
, FloatRegister rhs
,
2642 FloatRegister dest
, FloatRegister temp
)
2643 DEFINED_ON(x86_shared
);
2645 inline void mulInt64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
2646 FloatRegister dest
, FloatRegister temp
)
2647 DEFINED_ON(x86_shared
);
2649 inline void mulInt64x2(FloatRegister lhs
, FloatRegister rhs
,
2650 FloatRegister dest
, FloatRegister temp1
,
2651 FloatRegister temp2
) DEFINED_ON(arm64
);
2653 // Note for the extMul opcodes, the NxM designation is for the input lanes;
2654 // the output lanes are twice as wide.
2655 inline void extMulLowInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2657 DEFINED_ON(x86_shared
, arm64
);
2659 inline void extMulHighInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2661 DEFINED_ON(x86_shared
, arm64
);
2663 inline void unsignedExtMulLowInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2665 DEFINED_ON(x86_shared
, arm64
);
2667 inline void unsignedExtMulHighInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2669 DEFINED_ON(x86_shared
, arm64
);
2671 inline void extMulLowInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2673 DEFINED_ON(x86_shared
, arm64
);
2675 inline void extMulHighInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2677 DEFINED_ON(x86_shared
, arm64
);
2679 inline void unsignedExtMulLowInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2681 DEFINED_ON(x86_shared
, arm64
);
2683 inline void unsignedExtMulHighInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2685 DEFINED_ON(x86_shared
, arm64
);
2687 inline void extMulLowInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2689 DEFINED_ON(x86_shared
, arm64
);
2691 inline void extMulHighInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2693 DEFINED_ON(x86_shared
, arm64
);
2695 inline void unsignedExtMulLowInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2697 DEFINED_ON(x86_shared
, arm64
);
2699 inline void unsignedExtMulHighInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2701 DEFINED_ON(x86_shared
, arm64
);
2703 inline void q15MulrSatInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2705 DEFINED_ON(x86_shared
, arm64
);
2709 inline void negInt8x16(FloatRegister src
, FloatRegister dest
)
2710 DEFINED_ON(x86_shared
, arm64
);
2712 inline void negInt16x8(FloatRegister src
, FloatRegister dest
)
2713 DEFINED_ON(x86_shared
, arm64
);
2715 inline void negInt32x4(FloatRegister src
, FloatRegister dest
)
2716 DEFINED_ON(x86_shared
, arm64
);
2718 inline void negInt64x2(FloatRegister src
, FloatRegister dest
)
2719 DEFINED_ON(x86_shared
, arm64
);
2721 // Saturating integer add
2723 inline void addSatInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2724 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2726 inline void addSatInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2727 FloatRegister dest
) DEFINED_ON(x86_shared
);
2729 inline void unsignedAddSatInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2731 DEFINED_ON(x86_shared
, arm64
);
2733 inline void unsignedAddSatInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2734 FloatRegister dest
) DEFINED_ON(x86_shared
);
2736 inline void addSatInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2737 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2739 inline void addSatInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2740 FloatRegister dest
) DEFINED_ON(x86_shared
);
2742 inline void unsignedAddSatInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2744 DEFINED_ON(x86_shared
, arm64
);
2746 inline void unsignedAddSatInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2747 FloatRegister dest
) DEFINED_ON(x86_shared
);
2749 // Saturating integer subtract
2751 inline void subSatInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2752 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2754 inline void subSatInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2755 FloatRegister dest
) DEFINED_ON(x86_shared
);
2757 inline void unsignedSubSatInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2759 DEFINED_ON(x86_shared
, arm64
);
2761 inline void unsignedSubSatInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2762 FloatRegister dest
) DEFINED_ON(x86_shared
);
2764 inline void subSatInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2765 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2767 inline void subSatInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2768 FloatRegister dest
) DEFINED_ON(x86_shared
);
2770 inline void unsignedSubSatInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2772 DEFINED_ON(x86_shared
, arm64
);
2774 inline void unsignedSubSatInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2775 FloatRegister dest
) DEFINED_ON(x86_shared
);
2777 // Lane-wise integer minimum
2779 inline void minInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2780 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2782 inline void minInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2783 FloatRegister dest
) DEFINED_ON(x86_shared
);
2785 inline void unsignedMinInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2787 DEFINED_ON(x86_shared
, arm64
);
2789 inline void unsignedMinInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2790 FloatRegister dest
) DEFINED_ON(x86_shared
);
2792 inline void minInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2793 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2795 inline void minInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2796 FloatRegister dest
) DEFINED_ON(x86_shared
);
2798 inline void unsignedMinInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2800 DEFINED_ON(x86_shared
, arm64
);
2802 inline void unsignedMinInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2803 FloatRegister dest
) DEFINED_ON(x86_shared
);
2805 inline void minInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2806 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2808 inline void minInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2809 FloatRegister dest
) DEFINED_ON(x86_shared
);
2811 inline void unsignedMinInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2813 DEFINED_ON(x86_shared
, arm64
);
2815 inline void unsignedMinInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2816 FloatRegister dest
) DEFINED_ON(x86_shared
);
2818 // Lane-wise integer maximum
2820 inline void maxInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2821 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2823 inline void maxInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2824 FloatRegister dest
) DEFINED_ON(x86_shared
);
2826 inline void unsignedMaxInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2828 DEFINED_ON(x86_shared
, arm64
);
2830 inline void unsignedMaxInt8x16(FloatRegister lhs
, const SimdConstant
& rhs
,
2831 FloatRegister dest
) DEFINED_ON(x86_shared
);
2833 inline void maxInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2834 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2836 inline void maxInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2837 FloatRegister dest
) DEFINED_ON(x86_shared
);
2839 inline void unsignedMaxInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2841 DEFINED_ON(x86_shared
, arm64
);
2843 inline void unsignedMaxInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
2844 FloatRegister dest
) DEFINED_ON(x86_shared
);
2846 inline void maxInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2847 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
2849 inline void maxInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2850 FloatRegister dest
) DEFINED_ON(x86_shared
);
2852 inline void unsignedMaxInt32x4(FloatRegister lhs
, FloatRegister rhs
,
2854 DEFINED_ON(x86_shared
, arm64
);
2856 inline void unsignedMaxInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
2857 FloatRegister dest
) DEFINED_ON(x86_shared
);
2859 // Lane-wise integer rounding average
2861 inline void unsignedAverageInt8x16(FloatRegister lhs
, FloatRegister rhs
,
2863 DEFINED_ON(x86_shared
, arm64
);
2865 inline void unsignedAverageInt16x8(FloatRegister lhs
, FloatRegister rhs
,
2867 DEFINED_ON(x86_shared
, arm64
);
2869 // Lane-wise integer absolute value
2871 inline void absInt8x16(FloatRegister src
, FloatRegister dest
)
2872 DEFINED_ON(x86_shared
, arm64
);
2874 inline void absInt16x8(FloatRegister src
, FloatRegister dest
)
2875 DEFINED_ON(x86_shared
, arm64
);
2877 inline void absInt32x4(FloatRegister src
, FloatRegister dest
)
2878 DEFINED_ON(x86_shared
, arm64
);
2880 inline void absInt64x2(FloatRegister src
, FloatRegister dest
)
2881 DEFINED_ON(x86_shared
, arm64
);
2883 // Left shift by scalar. Immediates and variable shifts must have been
2884 // masked; shifts of zero will work but may or may not generate code.
2886 inline void leftShiftInt8x16(Register rhs
, FloatRegister lhsDest
,
2887 FloatRegister temp
) DEFINED_ON(x86_shared
);
2889 inline void leftShiftInt8x16(FloatRegister lhs
, Register rhs
,
2890 FloatRegister dest
) DEFINED_ON(arm64
);
2892 inline void leftShiftInt8x16(Imm32 count
, FloatRegister src
,
2894 DEFINED_ON(x86_shared
, arm64
);
2896 inline void leftShiftInt16x8(Register rhs
, FloatRegister lhsDest
)
2897 DEFINED_ON(x86_shared
);
2899 inline void leftShiftInt16x8(FloatRegister lhs
, Register rhs
,
2900 FloatRegister dest
) DEFINED_ON(arm64
);
2902 inline void leftShiftInt16x8(Imm32 count
, FloatRegister src
,
2904 DEFINED_ON(x86_shared
, arm64
);
2906 inline void leftShiftInt32x4(Register rhs
, FloatRegister lhsDest
)
2907 DEFINED_ON(x86_shared
);
2909 inline void leftShiftInt32x4(FloatRegister lhs
, Register rhs
,
2910 FloatRegister dest
) DEFINED_ON(arm64
);
2912 inline void leftShiftInt32x4(Imm32 count
, FloatRegister src
,
2914 DEFINED_ON(x86_shared
, arm64
);
2916 inline void leftShiftInt64x2(Register rhs
, FloatRegister lhsDest
)
2917 DEFINED_ON(x86_shared
);
2919 inline void leftShiftInt64x2(FloatRegister lhs
, Register rhs
,
2920 FloatRegister dest
) DEFINED_ON(arm64
);
2922 inline void leftShiftInt64x2(Imm32 count
, FloatRegister src
,
2924 DEFINED_ON(x86_shared
, arm64
);
2926 // Right shift by scalar. Immediates and variable shifts must have been
2927 // masked; shifts of zero will work but may or may not generate code.
2929 inline void rightShiftInt8x16(Register rhs
, FloatRegister lhsDest
,
2930 FloatRegister temp
) DEFINED_ON(x86_shared
);
2932 inline void rightShiftInt8x16(FloatRegister lhs
, Register rhs
,
2933 FloatRegister dest
) DEFINED_ON(arm64
);
2935 inline void rightShiftInt8x16(Imm32 count
, FloatRegister src
,
2937 DEFINED_ON(x86_shared
, arm64
);
2939 inline void unsignedRightShiftInt8x16(Register rhs
, FloatRegister lhsDest
,
2941 DEFINED_ON(x86_shared
);
2943 inline void unsignedRightShiftInt8x16(FloatRegister lhs
, Register rhs
,
2944 FloatRegister dest
) DEFINED_ON(arm64
);
2946 inline void unsignedRightShiftInt8x16(Imm32 count
, FloatRegister src
,
2948 DEFINED_ON(x86_shared
, arm64
);
2950 inline void rightShiftInt16x8(Register rhs
, FloatRegister lhsDest
)
2951 DEFINED_ON(x86_shared
);
2953 inline void rightShiftInt16x8(FloatRegister lhs
, Register rhs
,
2954 FloatRegister dest
) DEFINED_ON(arm64
);
2956 inline void rightShiftInt16x8(Imm32 count
, FloatRegister src
,
2958 DEFINED_ON(x86_shared
, arm64
);
2960 inline void unsignedRightShiftInt16x8(Register rhs
, FloatRegister lhsDest
)
2961 DEFINED_ON(x86_shared
);
2963 inline void unsignedRightShiftInt16x8(FloatRegister lhs
, Register rhs
,
2964 FloatRegister dest
) DEFINED_ON(arm64
);
2966 inline void unsignedRightShiftInt16x8(Imm32 count
, FloatRegister src
,
2968 DEFINED_ON(x86_shared
, arm64
);
2970 inline void rightShiftInt32x4(Register rhs
, FloatRegister lhsDest
)
2971 DEFINED_ON(x86_shared
);
2973 inline void rightShiftInt32x4(FloatRegister lhs
, Register rhs
,
2974 FloatRegister dest
) DEFINED_ON(arm64
);
2976 inline void rightShiftInt32x4(Imm32 count
, FloatRegister src
,
2978 DEFINED_ON(x86_shared
, arm64
);
2980 inline void unsignedRightShiftInt32x4(Register rhs
, FloatRegister lhsDest
)
2981 DEFINED_ON(x86_shared
);
2983 inline void unsignedRightShiftInt32x4(FloatRegister lhs
, Register rhs
,
2984 FloatRegister dest
) DEFINED_ON(arm64
);
2986 inline void unsignedRightShiftInt32x4(Imm32 count
, FloatRegister src
,
2988 DEFINED_ON(x86_shared
, arm64
);
2990 inline void rightShiftInt64x2(Register rhs
, FloatRegister lhsDest
,
2991 FloatRegister temp
) DEFINED_ON(x86_shared
);
2993 inline void rightShiftInt64x2(Imm32 count
, FloatRegister src
,
2995 DEFINED_ON(x86_shared
, arm64
);
2997 inline void rightShiftInt64x2(FloatRegister lhs
, Register rhs
,
2998 FloatRegister dest
) DEFINED_ON(arm64
);
3000 inline void unsignedRightShiftInt64x2(Register rhs
, FloatRegister lhsDest
)
3001 DEFINED_ON(x86_shared
);
3003 inline void unsignedRightShiftInt64x2(FloatRegister lhs
, Register rhs
,
3004 FloatRegister dest
) DEFINED_ON(arm64
);
3006 inline void unsignedRightShiftInt64x2(Imm32 count
, FloatRegister src
,
3008 DEFINED_ON(x86_shared
, arm64
);
3010 // Sign replication operation
3012 inline void signReplicationInt8x16(FloatRegister src
, FloatRegister dest
)
3013 DEFINED_ON(x86_shared
);
3015 inline void signReplicationInt16x8(FloatRegister src
, FloatRegister dest
)
3016 DEFINED_ON(x86_shared
);
3018 inline void signReplicationInt32x4(FloatRegister src
, FloatRegister dest
)
3019 DEFINED_ON(x86_shared
);
3021 inline void signReplicationInt64x2(FloatRegister src
, FloatRegister dest
)
3022 DEFINED_ON(x86_shared
);
3024 // Bitwise and, or, xor, not
3026 inline void bitwiseAndSimd128(FloatRegister rhs
, FloatRegister lhsDest
)
3027 DEFINED_ON(x86_shared
, arm64
);
3029 inline void bitwiseAndSimd128(FloatRegister lhs
, FloatRegister rhs
,
3031 DEFINED_ON(x86_shared
, arm64
);
3033 inline void bitwiseAndSimd128(FloatRegister lhs
, const SimdConstant
& rhs
,
3034 FloatRegister dest
) DEFINED_ON(x86_shared
);
3036 inline void bitwiseOrSimd128(FloatRegister rhs
, FloatRegister lhsDest
)
3037 DEFINED_ON(x86_shared
, arm64
);
3039 inline void bitwiseOrSimd128(FloatRegister lhs
, FloatRegister rhs
,
3041 DEFINED_ON(x86_shared
, arm64
);
3043 inline void bitwiseOrSimd128(FloatRegister lhs
, const SimdConstant
& rhs
,
3044 FloatRegister dest
) DEFINED_ON(x86_shared
);
3046 inline void bitwiseXorSimd128(FloatRegister rhs
, FloatRegister lhsDest
)
3047 DEFINED_ON(x86_shared
, arm64
);
3049 inline void bitwiseXorSimd128(FloatRegister lhs
, FloatRegister rhs
,
3051 DEFINED_ON(x86_shared
, arm64
);
3053 inline void bitwiseXorSimd128(FloatRegister lhs
, const SimdConstant
& rhs
,
3054 FloatRegister dest
) DEFINED_ON(x86_shared
);
3056 inline void bitwiseNotSimd128(FloatRegister src
, FloatRegister dest
)
3057 DEFINED_ON(x86_shared
, arm64
);
3059 // Bitwise AND with compliment: dest = lhs & ~rhs, note only arm64 can do it.
3060 inline void bitwiseAndNotSimd128(FloatRegister lhs
, FloatRegister rhs
,
3061 FloatRegister lhsDest
) DEFINED_ON(arm64
);
3063 // Bitwise AND with complement: dest = ~lhs & rhs, note this is not what Wasm
3064 // wants but what the x86 hardware offers. Hence the name.
3066 inline void bitwiseNotAndSimd128(FloatRegister rhs
, FloatRegister lhsDest
)
3067 DEFINED_ON(x86_shared
, arm64
);
3069 inline void bitwiseNotAndSimd128(FloatRegister lhs
, FloatRegister rhs
,
3070 FloatRegister lhsDest
)
3071 DEFINED_ON(x86_shared
);
3075 inline void bitwiseSelectSimd128(FloatRegister mask
, FloatRegister onTrue
,
3076 FloatRegister onFalse
, FloatRegister dest
,
3077 FloatRegister temp
) DEFINED_ON(x86_shared
);
3079 inline void bitwiseSelectSimd128(FloatRegister onTrue
, FloatRegister onFalse
,
3080 FloatRegister maskDest
) DEFINED_ON(arm64
);
3084 inline void popcntInt8x16(FloatRegister src
, FloatRegister dest
,
3085 FloatRegister temp
) DEFINED_ON(x86_shared
);
3087 inline void popcntInt8x16(FloatRegister src
, FloatRegister dest
)
3090 // Any lane true, ie, any bit set
3092 inline void anyTrueSimd128(FloatRegister src
, Register dest
)
3093 DEFINED_ON(x86_shared
, arm64
);
3097 inline void allTrueInt8x16(FloatRegister src
, Register dest
)
3098 DEFINED_ON(x86_shared
, arm64
);
3100 inline void allTrueInt16x8(FloatRegister src
, Register dest
)
3101 DEFINED_ON(x86_shared
, arm64
);
3103 inline void allTrueInt32x4(FloatRegister src
, Register dest
)
3104 DEFINED_ON(x86_shared
, arm64
);
3106 inline void allTrueInt64x2(FloatRegister src
, Register dest
)
3107 DEFINED_ON(x86_shared
, arm64
);
3109 // Bitmask, ie extract and compress high bits of all lanes
3111 inline void bitmaskInt8x16(FloatRegister src
, Register dest
)
3112 DEFINED_ON(x86_shared
);
3114 inline void bitmaskInt8x16(FloatRegister src
, Register dest
,
3115 FloatRegister temp
) DEFINED_ON(arm64
);
3117 inline void bitmaskInt16x8(FloatRegister src
, Register dest
)
3118 DEFINED_ON(x86_shared
);
3120 inline void bitmaskInt16x8(FloatRegister src
, Register dest
,
3121 FloatRegister temp
) DEFINED_ON(arm64
);
3123 inline void bitmaskInt32x4(FloatRegister src
, Register dest
)
3124 DEFINED_ON(x86_shared
);
3126 inline void bitmaskInt32x4(FloatRegister src
, Register dest
,
3127 FloatRegister temp
) DEFINED_ON(arm64
);
3129 inline void bitmaskInt64x2(FloatRegister src
, Register dest
)
3130 DEFINED_ON(x86_shared
);
3132 inline void bitmaskInt64x2(FloatRegister src
, Register dest
,
3133 FloatRegister temp
) DEFINED_ON(arm64
);
3135 // Comparisons (integer and floating-point)
3137 inline void compareInt8x16(Assembler::Condition cond
, FloatRegister rhs
,
3138 FloatRegister lhsDest
)
3139 DEFINED_ON(x86_shared
, arm64
);
3141 // On x86_shared, limited to !=, ==, <=, >
3142 inline void compareInt8x16(Assembler::Condition cond
, FloatRegister lhs
,
3143 const SimdConstant
& rhs
, FloatRegister dest
)
3144 DEFINED_ON(x86_shared
);
3146 // On arm64, use any integer comparison condition.
3147 inline void compareInt8x16(Assembler::Condition cond
, FloatRegister lhs
,
3148 FloatRegister rhs
, FloatRegister dest
)
3149 DEFINED_ON(x86_shared
, arm64
);
3151 inline void compareInt16x8(Assembler::Condition cond
, FloatRegister rhs
,
3152 FloatRegister lhsDest
)
3153 DEFINED_ON(x86_shared
, arm64
);
3155 inline void compareInt16x8(Assembler::Condition cond
, FloatRegister lhs
,
3156 FloatRegister rhs
, FloatRegister dest
)
3157 DEFINED_ON(x86_shared
, arm64
);
3159 // On x86_shared, limited to !=, ==, <=, >
3160 inline void compareInt16x8(Assembler::Condition cond
, FloatRegister lhs
,
3161 const SimdConstant
& rhs
, FloatRegister dest
)
3162 DEFINED_ON(x86_shared
);
3164 // On x86_shared, limited to !=, ==, <=, >
3165 inline void compareInt32x4(Assembler::Condition cond
, FloatRegister rhs
,
3166 FloatRegister lhsDest
)
3167 DEFINED_ON(x86_shared
, arm64
);
3169 inline void compareInt32x4(Assembler::Condition cond
, FloatRegister lhs
,
3170 const SimdConstant
& rhs
, FloatRegister dest
)
3171 DEFINED_ON(x86_shared
);
3173 // On arm64, use any integer comparison condition.
3174 inline void compareInt32x4(Assembler::Condition cond
, FloatRegister lhs
,
3175 FloatRegister rhs
, FloatRegister dest
)
3176 DEFINED_ON(x86_shared
, arm64
);
3178 inline void compareForEqualityInt64x2(Assembler::Condition cond
,
3179 FloatRegister lhs
, FloatRegister rhs
,
3181 DEFINED_ON(x86_shared
);
3183 inline void compareForOrderingInt64x2(Assembler::Condition cond
,
3184 FloatRegister lhs
, FloatRegister rhs
,
3185 FloatRegister dest
, FloatRegister temp1
,
3186 FloatRegister temp2
)
3187 DEFINED_ON(x86_shared
);
3189 inline void compareInt64x2(Assembler::Condition cond
, FloatRegister rhs
,
3190 FloatRegister lhsDest
) DEFINED_ON(arm64
);
3192 inline void compareInt64x2(Assembler::Condition cond
, FloatRegister lhs
,
3193 FloatRegister rhs
, FloatRegister dest
)
3196 inline void compareFloat32x4(Assembler::Condition cond
, FloatRegister rhs
,
3197 FloatRegister lhsDest
)
3198 DEFINED_ON(x86_shared
, arm64
);
3200 // On x86_shared, limited to ==, !=, <, <=
3201 inline void compareFloat32x4(Assembler::Condition cond
, FloatRegister lhs
,
3202 const SimdConstant
& rhs
, FloatRegister dest
)
3203 DEFINED_ON(x86_shared
);
3205 // On x86_shared, limited to ==, !=, <, <=
3206 // On arm64, use any float-point comparison condition.
3207 inline void compareFloat32x4(Assembler::Condition cond
, FloatRegister lhs
,
3208 FloatRegister rhs
, FloatRegister dest
)
3209 DEFINED_ON(x86_shared
, arm64
);
3211 inline void compareFloat64x2(Assembler::Condition cond
, FloatRegister rhs
,
3212 FloatRegister lhsDest
)
3213 DEFINED_ON(x86_shared
, arm64
);
3215 // On x86_shared, limited to ==, !=, <, <=
3216 inline void compareFloat64x2(Assembler::Condition cond
, FloatRegister lhs
,
3217 const SimdConstant
& rhs
, FloatRegister dest
)
3218 DEFINED_ON(x86_shared
);
3220 // On x86_shared, limited to ==, !=, <, <=
3221 // On arm64, use any float-point comparison condition.
3222 inline void compareFloat64x2(Assembler::Condition cond
, FloatRegister lhs
,
3223 FloatRegister rhs
, FloatRegister dest
)
3224 DEFINED_ON(x86_shared
, arm64
);
3228 inline void loadUnalignedSimd128(const Operand
& src
, FloatRegister dest
)
3229 DEFINED_ON(x86_shared
);
3231 inline FaultingCodeOffset
loadUnalignedSimd128(const Address
& src
,
3233 DEFINED_ON(x86_shared
, arm64
);
3235 inline FaultingCodeOffset
loadUnalignedSimd128(const BaseIndex
& src
,
3237 DEFINED_ON(x86_shared
, arm64
);
3241 inline FaultingCodeOffset
storeUnalignedSimd128(FloatRegister src
,
3242 const Address
& dest
)
3243 DEFINED_ON(x86_shared
, arm64
);
3245 inline FaultingCodeOffset
storeUnalignedSimd128(FloatRegister src
,
3246 const BaseIndex
& dest
)
3247 DEFINED_ON(x86_shared
, arm64
);
3249 // Floating point negation
3251 inline void negFloat32x4(FloatRegister src
, FloatRegister dest
)
3252 DEFINED_ON(x86_shared
, arm64
);
3254 inline void negFloat64x2(FloatRegister src
, FloatRegister dest
)
3255 DEFINED_ON(x86_shared
, arm64
);
3257 // Floating point absolute value
3259 inline void absFloat32x4(FloatRegister src
, FloatRegister dest
)
3260 DEFINED_ON(x86_shared
, arm64
);
3262 inline void absFloat64x2(FloatRegister src
, FloatRegister dest
)
3263 DEFINED_ON(x86_shared
, arm64
);
3265 // NaN-propagating minimum
3267 inline void minFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3268 FloatRegister dest
, FloatRegister temp1
,
3269 FloatRegister temp2
) DEFINED_ON(x86_shared
);
3271 inline void minFloat32x4(FloatRegister rhs
, FloatRegister lhsDest
)
3274 inline void minFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3275 FloatRegister dest
) DEFINED_ON(arm64
);
3277 inline void minFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3278 FloatRegister dest
, FloatRegister temp1
,
3279 FloatRegister temp2
) DEFINED_ON(x86_shared
);
3281 inline void minFloat64x2(FloatRegister rhs
, FloatRegister lhsDest
)
3284 inline void minFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3285 FloatRegister dest
) DEFINED_ON(arm64
);
3287 // NaN-propagating maximum
3289 inline void maxFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3290 FloatRegister dest
, FloatRegister temp1
,
3291 FloatRegister temp2
) DEFINED_ON(x86_shared
);
3293 inline void maxFloat32x4(FloatRegister rhs
, FloatRegister lhsDest
)
3296 inline void maxFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3297 FloatRegister dest
) DEFINED_ON(arm64
);
3299 inline void maxFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3300 FloatRegister dest
, FloatRegister temp1
,
3301 FloatRegister temp2
) DEFINED_ON(x86_shared
);
3303 inline void maxFloat64x2(FloatRegister rhs
, FloatRegister lhsDest
)
3306 inline void maxFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3307 FloatRegister dest
) DEFINED_ON(arm64
);
3311 inline void addFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3312 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3314 inline void addFloat32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
3315 FloatRegister dest
) DEFINED_ON(x86_shared
);
3317 inline void addFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3318 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3320 inline void addFloat64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
3321 FloatRegister dest
) DEFINED_ON(x86_shared
);
3323 // Floating subtract
3325 inline void subFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3326 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3328 inline void subFloat32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
3329 FloatRegister dest
) DEFINED_ON(x86_shared
);
3331 inline void subFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3332 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3334 inline void subFloat64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
3335 FloatRegister dest
) DEFINED_ON(x86_shared
);
3337 // Floating division
3339 inline void divFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3340 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3342 inline void divFloat32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
3343 FloatRegister dest
) DEFINED_ON(x86_shared
);
3345 inline void divFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3346 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3348 inline void divFloat64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
3349 FloatRegister dest
) DEFINED_ON(x86_shared
);
3351 // Floating Multiply
3353 inline void mulFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3354 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3356 inline void mulFloat32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
3357 FloatRegister dest
) DEFINED_ON(x86_shared
);
3359 inline void mulFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3360 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3362 inline void mulFloat64x2(FloatRegister lhs
, const SimdConstant
& rhs
,
3363 FloatRegister dest
) DEFINED_ON(x86_shared
);
3367 inline void extAddPairwiseInt8x16(FloatRegister src
, FloatRegister dest
)
3368 DEFINED_ON(x86_shared
, arm64
);
3370 inline void unsignedExtAddPairwiseInt8x16(FloatRegister src
,
3372 DEFINED_ON(x86_shared
, arm64
);
3374 inline void extAddPairwiseInt16x8(FloatRegister src
, FloatRegister dest
)
3375 DEFINED_ON(x86_shared
, arm64
);
3377 inline void unsignedExtAddPairwiseInt16x8(FloatRegister src
,
3379 DEFINED_ON(x86_shared
, arm64
);
3381 // Floating square root
3383 inline void sqrtFloat32x4(FloatRegister src
, FloatRegister dest
)
3384 DEFINED_ON(x86_shared
, arm64
);
3386 inline void sqrtFloat64x2(FloatRegister src
, FloatRegister dest
)
3387 DEFINED_ON(x86_shared
, arm64
);
3389 // Integer to floating point with rounding
3391 inline void convertInt32x4ToFloat32x4(FloatRegister src
, FloatRegister dest
)
3392 DEFINED_ON(x86_shared
, arm64
);
3394 inline void unsignedConvertInt32x4ToFloat32x4(FloatRegister src
,
3396 DEFINED_ON(x86_shared
, arm64
);
3398 inline void convertInt32x4ToFloat64x2(FloatRegister src
, FloatRegister dest
)
3399 DEFINED_ON(x86_shared
, arm64
);
3401 inline void unsignedConvertInt32x4ToFloat64x2(FloatRegister src
,
3403 DEFINED_ON(x86_shared
, arm64
);
3405 // Floating point to integer with saturation
3407 inline void truncSatFloat32x4ToInt32x4(FloatRegister src
, FloatRegister dest
)
3408 DEFINED_ON(x86_shared
, arm64
);
3410 inline void unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src
,
3413 DEFINED_ON(x86_shared
);
3415 inline void unsignedTruncSatFloat32x4ToInt32x4(FloatRegister src
,
3419 inline void truncSatFloat64x2ToInt32x4(FloatRegister src
, FloatRegister dest
,
3421 DEFINED_ON(x86_shared
, arm64
);
3423 inline void unsignedTruncSatFloat64x2ToInt32x4(FloatRegister src
,
3426 DEFINED_ON(x86_shared
, arm64
);
3428 inline void truncFloat32x4ToInt32x4Relaxed(FloatRegister src
,
3430 DEFINED_ON(x86_shared
, arm64
);
3432 inline void unsignedTruncFloat32x4ToInt32x4Relaxed(FloatRegister src
,
3434 DEFINED_ON(x86_shared
, arm64
);
3436 inline void truncFloat64x2ToInt32x4Relaxed(FloatRegister src
,
3438 DEFINED_ON(x86_shared
, arm64
);
3440 inline void unsignedTruncFloat64x2ToInt32x4Relaxed(FloatRegister src
,
3442 DEFINED_ON(x86_shared
, arm64
);
3444 // Floating point narrowing
3446 inline void convertFloat64x2ToFloat32x4(FloatRegister src
, FloatRegister dest
)
3447 DEFINED_ON(x86_shared
, arm64
);
3449 // Floating point widening
3451 inline void convertFloat32x4ToFloat64x2(FloatRegister src
, FloatRegister dest
)
3452 DEFINED_ON(x86_shared
, arm64
);
3454 // Integer to integer narrowing
3456 inline void narrowInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
3457 FloatRegister dest
) DEFINED_ON(x86_shared
);
3459 inline void narrowInt16x8(FloatRegister lhs
, FloatRegister rhs
,
3460 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3462 inline void unsignedNarrowInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
3463 FloatRegister dest
) DEFINED_ON(x86_shared
);
3465 inline void unsignedNarrowInt16x8(FloatRegister lhs
, FloatRegister rhs
,
3467 DEFINED_ON(x86_shared
, arm64
);
3469 inline void narrowInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
3470 FloatRegister dest
) DEFINED_ON(x86_shared
);
3472 inline void narrowInt32x4(FloatRegister lhs
, FloatRegister rhs
,
3473 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3475 inline void unsignedNarrowInt32x4(FloatRegister lhs
, const SimdConstant
& rhs
,
3476 FloatRegister dest
) DEFINED_ON(x86_shared
);
3478 inline void unsignedNarrowInt32x4(FloatRegister lhs
, FloatRegister rhs
,
3480 DEFINED_ON(x86_shared
, arm64
);
3482 // Integer to integer widening
3484 inline void widenLowInt8x16(FloatRegister src
, FloatRegister dest
)
3485 DEFINED_ON(x86_shared
, arm64
);
3487 inline void widenHighInt8x16(FloatRegister src
, FloatRegister dest
)
3488 DEFINED_ON(x86_shared
, arm64
);
3490 inline void unsignedWidenLowInt8x16(FloatRegister src
, FloatRegister dest
)
3491 DEFINED_ON(x86_shared
, arm64
);
3493 inline void unsignedWidenHighInt8x16(FloatRegister src
, FloatRegister dest
)
3494 DEFINED_ON(x86_shared
, arm64
);
3496 inline void widenLowInt16x8(FloatRegister src
, FloatRegister dest
)
3497 DEFINED_ON(x86_shared
, arm64
);
3499 inline void widenHighInt16x8(FloatRegister src
, FloatRegister dest
)
3500 DEFINED_ON(x86_shared
, arm64
);
3502 inline void unsignedWidenLowInt16x8(FloatRegister src
, FloatRegister dest
)
3503 DEFINED_ON(x86_shared
, arm64
);
3505 inline void unsignedWidenHighInt16x8(FloatRegister src
, FloatRegister dest
)
3506 DEFINED_ON(x86_shared
, arm64
);
3508 inline void widenLowInt32x4(FloatRegister src
, FloatRegister dest
)
3509 DEFINED_ON(x86_shared
, arm64
);
3511 inline void unsignedWidenLowInt32x4(FloatRegister src
, FloatRegister dest
)
3512 DEFINED_ON(x86_shared
, arm64
);
3514 inline void widenHighInt32x4(FloatRegister src
, FloatRegister dest
)
3515 DEFINED_ON(x86_shared
, arm64
);
3517 inline void unsignedWidenHighInt32x4(FloatRegister src
, FloatRegister dest
)
3518 DEFINED_ON(x86_shared
, arm64
);
3520 // Compare-based minimum/maximum
3522 // On x86, the signature is (rhsDest, lhs); on arm64 it is (rhs, lhsDest).
3524 // The masm preprocessor can't deal with multiple declarations with identical
3525 // signatures even if they are on different platforms, hence the weird
3528 inline void pseudoMinFloat32x4(FloatRegister rhsOrRhsDest
,
3529 FloatRegister lhsOrLhsDest
)
3530 DEFINED_ON(x86_shared
, arm64
);
3532 inline void pseudoMinFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3534 DEFINED_ON(x86_shared
, arm64
);
3536 inline void pseudoMinFloat64x2(FloatRegister rhsOrRhsDest
,
3537 FloatRegister lhsOrLhsDest
)
3538 DEFINED_ON(x86_shared
, arm64
);
3540 inline void pseudoMinFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3542 DEFINED_ON(x86_shared
, arm64
);
3544 inline void pseudoMaxFloat32x4(FloatRegister rhsOrRhsDest
,
3545 FloatRegister lhsOrLhsDest
)
3546 DEFINED_ON(x86_shared
, arm64
);
3548 inline void pseudoMaxFloat32x4(FloatRegister lhs
, FloatRegister rhs
,
3550 DEFINED_ON(x86_shared
, arm64
);
3552 inline void pseudoMaxFloat64x2(FloatRegister rhsOrRhsDest
,
3553 FloatRegister lhsOrLhsDest
)
3554 DEFINED_ON(x86_shared
, arm64
);
3556 inline void pseudoMaxFloat64x2(FloatRegister lhs
, FloatRegister rhs
,
3558 DEFINED_ON(x86_shared
, arm64
);
3560 // Widening/pairwise integer dot product
3562 inline void widenDotInt16x8(FloatRegister lhs
, FloatRegister rhs
,
3563 FloatRegister dest
) DEFINED_ON(x86_shared
, arm64
);
3565 inline void widenDotInt16x8(FloatRegister lhs
, const SimdConstant
& rhs
,
3566 FloatRegister dest
) DEFINED_ON(x86_shared
);
3568 inline void dotInt8x16Int7x16(FloatRegister lhs
, FloatRegister rhs
,
3570 DEFINED_ON(x86_shared
, arm64
);
3572 inline void dotInt8x16Int7x16ThenAdd(FloatRegister lhs
, FloatRegister rhs
,
3574 DEFINED_ON(x86_shared
);
3576 inline void dotInt8x16Int7x16ThenAdd(FloatRegister lhs
, FloatRegister rhs
,
3577 FloatRegister dest
, FloatRegister temp
)
3580 // Floating point rounding
3582 inline void ceilFloat32x4(FloatRegister src
, FloatRegister dest
)
3583 DEFINED_ON(x86_shared
, arm64
);
3585 inline void ceilFloat64x2(FloatRegister src
, FloatRegister dest
)
3586 DEFINED_ON(x86_shared
, arm64
);
3588 inline void floorFloat32x4(FloatRegister src
, FloatRegister dest
)
3589 DEFINED_ON(x86_shared
, arm64
);
3591 inline void floorFloat64x2(FloatRegister src
, FloatRegister dest
)
3592 DEFINED_ON(x86_shared
, arm64
);
3594 inline void truncFloat32x4(FloatRegister src
, FloatRegister dest
)
3595 DEFINED_ON(x86_shared
, arm64
);
3597 inline void truncFloat64x2(FloatRegister src
, FloatRegister dest
)
3598 DEFINED_ON(x86_shared
, arm64
);
3600 inline void nearestFloat32x4(FloatRegister src
, FloatRegister dest
)
3601 DEFINED_ON(x86_shared
, arm64
);
3603 inline void nearestFloat64x2(FloatRegister src
, FloatRegister dest
)
3604 DEFINED_ON(x86_shared
, arm64
);
3606 // Floating multiply-accumulate: srcDest [+-]= src1 * src2
3608 inline void fmaFloat32x4(FloatRegister src1
, FloatRegister src2
,
3609 FloatRegister srcDest
) DEFINED_ON(x86_shared
, arm64
);
3611 inline void fnmaFloat32x4(FloatRegister src1
, FloatRegister src2
,
3612 FloatRegister srcDest
)
3613 DEFINED_ON(x86_shared
, arm64
);
3615 inline void fmaFloat64x2(FloatRegister src1
, FloatRegister src2
,
3616 FloatRegister srcDest
) DEFINED_ON(x86_shared
, arm64
);
3618 inline void fnmaFloat64x2(FloatRegister src1
, FloatRegister src2
,
3619 FloatRegister srcDest
)
3620 DEFINED_ON(x86_shared
, arm64
);
3622 inline void minFloat32x4Relaxed(FloatRegister src
, FloatRegister srcDest
)
3623 DEFINED_ON(x86_shared
, arm64
);
3625 inline void minFloat32x4Relaxed(FloatRegister lhs
, FloatRegister rhs
,
3627 DEFINED_ON(x86_shared
, arm64
);
3629 inline void maxFloat32x4Relaxed(FloatRegister src
, FloatRegister srcDest
)
3630 DEFINED_ON(x86_shared
, arm64
);
3632 inline void maxFloat32x4Relaxed(FloatRegister lhs
, FloatRegister rhs
,
3634 DEFINED_ON(x86_shared
, arm64
);
3636 inline void minFloat64x2Relaxed(FloatRegister src
, FloatRegister srcDest
)
3637 DEFINED_ON(x86_shared
, arm64
);
3639 inline void minFloat64x2Relaxed(FloatRegister lhs
, FloatRegister rhs
,
3641 DEFINED_ON(x86_shared
, arm64
);
3643 inline void maxFloat64x2Relaxed(FloatRegister src
, FloatRegister srcDest
)
3644 DEFINED_ON(x86_shared
, arm64
);
3646 inline void maxFloat64x2Relaxed(FloatRegister lhs
, FloatRegister rhs
,
3648 DEFINED_ON(x86_shared
, arm64
);
3650 inline void q15MulrInt16x8Relaxed(FloatRegister lhs
, FloatRegister rhs
,
3652 DEFINED_ON(x86_shared
, arm64
);
3655 // ========================================================================
3656 // Truncate floating point.
3658 // Undefined behaviour when truncation is outside Int64 range.
3659 // Needs a temp register if SSE3 is not present.
3660 inline void truncateFloat32ToInt64(Address src
, Address dest
, Register temp
)
3661 DEFINED_ON(x86_shared
);
3662 inline void truncateFloat32ToUInt64(Address src
, Address dest
, Register temp
,
3663 FloatRegister floatTemp
)
3664 DEFINED_ON(x86
, x64
);
3665 inline void truncateDoubleToInt64(Address src
, Address dest
, Register temp
)
3666 DEFINED_ON(x86_shared
);
3667 inline void truncateDoubleToUInt64(Address src
, Address dest
, Register temp
,
3668 FloatRegister floatTemp
)
3669 DEFINED_ON(x86
, x64
);
3672 // ========================================================================
3673 // Convert floating point.
3675 // temp required on x86 and x64; must be undefined on mips64 and loong64.
3676 void convertUInt64ToFloat32(Register64 src
, FloatRegister dest
, Register temp
)
3677 DEFINED_ON(arm64
, mips64
, loong64
, riscv64
, wasm32
, x64
, x86
);
3679 void convertInt64ToFloat32(Register64 src
, FloatRegister dest
)
3680 DEFINED_ON(arm64
, mips64
, loong64
, riscv64
, wasm32
, x64
, x86
);
3682 bool convertUInt64ToDoubleNeedsTemp() PER_ARCH
;
3684 // temp required when convertUInt64ToDoubleNeedsTemp() returns true.
3685 void convertUInt64ToDouble(Register64 src
, FloatRegister dest
,
3686 Register temp
) PER_ARCH
;
3688 void convertInt64ToDouble(Register64 src
, FloatRegister dest
) PER_ARCH
;
3690 void convertIntPtrToDouble(Register src
, FloatRegister dest
) PER_ARCH
;
3693 // ========================================================================
3696 FaultingCodeOffset
wasmTrapInstruction() PER_SHARED_ARCH
;
3698 void wasmTrap(wasm::Trap trap
, wasm::BytecodeOffset bytecodeOffset
);
3700 // Load all pinned regs via InstanceReg. If the trapOffset is something,
3701 // give the first load a trap descriptor with type IndirectCallToNull, so that
3702 // a null instance will cause a trap.
3703 void loadWasmPinnedRegsFromInstance(
3704 mozilla::Maybe
<wasm::BytecodeOffset
> trapOffset
= mozilla::Nothing());
3706 // Returns a pair: the offset of the undefined (trapping) instruction, and
3707 // the number of extra bytes of stack allocated prior to the trap
3708 // instruction proper.
3709 std::pair
<CodeOffset
, uint32_t> wasmReserveStackChecked(
3710 uint32_t amount
, wasm::BytecodeOffset trapOffset
);
3712 // Emit a bounds check against the wasm heap limit, jumping to 'ok' if 'cond'
3713 // holds; this can be the label either of the access or of the trap. The
3714 // label should name a code position greater than the position of the bounds
3717 // If JitOptions.spectreMaskIndex is true, a no-op speculation barrier is
3718 // emitted in the code stream after the check to prevent an OOB access from
3719 // being executed speculatively. (On current tier-1 platforms the barrier is
3720 // a conditional saturation of 'index' to 'boundsCheckLimit', using the same
3721 // condition as the check.) If the condition is such that the bounds check
3722 // branches out of line to the trap, the barrier will actually be executed
3723 // when the bounds check passes.
3725 // On 32-bit systems for both wasm and asm.js, and on 64-bit systems for
3726 // asm.js, heap lengths are limited to 2GB. On 64-bit systems for wasm,
3727 // 32-bit heap lengths are limited to 4GB, and 64-bit heap lengths will be
3728 // limited to something much larger.
3730 void wasmBoundsCheck32(Condition cond
, Register index
,
3731 Register boundsCheckLimit
, Label
* ok
)
3732 DEFINED_ON(arm
, arm64
, mips32
, mips64
, x86_shared
, loong64
, riscv64
,
3735 void wasmBoundsCheck32(Condition cond
, Register index
,
3736 Address boundsCheckLimit
, Label
* ok
)
3737 DEFINED_ON(arm
, arm64
, mips32
, mips64
, x86_shared
, loong64
, riscv64
,
3740 void wasmBoundsCheck64(Condition cond
, Register64 index
,
3741 Register64 boundsCheckLimit
, Label
* ok
)
3742 DEFINED_ON(arm64
, mips64
, x64
, x86
, arm
, loong64
, riscv64
, wasm32
);
3744 void wasmBoundsCheck64(Condition cond
, Register64 index
,
3745 Address boundsCheckLimit
, Label
* ok
)
3746 DEFINED_ON(arm64
, mips64
, x64
, x86
, arm
, loong64
, riscv64
, wasm32
);
3748 // Each wasm load/store instruction appends its own wasm::Trap::OutOfBounds.
3749 void wasmLoad(const wasm::MemoryAccessDesc
& access
, Operand srcAddr
,
3750 AnyRegister out
) DEFINED_ON(x86
, x64
);
3751 void wasmLoadI64(const wasm::MemoryAccessDesc
& access
, Operand srcAddr
,
3752 Register64 out
) DEFINED_ON(x86
, x64
);
3753 void wasmStore(const wasm::MemoryAccessDesc
& access
, AnyRegister value
,
3754 Operand dstAddr
) DEFINED_ON(x86
, x64
);
3755 void wasmStoreI64(const wasm::MemoryAccessDesc
& access
, Register64 value
,
3756 Operand dstAddr
) DEFINED_ON(x86
);
3758 // For all the ARM/MIPS/LOONG64 wasmLoad and wasmStore functions below, `ptr`
3759 // MUST equal `ptrScratch`, and that register will be updated based on
3760 // conditions listed below (where it is only mentioned as `ptr`).
3762 // `ptr` will be updated if access.offset() != 0 or access.type() ==
3764 void wasmLoad(const wasm::MemoryAccessDesc
& access
, Register memoryBase
,
3765 Register ptr
, Register ptrScratch
, AnyRegister output
)
3766 DEFINED_ON(arm
, loong64
, riscv64
, mips_shared
);
3767 void wasmLoadI64(const wasm::MemoryAccessDesc
& access
, Register memoryBase
,
3768 Register ptr
, Register ptrScratch
, Register64 output
)
3769 DEFINED_ON(arm
, mips32
, mips64
, loong64
, riscv64
);
3770 void wasmStore(const wasm::MemoryAccessDesc
& access
, AnyRegister value
,
3771 Register memoryBase
, Register ptr
, Register ptrScratch
)
3772 DEFINED_ON(arm
, loong64
, riscv64
, mips_shared
);
3773 void wasmStoreI64(const wasm::MemoryAccessDesc
& access
, Register64 value
,
3774 Register memoryBase
, Register ptr
, Register ptrScratch
)
3775 DEFINED_ON(arm
, mips32
, mips64
, loong64
, riscv64
);
3777 // These accept general memoryBase + ptr + offset (in `access`); the offset is
3778 // always smaller than the guard region. They will insert an additional add
3779 // if the offset is nonzero, and of course that add may require a temporary
3780 // register for the offset if the offset is large, and instructions to set it
3782 void wasmLoad(const wasm::MemoryAccessDesc
& access
, Register memoryBase
,
3783 Register ptr
, AnyRegister output
) DEFINED_ON(arm64
);
3784 void wasmLoadI64(const wasm::MemoryAccessDesc
& access
, Register memoryBase
,
3785 Register ptr
, Register64 output
) DEFINED_ON(arm64
);
3786 void wasmStore(const wasm::MemoryAccessDesc
& access
, AnyRegister value
,
3787 Register memoryBase
, Register ptr
) DEFINED_ON(arm64
);
3788 void wasmStoreI64(const wasm::MemoryAccessDesc
& access
, Register64 value
,
3789 Register memoryBase
, Register ptr
) DEFINED_ON(arm64
);
3791 // `ptr` will always be updated.
3792 void wasmUnalignedLoad(const wasm::MemoryAccessDesc
& access
,
3793 Register memoryBase
, Register ptr
, Register ptrScratch
,
3794 Register output
, Register tmp
)
3795 DEFINED_ON(mips32
, mips64
);
3797 // MIPS: `ptr` will always be updated.
3798 void wasmUnalignedLoadFP(const wasm::MemoryAccessDesc
& access
,
3799 Register memoryBase
, Register ptr
,
3800 Register ptrScratch
, FloatRegister output
,
3801 Register tmp1
) DEFINED_ON(mips32
, mips64
);
3803 // `ptr` will always be updated.
3804 void wasmUnalignedLoadI64(const wasm::MemoryAccessDesc
& access
,
3805 Register memoryBase
, Register ptr
,
3806 Register ptrScratch
, Register64 output
,
3807 Register tmp
) DEFINED_ON(mips32
, mips64
);
3809 // MIPS: `ptr` will always be updated.
3810 void wasmUnalignedStore(const wasm::MemoryAccessDesc
& access
, Register value
,
3811 Register memoryBase
, Register ptr
,
3812 Register ptrScratch
, Register tmp
)
3813 DEFINED_ON(mips32
, mips64
);
3815 // `ptr` will always be updated.
3816 void wasmUnalignedStoreFP(const wasm::MemoryAccessDesc
& access
,
3817 FloatRegister floatValue
, Register memoryBase
,
3818 Register ptr
, Register ptrScratch
, Register tmp
)
3819 DEFINED_ON(mips32
, mips64
);
3821 // `ptr` will always be updated.
3822 void wasmUnalignedStoreI64(const wasm::MemoryAccessDesc
& access
,
3823 Register64 value
, Register memoryBase
,
3824 Register ptr
, Register ptrScratch
, Register tmp
)
3825 DEFINED_ON(mips32
, mips64
);
3827 // wasm specific methods, used in both the wasm baseline compiler and ion.
3829 // The truncate-to-int32 methods do not bind the rejoin label; clients must
3830 // do so if oolWasmTruncateCheckF64ToI32() can jump to it.
3831 void wasmTruncateDoubleToUInt32(FloatRegister input
, Register output
,
3832 bool isSaturating
, Label
* oolEntry
) PER_ARCH
;
3833 void wasmTruncateDoubleToInt32(FloatRegister input
, Register output
,
3835 Label
* oolEntry
) PER_SHARED_ARCH
;
3836 void oolWasmTruncateCheckF64ToI32(FloatRegister input
, Register output
,
3837 TruncFlags flags
, wasm::BytecodeOffset off
,
3839 DEFINED_ON(arm
, arm64
, x86_shared
, mips_shared
, loong64
, riscv64
, wasm32
);
3841 void wasmTruncateFloat32ToUInt32(FloatRegister input
, Register output
,
3842 bool isSaturating
, Label
* oolEntry
) PER_ARCH
;
3843 void wasmTruncateFloat32ToInt32(FloatRegister input
, Register output
,
3845 Label
* oolEntry
) PER_SHARED_ARCH
;
3846 void oolWasmTruncateCheckF32ToI32(FloatRegister input
, Register output
,
3847 TruncFlags flags
, wasm::BytecodeOffset off
,
3849 DEFINED_ON(arm
, arm64
, x86_shared
, mips_shared
, loong64
, riscv64
, wasm32
);
3851 // The truncate-to-int64 methods will always bind the `oolRejoin` label
3852 // after the last emitted instruction.
3853 void wasmTruncateDoubleToInt64(FloatRegister input
, Register64 output
,
3854 bool isSaturating
, Label
* oolEntry
,
3855 Label
* oolRejoin
, FloatRegister tempDouble
)
3856 DEFINED_ON(arm64
, x86
, x64
, mips64
, loong64
, riscv64
, wasm32
);
3857 void wasmTruncateDoubleToUInt64(FloatRegister input
, Register64 output
,
3858 bool isSaturating
, Label
* oolEntry
,
3859 Label
* oolRejoin
, FloatRegister tempDouble
)
3860 DEFINED_ON(arm64
, x86
, x64
, mips64
, loong64
, riscv64
, wasm32
);
3861 void oolWasmTruncateCheckF64ToI64(FloatRegister input
, Register64 output
,
3862 TruncFlags flags
, wasm::BytecodeOffset off
,
3864 DEFINED_ON(arm
, arm64
, x86_shared
, mips_shared
, loong64
, riscv64
, wasm32
);
3866 void wasmTruncateFloat32ToInt64(FloatRegister input
, Register64 output
,
3867 bool isSaturating
, Label
* oolEntry
,
3868 Label
* oolRejoin
, FloatRegister tempDouble
)
3869 DEFINED_ON(arm64
, x86
, x64
, mips64
, loong64
, riscv64
, wasm32
);
3870 void wasmTruncateFloat32ToUInt64(FloatRegister input
, Register64 output
,
3871 bool isSaturating
, Label
* oolEntry
,
3872 Label
* oolRejoin
, FloatRegister tempDouble
)
3873 DEFINED_ON(arm64
, x86
, x64
, mips64
, loong64
, riscv64
, wasm32
);
3874 void oolWasmTruncateCheckF32ToI64(FloatRegister input
, Register64 output
,
3875 TruncFlags flags
, wasm::BytecodeOffset off
,
3877 DEFINED_ON(arm
, arm64
, x86_shared
, mips_shared
, loong64
, riscv64
, wasm32
);
3879 // This function takes care of loading the callee's instance and pinned regs
3880 // but it is the caller's responsibility to save/restore instance or pinned
3882 CodeOffset
wasmCallImport(const wasm::CallSiteDesc
& desc
,
3883 const wasm::CalleeDesc
& callee
);
3885 #ifdef ENABLE_WASM_TAIL_CALLS
3886 CodeOffset
wasmReturnCallImport(const wasm::CallSiteDesc
& desc
,
3887 const wasm::CalleeDesc
& callee
,
3888 const ReturnCallAdjustmentInfo
& retCallInfo
);
3890 CodeOffset
wasmReturnCall(const wasm::CallSiteDesc
& desc
,
3891 uint32_t funcDefIndex
,
3892 const ReturnCallAdjustmentInfo
& retCallInfo
);
3894 void wasmCollapseFrameSlow(const ReturnCallAdjustmentInfo
& retCallInfo
,
3895 wasm::CallSiteDesc desc
);
3897 void wasmCollapseFrameFast(const ReturnCallAdjustmentInfo
& retCallInfo
);
3899 void wasmCheckSlowCallsite(Register ra
, Label
* notSlow
, Register temp1
,
3901 DEFINED_ON(x86
, x64
, arm
, arm64
, loong64
, mips64
, riscv64
);
3903 void wasmMarkSlowCall()
3904 DEFINED_ON(x86
, x64
, arm
, arm64
, loong64
, mips64
, riscv64
);
3907 // WasmTableCallIndexReg must contain the index of the indirect call. This is
3908 // for wasm calls only.
3910 // Indirect calls use a dual-path mechanism where a run-time test determines
3911 // whether a context switch is needed (slow path) or not (fast path). This
3912 // gives rise to two call instructions, both of which need safe points. As
3913 // per normal, the call offsets are the code offsets at the end of the call
3914 // instructions (the return points).
3916 // `boundsCheckFailedLabel` is non-null iff a bounds check is required.
3917 // `nullCheckFailedLabel` is non-null only on platforms that can't fold the
3918 // null check into the rest of the call instructions.
3919 void wasmCallIndirect(const wasm::CallSiteDesc
& desc
,
3920 const wasm::CalleeDesc
& callee
,
3921 Label
* boundsCheckFailedLabel
,
3922 Label
* nullCheckFailedLabel
,
3923 mozilla::Maybe
<uint32_t> tableSize
,
3924 CodeOffset
* fastCallOffset
, CodeOffset
* slowCallOffset
);
3926 #ifdef ENABLE_WASM_TAIL_CALLS
3927 // WasmTableCallIndexReg must contain the index of the indirect call. This is
3928 // for wasm calls only.
3930 // `boundsCheckFailedLabel` is non-null iff a bounds check is required.
3931 // `nullCheckFailedLabel` is non-null only on platforms that can't fold the
3932 // null check into the rest of the call instructions.
3933 void wasmReturnCallIndirect(const wasm::CallSiteDesc
& desc
,
3934 const wasm::CalleeDesc
& callee
,
3935 Label
* boundsCheckFailedLabel
,
3936 Label
* nullCheckFailedLabel
,
3937 mozilla::Maybe
<uint32_t> tableSize
,
3938 const ReturnCallAdjustmentInfo
& retCallInfo
);
3939 #endif // ENABLE_WASM_TAIL_CALLS
3941 // This function takes care of loading the callee's instance and address from
3943 void wasmCallRef(const wasm::CallSiteDesc
& desc
,
3944 const wasm::CalleeDesc
& callee
, CodeOffset
* fastCallOffset
,
3945 CodeOffset
* slowCallOffset
);
3947 #ifdef ENABLE_WASM_TAIL_CALLS
3948 void wasmReturnCallRef(const wasm::CallSiteDesc
& desc
,
3949 const wasm::CalleeDesc
& callee
,
3950 const ReturnCallAdjustmentInfo
& retCallInfo
);
3951 #endif // ENABLE_WASM_TAIL_CALLS
3953 // WasmTableCallIndexReg must contain the index of the indirect call.
3954 // This is for asm.js calls only.
3955 CodeOffset
asmCallIndirect(const wasm::CallSiteDesc
& desc
,
3956 const wasm::CalleeDesc
& callee
);
3958 // This function takes care of loading the pointer to the current instance
3959 // as the implicit first argument. It preserves instance and pinned registers.
3960 // (instance & pinned regs are non-volatile registers in the system ABI).
3961 CodeOffset
wasmCallBuiltinInstanceMethod(const wasm::CallSiteDesc
& desc
,
3962 const ABIArg
& instanceArg
,
3963 wasm::SymbolicAddress builtin
,
3964 wasm::FailureMode failureMode
);
3966 // Performs a bounds check for ranged wasm operations like memory.fill or
3967 // array.fill. This handles the bizarre edge case in the wasm spec where a
3968 // write to index N is valid as long as the length is zero - despite the index
3969 // itself being out of bounds.
3971 // `length` and `limit` will be unchanged.
3972 void wasmBoundsCheckRange32(Register index
, Register length
, Register limit
,
3974 wasm::BytecodeOffset bytecodeOffset
);
3976 // Returns information about which registers are necessary for a
3977 // branchWasmRefIsSubtype call.
3978 static BranchWasmRefIsSubtypeRegisters
regsForBranchWasmRefIsSubtype(
3979 wasm::RefType type
);
3981 // Perform a subtype check that `ref` is a subtype of `type`, branching to
3982 // `label` depending on `onSuccess`.
3984 // Will select one of the other branchWasmRefIsSubtype* functions depending on
3985 // destType. See each function for the register allocation requirements, as
3986 // well as which registers will be preserved.
3987 void branchWasmRefIsSubtype(Register ref
, wasm::RefType sourceType
,
3988 wasm::RefType destType
, Label
* label
,
3989 bool onSuccess
, Register superSTV
,
3990 Register scratch1
, Register scratch2
);
3992 // Perform a subtype check that `ref` is a subtype of `type`, branching to
3993 // `label` depending on `onSuccess`. `type` must be in the `any` hierarchy.
3995 // `superSTV` is required iff the destination type is a concrete
3996 // type. `scratch1` is required iff the destination type is eq or lower and
3997 // not none. `scratch2` is required iff the destination type is a concrete
3998 // type and its `subTypingDepth` is >= wasm::MinSuperTypeVectorLength. See
3999 // regsForBranchWasmRefIsSubtype.
4001 // `ref` and `superSTV` are preserved. Scratch registers are
4003 void branchWasmRefIsSubtypeAny(Register ref
, wasm::RefType sourceType
,
4004 wasm::RefType destType
, Label
* label
,
4005 bool onSuccess
, Register superSTV
,
4006 Register scratch1
, Register scratch2
);
4008 // Perform a subtype check that `ref` is a subtype of `type`, branching to
4009 // `label` depending on `onSuccess`. `type` must be in the `func` hierarchy.
4011 // `superSTV` and `scratch1` are required iff the destination type
4012 // is a concrete type (not func and not nofunc). `scratch2` is required iff
4013 // the destination type is a concrete type and its `subTypingDepth` is >=
4014 // wasm::MinSuperTypeVectorLength. See regsForBranchWasmRefIsSubtype.
4016 // `ref` and `superSTV` are preserved. Scratch registers are
4018 void branchWasmRefIsSubtypeFunc(Register ref
, wasm::RefType sourceType
,
4019 wasm::RefType destType
, Label
* label
,
4020 bool onSuccess
, Register superSTV
,
4021 Register scratch1
, Register scratch2
);
4023 // Perform a subtype check that `ref` is a subtype of `destType`, branching to
4024 // `label` depending on `onSuccess`. `type` must be in the `extern` hierarchy.
4025 void branchWasmRefIsSubtypeExtern(Register ref
, wasm::RefType sourceType
,
4026 wasm::RefType destType
, Label
* label
,
4029 // Perform a subtype check that `ref` is a subtype of `destType`, branching to
4030 // `label` depending on `onSuccess`. `type` must be in the `exn` hierarchy.
4031 void branchWasmRefIsSubtypeExn(Register ref
, wasm::RefType sourceType
,
4032 wasm::RefType destType
, Label
* label
,
4035 // Perform a subtype check that `subSTV` is a subtype of `superSTV`, branching
4036 // to `label` depending on `onSuccess`. This method is a specialization of the
4037 // general `wasm::TypeDef::isSubTypeOf` method for the case where the
4038 // `superSTV` is statically known, which is the case for all wasm
4041 // `scratch` is required iff the `superDepth` is >=
4042 // wasm::MinSuperTypeVectorLength. `subSTV` is clobbered by this method.
4043 // `superSTV` is preserved.
4044 void branchWasmSTVIsSubtype(Register subSTV
, Register superSTV
,
4045 Register scratch
, uint32_t superDepth
,
4046 Label
* label
, bool onSuccess
);
4048 // Same as branchWasmSTVIsSubtype, but looks up a dynamic position in the
4049 // super type vector.
4051 // `scratch` is always required. `subSTV` and `superDepth` are clobbered.
4052 // `superSTV` is preserved.
4053 void branchWasmSTVIsSubtypeDynamicDepth(Register subSTV
, Register superSTV
,
4054 Register superDepth
, Register scratch
,
4055 Label
* label
, bool onSuccess
);
4057 // Branch if the wasm anyref `src` is or is not the null value.
4058 void branchWasmAnyRefIsNull(bool isNull
, Register src
, Label
* label
);
4059 // Branch if the wasm anyref `src` is or is not an I31.
4060 void branchWasmAnyRefIsI31(bool isI31
, Register src
, Label
* label
);
4061 // Branch if the wasm anyref `src` is or is not a JSObject*.
4062 void branchWasmAnyRefIsObjectOrNull(bool isObject
, Register src
,
4064 // Branch if the wasm anyref `src` is or is not a GC thing.
4065 void branchWasmAnyRefIsGCThing(bool isGCThing
, Register src
, Label
* label
);
4066 // Branch if the wasm anyref `src` is or is not pointing to a nursery cell.
4067 void branchWasmAnyRefIsNurseryCell(bool isNurseryCell
, Register src
,
4068 Register scratch
, Label
* label
);
4070 // Create a wasm i31ref by truncating the 32-bit integer.
4071 void truncate32ToWasmI31Ref(Register src
, Register dest
);
4072 // Convert a wasm i31ref to a signed 32-bit integer.
4073 void convertWasmI31RefTo32Signed(Register src
, Register dest
);
4074 // Convert a wasm i31ref to an unsigned 32-bit integer.
4075 void convertWasmI31RefTo32Unsigned(Register src
, Register dest
);
4077 // Branch if the JS value `src` would need to be boxed out of line to be
4078 // converted to a wasm anyref.
4079 void branchValueConvertsToWasmAnyRefInline(ValueOperand src
,
4080 Register scratchInt
,
4081 FloatRegister scratchFloat
,
4083 // Convert a JS value to a wasm anyref. If the value requires boxing, this
4084 // will branch to `oolConvert`.
4085 void convertValueToWasmAnyRef(ValueOperand src
, Register dest
,
4086 FloatRegister scratchFloat
, Label
* oolConvert
);
4087 // Convert a JS object to a wasm anyref. This cannot fail.
4088 void convertObjectToWasmAnyRef(Register src
, Register dest
);
4089 // Convert a JS string to a wasm anyref. This cannot fail.
4090 void convertStringToWasmAnyRef(Register src
, Register dest
);
4092 // Convert a wasm anyref to a JS value. This cannot fail.
4094 // Due to spectre mitigations, these methods may clobber src.
4095 void convertWasmAnyRefToValue(Register instance
, Register src
,
4096 ValueOperand dst
, Register scratch
);
4097 void convertWasmAnyRefToValue(Register instance
, Register src
,
4098 const Address
& dst
, Register scratch
);
4100 // Branch if the object `src` is or is not a WasmGcObject.
4101 void branchObjectIsWasmGcObject(bool isGcObject
, Register src
,
4102 Register scratch
, Label
* label
);
4104 // `typeDefData` will be preserved. `instance` and `result` may be the same
4105 // register, in which case `instance` will be clobbered.
4106 void wasmNewStructObject(Register instance
, Register result
,
4107 Register typeDefData
, Register temp1
, Register temp2
,
4108 Label
* fail
, gc::AllocKind allocKind
,
4110 // Allocates a wasm array with a dynamic number of elements.
4112 // `numElements` and `typeDefData` will be preserved. `instance` and `result`
4113 // may be the same register, in which case `instance` will be clobbered.
4114 void wasmNewArrayObject(Register instance
, Register result
,
4115 Register numElements
, Register typeDefData
,
4116 Register temp
, Label
* fail
, uint32_t elemSize
,
4118 // Allocates a wasm array with a fixed number of elements.
4120 // `typeDefData` will be preserved. `instance` and `result` may be the same
4121 // register, in which case `instance` will be clobbered.
4122 void wasmNewArrayObjectFixed(Register instance
, Register result
,
4123 Register typeDefData
, Register temp1
,
4124 Register temp2
, Label
* fail
,
4125 uint32_t numElements
, uint32_t storageBytes
,
4128 // This function handles nursery allocations for wasm. For JS, see
4129 // MacroAssembler::bumpPointerAllocate.
4131 // `typeDefData` will be preserved. `instance` and `result` may be the same
4132 // register, in which case `instance` will be clobbered.
4134 // See also the dynamically-sized version,
4135 // MacroAssembler::wasmBumpPointerAllocateDynamic.
4136 void wasmBumpPointerAllocate(Register instance
, Register result
,
4137 Register typeDefData
, Register temp1
,
4138 Register temp2
, Label
* fail
, uint32_t size
);
4139 // This function handles nursery allocations for wasm of dynamic size. For
4140 // fixed-size allocations, see MacroAssembler::wasmBumpPointerAllocate.
4142 // `typeDefData` and `size` will be preserved. `instance` and `result` may be
4143 // the same register, in which case `instance` will be clobbered.
4144 void wasmBumpPointerAllocateDynamic(Register instance
, Register result
,
4145 Register typeDefData
, Register size
,
4146 Register temp1
, Label
* fail
);
4148 // Compute ptr += (indexTemp32 << shift) where shift can be any value < 32.
4149 // May destroy indexTemp32. The value of indexTemp32 must be positive, and it
4150 // is implementation-defined what happens if bits are lost or the value
4151 // becomes negative through the shift. On 64-bit systems, the high 32 bits of
4152 // indexTemp32 must be zero, not garbage.
4153 void shiftIndex32AndAdd(Register indexTemp32
, int shift
,
4154 Register pointer
) PER_SHARED_ARCH
;
4156 // The System ABI frequently states that the high bits of a 64-bit register
4157 // that holds a 32-bit return value are unpredictable, and C++ compilers will
4158 // indeed generate code that leaves garbage in the upper bits.
4160 // Adjust the contents of the 64-bit register `r` to conform to our internal
4161 // convention, which requires predictable high bits. In practice, this means
4162 // that the 32-bit value will be zero-extended or sign-extended to 64 bits as
4163 // appropriate for the platform.
4164 void widenInt32(Register r
) DEFINED_ON(arm64
, x64
, mips64
, loong64
, riscv64
);
4166 // As enterFakeExitFrame(), but using register conventions appropriate for
4168 void enterFakeExitFrameForWasm(Register cxreg
, Register scratch
,
4169 ExitFrameType type
) PER_SHARED_ARCH
;
4172 // ========================================================================
4173 // Barrier functions.
4175 void emitPreBarrierFastPath(JSRuntime
* rt
, MIRType type
, Register temp1
,
4176 Register temp2
, Register temp3
, Label
* noBarrier
);
4179 // ========================================================================
4180 // Clamping functions.
4182 inline void clampIntToUint8(Register reg
) PER_SHARED_ARCH
;
4185 // ========================================================================
4186 // Primitive atomic operations.
4188 // If the access is from JS and the eventual destination of the result is a
4189 // js::Value, it's probably best to use the JS-specific versions of these,
4190 // see further below.
4192 // Temp registers must be defined unless otherwise noted in the per-function
4195 // 8-bit, 16-bit, and 32-bit wide operations.
4197 // The 8-bit and 16-bit operations zero-extend or sign-extend the result to
4198 // 32 bits, according to `type`. On 64-bit systems, the upper 32 bits of the
4199 // result will be zero on some platforms (eg, on x64) and will be the sign
4200 // extension of the lower bits on other platforms (eg, MIPS).
4202 // CompareExchange with memory. Return the value that was in memory,
4203 // whether we wrote or not.
4205 // x86-shared: `output` must be eax.
4206 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
4207 // and 16-bit wide operations.
4209 void compareExchange(Scalar::Type type
, Synchronization sync
,
4210 const Address
& mem
, Register expected
,
4211 Register replacement
, Register output
)
4212 DEFINED_ON(arm
, arm64
, x86_shared
);
4214 void compareExchange(Scalar::Type type
, Synchronization sync
,
4215 const BaseIndex
& mem
, Register expected
,
4216 Register replacement
, Register output
)
4217 DEFINED_ON(arm
, arm64
, x86_shared
);
4219 void compareExchange(Scalar::Type type
, Synchronization sync
,
4220 const Address
& mem
, Register expected
,
4221 Register replacement
, Register valueTemp
,
4222 Register offsetTemp
, Register maskTemp
, Register output
)
4223 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4225 void compareExchange(Scalar::Type type
, Synchronization sync
,
4226 const BaseIndex
& mem
, Register expected
,
4227 Register replacement
, Register valueTemp
,
4228 Register offsetTemp
, Register maskTemp
, Register output
)
4229 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4231 // x86: `expected` and `output` must be edx:eax; `replacement` is ecx:ebx.
4232 // x64: `output` must be rax.
4233 // ARM: Registers must be distinct; `replacement` and `output` must be
4234 // (even,odd) pairs.
4236 void compareExchange64(Synchronization sync
, const Address
& mem
,
4237 Register64 expected
, Register64 replacement
,
4239 DEFINED_ON(arm
, arm64
, x64
, x86
, mips64
, loong64
, riscv64
);
4241 void compareExchange64(Synchronization sync
, const BaseIndex
& mem
,
4242 Register64 expected
, Register64 replacement
,
4244 DEFINED_ON(arm
, arm64
, x64
, x86
, mips64
, loong64
, riscv64
);
4246 // Exchange with memory. Return the value initially in memory.
4247 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
4248 // and 16-bit wide operations.
4250 void atomicExchange(Scalar::Type type
, Synchronization sync
,
4251 const Address
& mem
, Register value
, Register output
)
4252 DEFINED_ON(arm
, arm64
, x86_shared
);
4254 void atomicExchange(Scalar::Type type
, Synchronization sync
,
4255 const BaseIndex
& mem
, Register value
, Register output
)
4256 DEFINED_ON(arm
, arm64
, x86_shared
);
4258 void atomicExchange(Scalar::Type type
, Synchronization sync
,
4259 const Address
& mem
, Register value
, Register valueTemp
,
4260 Register offsetTemp
, Register maskTemp
, Register output
)
4261 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4263 void atomicExchange(Scalar::Type type
, Synchronization sync
,
4264 const BaseIndex
& mem
, Register value
, Register valueTemp
,
4265 Register offsetTemp
, Register maskTemp
, Register output
)
4266 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4268 // x86: `value` must be ecx:ebx; `output` must be edx:eax.
4269 // ARM: `value` and `output` must be distinct and (even,odd) pairs.
4270 // ARM64: `value` and `output` must be distinct.
4272 void atomicExchange64(Synchronization sync
, const Address
& mem
,
4273 Register64 value
, Register64 output
)
4274 DEFINED_ON(arm
, arm64
, x64
, x86
, mips64
, loong64
, riscv64
);
4276 void atomicExchange64(Synchronization sync
, const BaseIndex
& mem
,
4277 Register64 value
, Register64 output
)
4278 DEFINED_ON(arm
, arm64
, x64
, x86
, mips64
, loong64
, riscv64
);
4280 // Read-modify-write with memory. Return the value in memory before the
4284 // For 8-bit operations, `value` and `output` must have a byte subregister.
4285 // For Add and Sub, `temp` must be invalid.
4286 // For And, Or, and Xor, `output` must be eax and `temp` must have a byte
4289 // ARM: Registers `value` and `output` must differ.
4290 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
4291 // and 16-bit wide operations; `value` and `output` must differ.
4293 void atomicFetchOp(Scalar::Type type
, Synchronization sync
, AtomicOp op
,
4294 Register value
, const Address
& mem
, Register temp
,
4295 Register output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4297 void atomicFetchOp(Scalar::Type type
, Synchronization sync
, AtomicOp op
,
4298 Imm32 value
, const Address
& mem
, Register temp
,
4299 Register output
) DEFINED_ON(x86_shared
);
4301 void atomicFetchOp(Scalar::Type type
, Synchronization sync
, AtomicOp op
,
4302 Register value
, const BaseIndex
& mem
, Register temp
,
4303 Register output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4305 void atomicFetchOp(Scalar::Type type
, Synchronization sync
, AtomicOp op
,
4306 Imm32 value
, const BaseIndex
& mem
, Register temp
,
4307 Register output
) DEFINED_ON(x86_shared
);
4309 void atomicFetchOp(Scalar::Type type
, Synchronization sync
, AtomicOp op
,
4310 Register value
, const Address
& mem
, Register valueTemp
,
4311 Register offsetTemp
, Register maskTemp
, Register output
)
4312 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4314 void atomicFetchOp(Scalar::Type type
, Synchronization sync
, AtomicOp op
,
4315 Register value
, const BaseIndex
& mem
, Register valueTemp
,
4316 Register offsetTemp
, Register maskTemp
, Register output
)
4317 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4320 // `temp` must be ecx:ebx; `output` must be edx:eax.
4322 // For Add and Sub, `temp` is ignored.
4323 // For And, Or, and Xor, `output` must be rax.
4325 // `temp` and `output` must be (even,odd) pairs and distinct from `value`.
4327 // Registers `value`, `temp`, and `output` must all differ.
4329 void atomicFetchOp64(Synchronization sync
, AtomicOp op
, Register64 value
,
4330 const Address
& mem
, Register64 temp
, Register64 output
)
4331 DEFINED_ON(arm
, arm64
, x64
, mips64
, loong64
, riscv64
);
4333 void atomicFetchOp64(Synchronization sync
, AtomicOp op
, const Address
& value
,
4334 const Address
& mem
, Register64 temp
, Register64 output
)
4337 void atomicFetchOp64(Synchronization sync
, AtomicOp op
, Register64 value
,
4338 const BaseIndex
& mem
, Register64 temp
, Register64 output
)
4339 DEFINED_ON(arm
, arm64
, x64
, mips64
, loong64
, riscv64
);
4341 void atomicFetchOp64(Synchronization sync
, AtomicOp op
, const Address
& value
,
4342 const BaseIndex
& mem
, Register64 temp
, Register64 output
)
4346 // `value` can be any register.
4348 // `temp` must be an (even,odd) pair and distinct from `value`.
4350 // Registers `value` and `temp` must differ.
4352 void atomicEffectOp64(Synchronization sync
, AtomicOp op
, Register64 value
,
4353 const Address
& mem
) DEFINED_ON(x64
);
4355 void atomicEffectOp64(Synchronization sync
, AtomicOp op
, Register64 value
,
4356 const Address
& mem
, Register64 temp
)
4357 DEFINED_ON(arm
, arm64
, mips64
, loong64
, riscv64
);
4359 void atomicEffectOp64(Synchronization sync
, AtomicOp op
, Register64 value
,
4360 const BaseIndex
& mem
) DEFINED_ON(x64
);
4362 void atomicEffectOp64(Synchronization sync
, AtomicOp op
, Register64 value
,
4363 const BaseIndex
& mem
, Register64 temp
)
4364 DEFINED_ON(arm
, arm64
, mips64
, loong64
, riscv64
);
4366 // 64-bit atomic load. On 64-bit systems, use regular load with
4367 // Synchronization::Load, not this method.
4369 // x86: `temp` must be ecx:ebx; `output` must be edx:eax.
4370 // ARM: `output` must be (even,odd) pair.
4372 void atomicLoad64(Synchronization sync
, const Address
& mem
, Register64 temp
,
4373 Register64 output
) DEFINED_ON(x86
);
4375 void atomicLoad64(Synchronization sync
, const BaseIndex
& mem
, Register64 temp
,
4376 Register64 output
) DEFINED_ON(x86
);
4378 void atomicLoad64(Synchronization sync
, const Address
& mem
, Register64 output
)
4381 void atomicLoad64(Synchronization sync
, const BaseIndex
& mem
,
4382 Register64 output
) DEFINED_ON(arm
);
4384 // 64-bit atomic store. On 64-bit systems, use regular store with
4385 // Synchronization::Store, not this method.
4387 // x86: `value` must be ecx:ebx; `temp` must be edx:eax.
4388 // ARM: `value` and `temp` must be (even,odd) pairs.
4390 void atomicStore64(Synchronization sync
, const Address
& mem
, Register64 value
,
4391 Register64 temp
) DEFINED_ON(x86
, arm
);
4393 void atomicStore64(Synchronization sync
, const BaseIndex
& mem
,
4394 Register64 value
, Register64 temp
) DEFINED_ON(x86
, arm
);
4396 // ========================================================================
4397 // Wasm atomic operations.
4399 // Constraints, when omitted, are exactly as for the primitive operations
4402 void wasmCompareExchange(const wasm::MemoryAccessDesc
& access
,
4403 const Address
& mem
, Register expected
,
4404 Register replacement
, Register output
)
4405 DEFINED_ON(arm
, arm64
, x86_shared
);
4407 void wasmCompareExchange(const wasm::MemoryAccessDesc
& access
,
4408 const BaseIndex
& mem
, Register expected
,
4409 Register replacement
, Register output
)
4410 DEFINED_ON(arm
, arm64
, x86_shared
);
4412 void wasmCompareExchange(const wasm::MemoryAccessDesc
& access
,
4413 const Address
& mem
, Register expected
,
4414 Register replacement
, Register valueTemp
,
4415 Register offsetTemp
, Register maskTemp
,
4417 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4419 void wasmCompareExchange(const wasm::MemoryAccessDesc
& access
,
4420 const BaseIndex
& mem
, Register expected
,
4421 Register replacement
, Register valueTemp
,
4422 Register offsetTemp
, Register maskTemp
,
4424 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4426 void wasmAtomicExchange(const wasm::MemoryAccessDesc
& access
,
4427 const Address
& mem
, Register value
, Register output
)
4428 DEFINED_ON(arm
, arm64
, x86_shared
);
4430 void wasmAtomicExchange(const wasm::MemoryAccessDesc
& access
,
4431 const BaseIndex
& mem
, Register value
, Register output
)
4432 DEFINED_ON(arm
, arm64
, x86_shared
);
4434 void wasmAtomicExchange(const wasm::MemoryAccessDesc
& access
,
4435 const Address
& mem
, Register value
,
4436 Register valueTemp
, Register offsetTemp
,
4437 Register maskTemp
, Register output
)
4438 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4440 void wasmAtomicExchange(const wasm::MemoryAccessDesc
& access
,
4441 const BaseIndex
& mem
, Register value
,
4442 Register valueTemp
, Register offsetTemp
,
4443 Register maskTemp
, Register output
)
4444 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4446 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4447 Register value
, const Address
& mem
, Register temp
,
4448 Register output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4450 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4451 Imm32 value
, const Address
& mem
, Register temp
,
4452 Register output
) DEFINED_ON(x86_shared
);
4454 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4455 Register value
, const BaseIndex
& mem
, Register temp
,
4456 Register output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4458 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4459 Imm32 value
, const BaseIndex
& mem
, Register temp
,
4460 Register output
) DEFINED_ON(x86_shared
);
4462 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4463 Register value
, const Address
& mem
, Register valueTemp
,
4464 Register offsetTemp
, Register maskTemp
,
4466 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4468 void wasmAtomicFetchOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4469 Register value
, const BaseIndex
& mem
,
4470 Register valueTemp
, Register offsetTemp
,
4471 Register maskTemp
, Register output
)
4472 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4474 // Read-modify-write with memory. Return no value.
4476 // MIPS: `valueTemp`, `offsetTemp` and `maskTemp` must be defined for 8-bit
4477 // and 16-bit wide operations.
4479 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4480 Register value
, const Address
& mem
, Register temp
)
4481 DEFINED_ON(arm
, arm64
, x86_shared
);
4483 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4484 Imm32 value
, const Address
& mem
, Register temp
)
4485 DEFINED_ON(x86_shared
);
4487 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4488 Register value
, const BaseIndex
& mem
, Register temp
)
4489 DEFINED_ON(arm
, arm64
, x86_shared
);
4491 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4492 Imm32 value
, const BaseIndex
& mem
, Register temp
)
4493 DEFINED_ON(x86_shared
);
4495 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4496 Register value
, const Address
& mem
,
4497 Register valueTemp
, Register offsetTemp
,
4499 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4501 void wasmAtomicEffectOp(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4502 Register value
, const BaseIndex
& mem
,
4503 Register valueTemp
, Register offsetTemp
,
4505 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4507 // 64-bit wide operations.
4509 // 64-bit atomic load. On 64-bit systems, use regular wasm load with
4510 // Synchronization::Load, not this method.
4512 // x86: `temp` must be ecx:ebx; `output` must be edx:eax.
4513 // ARM: `temp` should be invalid; `output` must be (even,odd) pair.
4514 // MIPS32: `temp` should be invalid.
4516 void wasmAtomicLoad64(const wasm::MemoryAccessDesc
& access
,
4517 const Address
& mem
, Register64 temp
, Register64 output
)
4518 DEFINED_ON(arm
, mips32
, x86
, wasm32
);
4520 void wasmAtomicLoad64(const wasm::MemoryAccessDesc
& access
,
4521 const BaseIndex
& mem
, Register64 temp
,
4522 Register64 output
) DEFINED_ON(arm
, mips32
, x86
, wasm32
);
4524 // x86: `expected` must be the same as `output`, and must be edx:eax.
4525 // x86: `replacement` must be ecx:ebx.
4526 // x64: `output` must be rax.
4527 // ARM: Registers must be distinct; `replacement` and `output` must be
4528 // (even,odd) pairs.
4529 // ARM64: The base register in `mem` must not overlap `output`.
4530 // MIPS: Registers must be distinct.
4532 void wasmCompareExchange64(const wasm::MemoryAccessDesc
& access
,
4533 const Address
& mem
, Register64 expected
,
4534 Register64 replacement
,
4535 Register64 output
) PER_ARCH
;
4537 void wasmCompareExchange64(const wasm::MemoryAccessDesc
& access
,
4538 const BaseIndex
& mem
, Register64 expected
,
4539 Register64 replacement
,
4540 Register64 output
) PER_ARCH
;
4542 // x86: `value` must be ecx:ebx; `output` must be edx:eax.
4543 // ARM: Registers must be distinct; `value` and `output` must be (even,odd)
4545 // MIPS: Registers must be distinct.
4547 void wasmAtomicExchange64(const wasm::MemoryAccessDesc
& access
,
4548 const Address
& mem
, Register64 value
,
4549 Register64 output
) PER_ARCH
;
4551 void wasmAtomicExchange64(const wasm::MemoryAccessDesc
& access
,
4552 const BaseIndex
& mem
, Register64 value
,
4553 Register64 output
) PER_ARCH
;
4555 // x86: `output` must be edx:eax, `temp` must be ecx:ebx.
4556 // x64: For And, Or, and Xor `output` must be rax.
4557 // ARM: Registers must be distinct; `temp` and `output` must be (even,odd)
4559 // MIPS: Registers must be distinct.
4560 // MIPS32: `temp` should be invalid.
4562 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4563 Register64 value
, const Address
& mem
,
4564 Register64 temp
, Register64 output
)
4565 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, x64
);
4567 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4568 Register64 value
, const BaseIndex
& mem
,
4569 Register64 temp
, Register64 output
)
4570 DEFINED_ON(arm
, arm64
, mips32
, mips64
, loong64
, riscv64
, x64
);
4572 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4573 const Address
& value
, const Address
& mem
,
4574 Register64 temp
, Register64 output
) DEFINED_ON(x86
);
4576 void wasmAtomicFetchOp64(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4577 const Address
& value
, const BaseIndex
& mem
,
4578 Register64 temp
, Register64 output
) DEFINED_ON(x86
);
4580 // Here `value` can be any register.
4582 void wasmAtomicEffectOp64(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4583 Register64 value
, const BaseIndex
& mem
)
4586 void wasmAtomicEffectOp64(const wasm::MemoryAccessDesc
& access
, AtomicOp op
,
4587 Register64 value
, const BaseIndex
& mem
,
4588 Register64 temp
) DEFINED_ON(arm64
);
4590 // ========================================================================
4591 // JS atomic operations.
4593 // Here the arrayType must be a type that is valid for JS. As of 2017 that
4594 // is an 8-bit, 16-bit, or 32-bit integer type.
4596 // If arrayType is Scalar::Uint32 then:
4598 // - `output` must be a float register
4599 // - if the operation takes one temp register then `temp` must be defined
4600 // - if the operation takes two temp registers then `temp2` must be defined.
4602 // Otherwise `output` must be a GPR and `temp`/`temp2` should be InvalidReg.
4603 // (`temp1` must always be valid.)
4605 // For additional register constraints, see the primitive 32-bit operations
4606 // and/or wasm operations above.
4608 void compareExchangeJS(Scalar::Type arrayType
, Synchronization sync
,
4609 const Address
& mem
, Register expected
,
4610 Register replacement
, Register temp
,
4611 AnyRegister output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4613 void compareExchangeJS(Scalar::Type arrayType
, Synchronization sync
,
4614 const BaseIndex
& mem
, Register expected
,
4615 Register replacement
, Register temp
,
4616 AnyRegister output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4618 void compareExchangeJS(Scalar::Type arrayType
, Synchronization sync
,
4619 const Address
& mem
, Register expected
,
4620 Register replacement
, Register valueTemp
,
4621 Register offsetTemp
, Register maskTemp
, Register temp
,
4623 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4625 void compareExchangeJS(Scalar::Type arrayType
, Synchronization sync
,
4626 const BaseIndex
& mem
, Register expected
,
4627 Register replacement
, Register valueTemp
,
4628 Register offsetTemp
, Register maskTemp
, Register temp
,
4630 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4632 void atomicExchangeJS(Scalar::Type arrayType
, Synchronization sync
,
4633 const Address
& mem
, Register value
, Register temp
,
4634 AnyRegister output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4636 void atomicExchangeJS(Scalar::Type arrayType
, Synchronization sync
,
4637 const BaseIndex
& mem
, Register value
, Register temp
,
4638 AnyRegister output
) DEFINED_ON(arm
, arm64
, x86_shared
);
4640 void atomicExchangeJS(Scalar::Type arrayType
, Synchronization sync
,
4641 const Address
& mem
, Register value
, Register valueTemp
,
4642 Register offsetTemp
, Register maskTemp
, Register temp
,
4644 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4646 void atomicExchangeJS(Scalar::Type arrayType
, Synchronization sync
,
4647 const BaseIndex
& mem
, Register value
,
4648 Register valueTemp
, Register offsetTemp
,
4649 Register maskTemp
, Register temp
, AnyRegister output
)
4650 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4652 void atomicFetchOpJS(Scalar::Type arrayType
, Synchronization sync
,
4653 AtomicOp op
, Register value
, const Address
& mem
,
4654 Register temp1
, Register temp2
, AnyRegister output
)
4655 DEFINED_ON(arm
, arm64
, x86_shared
);
4657 void atomicFetchOpJS(Scalar::Type arrayType
, Synchronization sync
,
4658 AtomicOp op
, Register value
, const BaseIndex
& mem
,
4659 Register temp1
, Register temp2
, AnyRegister output
)
4660 DEFINED_ON(arm
, arm64
, x86_shared
);
4662 void atomicFetchOpJS(Scalar::Type arrayType
, Synchronization sync
,
4663 AtomicOp op
, Imm32 value
, const Address
& mem
,
4664 Register temp1
, Register temp2
, AnyRegister output
)
4665 DEFINED_ON(x86_shared
);
4667 void atomicFetchOpJS(Scalar::Type arrayType
, Synchronization sync
,
4668 AtomicOp op
, Imm32 value
, const BaseIndex
& mem
,
4669 Register temp1
, Register temp2
, AnyRegister output
)
4670 DEFINED_ON(x86_shared
);
4672 void atomicFetchOpJS(Scalar::Type arrayType
, Synchronization sync
,
4673 AtomicOp op
, Register value
, const Address
& mem
,
4674 Register valueTemp
, Register offsetTemp
,
4675 Register maskTemp
, Register temp
, AnyRegister output
)
4676 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4678 void atomicFetchOpJS(Scalar::Type arrayType
, Synchronization sync
,
4679 AtomicOp op
, Register value
, const BaseIndex
& mem
,
4680 Register valueTemp
, Register offsetTemp
,
4681 Register maskTemp
, Register temp
, AnyRegister output
)
4682 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4684 void atomicEffectOpJS(Scalar::Type arrayType
, Synchronization sync
,
4685 AtomicOp op
, Register value
, const Address
& mem
,
4686 Register temp
) DEFINED_ON(arm
, arm64
, x86_shared
);
4688 void atomicEffectOpJS(Scalar::Type arrayType
, Synchronization sync
,
4689 AtomicOp op
, Register value
, const BaseIndex
& mem
,
4690 Register temp
) DEFINED_ON(arm
, arm64
, x86_shared
);
4692 void atomicEffectOpJS(Scalar::Type arrayType
, Synchronization sync
,
4693 AtomicOp op
, Imm32 value
, const Address
& mem
,
4694 Register temp
) DEFINED_ON(x86_shared
);
4696 void atomicEffectOpJS(Scalar::Type arrayType
, Synchronization sync
,
4697 AtomicOp op
, Imm32 value
, const BaseIndex
& mem
,
4698 Register temp
) DEFINED_ON(x86_shared
);
4700 void atomicEffectOpJS(Scalar::Type arrayType
, Synchronization sync
,
4701 AtomicOp op
, Register value
, const Address
& mem
,
4702 Register valueTemp
, Register offsetTemp
,
4704 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4706 void atomicEffectOpJS(Scalar::Type arrayType
, Synchronization sync
,
4707 AtomicOp op
, Register value
, const BaseIndex
& mem
,
4708 Register valueTemp
, Register offsetTemp
,
4710 DEFINED_ON(mips_shared
, loong64
, riscv64
);
4712 void atomicIsLockFreeJS(Register value
, Register output
);
4714 // ========================================================================
4715 // Spectre Mitigations.
4717 // Spectre attacks are side-channel attacks based on cache pollution or
4718 // slow-execution of some instructions. We have multiple spectre mitigations
4721 // - Stop speculative executions, with memory barriers. Memory barriers
4722 // force all branches depending on loads to be resolved, and thus
4723 // resolve all miss-speculated paths.
4725 // - Use conditional move instructions. Some CPUs have a branch predictor,
4726 // and not a flag predictor. In such cases, using a conditional move
4727 // instruction to zero some pointer/index is enough to add a
4728 // data-dependency which prevents any futher executions until the load is
4731 void spectreMaskIndex32(Register index
, Register length
, Register output
);
4732 void spectreMaskIndex32(Register index
, const Address
& length
,
4734 void spectreMaskIndexPtr(Register index
, Register length
, Register output
);
4735 void spectreMaskIndexPtr(Register index
, const Address
& length
,
4738 // The length must be a power of two. Performs a bounds check and Spectre
4740 void boundsCheck32PowerOfTwo(Register index
, uint32_t length
, Label
* failure
);
4742 void speculationBarrier() PER_SHARED_ARCH
;
4744 //}}} check_macroassembler_decl_style
4746 // Unsafe here means the caller is responsible for Spectre mitigations if
4747 // needed. Prefer branchTestObjClass or one of the other masm helpers!
4748 inline void loadObjClassUnsafe(Register obj
, Register dest
);
4750 template <typename EmitPreBarrier
>
4751 inline void storeObjShape(Register shape
, Register obj
,
4752 EmitPreBarrier emitPreBarrier
);
4753 template <typename EmitPreBarrier
>
4754 inline void storeObjShape(Shape
* shape
, Register obj
,
4755 EmitPreBarrier emitPreBarrier
);
4757 inline void loadObjProto(Register obj
, Register dest
);
4759 inline void loadStringLength(Register str
, Register dest
);
4761 void loadStringChars(Register str
, Register dest
, CharEncoding encoding
);
4763 void loadNonInlineStringChars(Register str
, Register dest
,
4764 CharEncoding encoding
);
4765 void loadNonInlineStringCharsForStore(Register str
, Register dest
);
4766 void storeNonInlineStringChars(Register chars
, Register str
);
4768 void loadInlineStringChars(Register str
, Register dest
,
4769 CharEncoding encoding
);
4770 void loadInlineStringCharsForStore(Register str
, Register dest
);
4773 enum class CharKind
{ CharCode
, CodePoint
};
4775 void branchIfMaybeSplitSurrogatePair(Register leftChild
, Register index
,
4776 Register scratch
, Label
* maybeSplit
,
4779 void loadRopeChild(CharKind kind
, Register str
, Register index
,
4780 Register output
, Register maybeScratch
, Label
* isLinear
,
4781 Label
* splitSurrogate
);
4783 void branchIfCanLoadStringChar(CharKind kind
, Register str
, Register index
,
4784 Register scratch
, Register maybeScratch
,
4786 void branchIfNotCanLoadStringChar(CharKind kind
, Register str
, Register index
,
4787 Register scratch
, Register maybeScratch
,
4790 void loadStringChar(CharKind kind
, Register str
, Register index
,
4791 Register output
, Register scratch1
, Register scratch2
,
4795 void branchIfCanLoadStringChar(Register str
, Register index
, Register scratch
,
4797 branchIfCanLoadStringChar(CharKind::CharCode
, str
, index
, scratch
,
4800 void branchIfNotCanLoadStringChar(Register str
, Register index
,
4801 Register scratch
, Label
* label
) {
4802 branchIfNotCanLoadStringChar(CharKind::CharCode
, str
, index
, scratch
,
4806 void branchIfCanLoadStringCodePoint(Register str
, Register index
,
4807 Register scratch1
, Register scratch2
,
4809 branchIfCanLoadStringChar(CharKind::CodePoint
, str
, index
, scratch1
,
4812 void branchIfNotCanLoadStringCodePoint(Register str
, Register index
,
4813 Register scratch1
, Register scratch2
,
4815 branchIfNotCanLoadStringChar(CharKind::CodePoint
, str
, index
, scratch1
,
4819 void loadStringChar(Register str
, Register index
, Register output
,
4820 Register scratch1
, Register scratch2
, Label
* fail
) {
4821 loadStringChar(CharKind::CharCode
, str
, index
, output
, scratch1
, scratch2
,
4825 void loadStringChar(Register str
, int32_t index
, Register output
,
4826 Register scratch1
, Register scratch2
, Label
* fail
);
4828 void loadStringCodePoint(Register str
, Register index
, Register output
,
4829 Register scratch1
, Register scratch2
, Label
* fail
) {
4830 loadStringChar(CharKind::CodePoint
, str
, index
, output
, scratch1
, scratch2
,
4834 void loadRopeLeftChild(Register str
, Register dest
);
4835 void loadRopeRightChild(Register str
, Register dest
);
4836 void storeRopeChildren(Register left
, Register right
, Register str
);
4838 void loadDependentStringBase(Register str
, Register dest
);
4839 void storeDependentStringBase(Register base
, Register str
);
4841 void loadStringIndexValue(Register str
, Register dest
, Label
* fail
);
4844 * Store the character in |src| to |dest|.
4846 template <typename T
>
4847 void storeChar(const T
& src
, Address dest
, CharEncoding encoding
) {
4848 if (encoding
== CharEncoding::Latin1
) {
4856 * Load the character at |src| into |dest|.
4858 template <typename T
>
4859 void loadChar(const T
& src
, Register dest
, CharEncoding encoding
) {
4860 if (encoding
== CharEncoding::Latin1
) {
4861 load8ZeroExtend(src
, dest
);
4863 load16ZeroExtend(src
, dest
);
4868 * Load the character at |chars[index + offset]| into |dest|. The optional
4869 * offset argument is not scaled to the character encoding.
4871 void loadChar(Register chars
, Register index
, Register dest
,
4872 CharEncoding encoding
, int32_t offset
= 0);
4875 * Add |index| to |chars| so that |chars| now points at |chars[index]|.
4877 void addToCharPtr(Register chars
, Register index
, CharEncoding encoding
);
4880 * Branch if |src| is not a lead surrogate character.
4882 void branchIfNotLeadSurrogate(Register src
, Label
* label
);
4885 enum class SurrogateChar
{ Lead
, Trail
};
4886 void branchSurrogate(Assembler::Condition cond
, Register src
,
4887 Register scratch
, Label
* label
,
4888 SurrogateChar surrogateChar
);
4892 * Branch if |src| is a lead surrogate character.
4894 void branchIfLeadSurrogate(Register src
, Register scratch
, Label
* label
) {
4895 branchSurrogate(Assembler::Equal
, src
, scratch
, label
, SurrogateChar::Lead
);
4899 * Branch if |src| is not a lead surrogate character.
4901 void branchIfNotLeadSurrogate(Register src
, Register scratch
, Label
* label
) {
4902 branchSurrogate(Assembler::NotEqual
, src
, scratch
, label
,
4903 SurrogateChar::Lead
);
4907 * Branch if |src| is not a trail surrogate character.
4909 void branchIfNotTrailSurrogate(Register src
, Register scratch
, Label
* label
) {
4910 branchSurrogate(Assembler::NotEqual
, src
, scratch
, label
,
4911 SurrogateChar::Trail
);
4915 void loadStringFromUnit(Register unit
, Register dest
,
4916 const StaticStrings
& staticStrings
);
4917 void loadLengthTwoString(Register c1
, Register c2
, Register dest
,
4918 const StaticStrings
& staticStrings
);
4922 * Lookup the length-one string from the static strings cache.
4924 void lookupStaticString(Register ch
, Register dest
,
4925 const StaticStrings
& staticStrings
);
4928 * Lookup the length-one string from the static strings cache. Jumps to |fail|
4929 * when the string wasn't found in the strings cache.
4931 void lookupStaticString(Register ch
, Register dest
,
4932 const StaticStrings
& staticStrings
, Label
* fail
);
4935 * Lookup the length-two string from the static strings cache. Jumps to |fail|
4936 * when the string wasn't found in the strings cache.
4938 * Clobbers |ch1| and |ch2|.
4940 void lookupStaticString(Register ch1
, Register ch2
, Register dest
,
4941 const StaticStrings
& staticStrings
, Label
* fail
);
4944 * Lookup the integer string from the static integer strings cache. Jumps to
4945 * |fail| when the string wasn't found in the strings cache.
4947 void lookupStaticIntString(Register integer
, Register dest
, Register scratch
,
4948 const StaticStrings
& staticStrings
, Label
* fail
);
4949 void lookupStaticIntString(Register integer
, Register dest
,
4950 const StaticStrings
& staticStrings
, Label
* fail
) {
4951 lookupStaticIntString(integer
, dest
, dest
, staticStrings
, fail
);
4955 * Load the string representation of |input| in base |base|. Jumps to |fail|
4956 * when the string representation needs to be allocated dynamically.
4958 void loadInt32ToStringWithBase(Register input
, Register base
, Register dest
,
4959 Register scratch1
, Register scratch2
,
4960 const StaticStrings
& staticStrings
,
4961 const LiveRegisterSet
& volatileRegs
,
4962 bool lowerCase
, Label
* fail
);
4963 void loadInt32ToStringWithBase(Register input
, int32_t base
, Register dest
,
4964 Register scratch1
, Register scratch2
,
4965 const StaticStrings
& staticStrings
,
4966 bool lowerCase
, Label
* fail
);
4969 * Load the BigInt digits from |bigInt| into |digits|.
4971 void loadBigIntDigits(Register bigInt
, Register digits
);
4974 * Load the first [u]int64 value from |bigInt| into |dest|.
4976 void loadBigInt64(Register bigInt
, Register64 dest
);
4979 * Load the first digit from |bigInt| into |dest|. Handles the case when the
4980 * BigInt digits length is zero.
4982 * Note: A BigInt digit is a pointer-sized value.
4984 void loadFirstBigIntDigitOrZero(Register bigInt
, Register dest
);
4987 * Load the number stored in |bigInt| into |dest|. Handles the case when the
4988 * BigInt digits length is zero. Jumps to |fail| when the number can't be
4989 * saved into a single pointer-sized register.
4991 void loadBigInt(Register bigInt
, Register dest
, Label
* fail
);
4994 * Load the number stored in |bigInt| into |dest|. Doesn't handle the case
4995 * when the BigInt digits length is zero. Jumps to |fail| when the number
4996 * can't be saved into a single pointer-sized register.
4998 void loadBigIntNonZero(Register bigInt
, Register dest
, Label
* fail
);
5001 * Load the absolute number stored in |bigInt| into |dest|. Handles the case
5002 * when the BigInt digits length is zero. Jumps to |fail| when the number
5003 * can't be saved into a single pointer-sized register.
5005 void loadBigIntAbsolute(Register bigInt
, Register dest
, Label
* fail
);
5008 * In-place modifies the BigInt digit to a signed pointer-sized value. Jumps
5009 * to |fail| when the digit exceeds the representable range.
5011 void bigIntDigitToSignedPtr(Register bigInt
, Register digit
, Label
* fail
);
5014 * Initialize a BigInt from |val|. Clobbers |val|!
5016 void initializeBigInt64(Scalar::Type type
, Register bigInt
, Register64 val
);
5019 * Initialize a BigInt from the signed, pointer-sized register |val|.
5022 void initializeBigInt(Register bigInt
, Register val
);
5025 * Initialize a BigInt from the pointer-sized register |val|.
5027 void initializeBigIntAbsolute(Register bigInt
, Register val
);
5030 * Copy a BigInt. Jumps to |fail| on allocation failure or when the BigInt
5031 * digits need to be heap allocated.
5033 void copyBigIntWithInlineDigits(Register src
, Register dest
, Register temp
,
5034 gc::Heap initialHeap
, Label
* fail
);
5037 * Compare a BigInt and an Int32 value. Falls through to the false case.
5039 void compareBigIntAndInt32(JSOp op
, Register bigInt
, Register int32
,
5040 Register scratch1
, Register scratch2
,
5041 Label
* ifTrue
, Label
* ifFalse
);
5044 * Compare two BigInts for equality. Falls through if both BigInts are equal
5047 * - When we jump to |notSameLength|, |temp1| holds the length of the right
5049 * - When we jump to |notSameDigit|, |temp2| points to the current digit of
5050 * the left operand and |temp4| holds the current digit of the right
5053 void equalBigInts(Register left
, Register right
, Register temp1
,
5054 Register temp2
, Register temp3
, Register temp4
,
5055 Label
* notSameSign
, Label
* notSameLength
,
5056 Label
* notSameDigit
);
5058 void loadJSContext(Register dest
);
5060 void loadGlobalObjectData(Register dest
);
5062 void loadRealmFuse(RealmFuses::FuseIndex index
, Register dest
);
5064 void switchToRealm(Register realm
);
5065 void switchToRealm(const void* realm
, Register scratch
);
5066 void switchToObjectRealm(Register obj
, Register scratch
);
5067 void switchToBaselineFrameRealm(Register scratch
);
5068 void switchToWasmInstanceRealm(Register scratch1
, Register scratch2
);
5069 void debugAssertContextRealm(const void* realm
, Register scratch
);
5071 void loadJitActivation(Register dest
);
5073 void guardSpecificAtom(Register str
, JSAtom
* atom
, Register scratch
,
5074 const LiveRegisterSet
& volatileRegs
, Label
* fail
);
5076 void guardStringToInt32(Register str
, Register output
, Register scratch
,
5077 LiveRegisterSet volatileRegs
, Label
* fail
);
5079 template <typename T
>
5080 void loadTypedOrValue(const T
& src
, TypedOrValueRegister dest
) {
5081 if (dest
.hasValue()) {
5082 loadValue(src
, dest
.valueReg());
5084 loadUnboxedValue(src
, dest
.type(), dest
.typedReg());
5088 template <typename T
>
5089 void storeTypedOrValue(TypedOrValueRegister src
, const T
& dest
) {
5090 if (src
.hasValue()) {
5091 storeValue(src
.valueReg(), dest
);
5092 } else if (IsFloatingPointType(src
.type())) {
5093 FloatRegister reg
= src
.typedReg().fpu();
5094 if (src
.type() == MIRType::Float32
) {
5095 ScratchDoubleScope
fpscratch(*this);
5096 convertFloat32ToDouble(reg
, fpscratch
);
5097 boxDouble(fpscratch
, dest
);
5099 boxDouble(reg
, dest
);
5102 storeValue(ValueTypeFromMIRType(src
.type()), src
.typedReg().gpr(), dest
);
5106 template <typename T
>
5107 void storeConstantOrRegister(const ConstantOrRegister
& src
, const T
& dest
) {
5108 if (src
.constant()) {
5109 storeValue(src
.value(), dest
);
5111 storeTypedOrValue(src
.reg(), dest
);
5115 void storeCallPointerResult(Register reg
) {
5116 if (reg
!= ReturnReg
) {
5117 mov(ReturnReg
, reg
);
5121 inline void storeCallBoolResult(Register reg
);
5122 inline void storeCallInt32Result(Register reg
);
5124 void storeCallFloatResult(FloatRegister reg
) {
5125 if (reg
!= ReturnDoubleReg
) {
5126 moveDouble(ReturnDoubleReg
, reg
);
5130 inline void storeCallResultValue(AnyRegister dest
, JSValueType type
);
5132 void storeCallResultValue(ValueOperand dest
) {
5133 #if defined(JS_NUNBOX32)
5134 // reshuffle the return registers used for a call result to store into
5135 // dest, using ReturnReg as a scratch register if necessary. This must
5136 // only be called after returning from a call, at a point when the
5137 // return register is not live. XXX would be better to allow wrappers
5138 // to store the return value to different places.
5139 if (dest
.typeReg() == JSReturnReg_Data
) {
5140 if (dest
.payloadReg() == JSReturnReg_Type
) {
5141 // swap the two registers.
5142 mov(JSReturnReg_Type
, ReturnReg
);
5143 mov(JSReturnReg_Data
, JSReturnReg_Type
);
5144 mov(ReturnReg
, JSReturnReg_Data
);
5146 mov(JSReturnReg_Data
, dest
.payloadReg());
5147 mov(JSReturnReg_Type
, dest
.typeReg());
5150 mov(JSReturnReg_Type
, dest
.typeReg());
5151 mov(JSReturnReg_Data
, dest
.payloadReg());
5153 #elif defined(JS_PUNBOX64)
5154 if (dest
.valueReg() != JSReturnReg
) {
5155 mov(JSReturnReg
, dest
.valueReg());
5158 # error "Bad architecture"
5162 inline void storeCallResultValue(TypedOrValueRegister dest
);
5165 TrampolinePtr
preBarrierTrampoline(MIRType type
);
5167 template <typename T
>
5168 void unguardedCallPreBarrier(const T
& address
, MIRType type
) {
5170 if (type
== MIRType::Value
) {
5171 branchTestGCThing(Assembler::NotEqual
, address
, &done
);
5172 } else if (type
== MIRType::Object
|| type
== MIRType::String
) {
5173 branchPtr(Assembler::Equal
, address
, ImmWord(0), &done
);
5176 Push(PreBarrierReg
);
5177 computeEffectiveAddress(address
, PreBarrierReg
);
5179 TrampolinePtr preBarrier
= preBarrierTrampoline(type
);
5183 // On arm64, SP may be < PSP now (that's OK).
5184 // eg testcase: tests/auto-regress/bug702915.js
5189 template <typename T
>
5190 void guardedCallPreBarrier(const T
& address
, MIRType type
) {
5192 branchTestNeedsIncrementalBarrier(Assembler::Zero
, &done
);
5193 unguardedCallPreBarrier(address
, type
);
5197 // Like guardedCallPreBarrier, but unlike guardedCallPreBarrier this can be
5198 // called from runtime-wide trampolines because it loads cx->zone (instead of
5199 // baking in the current Zone) if JitContext::realm is nullptr.
5200 template <typename T
>
5201 void guardedCallPreBarrierAnyZone(const T
& address
, MIRType type
,
5204 branchTestNeedsIncrementalBarrierAnyZone(Assembler::Zero
, &done
, scratch
);
5205 unguardedCallPreBarrier(address
, type
);
5209 enum class Uint32Mode
{ FailOnDouble
, ForceDouble
};
5211 void boxUint32(Register source
, ValueOperand dest
, Uint32Mode uint32Mode
,
5214 template <typename T
>
5215 void loadFromTypedArray(Scalar::Type arrayType
, const T
& src
,
5216 AnyRegister dest
, Register temp
, Label
* fail
);
5218 template <typename T
>
5219 void loadFromTypedArray(Scalar::Type arrayType
, const T
& src
,
5220 const ValueOperand
& dest
, Uint32Mode uint32Mode
,
5221 Register temp
, Label
* fail
);
5223 template <typename T
>
5224 void loadFromTypedBigIntArray(Scalar::Type arrayType
, const T
& src
,
5225 Register bigInt
, Register64 temp
);
5227 template <typename S
, typename T
>
5228 void storeToTypedIntArray(Scalar::Type arrayType
, const S
& value
,
5230 switch (arrayType
) {
5233 case Scalar::Uint8Clamped
:
5234 store8(value
, dest
);
5237 case Scalar::Uint16
:
5238 store16(value
, dest
);
5241 case Scalar::Uint32
:
5242 store32(value
, dest
);
5245 MOZ_CRASH("Invalid typed array type");
5249 void storeToTypedFloatArray(Scalar::Type arrayType
, FloatRegister value
,
5250 const BaseIndex
& dest
);
5251 void storeToTypedFloatArray(Scalar::Type arrayType
, FloatRegister value
,
5252 const Address
& dest
);
5254 void storeToTypedBigIntArray(Scalar::Type arrayType
, Register64 value
,
5255 const BaseIndex
& dest
);
5256 void storeToTypedBigIntArray(Scalar::Type arrayType
, Register64 value
,
5257 const Address
& dest
);
5259 void memoryBarrierBefore(Synchronization sync
);
5260 void memoryBarrierAfter(Synchronization sync
);
5262 void debugAssertIsObject(const ValueOperand
& val
);
5263 void debugAssertObjHasFixedSlots(Register obj
, Register scratch
);
5265 void debugAssertObjectHasClass(Register obj
, Register scratch
,
5266 const JSClass
* clasp
);
5268 void debugAssertGCThingIsTenured(Register ptr
, Register temp
);
5270 void branchArrayIsNotPacked(Register array
, Register temp1
, Register temp2
,
5273 void setIsPackedArray(Register obj
, Register output
, Register temp
);
5275 void packedArrayPop(Register array
, ValueOperand output
, Register temp1
,
5276 Register temp2
, Label
* fail
);
5277 void packedArrayShift(Register array
, ValueOperand output
, Register temp1
,
5278 Register temp2
, LiveRegisterSet volatileRegs
,
5281 void loadArgumentsObjectElement(Register obj
, Register index
,
5282 ValueOperand output
, Register temp
,
5284 void loadArgumentsObjectElementHole(Register obj
, Register index
,
5285 ValueOperand output
, Register temp
,
5287 void loadArgumentsObjectElementExists(Register obj
, Register index
,
5288 Register output
, Register temp
,
5291 void loadArgumentsObjectLength(Register obj
, Register output
, Label
* fail
);
5292 void loadArgumentsObjectLength(Register obj
, Register output
);
5294 void branchTestArgumentsObjectFlags(Register obj
, Register temp
,
5295 uint32_t flags
, Condition cond
,
5298 void typedArrayElementSize(Register obj
, Register output
);
5301 // Shift |output| by the element shift of the ResizableTypedArray in |obj|.
5302 void resizableTypedArrayElementShiftBy(Register obj
, Register output
,
5306 void branchIfClassIsNotTypedArray(Register clasp
, Label
* notTypedArray
);
5307 void branchIfClassIsNotFixedLengthTypedArray(Register clasp
,
5308 Label
* notTypedArray
);
5309 void branchIfClassIsNotResizableTypedArray(Register clasp
,
5310 Label
* notTypedArray
);
5313 enum class BranchIfDetached
{ No
, Yes
};
5315 void branchIfHasDetachedArrayBuffer(BranchIfDetached branchIf
, Register obj
,
5316 Register temp
, Label
* label
);
5319 void branchIfHasDetachedArrayBuffer(Register obj
, Register temp
,
5321 branchIfHasDetachedArrayBuffer(BranchIfDetached::Yes
, obj
, temp
, label
);
5324 void branchIfHasAttachedArrayBuffer(Register obj
, Register temp
,
5326 branchIfHasDetachedArrayBuffer(BranchIfDetached::No
, obj
, temp
, label
);
5329 void branchIfResizableArrayBufferViewOutOfBounds(Register obj
, Register temp
,
5332 void branchIfResizableArrayBufferViewInBounds(Register obj
, Register temp
,
5335 void branchIfNativeIteratorNotReusable(Register ni
, Label
* notReusable
);
5336 void branchNativeIteratorIndices(Condition cond
, Register ni
, Register temp
,
5337 NativeIteratorIndices kind
, Label
* label
);
5339 void maybeLoadIteratorFromShape(Register obj
, Register dest
, Register temp
,
5340 Register temp2
, Register temp3
,
5343 void iteratorMore(Register obj
, ValueOperand output
, Register temp
);
5344 void iteratorClose(Register obj
, Register temp1
, Register temp2
,
5346 void registerIterator(Register enumeratorsList
, Register iter
, Register temp
);
5348 void toHashableNonGCThing(ValueOperand value
, ValueOperand result
,
5349 FloatRegister tempFloat
);
5351 void toHashableValue(ValueOperand value
, ValueOperand result
,
5352 FloatRegister tempFloat
, Label
* atomizeString
,
5356 void scrambleHashCode(Register result
);
5359 void prepareHashNonGCThing(ValueOperand value
, Register result
,
5361 void prepareHashString(Register str
, Register result
, Register temp
);
5362 void prepareHashSymbol(Register sym
, Register result
);
5363 void prepareHashBigInt(Register bigInt
, Register result
, Register temp1
,
5364 Register temp2
, Register temp3
);
5365 void prepareHashObject(Register setObj
, ValueOperand value
, Register result
,
5366 Register temp1
, Register temp2
, Register temp3
,
5368 void prepareHashValue(Register setObj
, ValueOperand value
, Register result
,
5369 Register temp1
, Register temp2
, Register temp3
,
5373 enum class IsBigInt
{ No
, Yes
, Maybe
};
5376 * Search for a value in a OrderedHashTable.
5378 * When we jump to |found|, |entryTemp| holds the found hashtable entry.
5380 template <typename OrderedHashTable
>
5381 void orderedHashTableLookup(Register setOrMapObj
, ValueOperand value
,
5382 Register hash
, Register entryTemp
, Register temp1
,
5383 Register temp3
, Register temp4
, Register temp5
,
5384 Label
* found
, IsBigInt isBigInt
);
5386 void setObjectHas(Register setObj
, ValueOperand value
, Register hash
,
5387 Register result
, Register temp1
, Register temp2
,
5388 Register temp3
, Register temp4
, IsBigInt isBigInt
);
5390 void mapObjectHas(Register mapObj
, ValueOperand value
, Register hash
,
5391 Register result
, Register temp1
, Register temp2
,
5392 Register temp3
, Register temp4
, IsBigInt isBigInt
);
5394 void mapObjectGet(Register mapObj
, ValueOperand value
, Register hash
,
5395 ValueOperand result
, Register temp1
, Register temp2
,
5396 Register temp3
, Register temp4
, Register temp5
,
5400 void setObjectHasNonBigInt(Register setObj
, ValueOperand value
, Register hash
,
5401 Register result
, Register temp1
, Register temp2
) {
5402 return setObjectHas(setObj
, value
, hash
, result
, temp1
, temp2
, InvalidReg
,
5403 InvalidReg
, IsBigInt::No
);
5405 void setObjectHasBigInt(Register setObj
, ValueOperand value
, Register hash
,
5406 Register result
, Register temp1
, Register temp2
,
5407 Register temp3
, Register temp4
) {
5408 return setObjectHas(setObj
, value
, hash
, result
, temp1
, temp2
, temp3
, temp4
,
5411 void setObjectHasValue(Register setObj
, ValueOperand value
, Register hash
,
5412 Register result
, Register temp1
, Register temp2
,
5413 Register temp3
, Register temp4
) {
5414 return setObjectHas(setObj
, value
, hash
, result
, temp1
, temp2
, temp3
, temp4
,
5418 void mapObjectHasNonBigInt(Register mapObj
, ValueOperand value
, Register hash
,
5419 Register result
, Register temp1
, Register temp2
) {
5420 return mapObjectHas(mapObj
, value
, hash
, result
, temp1
, temp2
, InvalidReg
,
5421 InvalidReg
, IsBigInt::No
);
5423 void mapObjectHasBigInt(Register mapObj
, ValueOperand value
, Register hash
,
5424 Register result
, Register temp1
, Register temp2
,
5425 Register temp3
, Register temp4
) {
5426 return mapObjectHas(mapObj
, value
, hash
, result
, temp1
, temp2
, temp3
, temp4
,
5429 void mapObjectHasValue(Register mapObj
, ValueOperand value
, Register hash
,
5430 Register result
, Register temp1
, Register temp2
,
5431 Register temp3
, Register temp4
) {
5432 return mapObjectHas(mapObj
, value
, hash
, result
, temp1
, temp2
, temp3
, temp4
,
5436 void mapObjectGetNonBigInt(Register mapObj
, ValueOperand value
, Register hash
,
5437 ValueOperand result
, Register temp1
,
5438 Register temp2
, Register temp3
) {
5439 return mapObjectGet(mapObj
, value
, hash
, result
, temp1
, temp2
, temp3
,
5440 InvalidReg
, InvalidReg
, IsBigInt::No
);
5442 void mapObjectGetBigInt(Register mapObj
, ValueOperand value
, Register hash
,
5443 ValueOperand result
, Register temp1
, Register temp2
,
5444 Register temp3
, Register temp4
, Register temp5
) {
5445 return mapObjectGet(mapObj
, value
, hash
, result
, temp1
, temp2
, temp3
, temp4
,
5446 temp5
, IsBigInt::Yes
);
5448 void mapObjectGetValue(Register mapObj
, ValueOperand value
, Register hash
,
5449 ValueOperand result
, Register temp1
, Register temp2
,
5450 Register temp3
, Register temp4
, Register temp5
) {
5451 return mapObjectGet(mapObj
, value
, hash
, result
, temp1
, temp2
, temp3
, temp4
,
5452 temp5
, IsBigInt::Maybe
);
5456 template <typename OrderedHashTable
>
5457 void loadOrderedHashTableCount(Register setOrMapObj
, Register result
);
5460 void loadSetObjectSize(Register setObj
, Register result
);
5461 void loadMapObjectSize(Register mapObj
, Register result
);
5463 // Inline version of js_TypedArray_uint8_clamp_double.
5464 // This function clobbers the input register.
5465 void clampDoubleToUint8(FloatRegister input
, Register output
) PER_ARCH
;
5467 using MacroAssemblerSpecific::ensureDouble
;
5469 template <typename S
>
5470 void ensureDouble(const S
& source
, FloatRegister dest
, Label
* failure
) {
5471 Label isDouble
, done
;
5472 branchTestDouble(Assembler::Equal
, source
, &isDouble
);
5473 branchTestInt32(Assembler::NotEqual
, source
, failure
);
5475 convertInt32ToDouble(source
, dest
);
5479 unboxDouble(source
, dest
);
5484 // Inline allocation.
5486 void checkAllocatorState(Register temp
, gc::AllocKind allocKind
, Label
* fail
);
5487 bool shouldNurseryAllocate(gc::AllocKind allocKind
, gc::Heap initialHeap
);
5488 void nurseryAllocateObject(
5489 Register result
, Register temp
, gc::AllocKind allocKind
,
5490 size_t nDynamicSlots
, Label
* fail
,
5491 const AllocSiteInput
& allocSite
= AllocSiteInput());
5492 void bumpPointerAllocate(Register result
, Register temp
, Label
* fail
,
5493 CompileZone
* zone
, JS::TraceKind traceKind
,
5495 const AllocSiteInput
& allocSite
= AllocSiteInput());
5496 void updateAllocSite(Register temp
, Register result
, CompileZone
* zone
,
5499 void freeListAllocate(Register result
, Register temp
, gc::AllocKind allocKind
,
5501 void allocateObject(Register result
, Register temp
, gc::AllocKind allocKind
,
5502 uint32_t nDynamicSlots
, gc::Heap initialHeap
, Label
* fail
,
5503 const AllocSiteInput
& allocSite
= AllocSiteInput());
5504 void nurseryAllocateString(Register result
, Register temp
,
5505 gc::AllocKind allocKind
, Label
* fail
);
5506 void allocateString(Register result
, Register temp
, gc::AllocKind allocKind
,
5507 gc::Heap initialHeap
, Label
* fail
);
5508 void nurseryAllocateBigInt(Register result
, Register temp
, Label
* fail
);
5509 void copySlotsFromTemplate(Register obj
,
5510 const TemplateNativeObject
& templateObj
,
5511 uint32_t start
, uint32_t end
);
5512 void fillSlotsWithConstantValue(Address addr
, Register temp
, uint32_t start
,
5513 uint32_t end
, const Value
& v
);
5514 void fillSlotsWithUndefined(Address addr
, Register temp
, uint32_t start
,
5516 void fillSlotsWithUninitialized(Address addr
, Register temp
, uint32_t start
,
5519 void initGCSlots(Register obj
, Register temp
,
5520 const TemplateNativeObject
& templateObj
);
5523 void callFreeStub(Register slots
);
5524 void createGCObject(Register result
, Register temp
,
5525 const TemplateObject
& templateObj
, gc::Heap initialHeap
,
5526 Label
* fail
, bool initContents
= true);
5528 void createPlainGCObject(Register result
, Register shape
, Register temp
,
5529 Register temp2
, uint32_t numFixedSlots
,
5530 uint32_t numDynamicSlots
, gc::AllocKind allocKind
,
5531 gc::Heap initialHeap
, Label
* fail
,
5532 const AllocSiteInput
& allocSite
,
5533 bool initContents
= true);
5535 // dynamicSlotsTemp is used to initialize the dynamic slots after allocating
5536 // the object. If numUsedDynamicSlots == 0, it may be InvalidReg.
5537 void createArrayWithFixedElements(
5538 Register result
, Register shape
, Register temp
, Register dynamicSlotsTemp
,
5539 uint32_t arrayLength
, uint32_t arrayCapacity
,
5540 uint32_t numUsedDynamicSlots
, uint32_t numDynamicSlots
,
5541 gc::AllocKind allocKind
, gc::Heap initialHeap
, Label
* fail
,
5542 const AllocSiteInput
& allocSite
= AllocSiteInput());
5544 void initGCThing(Register obj
, Register temp
,
5545 const TemplateObject
& templateObj
, bool initContents
= true);
5547 enum class TypedArrayLength
{ Fixed
, Dynamic
};
5549 void initTypedArraySlots(Register obj
, Register temp
, Register lengthReg
,
5550 LiveRegisterSet liveRegs
, Label
* fail
,
5551 FixedLengthTypedArrayObject
* templateObj
,
5552 TypedArrayLength lengthKind
);
5554 void newGCString(Register result
, Register temp
, gc::Heap initialHeap
,
5556 void newGCFatInlineString(Register result
, Register temp
,
5557 gc::Heap initialHeap
, Label
* fail
);
5559 void newGCBigInt(Register result
, Register temp
, gc::Heap initialHeap
,
5563 void branchIfNotStringCharsEquals(Register stringChars
,
5564 const JSLinearString
* linear
, Label
* label
);
5567 // Returns true if |linear| is a (non-empty) string which can be compared
5568 // using |compareStringChars|.
5569 static bool canCompareStringCharsInline(const JSLinearString
* linear
);
5571 // Load the string characters in preparation for |compareStringChars|.
5572 void loadStringCharsForCompare(Register input
, const JSLinearString
* linear
,
5573 Register stringChars
, Label
* fail
);
5575 // Compare string characters based on the equality operator. The string
5576 // characters must be at least as long as the length of |linear|.
5577 void compareStringChars(JSOp op
, Register stringChars
,
5578 const JSLinearString
* linear
, Register result
);
5580 // Compares two strings for equality based on the JSOP.
5581 // This checks for identical pointers, atoms and length and fails for
5583 void compareStrings(JSOp op
, Register left
, Register right
, Register result
,
5586 // Result of the typeof operation. Falls back to slow-path for proxies.
5587 void typeOfObject(Register objReg
, Register scratch
, Label
* slow
,
5588 Label
* isObject
, Label
* isCallable
, Label
* isUndefined
);
5590 // Implementation of IsCallable. Doesn't handle proxies.
5591 void isCallable(Register obj
, Register output
, Label
* isProxy
) {
5592 isCallableOrConstructor(true, obj
, output
, isProxy
);
5594 void isConstructor(Register obj
, Register output
, Label
* isProxy
) {
5595 isCallableOrConstructor(false, obj
, output
, isProxy
);
5598 void setIsCrossRealmArrayConstructor(Register obj
, Register output
);
5600 void setIsDefinitelyTypedArrayConstructor(Register obj
, Register output
);
5602 void loadMegamorphicCache(Register dest
);
5603 void lookupStringInAtomCacheLastLookups(Register str
, Register scratch
,
5604 Register output
, Label
* fail
);
5605 void loadMegamorphicSetPropCache(Register dest
);
5607 void loadAtomOrSymbolAndHash(ValueOperand value
, Register outId
,
5608 Register outHash
, Label
* cacheMiss
);
5610 void loadAtomHash(Register id
, Register hash
, Label
* done
);
5612 void emitExtractValueFromMegamorphicCacheEntry(
5613 Register obj
, Register entry
, Register scratch1
, Register scratch2
,
5614 ValueOperand output
, Label
* cacheHit
, Label
* cacheMiss
);
5616 template <typename IdOperandType
>
5617 void emitMegamorphicCacheLookupByValueCommon(
5618 IdOperandType id
, Register obj
, Register scratch1
, Register scratch2
,
5619 Register outEntryPtr
, Label
* cacheMiss
, Label
* cacheMissWithEntry
);
5621 void emitMegamorphicCacheLookup(PropertyKey id
, Register obj
,
5622 Register scratch1
, Register scratch2
,
5623 Register outEntryPtr
, ValueOperand output
,
5626 // NOTE: |id| must either be a ValueOperand or a Register. If it is a
5627 // Register, we assume that it is an atom.
5628 template <typename IdOperandType
>
5629 void emitMegamorphicCacheLookupByValue(IdOperandType id
, Register obj
,
5630 Register scratch1
, Register scratch2
,
5631 Register outEntryPtr
,
5632 ValueOperand output
, Label
* cacheHit
);
5634 void emitMegamorphicCacheLookupExists(ValueOperand id
, Register obj
,
5635 Register scratch1
, Register scratch2
,
5636 Register outEntryPtr
, Register output
,
5637 Label
* cacheHit
, bool hasOwn
);
5639 // Given a PropertyIteratorObject with valid indices, extract the current
5640 // PropertyIndex, storing the index in |outIndex| and the kind in |outKind|
5641 void extractCurrentIndexAndKindFromIterator(Register iterator
,
5645 template <typename IdType
>
5646 #ifdef JS_CODEGEN_X86
5647 // See MegamorphicSetElement in LIROps.yaml
5648 void emitMegamorphicCachedSetSlot(IdType id
, Register obj
, Register scratch1
,
5649 ValueOperand value
, Label
* cacheHit
,
5650 void (*emitPreBarrier
)(MacroAssembler
&,
5654 void emitMegamorphicCachedSetSlot(
5655 IdType id
, Register obj
, Register scratch1
, Register scratch2
,
5656 Register scratch3
, ValueOperand value
, Label
* cacheHit
,
5657 void (*emitPreBarrier
)(MacroAssembler
&, const Address
&, MIRType
));
5660 void loadDOMExpandoValueGuardGeneration(
5661 Register obj
, ValueOperand output
,
5662 JS::ExpandoAndGeneration
* expandoAndGeneration
, uint64_t generation
,
5665 void guardNonNegativeIntPtrToInt32(Register reg
, Label
* fail
);
5667 void loadArrayBufferByteLengthIntPtr(Register obj
, Register output
);
5668 void loadArrayBufferViewByteOffsetIntPtr(Register obj
, Register output
);
5669 void loadArrayBufferViewLengthIntPtr(Register obj
, Register output
);
5671 void loadGrowableSharedArrayBufferByteLengthIntPtr(Synchronization sync
,
5676 enum class ResizableArrayBufferView
{ TypedArray
, DataView
};
5678 void loadResizableArrayBufferViewLengthIntPtr(ResizableArrayBufferView view
,
5679 Synchronization sync
,
5680 Register obj
, Register output
,
5684 void loadResizableTypedArrayLengthIntPtr(Synchronization sync
, Register obj
,
5685 Register output
, Register scratch
) {
5686 loadResizableArrayBufferViewLengthIntPtr(
5687 ResizableArrayBufferView::TypedArray
, sync
, obj
, output
, scratch
);
5690 void loadResizableDataViewByteLengthIntPtr(Synchronization sync
, Register obj
,
5693 loadResizableArrayBufferViewLengthIntPtr(ResizableArrayBufferView::DataView
,
5694 sync
, obj
, output
, scratch
);
5697 void loadResizableTypedArrayByteOffsetMaybeOutOfBoundsIntPtr(
5698 Register obj
, Register output
, Register scratch
);
5701 void isCallableOrConstructor(bool isCallable
, Register obj
, Register output
,
5705 // Generates code used to complete a bailout.
5706 void generateBailoutTail(Register scratch
, Register bailoutInfo
);
5709 #ifndef JS_CODEGEN_ARM64
5710 // StackPointer manipulation functions.
5711 // On ARM64, the StackPointer is implemented as two synchronized registers.
5712 // Code shared across platforms must use these functions to be valid.
5713 template <typename T
>
5714 inline void addToStackPtr(T t
);
5715 template <typename T
>
5716 inline void addStackPtrTo(T t
);
5718 void subFromStackPtr(Imm32 imm32
)
5719 DEFINED_ON(mips32
, mips64
, loong64
, riscv64
, wasm32
, arm
, x86
, x64
);
5720 void subFromStackPtr(Register reg
);
5722 template <typename T
>
5723 void subStackPtrFrom(T t
) {
5724 subPtr(getStackPointer(), t
);
5727 template <typename T
>
5728 void andToStackPtr(T t
) {
5729 andPtr(t
, getStackPointer());
5732 template <typename T
>
5733 void moveToStackPtr(T t
) {
5734 movePtr(t
, getStackPointer());
5736 template <typename T
>
5737 void moveStackPtrTo(T t
) {
5738 movePtr(getStackPointer(), t
);
5741 template <typename T
>
5742 void loadStackPtr(T t
) {
5743 loadPtr(t
, getStackPointer());
5745 template <typename T
>
5746 void storeStackPtr(T t
) {
5747 storePtr(getStackPointer(), t
);
5750 // StackPointer testing functions.
5751 // On ARM64, sp can function as the zero register depending on context.
5752 // Code shared across platforms must use these functions to be valid.
5753 template <typename T
>
5754 inline void branchTestStackPtr(Condition cond
, T t
, Label
* label
);
5755 template <typename T
>
5756 inline void branchStackPtr(Condition cond
, T rhs
, Label
* label
);
5757 template <typename T
>
5758 inline void branchStackPtrRhs(Condition cond
, T lhs
, Label
* label
);
5760 // Move the stack pointer based on the requested amount.
5761 inline void reserveStack(uint32_t amount
);
5762 #else // !JS_CODEGEN_ARM64
5763 void reserveStack(uint32_t amount
);
5767 void enableProfilingInstrumentation() {
5768 emitProfilingInstrumentation_
= true;
5772 // This class is used to surround call sites throughout the assembler. This
5773 // is used by callWithABI, and callJit functions, except if suffixed by
5775 class MOZ_RAII AutoProfilerCallInstrumentation
{
5777 explicit AutoProfilerCallInstrumentation(MacroAssembler
& masm
);
5778 ~AutoProfilerCallInstrumentation() = default;
5780 friend class AutoProfilerCallInstrumentation
;
5782 void appendProfilerCallSite(CodeOffset label
) {
5783 propagateOOM(profilerCallSites_
.append(label
));
5786 // Fix up the code pointers to be written for locations where profilerCallSite
5787 // emitted moves of RIP to a register.
5788 void linkProfilerCallSites(JitCode
* code
);
5790 // This field is used to manage profiling instrumentation output. If
5791 // provided and enabled, then instrumentation will be emitted around call
5793 bool emitProfilingInstrumentation_
;
5795 // Record locations of the call sites.
5796 Vector
<CodeOffset
, 0, SystemAllocPolicy
> profilerCallSites_
;
5799 void loadJitCodeRaw(Register func
, Register dest
);
5800 void loadBaselineJitCodeRaw(Register func
, Register dest
,
5801 Label
* failure
= nullptr);
5802 void storeICScriptInJSContext(Register icScript
);
5804 void loadBaselineFramePtr(Register framePtr
, Register dest
);
5806 void pushBaselineFramePtr(Register framePtr
, Register scratch
) {
5807 loadBaselineFramePtr(framePtr
, scratch
);
5811 void PushBaselineFramePtr(Register framePtr
, Register scratch
) {
5812 loadBaselineFramePtr(framePtr
, scratch
);
5816 using MacroAssemblerSpecific::movePtr
;
5818 void movePtr(TrampolinePtr ptr
, Register dest
) {
5819 movePtr(ImmPtr(ptr
.value
), dest
);
5823 void handleFailure();
5826 Label
* exceptionLabel() {
5827 // Exceptions are currently handled the same way as sequential failures.
5828 return &failureLabel_
;
5831 Label
* failureLabel() { return &failureLabel_
; }
5834 void link(JitCode
* code
);
5836 void assumeUnreachable(const char* output
);
5838 void printf(const char* output
);
5839 void printf(const char* output
, Register value
);
5841 #define DISPATCH_FLOATING_POINT_OP(method, type, arg1d, arg1f, arg2) \
5842 MOZ_ASSERT(IsFloatingPointType(type)); \
5843 if (type == MIRType::Double) \
5844 method##Double(arg1d, arg2); \
5846 method##Float32(arg1f, arg2);
5848 void loadConstantFloatingPoint(double d
, float f
, FloatRegister dest
,
5850 DISPATCH_FLOATING_POINT_OP(loadConstant
, destType
, d
, f
, dest
);
5852 void boolValueToFloatingPoint(ValueOperand value
, FloatRegister dest
,
5854 DISPATCH_FLOATING_POINT_OP(boolValueTo
, destType
, value
, value
, dest
);
5856 void int32ValueToFloatingPoint(ValueOperand value
, FloatRegister dest
,
5858 DISPATCH_FLOATING_POINT_OP(int32ValueTo
, destType
, value
, value
, dest
);
5860 void convertInt32ToFloatingPoint(Register src
, FloatRegister dest
,
5862 DISPATCH_FLOATING_POINT_OP(convertInt32To
, destType
, src
, src
, dest
);
5865 #undef DISPATCH_FLOATING_POINT_OP
5867 void convertValueToFloatingPoint(ValueOperand value
, FloatRegister output
,
5868 Label
* fail
, MIRType outputType
);
5870 void outOfLineTruncateSlow(FloatRegister src
, Register dest
,
5871 bool widenFloatToDouble
, bool compilingWasm
,
5872 wasm::BytecodeOffset callOffset
);
5874 void convertInt32ValueToDouble(ValueOperand val
);
5876 void convertValueToDouble(ValueOperand value
, FloatRegister output
,
5878 convertValueToFloatingPoint(value
, output
, fail
, MIRType::Double
);
5881 void convertValueToFloat(ValueOperand value
, FloatRegister output
,
5883 convertValueToFloatingPoint(value
, output
, fail
, MIRType::Float32
);
5887 // Functions for converting values to int.
5889 void convertDoubleToInt(FloatRegister src
, Register output
,
5890 FloatRegister temp
, Label
* truncateFail
, Label
* fail
,
5891 IntConversionBehavior behavior
);
5893 // Strings may be handled by providing labels to jump to when the behavior
5894 // is truncation or clamping. The subroutine, usually an OOL call, is
5895 // passed the unboxed string in |stringReg| and should convert it to a
5896 // double store into |temp|.
5897 void convertValueToInt(
5898 ValueOperand value
, Label
* handleStringEntry
, Label
* handleStringRejoin
,
5899 Label
* truncateDoubleSlow
, Register stringReg
, FloatRegister temp
,
5900 Register output
, Label
* fail
, IntConversionBehavior behavior
,
5901 IntConversionInputKind conversion
= IntConversionInputKind::Any
);
5903 // This carries over the MToNumberInt32 operation on the ValueOperand
5904 // input; see comment at the top of this class.
5905 void convertValueToInt32(
5906 ValueOperand value
, FloatRegister temp
, Register output
, Label
* fail
,
5907 bool negativeZeroCheck
,
5908 IntConversionInputKind conversion
= IntConversionInputKind::Any
) {
5910 value
, nullptr, nullptr, nullptr, InvalidReg
, temp
, output
, fail
,
5911 negativeZeroCheck
? IntConversionBehavior::NegativeZeroCheck
5912 : IntConversionBehavior::Normal
,
5916 // This carries over the MTruncateToInt32 operation on the ValueOperand
5917 // input; see the comment at the top of this class.
5918 void truncateValueToInt32(ValueOperand value
, Label
* handleStringEntry
,
5919 Label
* handleStringRejoin
,
5920 Label
* truncateDoubleSlow
, Register stringReg
,
5921 FloatRegister temp
, Register output
, Label
* fail
) {
5922 convertValueToInt(value
, handleStringEntry
, handleStringRejoin
,
5923 truncateDoubleSlow
, stringReg
, temp
, output
, fail
,
5924 IntConversionBehavior::Truncate
);
5927 void truncateValueToInt32(ValueOperand value
, FloatRegister temp
,
5928 Register output
, Label
* fail
) {
5929 truncateValueToInt32(value
, nullptr, nullptr, nullptr, InvalidReg
, temp
,
5933 // Convenience functions for clamping values to uint8.
5934 void clampValueToUint8(ValueOperand value
, Label
* handleStringEntry
,
5935 Label
* handleStringRejoin
, Register stringReg
,
5936 FloatRegister temp
, Register output
, Label
* fail
) {
5937 convertValueToInt(value
, handleStringEntry
, handleStringRejoin
, nullptr,
5938 stringReg
, temp
, output
, fail
,
5939 IntConversionBehavior::ClampToUint8
);
5942 [[nodiscard
]] bool icBuildOOLFakeExitFrame(void* fakeReturnAddr
,
5943 AutoSaveLiveRegisters
& save
);
5945 // Align the stack pointer based on the number of arguments which are pushed
5946 // on the stack, such that the JitFrameLayout would be correctly aligned on
5947 // the JitStackAlignment.
5948 void alignJitStackBasedOnNArgs(Register nargs
, bool countIncludesThis
);
5949 void alignJitStackBasedOnNArgs(uint32_t argc
, bool countIncludesThis
);
5951 inline void assertStackAlignment(uint32_t alignment
, int32_t offset
= 0);
5953 void touchFrameValues(Register numStackValues
, Register scratch1
,
5957 // See comment block "64-bit GPRs carrying 32-bit values" above. This asserts
5958 // that the high bits of the register are appropriate for the architecture and
5959 // the value in the low bits.
5960 void debugAssertCanonicalInt32(Register r
);
5964 // StackMacroAssembler checks no GC will happen while it's on the stack.
5965 class MOZ_RAII StackMacroAssembler
: public MacroAssembler
{
5966 JS::AutoCheckCannotGC nogc
;
5969 StackMacroAssembler(JSContext
* cx
, TempAllocator
& alloc
);
5972 // WasmMacroAssembler does not contain GC pointers, so it doesn't need the no-GC
5973 // checking StackMacroAssembler has.
5974 class MOZ_RAII WasmMacroAssembler
: public MacroAssembler
{
5976 explicit WasmMacroAssembler(TempAllocator
& alloc
, bool limitedSize
= true);
5977 explicit WasmMacroAssembler(TempAllocator
& alloc
,
5978 const wasm::ModuleEnvironment
& env
,
5979 bool limitedSize
= true);
5980 ~WasmMacroAssembler() { assertNoGCThings(); }
5983 // Heap-allocated MacroAssembler used for Ion off-thread code generation.
5984 // GC cancels off-thread compilations.
5985 class IonHeapMacroAssembler
: public MacroAssembler
{
5987 IonHeapMacroAssembler(TempAllocator
& alloc
, CompileRealm
* realm
);
5990 //{{{ check_macroassembler_style
5991 inline uint32_t MacroAssembler::framePushed() const { return framePushed_
; }
5993 inline void MacroAssembler::setFramePushed(uint32_t framePushed
) {
5994 framePushed_
= framePushed
;
5997 inline void MacroAssembler::adjustFrame(int32_t value
) {
5998 MOZ_ASSERT_IF(value
< 0, framePushed_
>= uint32_t(-value
));
5999 setFramePushed(framePushed_
+ value
);
6002 inline void MacroAssembler::implicitPop(uint32_t bytes
) {
6003 MOZ_ASSERT(bytes
% sizeof(intptr_t) == 0);
6004 MOZ_ASSERT(bytes
<= INT32_MAX
);
6005 adjustFrame(-int32_t(bytes
));
6007 //}}} check_macroassembler_style
6009 static inline Assembler::DoubleCondition
JSOpToDoubleCondition(JSOp op
) {
6012 case JSOp::StrictEq
:
6013 return Assembler::DoubleEqual
;
6015 case JSOp::StrictNe
:
6016 return Assembler::DoubleNotEqualOrUnordered
;
6018 return Assembler::DoubleLessThan
;
6020 return Assembler::DoubleLessThanOrEqual
;
6022 return Assembler::DoubleGreaterThan
;
6024 return Assembler::DoubleGreaterThanOrEqual
;
6026 MOZ_CRASH("Unexpected comparison operation");
6030 // Note: the op may have been inverted during lowering (to put constants in a
6031 // position where they can be immediates), so it is important to use the
6032 // lir->jsop() instead of the mir->jsop() when it is present.
6033 static inline Assembler::Condition
JSOpToCondition(JSOp op
, bool isSigned
) {
6037 case JSOp::StrictEq
:
6038 return Assembler::Equal
;
6040 case JSOp::StrictNe
:
6041 return Assembler::NotEqual
;
6043 return Assembler::LessThan
;
6045 return Assembler::LessThanOrEqual
;
6047 return Assembler::GreaterThan
;
6049 return Assembler::GreaterThanOrEqual
;
6051 MOZ_CRASH("Unrecognized comparison operation");
6056 case JSOp::StrictEq
:
6057 return Assembler::Equal
;
6059 case JSOp::StrictNe
:
6060 return Assembler::NotEqual
;
6062 return Assembler::Below
;
6064 return Assembler::BelowOrEqual
;
6066 return Assembler::Above
;
6068 return Assembler::AboveOrEqual
;
6070 MOZ_CRASH("Unrecognized comparison operation");
6075 static inline size_t StackDecrementForCall(uint32_t alignment
,
6076 size_t bytesAlreadyPushed
,
6077 size_t bytesToPush
) {
6078 return bytesToPush
+
6079 ComputeByteAlignment(bytesAlreadyPushed
+ bytesToPush
, alignment
);
6082 // Helper for generatePreBarrier.
6083 inline DynFn
JitPreWriteBarrier(MIRType type
);
6088 #endif /* jit_MacroAssembler_h */