js/src/jit/MacroAssembler-inl.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef jit_MacroAssembler_inl_h
   8 #define jit_MacroAssembler_inl_h
   9
  10 #include "jit/MacroAssembler.h"
  11
  12 #include "mozilla/FloatingPoint.h"
  13 #include "mozilla/MathAlgorithms.h"
  14
  15 #include "gc/Zone.h"
  16 #include "jit/CalleeToken.h"
  17 #include "jit/CompileWrappers.h"
  18 #include "jit/JitFrames.h"
  19 #include "jit/JSJitFrameIter.h"
  20 #include "js/Prefs.h"
  21 #include "util/DifferentialTesting.h"
  22 #include "vm/BigIntType.h"
  23 #include "vm/JSObject.h"
  24 #include "vm/ProxyObject.h"
  25 #include "vm/Runtime.h"
  26 #include "vm/StringType.h"
  27
  28 #include "jit/ABIFunctionList-inl.h"
  29
  30 #if defined(JS_CODEGEN_X86)
  31 #  include "jit/x86/MacroAssembler-x86-inl.h"
  32 #elif defined(JS_CODEGEN_X64)
  33 #  include "jit/x64/MacroAssembler-x64-inl.h"
  34 #elif defined(JS_CODEGEN_ARM)
  35 #  include "jit/arm/MacroAssembler-arm-inl.h"
  36 #elif defined(JS_CODEGEN_ARM64)
  37 #  include "jit/arm64/MacroAssembler-arm64-inl.h"
  38 #elif defined(JS_CODEGEN_MIPS32)
  39 #  include "jit/mips32/MacroAssembler-mips32-inl.h"
  40 #elif defined(JS_CODEGEN_MIPS64)
  41 #  include "jit/mips64/MacroAssembler-mips64-inl.h"
  42 #elif defined(JS_CODEGEN_LOONG64)
  43 #  include "jit/loong64/MacroAssembler-loong64-inl.h"
  44 #elif defined(JS_CODEGEN_RISCV64)
  45 #  include "jit/riscv64/MacroAssembler-riscv64-inl.h"
  46 #elif defined(JS_CODEGEN_WASM32)
  47 #  include "jit/wasm32/MacroAssembler-wasm32-inl.h"
  48 #elif !defined(JS_CODEGEN_NONE)
  49 #  error "Unknown architecture!"
  50 #endif
  51
  52 #include "wasm/WasmBuiltins.h"
  53
  54 namespace js {
  55 namespace jit {
  56
  57 template <typename Sig>
  58 DynFn DynamicFunction(Sig fun) {
  59   ABIFunctionSignature<Sig> sig;
  60   return DynFn{sig.address(fun)};
  61 }
  62
  63 // Helper for generatePreBarrier.
  64 inline DynFn JitPreWriteBarrier(MIRType type) {
  65   switch (type) {
  66     case MIRType::Value: {
  67       using Fn = void (*)(JSRuntime* rt, Value* vp);
  68       return DynamicFunction<Fn>(JitValuePreWriteBarrier);
  69     }
  70     case MIRType::String: {
  71       using Fn = void (*)(JSRuntime* rt, JSString** stringp);
  72       return DynamicFunction<Fn>(JitStringPreWriteBarrier);
  73     }
  74     case MIRType::Object: {
  75       using Fn = void (*)(JSRuntime* rt, JSObject** objp);
  76       return DynamicFunction<Fn>(JitObjectPreWriteBarrier);
  77     }
  78     case MIRType::Shape: {
  79       using Fn = void (*)(JSRuntime* rt, Shape** shapep);
  80       return DynamicFunction<Fn>(JitShapePreWriteBarrier);
  81     }
  82     case MIRType::WasmAnyRef: {
  83       using Fn = void (*)(JSRuntime* rt, wasm::AnyRef* refp);
  84       return DynamicFunction<Fn>(JitWasmAnyRefPreWriteBarrier);
  85     }
  86     default:
  87       MOZ_CRASH();
  88   }
  89 }
  90
  91 //{{{ check_macroassembler_style
  92 // ===============================================================
  93 // Stack manipulation functions.
  94
  95 CodeOffset MacroAssembler::PushWithPatch(ImmWord word) {
  96   framePushed_ += sizeof(word.value);
  97   return pushWithPatch(word);
  98 }
  99
 100 CodeOffset MacroAssembler::PushWithPatch(ImmPtr imm) {
 101   return PushWithPatch(ImmWord(uintptr_t(imm.value)));
 102 }
 103
 104 // ===============================================================
 105 // Simple call functions.
 106
 107 void MacroAssembler::call(TrampolinePtr code) { call(ImmPtr(code.value)); }
 108
 109 CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc,
 110                                 const Register reg) {
 111   CodeOffset l = call(reg);
 112   append(desc, l);
 113   return l;
 114 }
 115
 116 CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc,
 117                                 uint32_t funcIndex) {
 118   CodeOffset l = callWithPatch();
 119   append(desc, l, funcIndex);
 120   return l;
 121 }
 122
 123 void MacroAssembler::call(const wasm::CallSiteDesc& desc, wasm::Trap trap) {
 124   CodeOffset l = callWithPatch();
 125   append(desc, l, trap);
 126 }
 127
 128 CodeOffset MacroAssembler::call(const wasm::CallSiteDesc& desc,
 129                                 wasm::SymbolicAddress imm) {
 130   MOZ_ASSERT(wasm::NeedsBuiltinThunk(imm),
 131              "only for functions which may appear in profiler");
 132   CodeOffset raOffset = call(imm);
 133   append(desc, raOffset);
 134   return raOffset;
 135 }
 136
 137 // ===============================================================
 138 // ABI function calls.
 139
 140 void MacroAssembler::passABIArg(Register reg) {
 141   passABIArg(MoveOperand(reg), ABIType::General);
 142 }
 143
 144 void MacroAssembler::passABIArg(FloatRegister reg, ABIType type) {
 145   passABIArg(MoveOperand(reg), type);
 146 }
 147
 148 void MacroAssembler::callWithABI(DynFn fun, ABIType result,
 149                                  CheckUnsafeCallWithABI check) {
 150   AutoProfilerCallInstrumentation profiler(*this);
 151   callWithABINoProfiler(fun.address, result, check);
 152 }
 153
 154 template <typename Sig, Sig fun>
 155 void MacroAssembler::callWithABI(ABIType result, CheckUnsafeCallWithABI check) {
 156   ABIFunction<Sig, fun> abiFun;
 157   AutoProfilerCallInstrumentation profiler(*this);
 158   callWithABINoProfiler(abiFun.address(), result, check);
 159 }
 160
 161 void MacroAssembler::callWithABI(Register fun, ABIType result) {
 162   AutoProfilerCallInstrumentation profiler(*this);
 163   callWithABINoProfiler(fun, result);
 164 }
 165
 166 void MacroAssembler::callWithABI(const Address& fun, ABIType result) {
 167   AutoProfilerCallInstrumentation profiler(*this);
 168   callWithABINoProfiler(fun, result);
 169 }
 170
 171 void MacroAssembler::appendSignatureType(ABIType type) {
 172 #ifdef JS_SIMULATOR
 173   signature_ <<= ABITypeArgShift;
 174   signature_ |= uint32_t(type);
 175 #endif
 176 }
 177
 178 ABIFunctionType MacroAssembler::signature() const {
 179 #ifdef JS_SIMULATOR
 180 #  ifdef DEBUG
 181   switch (signature_) {
 182     case Args_General0:
 183     case Args_General1:
 184     case Args_General2:
 185     case Args_General3:
 186     case Args_General4:
 187     case Args_General5:
 188     case Args_General6:
 189     case Args_General7:
 190     case Args_General8:
 191     case Args_Double_None:
 192     case Args_Int_Double:
 193     case Args_Float32_Float32:
 194     case Args_Int_Float32:
 195     case Args_Double_Double:
 196     case Args_Double_Int:
 197     case Args_Double_DoubleInt:
 198     case Args_Double_DoubleDouble:
 199     case Args_Double_IntDouble:
 200     case Args_Int_IntDouble:
 201     case Args_Int_DoubleInt:
 202     case Args_Int_DoubleIntInt:
 203     case Args_Int_IntDoubleIntInt:
 204     case Args_Double_DoubleDoubleDouble:
 205     case Args_Double_DoubleDoubleDoubleDouble:
 206     case Args_Int64_GeneralGeneral:
 207       break;
 208     default:
 209       MOZ_CRASH("Unexpected type");
 210   }
 211 #  endif  // DEBUG
 212
 213   return ABIFunctionType(signature_);
 214 #else
 215   // No simulator enabled.
 216   MOZ_CRASH("Only available for making calls within a simulator.");
 217 #endif
 218 }
 219
 220 // ===============================================================
 221 // Jit Frames.
 222
 223 uint32_t MacroAssembler::callJitNoProfiler(Register callee) {
 224 #ifdef JS_USE_LINK_REGISTER
 225   // The return address is pushed by the callee.
 226   call(callee);
 227 #else
 228   callAndPushReturnAddress(callee);
 229 #endif
 230   return currentOffset();
 231 }
 232
 233 uint32_t MacroAssembler::callJit(Register callee) {
 234   AutoProfilerCallInstrumentation profiler(*this);
 235   uint32_t ret = callJitNoProfiler(callee);
 236   return ret;
 237 }
 238
 239 uint32_t MacroAssembler::callJit(JitCode* callee) {
 240   AutoProfilerCallInstrumentation profiler(*this);
 241   call(callee);
 242   return currentOffset();
 243 }
 244
 245 uint32_t MacroAssembler::callJit(TrampolinePtr code) {
 246   AutoProfilerCallInstrumentation profiler(*this);
 247   call(code);
 248   return currentOffset();
 249 }
 250
 251 uint32_t MacroAssembler::callJit(ImmPtr callee) {
 252   AutoProfilerCallInstrumentation profiler(*this);
 253   call(callee);
 254   return currentOffset();
 255 }
 256
 257 void MacroAssembler::pushFrameDescriptor(FrameType type) {
 258   uint32_t descriptor = MakeFrameDescriptor(type);
 259   push(Imm32(descriptor));
 260 }
 261
 262 void MacroAssembler::PushFrameDescriptor(FrameType type) {
 263   uint32_t descriptor = MakeFrameDescriptor(type);
 264   Push(Imm32(descriptor));
 265 }
 266
 267 void MacroAssembler::pushFrameDescriptorForJitCall(FrameType type,
 268                                                    uint32_t argc) {
 269   uint32_t descriptor = MakeFrameDescriptorForJitCall(type, argc);
 270   push(Imm32(descriptor));
 271 }
 272
 273 void MacroAssembler::PushFrameDescriptorForJitCall(FrameType type,
 274                                                    uint32_t argc) {
 275   uint32_t descriptor = MakeFrameDescriptorForJitCall(type, argc);
 276   Push(Imm32(descriptor));
 277 }
 278
 279 void MacroAssembler::pushFrameDescriptorForJitCall(FrameType type,
 280                                                    Register argc,
 281                                                    Register scratch) {
 282   if (argc != scratch) {
 283     mov(argc, scratch);
 284   }
 285   lshift32(Imm32(NUMACTUALARGS_SHIFT), scratch);
 286   or32(Imm32(int32_t(type)), scratch);
 287   push(scratch);
 288 }
 289
 290 void MacroAssembler::PushFrameDescriptorForJitCall(FrameType type,
 291                                                    Register argc,
 292                                                    Register scratch) {
 293   pushFrameDescriptorForJitCall(type, argc, scratch);
 294   framePushed_ += sizeof(uintptr_t);
 295 }
 296
 297 void MacroAssembler::loadNumActualArgs(Register framePtr, Register dest) {
 298   loadPtr(Address(framePtr, JitFrameLayout::offsetOfDescriptor()), dest);
 299   rshift32(Imm32(NUMACTUALARGS_SHIFT), dest);
 300 }
 301
 302 void MacroAssembler::PushCalleeToken(Register callee, bool constructing) {
 303   if (constructing) {
 304     orPtr(Imm32(CalleeToken_FunctionConstructing), callee);
 305     Push(callee);
 306     andPtr(Imm32(uint32_t(CalleeTokenMask)), callee);
 307   } else {
 308     static_assert(CalleeToken_Function == 0,
 309                   "Non-constructing call requires no tagging");
 310     Push(callee);
 311   }
 312 }
 313
 314 void MacroAssembler::loadFunctionFromCalleeToken(Address token, Register dest) {
 315 #ifdef DEBUG
 316   Label ok;
 317   loadPtr(token, dest);
 318   andPtr(Imm32(uint32_t(~CalleeTokenMask)), dest);
 319   branchPtr(Assembler::Equal, dest, Imm32(CalleeToken_Function), &ok);
 320   branchPtr(Assembler::Equal, dest, Imm32(CalleeToken_FunctionConstructing),
 321             &ok);
 322   assumeUnreachable("Unexpected CalleeToken tag");
 323   bind(&ok);
 324 #endif
 325   loadPtr(token, dest);
 326   andPtr(Imm32(uint32_t(CalleeTokenMask)), dest);
 327 }
 328
 329 uint32_t MacroAssembler::buildFakeExitFrame(Register scratch) {
 330   mozilla::DebugOnly<uint32_t> initialDepth = framePushed();
 331
 332   PushFrameDescriptor(FrameType::IonJS);
 333   uint32_t retAddr = pushFakeReturnAddress(scratch);
 334   Push(FramePointer);
 335
 336   MOZ_ASSERT(framePushed() == initialDepth + ExitFrameLayout::Size());
 337   return retAddr;
 338 }
 339
 340 // ===============================================================
 341 // Exit frame footer.
 342
 343 void MacroAssembler::enterExitFrame(Register cxreg, Register scratch,
 344                                     VMFunctionId f) {
 345   linkExitFrame(cxreg, scratch);
 346   // Push `ExitFrameType::VMFunction + VMFunctionId`, for marking the arguments.
 347   // See ExitFooterFrame::data_.
 348   uintptr_t type = uintptr_t(ExitFrameType::VMFunction) + uintptr_t(f);
 349   MOZ_ASSERT(type <= INT32_MAX);
 350   Push(Imm32(type));
 351 }
 352
 353 void MacroAssembler::enterFakeExitFrame(Register cxreg, Register scratch,
 354                                         ExitFrameType type) {
 355   linkExitFrame(cxreg, scratch);
 356   Push(Imm32(int32_t(type)));
 357 }
 358
 359 void MacroAssembler::enterFakeExitFrameForNative(Register cxreg,
 360                                                  Register scratch,
 361                                                  bool isConstructing) {
 362   enterFakeExitFrame(cxreg, scratch,
 363                      isConstructing ? ExitFrameType::ConstructNative
 364                                     : ExitFrameType::CallNative);
 365 }
 366
 367 void MacroAssembler::leaveExitFrame(size_t extraFrame) {
 368   freeStack(ExitFooterFrame::Size() + extraFrame);
 369 }
 370
 371 // ===============================================================
 372 // Move instructions
 373
 374 void MacroAssembler::moveValue(const ConstantOrRegister& src,
 375                                const ValueOperand& dest) {
 376   if (src.constant()) {
 377     moveValue(src.value(), dest);
 378     return;
 379   }
 380
 381   moveValue(src.reg(), dest);
 382 }
 383
 384 // ===============================================================
 385 // Copy instructions
 386
 387 void MacroAssembler::copy64(const Address& src, const Address& dest,
 388                             Register scratch) {
 389 #if JS_BITS_PER_WORD == 32
 390   MOZ_RELEASE_ASSERT(src.base != scratch && dest.base != scratch);
 391   load32(LowWord(src), scratch);
 392   store32(scratch, LowWord(dest));
 393   load32(HighWord(src), scratch);
 394   store32(scratch, HighWord(dest));
 395 #else
 396   Register64 scratch64(scratch);
 397   load64(src, scratch64);
 398   store64(scratch64, dest);
 399 #endif
 400 }
 401
 402 // ===============================================================
 403 // Arithmetic functions
 404
 405 void MacroAssembler::addPtr(ImmPtr imm, Register dest) {
 406   addPtr(ImmWord(uintptr_t(imm.value)), dest);
 407 }
 408
 409 // ===============================================================
 410 // Branch functions
 411
 412 template <class L>
 413 void MacroAssembler::branchIfFalseBool(Register reg, L label) {
 414   // Note that C++ bool is only 1 byte, so ignore the higher-order bits.
 415   branchTest32(Assembler::Zero, reg, Imm32(0xFF), label);
 416 }
 417
 418 void MacroAssembler::branchIfTrueBool(Register reg, Label* label) {
 419   // Note that C++ bool is only 1 byte, so ignore the higher-order bits.
 420   branchTest32(Assembler::NonZero, reg, Imm32(0xFF), label);
 421 }
 422
 423 void MacroAssembler::branchIfRope(Register str, Label* label) {
 424   Address flags(str, JSString::offsetOfFlags());
 425   branchTest32(Assembler::Zero, flags, Imm32(JSString::LINEAR_BIT), label);
 426 }
 427
 428 void MacroAssembler::branchIfNotRope(Register str, Label* label) {
 429   Address flags(str, JSString::offsetOfFlags());
 430   branchTest32(Assembler::NonZero, flags, Imm32(JSString::LINEAR_BIT), label);
 431 }
 432
 433 void MacroAssembler::branchLatin1String(Register string, Label* label) {
 434   branchTest32(Assembler::NonZero, Address(string, JSString::offsetOfFlags()),
 435                Imm32(JSString::LATIN1_CHARS_BIT), label);
 436 }
 437
 438 void MacroAssembler::branchTwoByteString(Register string, Label* label) {
 439   branchTest32(Assembler::Zero, Address(string, JSString::offsetOfFlags()),
 440                Imm32(JSString::LATIN1_CHARS_BIT), label);
 441 }
 442
 443 void MacroAssembler::branchIfBigIntIsNegative(Register bigInt, Label* label) {
 444   branchTest32(Assembler::NonZero, Address(bigInt, BigInt::offsetOfFlags()),
 445                Imm32(BigInt::signBitMask()), label);
 446 }
 447
 448 void MacroAssembler::branchIfBigIntIsNonNegative(Register bigInt,
 449                                                  Label* label) {
 450   branchTest32(Assembler::Zero, Address(bigInt, BigInt::offsetOfFlags()),
 451                Imm32(BigInt::signBitMask()), label);
 452 }
 453
 454 void MacroAssembler::branchIfBigIntIsZero(Register bigInt, Label* label) {
 455   branch32(Assembler::Equal, Address(bigInt, BigInt::offsetOfLength()),
 456            Imm32(0), label);
 457 }
 458
 459 void MacroAssembler::branchIfBigIntIsNonZero(Register bigInt, Label* label) {
 460   branch32(Assembler::NotEqual, Address(bigInt, BigInt::offsetOfLength()),
 461            Imm32(0), label);
 462 }
 463
 464 void MacroAssembler::branchTestFunctionFlags(Register fun, uint32_t flags,
 465                                              Condition cond, Label* label) {
 466   Address address(fun, JSFunction::offsetOfFlagsAndArgCount());
 467   branchTest32(cond, address, Imm32(flags), label);
 468 }
 469
 470 void MacroAssembler::branchIfNotFunctionIsNonBuiltinCtor(Register fun,
 471                                                          Register scratch,
 472                                                          Label* label) {
 473   // Guard the function has the BASESCRIPT and CONSTRUCTOR flags and does NOT
 474   // have the SELF_HOSTED flag.
 475   // This is equivalent to JSFunction::isNonBuiltinConstructor.
 476   constexpr int32_t mask = FunctionFlags::BASESCRIPT |
 477                            FunctionFlags::SELF_HOSTED |
 478                            FunctionFlags::CONSTRUCTOR;
 479   constexpr int32_t expected =
 480       FunctionFlags::BASESCRIPT | FunctionFlags::CONSTRUCTOR;
 481
 482   load32(Address(fun, JSFunction::offsetOfFlagsAndArgCount()), scratch);
 483   and32(Imm32(mask), scratch);
 484   branch32(Assembler::NotEqual, scratch, Imm32(expected), label);
 485 }
 486
 487 void MacroAssembler::branchIfFunctionHasNoJitEntry(Register fun,
 488                                                    bool isConstructing,
 489                                                    Label* label) {
 490   uint16_t flags = FunctionFlags::HasJitEntryFlags(isConstructing);
 491   branchTestFunctionFlags(fun, flags, Assembler::Zero, label);
 492 }
 493
 494 void MacroAssembler::branchIfFunctionHasJitEntry(Register fun,
 495                                                  bool isConstructing,
 496                                                  Label* label) {
 497   uint16_t flags = FunctionFlags::HasJitEntryFlags(isConstructing);
 498   branchTestFunctionFlags(fun, flags, Assembler::NonZero, label);
 499 }
 500
 501 void MacroAssembler::branchIfScriptHasJitScript(Register script, Label* label) {
 502   static_assert(ScriptWarmUpData::JitScriptTag == 0,
 503                 "Code below depends on tag value");
 504   branchTestPtr(Assembler::Zero,
 505                 Address(script, JSScript::offsetOfWarmUpData()),
 506                 Imm32(ScriptWarmUpData::TagMask), label);
 507 }
 508
 509 void MacroAssembler::branchIfScriptHasNoJitScript(Register script,
 510                                                   Label* label) {
 511   static_assert(ScriptWarmUpData::JitScriptTag == 0,
 512                 "Code below depends on tag value");
 513   static_assert(BaseScript::offsetOfWarmUpData() ==
 514                     SelfHostedLazyScript::offsetOfWarmUpData(),
 515                 "SelfHostedLazyScript and BaseScript must use same layout for "
 516                 "warmUpData_");
 517   branchTestPtr(Assembler::NonZero,
 518                 Address(script, JSScript::offsetOfWarmUpData()),
 519                 Imm32(ScriptWarmUpData::TagMask), label);
 520 }
 521
 522 void MacroAssembler::loadJitScript(Register script, Register dest) {
 523 #ifdef DEBUG
 524   Label ok;
 525   branchIfScriptHasJitScript(script, &ok);
 526   assumeUnreachable("Script has no JitScript!");
 527   bind(&ok);
 528 #endif
 529
 530   static_assert(ScriptWarmUpData::JitScriptTag == 0,
 531                 "Code below depends on tag value");
 532   loadPtr(Address(script, JSScript::offsetOfWarmUpData()), dest);
 533 }
 534
 535 void MacroAssembler::loadFunctionArgCount(Register func, Register output) {
 536   load32(Address(func, JSFunction::offsetOfFlagsAndArgCount()), output);
 537   rshift32(Imm32(JSFunction::ArgCountShift), output);
 538 }
 539
 540 void MacroAssembler::branchIfObjectEmulatesUndefined(Register objReg,
 541                                                      Register scratch,
 542                                                      Label* slowCheck,
 543                                                      Label* label) {
 544   MOZ_ASSERT(objReg != scratch);
 545
 546   Label done;
 547   if (JS::Prefs::use_emulates_undefined_fuse()) {
 548     loadPtr(AbsoluteAddress(
 549                 runtime()->addressOfHasSeenObjectEmulateUndefinedFuse()),
 550             scratch);
 551     branchPtr(Assembler::Equal, scratch, ImmPtr(nullptr), &done);
 552   }
 553
 554   // The branches to out-of-line code here implement a conservative version
 555   // of the JSObject::isWrapper test performed in EmulatesUndefined.
 556   loadObjClassUnsafe(objReg, scratch);
 557
 558   branchTestClassIsProxy(true, scratch, slowCheck);
 559
 560   Address flags(scratch, JSClass::offsetOfFlags());
 561   branchTest32(Assembler::NonZero, flags, Imm32(JSCLASS_EMULATES_UNDEFINED),
 562                label);
 563   bind(&done);
 564 }
 565
 566 void MacroAssembler::branchFunctionKind(Condition cond,
 567                                         FunctionFlags::FunctionKind kind,
 568                                         Register fun, Register scratch,
 569                                         Label* label) {
 570   Address address(fun, JSFunction::offsetOfFlagsAndArgCount());
 571   load32(address, scratch);
 572   and32(Imm32(FunctionFlags::FUNCTION_KIND_MASK), scratch);
 573   branch32(cond, scratch, Imm32(kind), label);
 574 }
 575
 576 void MacroAssembler::branchTestObjClass(Condition cond, Register obj,
 577                                         const JSClass* clasp, Register scratch,
 578                                         Register spectreRegToZero,
 579                                         Label* label) {
 580   MOZ_ASSERT(obj != scratch);
 581   MOZ_ASSERT(scratch != spectreRegToZero);
 582
 583   loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
 584   loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
 585   branchPtr(cond, Address(scratch, BaseShape::offsetOfClasp()), ImmPtr(clasp),
 586             label);
 587
 588   if (JitOptions.spectreObjectMitigations) {
 589     spectreZeroRegister(cond, scratch, spectreRegToZero);
 590   }
 591 }
 592
 593 void MacroAssembler::branchTestObjClassNoSpectreMitigations(
 594     Condition cond, Register obj, const JSClass* clasp, Register scratch,
 595     Label* label) {
 596   loadPtr(Address(obj, JSObject::offsetOfShape()), scratch);
 597   loadPtr(Address(scratch, Shape::offsetOfBaseShape()), scratch);
 598   branchPtr(cond, Address(scratch, BaseShape::offsetOfClasp()), ImmPtr(clasp),
 599             label);
 600 }
 601
 602 void MacroAssembler::branchTestObjClass(Condition cond, Register obj,
 603                                         const Address& clasp, Register scratch,
 604                                         Register spectreRegToZero,
 605                                         Label* label) {
 606   MOZ_ASSERT(obj != scratch);
 607   MOZ_ASSERT(scratch != spectreRegToZero);
 608
 609   loadObjClassUnsafe(obj, scratch);
 610   branchPtr(cond, clasp, scratch, label);
 611
 612   if (JitOptions.spectreObjectMitigations) {
 613     spectreZeroRegister(cond, scratch, spectreRegToZero);
 614   }
 615 }
 616
 617 void MacroAssembler::branchTestObjClassNoSpectreMitigations(
 618     Condition cond, Register obj, const Address& clasp, Register scratch,
 619     Label* label) {
 620   MOZ_ASSERT(obj != scratch);
 621   loadObjClassUnsafe(obj, scratch);
 622   branchPtr(cond, clasp, scratch, label);
 623 }
 624
 625 void MacroAssembler::branchTestObjClass(Condition cond, Register obj,
 626                                         Register clasp, Register scratch,
 627                                         Register spectreRegToZero,
 628                                         Label* label) {
 629   MOZ_ASSERT(obj != scratch);
 630   MOZ_ASSERT(scratch != spectreRegToZero);
 631
 632   loadObjClassUnsafe(obj, scratch);
 633   branchPtr(cond, clasp, scratch, label);
 634
 635   if (JitOptions.spectreObjectMitigations) {
 636     spectreZeroRegister(cond, scratch, spectreRegToZero);
 637   }
 638 }
 639
 640 void MacroAssembler::branchTestClass(
 641     Condition cond, Register clasp,
 642     std::pair<const JSClass*, const JSClass*> classes, Label* label) {
 643   MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
 644
 645   if (cond == Assembler::Equal) {
 646     branchPtr(Assembler::Equal, clasp, ImmPtr(classes.first), label);
 647     branchPtr(Assembler::Equal, clasp, ImmPtr(classes.second), label);
 648     return;
 649   }
 650
 651   Label isClass;
 652   branchPtr(Assembler::Equal, clasp, ImmPtr(classes.first), &isClass);
 653   branchPtr(Assembler::NotEqual, clasp, ImmPtr(classes.second), label);
 654   bind(&isClass);
 655 }
 656
 657 void MacroAssembler::branchTestObjClass(
 658     Condition cond, Register obj,
 659     std::pair<const JSClass*, const JSClass*> classes, Register scratch,
 660     Register spectreRegToZero, Label* label) {
 661   MOZ_ASSERT(scratch != spectreRegToZero);
 662
 663   branchTestObjClassNoSpectreMitigations(cond, obj, classes, scratch, label);
 664
 665   if (JitOptions.spectreObjectMitigations) {
 666     spectreZeroRegister(cond, scratch, spectreRegToZero);
 667   }
 668 }
 669
 670 void MacroAssembler::branchTestObjClassNoSpectreMitigations(
 671     Condition cond, Register obj,
 672     std::pair<const JSClass*, const JSClass*> classes, Register scratch,
 673     Label* label) {
 674   MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
 675   MOZ_ASSERT(obj != scratch);
 676
 677   loadObjClassUnsafe(obj, scratch);
 678   branchTestClass(cond, scratch, classes, label);
 679 }
 680
 681 void MacroAssembler::branchTestClassIsFunction(Condition cond, Register clasp,
 682                                                Label* label) {
 683   return branchTestClass(cond, clasp, {&FunctionClass, &ExtendedFunctionClass},
 684                          label);
 685 }
 686
 687 void MacroAssembler::branchTestObjIsFunction(Condition cond, Register obj,
 688                                              Register scratch,
 689                                              Register spectreRegToZero,
 690                                              Label* label) {
 691   MOZ_ASSERT(scratch != spectreRegToZero);
 692
 693   branchTestObjIsFunctionNoSpectreMitigations(cond, obj, scratch, label);
 694
 695   if (JitOptions.spectreObjectMitigations) {
 696     spectreZeroRegister(cond, scratch, spectreRegToZero);
 697   }
 698 }
 699
 700 void MacroAssembler::branchTestObjIsFunctionNoSpectreMitigations(
 701     Condition cond, Register obj, Register scratch, Label* label) {
 702   MOZ_ASSERT(cond == Assembler::Equal || cond == Assembler::NotEqual);
 703   MOZ_ASSERT(obj != scratch);
 704
 705   loadObjClassUnsafe(obj, scratch);
 706   branchTestClassIsFunction(cond, scratch, label);
 707 }
 708
 709 void MacroAssembler::branchTestObjShape(Condition cond, Register obj,
 710                                         const Shape* shape, Register scratch,
 711                                         Register spectreRegToZero,
 712                                         Label* label) {
 713   MOZ_ASSERT(obj != scratch);
 714   MOZ_ASSERT(spectreRegToZero != scratch);
 715
 716   if (JitOptions.spectreObjectMitigations) {
 717     move32(Imm32(0), scratch);
 718   }
 719
 720   branchPtr(cond, Address(obj, JSObject::offsetOfShape()), ImmGCPtr(shape),
 721             label);
 722
 723   if (JitOptions.spectreObjectMitigations) {
 724     spectreMovePtr(cond, scratch, spectreRegToZero);
 725   }
 726 }
 727
 728 void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond,
 729                                                             Register obj,
 730                                                             const Shape* shape,
 731                                                             Label* label) {
 732   branchPtr(cond, Address(obj, JSObject::offsetOfShape()), ImmGCPtr(shape),
 733             label);
 734 }
 735
 736 void MacroAssembler::branchTestObjShape(Condition cond, Register obj,
 737                                         Register shape, Register scratch,
 738                                         Register spectreRegToZero,
 739                                         Label* label) {
 740   MOZ_ASSERT(obj != scratch);
 741   MOZ_ASSERT(obj != shape);
 742   MOZ_ASSERT(spectreRegToZero != scratch);
 743
 744   if (JitOptions.spectreObjectMitigations) {
 745     move32(Imm32(0), scratch);
 746   }
 747
 748   branchPtr(cond, Address(obj, JSObject::offsetOfShape()), shape, label);
 749
 750   if (JitOptions.spectreObjectMitigations) {
 751     spectreMovePtr(cond, scratch, spectreRegToZero);
 752   }
 753 }
 754
 755 void MacroAssembler::branchTestObjShapeNoSpectreMitigations(Condition cond,
 756                                                             Register obj,
 757                                                             Register shape,
 758                                                             Label* label) {
 759   branchPtr(cond, Address(obj, JSObject::offsetOfShape()), shape, label);
 760 }
 761
 762 void MacroAssembler::branchTestObjShapeUnsafe(Condition cond, Register obj,
 763                                               Register shape, Label* label) {
 764   branchTestObjShapeNoSpectreMitigations(cond, obj, shape, label);
 765 }
 766
 767 void MacroAssembler::branchTestClassIsProxy(bool proxy, Register clasp,
 768                                             Label* label) {
 769   branchTest32(proxy ? Assembler::NonZero : Assembler::Zero,
 770                Address(clasp, JSClass::offsetOfFlags()),
 771                Imm32(JSCLASS_IS_PROXY), label);
 772 }
 773
 774 void MacroAssembler::branchTestObjectIsProxy(bool proxy, Register object,
 775                                              Register scratch, Label* label) {
 776   constexpr uint32_t ShiftedMask = (Shape::kindMask() << Shape::kindShift());
 777   static_assert(uint32_t(Shape::Kind::Proxy) == 0,
 778                 "branchTest32 below depends on proxy kind being 0");
 779   loadPtr(Address(object, JSObject::offsetOfShape()), scratch);
 780   branchTest32(proxy ? Assembler::Zero : Assembler::NonZero,
 781                Address(scratch, Shape::offsetOfImmutableFlags()),
 782                Imm32(ShiftedMask), label);
 783 }
 784
 785 void MacroAssembler::branchTestProxyHandlerFamily(Condition cond,
 786                                                   Register proxy,
 787                                                   Register scratch,
 788                                                   const void* handlerp,
 789                                                   Label* label) {
 790 #ifdef DEBUG
 791   Label ok;
 792   branchTestObjectIsProxy(true, proxy, scratch, &ok);
 793   assumeUnreachable("Expected ProxyObject in branchTestProxyHandlerFamily");
 794   bind(&ok);
 795 #endif
 796
 797   Address handlerAddr(proxy, ProxyObject::offsetOfHandler());
 798   loadPtr(handlerAddr, scratch);
 799   Address familyAddr(scratch, BaseProxyHandler::offsetOfFamily());
 800   branchPtr(cond, familyAddr, ImmPtr(handlerp), label);
 801 }
 802
 803 void MacroAssembler::branchTestNeedsIncrementalBarrier(Condition cond,
 804                                                        Label* label) {
 805   MOZ_ASSERT(cond == Zero || cond == NonZero);
 806   CompileZone* zone = realm()->zone();
 807   const uint32_t* needsBarrierAddr = zone->addressOfNeedsIncrementalBarrier();
 808   branchTest32(cond, AbsoluteAddress(needsBarrierAddr), Imm32(0x1), label);
 809 }
 810
 811 void MacroAssembler::branchTestNeedsIncrementalBarrierAnyZone(
 812     Condition cond, Label* label, Register scratch) {
 813   MOZ_ASSERT(cond == Zero || cond == NonZero);
 814   if (maybeRealm_) {
 815     branchTestNeedsIncrementalBarrier(cond, label);
 816   } else {
 817     // We are compiling the interpreter or another runtime-wide trampoline, so
 818     // we have to load cx->zone.
 819     loadPtr(AbsoluteAddress(runtime()->addressOfZone()), scratch);
 820     Address needsBarrierAddr(scratch, Zone::offsetOfNeedsIncrementalBarrier());
 821     branchTest32(cond, needsBarrierAddr, Imm32(0x1), label);
 822   }
 823 }
 824
 825 void MacroAssembler::branchTestMagicValue(Condition cond,
 826                                           const ValueOperand& val,
 827                                           JSWhyMagic why, Label* label) {
 828   MOZ_ASSERT(cond == Equal || cond == NotEqual);
 829   branchTestValue(cond, val, MagicValue(why), label);
 830 }
 831
 832 void MacroAssembler::branchDoubleNotInInt64Range(Address src, Register temp,
 833                                                  Label* fail) {
 834   using mozilla::FloatingPoint;
 835
 836   // Tests if double is in [INT64_MIN; INT64_MAX] range
 837   uint32_t EXPONENT_MASK = 0x7ff00000;
 838   uint32_t EXPONENT_SHIFT = FloatingPoint<double>::kExponentShift - 32;
 839   uint32_t TOO_BIG_EXPONENT = (FloatingPoint<double>::kExponentBias + 63)
 840                               << EXPONENT_SHIFT;
 841
 842   load32(Address(src.base, src.offset + sizeof(int32_t)), temp);
 843   and32(Imm32(EXPONENT_MASK), temp);
 844   branch32(Assembler::GreaterThanOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
 845 }
 846
 847 void MacroAssembler::branchDoubleNotInUInt64Range(Address src, Register temp,
 848                                                   Label* fail) {
 849   using mozilla::FloatingPoint;
 850
 851   // Note: returns failure on -0.0
 852   // Tests if double is in [0; UINT64_MAX] range
 853   // Take the sign also in the equation. That way we can compare in one test?
 854   uint32_t EXPONENT_MASK = 0xfff00000;
 855   uint32_t EXPONENT_SHIFT = FloatingPoint<double>::kExponentShift - 32;
 856   uint32_t TOO_BIG_EXPONENT = (FloatingPoint<double>::kExponentBias + 64)
 857                               << EXPONENT_SHIFT;
 858
 859   load32(Address(src.base, src.offset + sizeof(int32_t)), temp);
 860   and32(Imm32(EXPONENT_MASK), temp);
 861   branch32(Assembler::AboveOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
 862 }
 863
 864 void MacroAssembler::branchFloat32NotInInt64Range(Address src, Register temp,
 865                                                   Label* fail) {
 866   using mozilla::FloatingPoint;
 867
 868   // Tests if float is in [INT64_MIN; INT64_MAX] range
 869   uint32_t EXPONENT_MASK = 0x7f800000;
 870   uint32_t EXPONENT_SHIFT = FloatingPoint<float>::kExponentShift;
 871   uint32_t TOO_BIG_EXPONENT = (FloatingPoint<float>::kExponentBias + 63)
 872                               << EXPONENT_SHIFT;
 873
 874   load32(src, temp);
 875   and32(Imm32(EXPONENT_MASK), temp);
 876   branch32(Assembler::GreaterThanOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
 877 }
 878
 879 void MacroAssembler::branchFloat32NotInUInt64Range(Address src, Register temp,
 880                                                    Label* fail) {
 881   using mozilla::FloatingPoint;
 882
 883   // Note: returns failure on -0.0
 884   // Tests if float is in [0; UINT64_MAX] range
 885   // Take the sign also in the equation. That way we can compare in one test?
 886   uint32_t EXPONENT_MASK = 0xff800000;
 887   uint32_t EXPONENT_SHIFT = FloatingPoint<float>::kExponentShift;
 888   uint32_t TOO_BIG_EXPONENT = (FloatingPoint<float>::kExponentBias + 64)
 889                               << EXPONENT_SHIFT;
 890
 891   load32(src, temp);
 892   and32(Imm32(EXPONENT_MASK), temp);
 893   branch32(Assembler::AboveOrEqual, temp, Imm32(TOO_BIG_EXPONENT), fail);
 894 }
 895
 896 // ========================================================================
 897 // Canonicalization primitives.
 898 void MacroAssembler::canonicalizeFloat(FloatRegister reg) {
 899   Label notNaN;
 900   branchFloat(DoubleOrdered, reg, reg, &notNaN);
 901   loadConstantFloat32(float(JS::GenericNaN()), reg);
 902   bind(&notNaN);
 903 }
 904
 905 void MacroAssembler::canonicalizeFloatIfDeterministic(FloatRegister reg) {
 906   // See the comment in TypedArrayObjectTemplate::getElement.
 907   if (js::SupportDifferentialTesting()) {
 908     canonicalizeFloat(reg);
 909   }
 910 }
 911
 912 void MacroAssembler::canonicalizeDouble(FloatRegister reg) {
 913   Label notNaN;
 914   branchDouble(DoubleOrdered, reg, reg, &notNaN);
 915   loadConstantDouble(JS::GenericNaN(), reg);
 916   bind(&notNaN);
 917 }
 918
 919 void MacroAssembler::canonicalizeDoubleIfDeterministic(FloatRegister reg) {
 920   // See the comment in TypedArrayObjectTemplate::getElement.
 921   if (js::SupportDifferentialTesting()) {
 922     canonicalizeDouble(reg);
 923   }
 924 }
 925
 926 // ========================================================================
 927 // Memory access primitives.
 928 template <class T>
 929 FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src,
 930                                                const T& dest) {
 931   canonicalizeDoubleIfDeterministic(src);
 932   return storeUncanonicalizedDouble(src, dest);
 933 }
 934
 935 template FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src,
 936                                                         const Address& dest);
 937 template FaultingCodeOffset MacroAssembler::storeDouble(FloatRegister src,
 938                                                         const BaseIndex& dest);
 939
 940 template <class T>
 941 void MacroAssembler::boxDouble(FloatRegister src, const T& dest) {
 942   storeDouble(src, dest);
 943 }
 944
 945 template <class T>
 946 FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src,
 947                                                 const T& dest) {
 948   canonicalizeFloatIfDeterministic(src);
 949   return storeUncanonicalizedFloat32(src, dest);
 950 }
 951
 952 template FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src,
 953                                                          const Address& dest);
 954 template FaultingCodeOffset MacroAssembler::storeFloat32(FloatRegister src,
 955                                                          const BaseIndex& dest);
 956
 957 template <typename T>
 958 void MacroAssembler::fallibleUnboxInt32(const T& src, Register dest,
 959                                         Label* fail) {
 960   // Int32Value can be unboxed efficiently with unboxInt32, so use that.
 961   branchTestInt32(Assembler::NotEqual, src, fail);
 962   unboxInt32(src, dest);
 963 }
 964
 965 template <typename T>
 966 void MacroAssembler::fallibleUnboxBoolean(const T& src, Register dest,
 967                                           Label* fail) {
 968   // BooleanValue can be unboxed efficiently with unboxBoolean, so use that.
 969   branchTestBoolean(Assembler::NotEqual, src, fail);
 970   unboxBoolean(src, dest);
 971 }
 972
 973 template <typename T>
 974 void MacroAssembler::fallibleUnboxObject(const T& src, Register dest,
 975                                          Label* fail) {
 976   fallibleUnboxPtr(src, dest, JSVAL_TYPE_OBJECT, fail);
 977 }
 978
 979 template <typename T>
 980 void MacroAssembler::fallibleUnboxString(const T& src, Register dest,
 981                                          Label* fail) {
 982   fallibleUnboxPtr(src, dest, JSVAL_TYPE_STRING, fail);
 983 }
 984
 985 template <typename T>
 986 void MacroAssembler::fallibleUnboxSymbol(const T& src, Register dest,
 987                                          Label* fail) {
 988   fallibleUnboxPtr(src, dest, JSVAL_TYPE_SYMBOL, fail);
 989 }
 990
 991 template <typename T>
 992 void MacroAssembler::fallibleUnboxBigInt(const T& src, Register dest,
 993                                          Label* fail) {
 994   fallibleUnboxPtr(src, dest, JSVAL_TYPE_BIGINT, fail);
 995 }
 996
 997 //}}} check_macroassembler_style
 998 // ===============================================================
 999
1000 #ifndef JS_CODEGEN_ARM64
1001
1002 template <typename T>
1003 void MacroAssembler::branchTestStackPtr(Condition cond, T t, Label* label) {
1004   branchTestPtr(cond, getStackPointer(), t, label);
1005 }
1006
1007 template <typename T>
1008 void MacroAssembler::branchStackPtr(Condition cond, T rhs, Label* label) {
1009   branchPtr(cond, getStackPointer(), rhs, label);
1010 }
1011
1012 template <typename T>
1013 void MacroAssembler::branchStackPtrRhs(Condition cond, T lhs, Label* label) {
1014   branchPtr(cond, lhs, getStackPointer(), label);
1015 }
1016
1017 template <typename T>
1018 void MacroAssembler::addToStackPtr(T t) {
1019   addPtr(t, getStackPointer());
1020 }
1021
1022 template <typename T>
1023 void MacroAssembler::addStackPtrTo(T t) {
1024   addPtr(getStackPointer(), t);
1025 }
1026
1027 void MacroAssembler::reserveStack(uint32_t amount) {
1028   subFromStackPtr(Imm32(amount));
1029   adjustFrame(amount);
1030 }
1031 #endif  // !JS_CODEGEN_ARM64
1032
1033 void MacroAssembler::loadObjClassUnsafe(Register obj, Register dest) {
1034   loadPtr(Address(obj, JSObject::offsetOfShape()), dest);
1035   loadPtr(Address(dest, Shape::offsetOfBaseShape()), dest);
1036   loadPtr(Address(dest, BaseShape::offsetOfClasp()), dest);
1037 }
1038
1039 template <typename EmitPreBarrier>
1040 void MacroAssembler::storeObjShape(Register shape, Register obj,
1041                                    EmitPreBarrier emitPreBarrier) {
1042   MOZ_ASSERT(shape != obj);
1043   Address shapeAddr(obj, JSObject::offsetOfShape());
1044   emitPreBarrier(*this, shapeAddr);
1045   storePtr(shape, shapeAddr);
1046 }
1047
1048 template <typename EmitPreBarrier>
1049 void MacroAssembler::storeObjShape(Shape* shape, Register obj,
1050                                    EmitPreBarrier emitPreBarrier) {
1051   Address shapeAddr(obj, JSObject::offsetOfShape());
1052   emitPreBarrier(*this, shapeAddr);
1053   storePtr(ImmGCPtr(shape), shapeAddr);
1054 }
1055
1056 void MacroAssembler::loadObjProto(Register obj, Register dest) {
1057   loadPtr(Address(obj, JSObject::offsetOfShape()), dest);
1058   loadPtr(Address(dest, Shape::offsetOfBaseShape()), dest);
1059   loadPtr(Address(dest, BaseShape::offsetOfProto()), dest);
1060 }
1061
1062 void MacroAssembler::loadStringLength(Register str, Register dest) {
1063   load32(Address(str, JSString::offsetOfLength()), dest);
1064 }
1065
1066 void MacroAssembler::assertStackAlignment(uint32_t alignment,
1067                                           int32_t offset /* = 0 */) {
1068 #ifdef DEBUG
1069   Label ok, bad;
1070   MOZ_ASSERT(mozilla::IsPowerOfTwo(alignment));
1071
1072   // Wrap around the offset to be a non-negative number.
1073   offset %= alignment;
1074   if (offset < 0) {
1075     offset += alignment;
1076   }
1077
1078   // Test if each bit from offset is set.
1079   uint32_t off = offset;
1080   while (off) {
1081     uint32_t lowestBit = 1 << mozilla::CountTrailingZeroes32(off);
1082     branchTestStackPtr(Assembler::Zero, Imm32(lowestBit), &bad);
1083     off ^= lowestBit;
1084   }
1085
1086   // Check that all remaining bits are zero.
1087   branchTestStackPtr(Assembler::Zero, Imm32((alignment - 1) ^ offset), &ok);
1088
1089   bind(&bad);
1090   breakpoint();
1091   bind(&ok);
1092 #endif
1093 }
1094
1095 void MacroAssembler::storeCallBoolResult(Register reg) {
1096   convertBoolToInt32(ReturnReg, reg);
1097 }
1098
1099 void MacroAssembler::storeCallInt32Result(Register reg) {
1100 #if JS_BITS_PER_WORD == 32
1101   storeCallPointerResult(reg);
1102 #else
1103   // Ensure the upper 32 bits are cleared.
1104   move32(ReturnReg, reg);
1105 #endif
1106 }
1107
1108 void MacroAssembler::storeCallResultValue(AnyRegister dest, JSValueType type) {
1109   unboxValue(JSReturnOperand, dest, type);
1110 }
1111
1112 void MacroAssembler::storeCallResultValue(TypedOrValueRegister dest) {
1113   if (dest.hasValue()) {
1114     storeCallResultValue(dest.valueReg());
1115   } else {
1116     storeCallResultValue(dest.typedReg(), ValueTypeFromMIRType(dest.type()));
1117   }
1118 }
1119
1120 }  // namespace jit
1121 }  // namespace js
1122
1123 #endif /* jit_MacroAssembler_inl_h */