js/src/jit/CodeGenerator.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef jit_CodeGenerator_h
   8 #define jit_CodeGenerator_h
   9
  10 #include "jit/PerfSpewer.h"
  11 #include "js/ScalarType.h"  // js::Scalar::Type
  12
  13 #if defined(JS_CODEGEN_X86)
  14 #  include "jit/x86/CodeGenerator-x86.h"
  15 #elif defined(JS_CODEGEN_X64)
  16 #  include "jit/x64/CodeGenerator-x64.h"
  17 #elif defined(JS_CODEGEN_ARM)
  18 #  include "jit/arm/CodeGenerator-arm.h"
  19 #elif defined(JS_CODEGEN_ARM64)
  20 #  include "jit/arm64/CodeGenerator-arm64.h"
  21 #elif defined(JS_CODEGEN_MIPS32)
  22 #  include "jit/mips32/CodeGenerator-mips32.h"
  23 #elif defined(JS_CODEGEN_MIPS64)
  24 #  include "jit/mips64/CodeGenerator-mips64.h"
  25 #elif defined(JS_CODEGEN_LOONG64)
  26 #  include "jit/loong64/CodeGenerator-loong64.h"
  27 #elif defined(JS_CODEGEN_RISCV64)
  28 #  include "jit/riscv64/CodeGenerator-riscv64.h"
  29 #elif defined(JS_CODEGEN_WASM32)
  30 #  include "jit/wasm32/CodeGenerator-wasm32.h"
  31 #elif defined(JS_CODEGEN_NONE)
  32 #  include "jit/none/CodeGenerator-none.h"
  33 #else
  34 #  error "Unknown architecture!"
  35 #endif
  36
  37 namespace js {
  38
  39 namespace wasm {
  40 class Decoder;
  41 class StackMaps;
  42 }  // namespace wasm
  43
  44 namespace jit {
  45
  46 class WarpSnapshot;
  47
  48 template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
  49 class OutOfLineCallVM;
  50
  51 enum class SwitchTableType { Inline, OutOfLine };
  52
  53 template <SwitchTableType tableType>
  54 class OutOfLineSwitch;
  55 class OutOfLineTestObject;
  56 class OutOfLineNewArray;
  57 class OutOfLineNewObject;
  58 class CheckOverRecursedFailure;
  59 class OutOfLineUnboxFloatingPoint;
  60 class OutOfLineStoreElementHole;
  61 class OutOfLineTypeOfV;
  62 class OutOfLineTypeOfIsNonPrimitiveV;
  63 class OutOfLineTypeOfIsNonPrimitiveO;
  64 class OutOfLineUpdateCache;
  65 class OutOfLineICFallback;
  66 class OutOfLineCallPostWriteBarrier;
  67 class OutOfLineCallPostWriteElementBarrier;
  68 class OutOfLineElementPostWriteBarrier;
  69 class OutOfLineIsCallable;
  70 class OutOfLineIsConstructor;
  71 class OutOfLineRegExpMatcher;
  72 class OutOfLineRegExpSearcher;
  73 class OutOfLineRegExpExecMatch;
  74 class OutOfLineRegExpExecTest;
  75 class OutOfLineRegExpPrototypeOptimizable;
  76 class OutOfLineRegExpInstanceOptimizable;
  77 class OutOfLineNaNToZero;
  78 class OutOfLineResumableWasmTrap;
  79 class OutOfLineAbortingWasmTrap;
  80 class OutOfLineGuardNumberToIntPtrIndex;
  81 class OutOfLineBoxNonStrictThis;
  82 class OutOfLineArrayPush;
  83 class OutOfLineAtomizeSlot;
  84 class OutOfLineWasmCallPostWriteBarrier;
  85 class OutOfLineWasmNewStruct;
  86
  87 class CodeGenerator final : public CodeGeneratorSpecific {
  88   [[nodiscard]] bool generateBody();
  89
  90   ConstantOrRegister toConstantOrRegister(LInstruction* lir, size_t n,
  91                                           MIRType type);
  92
  93 #ifdef CHECK_OSIPOINT_REGISTERS
  94   void resetOsiPointRegs(LSafepoint* safepoint);
  95   bool shouldVerifyOsiPointRegs(LSafepoint* safepoint);
  96   void verifyOsiPointRegs(LSafepoint* safepoint);
  97 #endif
  98
  99   void callVMInternal(VMFunctionId id, LInstruction* ins);
 100
 101   template <typename Fn, Fn fn>
 102   void callVM(LInstruction* ins);
 103
 104   template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
 105   inline OutOfLineCode* oolCallVM(LInstruction* ins, const ArgSeq& args,
 106                                   const StoreOutputTo& out);
 107
 108   template <typename LCallIns>
 109   void emitCallNative(LCallIns* call, JSNative native);
 110
 111  public:
 112   CodeGenerator(MIRGenerator* gen, LIRGraph* graph,
 113                 MacroAssembler* masm = nullptr);
 114   ~CodeGenerator();
 115
 116   [[nodiscard]] bool generate();
 117   [[nodiscard]] bool generateWasm(
 118       wasm::CallIndirectId callIndirectId, wasm::BytecodeOffset trapOffset,
 119       const wasm::ArgTypeVector& argTys, const RegisterOffsets& trapExitLayout,
 120       size_t trapExitLayoutNumWords, wasm::FuncOffsets* offsets,
 121       wasm::StackMaps* stackMaps, wasm::Decoder* decoder);
 122
 123   [[nodiscard]] bool link(JSContext* cx, const WarpSnapshot* snapshot);
 124
 125   void emitOOLTestObject(Register objreg, Label* ifTruthy, Label* ifFalsy,
 126                          Register scratch);
 127
 128   void emitTypeOfCheck(JSValueType type, Register tag, Register output,
 129                        Label* done, Label* oolObject);
 130   void emitTypeOfJSType(JSValueType type, Register output);
 131   void emitTypeOfObject(Register obj, Register output, Label* done);
 132   void emitTypeOfIsObject(MTypeOfIs* mir, Register obj, Register output,
 133                           Label* success, Label* fail, Label* slowCheck);
 134   void emitTypeOfIsObjectOOL(MTypeOfIs* mir, Register obj, Register output);
 135
 136   template <typename Fn, Fn fn, class ArgSeq, class StoreOutputTo>
 137   void visitOutOfLineCallVM(
 138       OutOfLineCallVM<Fn, fn, ArgSeq, StoreOutputTo>* ool);
 139
 140   void visitOutOfLineRegExpMatcher(OutOfLineRegExpMatcher* ool);
 141   void visitOutOfLineRegExpSearcher(OutOfLineRegExpSearcher* ool);
 142   void visitOutOfLineRegExpExecMatch(OutOfLineRegExpExecMatch* ool);
 143   void visitOutOfLineRegExpExecTest(OutOfLineRegExpExecTest* ool);
 144   void visitOutOfLineRegExpPrototypeOptimizable(
 145       OutOfLineRegExpPrototypeOptimizable* ool);
 146   void visitOutOfLineRegExpInstanceOptimizable(
 147       OutOfLineRegExpInstanceOptimizable* ool);
 148
 149   void visitOutOfLineTypeOfV(OutOfLineTypeOfV* ool);
 150   void visitOutOfLineTypeOfIsNonPrimitiveV(OutOfLineTypeOfIsNonPrimitiveV* ool);
 151   void visitOutOfLineTypeOfIsNonPrimitiveO(OutOfLineTypeOfIsNonPrimitiveO* ool);
 152
 153   template <SwitchTableType tableType>
 154   void visitOutOfLineSwitch(OutOfLineSwitch<tableType>* ool);
 155
 156   void visitOutOfLineIsCallable(OutOfLineIsCallable* ool);
 157   void visitOutOfLineIsConstructor(OutOfLineIsConstructor* ool);
 158
 159   void visitOutOfLineNaNToZero(OutOfLineNaNToZero* ool);
 160
 161   void visitOutOfLineResumableWasmTrap(OutOfLineResumableWasmTrap* ool);
 162   void visitOutOfLineAbortingWasmTrap(OutOfLineAbortingWasmTrap* ool);
 163   void visitCheckOverRecursedFailure(CheckOverRecursedFailure* ool);
 164
 165   void visitOutOfLineUnboxFloatingPoint(OutOfLineUnboxFloatingPoint* ool);
 166   void visitOutOfLineStoreElementHole(OutOfLineStoreElementHole* ool);
 167
 168   void visitOutOfLineBoxNonStrictThis(OutOfLineBoxNonStrictThis* ool);
 169
 170   void visitOutOfLineICFallback(OutOfLineICFallback* ool);
 171
 172   void visitOutOfLineCallPostWriteBarrier(OutOfLineCallPostWriteBarrier* ool);
 173   void visitOutOfLineCallPostWriteElementBarrier(
 174       OutOfLineCallPostWriteElementBarrier* ool);
 175
 176   void visitOutOfLineElementPostWriteBarrier(
 177       OutOfLineElementPostWriteBarrier* ool);
 178
 179   void visitOutOfLineNewArray(OutOfLineNewArray* ool);
 180   void visitOutOfLineNewObject(OutOfLineNewObject* ool);
 181
 182   void visitOutOfLineGuardNumberToIntPtrIndex(
 183       OutOfLineGuardNumberToIntPtrIndex* ool);
 184
 185   void visitOutOfLineArrayPush(OutOfLineArrayPush* ool);
 186
 187   void visitOutOfLineAtomizeSlot(OutOfLineAtomizeSlot* ool);
 188
 189   void visitOutOfLineWasmCallPostWriteBarrier(
 190       OutOfLineWasmCallPostWriteBarrier* ool);
 191
 192   void callWasmStructAllocFun(LInstruction* lir, wasm::SymbolicAddress fun,
 193                               Register typeDefData, Register output);
 194   void visitOutOfLineWasmNewStruct(OutOfLineWasmNewStruct* ool);
 195
 196  private:
 197   void emitPostWriteBarrier(const LAllocation* obj);
 198   void emitPostWriteBarrier(Register objreg);
 199   void emitPostWriteBarrierS(Address address, Register prev, Register next);
 200
 201   void emitElementPostWriteBarrier(MInstruction* mir,
 202                                    const LiveRegisterSet& liveVolatileRegs,
 203                                    Register obj, const LAllocation* index,
 204                                    Register scratch,
 205                                    const ConstantOrRegister& val,
 206                                    int32_t indexDiff = 0);
 207
 208   template <class LPostBarrierType, MIRType nurseryType>
 209   void visitPostWriteBarrierCommon(LPostBarrierType* lir, OutOfLineCode* ool);
 210   template <class LPostBarrierType>
 211   void visitPostWriteBarrierCommonV(LPostBarrierType* lir, OutOfLineCode* ool);
 212
 213   void emitCallInvokeFunction(LInstruction* call, Register callereg,
 214                               bool isConstructing, bool ignoresReturnValue,
 215                               uint32_t argc, uint32_t unusedStack);
 216   template <typename T>
 217   void emitApplyGeneric(T* apply);
 218   template <typename T>
 219   void emitCallInvokeFunction(T* apply);
 220   void emitAllocateSpaceForApply(Register argcreg, Register scratch);
 221   void emitAllocateSpaceForConstructAndPushNewTarget(
 222       Register argcreg, Register newTargetAndScratch);
 223   void emitCopyValuesForApply(Register argvSrcBase, Register argvIndex,
 224                               Register copyreg, size_t argvSrcOffset,
 225                               size_t argvDstOffset);
 226   void emitRestoreStackPointerFromFP();
 227   void emitPushArguments(Register argcreg, Register scratch, Register copyreg,
 228                          uint32_t extraFormals);
 229   void emitPushArrayAsArguments(Register tmpArgc, Register srcBaseAndArgc,
 230                                 Register scratch, size_t argvSrcOffset);
 231   void emitPushArguments(LApplyArgsGeneric* apply, Register scratch);
 232   void emitPushArguments(LApplyArgsObj* apply, Register scratch);
 233   void emitPushArguments(LApplyArrayGeneric* apply, Register scratch);
 234   void emitPushArguments(LConstructArgsGeneric* construct, Register scratch);
 235   void emitPushArguments(LConstructArrayGeneric* construct, Register scratch);
 236
 237   template <class GetInlinedArgument>
 238   void emitGetInlinedArgument(GetInlinedArgument* lir, Register index,
 239                               ValueOperand output);
 240
 241   void emitMaybeAtomizeSlot(LInstruction* ins, Register stringReg,
 242                             Address slotAddr, TypedOrValueRegister dest);
 243
 244   using RegisterOrInt32 = mozilla::Variant<Register, int32_t>;
 245
 246   static RegisterOrInt32 ToRegisterOrInt32(const LAllocation* allocation);
 247
 248 #ifdef DEBUG
 249   void emitAssertArgumentsSliceBounds(const RegisterOrInt32& begin,
 250                                       const RegisterOrInt32& count,
 251                                       Register numActualArgs);
 252 #endif
 253
 254   template <class ArgumentsSlice>
 255   void emitNewArray(ArgumentsSlice* lir, const RegisterOrInt32& count,
 256                     Register output, Register temp);
 257
 258   void visitNewArrayCallVM(LNewArray* lir);
 259   void visitNewObjectVMCall(LNewObject* lir);
 260
 261   void emitConcat(LInstruction* lir, Register lhs, Register rhs,
 262                   Register output);
 263
 264   void emitInstanceOf(LInstruction* ins, Register protoReg);
 265
 266   void loadJSScriptForBlock(MBasicBlock* block, Register reg);
 267   void loadOutermostJSScript(Register reg);
 268
 269 #ifdef DEBUG
 270   void emitAssertResultV(const ValueOperand output, const MDefinition* mir);
 271   void emitAssertGCThingResult(Register input, const MDefinition* mir);
 272 #endif
 273
 274 #ifdef DEBUG
 275   void emitDebugForceBailing(LInstruction* lir);
 276 #endif
 277
 278   IonScriptCounts* extractScriptCounts() {
 279     IonScriptCounts* counts = scriptCounts_;
 280     scriptCounts_ = nullptr;  // prevent delete in dtor
 281     return counts;
 282   }
 283
 284   void addGetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs,
 285                            TypedOrValueRegister value,
 286                            const ConstantOrRegister& id, ValueOperand output);
 287   void addSetPropertyCache(LInstruction* ins, LiveRegisterSet liveRegs,
 288                            Register objReg, Register temp,
 289                            const ConstantOrRegister& id,
 290                            const ConstantOrRegister& value, bool strict);
 291
 292   template <class IteratorObject, class OrderedHashTable>
 293   void emitGetNextEntryForIterator(LGetNextEntryForIterator* lir);
 294
 295   template <class OrderedHashTable>
 296   void emitLoadIteratorValues(Register result, Register temp, Register front);
 297
 298   void emitStringToInt64(LInstruction* lir, Register input, Register64 output);
 299
 300   OutOfLineCode* createBigIntOutOfLine(LInstruction* lir, Scalar::Type type,
 301                                        Register64 input, Register output);
 302
 303   void emitCreateBigInt(LInstruction* lir, Scalar::Type type, Register64 input,
 304                         Register output, Register maybeTemp);
 305
 306   template <size_t NumDefs>
 307   void emitIonToWasmCallBase(LIonToWasmCallBase<NumDefs>* lir);
 308
 309   IonScriptCounts* maybeCreateScriptCounts();
 310
 311   void emitWasmCompareAndSelect(LWasmCompareAndSelect* ins);
 312
 313   void testValueTruthyForType(JSValueType type, ScratchTagScope& tag,
 314                               const ValueOperand& value, Register tempToUnbox,
 315                               Register temp, FloatRegister floatTemp,
 316                               Label* ifTruthy, Label* ifFalsy,
 317                               OutOfLineTestObject* ool, bool skipTypeTest);
 318
 319   // Test whether value is truthy or not and jump to the corresponding label.
 320   // The control flow falls through when the object is truthy, as an
 321   // optimization.
 322   void testValueTruthy(const ValueOperand& value, Register tempToUnbox,
 323                        Register temp, FloatRegister floatTemp,
 324                        const TypeDataList& observedTypes, Label* ifTruthy,
 325                        Label* ifFalsy, OutOfLineTestObject* ool);
 326
 327   // This function behaves like testObjectEmulatesUndefined with the exception
 328   // that it can choose to let control flow fall through when the object
 329   // doesn't emulate undefined, as an optimization. Use the regular
 330   // testObjectEmulatesUndefined when it's required to branch to one of the
 331   // two labels.
 332   void testObjectEmulatesUndefinedKernel(Register objreg,
 333                                          Label* ifEmulatesUndefined,
 334                                          Label* ifDoesntEmulateUndefined,
 335                                          Register scratch,
 336                                          OutOfLineTestObject* ool);
 337
 338   // Test whether an object emulates |undefined|.  If it does, jump to
 339   // |ifEmulatesUndefined|; the caller is responsible for binding this label.
 340   // If it doesn't, fall through; the label |ifDoesntEmulateUndefined| (which
 341   // must be initially unbound) will be bound at this point.
 342   void branchTestObjectEmulatesUndefined(Register objreg,
 343                                          Label* ifEmulatesUndefined,
 344                                          Label* ifDoesntEmulateUndefined,
 345                                          Register scratch,
 346                                          OutOfLineTestObject* ool);
 347
 348   // Test whether an object emulates |undefined|, and jump to the
 349   // corresponding label.
 350   //
 351   // This method should be used when subsequent code can't be laid out in a
 352   // straight line; if it can, branchTest* should be used instead.
 353   void testObjectEmulatesUndefined(Register objreg, Label* ifEmulatesUndefined,
 354                                    Label* ifDoesntEmulateUndefined,
 355                                    Register scratch, OutOfLineTestObject* ool);
 356
 357   void emitStoreElementTyped(const LAllocation* value, MIRType valueType,
 358                              Register elements, const LAllocation* index);
 359
 360   // Bailout if an element about to be written to is a hole.
 361   void emitStoreHoleCheck(Register elements, const LAllocation* index,
 362                           LSnapshot* snapshot);
 363
 364   void emitAssertRangeI(MIRType type, const Range* r, Register input);
 365   void emitAssertRangeD(const Range* r, FloatRegister input,
 366                         FloatRegister temp);
 367
 368   void maybeEmitGlobalBarrierCheck(const LAllocation* maybeGlobal,
 369                                    OutOfLineCode* ool);
 370
 371   void incrementWarmUpCounter(AbsoluteAddress warmUpCount, JSScript* script,
 372                               Register tmp);
 373
 374   Vector<CodeOffset, 0, JitAllocPolicy> ionScriptLabels_;
 375
 376   // Used to bake in a pointer into the IonScript's list of nursery objects, for
 377   // MNurseryObject codegen.
 378   struct NurseryObjectLabel {
 379     CodeOffset offset;
 380     uint32_t nurseryIndex;
 381     NurseryObjectLabel(CodeOffset offset, uint32_t nurseryIndex)
 382         : offset(offset), nurseryIndex(nurseryIndex) {}
 383   };
 384   Vector<NurseryObjectLabel, 0, JitAllocPolicy> ionNurseryObjectLabels_;
 385
 386   void branchIfInvalidated(Register temp, Label* invalidated);
 387
 388 #ifdef DEBUG
 389   void emitDebugResultChecks(LInstruction* ins);
 390   void emitGCThingResultChecks(LInstruction* lir, MDefinition* mir);
 391   void emitValueResultChecks(LInstruction* lir, MDefinition* mir);
 392 #endif
 393
 394   // Script counts created during code generation.
 395   IonScriptCounts* scriptCounts_;
 396
 397   IonPerfSpewer perfSpewer_;
 398
 399   // Bit mask of JitZone stubs that are to be read-barriered.
 400   uint32_t zoneStubsToReadBarrier_;
 401
 402 #ifdef FUZZING_JS_FUZZILLI
 403   void emitFuzzilliHashDouble(FloatRegister floatDouble, Register scratch,
 404                               Register output);
 405   void emitFuzzilliHashObject(LInstruction* lir, Register obj, Register output);
 406   void emitFuzzilliHashBigInt(Register bigInt, Register output);
 407 #endif
 408
 409 #define LIR_OP(op) void visit##op(L##op* ins);
 410   LIR_OPCODE_LIST(LIR_OP)
 411 #undef LIR_OP
 412
 413   // In debug mode, we need to validate that we've not made a mistake with the
 414   // fuse.
 415   void assertObjectDoesNotEmulateUndefined(Register input, Register temp,
 416                                            const MInstruction* mir);
 417
 418   // Enumerates the fuses that a code generation can depend on. These will
 419   // be mapped to an actual fuse by validateAndRegisterFuseDependencies.
 420   enum class FuseDependencyKind {
 421     HasSeenObjectEmulateUndefinedFuse,
 422   };
 423
 424   // The set of fuses this code generation depends on.
 425   mozilla::EnumSet<FuseDependencyKind> fuseDependencies;
 426
 427   // Register a dependency on the HasSeenObjectEmulateUndefined fuse.
 428   void addHasSeenObjectEmulateUndefinedFuseDependency() {
 429     fuseDependencies += FuseDependencyKind::HasSeenObjectEmulateUndefinedFuse;
 430   }
 431
 432   // Called during linking on main-thread: Ensures that the fuses are still
 433   // intact, and registers a script dependency on a specific fuse before
 434   // finishing compilation.
 435   void validateAndRegisterFuseDependencies(JSContext* cx, HandleScript script,
 436                                            bool* isValid);
 437
 438   // Return true if the fuse is intact, andd if the fuse is intact note the
 439   // dependency
 440   bool hasSeenObjectEmulateUndefinedFuseIntactAndDependencyNoted() {
 441     if (!JitOptions.useHasSeenEmulatesUndefinedFuse) {
 442       // if we're not active, simply pretend the fuse is popped.
 443       return false;
 444     }
 445
 446     bool intact = gen->outerInfo().hasSeenObjectEmulateUndefinedFuseIntact();
 447     if (intact) {
 448       addHasSeenObjectEmulateUndefinedFuseDependency();
 449     }
 450     return intact;
 451   }
 452 };
 453
 454 class OutOfLineResumableWasmTrap : public OutOfLineCodeBase<CodeGenerator> {
 455   LInstruction* lir_;
 456   size_t framePushed_;
 457   wasm::BytecodeOffset bytecodeOffset_;
 458   wasm::Trap trap_;
 459
 460  public:
 461   OutOfLineResumableWasmTrap(LInstruction* lir, size_t framePushed,
 462                              wasm::BytecodeOffset bytecodeOffset,
 463                              wasm::Trap trap)
 464       : lir_(lir),
 465         framePushed_(framePushed),
 466         bytecodeOffset_(bytecodeOffset),
 467         trap_(trap) {}
 468
 469   void accept(CodeGenerator* codegen) override {
 470     codegen->visitOutOfLineResumableWasmTrap(this);
 471   }
 472   LInstruction* lir() const { return lir_; }
 473   size_t framePushed() const { return framePushed_; }
 474   wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
 475   wasm::Trap trap() const { return trap_; }
 476 };
 477
 478 class OutOfLineAbortingWasmTrap : public OutOfLineCodeBase<CodeGenerator> {
 479   wasm::BytecodeOffset bytecodeOffset_;
 480   wasm::Trap trap_;
 481
 482  public:
 483   OutOfLineAbortingWasmTrap(wasm::BytecodeOffset bytecodeOffset,
 484                             wasm::Trap trap)
 485       : bytecodeOffset_(bytecodeOffset), trap_(trap) {}
 486
 487   void accept(CodeGenerator* codegen) override {
 488     codegen->visitOutOfLineAbortingWasmTrap(this);
 489   }
 490   wasm::BytecodeOffset bytecodeOffset() const { return bytecodeOffset_; }
 491   wasm::Trap trap() const { return trap_; }
 492 };
 493
 494 }  // namespace jit
 495 }  // namespace js
 496
 497 #endif /* jit_CodeGenerator_h */