js/src/jit/WarpBuilder.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "jit/WarpBuilder.h"
   8
   9 #include "mozilla/DebugOnly.h"
  10
  11 #include "jit/BaselineFrame.h"
  12 #include "jit/CacheIR.h"
  13 #include "jit/CompileInfo.h"
  14 #include "jit/InlineScriptTree.h"
  15 #include "jit/MIR-wasm.h"
  16 #include "jit/MIR.h"
  17 #include "jit/MIRGenerator.h"
  18 #include "jit/MIRGraph.h"
  19 #include "jit/WarpCacheIRTranspiler.h"
  20 #include "jit/WarpSnapshot.h"
  21 #include "js/friend/ErrorMessages.h"  // JSMSG_BAD_CONST_ASSIGN
  22 #include "vm/GeneratorObject.h"
  23 #include "vm/Interpreter.h"
  24 #include "vm/Opcodes.h"
  25 #include "vm/TypeofEqOperand.h"  // TypeofEqOperand
  26
  27 #include "gc/ObjectKind-inl.h"
  28 #include "vm/BytecodeIterator-inl.h"
  29 #include "vm/BytecodeLocation-inl.h"
  30 #include "vm/JSObject-inl.h"
  31
  32 using namespace js;
  33 using namespace js::jit;
  34
  35 // Used for building the outermost script.
  36 WarpBuilder::WarpBuilder(WarpSnapshot& snapshot, MIRGenerator& mirGen,
  37                          WarpCompilation* warpCompilation)
  38     : WarpBuilderShared(snapshot, mirGen, nullptr),
  39       warpCompilation_(warpCompilation),
  40       graph_(mirGen.graph()),
  41       info_(mirGen.outerInfo()),
  42       scriptSnapshot_(snapshot.rootScript()),
  43       script_(snapshot.rootScript()->script()),
  44       loopStack_(mirGen.alloc()) {
  45   opSnapshotIter_ = scriptSnapshot_->opSnapshots().getFirst();
  46 }
  47
  48 // Used for building inlined scripts.
  49 WarpBuilder::WarpBuilder(WarpBuilder* caller, WarpScriptSnapshot* snapshot,
  50                          CompileInfo& compileInfo, CallInfo* inlineCallInfo,
  51                          MResumePoint* callerResumePoint)
  52     : WarpBuilderShared(caller->snapshot(), caller->mirGen(), nullptr),
  53       warpCompilation_(caller->warpCompilation()),
  54       graph_(caller->mirGen().graph()),
  55       info_(compileInfo),
  56       scriptSnapshot_(snapshot),
  57       script_(snapshot->script()),
  58       loopStack_(caller->mirGen().alloc()),
  59       callerBuilder_(caller),
  60       callerResumePoint_(callerResumePoint),
  61       inlineCallInfo_(inlineCallInfo) {
  62   opSnapshotIter_ = snapshot->opSnapshots().getFirst();
  63 }
  64
  65 BytecodeSite* WarpBuilder::newBytecodeSite(BytecodeLocation loc) {
  66   jsbytecode* pc = loc.toRawBytecode();
  67   MOZ_ASSERT(info().inlineScriptTree()->script()->containsPC(pc));
  68   return new (alloc()) BytecodeSite(info().inlineScriptTree(), pc);
  69 }
  70
  71 const WarpOpSnapshot* WarpBuilder::getOpSnapshotImpl(
  72     BytecodeLocation loc, WarpOpSnapshot::Kind kind) {
  73   uint32_t offset = loc.bytecodeToOffset(script_);
  74
  75   // Skip snapshots until we get to a snapshot with offset >= offset. This is
  76   // a loop because WarpBuilder can skip unreachable bytecode ops.
  77   while (opSnapshotIter_ && opSnapshotIter_->offset() < offset) {
  78     opSnapshotIter_ = opSnapshotIter_->getNext();
  79   }
  80
  81   if (!opSnapshotIter_ || opSnapshotIter_->offset() != offset ||
  82       opSnapshotIter_->kind() != kind) {
  83     return nullptr;
  84   }
  85
  86   return opSnapshotIter_;
  87 }
  88
  89 void WarpBuilder::initBlock(MBasicBlock* block) {
  90   graph().addBlock(block);
  91
  92   block->setLoopDepth(loopDepth());
  93
  94   current = block;
  95 }
  96
  97 bool WarpBuilder::startNewBlock(MBasicBlock* predecessor, BytecodeLocation loc,
  98                                 size_t numToPop) {
  99   MBasicBlock* block =
 100       MBasicBlock::NewPopN(graph(), info(), predecessor, newBytecodeSite(loc),
 101                            MBasicBlock::NORMAL, numToPop);
 102   if (!block) {
 103     return false;
 104   }
 105
 106   initBlock(block);
 107   return true;
 108 }
 109
 110 bool WarpBuilder::startNewEntryBlock(size_t stackDepth, BytecodeLocation loc) {
 111   MBasicBlock* block =
 112       MBasicBlock::New(graph(), stackDepth, info(), /* maybePred = */ nullptr,
 113                        newBytecodeSite(loc), MBasicBlock::NORMAL);
 114   if (!block) {
 115     return false;
 116   }
 117
 118   initBlock(block);
 119   return true;
 120 }
 121
 122 bool WarpBuilder::startNewLoopHeaderBlock(BytecodeLocation loopHead) {
 123   MBasicBlock* header = MBasicBlock::NewPendingLoopHeader(
 124       graph(), info(), current, newBytecodeSite(loopHead));
 125   if (!header) {
 126     return false;
 127   }
 128
 129   initBlock(header);
 130   return loopStack_.emplaceBack(header);
 131 }
 132
 133 bool WarpBuilder::startNewOsrPreHeaderBlock(BytecodeLocation loopHead) {
 134   MOZ_ASSERT(loopHead.is(JSOp::LoopHead));
 135   MOZ_ASSERT(loopHead.toRawBytecode() == info().osrPc());
 136
 137   // Create two blocks:
 138   // * The OSR entry block. This is always the graph's second block and has no
 139   //   predecessors. This is the entry point for OSR from the Baseline JIT.
 140   // * The OSR preheader block. This has two predecessors: the OSR entry block
 141   //   and the current block.
 142
 143   MBasicBlock* pred = current;
 144
 145   // Create the OSR entry block.
 146   if (!startNewEntryBlock(pred->stackDepth(), loopHead)) {
 147     return false;
 148   }
 149
 150   MBasicBlock* osrBlock = current;
 151   graph().setOsrBlock(osrBlock);
 152   graph().moveBlockAfter(*graph().begin(), osrBlock);
 153
 154   MOsrEntry* entry = MOsrEntry::New(alloc());
 155   osrBlock->add(entry);
 156
 157   // Initialize environment chain.
 158   {
 159     uint32_t slot = info().environmentChainSlot();
 160     MInstruction* envv;
 161     if (usesEnvironmentChain()) {
 162       envv = MOsrEnvironmentChain::New(alloc(), entry);
 163     } else {
 164       // Use an undefined value if the script does not need its environment
 165       // chain, to match the main entry point.
 166       envv = MConstant::New(alloc(), UndefinedValue());
 167     }
 168     osrBlock->add(envv);
 169     osrBlock->initSlot(slot, envv);
 170   }
 171
 172   // Initialize return value.
 173   {
 174     MInstruction* returnValue;
 175     if (!script_->noScriptRval()) {
 176       returnValue = MOsrReturnValue::New(alloc(), entry);
 177     } else {
 178       returnValue = MConstant::New(alloc(), UndefinedValue());
 179     }
 180     osrBlock->add(returnValue);
 181     osrBlock->initSlot(info().returnValueSlot(), returnValue);
 182   }
 183
 184   // Initialize arguments object.
 185   MInstruction* argsObj = nullptr;
 186   if (info().needsArgsObj()) {
 187     argsObj = MOsrArgumentsObject::New(alloc(), entry);
 188     osrBlock->add(argsObj);
 189     osrBlock->initSlot(info().argsObjSlot(), argsObj);
 190   }
 191
 192   if (info().hasFunMaybeLazy()) {
 193     // Initialize |this| parameter.
 194     MParameter* thisv = MParameter::New(alloc(), MParameter::THIS_SLOT);
 195     osrBlock->add(thisv);
 196     osrBlock->initSlot(info().thisSlot(), thisv);
 197
 198     // Initialize arguments. There are three cases:
 199     //
 200     // 1) There's no ArgumentsObject or it doesn't alias formals. In this case
 201     //    we can just use the frame's argument slot.
 202     // 2) The ArgumentsObject aliases formals and the argument is stored in the
 203     //    CallObject. Use |undefined| because we can't load from the arguments
 204     //    object and code will use the CallObject anyway.
 205     // 3) The ArgumentsObject aliases formals and the argument isn't stored in
 206     //    the CallObject. We have to load it from the ArgumentsObject.
 207     for (uint32_t i = 0; i < info().nargs(); i++) {
 208       uint32_t slot = info().argSlotUnchecked(i);
 209       MInstruction* osrv;
 210       if (!info().argsObjAliasesFormals()) {
 211         osrv = MParameter::New(alloc().fallible(), i);
 212       } else if (script_->formalIsAliased(i)) {
 213         osrv = MConstant::New(alloc().fallible(), UndefinedValue());
 214       } else {
 215         osrv = MGetArgumentsObjectArg::New(alloc().fallible(), argsObj, i);
 216       }
 217       if (!osrv) {
 218         return false;
 219       }
 220       current->add(osrv);
 221       current->initSlot(slot, osrv);
 222     }
 223   }
 224
 225   // Initialize locals.
 226   uint32_t nlocals = info().nlocals();
 227   for (uint32_t i = 0; i < nlocals; i++) {
 228     uint32_t slot = info().localSlot(i);
 229     ptrdiff_t offset = BaselineFrame::reverseOffsetOfLocal(i);
 230     MOsrValue* osrv = MOsrValue::New(alloc().fallible(), entry, offset);
 231     if (!osrv) {
 232       return false;
 233     }
 234     current->add(osrv);
 235     current->initSlot(slot, osrv);
 236   }
 237
 238   // Initialize expression stack slots.
 239   uint32_t numStackSlots = current->stackDepth() - info().firstStackSlot();
 240   for (uint32_t i = 0; i < numStackSlots; i++) {
 241     uint32_t slot = info().stackSlot(i);
 242     ptrdiff_t offset = BaselineFrame::reverseOffsetOfLocal(nlocals + i);
 243     MOsrValue* osrv = MOsrValue::New(alloc().fallible(), entry, offset);
 244     if (!osrv) {
 245       return false;
 246     }
 247     current->add(osrv);
 248     current->initSlot(slot, osrv);
 249   }
 250
 251   MStart* start = MStart::New(alloc());
 252   current->add(start);
 253
 254   // Note: phi specialization can add type guard instructions to the OSR entry
 255   // block if needed. See TypeAnalyzer::shouldSpecializeOsrPhis.
 256
 257   // Create the preheader block, with the predecessor block and OSR block as
 258   // predecessors.
 259   if (!startNewBlock(pred, loopHead)) {
 260     return false;
 261   }
 262
 263   pred->end(MGoto::New(alloc(), current));
 264   osrBlock->end(MGoto::New(alloc(), current));
 265
 266   if (!current->addPredecessor(alloc(), osrBlock)) {
 267     return false;
 268   }
 269
 270   return true;
 271 }
 272
 273 bool WarpBuilder::addPendingEdge(BytecodeLocation target, MBasicBlock* block,
 274                                  uint32_t successor, uint32_t numToPop) {
 275   MOZ_ASSERT(successor < block->lastIns()->numSuccessors());
 276   MOZ_ASSERT(numToPop <= block->stackDepth());
 277
 278   jsbytecode* targetPC = target.toRawBytecode();
 279   PendingEdgesMap::AddPtr p = pendingEdges_.lookupForAdd(targetPC);
 280   if (p) {
 281     return p->value().emplaceBack(block, successor, numToPop);
 282   }
 283
 284   PendingEdges edges;
 285   static_assert(PendingEdges::InlineLength >= 1,
 286                 "Appending one element should be infallible");
 287   MOZ_ALWAYS_TRUE(edges.emplaceBack(block, successor, numToPop));
 288
 289   return pendingEdges_.add(p, targetPC, std::move(edges));
 290 }
 291
 292 bool WarpBuilder::build() {
 293   if (!buildPrologue()) {
 294     return false;
 295   }
 296
 297   if (!buildBody()) {
 298     return false;
 299   }
 300
 301   if (!MPhi::markIteratorPhis(*iterators())) {
 302     return false;
 303   }
 304
 305   MOZ_ASSERT_IF(info().osrPc(), graph().osrBlock());
 306   MOZ_ASSERT(loopStack_.empty());
 307   MOZ_ASSERT(loopDepth() == 0);
 308
 309   return true;
 310 }
 311
 312 bool WarpBuilder::buildInline() {
 313   if (!buildInlinePrologue()) {
 314     return false;
 315   }
 316
 317   if (!buildBody()) {
 318     return false;
 319   }
 320
 321   MOZ_ASSERT(loopStack_.empty());
 322   return true;
 323 }
 324
 325 MInstruction* WarpBuilder::buildNamedLambdaEnv(MDefinition* callee,
 326                                                MDefinition* env,
 327                                                NamedLambdaObject* templateObj) {
 328   MOZ_ASSERT(templateObj->numDynamicSlots() == 0);
 329
 330   MInstruction* namedLambda = MNewNamedLambdaObject::New(alloc(), templateObj);
 331   current->add(namedLambda);
 332
 333 #ifdef DEBUG
 334   // Assert in debug mode we can elide the post write barriers.
 335   current->add(MAssertCanElidePostWriteBarrier::New(alloc(), namedLambda, env));
 336   current->add(
 337       MAssertCanElidePostWriteBarrier::New(alloc(), namedLambda, callee));
 338 #endif
 339
 340   // Initialize the object's reserved slots. No post barrier is needed here:
 341   // the object will be allocated in the nursery if possible, and if the
 342   // tenured heap is used instead, a minor collection will have been performed
 343   // that moved env/callee to the tenured heap.
 344   size_t enclosingSlot = NamedLambdaObject::enclosingEnvironmentSlot();
 345   size_t lambdaSlot = NamedLambdaObject::lambdaSlot();
 346   current->add(MStoreFixedSlot::NewUnbarriered(alloc(), namedLambda,
 347                                                enclosingSlot, env));
 348   current->add(MStoreFixedSlot::NewUnbarriered(alloc(), namedLambda, lambdaSlot,
 349                                                callee));
 350
 351   return namedLambda;
 352 }
 353
 354 MInstruction* WarpBuilder::buildCallObject(MDefinition* callee,
 355                                            MDefinition* env,
 356                                            CallObject* templateObj) {
 357   MConstant* templateCst = constant(ObjectValue(*templateObj));
 358
 359   MNewCallObject* callObj = MNewCallObject::New(alloc(), templateCst);
 360   current->add(callObj);
 361
 362 #ifdef DEBUG
 363   // Assert in debug mode we can elide the post write barriers.
 364   current->add(MAssertCanElidePostWriteBarrier::New(alloc(), callObj, env));
 365   current->add(MAssertCanElidePostWriteBarrier::New(alloc(), callObj, callee));
 366 #endif
 367
 368   // Initialize the object's reserved slots. No post barrier is needed here,
 369   // for the same reason as in buildNamedLambdaEnv.
 370   size_t enclosingSlot = CallObject::enclosingEnvironmentSlot();
 371   size_t calleeSlot = CallObject::calleeSlot();
 372   current->add(
 373       MStoreFixedSlot::NewUnbarriered(alloc(), callObj, enclosingSlot, env));
 374   current->add(
 375       MStoreFixedSlot::NewUnbarriered(alloc(), callObj, calleeSlot, callee));
 376
 377   return callObj;
 378 }
 379
 380 bool WarpBuilder::buildEnvironmentChain() {
 381   const WarpEnvironment& env = scriptSnapshot()->environment();
 382
 383   if (env.is<NoEnvironment>()) {
 384     return true;
 385   }
 386
 387   MInstruction* envDef = env.match(
 388       [](const NoEnvironment&) -> MInstruction* {
 389         MOZ_CRASH("Already handled");
 390       },
 391       [this](JSObject* obj) -> MInstruction* {
 392         return constant(ObjectValue(*obj));
 393       },
 394       [this](const FunctionEnvironment& env) -> MInstruction* {
 395         MDefinition* callee = getCallee();
 396         MInstruction* envDef = MFunctionEnvironment::New(alloc(), callee);
 397         current->add(envDef);
 398         if (NamedLambdaObject* obj = env.namedLambdaTemplate) {
 399           envDef = buildNamedLambdaEnv(callee, envDef, obj);
 400         }
 401         if (CallObject* obj = env.callObjectTemplate) {
 402           envDef = buildCallObject(callee, envDef, obj);
 403           if (!envDef) {
 404             return nullptr;
 405           }
 406         }
 407         return envDef;
 408       });
 409   if (!envDef) {
 410     return false;
 411   }
 412
 413   // Update the environment slot from UndefinedValue only after the initial
 414   // environment is created so that bailout doesn't see a partial environment.
 415   // See: |BaselineStackBuilder::buildBaselineFrame|
 416   current->setEnvironmentChain(envDef);
 417   return true;
 418 }
 419
 420 bool WarpBuilder::buildPrologue() {
 421   BytecodeLocation startLoc(script_, script_->code());
 422   if (!startNewEntryBlock(info().firstStackSlot(), startLoc)) {
 423     return false;
 424   }
 425
 426   if (info().hasFunMaybeLazy()) {
 427     // Initialize |this|.
 428     MParameter* param = MParameter::New(alloc(), MParameter::THIS_SLOT);
 429     current->add(param);
 430     current->initSlot(info().thisSlot(), param);
 431
 432     // Initialize arguments.
 433     for (uint32_t i = 0; i < info().nargs(); i++) {
 434       MParameter* param = MParameter::New(alloc().fallible(), i);
 435       if (!param) {
 436         return false;
 437       }
 438       current->add(param);
 439       current->initSlot(info().argSlotUnchecked(i), param);
 440     }
 441   }
 442
 443   MConstant* undef = constant(UndefinedValue());
 444
 445   // Initialize local slots.
 446   for (uint32_t i = 0; i < info().nlocals(); i++) {
 447     current->initSlot(info().localSlot(i), undef);
 448   }
 449
 450   // Initialize the environment chain, return value, and arguments object slots.
 451   current->initSlot(info().environmentChainSlot(), undef);
 452   current->initSlot(info().returnValueSlot(), undef);
 453   if (info().needsArgsObj()) {
 454     current->initSlot(info().argsObjSlot(), undef);
 455   }
 456
 457   current->add(MStart::New(alloc()));
 458
 459   // Guard against over-recursion.
 460   MCheckOverRecursed* check = MCheckOverRecursed::New(alloc());
 461   current->add(check);
 462
 463   if (!buildEnvironmentChain()) {
 464     return false;
 465   }
 466
 467 #ifdef JS_CACHEIR_SPEW
 468   if (snapshot().needsFinalWarmUpCount()) {
 469     MIncrementWarmUpCounter* ins =
 470         MIncrementWarmUpCounter::New(alloc(), script_);
 471     current->add(ins);
 472   }
 473 #endif
 474
 475   return true;
 476 }
 477
 478 bool WarpBuilder::buildInlinePrologue() {
 479   // Generate entry block.
 480   BytecodeLocation startLoc(script_, script_->code());
 481   if (!startNewEntryBlock(info().firstStackSlot(), startLoc)) {
 482     return false;
 483   }
 484   current->setCallerResumePoint(callerResumePoint());
 485
 486   // Connect the entry block to the last block in the caller's graph.
 487   MBasicBlock* pred = callerBuilder()->current;
 488   MOZ_ASSERT(pred == callerResumePoint()->block());
 489
 490   pred->end(MGoto::New(alloc(), current));
 491   if (!current->addPredecessorWithoutPhis(pred)) {
 492     return false;
 493   }
 494
 495   MConstant* undef = constant(UndefinedValue());
 496
 497   // Initialize env chain slot to Undefined.  It's set later by
 498   // |buildEnvironmentChain|.
 499   current->initSlot(info().environmentChainSlot(), undef);
 500
 501   // Initialize |return value| slot.
 502   current->initSlot(info().returnValueSlot(), undef);
 503
 504   // Initialize |arguments| slot if needed.
 505   if (info().needsArgsObj()) {
 506     current->initSlot(info().argsObjSlot(), undef);
 507   }
 508
 509   // Initialize |this| slot.
 510   current->initSlot(info().thisSlot(), inlineCallInfo()->thisArg());
 511
 512   uint32_t callerArgs = inlineCallInfo()->argc();
 513   uint32_t actualArgs = info().nargs();
 514   uint32_t passedArgs = std::min<uint32_t>(callerArgs, actualArgs);
 515
 516   // Initialize actually set arguments.
 517   for (uint32_t i = 0; i < passedArgs; i++) {
 518     MDefinition* arg = inlineCallInfo()->getArg(i);
 519     current->initSlot(info().argSlotUnchecked(i), arg);
 520   }
 521
 522   // Pass undefined for missing arguments.
 523   for (uint32_t i = passedArgs; i < actualArgs; i++) {
 524     current->initSlot(info().argSlotUnchecked(i), undef);
 525   }
 526
 527   // Initialize local slots.
 528   for (uint32_t i = 0; i < info().nlocals(); i++) {
 529     current->initSlot(info().localSlot(i), undef);
 530   }
 531
 532   MOZ_ASSERT(current->entryResumePoint()->stackDepth() == info().totalSlots());
 533
 534   if (!buildEnvironmentChain()) {
 535     return false;
 536   }
 537
 538   return true;
 539 }
 540
 541 #ifdef DEBUG
 542 // In debug builds, after compiling a bytecode op, this class is used to check
 543 // that all values popped by this opcode either:
 544 //
 545 //   (1) Have the ImplicitlyUsed flag set on them.
 546 //   (2) Have more uses than before compiling this op (the value is
 547 //       used as operand of a new MIR instruction).
 548 //
 549 // This is used to catch problems where WarpBuilder pops a value without
 550 // adding any SSA uses and doesn't call setImplicitlyUsedUnchecked on it.
 551 class MOZ_RAII WarpPoppedValueUseChecker {
 552   Vector<MDefinition*, 4, SystemAllocPolicy> popped_;
 553   Vector<size_t, 4, SystemAllocPolicy> poppedUses_;
 554   MBasicBlock* current_;
 555   BytecodeLocation loc_;
 556
 557  public:
 558   WarpPoppedValueUseChecker(MBasicBlock* current, BytecodeLocation loc)
 559       : current_(current), loc_(loc) {}
 560
 561   [[nodiscard]] bool init() {
 562     // Don't require SSA uses for values popped by these ops.
 563     switch (loc_.getOp()) {
 564       case JSOp::Pop:
 565       case JSOp::PopN:
 566       case JSOp::DupAt:
 567       case JSOp::Dup:
 568       case JSOp::Dup2:
 569       case JSOp::Pick:
 570       case JSOp::Unpick:
 571       case JSOp::Swap:
 572       case JSOp::SetArg:
 573       case JSOp::SetLocal:
 574       case JSOp::InitLexical:
 575       case JSOp::SetRval:
 576       case JSOp::Void:
 577         // Basic stack/local/argument management opcodes.
 578         return true;
 579
 580       case JSOp::Case:
 581       case JSOp::Default:
 582         // These ops have to pop the switch value when branching but don't
 583         // actually use it.
 584         return true;
 585
 586       default:
 587         break;
 588     }
 589
 590     unsigned nuses = loc_.useCount();
 591
 592     for (unsigned i = 0; i < nuses; i++) {
 593       MDefinition* def = current_->peek(-int32_t(i + 1));
 594       if (!popped_.append(def) || !poppedUses_.append(def->defUseCount())) {
 595         return false;
 596       }
 597     }
 598
 599     return true;
 600   }
 601
 602   void checkAfterOp() {
 603     for (size_t i = 0; i < popped_.length(); i++) {
 604       // First value popped by JSOp::EndIter is not used at all, it's similar
 605       // to JSOp::Pop above.
 606       if (loc_.is(JSOp::EndIter) && i == 0) {
 607         continue;
 608       }
 609       MOZ_ASSERT(popped_[i]->isImplicitlyUsed() ||
 610                  popped_[i]->defUseCount() > poppedUses_[i]);
 611     }
 612   }
 613 };
 614 #endif
 615
 616 bool WarpBuilder::buildBody() {
 617   for (BytecodeLocation loc : AllBytecodesIterable(script_)) {
 618     if (mirGen().shouldCancel("WarpBuilder (opcode loop)")) {
 619       return false;
 620     }
 621
 622     // Skip unreachable ops (for example code after a 'return' or 'throw') until
 623     // we get to the next jump target.
 624     if (hasTerminatedBlock()) {
 625       // Finish any "broken" loops with an unreachable backedge. For example:
 626       //
 627       //   do {
 628       //     ...
 629       //     return;
 630       //     ...
 631       //   } while (x);
 632       //
 633       // This loop never actually loops.
 634       if (loc.isBackedge() && !loopStack_.empty()) {
 635         BytecodeLocation loopHead(script_,
 636                                   loopStack_.back().header()->entryPC());
 637         if (loc.isBackedgeForLoophead(loopHead)) {
 638           decLoopDepth();
 639           loopStack_.popBack();
 640         }
 641       }
 642       if (!loc.isJumpTarget()) {
 643         continue;
 644       }
 645     }
 646
 647     if (!alloc().ensureBallast()) {
 648       return false;
 649     }
 650
 651 #ifdef DEBUG
 652     WarpPoppedValueUseChecker useChecker(current, loc);
 653     if (!useChecker.init()) {
 654       return false;
 655     }
 656 #endif
 657     bool wantPreciseLineNumbers = js::jit::PerfEnabled();
 658     if (wantPreciseLineNumbers && !hasTerminatedBlock()) {
 659       current->updateTrackedSite(newBytecodeSite(loc));
 660     }
 661
 662     JSOp op = loc.getOp();
 663
 664 #define BUILD_OP(OP, ...)                       \
 665   case JSOp::OP:                                \
 666     if (MOZ_UNLIKELY(!this->build_##OP(loc))) { \
 667       return false;                             \
 668     }                                           \
 669     break;
 670     switch (op) { FOR_EACH_OPCODE(BUILD_OP) }
 671 #undef BUILD_OP
 672
 673 #ifdef DEBUG
 674     useChecker.checkAfterOp();
 675 #endif
 676   }
 677
 678   return true;
 679 }
 680
 681 #define DEF_OP(OP)                                 \
 682   bool WarpBuilder::build_##OP(BytecodeLocation) { \
 683     MOZ_CRASH("Unsupported op");                   \
 684   }
 685 WARP_UNSUPPORTED_OPCODE_LIST(DEF_OP)
 686 #undef DEF_OP
 687
 688 bool WarpBuilder::build_Nop(BytecodeLocation) { return true; }
 689
 690 bool WarpBuilder::build_NopDestructuring(BytecodeLocation) { return true; }
 691
 692 bool WarpBuilder::build_NopIsAssignOp(BytecodeLocation) { return true; }
 693
 694 bool WarpBuilder::build_TryDestructuring(BytecodeLocation) {
 695   // Set the hasTryBlock flag to turn off optimizations that eliminate dead
 696   // resume points operands because the exception handler code for
 697   // TryNoteKind::Destructuring is effectively a (specialized) catch-block.
 698   graph().setHasTryBlock();
 699   return true;
 700 }
 701
 702 bool WarpBuilder::build_Lineno(BytecodeLocation) { return true; }
 703
 704 bool WarpBuilder::build_DebugLeaveLexicalEnv(BytecodeLocation) { return true; }
 705
 706 bool WarpBuilder::build_Undefined(BytecodeLocation) {
 707   pushConstant(UndefinedValue());
 708   return true;
 709 }
 710
 711 bool WarpBuilder::build_Void(BytecodeLocation) {
 712   current->pop();
 713   pushConstant(UndefinedValue());
 714   return true;
 715 }
 716
 717 bool WarpBuilder::build_Null(BytecodeLocation) {
 718   pushConstant(NullValue());
 719   return true;
 720 }
 721
 722 bool WarpBuilder::build_Hole(BytecodeLocation) {
 723   pushConstant(MagicValue(JS_ELEMENTS_HOLE));
 724   return true;
 725 }
 726
 727 bool WarpBuilder::build_Uninitialized(BytecodeLocation) {
 728   pushConstant(MagicValue(JS_UNINITIALIZED_LEXICAL));
 729   return true;
 730 }
 731
 732 bool WarpBuilder::build_IsConstructing(BytecodeLocation) {
 733   pushConstant(MagicValue(JS_IS_CONSTRUCTING));
 734   return true;
 735 }
 736
 737 bool WarpBuilder::build_False(BytecodeLocation) {
 738   pushConstant(BooleanValue(false));
 739   return true;
 740 }
 741
 742 bool WarpBuilder::build_True(BytecodeLocation) {
 743   pushConstant(BooleanValue(true));
 744   return true;
 745 }
 746
 747 bool WarpBuilder::build_Pop(BytecodeLocation) {
 748   current->pop();
 749   return true;
 750 }
 751
 752 bool WarpBuilder::build_PopN(BytecodeLocation loc) {
 753   for (uint32_t i = 0, n = loc.getPopCount(); i < n; i++) {
 754     current->pop();
 755   }
 756   return true;
 757 }
 758
 759 bool WarpBuilder::build_Dup(BytecodeLocation) {
 760   current->pushSlot(current->stackDepth() - 1);
 761   return true;
 762 }
 763
 764 bool WarpBuilder::build_Dup2(BytecodeLocation) {
 765   uint32_t lhsSlot = current->stackDepth() - 2;
 766   uint32_t rhsSlot = current->stackDepth() - 1;
 767   current->pushSlot(lhsSlot);
 768   current->pushSlot(rhsSlot);
 769   return true;
 770 }
 771
 772 bool WarpBuilder::build_DupAt(BytecodeLocation loc) {
 773   current->pushSlot(current->stackDepth() - 1 - loc.getDupAtIndex());
 774   return true;
 775 }
 776
 777 bool WarpBuilder::build_Swap(BytecodeLocation) {
 778   current->swapAt(-1);
 779   return true;
 780 }
 781
 782 bool WarpBuilder::build_Pick(BytecodeLocation loc) {
 783   int32_t depth = -int32_t(loc.getPickDepth());
 784   current->pick(depth);
 785   return true;
 786 }
 787
 788 bool WarpBuilder::build_Unpick(BytecodeLocation loc) {
 789   int32_t depth = -int32_t(loc.getUnpickDepth());
 790   current->unpick(depth);
 791   return true;
 792 }
 793
 794 bool WarpBuilder::build_Zero(BytecodeLocation) {
 795   pushConstant(Int32Value(0));
 796   return true;
 797 }
 798
 799 bool WarpBuilder::build_One(BytecodeLocation) {
 800   pushConstant(Int32Value(1));
 801   return true;
 802 }
 803
 804 bool WarpBuilder::build_Int8(BytecodeLocation loc) {
 805   pushConstant(Int32Value(loc.getInt8()));
 806   return true;
 807 }
 808
 809 bool WarpBuilder::build_Uint16(BytecodeLocation loc) {
 810   pushConstant(Int32Value(loc.getUint16()));
 811   return true;
 812 }
 813
 814 bool WarpBuilder::build_Uint24(BytecodeLocation loc) {
 815   pushConstant(Int32Value(loc.getUint24()));
 816   return true;
 817 }
 818
 819 bool WarpBuilder::build_Int32(BytecodeLocation loc) {
 820   pushConstant(Int32Value(loc.getInt32()));
 821   return true;
 822 }
 823
 824 bool WarpBuilder::build_Double(BytecodeLocation loc) {
 825   pushConstant(loc.getInlineValue());
 826   return true;
 827 }
 828
 829 bool WarpBuilder::build_BigInt(BytecodeLocation loc) {
 830   BigInt* bi = loc.getBigInt(script_);
 831   pushConstant(BigIntValue(bi));
 832   return true;
 833 }
 834
 835 bool WarpBuilder::build_String(BytecodeLocation loc) {
 836   JSString* str = loc.getString(script_);
 837   pushConstant(StringValue(str));
 838   return true;
 839 }
 840
 841 bool WarpBuilder::build_Symbol(BytecodeLocation loc) {
 842   uint32_t which = loc.getSymbolIndex();
 843   JS::Symbol* sym = mirGen().runtime->wellKnownSymbols().get(which);
 844   pushConstant(SymbolValue(sym));
 845   return true;
 846 }
 847
 848 bool WarpBuilder::build_RegExp(BytecodeLocation loc) {
 849   RegExpObject* reObj = loc.getRegExp(script_);
 850
 851   auto* snapshot = getOpSnapshot<WarpRegExp>(loc);
 852
 853   MRegExp* regexp = MRegExp::New(alloc(), reObj, snapshot->hasShared());
 854   current->add(regexp);
 855   current->push(regexp);
 856
 857   return true;
 858 }
 859
 860 bool WarpBuilder::build_Return(BytecodeLocation) {
 861   MDefinition* def = current->pop();
 862
 863   MReturn* ret = MReturn::New(alloc(), def);
 864   current->end(ret);
 865
 866   if (!graph().addReturn(current)) {
 867     return false;
 868   }
 869
 870   setTerminatedBlock();
 871   return true;
 872 }
 873
 874 bool WarpBuilder::build_RetRval(BytecodeLocation) {
 875   MDefinition* rval;
 876   if (script_->noScriptRval()) {
 877     rval = constant(UndefinedValue());
 878   } else {
 879     rval = current->getSlot(info().returnValueSlot());
 880   }
 881
 882   MReturn* ret = MReturn::New(alloc(), rval);
 883   current->end(ret);
 884
 885   if (!graph().addReturn(current)) {
 886     return false;
 887   }
 888
 889   setTerminatedBlock();
 890   return true;
 891 }
 892
 893 bool WarpBuilder::build_SetRval(BytecodeLocation) {
 894   MOZ_ASSERT(!script_->noScriptRval());
 895   MDefinition* rval = current->pop();
 896   current->setSlot(info().returnValueSlot(), rval);
 897   return true;
 898 }
 899
 900 bool WarpBuilder::build_GetRval(BytecodeLocation) {
 901   MOZ_ASSERT(!script_->noScriptRval());
 902   MDefinition* rval = current->getSlot(info().returnValueSlot());
 903   current->push(rval);
 904   return true;
 905 }
 906
 907 bool WarpBuilder::build_GetLocal(BytecodeLocation loc) {
 908   current->pushLocal(loc.local());
 909   return true;
 910 }
 911
 912 bool WarpBuilder::build_SetLocal(BytecodeLocation loc) {
 913   current->setLocal(loc.local());
 914   return true;
 915 }
 916
 917 bool WarpBuilder::build_InitLexical(BytecodeLocation loc) {
 918   current->setLocal(loc.local());
 919   return true;
 920 }
 921
 922 bool WarpBuilder::build_GetArg(BytecodeLocation loc) {
 923   uint32_t arg = loc.arg();
 924   if (info().argsObjAliasesFormals()) {
 925     MDefinition* argsObj = current->argumentsObject();
 926     auto* getArg = MGetArgumentsObjectArg::New(alloc(), argsObj, arg);
 927     current->add(getArg);
 928     current->push(getArg);
 929   } else {
 930     current->pushArg(arg);
 931   }
 932   return true;
 933 }
 934
 935 bool WarpBuilder::build_GetFrameArg(BytecodeLocation loc) {
 936   current->pushArgUnchecked(loc.arg());
 937   return true;
 938 }
 939
 940 bool WarpBuilder::build_SetArg(BytecodeLocation loc) {
 941   uint32_t arg = loc.arg();
 942   MDefinition* val = current->peek(-1);
 943
 944   if (!info().argsObjAliasesFormals()) {
 945     // Either |arguments| is never referenced within this function, or
 946     // it doesn't map to the actual arguments values. Either way, we
 947     // don't need to worry about synchronizing the argument values
 948     // when writing to them.
 949     current->setArg(arg);
 950     return true;
 951   }
 952
 953   // If an arguments object is in use, and it aliases formals, then all SetArgs
 954   // must go through the arguments object.
 955   MDefinition* argsObj = current->argumentsObject();
 956   current->add(MPostWriteBarrier::New(alloc(), argsObj, val));
 957   auto* ins = MSetArgumentsObjectArg::New(alloc(), argsObj, val, arg);
 958   current->add(ins);
 959   return resumeAfter(ins, loc);
 960 }
 961
 962 bool WarpBuilder::build_ArgumentsLength(BytecodeLocation) {
 963   if (inlineCallInfo()) {
 964     pushConstant(Int32Value(inlineCallInfo()->argc()));
 965   } else {
 966     auto* argsLength = MArgumentsLength::New(alloc());
 967     current->add(argsLength);
 968     current->push(argsLength);
 969   }
 970   return true;
 971 }
 972
 973 bool WarpBuilder::build_GetActualArg(BytecodeLocation) {
 974   MDefinition* index = current->pop();
 975   MInstruction* arg;
 976   if (inlineCallInfo()) {
 977     arg = MGetInlinedArgument::New(alloc(), index, *inlineCallInfo());
 978     if (!arg) {
 979       return false;
 980     }
 981   } else {
 982     arg = MGetFrameArgument::New(alloc(), index);
 983   }
 984   current->add(arg);
 985   current->push(arg);
 986   return true;
 987 }
 988
 989 bool WarpBuilder::build_ToNumeric(BytecodeLocation loc) {
 990   return buildUnaryOp(loc);
 991 }
 992
 993 bool WarpBuilder::buildUnaryOp(BytecodeLocation loc) {
 994   MDefinition* value = current->pop();
 995   return buildIC(loc, CacheKind::UnaryArith, {value});
 996 }
 997
 998 bool WarpBuilder::build_Inc(BytecodeLocation loc) { return buildUnaryOp(loc); }
 999
1000 bool WarpBuilder::build_Dec(BytecodeLocation loc) { return buildUnaryOp(loc); }
1001
1002 bool WarpBuilder::build_Pos(BytecodeLocation loc) { return buildUnaryOp(loc); }
1003
1004 bool WarpBuilder::build_Neg(BytecodeLocation loc) { return buildUnaryOp(loc); }
1005
1006 bool WarpBuilder::build_BitNot(BytecodeLocation loc) {
1007   return buildUnaryOp(loc);
1008 }
1009
1010 bool WarpBuilder::buildBinaryOp(BytecodeLocation loc) {
1011   MDefinition* right = current->pop();
1012   MDefinition* left = current->pop();
1013   return buildIC(loc, CacheKind::BinaryArith, {left, right});
1014 }
1015
1016 bool WarpBuilder::build_Add(BytecodeLocation loc) { return buildBinaryOp(loc); }
1017
1018 bool WarpBuilder::build_Sub(BytecodeLocation loc) { return buildBinaryOp(loc); }
1019
1020 bool WarpBuilder::build_Mul(BytecodeLocation loc) { return buildBinaryOp(loc); }
1021
1022 bool WarpBuilder::build_Div(BytecodeLocation loc) { return buildBinaryOp(loc); }
1023
1024 bool WarpBuilder::build_Mod(BytecodeLocation loc) { return buildBinaryOp(loc); }
1025
1026 bool WarpBuilder::build_Pow(BytecodeLocation loc) { return buildBinaryOp(loc); }
1027
1028 bool WarpBuilder::build_BitAnd(BytecodeLocation loc) {
1029   return buildBinaryOp(loc);
1030 }
1031
1032 bool WarpBuilder::build_BitOr(BytecodeLocation loc) {
1033   return buildBinaryOp(loc);
1034 }
1035
1036 bool WarpBuilder::build_BitXor(BytecodeLocation loc) {
1037   return buildBinaryOp(loc);
1038 }
1039
1040 bool WarpBuilder::build_Lsh(BytecodeLocation loc) { return buildBinaryOp(loc); }
1041
1042 bool WarpBuilder::build_Rsh(BytecodeLocation loc) { return buildBinaryOp(loc); }
1043
1044 bool WarpBuilder::build_Ursh(BytecodeLocation loc) {
1045   return buildBinaryOp(loc);
1046 }
1047
1048 bool WarpBuilder::buildCompareOp(BytecodeLocation loc) {
1049   MDefinition* right = current->pop();
1050   MDefinition* left = current->pop();
1051   return buildIC(loc, CacheKind::Compare, {left, right});
1052 }
1053
1054 bool WarpBuilder::build_Eq(BytecodeLocation loc) { return buildCompareOp(loc); }
1055
1056 bool WarpBuilder::build_Ne(BytecodeLocation loc) { return buildCompareOp(loc); }
1057
1058 bool WarpBuilder::build_Lt(BytecodeLocation loc) { return buildCompareOp(loc); }
1059
1060 bool WarpBuilder::build_Le(BytecodeLocation loc) { return buildCompareOp(loc); }
1061
1062 bool WarpBuilder::build_Gt(BytecodeLocation loc) { return buildCompareOp(loc); }
1063
1064 bool WarpBuilder::build_Ge(BytecodeLocation loc) { return buildCompareOp(loc); }
1065
1066 bool WarpBuilder::build_StrictEq(BytecodeLocation loc) {
1067   return buildCompareOp(loc);
1068 }
1069
1070 bool WarpBuilder::build_StrictNe(BytecodeLocation loc) {
1071   return buildCompareOp(loc);
1072 }
1073
1074 // Returns true iff the MTest added for |op| has a true-target corresponding
1075 // with the join point in the bytecode.
1076 static bool TestTrueTargetIsJoinPoint(JSOp op) {
1077   switch (op) {
1078     case JSOp::JumpIfTrue:
1079     case JSOp::Or:
1080     case JSOp::Case:
1081       return true;
1082
1083     case JSOp::JumpIfFalse:
1084     case JSOp::And:
1085     case JSOp::Coalesce:
1086       return false;
1087
1088     default:
1089       MOZ_CRASH("Unexpected op");
1090   }
1091 }
1092
1093 bool WarpBuilder::build_JumpTarget(BytecodeLocation loc) {
1094   PendingEdgesMap::Ptr p = pendingEdges_.lookup(loc.toRawBytecode());
1095   if (!p) {
1096     // No (reachable) jumps so this is just a no-op.
1097     return true;
1098   }
1099
1100   PendingEdges edges(std::move(p->value()));
1101   pendingEdges_.remove(p);
1102
1103   MOZ_ASSERT(!edges.empty());
1104
1105   // Create join block if there's fall-through from the previous bytecode op.
1106   if (!hasTerminatedBlock()) {
1107     MBasicBlock* pred = current;
1108     if (!startNewBlock(pred, loc)) {
1109       return false;
1110     }
1111     pred->end(MGoto::New(alloc(), current));
1112   }
1113
1114   for (const PendingEdge& edge : edges) {
1115     MBasicBlock* source = edge.block();
1116     uint32_t numToPop = edge.numToPop();
1117
1118     if (hasTerminatedBlock()) {
1119       if (!startNewBlock(source, loc, numToPop)) {
1120         return false;
1121       }
1122     } else {
1123       MOZ_ASSERT(source->stackDepth() - numToPop == current->stackDepth());
1124       if (!current->addPredecessorPopN(alloc(), source, numToPop)) {
1125         return false;
1126       }
1127     }
1128
1129     MOZ_ASSERT(source->lastIns()->isTest() || source->lastIns()->isGoto() ||
1130                source->lastIns()->isTableSwitch());
1131     source->lastIns()->initSuccessor(edge.successor(), current);
1132   }
1133
1134   MOZ_ASSERT(!hasTerminatedBlock());
1135   return true;
1136 }
1137
1138 bool WarpBuilder::addIteratorLoopPhis(BytecodeLocation loopHead) {
1139   // When unwinding the stack for a thrown exception, the exception handler must
1140   // close live iterators. For ForIn and Destructuring loops, the exception
1141   // handler needs access to values on the stack. To prevent them from being
1142   // optimized away (and replaced with the JS_OPTIMIZED_OUT MagicValue), we need
1143   // to mark the phis (and phis they flow into) as having implicit uses.
1144   // See ProcessTryNotes in vm/Interpreter.cpp and CloseLiveIteratorIon in
1145   // jit/JitFrames.cpp
1146
1147   MOZ_ASSERT(current->stackDepth() >= info().firstStackSlot());
1148
1149   bool emptyStack = current->stackDepth() == info().firstStackSlot();
1150   if (emptyStack) {
1151     return true;
1152   }
1153
1154   jsbytecode* loopHeadPC = loopHead.toRawBytecode();
1155
1156   for (TryNoteIterAllNoGC tni(script_, loopHeadPC); !tni.done(); ++tni) {
1157     const TryNote& tn = **tni;
1158
1159     // Stop if we reach an outer loop because outer loops were already
1160     // processed when we visited their loop headers.
1161     if (tn.isLoop()) {
1162       BytecodeLocation tnStart = script_->offsetToLocation(tn.start);
1163       if (tnStart != loopHead) {
1164         MOZ_ASSERT(tnStart.is(JSOp::LoopHead));
1165         MOZ_ASSERT(tnStart < loopHead);
1166         return true;
1167       }
1168     }
1169
1170     switch (tn.kind()) {
1171       case TryNoteKind::Destructuring:
1172       case TryNoteKind::ForIn: {
1173         // For for-in loops we add the iterator object to iterators(). For
1174         // destructuring loops we add the "done" value that's on top of the
1175         // stack and used in the exception handler.
1176         MOZ_ASSERT(tn.stackDepth >= 1);
1177         uint32_t slot = info().stackSlot(tn.stackDepth - 1);
1178         MPhi* phi = current->getSlot(slot)->toPhi();
1179         if (!iterators()->append(phi)) {
1180           return false;
1181         }
1182         break;
1183       }
1184       case TryNoteKind::Loop:
1185       case TryNoteKind::ForOf:
1186         // Regular loops do not have iterators to close. ForOf loops handle
1187         // unwinding using catch blocks.
1188         break;
1189       default:
1190         break;
1191     }
1192   }
1193
1194   return true;
1195 }
1196
1197 bool WarpBuilder::build_LoopHead(BytecodeLocation loc) {
1198   // All loops have the following bytecode structure:
1199   //
1200   //    LoopHead
1201   //    ...
1202   //    JumpIfTrue/Goto to LoopHead
1203
1204   if (hasTerminatedBlock()) {
1205     // The whole loop is unreachable.
1206     return true;
1207   }
1208
1209   // Handle OSR from Baseline JIT code.
1210   if (loc.toRawBytecode() == info().osrPc()) {
1211     if (!startNewOsrPreHeaderBlock(loc)) {
1212       return false;
1213     }
1214   }
1215
1216   incLoopDepth();
1217
1218   MBasicBlock* pred = current;
1219   if (!startNewLoopHeaderBlock(loc)) {
1220     return false;
1221   }
1222
1223   pred->end(MGoto::New(alloc(), current));
1224
1225   if (!addIteratorLoopPhis(loc)) {
1226     return false;
1227   }
1228
1229   MInterruptCheck* check = MInterruptCheck::New(alloc());
1230   current->add(check);
1231
1232 #ifdef JS_CACHEIR_SPEW
1233   if (snapshot().needsFinalWarmUpCount()) {
1234     MIncrementWarmUpCounter* ins =
1235         MIncrementWarmUpCounter::New(alloc(), script_);
1236     current->add(ins);
1237   }
1238 #endif
1239
1240   return true;
1241 }
1242
1243 bool WarpBuilder::buildTestOp(BytecodeLocation loc) {
1244   MDefinition* originalValue = current->peek(-1);
1245
1246   if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
1247     // If we have CacheIR, we can use it to refine the input. Note that
1248     // the transpiler doesn't generate any control instructions. Instead,
1249     // we fall through and generate them below.
1250     MDefinition* value = current->pop();
1251     if (!TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, {value})) {
1252       return false;
1253     }
1254   }
1255
1256   if (loc.isBackedge()) {
1257     return buildTestBackedge(loc);
1258   }
1259
1260   JSOp op = loc.getOp();
1261   BytecodeLocation target1 = loc.next();
1262   BytecodeLocation target2 = loc.getJumpTarget();
1263
1264   if (TestTrueTargetIsJoinPoint(op)) {
1265     std::swap(target1, target2);
1266   }
1267
1268   MDefinition* value = current->pop();
1269
1270   // JSOp::And and JSOp::Or leave the top stack value unchanged.  The
1271   // top stack value may have been converted to bool by a transpiled
1272   // ToBool IC, so we push the original value.
1273   bool mustKeepCondition = (op == JSOp::And || op == JSOp::Or);
1274   if (mustKeepCondition) {
1275     current->push(originalValue);
1276   }
1277
1278   // If this op always branches to the same location we treat this as a
1279   // JSOp::Goto.
1280   if (target1 == target2) {
1281     value->setImplicitlyUsedUnchecked();
1282     return buildForwardGoto(target1);
1283   }
1284
1285   MTest* test = MTest::New(alloc(), value, /* ifTrue = */ nullptr,
1286                            /* ifFalse = */ nullptr);
1287   current->end(test);
1288
1289   // JSOp::Case must pop a second value on the true-branch (the input to the
1290   // switch-statement).
1291   uint32_t numToPop = (loc.getOp() == JSOp::Case) ? 1 : 0;
1292
1293   if (!addPendingEdge(target1, current, MTest::TrueBranchIndex, numToPop)) {
1294     return false;
1295   }
1296   if (!addPendingEdge(target2, current, MTest::FalseBranchIndex)) {
1297     return false;
1298   }
1299
1300   if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
1301     test->setObservedTypes(typesSnapshot->list());
1302   }
1303
1304   setTerminatedBlock();
1305   return true;
1306 }
1307
1308 bool WarpBuilder::buildTestBackedge(BytecodeLocation loc) {
1309   MOZ_ASSERT(loc.is(JSOp::JumpIfTrue));
1310   MOZ_ASSERT(loopDepth() > 0);
1311
1312   MDefinition* value = current->pop();
1313
1314   BytecodeLocation loopHead = loc.getJumpTarget();
1315   MOZ_ASSERT(loopHead.is(JSOp::LoopHead));
1316
1317   BytecodeLocation successor = loc.next();
1318
1319   // We can finish the loop now. Use the loophead pc instead of the current pc
1320   // because the stack depth at the start of that op matches the current stack
1321   // depth (after popping our operand).
1322   MBasicBlock* pred = current;
1323   if (!startNewBlock(current, loopHead)) {
1324     return false;
1325   }
1326
1327   MTest* test = MTest::New(alloc(), value, /* ifTrue = */ current,
1328                            /* ifFalse = */ nullptr);
1329   pred->end(test);
1330
1331   if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
1332     test->setObservedTypes(typesSnapshot->list());
1333   }
1334
1335   if (!addPendingEdge(successor, pred, MTest::FalseBranchIndex)) {
1336     return false;
1337   }
1338
1339   return buildBackedge();
1340 }
1341
1342 bool WarpBuilder::build_JumpIfFalse(BytecodeLocation loc) {
1343   return buildTestOp(loc);
1344 }
1345
1346 bool WarpBuilder::build_JumpIfTrue(BytecodeLocation loc) {
1347   return buildTestOp(loc);
1348 }
1349
1350 bool WarpBuilder::build_And(BytecodeLocation loc) { return buildTestOp(loc); }
1351
1352 bool WarpBuilder::build_Or(BytecodeLocation loc) { return buildTestOp(loc); }
1353
1354 bool WarpBuilder::build_Case(BytecodeLocation loc) { return buildTestOp(loc); }
1355
1356 bool WarpBuilder::build_Default(BytecodeLocation loc) {
1357   current->pop();
1358   return buildForwardGoto(loc.getJumpTarget());
1359 }
1360
1361 bool WarpBuilder::build_Coalesce(BytecodeLocation loc) {
1362   BytecodeLocation target1 = loc.next();
1363   BytecodeLocation target2 = loc.getJumpTarget();
1364   MOZ_ASSERT(target2 > target1);
1365
1366   MDefinition* value = current->peek(-1);
1367
1368   MInstruction* isNullOrUndefined = MIsNullOrUndefined::New(alloc(), value);
1369   current->add(isNullOrUndefined);
1370
1371   current->end(MTest::New(alloc(), isNullOrUndefined, /* ifTrue = */ nullptr,
1372                           /* ifFalse = */ nullptr));
1373
1374   if (!addPendingEdge(target1, current, MTest::TrueBranchIndex)) {
1375     return false;
1376   }
1377   if (!addPendingEdge(target2, current, MTest::FalseBranchIndex)) {
1378     return false;
1379   }
1380
1381   setTerminatedBlock();
1382   return true;
1383 }
1384
1385 bool WarpBuilder::buildBackedge() {
1386   decLoopDepth();
1387
1388   MBasicBlock* header = loopStack_.popCopy().header();
1389   current->end(MGoto::New(alloc(), header));
1390
1391   if (!header->setBackedge(current)) {
1392     return false;
1393   }
1394
1395   setTerminatedBlock();
1396   return true;
1397 }
1398
1399 bool WarpBuilder::buildForwardGoto(BytecodeLocation target) {
1400   current->end(MGoto::New(alloc(), nullptr));
1401
1402   if (!addPendingEdge(target, current, MGoto::TargetIndex)) {
1403     return false;
1404   }
1405
1406   setTerminatedBlock();
1407   return true;
1408 }
1409
1410 bool WarpBuilder::build_Goto(BytecodeLocation loc) {
1411   if (loc.isBackedge()) {
1412     return buildBackedge();
1413   }
1414
1415   return buildForwardGoto(loc.getJumpTarget());
1416 }
1417
1418 bool WarpBuilder::build_IsNullOrUndefined(BytecodeLocation loc) {
1419   MDefinition* value = current->peek(-1);
1420   auto* isNullOrUndef = MIsNullOrUndefined::New(alloc(), value);
1421   current->add(isNullOrUndef);
1422   current->push(isNullOrUndef);
1423   return true;
1424 }
1425
1426 bool WarpBuilder::build_DebugCheckSelfHosted(BytecodeLocation loc) {
1427 #ifdef DEBUG
1428   MDefinition* val = current->pop();
1429   MDebugCheckSelfHosted* check = MDebugCheckSelfHosted::New(alloc(), val);
1430   current->add(check);
1431   current->push(check);
1432   if (!resumeAfter(check, loc)) {
1433     return false;
1434   }
1435 #endif
1436   return true;
1437 }
1438
1439 bool WarpBuilder::build_DynamicImport(BytecodeLocation loc) {
1440   MDefinition* options = current->pop();
1441   MDefinition* specifier = current->pop();
1442   MDynamicImport* ins = MDynamicImport::New(alloc(), specifier, options);
1443   current->add(ins);
1444   current->push(ins);
1445   return resumeAfter(ins, loc);
1446 }
1447
1448 bool WarpBuilder::build_Not(BytecodeLocation loc) {
1449   if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
1450     // If we have CacheIR, we can use it to refine the input before
1451     // emitting the MNot.
1452     MDefinition* value = current->pop();
1453     if (!TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, {value})) {
1454       return false;
1455     }
1456   }
1457
1458   MDefinition* value = current->pop();
1459   MNot* ins = MNot::New(alloc(), value);
1460   current->add(ins);
1461   current->push(ins);
1462
1463   if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
1464     ins->setObservedTypes(typesSnapshot->list());
1465   }
1466
1467   return true;
1468 }
1469
1470 bool WarpBuilder::build_ToString(BytecodeLocation loc) {
1471   MDefinition* value = current->pop();
1472
1473   if (value->type() == MIRType::String) {
1474     value->setImplicitlyUsedUnchecked();
1475     current->push(value);
1476     return true;
1477   }
1478
1479   MToString* ins =
1480       MToString::New(alloc(), value, MToString::SideEffectHandling::Supported);
1481   current->add(ins);
1482   current->push(ins);
1483   if (ins->isEffectful()) {
1484     return resumeAfter(ins, loc);
1485   }
1486   return true;
1487 }
1488
1489 bool WarpBuilder::usesEnvironmentChain() const {
1490   return script_->jitScript()->usesEnvironmentChain();
1491 }
1492
1493 bool WarpBuilder::build_GlobalOrEvalDeclInstantiation(BytecodeLocation loc) {
1494   MOZ_ASSERT(!script_->isForEval(), "Eval scripts not supported");
1495   auto* redeclCheck = MGlobalDeclInstantiation::New(alloc());
1496   current->add(redeclCheck);
1497   return resumeAfter(redeclCheck, loc);
1498 }
1499
1500 bool WarpBuilder::build_BindVar(BytecodeLocation) {
1501   MOZ_ASSERT(usesEnvironmentChain());
1502
1503   MDefinition* env = current->environmentChain();
1504   MCallBindVar* ins = MCallBindVar::New(alloc(), env);
1505   current->add(ins);
1506   current->push(ins);
1507   return true;
1508 }
1509
1510 bool WarpBuilder::build_MutateProto(BytecodeLocation loc) {
1511   MDefinition* value = current->pop();
1512   MDefinition* obj = current->peek(-1);
1513   MMutateProto* mutate = MMutateProto::New(alloc(), obj, value);
1514   current->add(mutate);
1515   return resumeAfter(mutate, loc);
1516 }
1517
1518 MDefinition* WarpBuilder::getCallee() {
1519   if (inlineCallInfo()) {
1520     return inlineCallInfo()->callee();
1521   }
1522
1523   MInstruction* callee = MCallee::New(alloc());
1524   current->add(callee);
1525   return callee;
1526 }
1527
1528 bool WarpBuilder::build_Callee(BytecodeLocation) {
1529   MDefinition* callee = getCallee();
1530   current->push(callee);
1531   return true;
1532 }
1533
1534 bool WarpBuilder::build_ToAsyncIter(BytecodeLocation loc) {
1535   MDefinition* nextMethod = current->pop();
1536   MDefinition* iterator = current->pop();
1537   MToAsyncIter* ins = MToAsyncIter::New(alloc(), iterator, nextMethod);
1538   current->add(ins);
1539   current->push(ins);
1540   return resumeAfter(ins, loc);
1541 }
1542
1543 bool WarpBuilder::build_ToPropertyKey(BytecodeLocation loc) {
1544   MDefinition* value = current->pop();
1545   return buildIC(loc, CacheKind::ToPropertyKey, {value});
1546 }
1547
1548 bool WarpBuilder::build_Typeof(BytecodeLocation loc) {
1549   MDefinition* input = current->pop();
1550
1551   if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
1552     auto* typeOf = MTypeOf::New(alloc(), input);
1553     typeOf->setObservedTypes(typesSnapshot->list());
1554     current->add(typeOf);
1555
1556     auto* ins = MTypeOfName::New(alloc(), typeOf);
1557     current->add(ins);
1558     current->push(ins);
1559     return true;
1560   }
1561
1562   return buildIC(loc, CacheKind::TypeOf, {input});
1563 }
1564
1565 bool WarpBuilder::build_TypeofExpr(BytecodeLocation loc) {
1566   return build_Typeof(loc);
1567 }
1568
1569 bool WarpBuilder::build_TypeofEq(BytecodeLocation loc) {
1570   auto operand = loc.getTypeofEqOperand();
1571   JSType type = operand.type();
1572   JSOp compareOp = operand.compareOp();
1573   MDefinition* input = current->pop();
1574
1575   if (const auto* typesSnapshot = getOpSnapshot<WarpPolymorphicTypes>(loc)) {
1576     auto* typeOf = MTypeOf::New(alloc(), input);
1577     typeOf->setObservedTypes(typesSnapshot->list());
1578     current->add(typeOf);
1579
1580     auto* typeInt = MConstant::New(alloc(), Int32Value(type));
1581     current->add(typeInt);
1582
1583     auto* ins = MCompare::New(alloc(), typeOf, typeInt, compareOp,
1584                               MCompare::Compare_Int32);
1585     current->add(ins);
1586     current->push(ins);
1587     return true;
1588   }
1589
1590   return buildIC(loc, CacheKind::TypeOfEq, {input});
1591 }
1592
1593 bool WarpBuilder::build_Arguments(BytecodeLocation loc) {
1594   auto* snapshot = getOpSnapshot<WarpArguments>(loc);
1595   MOZ_ASSERT(info().needsArgsObj());
1596   MOZ_ASSERT(snapshot);
1597   MOZ_ASSERT(usesEnvironmentChain());
1598
1599   ArgumentsObject* templateObj = snapshot->templateObj();
1600   MDefinition* env = current->environmentChain();
1601
1602   MInstruction* argsObj;
1603   if (inlineCallInfo()) {
1604     argsObj = MCreateInlinedArgumentsObject::New(
1605         alloc(), env, getCallee(), inlineCallInfo()->argv(), templateObj);
1606     if (!argsObj) {
1607       return false;
1608     }
1609   } else {
1610     argsObj = MCreateArgumentsObject::New(alloc(), env, templateObj);
1611   }
1612   current->add(argsObj);
1613   current->setArgumentsObject(argsObj);
1614   current->push(argsObj);
1615
1616   return true;
1617 }
1618
1619 bool WarpBuilder::build_ObjWithProto(BytecodeLocation loc) {
1620   MDefinition* proto = current->pop();
1621   MInstruction* ins = MObjectWithProto::New(alloc(), proto);
1622   current->add(ins);
1623   current->push(ins);
1624   return resumeAfter(ins, loc);
1625 }
1626
1627 MDefinition* WarpBuilder::walkEnvironmentChain(uint32_t numHops) {
1628   MDefinition* env = current->environmentChain();
1629
1630   for (uint32_t i = 0; i < numHops; i++) {
1631     if (!alloc().ensureBallast()) {
1632       return nullptr;
1633     }
1634
1635     MInstruction* ins = MEnclosingEnvironment::New(alloc(), env);
1636     current->add(ins);
1637     env = ins;
1638   }
1639
1640   return env;
1641 }
1642
1643 bool WarpBuilder::build_GetAliasedVar(BytecodeLocation loc) {
1644   EnvironmentCoordinate ec = loc.getEnvironmentCoordinate();
1645   MDefinition* obj = walkEnvironmentChain(ec.hops());
1646   if (!obj) {
1647     return false;
1648   }
1649
1650   MInstruction* load;
1651   if (EnvironmentObject::nonExtensibleIsFixedSlot(ec)) {
1652     load = MLoadFixedSlot::New(alloc(), obj, ec.slot());
1653   } else {
1654     MInstruction* slots = MSlots::New(alloc(), obj);
1655     current->add(slots);
1656
1657     uint32_t slot = EnvironmentObject::nonExtensibleDynamicSlotIndex(ec);
1658     load = MLoadDynamicSlot::New(alloc(), slots, slot);
1659   }
1660
1661   current->add(load);
1662   current->push(load);
1663   return true;
1664 }
1665
1666 bool WarpBuilder::build_SetAliasedVar(BytecodeLocation loc) {
1667   EnvironmentCoordinate ec = loc.getEnvironmentCoordinate();
1668   MDefinition* val = current->peek(-1);
1669   MDefinition* obj = walkEnvironmentChain(ec.hops());
1670   if (!obj) {
1671     return false;
1672   }
1673
1674   current->add(MPostWriteBarrier::New(alloc(), obj, val));
1675
1676   MInstruction* store;
1677   if (EnvironmentObject::nonExtensibleIsFixedSlot(ec)) {
1678     store = MStoreFixedSlot::NewBarriered(alloc(), obj, ec.slot(), val);
1679   } else {
1680     MInstruction* slots = MSlots::New(alloc(), obj);
1681     current->add(slots);
1682
1683     uint32_t slot = EnvironmentObject::nonExtensibleDynamicSlotIndex(ec);
1684     store = MStoreDynamicSlot::NewBarriered(alloc(), slots, slot, val);
1685   }
1686
1687   current->add(store);
1688   return resumeAfter(store, loc);
1689 }
1690
1691 bool WarpBuilder::build_InitAliasedLexical(BytecodeLocation loc) {
1692   return build_SetAliasedVar(loc);
1693 }
1694
1695 bool WarpBuilder::build_EnvCallee(BytecodeLocation loc) {
1696   uint32_t numHops = loc.getEnvCalleeNumHops();
1697   MDefinition* env = walkEnvironmentChain(numHops);
1698   if (!env) {
1699     return false;
1700   }
1701
1702   auto* callee = MLoadFixedSlot::New(alloc(), env, CallObject::calleeSlot());
1703   current->add(callee);
1704   current->push(callee);
1705   return true;
1706 }
1707
1708 bool WarpBuilder::build_Iter(BytecodeLocation loc) {
1709   MDefinition* obj = current->pop();
1710   return buildIC(loc, CacheKind::GetIterator, {obj});
1711 }
1712
1713 bool WarpBuilder::build_MoreIter(BytecodeLocation loc) {
1714   MDefinition* iter = current->peek(-1);
1715   MInstruction* ins = MIteratorMore::New(alloc(), iter);
1716   current->add(ins);
1717   current->push(ins);
1718   return resumeAfter(ins, loc);
1719 }
1720
1721 bool WarpBuilder::build_EndIter(BytecodeLocation loc) {
1722   current->pop();  // Iterator value is not used.
1723   MDefinition* iter = current->pop();
1724   MInstruction* ins = MIteratorEnd::New(alloc(), iter);
1725   current->add(ins);
1726   return resumeAfter(ins, loc);
1727 }
1728
1729 bool WarpBuilder::build_CloseIter(BytecodeLocation loc) {
1730   MDefinition* iter = current->pop();
1731   iter = unboxObjectInfallible(iter, IsMovable::Yes);
1732   return buildIC(loc, CacheKind::CloseIter, {iter});
1733 }
1734
1735 bool WarpBuilder::build_IsNoIter(BytecodeLocation) {
1736   MDefinition* def = current->peek(-1);
1737   MOZ_ASSERT(def->isIteratorMore());
1738   MInstruction* ins = MIsNoIter::New(alloc(), def);
1739   current->add(ins);
1740   current->push(ins);
1741   return true;
1742 }
1743
1744 bool WarpBuilder::build_OptimizeGetIterator(BytecodeLocation loc) {
1745   MDefinition* value = current->pop();
1746   return buildIC(loc, CacheKind::OptimizeGetIterator, {value});
1747 }
1748
1749 bool WarpBuilder::transpileCall(BytecodeLocation loc,
1750                                 const WarpCacheIR* cacheIRSnapshot,
1751                                 CallInfo* callInfo) {
1752   // Synthesize the constant number of arguments for this call op.
1753   auto* argc = MConstant::New(alloc(), Int32Value(callInfo->argc()));
1754   current->add(argc);
1755
1756   return TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, {argc}, callInfo);
1757 }
1758
1759 void WarpBuilder::buildCreateThis(CallInfo& callInfo) {
1760   MOZ_ASSERT(callInfo.constructing());
1761
1762   // Inline the this-object allocation on the caller-side.
1763   MDefinition* callee = callInfo.callee();
1764   MDefinition* newTarget = callInfo.getNewTarget();
1765   auto* createThis = MCreateThis::New(alloc(), callee, newTarget);
1766   current->add(createThis);
1767   callInfo.thisArg()->setImplicitlyUsedUnchecked();
1768   callInfo.setThis(createThis);
1769 }
1770
1771 bool WarpBuilder::buildCallOp(BytecodeLocation loc) {
1772   uint32_t argc = loc.getCallArgc();
1773   JSOp op = loc.getOp();
1774   bool constructing = IsConstructOp(op);
1775   bool ignoresReturnValue = (op == JSOp::CallIgnoresRv || loc.resultIsPopped());
1776
1777   CallInfo callInfo(alloc(), constructing, ignoresReturnValue);
1778   if (!callInfo.init(current, argc)) {
1779     return false;
1780   }
1781
1782   if (const auto* inliningSnapshot = getOpSnapshot<WarpInlinedCall>(loc)) {
1783     // Transpile the CacheIR to generate the correct guards before
1784     // inlining.  In this case, CacheOp::CallInlinedFunction updates
1785     // the CallInfo, but does not generate a call.
1786     callInfo.markAsInlined();
1787     if (!transpileCall(loc, inliningSnapshot->cacheIRSnapshot(), &callInfo)) {
1788       return false;
1789     }
1790
1791     // Generate the body of the inlined function.
1792     return buildInlinedCall(loc, inliningSnapshot, callInfo);
1793   }
1794
1795   if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
1796     return transpileCall(loc, cacheIRSnapshot, &callInfo);
1797   }
1798
1799   if (getOpSnapshot<WarpBailout>(loc)) {
1800     callInfo.setImplicitlyUsedUnchecked();
1801     return buildBailoutForColdIC(loc, CacheKind::Call);
1802   }
1803
1804   bool needsThisCheck = false;
1805   if (callInfo.constructing()) {
1806     buildCreateThis(callInfo);
1807     needsThisCheck = true;
1808   }
1809
1810   MCall* call = makeCall(callInfo, needsThisCheck);
1811   if (!call) {
1812     return false;
1813   }
1814
1815   current->add(call);
1816   current->push(call);
1817   return resumeAfter(call, loc);
1818 }
1819
1820 bool WarpBuilder::build_Call(BytecodeLocation loc) { return buildCallOp(loc); }
1821
1822 bool WarpBuilder::build_CallContent(BytecodeLocation loc) {
1823   return buildCallOp(loc);
1824 }
1825
1826 bool WarpBuilder::build_CallIgnoresRv(BytecodeLocation loc) {
1827   return buildCallOp(loc);
1828 }
1829
1830 bool WarpBuilder::build_CallIter(BytecodeLocation loc) {
1831   return buildCallOp(loc);
1832 }
1833
1834 bool WarpBuilder::build_CallContentIter(BytecodeLocation loc) {
1835   return buildCallOp(loc);
1836 }
1837
1838 bool WarpBuilder::build_New(BytecodeLocation loc) { return buildCallOp(loc); }
1839
1840 bool WarpBuilder::build_NewContent(BytecodeLocation loc) {
1841   return buildCallOp(loc);
1842 }
1843
1844 bool WarpBuilder::build_SuperCall(BytecodeLocation loc) {
1845   return buildCallOp(loc);
1846 }
1847
1848 bool WarpBuilder::build_FunctionThis(BytecodeLocation loc) {
1849   MOZ_ASSERT(info().hasFunMaybeLazy());
1850
1851   if (script_->strict()) {
1852     // No need to wrap primitive |this| in strict mode.
1853     current->pushSlot(info().thisSlot());
1854     return true;
1855   }
1856
1857   MOZ_ASSERT(!script_->hasNonSyntacticScope(),
1858              "WarpOracle should have aborted compilation");
1859
1860   MDefinition* def = current->getSlot(info().thisSlot());
1861   JSObject* globalThis = snapshot().globalLexicalEnvThis();
1862
1863   auto* thisObj = MBoxNonStrictThis::New(alloc(), def, globalThis);
1864   current->add(thisObj);
1865   current->push(thisObj);
1866
1867   return true;
1868 }
1869
1870 bool WarpBuilder::build_GlobalThis(BytecodeLocation loc) {
1871   MOZ_ASSERT(!script_->hasNonSyntacticScope());
1872   JSObject* obj = snapshot().globalLexicalEnvThis();
1873   pushConstant(ObjectValue(*obj));
1874   return true;
1875 }
1876
1877 MConstant* WarpBuilder::globalLexicalEnvConstant() {
1878   JSObject* globalLexical = snapshot().globalLexicalEnv();
1879   return constant(ObjectValue(*globalLexical));
1880 }
1881
1882 bool WarpBuilder::build_GetName(BytecodeLocation loc) {
1883   MOZ_ASSERT(usesEnvironmentChain());
1884
1885   MDefinition* env = current->environmentChain();
1886   env = unboxObjectInfallible(env, IsMovable::Yes);
1887   return buildIC(loc, CacheKind::GetName, {env});
1888 }
1889
1890 bool WarpBuilder::build_GetGName(BytecodeLocation loc) {
1891   MOZ_ASSERT(!script_->hasNonSyntacticScope());
1892
1893   MDefinition* env = globalLexicalEnvConstant();
1894   return buildIC(loc, CacheKind::GetName, {env});
1895 }
1896
1897 bool WarpBuilder::build_BindName(BytecodeLocation loc) {
1898   MOZ_ASSERT(usesEnvironmentChain());
1899
1900   MDefinition* env = current->environmentChain();
1901   env = unboxObjectInfallible(env, IsMovable::Yes);
1902   return buildIC(loc, CacheKind::BindName, {env});
1903 }
1904
1905 bool WarpBuilder::build_BindGName(BytecodeLocation loc) {
1906   MOZ_ASSERT(!script_->hasNonSyntacticScope());
1907
1908   if (const auto* snapshot = getOpSnapshot<WarpBindGName>(loc)) {
1909     JSObject* globalEnv = snapshot->globalEnv();
1910     pushConstant(ObjectValue(*globalEnv));
1911     return true;
1912   }
1913
1914   MDefinition* env = globalLexicalEnvConstant();
1915   return buildIC(loc, CacheKind::BindName, {env});
1916 }
1917
1918 bool WarpBuilder::build_GetProp(BytecodeLocation loc) {
1919   MDefinition* val = current->pop();
1920   return buildIC(loc, CacheKind::GetProp, {val});
1921 }
1922
1923 bool WarpBuilder::build_GetElem(BytecodeLocation loc) {
1924   MDefinition* id = current->pop();
1925   MDefinition* val = current->pop();
1926   return buildIC(loc, CacheKind::GetElem, {val, id});
1927 }
1928
1929 bool WarpBuilder::build_SetProp(BytecodeLocation loc) {
1930   MDefinition* val = current->pop();
1931   MDefinition* obj = current->pop();
1932   current->push(val);
1933   return buildIC(loc, CacheKind::SetProp, {obj, val});
1934 }
1935
1936 bool WarpBuilder::build_StrictSetProp(BytecodeLocation loc) {
1937   return build_SetProp(loc);
1938 }
1939
1940 bool WarpBuilder::build_SetName(BytecodeLocation loc) {
1941   return build_SetProp(loc);
1942 }
1943
1944 bool WarpBuilder::build_StrictSetName(BytecodeLocation loc) {
1945   return build_SetProp(loc);
1946 }
1947
1948 bool WarpBuilder::build_SetGName(BytecodeLocation loc) {
1949   return build_SetProp(loc);
1950 }
1951
1952 bool WarpBuilder::build_StrictSetGName(BytecodeLocation loc) {
1953   return build_SetProp(loc);
1954 }
1955
1956 bool WarpBuilder::build_InitGLexical(BytecodeLocation loc) {
1957   MOZ_ASSERT(!script_->hasNonSyntacticScope());
1958
1959   MDefinition* globalLexical = globalLexicalEnvConstant();
1960   MDefinition* val = current->peek(-1);
1961
1962   return buildIC(loc, CacheKind::SetProp, {globalLexical, val});
1963 }
1964
1965 bool WarpBuilder::build_SetElem(BytecodeLocation loc) {
1966   MDefinition* val = current->pop();
1967   MDefinition* id = current->pop();
1968   MDefinition* obj = current->pop();
1969   current->push(val);
1970   return buildIC(loc, CacheKind::SetElem, {obj, id, val});
1971 }
1972
1973 bool WarpBuilder::build_StrictSetElem(BytecodeLocation loc) {
1974   return build_SetElem(loc);
1975 }
1976
1977 bool WarpBuilder::build_DelProp(BytecodeLocation loc) {
1978   PropertyName* name = loc.getPropertyName(script_);
1979   MDefinition* obj = current->pop();
1980   bool strict = loc.getOp() == JSOp::StrictDelProp;
1981
1982   MInstruction* ins = MDeleteProperty::New(alloc(), obj, name, strict);
1983   current->add(ins);
1984   current->push(ins);
1985   return resumeAfter(ins, loc);
1986 }
1987
1988 bool WarpBuilder::build_StrictDelProp(BytecodeLocation loc) {
1989   return build_DelProp(loc);
1990 }
1991
1992 bool WarpBuilder::build_DelElem(BytecodeLocation loc) {
1993   MDefinition* id = current->pop();
1994   MDefinition* obj = current->pop();
1995   bool strict = loc.getOp() == JSOp::StrictDelElem;
1996
1997   MInstruction* ins = MDeleteElement::New(alloc(), obj, id, strict);
1998   current->add(ins);
1999   current->push(ins);
2000   return resumeAfter(ins, loc);
2001 }
2002
2003 bool WarpBuilder::build_StrictDelElem(BytecodeLocation loc) {
2004   return build_DelElem(loc);
2005 }
2006
2007 bool WarpBuilder::build_SetFunName(BytecodeLocation loc) {
2008   FunctionPrefixKind prefixKind = loc.getFunctionPrefixKind();
2009   MDefinition* name = current->pop();
2010   MDefinition* fun = current->pop();
2011
2012   MSetFunName* ins = MSetFunName::New(alloc(), fun, name, uint8_t(prefixKind));
2013   current->add(ins);
2014   current->push(fun);
2015   return resumeAfter(ins, loc);
2016 }
2017
2018 bool WarpBuilder::build_PushLexicalEnv(BytecodeLocation loc) {
2019   MOZ_ASSERT(usesEnvironmentChain());
2020
2021   const auto* snapshot = getOpSnapshot<WarpLexicalEnvironment>(loc);
2022   MOZ_ASSERT(snapshot);
2023
2024   MDefinition* env = current->environmentChain();
2025   MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
2026
2027   auto* ins = MNewLexicalEnvironmentObject::New(alloc(), templateCst);
2028   current->add(ins);
2029
2030 #ifdef DEBUG
2031   // Assert in debug mode we can elide the post write barrier.
2032   current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, env));
2033 #endif
2034
2035   // Initialize the object's reserved slots. No post barrier is needed here,
2036   // for the same reason as in buildNamedLambdaEnv.
2037   current->add(MStoreFixedSlot::NewUnbarriered(
2038       alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(), env));
2039
2040   current->setEnvironmentChain(ins);
2041   return true;
2042 }
2043
2044 bool WarpBuilder::build_PushClassBodyEnv(BytecodeLocation loc) {
2045   MOZ_ASSERT(usesEnvironmentChain());
2046
2047   const auto* snapshot = getOpSnapshot<WarpClassBodyEnvironment>(loc);
2048   MOZ_ASSERT(snapshot);
2049
2050   MDefinition* env = current->environmentChain();
2051   MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
2052
2053   auto* ins = MNewClassBodyEnvironmentObject::New(alloc(), templateCst);
2054   current->add(ins);
2055
2056 #ifdef DEBUG
2057   // Assert in debug mode we can elide the post write barrier.
2058   current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, env));
2059 #endif
2060
2061   // Initialize the object's reserved slots. No post barrier is needed here,
2062   // for the same reason as in buildNamedLambdaEnv.
2063   current->add(MStoreFixedSlot::NewUnbarriered(
2064       alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(), env));
2065
2066   current->setEnvironmentChain(ins);
2067   return true;
2068 }
2069
2070 bool WarpBuilder::build_PopLexicalEnv(BytecodeLocation) {
2071   MDefinition* enclosingEnv = walkEnvironmentChain(1);
2072   if (!enclosingEnv) {
2073     return false;
2074   }
2075   current->setEnvironmentChain(enclosingEnv);
2076   return true;
2077 }
2078
2079 bool WarpBuilder::build_FreshenLexicalEnv(BytecodeLocation loc) {
2080   MOZ_ASSERT(usesEnvironmentChain());
2081
2082   const auto* snapshot = getOpSnapshot<WarpLexicalEnvironment>(loc);
2083   MOZ_ASSERT(snapshot);
2084
2085   MDefinition* enclosingEnv = walkEnvironmentChain(1);
2086   if (!enclosingEnv) {
2087     return false;
2088   }
2089
2090   MDefinition* env = current->environmentChain();
2091   MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
2092
2093   auto* templateObj = snapshot->templateObj();
2094   auto* scope = &templateObj->scope();
2095   MOZ_ASSERT(scope->hasEnvironment());
2096
2097   auto* ins = MNewLexicalEnvironmentObject::New(alloc(), templateCst);
2098   current->add(ins);
2099
2100 #ifdef DEBUG
2101   // Assert in debug mode we can elide the post write barrier.
2102   current->add(
2103       MAssertCanElidePostWriteBarrier::New(alloc(), ins, enclosingEnv));
2104 #endif
2105
2106   // Initialize the object's reserved slots. No post barrier is needed here,
2107   // for the same reason as in buildNamedLambdaEnv.
2108   current->add(MStoreFixedSlot::NewUnbarriered(
2109       alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(),
2110       enclosingEnv));
2111
2112   // Copy environment slots.
2113   MSlots* envSlots = nullptr;
2114   MSlots* slots = nullptr;
2115   for (BindingIter iter(scope); iter; iter++) {
2116     auto loc = iter.location();
2117     if (loc.kind() != BindingLocation::Kind::Environment) {
2118       MOZ_ASSERT(loc.kind() == BindingLocation::Kind::Frame);
2119       continue;
2120     }
2121
2122     if (!alloc().ensureBallast()) {
2123       return false;
2124     }
2125
2126     uint32_t slot = loc.slot();
2127     uint32_t numFixedSlots = templateObj->numFixedSlots();
2128     if (slot >= numFixedSlots) {
2129       if (!envSlots) {
2130         envSlots = MSlots::New(alloc(), env);
2131         current->add(envSlots);
2132       }
2133       if (!slots) {
2134         slots = MSlots::New(alloc(), ins);
2135         current->add(slots);
2136       }
2137
2138       uint32_t dynamicSlot = slot - numFixedSlots;
2139
2140       auto* load = MLoadDynamicSlot::New(alloc(), envSlots, dynamicSlot);
2141       current->add(load);
2142
2143 #ifdef DEBUG
2144       // Assert in debug mode we can elide the post write barrier.
2145       current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, load));
2146 #endif
2147
2148       current->add(
2149           MStoreDynamicSlot::NewUnbarriered(alloc(), slots, dynamicSlot, load));
2150     } else {
2151       auto* load = MLoadFixedSlot::New(alloc(), env, slot);
2152       current->add(load);
2153
2154 #ifdef DEBUG
2155       // Assert in debug mode we can elide the post write barrier.
2156       current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, load));
2157 #endif
2158
2159       current->add(MStoreFixedSlot::NewUnbarriered(alloc(), ins, slot, load));
2160     }
2161   }
2162
2163   current->setEnvironmentChain(ins);
2164   return true;
2165 }
2166
2167 bool WarpBuilder::build_RecreateLexicalEnv(BytecodeLocation loc) {
2168   MOZ_ASSERT(usesEnvironmentChain());
2169
2170   const auto* snapshot = getOpSnapshot<WarpLexicalEnvironment>(loc);
2171   MOZ_ASSERT(snapshot);
2172
2173   MDefinition* enclosingEnv = walkEnvironmentChain(1);
2174   if (!enclosingEnv) {
2175     return false;
2176   }
2177
2178   MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
2179
2180   auto* ins = MNewLexicalEnvironmentObject::New(alloc(), templateCst);
2181   current->add(ins);
2182
2183 #ifdef DEBUG
2184   // Assert in debug mode we can elide the post write barrier.
2185   current->add(
2186       MAssertCanElidePostWriteBarrier::New(alloc(), ins, enclosingEnv));
2187 #endif
2188
2189   // Initialize the object's reserved slots. No post barrier is needed here,
2190   // for the same reason as in buildNamedLambdaEnv.
2191   current->add(MStoreFixedSlot::NewUnbarriered(
2192       alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(),
2193       enclosingEnv));
2194
2195   current->setEnvironmentChain(ins);
2196   return true;
2197 }
2198
2199 bool WarpBuilder::build_PushVarEnv(BytecodeLocation loc) {
2200   MOZ_ASSERT(usesEnvironmentChain());
2201
2202   const auto* snapshot = getOpSnapshot<WarpVarEnvironment>(loc);
2203   MOZ_ASSERT(snapshot);
2204
2205   MDefinition* env = current->environmentChain();
2206   MConstant* templateCst = constant(ObjectValue(*snapshot->templateObj()));
2207
2208   auto* ins = MNewVarEnvironmentObject::New(alloc(), templateCst);
2209   current->add(ins);
2210
2211 #ifdef DEBUG
2212   // Assert in debug mode we can elide the post write barrier.
2213   current->add(MAssertCanElidePostWriteBarrier::New(alloc(), ins, env));
2214 #endif
2215
2216   // Initialize the object's reserved slots. No post barrier is needed here,
2217   // for the same reason as in buildNamedLambdaEnv.
2218   current->add(MStoreFixedSlot::NewUnbarriered(
2219       alloc(), ins, EnvironmentObject::enclosingEnvironmentSlot(), env));
2220
2221   current->setEnvironmentChain(ins);
2222   return true;
2223 }
2224
2225 bool WarpBuilder::build_ImplicitThis(BytecodeLocation loc) {
2226   MOZ_ASSERT(usesEnvironmentChain());
2227
2228   PropertyName* name = loc.getPropertyName(script_);
2229   MDefinition* env = current->environmentChain();
2230
2231   auto* ins = MImplicitThis::New(alloc(), env, name);
2232   current->add(ins);
2233   current->push(ins);
2234   return resumeAfter(ins, loc);
2235 }
2236
2237 bool WarpBuilder::build_CheckClassHeritage(BytecodeLocation loc) {
2238   MDefinition* def = current->pop();
2239   auto* ins = MCheckClassHeritage::New(alloc(), def);
2240   current->add(ins);
2241   current->push(ins);
2242   return resumeAfter(ins, loc);
2243 }
2244
2245 bool WarpBuilder::build_CheckThis(BytecodeLocation loc) {
2246   MDefinition* def = current->pop();
2247   auto* ins = MCheckThis::New(alloc(), def);
2248   current->add(ins);
2249   current->push(ins);
2250   return resumeAfter(ins, loc);
2251 }
2252
2253 bool WarpBuilder::build_CheckThisReinit(BytecodeLocation loc) {
2254   MDefinition* def = current->pop();
2255   auto* ins = MCheckThisReinit::New(alloc(), def);
2256   current->add(ins);
2257   current->push(ins);
2258   return resumeAfter(ins, loc);
2259 }
2260
2261 bool WarpBuilder::build_Generator(BytecodeLocation loc) {
2262   MOZ_ASSERT(usesEnvironmentChain());
2263
2264   MDefinition* callee = getCallee();
2265   MDefinition* environmentChain = current->environmentChain();
2266   MDefinition* argsObj = info().needsArgsObj() ? current->argumentsObject()
2267                                                : constant(Int32Value(0));
2268
2269   MGenerator* generator =
2270       MGenerator::New(alloc(), callee, environmentChain, argsObj);
2271
2272   current->add(generator);
2273   current->push(generator);
2274   return resumeAfter(generator, loc);
2275 }
2276
2277 bool WarpBuilder::build_AfterYield(BytecodeLocation loc) {
2278   // Unreachable blocks don't need to generate a bail.
2279   if (hasTerminatedBlock()) {
2280     return true;
2281   }
2282
2283   // This comes after a yield, which we generate as a return,
2284   // so we know this should be unreachable code.
2285   //
2286   // We emit an unreachable bail for this, which will assert if we
2287   // ever execute this.
2288   //
2289   // An Unreachable bail, instead of MUnreachable, because MUnreachable
2290   // is a control instruction, and injecting it in the middle of a block
2291   // causes various graph state assertions to fail.
2292   MBail* bail = MBail::New(alloc(), BailoutKind::Unreachable);
2293   current->add(bail);
2294
2295   return true;
2296 }
2297
2298 bool WarpBuilder::build_FinalYieldRval(BytecodeLocation loc) {
2299   MDefinition* gen = current->pop();
2300
2301   auto setSlotNull = [this, gen](size_t slot) {
2302     auto* ins = MStoreFixedSlot::NewBarriered(alloc(), gen, slot,
2303                                               constant(NullValue()));
2304     current->add(ins);
2305   };
2306
2307   // Close the generator
2308   setSlotNull(AbstractGeneratorObject::calleeSlot());
2309   setSlotNull(AbstractGeneratorObject::envChainSlot());
2310   setSlotNull(AbstractGeneratorObject::argsObjectSlot());
2311   setSlotNull(AbstractGeneratorObject::stackStorageSlot());
2312   setSlotNull(AbstractGeneratorObject::resumeIndexSlot());
2313
2314   // Return
2315   return build_RetRval(loc);
2316 }
2317
2318 bool WarpBuilder::build_AsyncResolve(BytecodeLocation loc) {
2319   MDefinition* generator = current->pop();
2320   MDefinition* value = current->pop();
2321
2322   auto* resolve = MAsyncResolve::New(alloc(), generator, value);
2323   current->add(resolve);
2324   current->push(resolve);
2325   return resumeAfter(resolve, loc);
2326 }
2327
2328 bool WarpBuilder::build_AsyncReject(BytecodeLocation loc) {
2329   MDefinition* generator = current->pop();
2330   MDefinition* stack = current->pop();
2331   MDefinition* reason = current->pop();
2332
2333   auto* reject = MAsyncReject::New(alloc(), generator, reason, stack);
2334   current->add(reject);
2335   current->push(reject);
2336   return resumeAfter(reject, loc);
2337 }
2338
2339 bool WarpBuilder::build_ResumeKind(BytecodeLocation loc) {
2340   GeneratorResumeKind resumeKind = loc.resumeKind();
2341
2342   current->push(constant(Int32Value(static_cast<int32_t>(resumeKind))));
2343   return true;
2344 }
2345
2346 bool WarpBuilder::build_CheckResumeKind(BytecodeLocation loc) {
2347   // Outside of `yield*`, this is normally unreachable code in Warp,
2348   // so we just manipulate the stack appropriately to ensure correct
2349   // MIR generation.
2350   //
2351   // However, `yield*` emits a forced generator return which can be
2352   // warp compiled, so in order to correctly handle these semantics
2353   // we also generate a bailout, so that the forced generator return
2354   // runs in baseline.
2355   MDefinition* resumeKind = current->pop();
2356   MDefinition* gen = current->pop();
2357   MDefinition* rval = current->peek(-1);
2358
2359   // Mark operands as implicitly used.
2360   resumeKind->setImplicitlyUsedUnchecked();
2361   gen->setImplicitlyUsedUnchecked();
2362   rval->setImplicitlyUsedUnchecked();
2363
2364   // Bail out if we encounter CheckResumeKind.
2365   MBail* bail = MBail::New(alloc(), BailoutKind::Inevitable);
2366   current->add(bail);
2367   current->setAlwaysBails();
2368
2369   return true;
2370 }
2371
2372 bool WarpBuilder::build_CanSkipAwait(BytecodeLocation loc) {
2373   MDefinition* val = current->pop();
2374
2375   MCanSkipAwait* canSkip = MCanSkipAwait::New(alloc(), val);
2376   current->add(canSkip);
2377
2378   current->push(val);
2379   current->push(canSkip);
2380
2381   return resumeAfter(canSkip, loc);
2382 }
2383
2384 bool WarpBuilder::build_MaybeExtractAwaitValue(BytecodeLocation loc) {
2385   MDefinition* canSkip = current->pop();
2386   MDefinition* value = current->pop();
2387
2388   MMaybeExtractAwaitValue* extracted =
2389       MMaybeExtractAwaitValue::New(alloc(), value, canSkip);
2390   current->add(extracted);
2391
2392   current->push(extracted);
2393   current->push(canSkip);
2394
2395   return resumeAfter(extracted, loc);
2396 }
2397
2398 bool WarpBuilder::build_InitialYield(BytecodeLocation loc) {
2399   MDefinition* gen = current->pop();
2400   return buildSuspend(loc, gen, gen);
2401 }
2402
2403 bool WarpBuilder::build_Await(BytecodeLocation loc) {
2404   MDefinition* gen = current->pop();
2405   MDefinition* promiseOrGenerator = current->pop();
2406
2407   return buildSuspend(loc, gen, promiseOrGenerator);
2408 }
2409 bool WarpBuilder::build_Yield(BytecodeLocation loc) { return build_Await(loc); }
2410
2411 bool WarpBuilder::buildSuspend(BytecodeLocation loc, MDefinition* gen,
2412                                MDefinition* retVal) {
2413   // If required, unbox the generator object explicitly and infallibly.
2414   //
2415   // This is done to avoid fuzz-bugs where ApplyTypeInformation does the
2416   // unboxing, and generates fallible unboxes which can lead to torn object
2417   // state due to `bailAfter`.
2418   MDefinition* genObj = gen;
2419   if (genObj->type() != MIRType::Object) {
2420     auto* unbox =
2421         MUnbox::New(alloc(), gen, MIRType::Object, MUnbox::Mode::Infallible);
2422     current->add(unbox);
2423
2424     genObj = unbox;
2425   }
2426
2427   int32_t slotsToCopy = current->stackDepth() - info().firstLocalSlot();
2428   MOZ_ASSERT(slotsToCopy >= 0);
2429   if (slotsToCopy > 0) {
2430     auto* arrayObj = MLoadFixedSlotAndUnbox::New(
2431         alloc(), genObj, AbstractGeneratorObject::stackStorageSlot(),
2432         MUnbox::Mode::Infallible, MIRType::Object);
2433     current->add(arrayObj);
2434
2435     auto* stackStorage = MElements::New(alloc(), arrayObj);
2436     current->add(stackStorage);
2437
2438     for (int32_t i = 0; i < slotsToCopy; i++) {
2439       if (!alloc().ensureBallast()) {
2440         return false;
2441       }
2442       // Use peekUnchecked because we're also writing out the argument slots
2443       int32_t peek = -slotsToCopy + i;
2444       MDefinition* stackElem = current->peekUnchecked(peek);
2445       auto* store = MStoreElement::NewUnbarriered(
2446           alloc(), stackStorage, constant(Int32Value(i)), stackElem,
2447           /* needsHoleCheck = */ false);
2448
2449       current->add(store);
2450       current->add(MPostWriteBarrier::New(alloc(), arrayObj, stackElem));
2451     }
2452
2453     auto* len = constant(Int32Value(slotsToCopy - 1));
2454
2455     auto* setInitLength =
2456         MSetInitializedLength::New(alloc(), stackStorage, len);
2457     current->add(setInitLength);
2458
2459     auto* setLength = MSetArrayLength::New(alloc(), stackStorage, len);
2460     current->add(setLength);
2461   }
2462
2463   // Update Generator Object state
2464   uint32_t resumeIndex = loc.getResumeIndex();
2465
2466   // This store is unbarriered, as it's only ever storing an integer, and as
2467   // such doesn't partake of object tracing.
2468   current->add(MStoreFixedSlot::NewUnbarriered(
2469       alloc(), genObj, AbstractGeneratorObject::resumeIndexSlot(),
2470       constant(Int32Value(resumeIndex))));
2471
2472   // This store is barriered because it stores an object value.
2473   current->add(MStoreFixedSlot::NewBarriered(
2474       alloc(), genObj, AbstractGeneratorObject::envChainSlot(),
2475       current->environmentChain()));
2476
2477   current->add(
2478       MPostWriteBarrier::New(alloc(), genObj, current->environmentChain()));
2479
2480   // GeneratorReturn will return from the method, however to support MIR
2481   // generation isn't treated like the end of a block
2482   MGeneratorReturn* ret = MGeneratorReturn::New(alloc(), retVal);
2483   current->add(ret);
2484
2485   // To ensure the rest of the MIR generation looks correct, fill the stack with
2486   // the appropriately typed MUnreachable's for the stack pushes from this
2487   // opcode.
2488   auto* unreachableResumeKind =
2489       MUnreachableResult::New(alloc(), MIRType::Int32);
2490   current->add(unreachableResumeKind);
2491   current->push(unreachableResumeKind);
2492
2493   auto* unreachableGenerator =
2494       MUnreachableResult::New(alloc(), MIRType::Object);
2495   current->add(unreachableGenerator);
2496   current->push(unreachableGenerator);
2497
2498   auto* unreachableRval = MUnreachableResult::New(alloc(), MIRType::Value);
2499   current->add(unreachableRval);
2500   current->push(unreachableRval);
2501
2502   return true;
2503 }
2504
2505 bool WarpBuilder::build_AsyncAwait(BytecodeLocation loc) {
2506   MDefinition* gen = current->pop();
2507   MDefinition* value = current->pop();
2508
2509   MAsyncAwait* asyncAwait = MAsyncAwait::New(alloc(), value, gen);
2510   current->add(asyncAwait);
2511   current->push(asyncAwait);
2512   return resumeAfter(asyncAwait, loc);
2513 }
2514
2515 bool WarpBuilder::build_CheckReturn(BytecodeLocation loc) {
2516   MOZ_ASSERT(!script_->noScriptRval());
2517
2518   MDefinition* returnValue = current->getSlot(info().returnValueSlot());
2519   MDefinition* thisValue = current->pop();
2520
2521   auto* ins = MCheckReturn::New(alloc(), returnValue, thisValue);
2522   current->add(ins);
2523   current->push(ins);
2524   return resumeAfter(ins, loc);
2525 }
2526
2527 void WarpBuilder::buildCheckLexicalOp(BytecodeLocation loc) {
2528   JSOp op = loc.getOp();
2529   MOZ_ASSERT(op == JSOp::CheckLexical || op == JSOp::CheckAliasedLexical);
2530
2531   MDefinition* input = current->pop();
2532   MInstruction* lexicalCheck = MLexicalCheck::New(alloc(), input);
2533   current->add(lexicalCheck);
2534   current->push(lexicalCheck);
2535
2536   if (snapshot().bailoutInfo().failedLexicalCheck()) {
2537     // If we have previously had a failed lexical check in Ion, we want to avoid
2538     // hoisting any lexical checks, which can cause spurious failures. In this
2539     // case, we also have to be careful not to hoist any loads of this lexical
2540     // past the check. For unaliased lexical variables, we can set the local
2541     // slot to create a dependency (see below). For aliased lexicals, that
2542     // doesn't work, so we disable LICM instead.
2543     lexicalCheck->setNotMovable();
2544     if (op == JSOp::CheckAliasedLexical) {
2545       mirGen().disableLICM();
2546     }
2547   }
2548
2549   if (op == JSOp::CheckLexical) {
2550     // Set the local slot so that a subsequent GetLocal without a CheckLexical
2551     // (the frontend can elide lexical checks) doesn't let a definition with
2552     // MIRType::MagicUninitializedLexical escape to arbitrary MIR instructions.
2553     // Note that in this case the GetLocal would be unreachable because we throw
2554     // an exception here, but we still generate MIR instructions for it.
2555     uint32_t slot = info().localSlot(loc.local());
2556     current->setSlot(slot, lexicalCheck);
2557   }
2558 }
2559
2560 bool WarpBuilder::build_CheckLexical(BytecodeLocation loc) {
2561   buildCheckLexicalOp(loc);
2562   return true;
2563 }
2564
2565 bool WarpBuilder::build_CheckAliasedLexical(BytecodeLocation loc) {
2566   buildCheckLexicalOp(loc);
2567   return true;
2568 }
2569
2570 bool WarpBuilder::build_InitHomeObject(BytecodeLocation loc) {
2571   MDefinition* homeObject = current->pop();
2572   MDefinition* function = current->pop();
2573
2574   current->add(MPostWriteBarrier::New(alloc(), function, homeObject));
2575
2576   auto* ins = MInitHomeObject::New(alloc(), function, homeObject);
2577   current->add(ins);
2578   current->push(ins);
2579   return true;
2580 }
2581
2582 bool WarpBuilder::build_SuperBase(BytecodeLocation) {
2583   MDefinition* callee = current->pop();
2584
2585   auto* homeObject = MHomeObject::New(alloc(), callee);
2586   current->add(homeObject);
2587
2588   auto* superBase = MHomeObjectSuperBase::New(alloc(), homeObject);
2589   current->add(superBase);
2590   current->push(superBase);
2591   return true;
2592 }
2593
2594 bool WarpBuilder::build_SuperFun(BytecodeLocation) {
2595   MDefinition* callee = current->pop();
2596   auto* ins = MSuperFunction::New(alloc(), callee);
2597   current->add(ins);
2598   current->push(ins);
2599   return true;
2600 }
2601
2602 bool WarpBuilder::build_BuiltinObject(BytecodeLocation loc) {
2603   if (auto* snapshot = getOpSnapshot<WarpBuiltinObject>(loc)) {
2604     JSObject* builtin = snapshot->builtin();
2605     pushConstant(ObjectValue(*builtin));
2606     return true;
2607   }
2608
2609   auto kind = loc.getBuiltinObjectKind();
2610   auto* ins = MBuiltinObject::New(alloc(), kind);
2611   current->add(ins);
2612   current->push(ins);
2613   return resumeAfter(ins, loc);
2614 }
2615
2616 bool WarpBuilder::build_GetIntrinsic(BytecodeLocation loc) {
2617   if (auto* snapshot = getOpSnapshot<WarpGetIntrinsic>(loc)) {
2618     Value intrinsic = snapshot->intrinsic();
2619     pushConstant(intrinsic);
2620     return true;
2621   }
2622
2623   PropertyName* name = loc.getPropertyName(script_);
2624   MCallGetIntrinsicValue* ins = MCallGetIntrinsicValue::New(alloc(), name);
2625   current->add(ins);
2626   current->push(ins);
2627   return resumeAfter(ins, loc);
2628 }
2629
2630 bool WarpBuilder::build_ImportMeta(BytecodeLocation loc) {
2631   ModuleObject* moduleObj = scriptSnapshot()->moduleObject();
2632   MOZ_ASSERT(moduleObj);
2633
2634   MModuleMetadata* ins = MModuleMetadata::New(alloc(), moduleObj);
2635   current->add(ins);
2636   current->push(ins);
2637   return resumeAfter(ins, loc);
2638 }
2639
2640 bool WarpBuilder::build_CallSiteObj(BytecodeLocation loc) {
2641   return build_Object(loc);
2642 }
2643
2644 bool WarpBuilder::build_NewArray(BytecodeLocation loc) {
2645   return buildIC(loc, CacheKind::NewArray, {});
2646 }
2647
2648 bool WarpBuilder::build_NewObject(BytecodeLocation loc) {
2649   return buildIC(loc, CacheKind::NewObject, {});
2650 }
2651
2652 bool WarpBuilder::build_NewInit(BytecodeLocation loc) {
2653   return build_NewObject(loc);
2654 }
2655
2656 bool WarpBuilder::build_Object(BytecodeLocation loc) {
2657   JSObject* obj = loc.getObject(script_);
2658   MConstant* objConst = constant(ObjectValue(*obj));
2659
2660   current->push(objConst);
2661   return true;
2662 }
2663
2664 bool WarpBuilder::buildInitPropGetterSetterOp(BytecodeLocation loc) {
2665   PropertyName* name = loc.getPropertyName(script_);
2666   MDefinition* value = current->pop();
2667   MDefinition* obj = current->peek(-1);
2668
2669   auto* ins = MInitPropGetterSetter::New(alloc(), obj, value, name);
2670   current->add(ins);
2671   return resumeAfter(ins, loc);
2672 }
2673
2674 bool WarpBuilder::build_InitPropGetter(BytecodeLocation loc) {
2675   return buildInitPropGetterSetterOp(loc);
2676 }
2677
2678 bool WarpBuilder::build_InitPropSetter(BytecodeLocation loc) {
2679   return buildInitPropGetterSetterOp(loc);
2680 }
2681
2682 bool WarpBuilder::build_InitHiddenPropGetter(BytecodeLocation loc) {
2683   return buildInitPropGetterSetterOp(loc);
2684 }
2685
2686 bool WarpBuilder::build_InitHiddenPropSetter(BytecodeLocation loc) {
2687   return buildInitPropGetterSetterOp(loc);
2688 }
2689
2690 bool WarpBuilder::buildInitElemGetterSetterOp(BytecodeLocation loc) {
2691   MDefinition* value = current->pop();
2692   MDefinition* id = current->pop();
2693   MDefinition* obj = current->peek(-1);
2694
2695   auto* ins = MInitElemGetterSetter::New(alloc(), obj, id, value);
2696   current->add(ins);
2697   return resumeAfter(ins, loc);
2698 }
2699
2700 bool WarpBuilder::build_InitElemGetter(BytecodeLocation loc) {
2701   return buildInitElemGetterSetterOp(loc);
2702 }
2703
2704 bool WarpBuilder::build_InitElemSetter(BytecodeLocation loc) {
2705   return buildInitElemGetterSetterOp(loc);
2706 }
2707
2708 bool WarpBuilder::build_InitHiddenElemGetter(BytecodeLocation loc) {
2709   return buildInitElemGetterSetterOp(loc);
2710 }
2711
2712 bool WarpBuilder::build_InitHiddenElemSetter(BytecodeLocation loc) {
2713   return buildInitElemGetterSetterOp(loc);
2714 }
2715
2716 bool WarpBuilder::build_In(BytecodeLocation loc) {
2717   MDefinition* obj = current->pop();
2718   MDefinition* id = current->pop();
2719   return buildIC(loc, CacheKind::In, {id, obj});
2720 }
2721
2722 bool WarpBuilder::build_HasOwn(BytecodeLocation loc) {
2723   MDefinition* obj = current->pop();
2724   MDefinition* id = current->pop();
2725   return buildIC(loc, CacheKind::HasOwn, {id, obj});
2726 }
2727
2728 bool WarpBuilder::build_CheckPrivateField(BytecodeLocation loc) {
2729   MDefinition* id = current->peek(-1);
2730   MDefinition* obj = current->peek(-2);
2731   return buildIC(loc, CacheKind::CheckPrivateField, {obj, id});
2732 }
2733
2734 bool WarpBuilder::build_NewPrivateName(BytecodeLocation loc) {
2735   JSAtom* name = loc.getAtom(script_);
2736
2737   auto* ins = MNewPrivateName::New(alloc(), name);
2738   current->add(ins);
2739   current->push(ins);
2740   return resumeAfter(ins, loc);
2741 }
2742
2743 bool WarpBuilder::build_Instanceof(BytecodeLocation loc) {
2744   MDefinition* rhs = current->pop();
2745   MDefinition* obj = current->pop();
2746   return buildIC(loc, CacheKind::InstanceOf, {obj, rhs});
2747 }
2748
2749 bool WarpBuilder::build_NewTarget(BytecodeLocation loc) {
2750   MOZ_ASSERT(script_->isFunction());
2751   MOZ_ASSERT(info().hasFunMaybeLazy());
2752   MOZ_ASSERT(!scriptSnapshot()->isArrowFunction());
2753
2754   if (inlineCallInfo()) {
2755     if (inlineCallInfo()->constructing()) {
2756       current->push(inlineCallInfo()->getNewTarget());
2757     } else {
2758       pushConstant(UndefinedValue());
2759     }
2760     return true;
2761   }
2762
2763   MNewTarget* ins = MNewTarget::New(alloc());
2764   current->add(ins);
2765   current->push(ins);
2766   return true;
2767 }
2768
2769 bool WarpBuilder::build_CheckIsObj(BytecodeLocation loc) {
2770   CheckIsObjectKind kind = loc.getCheckIsObjectKind();
2771
2772   MDefinition* toCheck = current->peek(-1);
2773   if (toCheck->type() == MIRType::Object) {
2774     toCheck->setImplicitlyUsedUnchecked();
2775     return true;
2776   }
2777
2778   MDefinition* val = current->pop();
2779   MCheckIsObj* ins = MCheckIsObj::New(alloc(), val, uint8_t(kind));
2780   current->add(ins);
2781   current->push(ins);
2782   return resumeAfter(ins, loc);
2783 }
2784
2785 bool WarpBuilder::build_CheckObjCoercible(BytecodeLocation loc) {
2786   MDefinition* val = current->pop();
2787   MCheckObjCoercible* ins = MCheckObjCoercible::New(alloc(), val);
2788   current->add(ins);
2789   current->push(ins);
2790   return resumeAfter(ins, loc);
2791 }
2792
2793 MInstruction* WarpBuilder::buildLoadSlot(MDefinition* obj,
2794                                          uint32_t numFixedSlots,
2795                                          uint32_t slot) {
2796   if (slot < numFixedSlots) {
2797     MLoadFixedSlot* load = MLoadFixedSlot::New(alloc(), obj, slot);
2798     current->add(load);
2799     return load;
2800   }
2801
2802   MSlots* slots = MSlots::New(alloc(), obj);
2803   current->add(slots);
2804
2805   MLoadDynamicSlot* load =
2806       MLoadDynamicSlot::New(alloc(), slots, slot - numFixedSlots);
2807   current->add(load);
2808   return load;
2809 }
2810
2811 bool WarpBuilder::build_GetImport(BytecodeLocation loc) {
2812   auto* snapshot = getOpSnapshot<WarpGetImport>(loc);
2813
2814   ModuleEnvironmentObject* targetEnv = snapshot->targetEnv();
2815
2816   // Load the target environment slot.
2817   MConstant* obj = constant(ObjectValue(*targetEnv));
2818   auto* load = buildLoadSlot(obj, snapshot->numFixedSlots(), snapshot->slot());
2819
2820   if (snapshot->needsLexicalCheck()) {
2821     // TODO: IonBuilder has code to mark non-movable. See buildCheckLexicalOp.
2822     MInstruction* lexicalCheck = MLexicalCheck::New(alloc(), load);
2823     current->add(lexicalCheck);
2824     current->push(lexicalCheck);
2825   } else {
2826     current->push(load);
2827   }
2828
2829   return true;
2830 }
2831
2832 bool WarpBuilder::build_GetPropSuper(BytecodeLocation loc) {
2833   MDefinition* obj = current->pop();
2834   MDefinition* receiver = current->pop();
2835   return buildIC(loc, CacheKind::GetPropSuper, {obj, receiver});
2836 }
2837
2838 bool WarpBuilder::build_GetElemSuper(BytecodeLocation loc) {
2839   MDefinition* obj = current->pop();
2840   MDefinition* id = current->pop();
2841   MDefinition* receiver = current->pop();
2842   return buildIC(loc, CacheKind::GetElemSuper, {obj, id, receiver});
2843 }
2844
2845 bool WarpBuilder::build_InitProp(BytecodeLocation loc) {
2846   MDefinition* val = current->pop();
2847   MDefinition* obj = current->peek(-1);
2848   return buildIC(loc, CacheKind::SetProp, {obj, val});
2849 }
2850
2851 bool WarpBuilder::build_InitLockedProp(BytecodeLocation loc) {
2852   return build_InitProp(loc);
2853 }
2854
2855 bool WarpBuilder::build_InitHiddenProp(BytecodeLocation loc) {
2856   return build_InitProp(loc);
2857 }
2858
2859 bool WarpBuilder::build_InitElem(BytecodeLocation loc) {
2860   MDefinition* val = current->pop();
2861   MDefinition* id = current->pop();
2862   MDefinition* obj = current->peek(-1);
2863   return buildIC(loc, CacheKind::SetElem, {obj, id, val});
2864 }
2865
2866 bool WarpBuilder::build_InitLockedElem(BytecodeLocation loc) {
2867   return build_InitElem(loc);
2868 }
2869
2870 bool WarpBuilder::build_InitHiddenElem(BytecodeLocation loc) {
2871   return build_InitElem(loc);
2872 }
2873
2874 bool WarpBuilder::build_InitElemArray(BytecodeLocation loc) {
2875   MDefinition* val = current->pop();
2876   MDefinition* obj = current->peek(-1);
2877
2878   // Note: getInitElemArrayIndex asserts the index fits in int32_t.
2879   uint32_t index = loc.getInitElemArrayIndex();
2880   MConstant* indexConst = constant(Int32Value(index));
2881
2882   // Note: InitArrayElemOperation asserts the index does not exceed the array's
2883   // dense element capacity.
2884
2885   auto* elements = MElements::New(alloc(), obj);
2886   current->add(elements);
2887
2888   if (val->type() == MIRType::MagicHole) {
2889     val->setImplicitlyUsedUnchecked();
2890     auto* store = MStoreHoleValueElement::New(alloc(), elements, indexConst);
2891     current->add(store);
2892   } else {
2893     current->add(MPostWriteBarrier::New(alloc(), obj, val));
2894     auto* store =
2895         MStoreElement::NewUnbarriered(alloc(), elements, indexConst, val,
2896                                       /* needsHoleCheck = */ false);
2897     current->add(store);
2898   }
2899
2900   auto* setLength = MSetInitializedLength::New(alloc(), elements, indexConst);
2901   current->add(setLength);
2902
2903   return resumeAfter(setLength, loc);
2904 }
2905
2906 bool WarpBuilder::build_InitElemInc(BytecodeLocation loc) {
2907   MDefinition* val = current->pop();
2908   MDefinition* index = current->pop();
2909   MDefinition* obj = current->peek(-1);
2910
2911   // Push index + 1.
2912   MConstant* constOne = constant(Int32Value(1));
2913   MAdd* nextIndex = MAdd::New(alloc(), index, constOne, TruncateKind::Truncate);
2914   current->add(nextIndex);
2915   current->push(nextIndex);
2916
2917   return buildIC(loc, CacheKind::SetElem, {obj, index, val});
2918 }
2919
2920 bool WarpBuilder::build_Lambda(BytecodeLocation loc) {
2921   MOZ_ASSERT(usesEnvironmentChain());
2922
2923   MDefinition* env = current->environmentChain();
2924
2925   JSFunction* fun = loc.getFunction(script_);
2926   MConstant* funConst = constant(ObjectValue(*fun));
2927
2928   auto* ins = MLambda::New(alloc(), env, funConst);
2929   current->add(ins);
2930   current->push(ins);
2931   return resumeAfter(ins, loc);
2932 }
2933
2934 bool WarpBuilder::build_FunWithProto(BytecodeLocation loc) {
2935   MOZ_ASSERT(usesEnvironmentChain());
2936
2937   MDefinition* proto = current->pop();
2938   MDefinition* env = current->environmentChain();
2939
2940   JSFunction* fun = loc.getFunction(script_);
2941   MConstant* funConst = constant(ObjectValue(*fun));
2942
2943   auto* ins = MFunctionWithProto::New(alloc(), env, proto, funConst);
2944   current->add(ins);
2945   current->push(ins);
2946   return resumeAfter(ins, loc);
2947 }
2948
2949 bool WarpBuilder::build_SpreadCall(BytecodeLocation loc) {
2950   bool constructing = false;
2951   CallInfo callInfo(alloc(), constructing, loc.resultIsPopped());
2952   callInfo.initForSpreadCall(current);
2953
2954   // The argument must be an array object. Add an infallible MUnbox if needed,
2955   // but ensure it's not loop hoisted before the branch in the bytecode guarding
2956   // that it's not undefined.
2957   MOZ_ASSERT(callInfo.argc() == 1);
2958   callInfo.setArg(0, unboxObjectInfallible(callInfo.getArg(0), IsMovable::No));
2959
2960   if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
2961     return transpileCall(loc, cacheIRSnapshot, &callInfo);
2962   }
2963
2964   bool needsThisCheck = false;
2965   MInstruction* call = makeSpreadCall(callInfo, needsThisCheck);
2966   if (!call) {
2967     return false;
2968   }
2969   call->setBailoutKind(BailoutKind::TooManyArguments);
2970   current->add(call);
2971   current->push(call);
2972   return resumeAfter(call, loc);
2973 }
2974
2975 bool WarpBuilder::build_SpreadNew(BytecodeLocation loc) {
2976   bool constructing = true;
2977   CallInfo callInfo(alloc(), constructing, loc.resultIsPopped());
2978   callInfo.initForSpreadCall(current);
2979
2980   // See build_SpreadCall.
2981   MOZ_ASSERT(callInfo.argc() == 1);
2982   callInfo.setArg(0, unboxObjectInfallible(callInfo.getArg(0), IsMovable::No));
2983
2984   if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
2985     return transpileCall(loc, cacheIRSnapshot, &callInfo);
2986   }
2987
2988   buildCreateThis(callInfo);
2989
2990   bool needsThisCheck = true;
2991   MInstruction* call = makeSpreadCall(callInfo, needsThisCheck);
2992   if (!call) {
2993     return false;
2994   }
2995   call->setBailoutKind(BailoutKind::TooManyArguments);
2996   current->add(call);
2997   current->push(call);
2998   return resumeAfter(call, loc);
2999 }
3000
3001 bool WarpBuilder::build_SpreadSuperCall(BytecodeLocation loc) {
3002   return build_SpreadNew(loc);
3003 }
3004
3005 bool WarpBuilder::build_OptimizeSpreadCall(BytecodeLocation loc) {
3006   MDefinition* value = current->pop();
3007   return buildIC(loc, CacheKind::OptimizeSpreadCall, {value});
3008 }
3009
3010 bool WarpBuilder::build_Debugger(BytecodeLocation loc) {
3011   // The |debugger;| statement will bail out to Baseline if the realm is a
3012   // debuggee realm with an onDebuggerStatement hook.
3013   MDebugger* debugger = MDebugger::New(alloc());
3014   current->add(debugger);
3015   return resumeAfter(debugger, loc);
3016 }
3017
3018 bool WarpBuilder::build_TableSwitch(BytecodeLocation loc) {
3019   int32_t low = loc.getTableSwitchLow();
3020   int32_t high = loc.getTableSwitchHigh();
3021   size_t numCases = high - low + 1;
3022
3023   MDefinition* input = current->pop();
3024   MTableSwitch* tableswitch = MTableSwitch::New(alloc(), input, low, high);
3025   current->end(tableswitch);
3026
3027   // Table mapping from target bytecode offset to MTableSwitch successor index.
3028   // This prevents adding multiple predecessor/successor edges to the same
3029   // target block, which isn't valid in MIR.
3030   using TargetToSuccessorMap =
3031       InlineMap<uint32_t, uint32_t, 8, DefaultHasher<uint32_t>,
3032                 SystemAllocPolicy>;
3033   TargetToSuccessorMap targetToSuccessor;
3034
3035   // Create |default| edge.
3036   {
3037     BytecodeLocation defaultLoc = loc.getTableSwitchDefaultTarget();
3038     uint32_t defaultOffset = defaultLoc.bytecodeToOffset(script_);
3039
3040     size_t index;
3041     if (!tableswitch->addDefault(nullptr, &index)) {
3042       return false;
3043     }
3044     if (!addPendingEdge(defaultLoc, current, index)) {
3045       return false;
3046     }
3047     if (!targetToSuccessor.put(defaultOffset, index)) {
3048       return false;
3049     }
3050   }
3051
3052   // Add all cases.
3053   for (size_t i = 0; i < numCases; i++) {
3054     BytecodeLocation caseLoc = loc.getTableSwitchCaseTarget(script_, i);
3055     uint32_t caseOffset = caseLoc.bytecodeToOffset(script_);
3056
3057     size_t index;
3058     if (auto p = targetToSuccessor.lookupForAdd(caseOffset)) {
3059       index = p->value();
3060     } else {
3061       if (!tableswitch->addSuccessor(nullptr, &index)) {
3062         return false;
3063       }
3064       if (!addPendingEdge(caseLoc, current, index)) {
3065         return false;
3066       }
3067       if (!targetToSuccessor.add(p, caseOffset, index)) {
3068         return false;
3069       }
3070     }
3071     if (!tableswitch->addCase(index)) {
3072       return false;
3073     }
3074   }
3075
3076   setTerminatedBlock();
3077   return true;
3078 }
3079
3080 bool WarpBuilder::build_Rest(BytecodeLocation loc) {
3081   auto* snapshot = getOpSnapshot<WarpRest>(loc);
3082   Shape* shape = snapshot ? snapshot->shape() : nullptr;
3083
3084   // NOTE: Keep this code in sync with |ArgumentsReplacer|.
3085
3086   if (inlineCallInfo()) {
3087     // If we are inlining, we know the actual arguments.
3088     unsigned numActuals = inlineCallInfo()->argc();
3089     unsigned numFormals = info().nargs() - 1;
3090     unsigned numRest = numActuals > numFormals ? numActuals - numFormals : 0;
3091
3092     // TODO: support pre-tenuring.
3093     gc::Heap heap = gc::Heap::Default;
3094
3095     // Allocate an array of the correct size.
3096     MInstruction* newArray;
3097     if (shape && gc::CanUseFixedElementsForArray(numRest)) {
3098       auto* shapeConstant = MConstant::NewShape(alloc(), shape);
3099       current->add(shapeConstant);
3100       newArray = MNewArrayObject::New(alloc(), shapeConstant, numRest, heap);
3101     } else {
3102       MConstant* templateConst = constant(NullValue());
3103       newArray = MNewArray::NewVM(alloc(), numRest, templateConst, heap);
3104     }
3105     current->add(newArray);
3106     current->push(newArray);
3107
3108     if (numRest == 0) {
3109       // No more updating to do.
3110       return true;
3111     }
3112
3113     MElements* elements = MElements::New(alloc(), newArray);
3114     current->add(elements);
3115
3116     // Unroll the argument copy loop. We don't need to do any bounds or hole
3117     // checking here.
3118     MConstant* index = nullptr;
3119     for (uint32_t i = numFormals; i < numActuals; i++) {
3120       if (!alloc().ensureBallast()) {
3121         return false;
3122       }
3123
3124       index = MConstant::New(alloc(), Int32Value(i - numFormals));
3125       current->add(index);
3126
3127       MDefinition* arg = inlineCallInfo()->argv()[i];
3128       MStoreElement* store =
3129           MStoreElement::NewUnbarriered(alloc(), elements, index, arg,
3130                                         /* needsHoleCheck = */ false);
3131       current->add(store);
3132       current->add(MPostWriteBarrier::New(alloc(), newArray, arg));
3133     }
3134
3135     // Update the initialized length for all the (necessarily non-hole)
3136     // elements added.
3137     MSetInitializedLength* initLength =
3138         MSetInitializedLength::New(alloc(), elements, index);
3139     current->add(initLength);
3140
3141     return true;
3142   }
3143
3144   MArgumentsLength* numActuals = MArgumentsLength::New(alloc());
3145   current->add(numActuals);
3146
3147   // Pass in the number of actual arguments, the number of formals (not
3148   // including the rest parameter slot itself), and the shape.
3149   unsigned numFormals = info().nargs() - 1;
3150   MRest* rest = MRest::New(alloc(), numActuals, numFormals, shape);
3151   current->add(rest);
3152   current->push(rest);
3153   return true;
3154 }
3155
3156 bool WarpBuilder::build_Try(BytecodeLocation loc) {
3157   graph().setHasTryBlock();
3158
3159   MBasicBlock* pred = current;
3160   if (!startNewBlock(pred, loc.next())) {
3161     return false;
3162   }
3163
3164   pred->end(MGoto::New(alloc(), current));
3165   return true;
3166 }
3167
3168 bool WarpBuilder::build_Finally(BytecodeLocation loc) {
3169   MOZ_ASSERT(graph().hasTryBlock());
3170   return true;
3171 }
3172
3173 bool WarpBuilder::build_Exception(BytecodeLocation) {
3174   MOZ_CRASH("Unreachable because we skip catch-blocks");
3175 }
3176
3177 bool WarpBuilder::build_ExceptionAndStack(BytecodeLocation) {
3178   MOZ_CRASH("Unreachable because we skip catch-blocks");
3179 }
3180
3181 bool WarpBuilder::build_Throw(BytecodeLocation loc) {
3182   MDefinition* def = current->pop();
3183
3184   MThrow* ins = MThrow::New(alloc(), def);
3185   current->add(ins);
3186   if (!resumeAfter(ins, loc)) {
3187     return false;
3188   }
3189
3190   // Terminate the block.
3191   current->end(MUnreachable::New(alloc()));
3192   setTerminatedBlock();
3193   return true;
3194 }
3195
3196 bool WarpBuilder::build_ThrowWithStack(BytecodeLocation loc) {
3197   MDefinition* stack = current->pop();
3198   MDefinition* value = current->pop();
3199
3200   auto* ins = MThrowWithStack::New(alloc(), value, stack);
3201   current->add(ins);
3202   if (!resumeAfter(ins, loc)) {
3203     return false;
3204   }
3205
3206   // Terminate the block.
3207   current->end(MUnreachable::New(alloc()));
3208   setTerminatedBlock();
3209   return true;
3210 }
3211
3212 bool WarpBuilder::build_ThrowSetConst(BytecodeLocation loc) {
3213   auto* ins = MThrowRuntimeLexicalError::New(alloc(), JSMSG_BAD_CONST_ASSIGN);
3214   current->add(ins);
3215   if (!resumeAfter(ins, loc)) {
3216     return false;
3217   }
3218
3219   // Terminate the block.
3220   current->end(MUnreachable::New(alloc()));
3221   setTerminatedBlock();
3222   return true;
3223 }
3224
3225 bool WarpBuilder::build_ThrowMsg(BytecodeLocation loc) {
3226   auto* ins = MThrowMsg::New(alloc(), loc.throwMsgKind());
3227   current->add(ins);
3228   if (!resumeAfter(ins, loc)) {
3229     return false;
3230   }
3231
3232   // Terminate the block.
3233   current->end(MUnreachable::New(alloc()));
3234   setTerminatedBlock();
3235   return true;
3236 }
3237
3238 bool WarpBuilder::buildIC(BytecodeLocation loc, CacheKind kind,
3239                           std::initializer_list<MDefinition*> inputs) {
3240   MOZ_ASSERT(loc.opHasIC());
3241
3242   mozilla::DebugOnly<size_t> numInputs = inputs.size();
3243   MOZ_ASSERT(numInputs == NumInputsForCacheKind(kind));
3244
3245   if (auto* cacheIRSnapshot = getOpSnapshot<WarpCacheIR>(loc)) {
3246     return TranspileCacheIRToMIR(this, loc, cacheIRSnapshot, inputs);
3247   }
3248
3249   if (getOpSnapshot<WarpBailout>(loc)) {
3250     for (MDefinition* input : inputs) {
3251       input->setImplicitlyUsedUnchecked();
3252     }
3253     return buildBailoutForColdIC(loc, kind);
3254   }
3255
3256   if (const auto* inliningSnapshot = getOpSnapshot<WarpInlinedCall>(loc)) {
3257     // The CallInfo will be initialized by the transpiler.
3258     bool ignoresRval = BytecodeIsPopped(loc.toRawBytecode());
3259     CallInfo callInfo(alloc(), /*constructing =*/false, ignoresRval);
3260     callInfo.markAsInlined();
3261
3262     if (!TranspileCacheIRToMIR(this, loc, inliningSnapshot->cacheIRSnapshot(),
3263                                inputs, &callInfo)) {
3264       return false;
3265     }
3266     return buildInlinedCall(loc, inliningSnapshot, callInfo);
3267   }
3268
3269   // Work around std::initializer_list not defining operator[].
3270   auto getInput = [&](size_t index) -> MDefinition* {
3271     MOZ_ASSERT(index < numInputs);
3272     return inputs.begin()[index];
3273   };
3274
3275   switch (kind) {
3276     case CacheKind::UnaryArith: {
3277       MOZ_ASSERT(numInputs == 1);
3278       auto* ins = MUnaryCache::New(alloc(), getInput(0));
3279       current->add(ins);
3280       current->push(ins);
3281       return resumeAfter(ins, loc);
3282     }
3283     case CacheKind::ToPropertyKey: {
3284       MOZ_ASSERT(numInputs == 1);
3285       auto* ins = MToPropertyKeyCache::New(alloc(), getInput(0));
3286       current->add(ins);
3287       current->push(ins);
3288       return resumeAfter(ins, loc);
3289     }
3290     case CacheKind::BinaryArith: {
3291       MOZ_ASSERT(numInputs == 2);
3292       auto* ins =
3293           MBinaryCache::New(alloc(), getInput(0), getInput(1), MIRType::Value);
3294       current->add(ins);
3295       current->push(ins);
3296       return resumeAfter(ins, loc);
3297     }
3298     case CacheKind::Compare: {
3299       MOZ_ASSERT(numInputs == 2);
3300       auto* ins = MBinaryCache::New(alloc(), getInput(0), getInput(1),
3301                                     MIRType::Boolean);
3302       current->add(ins);
3303       current->push(ins);
3304       return resumeAfter(ins, loc);
3305     }
3306     case CacheKind::In: {
3307       MOZ_ASSERT(numInputs == 2);
3308       auto* ins = MInCache::New(alloc(), getInput(0), getInput(1));
3309       current->add(ins);
3310       current->push(ins);
3311       return resumeAfter(ins, loc);
3312     }
3313     case CacheKind::HasOwn: {
3314       MOZ_ASSERT(numInputs == 2);
3315       // Note: the MHasOwnCache constructor takes obj/id instead of id/obj.
3316       auto* ins = MHasOwnCache::New(alloc(), getInput(1), getInput(0));
3317       current->add(ins);
3318       current->push(ins);
3319       return resumeAfter(ins, loc);
3320     }
3321     case CacheKind::CheckPrivateField: {
3322       MOZ_ASSERT(numInputs == 2);
3323       auto* ins =
3324           MCheckPrivateFieldCache::New(alloc(), getInput(0), getInput(1));
3325       current->add(ins);
3326       current->push(ins);
3327       return resumeAfter(ins, loc);
3328     }
3329     case CacheKind::InstanceOf: {
3330       MOZ_ASSERT(numInputs == 2);
3331       auto* ins = MInstanceOfCache::New(alloc(), getInput(0), getInput(1));
3332       current->add(ins);
3333       current->push(ins);
3334       return resumeAfter(ins, loc);
3335     }
3336     case CacheKind::BindName: {
3337       MOZ_ASSERT(numInputs == 1);
3338       auto* ins = MBindNameCache::New(alloc(), getInput(0));
3339       current->add(ins);
3340       current->push(ins);
3341       return resumeAfter(ins, loc);
3342     }
3343     case CacheKind::GetIterator: {
3344       MOZ_ASSERT(numInputs == 1);
3345       auto* ins = MGetIteratorCache::New(alloc(), getInput(0));
3346       current->add(ins);
3347       current->push(ins);
3348       return resumeAfter(ins, loc);
3349     }
3350     case CacheKind::GetName: {
3351       MOZ_ASSERT(numInputs == 1);
3352       auto* ins = MGetNameCache::New(alloc(), getInput(0));
3353       current->add(ins);
3354       current->push(ins);
3355       return resumeAfter(ins, loc);
3356     }
3357     case CacheKind::GetProp: {
3358       MOZ_ASSERT(numInputs == 1);
3359       PropertyName* name = loc.getPropertyName(script_);
3360       MConstant* id = constant(StringValue(name));
3361       MDefinition* val = getInput(0);
3362       auto* ins = MGetPropertyCache::New(alloc(), val, id);
3363       current->add(ins);
3364       current->push(ins);
3365       return resumeAfter(ins, loc);
3366     }
3367     case CacheKind::GetElem: {
3368       MOZ_ASSERT(numInputs == 2);
3369       MDefinition* val = getInput(0);
3370       auto* ins = MGetPropertyCache::New(alloc(), val, getInput(1));
3371       current->add(ins);
3372       current->push(ins);
3373       return resumeAfter(ins, loc);
3374     }
3375     case CacheKind::SetProp: {
3376       MOZ_ASSERT(numInputs == 2);
3377       PropertyName* name = loc.getPropertyName(script_);
3378       MConstant* id = constant(StringValue(name));
3379       bool strict = loc.isStrictSetOp();
3380       auto* ins =
3381           MSetPropertyCache::New(alloc(), getInput(0), id, getInput(1), strict);
3382       current->add(ins);
3383       return resumeAfter(ins, loc);
3384     }
3385     case CacheKind::SetElem: {
3386       MOZ_ASSERT(numInputs == 3);
3387       bool strict = loc.isStrictSetOp();
3388       auto* ins = MSetPropertyCache::New(alloc(), getInput(0), getInput(1),
3389                                          getInput(2), strict);
3390       current->add(ins);
3391       return resumeAfter(ins, loc);
3392     }
3393     case CacheKind::GetPropSuper: {
3394       MOZ_ASSERT(numInputs == 2);
3395       PropertyName* name = loc.getPropertyName(script_);
3396       MConstant* id = constant(StringValue(name));
3397       auto* ins =
3398           MGetPropSuperCache::New(alloc(), getInput(0), getInput(1), id);
3399       current->add(ins);
3400       current->push(ins);
3401       return resumeAfter(ins, loc);
3402     }
3403     case CacheKind::GetElemSuper: {
3404       MOZ_ASSERT(numInputs == 3);
3405       // Note: CacheIR expects obj/id/receiver but MGetPropSuperCache takes
3406       // obj/receiver/id so swap the last two inputs.
3407       auto* ins = MGetPropSuperCache::New(alloc(), getInput(0), getInput(2),
3408                                           getInput(1));
3409       current->add(ins);
3410       current->push(ins);
3411       return resumeAfter(ins, loc);
3412     }
3413     case CacheKind::OptimizeSpreadCall: {
3414       MOZ_ASSERT(numInputs == 1);
3415       auto* ins = MOptimizeSpreadCallCache::New(alloc(), getInput(0));
3416       current->add(ins);
3417       current->push(ins);
3418       return resumeAfter(ins, loc);
3419     }
3420     case CacheKind::TypeOf: {
3421       // Note: Warp does not have a TypeOf IC, it just inlines the operation.
3422       MOZ_ASSERT(numInputs == 1);
3423       auto* typeOf = MTypeOf::New(alloc(), getInput(0));
3424       current->add(typeOf);
3425
3426       auto* ins = MTypeOfName::New(alloc(), typeOf);
3427       current->add(ins);
3428       current->push(ins);
3429       return true;
3430     }
3431     case CacheKind::TypeOfEq: {
3432       MOZ_ASSERT(numInputs == 1);
3433       auto operand = loc.getTypeofEqOperand();
3434       JSType type = operand.type();
3435       JSOp compareOp = operand.compareOp();
3436       auto* typeOf = MTypeOf::New(alloc(), getInput(0));
3437       current->add(typeOf);
3438
3439       auto* typeInt = MConstant::New(alloc(), Int32Value(type));
3440       current->add(typeInt);
3441
3442       auto* ins = MCompare::New(alloc(), typeOf, typeInt, compareOp,
3443                                 MCompare::Compare_Int32);
3444       current->add(ins);
3445       current->push(ins);
3446       return true;
3447     }
3448     case CacheKind::NewObject: {
3449       auto* templateConst = constant(NullValue());
3450       MNewObject* ins = MNewObject::NewVM(
3451           alloc(), templateConst, gc::Heap::Default, MNewObject::ObjectLiteral);
3452       current->add(ins);
3453       current->push(ins);
3454       return resumeAfter(ins, loc);
3455     }
3456     case CacheKind::NewArray: {
3457       uint32_t length = loc.getNewArrayLength();
3458       MConstant* templateConst = constant(NullValue());
3459       MNewArray* ins =
3460           MNewArray::NewVM(alloc(), length, templateConst, gc::Heap::Default);
3461       current->add(ins);
3462       current->push(ins);
3463       return true;
3464     }
3465     case CacheKind::CloseIter: {
3466       MOZ_ASSERT(numInputs == 1);
3467       static_assert(sizeof(CompletionKind) == sizeof(uint8_t));
3468       CompletionKind kind = loc.getCompletionKind();
3469       auto* ins = MCloseIterCache::New(alloc(), getInput(0), uint8_t(kind));
3470       current->add(ins);
3471       return resumeAfter(ins, loc);
3472     }
3473     case CacheKind::OptimizeGetIterator: {
3474       MOZ_ASSERT(numInputs == 1);
3475       auto* ins = MOptimizeGetIteratorCache::New(alloc(), getInput(0));
3476       current->add(ins);
3477       current->push(ins);
3478       return resumeAfter(ins, loc);
3479     }
3480     case CacheKind::GetIntrinsic:
3481     case CacheKind::ToBool:
3482     case CacheKind::Call:
3483       // We're currently not using an IC or transpiling CacheIR for these kinds.
3484       MOZ_CRASH("Unexpected kind");
3485   }
3486
3487   return true;
3488 }
3489
3490 bool WarpBuilder::buildBailoutForColdIC(BytecodeLocation loc, CacheKind kind) {
3491   MOZ_ASSERT(loc.opHasIC());
3492
3493   MBail* bail = MBail::New(alloc(), BailoutKind::FirstExecution);
3494   current->add(bail);
3495   current->setAlwaysBails();
3496
3497   MIRType resultType;
3498   switch (kind) {
3499     case CacheKind::UnaryArith:
3500     case CacheKind::BinaryArith:
3501     case CacheKind::GetName:
3502     case CacheKind::GetProp:
3503     case CacheKind::GetElem:
3504     case CacheKind::GetPropSuper:
3505     case CacheKind::GetElemSuper:
3506     case CacheKind::GetIntrinsic:
3507     case CacheKind::Call:
3508     case CacheKind::ToPropertyKey:
3509     case CacheKind::OptimizeSpreadCall:
3510       resultType = MIRType::Value;
3511       break;
3512     case CacheKind::BindName:
3513     case CacheKind::GetIterator:
3514     case CacheKind::NewArray:
3515     case CacheKind::NewObject:
3516       resultType = MIRType::Object;
3517       break;
3518     case CacheKind::TypeOf:
3519       resultType = MIRType::String;
3520       break;
3521     case CacheKind::ToBool:
3522     case CacheKind::Compare:
3523     case CacheKind::In:
3524     case CacheKind::HasOwn:
3525     case CacheKind::CheckPrivateField:
3526     case CacheKind::InstanceOf:
3527     case CacheKind::OptimizeGetIterator:
3528     case CacheKind::TypeOfEq:
3529       resultType = MIRType::Boolean;
3530       break;
3531     case CacheKind::SetProp:
3532     case CacheKind::SetElem:
3533     case CacheKind::CloseIter:
3534       return true;  // No result.
3535   }
3536
3537   auto* ins = MUnreachableResult::New(alloc(), resultType);
3538   current->add(ins);
3539   current->push(ins);
3540
3541   return true;
3542 }
3543
3544 class MOZ_RAII AutoAccumulateReturns {
3545   MIRGraph& graph_;
3546   MIRGraphReturns* prev_;
3547
3548  public:
3549   AutoAccumulateReturns(MIRGraph& graph, MIRGraphReturns& returns)
3550       : graph_(graph) {
3551     prev_ = graph_.returnAccumulator();
3552     graph_.setReturnAccumulator(&returns);
3553   }
3554   ~AutoAccumulateReturns() { graph_.setReturnAccumulator(prev_); }
3555 };
3556
3557 bool WarpBuilder::buildInlinedCall(BytecodeLocation loc,
3558                                    const WarpInlinedCall* inlineSnapshot,
3559                                    CallInfo& callInfo) {
3560   jsbytecode* pc = loc.toRawBytecode();
3561
3562   if (callInfo.isSetter()) {
3563     // build_SetProp pushes the rhs argument onto the stack. Remove it
3564     // in preparation for pushCallStack.
3565     current->pop();
3566   }
3567
3568   callInfo.setImplicitlyUsedUnchecked();
3569
3570   // Capture formals in the outer resume point.
3571   if (!callInfo.pushCallStack(current)) {
3572     return false;
3573   }
3574   MResumePoint* outerResumePoint =
3575       MResumePoint::New(alloc(), current, pc, callInfo.inliningResumeMode());
3576   if (!outerResumePoint) {
3577     return false;
3578   }
3579   current->setOuterResumePoint(outerResumePoint);
3580
3581   // Pop formals again, except leave |callee| on stack for duration of call.
3582   callInfo.popCallStack(current);
3583   current->push(callInfo.callee());
3584
3585   // Build the graph.
3586   CompileInfo* calleeCompileInfo = inlineSnapshot->info();
3587   MIRGraphReturns returns(alloc());
3588   AutoAccumulateReturns aar(graph(), returns);
3589   WarpBuilder inlineBuilder(this, inlineSnapshot->scriptSnapshot(),
3590                             *calleeCompileInfo, &callInfo, outerResumePoint);
3591   if (!inlineBuilder.buildInline()) {
3592     // Note: Inlining only aborts on OOM.  If inlining would fail for
3593     // any other reason, we detect it in advance and don't inline.
3594     return false;
3595   }
3596
3597   // We mark scripts as uninlineable in BytecodeAnalysis if we cannot
3598   // reach a return statement (without going through a catch/finally).
3599   MOZ_ASSERT(!returns.empty());
3600
3601   // Create return block
3602   BytecodeLocation postCall = loc.next();
3603   MBasicBlock* prev = current;
3604   if (!startNewEntryBlock(prev->stackDepth(), postCall)) {
3605     return false;
3606   }
3607   // Restore previous value of callerResumePoint.
3608   current->setCallerResumePoint(callerResumePoint());
3609   current->inheritSlots(prev);
3610
3611   // Pop |callee|.
3612   current->pop();
3613
3614   // Accumulate return values.
3615   MDefinition* returnValue =
3616       patchInlinedReturns(calleeCompileInfo, callInfo, returns, current);
3617   if (!returnValue) {
3618     return false;
3619   }
3620   current->push(returnValue);
3621
3622   // Initialize entry slots
3623   if (!current->initEntrySlots(alloc())) {
3624     return false;
3625   }
3626
3627   return true;
3628 }
3629
3630 MDefinition* WarpBuilder::patchInlinedReturns(CompileInfo* calleeCompileInfo,
3631                                               CallInfo& callInfo,
3632                                               MIRGraphReturns& exits,
3633                                               MBasicBlock* returnBlock) {
3634   if (exits.length() == 1) {
3635     return patchInlinedReturn(calleeCompileInfo, callInfo, exits[0],
3636                               returnBlock);
3637   }
3638
3639   // Accumulate multiple returns with a phi.
3640   MPhi* phi = MPhi::New(alloc());
3641   if (!phi->reserveLength(exits.length())) {
3642     return nullptr;
3643   }
3644
3645   for (auto* exit : exits) {
3646     MDefinition* rdef =
3647         patchInlinedReturn(calleeCompileInfo, callInfo, exit, returnBlock);
3648     if (!rdef) {
3649       return nullptr;
3650     }
3651     phi->addInput(rdef);
3652   }
3653   returnBlock->addPhi(phi);
3654   return phi;
3655 }
3656
3657 MDefinition* WarpBuilder::patchInlinedReturn(CompileInfo* calleeCompileInfo,
3658                                              CallInfo& callInfo,
3659                                              MBasicBlock* exit,
3660                                              MBasicBlock* returnBlock) {
3661   // Replace the MReturn in the exit block with an MGoto branching to
3662   // the return block.
3663   MDefinition* rdef = exit->lastIns()->toReturn()->input();
3664   exit->discardLastIns();
3665
3666   // Constructors must be patched by the caller to always return an object.
3667   // Derived class constructors contain extra bytecode to ensure an object
3668   // is always returned, so no additional patching is needed.
3669   if (callInfo.constructing() &&
3670       !calleeCompileInfo->isDerivedClassConstructor()) {
3671     auto* filter = MReturnFromCtor::New(alloc(), rdef, callInfo.thisArg());
3672     exit->add(filter);
3673     rdef = filter;
3674   } else if (callInfo.isSetter()) {
3675     // Setters return the rhs argument, not whatever value is returned.
3676     rdef = callInfo.getArg(0);
3677   }
3678
3679   exit->end(MGoto::New(alloc(), returnBlock));
3680   if (!returnBlock->addPredecessorWithoutPhis(exit)) {
3681     return nullptr;
3682   }
3683
3684   return rdef;
3685 }