js/src/jit/WarpOracle.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "jit/WarpOracle.h"
   8
   9 #include "mozilla/ScopeExit.h"
  10 #include "mozilla/Try.h"
  11
  12 #include <algorithm>
  13
  14 #include "jit/CacheIR.h"
  15 #include "jit/CacheIRCompiler.h"
  16 #include "jit/CacheIRReader.h"
  17 #include "jit/CompileInfo.h"
  18 #include "jit/InlineScriptTree.h"
  19 #include "jit/JitHints.h"
  20 #include "jit/JitRuntime.h"
  21 #include "jit/JitScript.h"
  22 #include "jit/JitSpewer.h"
  23 #include "jit/JitZone.h"
  24 #include "jit/MIRGenerator.h"
  25 #include "jit/TrialInlining.h"
  26 #include "jit/TypeData.h"
  27 #include "jit/WarpBuilder.h"
  28 #include "js/ColumnNumber.h"  // JS::LimitedColumnNumberOneOrigin
  29 #include "util/DifferentialTesting.h"
  30 #include "vm/BuiltinObjectKind.h"
  31 #include "vm/BytecodeIterator.h"
  32 #include "vm/BytecodeLocation.h"
  33
  34 #include "jit/InlineScriptTree-inl.h"
  35 #include "vm/BytecodeIterator-inl.h"
  36 #include "vm/BytecodeLocation-inl.h"
  37 #include "vm/EnvironmentObject-inl.h"
  38 #include "vm/Interpreter-inl.h"
  39
  40 using namespace js;
  41 using namespace js::jit;
  42
  43 using mozilla::Maybe;
  44
  45 // WarpScriptOracle creates a WarpScriptSnapshot for a single JSScript. Note
  46 // that a single WarpOracle can use multiple WarpScriptOracles when scripts are
  47 // inlined.
  48 class MOZ_STACK_CLASS WarpScriptOracle {
  49   JSContext* cx_;
  50   WarpOracle* oracle_;
  51   MIRGenerator& mirGen_;
  52   TempAllocator& alloc_;
  53   HandleScript script_;
  54   const CompileInfo* info_;
  55   ICScript* icScript_;
  56
  57   // Index of the next ICEntry for getICEntry. This assumes the script's
  58   // bytecode is processed from first to last instruction.
  59   uint32_t icEntryIndex_ = 0;
  60
  61   template <typename... Args>
  62   mozilla::GenericErrorResult<AbortReason> abort(Args&&... args) {
  63     return oracle_->abort(script_, args...);
  64   }
  65
  66   WarpEnvironment createEnvironment();
  67   AbortReasonOr<Ok> maybeInlineIC(WarpOpSnapshotList& snapshots,
  68                                   BytecodeLocation loc);
  69   AbortReasonOr<bool> maybeInlineCall(WarpOpSnapshotList& snapshots,
  70                                       BytecodeLocation loc, ICCacheIRStub* stub,
  71                                       ICFallbackStub* fallbackStub,
  72                                       uint8_t* stubDataCopy);
  73   AbortReasonOr<bool> maybeInlinePolymorphicTypes(WarpOpSnapshotList& snapshots,
  74                                                   BytecodeLocation loc,
  75                                                   ICCacheIRStub* firstStub,
  76                                                   ICFallbackStub* fallbackStub);
  77   [[nodiscard]] bool replaceNurseryAndAllocSitePointers(
  78       ICCacheIRStub* stub, const CacheIRStubInfo* stubInfo,
  79       uint8_t* stubDataCopy);
  80   bool maybeReplaceNurseryPointer(const CacheIRStubInfo* stubInfo,
  81                                   uint8_t* stubDataCopy, JSObject* obj,
  82                                   size_t offset);
  83
  84  public:
  85   WarpScriptOracle(JSContext* cx, WarpOracle* oracle, HandleScript script,
  86                    const CompileInfo* info, ICScript* icScript)
  87       : cx_(cx),
  88         oracle_(oracle),
  89         mirGen_(oracle->mirGen()),
  90         alloc_(mirGen_.alloc()),
  91         script_(script),
  92         info_(info),
  93         icScript_(icScript) {}
  94
  95   AbortReasonOr<WarpScriptSnapshot*> createScriptSnapshot();
  96
  97   ICEntry& getICEntryAndFallback(BytecodeLocation loc,
  98                                  ICFallbackStub** fallback);
  99 };
 100
 101 WarpOracle::WarpOracle(JSContext* cx, MIRGenerator& mirGen,
 102                        HandleScript outerScript)
 103     : cx_(cx),
 104       mirGen_(mirGen),
 105       alloc_(mirGen.alloc()),
 106       outerScript_(outerScript) {}
 107
 108 mozilla::GenericErrorResult<AbortReason> WarpOracle::abort(HandleScript script,
 109                                                            AbortReason r) {
 110   auto res = mirGen_.abort(r);
 111   JitSpew(JitSpew_IonAbort, "aborted @ %s", script->filename());
 112   return res;
 113 }
 114
 115 mozilla::GenericErrorResult<AbortReason> WarpOracle::abort(HandleScript script,
 116                                                            AbortReason r,
 117                                                            const char* message,
 118                                                            ...) {
 119   va_list ap;
 120   va_start(ap, message);
 121   auto res = mirGen_.abortFmt(r, message, ap);
 122   va_end(ap);
 123   JitSpew(JitSpew_IonAbort, "aborted @ %s", script->filename());
 124   return res;
 125 }
 126
 127 void WarpOracle::addScriptSnapshot(WarpScriptSnapshot* scriptSnapshot,
 128                                    ICScript* icScript, size_t bytecodeLength) {
 129   scriptSnapshots_.insertBack(scriptSnapshot);
 130   accumulatedBytecodeSize_ += bytecodeLength;
 131 #ifdef DEBUG
 132   runningScriptHash_ = mozilla::AddToHash(runningScriptHash_, icScript->hash());
 133 #endif
 134 }
 135
 136 AbortReasonOr<WarpSnapshot*> WarpOracle::createSnapshot() {
 137 #ifdef JS_JITSPEW
 138   const char* mode;
 139   if (outerScript_->hasIonScript()) {
 140     mode = "Recompiling";
 141   } else {
 142     mode = "Compiling";
 143   }
 144   JitSpew(JitSpew_IonScripts,
 145           "Warp %s script %s:%u:%u (%p) (warmup-counter=%" PRIu32 ",%s%s)",
 146           mode, outerScript_->filename(), outerScript_->lineno(),
 147           outerScript_->column().oneOriginValue(),
 148           static_cast<JSScript*>(outerScript_), outerScript_->getWarmUpCount(),
 149           outerScript_->isGenerator() ? " isGenerator" : "",
 150           outerScript_->isAsync() ? " isAsync" : "");
 151 #endif
 152
 153   accumulatedBytecodeSize_ = outerScript_->length();
 154
 155   MOZ_ASSERT(outerScript_->hasJitScript());
 156   ICScript* icScript = outerScript_->jitScript()->icScript();
 157   WarpScriptOracle scriptOracle(cx_, this, outerScript_, &mirGen_.outerInfo(),
 158                                 icScript);
 159
 160   WarpScriptSnapshot* scriptSnapshot;
 161   MOZ_TRY_VAR(scriptSnapshot, scriptOracle.createScriptSnapshot());
 162
 163   // Insert the outermost scriptSnapshot at the front of the list.
 164   scriptSnapshots_.insertFront(scriptSnapshot);
 165
 166   bool recordFinalWarmUpCount = false;
 167 #ifdef JS_CACHEIR_SPEW
 168   recordFinalWarmUpCount = outerScript_->needsFinalWarmUpCount();
 169 #endif
 170
 171   auto* snapshot = new (alloc_.fallible())
 172       WarpSnapshot(cx_, alloc_, std::move(scriptSnapshots_), bailoutInfo_,
 173                    recordFinalWarmUpCount);
 174   if (!snapshot) {
 175     return abort(outerScript_, AbortReason::Alloc);
 176   }
 177
 178   if (!snapshot->nurseryObjects().appendAll(nurseryObjects_)) {
 179     return abort(outerScript_, AbortReason::Alloc);
 180   }
 181
 182 #ifdef JS_JITSPEW
 183   if (JitSpewEnabled(JitSpew_WarpSnapshots)) {
 184     Fprinter& out = JitSpewPrinter();
 185     snapshot->dump(out);
 186   }
 187 #endif
 188
 189 #ifdef DEBUG
 190   // When transpiled CacheIR bails out, we do not want to recompile
 191   // with the exact same data and get caught in an invalidation loop.
 192   //
 193   // To avoid this, we store a hash of the stub pointers and entry
 194   // counts in this snapshot, save that hash in the JitScript if we
 195   // have a TranspiledCacheIR or MonomorphicInlinedStubFolding bailout,
 196   // and assert that the hash has changed when we recompile.
 197   //
 198   // Note: this assertion catches potential performance issues.
 199   // Failing this assertion is not a correctness/security problem.
 200   // We therefore ignore cases involving resource exhaustion (OOM,
 201   // stack overflow, etc), or stubs purged by GC.
 202   HashNumber hash = mozilla::AddToHash(icScript->hash(), runningScriptHash_);
 203   if (outerScript_->jitScript()->hasFailedICHash()) {
 204     HashNumber oldHash = outerScript_->jitScript()->getFailedICHash();
 205     MOZ_ASSERT_IF(hash == oldHash && !js::SupportDifferentialTesting(),
 206                   cx_->hadResourceExhaustion());
 207   }
 208   snapshot->setICHash(hash);
 209 #endif
 210
 211   return snapshot;
 212 }
 213
 214 template <typename T, typename... Args>
 215 [[nodiscard]] static bool AddOpSnapshot(TempAllocator& alloc,
 216                                         WarpOpSnapshotList& snapshots,
 217                                         uint32_t offset, Args&&... args) {
 218   T* snapshot = new (alloc.fallible()) T(offset, std::forward<Args>(args)...);
 219   if (!snapshot) {
 220     return false;
 221   }
 222
 223   snapshots.insertBack(snapshot);
 224   return true;
 225 }
 226
 227 [[nodiscard]] static bool AddWarpGetImport(TempAllocator& alloc,
 228                                            WarpOpSnapshotList& snapshots,
 229                                            uint32_t offset, JSScript* script,
 230                                            PropertyName* name) {
 231   ModuleEnvironmentObject* env = GetModuleEnvironmentForScript(script);
 232   MOZ_ASSERT(env);
 233
 234   mozilla::Maybe<PropertyInfo> prop;
 235   ModuleEnvironmentObject* targetEnv;
 236   MOZ_ALWAYS_TRUE(env->lookupImport(NameToId(name), &targetEnv, &prop));
 237
 238   uint32_t numFixedSlots = targetEnv->numFixedSlots();
 239   uint32_t slot = prop->slot();
 240
 241   // In the rare case where this import hasn't been initialized already (we have
 242   // an import cycle where modules reference each other's imports), we need a
 243   // check.
 244   bool needsLexicalCheck =
 245       targetEnv->getSlot(slot).isMagic(JS_UNINITIALIZED_LEXICAL);
 246
 247   return AddOpSnapshot<WarpGetImport>(alloc, snapshots, offset, targetEnv,
 248                                       numFixedSlots, slot, needsLexicalCheck);
 249 }
 250
 251 ICEntry& WarpScriptOracle::getICEntryAndFallback(BytecodeLocation loc,
 252                                                  ICFallbackStub** fallback) {
 253   const uint32_t offset = loc.bytecodeToOffset(script_);
 254
 255   do {
 256     *fallback = icScript_->fallbackStub(icEntryIndex_);
 257     icEntryIndex_++;
 258   } while ((*fallback)->pcOffset() < offset);
 259
 260   MOZ_ASSERT((*fallback)->pcOffset() == offset);
 261   return icScript_->icEntry(icEntryIndex_ - 1);
 262 }
 263
 264 WarpEnvironment WarpScriptOracle::createEnvironment() {
 265   // Don't do anything if the script doesn't use the environment chain.
 266   if (!script_->jitScript()->usesEnvironmentChain()) {
 267     return WarpEnvironment(NoEnvironment());
 268   }
 269
 270   if (script_->isModule()) {
 271     ModuleObject* module = script_->module();
 272     JSObject* obj = &module->initialEnvironment();
 273     return WarpEnvironment(ConstantObjectEnvironment(obj));
 274   }
 275
 276   JSFunction* fun = script_->function();
 277   if (!fun) {
 278     // For global scripts without a non-syntactic global scope, the environment
 279     // chain is the global lexical environment.
 280     MOZ_ASSERT(!script_->isForEval());
 281     MOZ_ASSERT(!script_->hasNonSyntacticScope());
 282     JSObject* obj = &script_->global().lexicalEnvironment();
 283     return WarpEnvironment(ConstantObjectEnvironment(obj));
 284   }
 285
 286   JSObject* templateEnv = script_->jitScript()->templateEnvironment();
 287
 288   CallObject* callObjectTemplate = nullptr;
 289   if (fun->needsCallObject()) {
 290     callObjectTemplate = &templateEnv->as<CallObject>();
 291   }
 292
 293   NamedLambdaObject* namedLambdaTemplate = nullptr;
 294   if (fun->needsNamedLambdaEnvironment()) {
 295     if (callObjectTemplate) {
 296       templateEnv = templateEnv->enclosingEnvironment();
 297     }
 298     namedLambdaTemplate = &templateEnv->as<NamedLambdaObject>();
 299   }
 300
 301   return WarpEnvironment(
 302       FunctionEnvironment(callObjectTemplate, namedLambdaTemplate));
 303 }
 304
 305 AbortReasonOr<WarpScriptSnapshot*> WarpScriptOracle::createScriptSnapshot() {
 306   MOZ_ASSERT(script_->hasJitScript());
 307
 308   if (!script_->jitScript()->ensureHasCachedIonData(cx_, script_)) {
 309     return abort(AbortReason::Error);
 310   }
 311
 312   if (script_->failedBoundsCheck()) {
 313     oracle_->bailoutInfo().setFailedBoundsCheck();
 314   }
 315   if (script_->failedLexicalCheck()) {
 316     oracle_->bailoutInfo().setFailedLexicalCheck();
 317   }
 318
 319   WarpEnvironment environment = createEnvironment();
 320
 321   // Unfortunately LinkedList<> asserts the list is empty in its destructor.
 322   // Clear the list if we abort compilation.
 323   WarpOpSnapshotList opSnapshots;
 324   auto autoClearOpSnapshots =
 325       mozilla::MakeScopeExit([&] { opSnapshots.clear(); });
 326
 327   ModuleObject* moduleObject = nullptr;
 328
 329   // Analyze the bytecode. Abort compilation for unsupported ops and create
 330   // WarpOpSnapshots.
 331   for (BytecodeLocation loc : AllBytecodesIterable(script_)) {
 332     JSOp op = loc.getOp();
 333     uint32_t offset = loc.bytecodeToOffset(script_);
 334     switch (op) {
 335       case JSOp::Arguments: {
 336         MOZ_ASSERT(script_->needsArgsObj());
 337         bool mapped = script_->hasMappedArgsObj();
 338         ArgumentsObject* templateObj =
 339             script_->global().maybeArgumentsTemplateObject(mapped);
 340         if (!AddOpSnapshot<WarpArguments>(alloc_, opSnapshots, offset,
 341                                           templateObj)) {
 342           return abort(AbortReason::Alloc);
 343         }
 344         break;
 345       }
 346       case JSOp::RegExp: {
 347         bool hasShared = loc.getRegExp(script_)->hasShared();
 348         if (!AddOpSnapshot<WarpRegExp>(alloc_, opSnapshots, offset,
 349                                        hasShared)) {
 350           return abort(AbortReason::Alloc);
 351         }
 352         break;
 353       }
 354
 355       case JSOp::FunctionThis:
 356         if (!script_->strict() && script_->hasNonSyntacticScope()) {
 357           // Abort because MBoxNonStrictThis doesn't support non-syntactic
 358           // scopes (a deprecated SpiderMonkey mechanism). If this becomes an
 359           // issue we could support it by refactoring GetFunctionThis to not
 360           // take a frame pointer and then call that.
 361           return abort(AbortReason::Disable,
 362                        "JSOp::FunctionThis with non-syntactic scope");
 363         }
 364         break;
 365
 366       case JSOp::GlobalThis:
 367         MOZ_ASSERT(!script_->hasNonSyntacticScope());
 368         break;
 369
 370       case JSOp::BuiltinObject: {
 371         // If we already resolved this built-in we can bake it in.
 372         auto kind = loc.getBuiltinObjectKind();
 373         if (JSObject* proto = MaybeGetBuiltinObject(cx_->global(), kind)) {
 374           if (!AddOpSnapshot<WarpBuiltinObject>(alloc_, opSnapshots, offset,
 375                                                 proto)) {
 376             return abort(AbortReason::Alloc);
 377           }
 378         }
 379         break;
 380       }
 381
 382       case JSOp::GetIntrinsic: {
 383         // If we already cloned this intrinsic we can bake it in.
 384         // NOTE: When the initializer runs in a content global, we also have to
 385         //       worry about nursery objects. These quickly tenure and stay that
 386         //       way so this is only a temporary problem.
 387         PropertyName* name = loc.getPropertyName(script_);
 388         Value val;
 389         if (cx_->global()->maybeGetIntrinsicValue(name, &val, cx_) &&
 390             JS::GCPolicy<Value>::isTenured(val)) {
 391           if (!AddOpSnapshot<WarpGetIntrinsic>(alloc_, opSnapshots, offset,
 392                                                val)) {
 393             return abort(AbortReason::Alloc);
 394           }
 395         }
 396         break;
 397       }
 398
 399       case JSOp::ImportMeta: {
 400         if (!moduleObject) {
 401           moduleObject = GetModuleObjectForScript(script_);
 402           MOZ_ASSERT(moduleObject->isTenured());
 403         }
 404         break;
 405       }
 406
 407       case JSOp::GetImport: {
 408         PropertyName* name = loc.getPropertyName(script_);
 409         if (!AddWarpGetImport(alloc_, opSnapshots, offset, script_, name)) {
 410           return abort(AbortReason::Alloc);
 411         }
 412         break;
 413       }
 414
 415       case JSOp::Lambda: {
 416         JSFunction* fun = loc.getFunction(script_);
 417         if (IsAsmJSModule(fun)) {
 418           return abort(AbortReason::Disable, "asm.js module function lambda");
 419         }
 420         break;
 421       }
 422
 423       case JSOp::GetElemSuper: {
 424 #if defined(JS_CODEGEN_X86)
 425         // x86 does not have enough registers.
 426         return abort(AbortReason::Disable,
 427                      "GetElemSuper is not supported on x86");
 428 #else
 429         MOZ_TRY(maybeInlineIC(opSnapshots, loc));
 430         break;
 431 #endif
 432       }
 433
 434       case JSOp::Rest: {
 435         if (Shape* shape =
 436                 script_->global().maybeArrayShapeWithDefaultProto()) {
 437           if (!AddOpSnapshot<WarpRest>(alloc_, opSnapshots, offset, shape)) {
 438             return abort(AbortReason::Alloc);
 439           }
 440         }
 441         break;
 442       }
 443
 444       case JSOp::BindGName: {
 445         Rooted<GlobalObject*> global(cx_, &script_->global());
 446         Rooted<PropertyName*> name(cx_, loc.getPropertyName(script_));
 447         if (JSObject* env = MaybeOptimizeBindGlobalName(cx_, global, name)) {
 448           MOZ_ASSERT(env->isTenured());
 449           if (!AddOpSnapshot<WarpBindGName>(alloc_, opSnapshots, offset, env)) {
 450             return abort(AbortReason::Alloc);
 451           }
 452         } else {
 453           MOZ_TRY(maybeInlineIC(opSnapshots, loc));
 454         }
 455         break;
 456       }
 457
 458       case JSOp::PushVarEnv: {
 459         Rooted<VarScope*> scope(cx_, &loc.getScope(script_)->as<VarScope>());
 460
 461         auto* templateObj =
 462             VarEnvironmentObject::createTemplateObject(cx_, scope);
 463         if (!templateObj) {
 464           return abort(AbortReason::Alloc);
 465         }
 466         MOZ_ASSERT(templateObj->isTenured());
 467
 468         if (!AddOpSnapshot<WarpVarEnvironment>(alloc_, opSnapshots, offset,
 469                                                templateObj)) {
 470           return abort(AbortReason::Alloc);
 471         }
 472         break;
 473       }
 474
 475       case JSOp::PushLexicalEnv:
 476       case JSOp::FreshenLexicalEnv:
 477       case JSOp::RecreateLexicalEnv: {
 478         Rooted<LexicalScope*> scope(cx_,
 479                                     &loc.getScope(script_)->as<LexicalScope>());
 480
 481         auto* templateObj =
 482             BlockLexicalEnvironmentObject::createTemplateObject(cx_, scope);
 483         if (!templateObj) {
 484           return abort(AbortReason::Alloc);
 485         }
 486         MOZ_ASSERT(templateObj->isTenured());
 487
 488         if (!AddOpSnapshot<WarpLexicalEnvironment>(alloc_, opSnapshots, offset,
 489                                                    templateObj)) {
 490           return abort(AbortReason::Alloc);
 491         }
 492         break;
 493       }
 494
 495       case JSOp::PushClassBodyEnv: {
 496         Rooted<ClassBodyScope*> scope(
 497             cx_, &loc.getScope(script_)->as<ClassBodyScope>());
 498
 499         auto* templateObj =
 500             ClassBodyLexicalEnvironmentObject::createTemplateObject(cx_, scope);
 501         if (!templateObj) {
 502           return abort(AbortReason::Alloc);
 503         }
 504         MOZ_ASSERT(templateObj->isTenured());
 505
 506         if (!AddOpSnapshot<WarpClassBodyEnvironment>(alloc_, opSnapshots,
 507                                                      offset, templateObj)) {
 508           return abort(AbortReason::Alloc);
 509         }
 510         break;
 511       }
 512
 513       case JSOp::GetName:
 514       case JSOp::GetGName:
 515       case JSOp::GetProp:
 516       case JSOp::GetElem:
 517       case JSOp::SetProp:
 518       case JSOp::StrictSetProp:
 519       case JSOp::Call:
 520       case JSOp::CallContent:
 521       case JSOp::CallIgnoresRv:
 522       case JSOp::CallIter:
 523       case JSOp::CallContentIter:
 524       case JSOp::New:
 525       case JSOp::NewContent:
 526       case JSOp::SuperCall:
 527       case JSOp::SpreadCall:
 528       case JSOp::SpreadNew:
 529       case JSOp::SpreadSuperCall:
 530       case JSOp::ToNumeric:
 531       case JSOp::Pos:
 532       case JSOp::Inc:
 533       case JSOp::Dec:
 534       case JSOp::Neg:
 535       case JSOp::BitNot:
 536       case JSOp::Iter:
 537       case JSOp::Eq:
 538       case JSOp::Ne:
 539       case JSOp::Lt:
 540       case JSOp::Le:
 541       case JSOp::Gt:
 542       case JSOp::Ge:
 543       case JSOp::StrictEq:
 544       case JSOp::StrictNe:
 545       case JSOp::BindName:
 546       case JSOp::Add:
 547       case JSOp::Sub:
 548       case JSOp::Mul:
 549       case JSOp::Div:
 550       case JSOp::Mod:
 551       case JSOp::Pow:
 552       case JSOp::BitAnd:
 553       case JSOp::BitOr:
 554       case JSOp::BitXor:
 555       case JSOp::Lsh:
 556       case JSOp::Rsh:
 557       case JSOp::Ursh:
 558       case JSOp::In:
 559       case JSOp::HasOwn:
 560       case JSOp::CheckPrivateField:
 561       case JSOp::Instanceof:
 562       case JSOp::GetPropSuper:
 563       case JSOp::InitProp:
 564       case JSOp::InitLockedProp:
 565       case JSOp::InitHiddenProp:
 566       case JSOp::InitElem:
 567       case JSOp::InitHiddenElem:
 568       case JSOp::InitLockedElem:
 569       case JSOp::InitElemInc:
 570       case JSOp::SetName:
 571       case JSOp::StrictSetName:
 572       case JSOp::SetGName:
 573       case JSOp::StrictSetGName:
 574       case JSOp::InitGLexical:
 575       case JSOp::SetElem:
 576       case JSOp::StrictSetElem:
 577       case JSOp::ToPropertyKey:
 578       case JSOp::OptimizeSpreadCall:
 579       case JSOp::Typeof:
 580       case JSOp::TypeofExpr:
 581       case JSOp::NewObject:
 582       case JSOp::NewInit:
 583       case JSOp::NewArray:
 584       case JSOp::JumpIfFalse:
 585       case JSOp::JumpIfTrue:
 586       case JSOp::And:
 587       case JSOp::Or:
 588       case JSOp::Not:
 589       case JSOp::CloseIter:
 590       case JSOp::OptimizeGetIterator:
 591         MOZ_TRY(maybeInlineIC(opSnapshots, loc));
 592         break;
 593
 594       case JSOp::Nop:
 595       case JSOp::NopDestructuring:
 596       case JSOp::NopIsAssignOp:
 597       case JSOp::TryDestructuring:
 598       case JSOp::Lineno:
 599       case JSOp::DebugLeaveLexicalEnv:
 600       case JSOp::Undefined:
 601       case JSOp::Void:
 602       case JSOp::Null:
 603       case JSOp::Hole:
 604       case JSOp::Uninitialized:
 605       case JSOp::IsConstructing:
 606       case JSOp::False:
 607       case JSOp::True:
 608       case JSOp::Zero:
 609       case JSOp::One:
 610       case JSOp::Int8:
 611       case JSOp::Uint16:
 612       case JSOp::Uint24:
 613       case JSOp::Int32:
 614       case JSOp::Double:
 615       case JSOp::BigInt:
 616       case JSOp::String:
 617       case JSOp::Symbol:
 618       case JSOp::Pop:
 619       case JSOp::PopN:
 620       case JSOp::Dup:
 621       case JSOp::Dup2:
 622       case JSOp::DupAt:
 623       case JSOp::Swap:
 624       case JSOp::Pick:
 625       case JSOp::Unpick:
 626       case JSOp::GetLocal:
 627       case JSOp::SetLocal:
 628       case JSOp::InitLexical:
 629       case JSOp::GetArg:
 630       case JSOp::GetFrameArg:
 631       case JSOp::SetArg:
 632       case JSOp::ArgumentsLength:
 633       case JSOp::GetActualArg:
 634       case JSOp::JumpTarget:
 635       case JSOp::LoopHead:
 636       case JSOp::Case:
 637       case JSOp::Default:
 638       case JSOp::Coalesce:
 639       case JSOp::Goto:
 640       case JSOp::DebugCheckSelfHosted:
 641       case JSOp::DynamicImport:
 642       case JSOp::ToString:
 643       case JSOp::GlobalOrEvalDeclInstantiation:
 644       case JSOp::BindVar:
 645       case JSOp::MutateProto:
 646       case JSOp::Callee:
 647       case JSOp::ToAsyncIter:
 648       case JSOp::ObjWithProto:
 649       case JSOp::GetAliasedVar:
 650       case JSOp::SetAliasedVar:
 651       case JSOp::InitAliasedLexical:
 652       case JSOp::EnvCallee:
 653       case JSOp::MoreIter:
 654       case JSOp::EndIter:
 655       case JSOp::IsNoIter:
 656       case JSOp::IsNullOrUndefined:
 657       case JSOp::DelProp:
 658       case JSOp::StrictDelProp:
 659       case JSOp::DelElem:
 660       case JSOp::StrictDelElem:
 661       case JSOp::SetFunName:
 662       case JSOp::PopLexicalEnv:
 663       case JSOp::ImplicitThis:
 664       case JSOp::CheckClassHeritage:
 665       case JSOp::CheckThis:
 666       case JSOp::CheckThisReinit:
 667       case JSOp::Generator:
 668       case JSOp::AfterYield:
 669       case JSOp::FinalYieldRval:
 670       case JSOp::AsyncResolve:
 671       case JSOp::AsyncReject:
 672       case JSOp::CheckResumeKind:
 673       case JSOp::CanSkipAwait:
 674       case JSOp::MaybeExtractAwaitValue:
 675       case JSOp::AsyncAwait:
 676       case JSOp::Await:
 677       case JSOp::CheckReturn:
 678       case JSOp::CheckLexical:
 679       case JSOp::CheckAliasedLexical:
 680       case JSOp::InitHomeObject:
 681       case JSOp::SuperBase:
 682       case JSOp::SuperFun:
 683       case JSOp::InitElemArray:
 684       case JSOp::InitPropGetter:
 685       case JSOp::InitPropSetter:
 686       case JSOp::InitHiddenPropGetter:
 687       case JSOp::InitHiddenPropSetter:
 688       case JSOp::InitElemGetter:
 689       case JSOp::InitElemSetter:
 690       case JSOp::InitHiddenElemGetter:
 691       case JSOp::InitHiddenElemSetter:
 692       case JSOp::NewTarget:
 693       case JSOp::Object:
 694       case JSOp::CallSiteObj:
 695       case JSOp::CheckIsObj:
 696       case JSOp::CheckObjCoercible:
 697       case JSOp::FunWithProto:
 698       case JSOp::Debugger:
 699       case JSOp::TableSwitch:
 700       case JSOp::Exception:
 701       case JSOp::ExceptionAndStack:
 702       case JSOp::Throw:
 703       case JSOp::ThrowWithStack:
 704       case JSOp::ThrowSetConst:
 705       case JSOp::SetRval:
 706       case JSOp::GetRval:
 707       case JSOp::Return:
 708       case JSOp::RetRval:
 709       case JSOp::InitialYield:
 710       case JSOp::Yield:
 711       case JSOp::ResumeKind:
 712       case JSOp::ThrowMsg:
 713       case JSOp::Try:
 714       case JSOp::Finally:
 715       case JSOp::NewPrivateName:
 716         // Supported by WarpBuilder. Nothing to do.
 717         break;
 718
 719         // Unsupported ops. Don't use a 'default' here, we want to trigger a
 720         // compiler warning when adding a new JSOp.
 721 #define DEF_CASE(OP) case JSOp::OP:
 722         WARP_UNSUPPORTED_OPCODE_LIST(DEF_CASE)
 723 #undef DEF_CASE
 724 #ifdef DEBUG
 725         return abort(AbortReason::Disable, "Unsupported opcode: %s",
 726                      CodeName(op));
 727 #else
 728         return abort(AbortReason::Disable, "Unsupported opcode: %u",
 729                      uint8_t(op));
 730 #endif
 731     }
 732   }
 733
 734   auto* scriptSnapshot = new (alloc_.fallible()) WarpScriptSnapshot(
 735       script_, environment, std::move(opSnapshots), moduleObject);
 736   if (!scriptSnapshot) {
 737     return abort(AbortReason::Alloc);
 738   }
 739
 740   autoClearOpSnapshots.release();
 741   return scriptSnapshot;
 742 }
 743
 744 static void LineNumberAndColumn(HandleScript script, BytecodeLocation loc,
 745                                 unsigned* line,
 746                                 JS::LimitedColumnNumberOneOrigin* column) {
 747 #ifdef DEBUG
 748   *line = PCToLineNumber(script, loc.toRawBytecode(), column);
 749 #else
 750   *line = script->lineno();
 751   *column = script->column();
 752 #endif
 753 }
 754
 755 static void MaybeSetInliningStateFromJitHints(JSContext* cx,
 756                                               ICFallbackStub* fallbackStub,
 757                                               JSScript* script,
 758                                               BytecodeLocation loc) {
 759   // Only update the state if it has already been marked as a candidate.
 760   if (fallbackStub->trialInliningState() != TrialInliningState::Candidate) {
 761     return;
 762   }
 763
 764   // Make sure the op is inlineable.
 765   if (!TrialInliner::IsValidInliningOp(loc.getOp())) {
 766     return;
 767   }
 768
 769   if (!cx->runtime()->jitRuntime()->hasJitHintsMap()) {
 770     return;
 771   }
 772
 773   JitHintsMap* jitHints = cx->runtime()->jitRuntime()->getJitHintsMap();
 774   uint32_t offset = loc.bytecodeToOffset(script);
 775
 776   if (jitHints->hasMonomorphicInlineHintAtOffset(script, offset)) {
 777     fallbackStub->setTrialInliningState(TrialInliningState::MonomorphicInlined);
 778   }
 779 }
 780
 781 AbortReasonOr<Ok> WarpScriptOracle::maybeInlineIC(WarpOpSnapshotList& snapshots,
 782                                                   BytecodeLocation loc) {
 783   // Do one of the following:
 784   //
 785   // * If the Baseline IC has a single ICStub we can inline, add a WarpCacheIR
 786   //   snapshot to transpile it to MIR.
 787   //
 788   // * If that single ICStub is a call IC with a known target, instead add a
 789   //   WarpInline snapshot to transpile the guards to MIR and inline the target.
 790   //
 791   // * If the Baseline IC is cold (never executed), add a WarpBailout snapshot
 792   //   so that we can collect information in Baseline.
 793   //
 794   // * Else, don't add a snapshot and rely on WarpBuilder adding an Ion IC.
 795
 796   MOZ_ASSERT(loc.opHasIC());
 797
 798   // Don't create snapshots when testing ICs.
 799   if (JitOptions.forceInlineCaches) {
 800     return Ok();
 801   }
 802
 803   ICFallbackStub* fallbackStub;
 804   const ICEntry& entry = getICEntryAndFallback(loc, &fallbackStub);
 805   ICStub* firstStub = entry.firstStub();
 806
 807   uint32_t offset = loc.bytecodeToOffset(script_);
 808
 809   // Set the trial inlining state directly if there is a hint cached from a
 810   // previous compilation.
 811   MaybeSetInliningStateFromJitHints(cx_, fallbackStub, script_, loc);
 812
 813   // Clear the used-by-transpiler flag on the IC. It can still be set from a
 814   // previous compilation because we don't clear the flag on every IC when
 815   // invalidating.
 816   fallbackStub->clearUsedByTranspiler();
 817
 818   if (firstStub == fallbackStub) {
 819     [[maybe_unused]] unsigned line;
 820     [[maybe_unused]] JS::LimitedColumnNumberOneOrigin column;
 821     LineNumberAndColumn(script_, loc, &line, &column);
 822
 823     // No optimized stubs.
 824     JitSpew(JitSpew_WarpTranspiler,
 825             "fallback stub (entered-count: %" PRIu32
 826             ") for JSOp::%s @ %s:%u:%u",
 827             fallbackStub->enteredCount(), CodeName(loc.getOp()),
 828             script_->filename(), line, column.oneOriginValue());
 829
 830     // If the fallback stub was used but there's no optimized stub, use an IC.
 831     if (fallbackStub->enteredCount() != 0) {
 832       return Ok();
 833     }
 834
 835     // Cold IC. Bailout to collect information.
 836     if (!AddOpSnapshot<WarpBailout>(alloc_, snapshots, offset)) {
 837       return abort(AbortReason::Alloc);
 838     }
 839     return Ok();
 840   }
 841
 842   ICCacheIRStub* stub = firstStub->toCacheIRStub();
 843
 844   // Don't transpile if this IC ever encountered a case where it had
 845   // no stub to attach.
 846   if (fallbackStub->state().hasFailures()) {
 847     [[maybe_unused]] unsigned line;
 848     [[maybe_unused]] JS::LimitedColumnNumberOneOrigin column;
 849     LineNumberAndColumn(script_, loc, &line, &column);
 850
 851     JitSpew(JitSpew_WarpTranspiler, "Failed to attach for JSOp::%s @ %s:%u:%u",
 852             CodeName(loc.getOp()), script_->filename(), line,
 853             column.oneOriginValue());
 854     return Ok();
 855   }
 856
 857   // Don't transpile if there are other stubs with entered-count > 0. Counters
 858   // are reset when a new stub is attached so this means the stub that was added
 859   // most recently didn't handle all cases.
 860   // If this code is changed, ICScript::hash may also need changing.
 861   bool firstStubHandlesAllCases = true;
 862   for (ICStub* next = stub->next(); next; next = next->maybeNext()) {
 863     if (next->enteredCount() != 0) {
 864       firstStubHandlesAllCases = false;
 865       break;
 866     }
 867   }
 868
 869   if (!firstStubHandlesAllCases) {
 870     // In some polymorphic cases, we can generate better code than the
 871     // default fallback if we know the observed types of the operands
 872     // and their relative frequency.
 873     if (ICSupportsPolymorphicTypeData(loc.getOp()) &&
 874         fallbackStub->enteredCount() == 0) {
 875       bool inlinedPolymorphicTypes = false;
 876       MOZ_TRY_VAR(
 877           inlinedPolymorphicTypes,
 878           maybeInlinePolymorphicTypes(snapshots, loc, stub, fallbackStub));
 879       if (inlinedPolymorphicTypes) {
 880         return Ok();
 881       }
 882     }
 883
 884     [[maybe_unused]] unsigned line;
 885     [[maybe_unused]] JS::LimitedColumnNumberOneOrigin column;
 886     LineNumberAndColumn(script_, loc, &line, &column);
 887
 888     JitSpew(JitSpew_WarpTranspiler,
 889             "multiple active stubs for JSOp::%s @ %s:%u:%u",
 890             CodeName(loc.getOp()), script_->filename(), line,
 891             column.oneOriginValue());
 892     return Ok();
 893   }
 894
 895   const CacheIRStubInfo* stubInfo = stub->stubInfo();
 896   const uint8_t* stubData = stub->stubDataStart();
 897
 898   // Only create a snapshot if all opcodes are supported by the transpiler.
 899   CacheIRReader reader(stubInfo);
 900   while (reader.more()) {
 901     CacheOp op = reader.readOp();
 902     CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
 903     reader.skip(opInfo.argLength);
 904
 905     if (!opInfo.transpile) {
 906       [[maybe_unused]] unsigned line;
 907       [[maybe_unused]] JS::LimitedColumnNumberOneOrigin column;
 908       LineNumberAndColumn(script_, loc, &line, &column);
 909
 910       MOZ_ASSERT(
 911           fallbackStub->trialInliningState() != TrialInliningState::Inlined,
 912           "Trial-inlined stub not supported by transpiler");
 913
 914       // Unsupported CacheIR opcode.
 915       JitSpew(JitSpew_WarpTranspiler,
 916               "unsupported CacheIR opcode %s for JSOp::%s @ %s:%u:%u",
 917               CacheIROpNames[size_t(op)], CodeName(loc.getOp()),
 918               script_->filename(), line, column.oneOriginValue());
 919       return Ok();
 920     }
 921
 922     // While on the main thread, ensure code stubs exist for ops that require
 923     // them.
 924     switch (op) {
 925       case CacheOp::CallRegExpMatcherResult:
 926         if (!cx_->zone()->jitZone()->ensureRegExpMatcherStubExists(cx_)) {
 927           return abort(AbortReason::Error);
 928         }
 929         break;
 930       case CacheOp::CallRegExpSearcherResult:
 931         if (!cx_->zone()->jitZone()->ensureRegExpSearcherStubExists(cx_)) {
 932           return abort(AbortReason::Error);
 933         }
 934         break;
 935       case CacheOp::RegExpBuiltinExecMatchResult:
 936         if (!cx_->zone()->jitZone()->ensureRegExpExecMatchStubExists(cx_)) {
 937           return abort(AbortReason::Error);
 938         }
 939         break;
 940       case CacheOp::RegExpBuiltinExecTestResult:
 941         if (!cx_->zone()->jitZone()->ensureRegExpExecTestStubExists(cx_)) {
 942           return abort(AbortReason::Error);
 943         }
 944         break;
 945       default:
 946         break;
 947     }
 948   }
 949
 950   // Check GC is not possible between updating stub pointers and creating the
 951   // snapshot.
 952   JS::AutoAssertNoGC nogc;
 953
 954   // Copy the ICStub data to protect against the stub being unlinked or mutated.
 955   // We don't need to copy the CacheIRStubInfo: because we store and trace the
 956   // stub's JitCode*, the baselineCacheIRStubCodes_ map in JitZone will keep it
 957   // alive.
 958   uint8_t* stubDataCopy = nullptr;
 959   size_t bytesNeeded = stubInfo->stubDataSize();
 960   if (bytesNeeded > 0) {
 961     stubDataCopy = alloc_.allocateArray<uint8_t>(bytesNeeded);
 962     if (!stubDataCopy) {
 963       return abort(AbortReason::Alloc);
 964     }
 965
 966     // Note: nursery pointers are handled below and the read barrier for weak
 967     // pointers is handled above so we can do a bitwise copy here.
 968     std::copy_n(stubData, bytesNeeded, stubDataCopy);
 969
 970     if (!replaceNurseryAndAllocSitePointers(stub, stubInfo, stubDataCopy)) {
 971       return abort(AbortReason::Alloc);
 972     }
 973   }
 974
 975   JitCode* jitCode = stub->jitCode();
 976
 977   if (fallbackStub->trialInliningState() == TrialInliningState::Inlined ||
 978       fallbackStub->trialInliningState() ==
 979           TrialInliningState::MonomorphicInlined) {
 980     bool inlinedCall;
 981     MOZ_TRY_VAR(inlinedCall, maybeInlineCall(snapshots, loc, stub, fallbackStub,
 982                                              stubDataCopy));
 983     if (inlinedCall) {
 984       return Ok();
 985     }
 986   }
 987
 988   if (!AddOpSnapshot<WarpCacheIR>(alloc_, snapshots, offset, jitCode, stubInfo,
 989                                   stubDataCopy)) {
 990     return abort(AbortReason::Alloc);
 991   }
 992
 993   fallbackStub->setUsedByTranspiler();
 994
 995   return Ok();
 996 }
 997
 998 AbortReasonOr<bool> WarpScriptOracle::maybeInlineCall(
 999     WarpOpSnapshotList& snapshots, BytecodeLocation loc, ICCacheIRStub* stub,
1000     ICFallbackStub* fallbackStub, uint8_t* stubDataCopy) {
1001   Maybe<InlinableOpData> inlineData = FindInlinableOpData(stub, loc);
1002   if (inlineData.isNothing()) {
1003     return false;
1004   }
1005
1006   RootedFunction targetFunction(cx_, inlineData->target);
1007   if (!TrialInliner::canInline(targetFunction, script_, loc)) {
1008     return false;
1009   }
1010
1011   bool isTrialInlined =
1012       fallbackStub->trialInliningState() == TrialInliningState::Inlined;
1013   MOZ_ASSERT_IF(!isTrialInlined, fallbackStub->trialInliningState() ==
1014                                      TrialInliningState::MonomorphicInlined);
1015
1016   RootedScript targetScript(cx_, targetFunction->nonLazyScript());
1017   ICScript* icScript = nullptr;
1018   if (isTrialInlined) {
1019     icScript = inlineData->icScript;
1020   } else {
1021     JitScript* jitScript = targetScript->jitScript();
1022     icScript = jitScript->icScript();
1023   }
1024
1025   if (!icScript) {
1026     return false;
1027   }
1028
1029   // This is just a cheap check to limit the damage we can do to ourselves if
1030   // we try to monomorphically inline an indirectly recursive call.
1031   const uint32_t maxInliningDepth = 8;
1032   if (!isTrialInlined &&
1033       info_->inlineScriptTree()->depth() > maxInliningDepth) {
1034     return false;
1035   }
1036
1037   // And this is a second cheap check to ensure monomorphic inlining doesn't
1038   // cause us to blow past our script size budget.
1039   if (oracle_->accumulatedBytecodeSize() + targetScript->length() >
1040       JitOptions.ionMaxScriptSize) {
1041     return false;
1042   }
1043
1044   // Add the inlined script to the inline script tree.
1045   LifoAlloc* lifoAlloc = alloc_.lifoAlloc();
1046   InlineScriptTree* inlineScriptTree = info_->inlineScriptTree()->addCallee(
1047       &alloc_, loc.toRawBytecode(), targetScript, !isTrialInlined);
1048   if (!inlineScriptTree) {
1049     return abort(AbortReason::Alloc);
1050   }
1051
1052   // Create a CompileInfo for the inlined script.
1053   jsbytecode* osrPc = nullptr;
1054   bool needsArgsObj = targetScript->needsArgsObj();
1055   CompileInfo* info = lifoAlloc->new_<CompileInfo>(
1056       mirGen_.runtime, targetScript, targetFunction, osrPc, needsArgsObj,
1057       inlineScriptTree);
1058   if (!info) {
1059     return abort(AbortReason::Alloc);
1060   }
1061
1062   // Take a snapshot of the CacheIR.
1063   uint32_t offset = loc.bytecodeToOffset(script_);
1064   JitCode* jitCode = stub->jitCode();
1065   const CacheIRStubInfo* stubInfo = stub->stubInfo();
1066   WarpCacheIR* cacheIRSnapshot = new (alloc_.fallible())
1067       WarpCacheIR(offset, jitCode, stubInfo, stubDataCopy);
1068   if (!cacheIRSnapshot) {
1069     return abort(AbortReason::Alloc);
1070   }
1071
1072   // Read barrier for weak stub data copied into the snapshot.
1073   Zone* zone = jitCode->zone();
1074   if (zone->needsIncrementalBarrier()) {
1075     TraceWeakCacheIRStub(zone->barrierTracer(), stub, stub->stubInfo());
1076   }
1077
1078   // Take a snapshot of the inlined script (which may do more
1079   // inlining recursively).
1080   WarpScriptOracle scriptOracle(cx_, oracle_, targetScript, info, icScript);
1081
1082   AbortReasonOr<WarpScriptSnapshot*> maybeScriptSnapshot =
1083       scriptOracle.createScriptSnapshot();
1084
1085   if (maybeScriptSnapshot.isErr()) {
1086     JitSpew(JitSpew_WarpTranspiler, "Can't create snapshot for JSOp::%s",
1087             CodeName(loc.getOp()));
1088
1089     switch (maybeScriptSnapshot.unwrapErr()) {
1090       case AbortReason::Disable: {
1091         // If the target script can't be warp-compiled, mark it as
1092         // uninlineable, clean up, and fall through to the non-inlined path.
1093         ICEntry* entry = icScript_->icEntryForStub(fallbackStub);
1094         if (entry->firstStub() == stub) {
1095           fallbackStub->unlinkStub(cx_->zone(), entry, /*prev=*/nullptr, stub);
1096         }
1097         targetScript->setUninlineable();
1098         info_->inlineScriptTree()->removeCallee(inlineScriptTree);
1099         if (isTrialInlined) {
1100           icScript_->removeInlinedChild(loc.bytecodeToOffset(script_));
1101         }
1102         fallbackStub->setTrialInliningState(TrialInliningState::Failure);
1103         return false;
1104       }
1105       case AbortReason::Error:
1106       case AbortReason::Alloc:
1107         return Err(maybeScriptSnapshot.unwrapErr());
1108       default:
1109         MOZ_CRASH("Unexpected abort reason");
1110     }
1111   }
1112
1113   WarpScriptSnapshot* scriptSnapshot = maybeScriptSnapshot.unwrap();
1114   oracle_->addScriptSnapshot(scriptSnapshot, icScript, targetScript->length());
1115 #ifdef DEBUG
1116   if (!isTrialInlined && targetScript->jitScript()->hasPurgedStubs()) {
1117     oracle_->ignoreFailedICHash();
1118   }
1119 #endif
1120
1121   if (!AddOpSnapshot<WarpInlinedCall>(alloc_, snapshots, offset,
1122                                       cacheIRSnapshot, scriptSnapshot, info)) {
1123     return abort(AbortReason::Alloc);
1124   }
1125   fallbackStub->setUsedByTranspiler();
1126
1127   // Store the location of this monomorphic inline as a hint for future
1128   // compilations.
1129   if (!isTrialInlined && cx_->runtime()->jitRuntime()->hasJitHintsMap()) {
1130     JitHintsMap* jitHints = cx_->runtime()->jitRuntime()->getJitHintsMap();
1131     if (!jitHints->addMonomorphicInlineLocation(script_, loc)) {
1132       return abort(AbortReason::Alloc);
1133     }
1134   }
1135
1136   return true;
1137 }
1138
1139 void WarpOracle::ignoreFailedICHash() {
1140   outerScript_->jitScript()->notePurgedStubs();
1141 }
1142
1143 struct TypeFrequency {
1144   TypeData typeData_;
1145   uint32_t successCount_;
1146   TypeFrequency(TypeData typeData, uint32_t successCount)
1147       : typeData_(typeData), successCount_(successCount) {}
1148
1149   // Sort highest frequency first.
1150   bool operator<(const TypeFrequency& other) const {
1151     return other.successCount_ < successCount_;
1152   }
1153 };
1154
1155 AbortReasonOr<bool> WarpScriptOracle::maybeInlinePolymorphicTypes(
1156     WarpOpSnapshotList& snapshots, BytecodeLocation loc,
1157     ICCacheIRStub* firstStub, ICFallbackStub* fallbackStub) {
1158   MOZ_ASSERT(ICSupportsPolymorphicTypeData(loc.getOp()));
1159
1160   // We use polymorphic type data if there are multiple active stubs,
1161   // all of which have type data available.
1162   Vector<TypeFrequency, 6, SystemAllocPolicy> candidates;
1163   for (ICStub* stub = firstStub; !stub->isFallback();
1164        stub = stub->maybeNext()) {
1165     ICCacheIRStub* cacheIRStub = stub->toCacheIRStub();
1166     uint32_t successCount =
1167         cacheIRStub->enteredCount() - cacheIRStub->next()->enteredCount();
1168     if (successCount == 0) {
1169       continue;
1170     }
1171     TypeData types = cacheIRStub->typeData();
1172     if (!types.hasData()) {
1173       return false;
1174     }
1175     if (!candidates.append(TypeFrequency(types, successCount))) {
1176       return abort(AbortReason::Alloc);
1177     }
1178   }
1179   if (candidates.length() < 2) {
1180     return false;
1181   }
1182
1183   // Sort candidates by success frequency.
1184   std::sort(candidates.begin(), candidates.end());
1185
1186   TypeDataList list;
1187   for (auto& candidate : candidates) {
1188     list.addTypeData(candidate.typeData_);
1189   }
1190
1191   uint32_t offset = loc.bytecodeToOffset(script_);
1192   if (!AddOpSnapshot<WarpPolymorphicTypes>(alloc_, snapshots, offset, list)) {
1193     return abort(AbortReason::Alloc);
1194   }
1195
1196   return true;
1197 }
1198
1199 bool WarpScriptOracle::replaceNurseryAndAllocSitePointers(
1200     ICCacheIRStub* stub, const CacheIRStubInfo* stubInfo,
1201     uint8_t* stubDataCopy) {
1202   // If the stub data contains nursery object pointers, replace them with the
1203   // corresponding nursery index. See WarpObjectField.
1204   //
1205   // If the stub data contains allocation site pointers replace them with the
1206   // initial heap to use, because the site's state may be mutated by the main
1207   // thread while we are compiling.
1208   //
1209   // If the stub data contains weak pointers then trigger a read barrier. This
1210   // is necessary as these will now be strong references in the snapshot.
1211   //
1212   // Also asserts non-object fields don't contain nursery pointers.
1213
1214   uint32_t field = 0;
1215   size_t offset = 0;
1216   while (true) {
1217     StubField::Type fieldType = stubInfo->fieldType(field);
1218     switch (fieldType) {
1219       case StubField::Type::RawInt32:
1220       case StubField::Type::RawPointer:
1221       case StubField::Type::RawInt64:
1222       case StubField::Type::Double:
1223         break;
1224       case StubField::Type::Shape:
1225         static_assert(std::is_convertible_v<Shape*, gc::TenuredCell*>,
1226                       "Code assumes shapes are tenured");
1227         break;
1228       case StubField::Type::WeakShape: {
1229         static_assert(std::is_convertible_v<Shape*, gc::TenuredCell*>,
1230                       "Code assumes shapes are tenured");
1231         stubInfo->getStubField<StubField::Type::WeakShape>(stub, offset).get();
1232         break;
1233       }
1234       case StubField::Type::WeakGetterSetter: {
1235         static_assert(std::is_convertible_v<GetterSetter*, gc::TenuredCell*>,
1236                       "Code assumes GetterSetters are tenured");
1237         stubInfo->getStubField<StubField::Type::WeakGetterSetter>(stub, offset)
1238             .get();
1239         break;
1240       }
1241       case StubField::Type::Symbol:
1242         static_assert(std::is_convertible_v<JS::Symbol*, gc::TenuredCell*>,
1243                       "Code assumes symbols are tenured");
1244         break;
1245       case StubField::Type::WeakBaseScript: {
1246         static_assert(std::is_convertible_v<BaseScript*, gc::TenuredCell*>,
1247                       "Code assumes scripts are tenured");
1248         stubInfo->getStubField<StubField::Type::WeakBaseScript>(stub, offset)
1249             .get();
1250         break;
1251       }
1252       case StubField::Type::JitCode:
1253         static_assert(std::is_convertible_v<JitCode*, gc::TenuredCell*>,
1254                       "Code assumes JitCodes are tenured");
1255         break;
1256       case StubField::Type::JSObject: {
1257         JSObject* obj =
1258             stubInfo->getStubField<StubField::Type::JSObject>(stub, offset);
1259         if (!maybeReplaceNurseryPointer(stubInfo, stubDataCopy, obj, offset)) {
1260           return false;
1261         }
1262         break;
1263       }
1264       case StubField::Type::WeakObject: {
1265         JSObject* obj =
1266             stubInfo->getStubField<StubField::Type::WeakObject>(stub, offset);
1267         if (!maybeReplaceNurseryPointer(stubInfo, stubDataCopy, obj, offset)) {
1268           return false;
1269         }
1270         break;
1271       }
1272       case StubField::Type::String: {
1273 #ifdef DEBUG
1274         JSString* str =
1275             stubInfo->getStubField<StubField::Type::String>(stub, offset);
1276         MOZ_ASSERT(!IsInsideNursery(str));
1277 #endif
1278         break;
1279       }
1280       case StubField::Type::Id: {
1281 #ifdef DEBUG
1282         // jsid never contains nursery-allocated things.
1283         jsid id = stubInfo->getStubField<StubField::Type::Id>(stub, offset);
1284         MOZ_ASSERT_IF(id.isGCThing(),
1285                       !IsInsideNursery(id.toGCCellPtr().asCell()));
1286 #endif
1287         break;
1288       }
1289       case StubField::Type::Value: {
1290 #ifdef DEBUG
1291         Value v = stubInfo->getStubField<StubField::Type::Value>(stub, offset);
1292         MOZ_ASSERT_IF(v.isGCThing(), !IsInsideNursery(v.toGCThing()));
1293 #endif
1294         break;
1295       }
1296       case StubField::Type::AllocSite: {
1297         uintptr_t oldWord = stubInfo->getStubRawWord(stub, offset);
1298         auto* site = reinterpret_cast<gc::AllocSite*>(oldWord);
1299         gc::Heap initialHeap = site->initialHeap();
1300         uintptr_t newWord = uintptr_t(initialHeap);
1301         stubInfo->replaceStubRawWord(stubDataCopy, offset, oldWord, newWord);
1302         break;
1303       }
1304       case StubField::Type::Limit:
1305         return true;  // Done.
1306     }
1307     field++;
1308     offset += StubField::sizeInBytes(fieldType);
1309   }
1310 }
1311
1312 bool WarpScriptOracle::maybeReplaceNurseryPointer(
1313     const CacheIRStubInfo* stubInfo, uint8_t* stubDataCopy, JSObject* obj,
1314     size_t offset) {
1315   if (!IsInsideNursery(obj)) {
1316     return true;
1317   }
1318
1319   uint32_t nurseryIndex;
1320   if (!oracle_->registerNurseryObject(obj, &nurseryIndex)) {
1321     return false;
1322   }
1323
1324   uintptr_t oldWord = WarpObjectField::fromObject(obj).rawData();
1325   uintptr_t newWord = WarpObjectField::fromNurseryIndex(nurseryIndex).rawData();
1326   stubInfo->replaceStubRawWord(stubDataCopy, offset, oldWord, newWord);
1327   return true;
1328 }
1329
1330 bool WarpOracle::registerNurseryObject(JSObject* obj, uint32_t* nurseryIndex) {
1331   MOZ_ASSERT(IsInsideNursery(obj));
1332
1333   auto p = nurseryObjectsMap_.lookupForAdd(obj);
1334   if (p) {
1335     *nurseryIndex = p->value();
1336     return true;
1337   }
1338
1339   if (!nurseryObjects_.append(obj)) {
1340     return false;
1341   }
1342   *nurseryIndex = nurseryObjects_.length() - 1;
1343   return nurseryObjectsMap_.add(p, obj, *nurseryIndex);
1344 }