js/src/jit/WarpOracle.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "jit/WarpOracle.h"
   8
   9 #include "mozilla/ScopeExit.h"
  10
  11 #include <algorithm>
  12
  13 #include "jit/CacheIR.h"
  14 #include "jit/CacheIRCompiler.h"
  15 #include "jit/CacheIRReader.h"
  16 #include "jit/CompileInfo.h"
  17 #include "jit/InlineScriptTree.h"
  18 #include "jit/JitRealm.h"
  19 #include "jit/JitScript.h"
  20 #include "jit/JitSpewer.h"
  21 #include "jit/MIRGenerator.h"
  22 #include "jit/TrialInlining.h"
  23 #include "jit/TypeData.h"
  24 #include "jit/WarpBuilder.h"
  25 #include "util/DifferentialTesting.h"
  26 #include "vm/BuiltinObjectKind.h"
  27 #include "vm/BytecodeIterator.h"
  28 #include "vm/BytecodeLocation.h"
  29
  30 #include "jit/InlineScriptTree-inl.h"
  31 #include "vm/BytecodeIterator-inl.h"
  32 #include "vm/BytecodeLocation-inl.h"
  33 #include "vm/EnvironmentObject-inl.h"
  34 #include "vm/Interpreter-inl.h"
  35
  36 using namespace js;
  37 using namespace js::jit;
  38
  39 using mozilla::Maybe;
  40
  41 // WarpScriptOracle creates a WarpScriptSnapshot for a single JSScript. Note
  42 // that a single WarpOracle can use multiple WarpScriptOracles when scripts are
  43 // inlined.
  44 class MOZ_STACK_CLASS WarpScriptOracle {
  45   JSContext* cx_;
  46   WarpOracle* oracle_;
  47   MIRGenerator& mirGen_;
  48   TempAllocator& alloc_;
  49   HandleScript script_;
  50   const CompileInfo* info_;
  51   ICScript* icScript_;
  52
  53   // Index of the next ICEntry for getICEntry. This assumes the script's
  54   // bytecode is processed from first to last instruction.
  55   uint32_t icEntryIndex_ = 0;
  56
  57   template <typename... Args>
  58   mozilla::GenericErrorResult<AbortReason> abort(Args&&... args) {
  59     return oracle_->abort(script_, args...);
  60   }
  61
  62   WarpEnvironment createEnvironment();
  63   AbortReasonOr<Ok> maybeInlineIC(WarpOpSnapshotList& snapshots,
  64                                   BytecodeLocation loc);
  65   AbortReasonOr<bool> maybeInlineCall(WarpOpSnapshotList& snapshots,
  66                                       BytecodeLocation loc, ICCacheIRStub* stub,
  67                                       ICFallbackStub* fallbackStub,
  68                                       uint8_t* stubDataCopy);
  69   AbortReasonOr<bool> maybeInlinePolymorphicTypes(WarpOpSnapshotList& snapshots,
  70                                                   BytecodeLocation loc,
  71                                                   ICCacheIRStub* firstStub,
  72                                                   ICFallbackStub* fallbackStub);
  73   [[nodiscard]] bool replaceNurseryAndAllocSitePointers(
  74       ICCacheIRStub* stub, const CacheIRStubInfo* stubInfo,
  75       uint8_t* stubDataCopy);
  76
  77  public:
  78   WarpScriptOracle(JSContext* cx, WarpOracle* oracle, HandleScript script,
  79                    const CompileInfo* info, ICScript* icScript)
  80       : cx_(cx),
  81         oracle_(oracle),
  82         mirGen_(oracle->mirGen()),
  83         alloc_(mirGen_.alloc()),
  84         script_(script),
  85         info_(info),
  86         icScript_(icScript) {}
  87
  88   AbortReasonOr<WarpScriptSnapshot*> createScriptSnapshot();
  89
  90   ICEntry& getICEntryAndFallback(BytecodeLocation loc,
  91                                  ICFallbackStub** fallback);
  92 };
  93
  94 WarpOracle::WarpOracle(JSContext* cx, MIRGenerator& mirGen,
  95                        HandleScript outerScript)
  96     : cx_(cx),
  97       mirGen_(mirGen),
  98       alloc_(mirGen.alloc()),
  99       outerScript_(outerScript) {}
 100
 101 mozilla::GenericErrorResult<AbortReason> WarpOracle::abort(HandleScript script,
 102                                                            AbortReason r) {
 103   auto res = mirGen_.abort(r);
 104   JitSpew(JitSpew_IonAbort, "aborted @ %s", script->filename());
 105   return res;
 106 }
 107
 108 mozilla::GenericErrorResult<AbortReason> WarpOracle::abort(HandleScript script,
 109                                                            AbortReason r,
 110                                                            const char* message,
 111                                                            ...) {
 112   va_list ap;
 113   va_start(ap, message);
 114   auto res = mirGen_.abortFmt(r, message, ap);
 115   va_end(ap);
 116   JitSpew(JitSpew_IonAbort, "aborted @ %s", script->filename());
 117   return res;
 118 }
 119
 120 void WarpOracle::addScriptSnapshot(WarpScriptSnapshot* scriptSnapshot,
 121                                    ICScript* icScript, size_t bytecodeLength) {
 122   scriptSnapshots_.insertBack(scriptSnapshot);
 123   accumulatedBytecodeSize_ += bytecodeLength;
 124 #ifdef DEBUG
 125   runningScriptHash_ = mozilla::AddToHash(runningScriptHash_, icScript->hash());
 126 #endif
 127 }
 128
 129 AbortReasonOr<WarpSnapshot*> WarpOracle::createSnapshot() {
 130 #ifdef JS_JITSPEW
 131   const char* mode;
 132   if (outerScript_->hasIonScript()) {
 133     mode = "Recompiling";
 134   } else {
 135     mode = "Compiling";
 136   }
 137   JitSpew(JitSpew_IonScripts,
 138           "Warp %s script %s:%u:%u (%p) (warmup-counter=%" PRIu32 ",%s%s)",
 139           mode, outerScript_->filename(), outerScript_->lineno(),
 140           outerScript_->column(), static_cast<JSScript*>(outerScript_),
 141           outerScript_->getWarmUpCount(),
 142           outerScript_->isGenerator() ? " isGenerator" : "",
 143           outerScript_->isAsync() ? " isAsync" : "");
 144 #endif
 145
 146   accumulatedBytecodeSize_ = outerScript_->length();
 147
 148   MOZ_ASSERT(outerScript_->hasJitScript());
 149   ICScript* icScript = outerScript_->jitScript()->icScript();
 150   WarpScriptOracle scriptOracle(cx_, this, outerScript_, &mirGen_.outerInfo(),
 151                                 icScript);
 152
 153   WarpScriptSnapshot* scriptSnapshot;
 154   MOZ_TRY_VAR(scriptSnapshot, scriptOracle.createScriptSnapshot());
 155
 156   // Insert the outermost scriptSnapshot at the front of the list.
 157   scriptSnapshots_.insertFront(scriptSnapshot);
 158
 159   bool recordFinalWarmUpCount = false;
 160 #ifdef JS_CACHEIR_SPEW
 161   recordFinalWarmUpCount = outerScript_->needsFinalWarmUpCount();
 162 #endif
 163
 164   auto* snapshot = new (alloc_.fallible())
 165       WarpSnapshot(cx_, alloc_, std::move(scriptSnapshots_), bailoutInfo_,
 166                    recordFinalWarmUpCount);
 167   if (!snapshot) {
 168     return abort(outerScript_, AbortReason::Alloc);
 169   }
 170
 171   if (!snapshot->nurseryObjects().appendAll(nurseryObjects_)) {
 172     return abort(outerScript_, AbortReason::Alloc);
 173   }
 174
 175 #ifdef JS_JITSPEW
 176   if (JitSpewEnabled(JitSpew_WarpSnapshots)) {
 177     Fprinter& out = JitSpewPrinter();
 178     snapshot->dump(out);
 179   }
 180 #endif
 181
 182 #ifdef DEBUG
 183   // When transpiled CacheIR bails out, we do not want to recompile
 184   // with the exact same data and get caught in an invalidation loop.
 185   //
 186   // To avoid this, we store a hash of the stub pointers and entry
 187   // counts in this snapshot, save that hash in the JitScript if we
 188   // have a TranspiledCacheIR or MonomorphicInlinedStubFolding bailout,
 189   // and assert that the hash has changed when we recompile.
 190   //
 191   // Note: this assertion catches potential performance issues.
 192   // Failing this assertion is not a correctness/security problem.
 193   // We therefore ignore cases involving resource exhaustion (OOM,
 194   // stack overflow, etc), or stubs purged by GC.
 195   HashNumber hash = mozilla::AddToHash(icScript->hash(), runningScriptHash_);
 196   if (outerScript_->jitScript()->hasFailedICHash()) {
 197     HashNumber oldHash = outerScript_->jitScript()->getFailedICHash();
 198     MOZ_ASSERT_IF(hash == oldHash && !js::SupportDifferentialTesting(),
 199                   cx_->hadResourceExhaustion());
 200   }
 201   snapshot->setICHash(hash);
 202 #endif
 203
 204   return snapshot;
 205 }
 206
 207 template <typename T, typename... Args>
 208 [[nodiscard]] static bool AddOpSnapshot(TempAllocator& alloc,
 209                                         WarpOpSnapshotList& snapshots,
 210                                         uint32_t offset, Args&&... args) {
 211   T* snapshot = new (alloc.fallible()) T(offset, std::forward<Args>(args)...);
 212   if (!snapshot) {
 213     return false;
 214   }
 215
 216   snapshots.insertBack(snapshot);
 217   return true;
 218 }
 219
 220 [[nodiscard]] static bool AddWarpGetImport(TempAllocator& alloc,
 221                                            WarpOpSnapshotList& snapshots,
 222                                            uint32_t offset, JSScript* script,
 223                                            PropertyName* name) {
 224   ModuleEnvironmentObject* env = GetModuleEnvironmentForScript(script);
 225   MOZ_ASSERT(env);
 226
 227   mozilla::Maybe<PropertyInfo> prop;
 228   ModuleEnvironmentObject* targetEnv;
 229   MOZ_ALWAYS_TRUE(env->lookupImport(NameToId(name), &targetEnv, &prop));
 230
 231   uint32_t numFixedSlots = targetEnv->numFixedSlots();
 232   uint32_t slot = prop->slot();
 233
 234   // In the rare case where this import hasn't been initialized already (we have
 235   // an import cycle where modules reference each other's imports), we need a
 236   // check.
 237   bool needsLexicalCheck =
 238       targetEnv->getSlot(slot).isMagic(JS_UNINITIALIZED_LEXICAL);
 239
 240   return AddOpSnapshot<WarpGetImport>(alloc, snapshots, offset, targetEnv,
 241                                       numFixedSlots, slot, needsLexicalCheck);
 242 }
 243
 244 ICEntry& WarpScriptOracle::getICEntryAndFallback(BytecodeLocation loc,
 245                                                  ICFallbackStub** fallback) {
 246   const uint32_t offset = loc.bytecodeToOffset(script_);
 247
 248   do {
 249     *fallback = icScript_->fallbackStub(icEntryIndex_);
 250     icEntryIndex_++;
 251   } while ((*fallback)->pcOffset() < offset);
 252
 253   MOZ_ASSERT((*fallback)->pcOffset() == offset);
 254   return icScript_->icEntry(icEntryIndex_ - 1);
 255 }
 256
 257 WarpEnvironment WarpScriptOracle::createEnvironment() {
 258   // Don't do anything if the script doesn't use the environment chain.
 259   if (!script_->jitScript()->usesEnvironmentChain()) {
 260     return WarpEnvironment(NoEnvironment());
 261   }
 262
 263   if (script_->isModule()) {
 264     ModuleObject* module = script_->module();
 265     JSObject* obj = &module->initialEnvironment();
 266     return WarpEnvironment(ConstantObjectEnvironment(obj));
 267   }
 268
 269   JSFunction* fun = script_->function();
 270   if (!fun) {
 271     // For global scripts without a non-syntactic global scope, the environment
 272     // chain is the global lexical environment.
 273     MOZ_ASSERT(!script_->isForEval());
 274     MOZ_ASSERT(!script_->hasNonSyntacticScope());
 275     JSObject* obj = &script_->global().lexicalEnvironment();
 276     return WarpEnvironment(ConstantObjectEnvironment(obj));
 277   }
 278
 279   JSObject* templateEnv = script_->jitScript()->templateEnvironment();
 280
 281   CallObject* callObjectTemplate = nullptr;
 282   if (fun->needsCallObject()) {
 283     callObjectTemplate = &templateEnv->as<CallObject>();
 284   }
 285
 286   NamedLambdaObject* namedLambdaTemplate = nullptr;
 287   if (fun->needsNamedLambdaEnvironment()) {
 288     if (callObjectTemplate) {
 289       templateEnv = templateEnv->enclosingEnvironment();
 290     }
 291     namedLambdaTemplate = &templateEnv->as<NamedLambdaObject>();
 292   }
 293
 294   return WarpEnvironment(
 295       FunctionEnvironment(callObjectTemplate, namedLambdaTemplate));
 296 }
 297
 298 AbortReasonOr<WarpScriptSnapshot*> WarpScriptOracle::createScriptSnapshot() {
 299   MOZ_ASSERT(script_->hasJitScript());
 300
 301   if (!script_->jitScript()->ensureHasCachedIonData(cx_, script_)) {
 302     return abort(AbortReason::Error);
 303   }
 304
 305   if (script_->failedBoundsCheck()) {
 306     oracle_->bailoutInfo().setFailedBoundsCheck();
 307   }
 308   if (script_->failedLexicalCheck()) {
 309     oracle_->bailoutInfo().setFailedLexicalCheck();
 310   }
 311
 312   WarpEnvironment environment = createEnvironment();
 313
 314   // Unfortunately LinkedList<> asserts the list is empty in its destructor.
 315   // Clear the list if we abort compilation.
 316   WarpOpSnapshotList opSnapshots;
 317   auto autoClearOpSnapshots =
 318       mozilla::MakeScopeExit([&] { opSnapshots.clear(); });
 319
 320   ModuleObject* moduleObject = nullptr;
 321
 322   // Analyze the bytecode. Abort compilation for unsupported ops and create
 323   // WarpOpSnapshots.
 324   for (BytecodeLocation loc : AllBytecodesIterable(script_)) {
 325     JSOp op = loc.getOp();
 326     uint32_t offset = loc.bytecodeToOffset(script_);
 327     switch (op) {
 328       case JSOp::Arguments: {
 329         MOZ_ASSERT(script_->needsArgsObj());
 330         bool mapped = script_->hasMappedArgsObj();
 331         ArgumentsObject* templateObj =
 332             script_->global().maybeArgumentsTemplateObject(mapped);
 333         if (!AddOpSnapshot<WarpArguments>(alloc_, opSnapshots, offset,
 334                                           templateObj)) {
 335           return abort(AbortReason::Alloc);
 336         }
 337         break;
 338       }
 339       case JSOp::RegExp: {
 340         bool hasShared = loc.getRegExp(script_)->hasShared();
 341         if (!AddOpSnapshot<WarpRegExp>(alloc_, opSnapshots, offset,
 342                                        hasShared)) {
 343           return abort(AbortReason::Alloc);
 344         }
 345         break;
 346       }
 347
 348       case JSOp::FunctionThis:
 349         if (!script_->strict() && script_->hasNonSyntacticScope()) {
 350           // Abort because MBoxNonStrictThis doesn't support non-syntactic
 351           // scopes (a deprecated SpiderMonkey mechanism). If this becomes an
 352           // issue we could support it by refactoring GetFunctionThis to not
 353           // take a frame pointer and then call that.
 354           return abort(AbortReason::Disable,
 355                        "JSOp::FunctionThis with non-syntactic scope");
 356         }
 357         break;
 358
 359       case JSOp::GlobalThis:
 360         MOZ_ASSERT(!script_->hasNonSyntacticScope());
 361         break;
 362
 363       case JSOp::BuiltinObject: {
 364         // If we already resolved this built-in we can bake it in.
 365         auto kind = loc.getBuiltinObjectKind();
 366         if (JSObject* proto = MaybeGetBuiltinObject(cx_->global(), kind)) {
 367           if (!AddOpSnapshot<WarpBuiltinObject>(alloc_, opSnapshots, offset,
 368                                                 proto)) {
 369             return abort(AbortReason::Alloc);
 370           }
 371         }
 372         break;
 373       }
 374
 375       case JSOp::GetIntrinsic: {
 376         // If we already cloned this intrinsic we can bake it in.
 377         // NOTE: When the initializer runs in a content global, we also have to
 378         //       worry about nursery objects. These quickly tenure and stay that
 379         //       way so this is only a temporary problem.
 380         PropertyName* name = loc.getPropertyName(script_);
 381         Value val;
 382         if (cx_->global()->maybeGetIntrinsicValue(name, &val, cx_) &&
 383             JS::GCPolicy<Value>::isTenured(val)) {
 384           if (!AddOpSnapshot<WarpGetIntrinsic>(alloc_, opSnapshots, offset,
 385                                                val)) {
 386             return abort(AbortReason::Alloc);
 387           }
 388         }
 389         break;
 390       }
 391
 392       case JSOp::ImportMeta: {
 393         if (!moduleObject) {
 394           moduleObject = GetModuleObjectForScript(script_);
 395           MOZ_ASSERT(moduleObject->isTenured());
 396         }
 397         break;
 398       }
 399
 400       case JSOp::GetImport: {
 401         PropertyName* name = loc.getPropertyName(script_);
 402         if (!AddWarpGetImport(alloc_, opSnapshots, offset, script_, name)) {
 403           return abort(AbortReason::Alloc);
 404         }
 405         break;
 406       }
 407
 408       case JSOp::Lambda: {
 409         JSFunction* fun = loc.getFunction(script_);
 410         if (IsAsmJSModule(fun)) {
 411           return abort(AbortReason::Disable, "asm.js module function lambda");
 412         }
 413         break;
 414       }
 415
 416       case JSOp::GetElemSuper: {
 417 #if defined(JS_CODEGEN_X86)
 418         // x86 does not have enough registers.
 419         return abort(AbortReason::Disable,
 420                      "GetElemSuper is not supported on x86");
 421 #else
 422         MOZ_TRY(maybeInlineIC(opSnapshots, loc));
 423         break;
 424 #endif
 425       }
 426
 427       case JSOp::Rest: {
 428         if (Shape* shape =
 429                 script_->global().maybeArrayShapeWithDefaultProto()) {
 430           if (!AddOpSnapshot<WarpRest>(alloc_, opSnapshots, offset, shape)) {
 431             return abort(AbortReason::Alloc);
 432           }
 433         }
 434         break;
 435       }
 436
 437       case JSOp::BindGName: {
 438         Rooted<GlobalObject*> global(cx_, &script_->global());
 439         Rooted<PropertyName*> name(cx_, loc.getPropertyName(script_));
 440         if (JSObject* env = MaybeOptimizeBindGlobalName(cx_, global, name)) {
 441           MOZ_ASSERT(env->isTenured());
 442           if (!AddOpSnapshot<WarpBindGName>(alloc_, opSnapshots, offset, env)) {
 443             return abort(AbortReason::Alloc);
 444           }
 445         } else {
 446           MOZ_TRY(maybeInlineIC(opSnapshots, loc));
 447         }
 448         break;
 449       }
 450
 451       case JSOp::PushVarEnv: {
 452         Rooted<VarScope*> scope(cx_, &loc.getScope(script_)->as<VarScope>());
 453
 454         auto* templateObj =
 455             VarEnvironmentObject::createTemplateObject(cx_, scope);
 456         if (!templateObj) {
 457           return abort(AbortReason::Alloc);
 458         }
 459         MOZ_ASSERT(templateObj->isTenured());
 460
 461         if (!AddOpSnapshot<WarpVarEnvironment>(alloc_, opSnapshots, offset,
 462                                                templateObj)) {
 463           return abort(AbortReason::Alloc);
 464         }
 465         break;
 466       }
 467
 468       case JSOp::PushLexicalEnv:
 469       case JSOp::FreshenLexicalEnv:
 470       case JSOp::RecreateLexicalEnv: {
 471         Rooted<LexicalScope*> scope(cx_,
 472                                     &loc.getScope(script_)->as<LexicalScope>());
 473
 474         auto* templateObj =
 475             BlockLexicalEnvironmentObject::createTemplateObject(cx_, scope);
 476         if (!templateObj) {
 477           return abort(AbortReason::Alloc);
 478         }
 479         MOZ_ASSERT(templateObj->isTenured());
 480
 481         if (!AddOpSnapshot<WarpLexicalEnvironment>(alloc_, opSnapshots, offset,
 482                                                    templateObj)) {
 483           return abort(AbortReason::Alloc);
 484         }
 485         break;
 486       }
 487
 488       case JSOp::PushClassBodyEnv: {
 489         Rooted<ClassBodyScope*> scope(
 490             cx_, &loc.getScope(script_)->as<ClassBodyScope>());
 491
 492         auto* templateObj =
 493             ClassBodyLexicalEnvironmentObject::createTemplateObject(cx_, scope);
 494         if (!templateObj) {
 495           return abort(AbortReason::Alloc);
 496         }
 497         MOZ_ASSERT(templateObj->isTenured());
 498
 499         if (!AddOpSnapshot<WarpClassBodyEnvironment>(alloc_, opSnapshots,
 500                                                      offset, templateObj)) {
 501           return abort(AbortReason::Alloc);
 502         }
 503         break;
 504       }
 505
 506       case JSOp::GetName:
 507       case JSOp::GetGName:
 508       case JSOp::GetProp:
 509       case JSOp::GetElem:
 510       case JSOp::SetProp:
 511       case JSOp::StrictSetProp:
 512       case JSOp::Call:
 513       case JSOp::CallContent:
 514       case JSOp::CallIgnoresRv:
 515       case JSOp::CallIter:
 516       case JSOp::CallContentIter:
 517       case JSOp::New:
 518       case JSOp::NewContent:
 519       case JSOp::SuperCall:
 520       case JSOp::SpreadCall:
 521       case JSOp::SpreadNew:
 522       case JSOp::SpreadSuperCall:
 523       case JSOp::ToNumeric:
 524       case JSOp::Pos:
 525       case JSOp::Inc:
 526       case JSOp::Dec:
 527       case JSOp::Neg:
 528       case JSOp::BitNot:
 529       case JSOp::Iter:
 530       case JSOp::Eq:
 531       case JSOp::Ne:
 532       case JSOp::Lt:
 533       case JSOp::Le:
 534       case JSOp::Gt:
 535       case JSOp::Ge:
 536       case JSOp::StrictEq:
 537       case JSOp::StrictNe:
 538       case JSOp::BindName:
 539       case JSOp::Add:
 540       case JSOp::Sub:
 541       case JSOp::Mul:
 542       case JSOp::Div:
 543       case JSOp::Mod:
 544       case JSOp::Pow:
 545       case JSOp::BitAnd:
 546       case JSOp::BitOr:
 547       case JSOp::BitXor:
 548       case JSOp::Lsh:
 549       case JSOp::Rsh:
 550       case JSOp::Ursh:
 551       case JSOp::In:
 552       case JSOp::HasOwn:
 553       case JSOp::CheckPrivateField:
 554       case JSOp::Instanceof:
 555       case JSOp::GetPropSuper:
 556       case JSOp::InitProp:
 557       case JSOp::InitLockedProp:
 558       case JSOp::InitHiddenProp:
 559       case JSOp::InitElem:
 560       case JSOp::InitHiddenElem:
 561       case JSOp::InitLockedElem:
 562       case JSOp::InitElemInc:
 563       case JSOp::SetName:
 564       case JSOp::StrictSetName:
 565       case JSOp::SetGName:
 566       case JSOp::StrictSetGName:
 567       case JSOp::InitGLexical:
 568       case JSOp::SetElem:
 569       case JSOp::StrictSetElem:
 570       case JSOp::ToPropertyKey:
 571       case JSOp::OptimizeSpreadCall:
 572       case JSOp::Typeof:
 573       case JSOp::TypeofExpr:
 574       case JSOp::NewObject:
 575       case JSOp::NewInit:
 576       case JSOp::NewArray:
 577       case JSOp::JumpIfFalse:
 578       case JSOp::JumpIfTrue:
 579       case JSOp::And:
 580       case JSOp::Or:
 581       case JSOp::Not:
 582       case JSOp::CloseIter:
 583         MOZ_TRY(maybeInlineIC(opSnapshots, loc));
 584         break;
 585
 586       case JSOp::Nop:
 587       case JSOp::NopDestructuring:
 588       case JSOp::TryDestructuring:
 589       case JSOp::Lineno:
 590       case JSOp::DebugLeaveLexicalEnv:
 591       case JSOp::Undefined:
 592       case JSOp::Void:
 593       case JSOp::Null:
 594       case JSOp::Hole:
 595       case JSOp::Uninitialized:
 596       case JSOp::IsConstructing:
 597       case JSOp::False:
 598       case JSOp::True:
 599       case JSOp::Zero:
 600       case JSOp::One:
 601       case JSOp::Int8:
 602       case JSOp::Uint16:
 603       case JSOp::Uint24:
 604       case JSOp::Int32:
 605       case JSOp::Double:
 606       case JSOp::BigInt:
 607       case JSOp::String:
 608       case JSOp::Symbol:
 609       case JSOp::Pop:
 610       case JSOp::PopN:
 611       case JSOp::Dup:
 612       case JSOp::Dup2:
 613       case JSOp::DupAt:
 614       case JSOp::Swap:
 615       case JSOp::Pick:
 616       case JSOp::Unpick:
 617       case JSOp::GetLocal:
 618       case JSOp::SetLocal:
 619       case JSOp::InitLexical:
 620       case JSOp::GetArg:
 621       case JSOp::GetFrameArg:
 622       case JSOp::SetArg:
 623       case JSOp::ArgumentsLength:
 624       case JSOp::GetActualArg:
 625       case JSOp::JumpTarget:
 626       case JSOp::LoopHead:
 627       case JSOp::Case:
 628       case JSOp::Default:
 629       case JSOp::Coalesce:
 630       case JSOp::Goto:
 631       case JSOp::DebugCheckSelfHosted:
 632       case JSOp::DynamicImport:
 633       case JSOp::ToString:
 634       case JSOp::GlobalOrEvalDeclInstantiation:
 635       case JSOp::BindVar:
 636       case JSOp::MutateProto:
 637       case JSOp::Callee:
 638       case JSOp::ToAsyncIter:
 639       case JSOp::ObjWithProto:
 640       case JSOp::GetAliasedVar:
 641       case JSOp::SetAliasedVar:
 642       case JSOp::InitAliasedLexical:
 643       case JSOp::EnvCallee:
 644       case JSOp::MoreIter:
 645       case JSOp::EndIter:
 646       case JSOp::IsNoIter:
 647       case JSOp::IsNullOrUndefined:
 648       case JSOp::DelProp:
 649       case JSOp::StrictDelProp:
 650       case JSOp::DelElem:
 651       case JSOp::StrictDelElem:
 652       case JSOp::SetFunName:
 653       case JSOp::PopLexicalEnv:
 654       case JSOp::ImplicitThis:
 655       case JSOp::CheckClassHeritage:
 656       case JSOp::CheckThis:
 657       case JSOp::CheckThisReinit:
 658       case JSOp::Generator:
 659       case JSOp::AfterYield:
 660       case JSOp::FinalYieldRval:
 661       case JSOp::AsyncResolve:
 662       case JSOp::CheckResumeKind:
 663       case JSOp::CanSkipAwait:
 664       case JSOp::MaybeExtractAwaitValue:
 665       case JSOp::AsyncAwait:
 666       case JSOp::Await:
 667       case JSOp::CheckReturn:
 668       case JSOp::CheckLexical:
 669       case JSOp::CheckAliasedLexical:
 670       case JSOp::InitHomeObject:
 671       case JSOp::SuperBase:
 672       case JSOp::SuperFun:
 673       case JSOp::InitElemArray:
 674       case JSOp::InitPropGetter:
 675       case JSOp::InitPropSetter:
 676       case JSOp::InitHiddenPropGetter:
 677       case JSOp::InitHiddenPropSetter:
 678       case JSOp::InitElemGetter:
 679       case JSOp::InitElemSetter:
 680       case JSOp::InitHiddenElemGetter:
 681       case JSOp::InitHiddenElemSetter:
 682       case JSOp::NewTarget:
 683       case JSOp::Object:
 684       case JSOp::CallSiteObj:
 685       case JSOp::CheckIsObj:
 686       case JSOp::CheckObjCoercible:
 687       case JSOp::FunWithProto:
 688       case JSOp::Debugger:
 689       case JSOp::TableSwitch:
 690       case JSOp::Exception:
 691       case JSOp::Throw:
 692       case JSOp::ThrowSetConst:
 693       case JSOp::SetRval:
 694       case JSOp::GetRval:
 695       case JSOp::Return:
 696       case JSOp::RetRval:
 697       case JSOp::InitialYield:
 698       case JSOp::Yield:
 699       case JSOp::ResumeKind:
 700       case JSOp::ThrowMsg:
 701       case JSOp::Try:
 702       case JSOp::Finally:
 703       case JSOp::NewPrivateName:
 704         // Supported by WarpBuilder. Nothing to do.
 705         break;
 706
 707         // Unsupported ops. Don't use a 'default' here, we want to trigger a
 708         // compiler warning when adding a new JSOp.
 709 #define DEF_CASE(OP) case JSOp::OP:
 710         WARP_UNSUPPORTED_OPCODE_LIST(DEF_CASE)
 711 #undef DEF_CASE
 712 #ifdef DEBUG
 713         return abort(AbortReason::Disable, "Unsupported opcode: %s",
 714                      CodeName(op));
 715 #else
 716         return abort(AbortReason::Disable, "Unsupported opcode: %u",
 717                      uint8_t(op));
 718 #endif
 719     }
 720   }
 721
 722   auto* scriptSnapshot = new (alloc_.fallible()) WarpScriptSnapshot(
 723       script_, environment, std::move(opSnapshots), moduleObject);
 724   if (!scriptSnapshot) {
 725     return abort(AbortReason::Alloc);
 726   }
 727
 728   autoClearOpSnapshots.release();
 729   return scriptSnapshot;
 730 }
 731
 732 static void LineNumberAndColumn(HandleScript script, BytecodeLocation loc,
 733                                 unsigned* line, unsigned* column) {
 734 #ifdef DEBUG
 735   *line = PCToLineNumber(script, loc.toRawBytecode(), column);
 736 #else
 737   *line = script->lineno();
 738   *column = script->column();
 739 #endif
 740 }
 741
 742 AbortReasonOr<Ok> WarpScriptOracle::maybeInlineIC(WarpOpSnapshotList& snapshots,
 743                                                   BytecodeLocation loc) {
 744   // Do one of the following:
 745   //
 746   // * If the Baseline IC has a single ICStub we can inline, add a WarpCacheIR
 747   //   snapshot to transpile it to MIR.
 748   //
 749   // * If that single ICStub is a call IC with a known target, instead add a
 750   //   WarpInline snapshot to transpile the guards to MIR and inline the target.
 751   //
 752   // * If the Baseline IC is cold (never executed), add a WarpBailout snapshot
 753   //   so that we can collect information in Baseline.
 754   //
 755   // * Else, don't add a snapshot and rely on WarpBuilder adding an Ion IC.
 756
 757   MOZ_ASSERT(loc.opHasIC());
 758
 759   // Don't create snapshots when testing ICs.
 760   if (JitOptions.forceInlineCaches) {
 761     return Ok();
 762   }
 763
 764   ICFallbackStub* fallbackStub;
 765   const ICEntry& entry = getICEntryAndFallback(loc, &fallbackStub);
 766   ICStub* firstStub = entry.firstStub();
 767
 768   uint32_t offset = loc.bytecodeToOffset(script_);
 769
 770   // Clear the used-by-transpiler flag on the IC. It can still be set from a
 771   // previous compilation because we don't clear the flag on every IC when
 772   // invalidating.
 773   fallbackStub->clearUsedByTranspiler();
 774
 775   if (firstStub == fallbackStub) {
 776     [[maybe_unused]] unsigned line, column;
 777     LineNumberAndColumn(script_, loc, &line, &column);
 778
 779     // No optimized stubs.
 780     JitSpew(JitSpew_WarpTranspiler,
 781             "fallback stub (entered-count: %" PRIu32
 782             ") for JSOp::%s @ %s:%u:%u",
 783             fallbackStub->enteredCount(), CodeName(loc.getOp()),
 784             script_->filename(), line, column);
 785
 786     // If the fallback stub was used but there's no optimized stub, use an IC.
 787     if (fallbackStub->enteredCount() != 0) {
 788       return Ok();
 789     }
 790
 791     // Cold IC. Bailout to collect information.
 792     if (!AddOpSnapshot<WarpBailout>(alloc_, snapshots, offset)) {
 793       return abort(AbortReason::Alloc);
 794     }
 795     return Ok();
 796   }
 797
 798   ICCacheIRStub* stub = firstStub->toCacheIRStub();
 799
 800   // Don't transpile if this IC ever encountered a case where it had
 801   // no stub to attach.
 802   if (fallbackStub->state().hasFailures()) {
 803     [[maybe_unused]] unsigned line, column;
 804     LineNumberAndColumn(script_, loc, &line, &column);
 805
 806     JitSpew(JitSpew_WarpTranspiler, "Failed to attach for JSOp::%s @ %s:%u:%u",
 807             CodeName(loc.getOp()), script_->filename(), line, column);
 808     return Ok();
 809   }
 810
 811   // Don't transpile if there are other stubs with entered-count > 0. Counters
 812   // are reset when a new stub is attached so this means the stub that was added
 813   // most recently didn't handle all cases.
 814   // If this code is changed, ICScript::hash may also need changing.
 815   bool firstStubHandlesAllCases = true;
 816   for (ICStub* next = stub->next(); next; next = next->maybeNext()) {
 817     if (next->enteredCount() != 0) {
 818       firstStubHandlesAllCases = false;
 819       break;
 820     }
 821   }
 822
 823   if (!firstStubHandlesAllCases) {
 824     // In some polymorphic cases, we can generate better code than the
 825     // default fallback if we know the observed types of the operands
 826     // and their relative frequency.
 827     if (ICSupportsPolymorphicTypeData(loc.getOp()) &&
 828         fallbackStub->enteredCount() == 0) {
 829       bool inlinedPolymorphicTypes = false;
 830       MOZ_TRY_VAR(
 831           inlinedPolymorphicTypes,
 832           maybeInlinePolymorphicTypes(snapshots, loc, stub, fallbackStub));
 833       if (inlinedPolymorphicTypes) {
 834         return Ok();
 835       }
 836     }
 837
 838     [[maybe_unused]] unsigned line, column;
 839     LineNumberAndColumn(script_, loc, &line, &column);
 840
 841     JitSpew(JitSpew_WarpTranspiler,
 842             "multiple active stubs for JSOp::%s @ %s:%u:%u",
 843             CodeName(loc.getOp()), script_->filename(), line, column);
 844     return Ok();
 845   }
 846
 847   const CacheIRStubInfo* stubInfo = stub->stubInfo();
 848   const uint8_t* stubData = stub->stubDataStart();
 849
 850   // Only create a snapshot if all opcodes are supported by the transpiler.
 851   CacheIRReader reader(stubInfo);
 852   while (reader.more()) {
 853     CacheOp op = reader.readOp();
 854     CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
 855     reader.skip(opInfo.argLength);
 856
 857     if (!opInfo.transpile) {
 858       [[maybe_unused]] unsigned line, column;
 859       LineNumberAndColumn(script_, loc, &line, &column);
 860
 861       MOZ_ASSERT(
 862           fallbackStub->trialInliningState() != TrialInliningState::Inlined,
 863           "Trial-inlined stub not supported by transpiler");
 864
 865       // Unsupported CacheIR opcode.
 866       JitSpew(JitSpew_WarpTranspiler,
 867               "unsupported CacheIR opcode %s for JSOp::%s @ %s:%u:%u",
 868               CacheIROpNames[size_t(op)], CodeName(loc.getOp()),
 869               script_->filename(), line, column);
 870       return Ok();
 871     }
 872
 873     // While on the main thread, ensure code stubs exist for ops that require
 874     // them.
 875     switch (op) {
 876       case CacheOp::CallRegExpMatcherResult:
 877         if (!cx_->realm()->jitRealm()->ensureRegExpMatcherStubExists(cx_)) {
 878           return abort(AbortReason::Error);
 879         }
 880         break;
 881       case CacheOp::CallRegExpSearcherResult:
 882         if (!cx_->realm()->jitRealm()->ensureRegExpSearcherStubExists(cx_)) {
 883           return abort(AbortReason::Error);
 884         }
 885         break;
 886       case CacheOp::RegExpBuiltinExecMatchResult:
 887         if (!cx_->realm()->jitRealm()->ensureRegExpExecMatchStubExists(cx_)) {
 888           return abort(AbortReason::Error);
 889         }
 890         break;
 891       case CacheOp::RegExpBuiltinExecTestResult:
 892         if (!cx_->realm()->jitRealm()->ensureRegExpExecTestStubExists(cx_)) {
 893           return abort(AbortReason::Error);
 894         }
 895         break;
 896       default:
 897         break;
 898     }
 899   }
 900
 901   JS::AutoAssertNoGC nogc;
 902   Zone* zone = cx_->zone();
 903   if (zone->needsIncrementalBarrier()) {
 904     TraceWeakCacheIRStub(zone->barrierTracer(), stub, stub->stubInfo());
 905   }
 906
 907   // Copy the ICStub data to protect against the stub being unlinked or mutated.
 908   // We don't need to copy the CacheIRStubInfo: because we store and trace the
 909   // stub's JitCode*, the baselineCacheIRStubCodes_ map in JitZone will keep it
 910   // alive.
 911   uint8_t* stubDataCopy = nullptr;
 912   size_t bytesNeeded = stubInfo->stubDataSize();
 913   if (bytesNeeded > 0) {
 914     stubDataCopy = alloc_.allocateArray<uint8_t>(bytesNeeded);
 915     if (!stubDataCopy) {
 916       return abort(AbortReason::Alloc);
 917     }
 918
 919     // Note: nursery pointers are handled below and the read barrier for weak
 920     // pointers is handled above so we can do a bitwise copy here.
 921     std::copy_n(stubData, bytesNeeded, stubDataCopy);
 922
 923     if (!replaceNurseryAndAllocSitePointers(stub, stubInfo, stubDataCopy)) {
 924       return abort(AbortReason::Alloc);
 925     }
 926   }
 927
 928   JitCode* jitCode = stub->jitCode();
 929
 930   if (fallbackStub->trialInliningState() == TrialInliningState::Inlined ||
 931       fallbackStub->trialInliningState() ==
 932           TrialInliningState::MonomorphicInlined) {
 933     bool inlinedCall;
 934     MOZ_TRY_VAR(inlinedCall, maybeInlineCall(snapshots, loc, stub, fallbackStub,
 935                                              stubDataCopy));
 936     if (inlinedCall) {
 937       return Ok();
 938     }
 939   }
 940
 941   if (!AddOpSnapshot<WarpCacheIR>(alloc_, snapshots, offset, jitCode, stubInfo,
 942                                   stubDataCopy)) {
 943     return abort(AbortReason::Alloc);
 944   }
 945
 946   fallbackStub->setUsedByTranspiler();
 947
 948   return Ok();
 949 }
 950
 951 AbortReasonOr<bool> WarpScriptOracle::maybeInlineCall(
 952     WarpOpSnapshotList& snapshots, BytecodeLocation loc, ICCacheIRStub* stub,
 953     ICFallbackStub* fallbackStub, uint8_t* stubDataCopy) {
 954   Maybe<InlinableOpData> inlineData = FindInlinableOpData(stub, loc);
 955   if (inlineData.isNothing()) {
 956     return false;
 957   }
 958
 959   RootedFunction targetFunction(cx_, inlineData->target);
 960   if (!TrialInliner::canInline(targetFunction, script_, loc)) {
 961     return false;
 962   }
 963
 964   bool isTrialInlined =
 965       fallbackStub->trialInliningState() == TrialInliningState::Inlined;
 966   MOZ_ASSERT_IF(!isTrialInlined, fallbackStub->trialInliningState() ==
 967                                      TrialInliningState::MonomorphicInlined);
 968
 969   RootedScript targetScript(cx_, targetFunction->nonLazyScript());
 970   ICScript* icScript = nullptr;
 971   if (isTrialInlined) {
 972     icScript = inlineData->icScript;
 973   } else {
 974     JitScript* jitScript = targetScript->jitScript();
 975     icScript = jitScript->icScript();
 976   }
 977
 978   if (!icScript) {
 979     return false;
 980   }
 981
 982   // This is just a cheap check to limit the damage we can do to ourselves if
 983   // we try to monomorphically inline an indirectly recursive call.
 984   const uint32_t maxInliningDepth = 8;
 985   if (!isTrialInlined &&
 986       info_->inlineScriptTree()->depth() > maxInliningDepth) {
 987     return false;
 988   }
 989
 990   // And this is a second cheap check to ensure monomorphic inlining doesn't
 991   // cause us to blow past our script size budget.
 992   if (oracle_->accumulatedBytecodeSize() + targetScript->length() >
 993       JitOptions.ionMaxScriptSize) {
 994     return false;
 995   }
 996
 997   // Add the inlined script to the inline script tree.
 998   LifoAlloc* lifoAlloc = alloc_.lifoAlloc();
 999   InlineScriptTree* inlineScriptTree = info_->inlineScriptTree()->addCallee(
1000       &alloc_, loc.toRawBytecode(), targetScript);
1001   if (!inlineScriptTree) {
1002     return abort(AbortReason::Alloc);
1003   }
1004
1005   // Create a CompileInfo for the inlined script.
1006   jsbytecode* osrPc = nullptr;
1007   bool needsArgsObj = targetScript->needsArgsObj();
1008   CompileInfo* info = lifoAlloc->new_<CompileInfo>(
1009       mirGen_.runtime, targetScript, targetFunction, osrPc, needsArgsObj,
1010       inlineScriptTree);
1011   if (!info) {
1012     return abort(AbortReason::Alloc);
1013   }
1014
1015   // Take a snapshot of the CacheIR.
1016   uint32_t offset = loc.bytecodeToOffset(script_);
1017   JitCode* jitCode = stub->jitCode();
1018   const CacheIRStubInfo* stubInfo = stub->stubInfo();
1019   WarpCacheIR* cacheIRSnapshot = new (alloc_.fallible())
1020       WarpCacheIR(offset, jitCode, stubInfo, stubDataCopy);
1021   if (!cacheIRSnapshot) {
1022     return abort(AbortReason::Alloc);
1023   }
1024
1025   // Read barrier for weak stub data copied into the snapshot.
1026   Zone* zone = jitCode->zone();
1027   if (zone->needsIncrementalBarrier()) {
1028     TraceWeakCacheIRStub(zone->barrierTracer(), stub, stub->stubInfo());
1029   }
1030
1031   // Take a snapshot of the inlined script (which may do more
1032   // inlining recursively).
1033   WarpScriptOracle scriptOracle(cx_, oracle_, targetScript, info, icScript);
1034
1035   AbortReasonOr<WarpScriptSnapshot*> maybeScriptSnapshot =
1036       scriptOracle.createScriptSnapshot();
1037
1038   if (maybeScriptSnapshot.isErr()) {
1039     JitSpew(JitSpew_WarpTranspiler, "Can't create snapshot for JSOp::%s",
1040             CodeName(loc.getOp()));
1041
1042     switch (maybeScriptSnapshot.unwrapErr()) {
1043       case AbortReason::Disable: {
1044         // If the target script can't be warp-compiled, mark it as
1045         // uninlineable, clean up, and fall through to the non-inlined path.
1046         ICEntry* entry = icScript_->icEntryForStub(fallbackStub);
1047         if (entry->firstStub() == stub) {
1048           fallbackStub->unlinkStub(cx_->zone(), entry, /*prev=*/nullptr, stub);
1049         }
1050         targetScript->setUninlineable();
1051         info_->inlineScriptTree()->removeCallee(inlineScriptTree);
1052         if (isTrialInlined) {
1053           icScript_->removeInlinedChild(loc.bytecodeToOffset(script_));
1054         }
1055         fallbackStub->setTrialInliningState(TrialInliningState::Failure);
1056         return false;
1057       }
1058       case AbortReason::Error:
1059       case AbortReason::Alloc:
1060         return Err(maybeScriptSnapshot.unwrapErr());
1061       default:
1062         MOZ_CRASH("Unexpected abort reason");
1063     }
1064   }
1065
1066   WarpScriptSnapshot* scriptSnapshot = maybeScriptSnapshot.unwrap();
1067   if (!isTrialInlined) {
1068     scriptSnapshot->markIsMonomorphicInlined();
1069   }
1070
1071   oracle_->addScriptSnapshot(scriptSnapshot, icScript, targetScript->length());
1072
1073   if (!AddOpSnapshot<WarpInlinedCall>(alloc_, snapshots, offset,
1074                                       cacheIRSnapshot, scriptSnapshot, info)) {
1075     return abort(AbortReason::Alloc);
1076   }
1077   fallbackStub->setUsedByTranspiler();
1078   return true;
1079 }
1080
1081 struct TypeFrequency {
1082   TypeData typeData_;
1083   uint32_t successCount_;
1084   TypeFrequency(TypeData typeData, uint32_t successCount)
1085       : typeData_(typeData), successCount_(successCount) {}
1086
1087   // Sort highest frequency first.
1088   bool operator<(const TypeFrequency& other) const {
1089     return other.successCount_ < successCount_;
1090   }
1091 };
1092
1093 AbortReasonOr<bool> WarpScriptOracle::maybeInlinePolymorphicTypes(
1094     WarpOpSnapshotList& snapshots, BytecodeLocation loc,
1095     ICCacheIRStub* firstStub, ICFallbackStub* fallbackStub) {
1096   MOZ_ASSERT(ICSupportsPolymorphicTypeData(loc.getOp()));
1097
1098   // We use polymorphic type data if there are multiple active stubs,
1099   // all of which have type data available.
1100   Vector<TypeFrequency, 6, SystemAllocPolicy> candidates;
1101   for (ICStub* stub = firstStub; !stub->isFallback();
1102        stub = stub->maybeNext()) {
1103     ICCacheIRStub* cacheIRStub = stub->toCacheIRStub();
1104     uint32_t successCount =
1105         cacheIRStub->enteredCount() - cacheIRStub->next()->enteredCount();
1106     if (successCount == 0) {
1107       continue;
1108     }
1109     TypeData types = cacheIRStub->typeData();
1110     if (!types.hasData()) {
1111       return false;
1112     }
1113     if (!candidates.append(TypeFrequency(types, successCount))) {
1114       return abort(AbortReason::Alloc);
1115     }
1116   }
1117   if (candidates.length() < 2) {
1118     return false;
1119   }
1120
1121   // Sort candidates by success frequency.
1122   std::sort(candidates.begin(), candidates.end());
1123
1124   TypeDataList list;
1125   for (auto& candidate : candidates) {
1126     list.addTypeData(candidate.typeData_);
1127   }
1128
1129   uint32_t offset = loc.bytecodeToOffset(script_);
1130   if (!AddOpSnapshot<WarpPolymorphicTypes>(alloc_, snapshots, offset, list)) {
1131     return abort(AbortReason::Alloc);
1132   }
1133
1134   return true;
1135 }
1136
1137 bool WarpScriptOracle::replaceNurseryAndAllocSitePointers(
1138     ICCacheIRStub* stub, const CacheIRStubInfo* stubInfo,
1139     uint8_t* stubDataCopy) {
1140   // If the stub data contains nursery object pointers, replace them with the
1141   // corresponding nursery index. See WarpObjectField.
1142   //
1143   // If the stub data contains allocation site pointers replace them with the
1144   // initial heap to use, because the site's state may be mutated by the main
1145   // thread while we are compiling.
1146   //
1147   // Also asserts non-object fields don't contain nursery pointers.
1148
1149   uint32_t field = 0;
1150   size_t offset = 0;
1151   while (true) {
1152     StubField::Type fieldType = stubInfo->fieldType(field);
1153     switch (fieldType) {
1154       case StubField::Type::RawInt32:
1155       case StubField::Type::RawPointer:
1156       case StubField::Type::RawInt64:
1157       case StubField::Type::Double:
1158         break;
1159       case StubField::Type::Shape:
1160       case StubField::Type::WeakShape:
1161         static_assert(std::is_convertible_v<Shape*, gc::TenuredCell*>,
1162                       "Code assumes shapes are tenured");
1163         break;
1164       case StubField::Type::GetterSetter:
1165         static_assert(std::is_convertible_v<GetterSetter*, gc::TenuredCell*>,
1166                       "Code assumes GetterSetters are tenured");
1167         break;
1168       case StubField::Type::Symbol:
1169         static_assert(std::is_convertible_v<JS::Symbol*, gc::TenuredCell*>,
1170                       "Code assumes symbols are tenured");
1171         break;
1172       case StubField::Type::BaseScript:
1173         static_assert(std::is_convertible_v<BaseScript*, gc::TenuredCell*>,
1174                       "Code assumes scripts are tenured");
1175         break;
1176       case StubField::Type::JitCode:
1177         static_assert(std::is_convertible_v<JitCode*, gc::TenuredCell*>,
1178                       "Code assumes JitCodes are tenured");
1179         break;
1180       case StubField::Type::JSObject: {
1181         JSObject* obj =
1182             stubInfo->getStubField<ICCacheIRStub, JSObject*>(stub, offset);
1183         if (IsInsideNursery(obj)) {
1184           uint32_t nurseryIndex;
1185           if (!oracle_->registerNurseryObject(obj, &nurseryIndex)) {
1186             return false;
1187           }
1188           uintptr_t oldWord = WarpObjectField::fromObject(obj).rawData();
1189           uintptr_t newWord =
1190               WarpObjectField::fromNurseryIndex(nurseryIndex).rawData();
1191           stubInfo->replaceStubRawWord(stubDataCopy, offset, oldWord, newWord);
1192         }
1193         break;
1194       }
1195       case StubField::Type::String: {
1196 #ifdef DEBUG
1197         JSString* str =
1198             stubInfo->getStubField<ICCacheIRStub, JSString*>(stub, offset);
1199         MOZ_ASSERT(!IsInsideNursery(str));
1200 #endif
1201         break;
1202       }
1203       case StubField::Type::Id: {
1204 #ifdef DEBUG
1205         // jsid never contains nursery-allocated things.
1206         jsid id = stubInfo->getStubField<ICCacheIRStub, jsid>(stub, offset);
1207         MOZ_ASSERT_IF(id.isGCThing(),
1208                       !IsInsideNursery(id.toGCCellPtr().asCell()));
1209 #endif
1210         break;
1211       }
1212       case StubField::Type::Value: {
1213 #ifdef DEBUG
1214         Value v =
1215             stubInfo->getStubField<ICCacheIRStub, JS::Value>(stub, offset);
1216         MOZ_ASSERT_IF(v.isGCThing(), !IsInsideNursery(v.toGCThing()));
1217 #endif
1218         break;
1219       }
1220       case StubField::Type::AllocSite: {
1221         uintptr_t oldWord = stubInfo->getStubRawWord(stub, offset);
1222         auto* site = reinterpret_cast<gc::AllocSite*>(oldWord);
1223         gc::Heap initialHeap = site->initialHeap();
1224         uintptr_t newWord = uintptr_t(initialHeap);
1225         stubInfo->replaceStubRawWord(stubDataCopy, offset, oldWord, newWord);
1226         break;
1227       }
1228       case StubField::Type::Limit:
1229         return true;  // Done.
1230     }
1231     field++;
1232     offset += StubField::sizeInBytes(fieldType);
1233   }
1234 }
1235
1236 bool WarpOracle::registerNurseryObject(JSObject* obj, uint32_t* nurseryIndex) {
1237   MOZ_ASSERT(IsInsideNursery(obj));
1238
1239   auto p = nurseryObjectsMap_.lookupForAdd(obj);
1240   if (p) {
1241     *nurseryIndex = p->value();
1242     return true;
1243   }
1244
1245   if (!nurseryObjects_.append(obj)) {
1246     return false;
1247   }
1248   *nurseryIndex = nurseryObjects_.length() - 1;
1249   return nurseryObjectsMap_.add(p, obj, *nurseryIndex);
1250 }