js/src/jit/WarpOracle.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "jit/WarpOracle.h"
   8
   9 #include "mozilla/ScopeExit.h"
  10
  11 #include <algorithm>
  12
  13 #include "jit/CacheIR.h"
  14 #include "jit/CacheIRCompiler.h"
  15 #include "jit/CacheIRReader.h"
  16 #include "jit/CompileInfo.h"
  17 #include "jit/InlineScriptTree.h"
  18 #include "jit/JitRealm.h"
  19 #include "jit/JitScript.h"
  20 #include "jit/JitSpewer.h"
  21 #include "jit/MIRGenerator.h"
  22 #include "jit/TrialInlining.h"
  23 #include "jit/TypeData.h"
  24 #include "jit/WarpBuilder.h"
  25 #include "util/DifferentialTesting.h"
  26 #include "vm/BuiltinObjectKind.h"
  27 #include "vm/BytecodeIterator.h"
  28 #include "vm/BytecodeLocation.h"
  29
  30 #include "jit/InlineScriptTree-inl.h"
  31 #include "vm/BytecodeIterator-inl.h"
  32 #include "vm/BytecodeLocation-inl.h"
  33 #include "vm/EnvironmentObject-inl.h"
  34 #include "vm/Interpreter-inl.h"
  35
  36 using namespace js;
  37 using namespace js::jit;
  38
  39 using mozilla::Maybe;
  40
  41 // WarpScriptOracle creates a WarpScriptSnapshot for a single JSScript. Note
  42 // that a single WarpOracle can use multiple WarpScriptOracles when scripts are
  43 // inlined.
  44 class MOZ_STACK_CLASS WarpScriptOracle {
  45   JSContext* cx_;
  46   WarpOracle* oracle_;
  47   MIRGenerator& mirGen_;
  48   TempAllocator& alloc_;
  49   HandleScript script_;
  50   const CompileInfo* info_;
  51   ICScript* icScript_;
  52
  53   // Index of the next ICEntry for getICEntry. This assumes the script's
  54   // bytecode is processed from first to last instruction.
  55   uint32_t icEntryIndex_ = 0;
  56
  57   template <typename... Args>
  58   mozilla::GenericErrorResult<AbortReason> abort(Args&&... args) {
  59     return oracle_->abort(script_, args...);
  60   }
  61
  62   WarpEnvironment createEnvironment();
  63   AbortReasonOr<Ok> maybeInlineIC(WarpOpSnapshotList& snapshots,
  64                                   BytecodeLocation loc);
  65   AbortReasonOr<bool> maybeInlineCall(WarpOpSnapshotList& snapshots,
  66                                       BytecodeLocation loc, ICCacheIRStub* stub,
  67                                       ICFallbackStub* fallbackStub,
  68                                       uint8_t* stubDataCopy);
  69   AbortReasonOr<bool> maybeInlinePolymorphicTypes(WarpOpSnapshotList& snapshots,
  70                                                   BytecodeLocation loc,
  71                                                   ICCacheIRStub* firstStub,
  72                                                   ICFallbackStub* fallbackStub);
  73   [[nodiscard]] bool replaceNurseryAndAllocSitePointers(
  74       ICCacheIRStub* stub, const CacheIRStubInfo* stubInfo,
  75       uint8_t* stubDataCopy);
  76
  77  public:
  78   WarpScriptOracle(JSContext* cx, WarpOracle* oracle, HandleScript script,
  79                    const CompileInfo* info, ICScript* icScript)
  80       : cx_(cx),
  81         oracle_(oracle),
  82         mirGen_(oracle->mirGen()),
  83         alloc_(mirGen_.alloc()),
  84         script_(script),
  85         info_(info),
  86         icScript_(icScript) {}
  87
  88   AbortReasonOr<WarpScriptSnapshot*> createScriptSnapshot();
  89
  90   ICEntry& getICEntryAndFallback(BytecodeLocation loc,
  91                                  ICFallbackStub** fallback);
  92 };
  93
  94 WarpOracle::WarpOracle(JSContext* cx, MIRGenerator& mirGen,
  95                        HandleScript outerScript)
  96     : cx_(cx),
  97       mirGen_(mirGen),
  98       alloc_(mirGen.alloc()),
  99       outerScript_(outerScript) {}
 100
 101 mozilla::GenericErrorResult<AbortReason> WarpOracle::abort(HandleScript script,
 102                                                            AbortReason r) {
 103   auto res = mirGen_.abort(r);
 104   JitSpew(JitSpew_IonAbort, "aborted @ %s", script->filename());
 105   return res;
 106 }
 107
 108 mozilla::GenericErrorResult<AbortReason> WarpOracle::abort(HandleScript script,
 109                                                            AbortReason r,
 110                                                            const char* message,
 111                                                            ...) {
 112   va_list ap;
 113   va_start(ap, message);
 114   auto res = mirGen_.abortFmt(r, message, ap);
 115   va_end(ap);
 116   JitSpew(JitSpew_IonAbort, "aborted @ %s", script->filename());
 117   return res;
 118 }
 119
 120 void WarpOracle::addScriptSnapshot(WarpScriptSnapshot* scriptSnapshot,
 121                                    ICScript* icScript, size_t bytecodeLength) {
 122   scriptSnapshots_.insertBack(scriptSnapshot);
 123   accumulatedBytecodeSize_ += bytecodeLength;
 124 #ifdef DEBUG
 125   runningScriptHash_ = mozilla::AddToHash(runningScriptHash_, icScript->hash());
 126 #endif
 127 }
 128
 129 AbortReasonOr<WarpSnapshot*> WarpOracle::createSnapshot() {
 130 #ifdef JS_JITSPEW
 131   const char* mode;
 132   if (outerScript_->hasIonScript()) {
 133     mode = "Recompiling";
 134   } else {
 135     mode = "Compiling";
 136   }
 137   JitSpew(JitSpew_IonScripts,
 138           "Warp %s script %s:%u:%u (%p) (warmup-counter=%" PRIu32 ",%s%s)",
 139           mode, outerScript_->filename(), outerScript_->lineno(),
 140           outerScript_->column(), static_cast<JSScript*>(outerScript_),
 141           outerScript_->getWarmUpCount(),
 142           outerScript_->isGenerator() ? " isGenerator" : "",
 143           outerScript_->isAsync() ? " isAsync" : "");
 144 #endif
 145
 146   accumulatedBytecodeSize_ = outerScript_->length();
 147
 148   MOZ_ASSERT(outerScript_->hasJitScript());
 149   ICScript* icScript = outerScript_->jitScript()->icScript();
 150   WarpScriptOracle scriptOracle(cx_, this, outerScript_, &mirGen_.outerInfo(),
 151                                 icScript);
 152
 153   WarpScriptSnapshot* scriptSnapshot;
 154   MOZ_TRY_VAR(scriptSnapshot, scriptOracle.createScriptSnapshot());
 155
 156   // Insert the outermost scriptSnapshot at the front of the list.
 157   scriptSnapshots_.insertFront(scriptSnapshot);
 158
 159   bool recordFinalWarmUpCount = false;
 160 #ifdef JS_CACHEIR_SPEW
 161   recordFinalWarmUpCount = outerScript_->needsFinalWarmUpCount();
 162 #endif
 163
 164   auto* snapshot = new (alloc_.fallible())
 165       WarpSnapshot(cx_, alloc_, std::move(scriptSnapshots_), bailoutInfo_,
 166                    recordFinalWarmUpCount);
 167   if (!snapshot) {
 168     return abort(outerScript_, AbortReason::Alloc);
 169   }
 170
 171   if (!snapshot->nurseryObjects().appendAll(nurseryObjects_)) {
 172     return abort(outerScript_, AbortReason::Alloc);
 173   }
 174
 175 #ifdef JS_JITSPEW
 176   if (JitSpewEnabled(JitSpew_WarpSnapshots)) {
 177     Fprinter& out = JitSpewPrinter();
 178     snapshot->dump(out);
 179   }
 180 #endif
 181
 182 #ifdef DEBUG
 183   // When transpiled CacheIR bails out, we do not want to recompile
 184   // with the exact same data and get caught in an invalidation loop.
 185   //
 186   // To avoid this, we store a hash of the stub pointers and entry
 187   // counts in this snapshot, save that hash in the JitScript if we
 188   // have a TranspiledCacheIR or MonomorphicInlinedStubFolding bailout,
 189   // and assert that the hash has changed when we recompile.
 190   //
 191   // Note: this assertion catches potential performance issues.
 192   // Failing this assertion is not a correctness/security problem.
 193   // We therefore ignore cases involving resource exhaustion (OOM,
 194   // stack overflow, etc), or stubs purged by GC.
 195   HashNumber hash = mozilla::AddToHash(icScript->hash(), runningScriptHash_);
 196   if (outerScript_->jitScript()->hasFailedICHash()) {
 197     HashNumber oldHash = outerScript_->jitScript()->getFailedICHash();
 198     MOZ_ASSERT_IF(hash == oldHash && !js::SupportDifferentialTesting(),
 199                   cx_->hadResourceExhaustion());
 200   }
 201   snapshot->setICHash(hash);
 202 #endif
 203
 204   return snapshot;
 205 }
 206
 207 template <typename T, typename... Args>
 208 [[nodiscard]] static bool AddOpSnapshot(TempAllocator& alloc,
 209                                         WarpOpSnapshotList& snapshots,
 210                                         uint32_t offset, Args&&... args) {
 211   T* snapshot = new (alloc.fallible()) T(offset, std::forward<Args>(args)...);
 212   if (!snapshot) {
 213     return false;
 214   }
 215
 216   snapshots.insertBack(snapshot);
 217   return true;
 218 }
 219
 220 [[nodiscard]] static bool AddWarpGetImport(TempAllocator& alloc,
 221                                            WarpOpSnapshotList& snapshots,
 222                                            uint32_t offset, JSScript* script,
 223                                            PropertyName* name) {
 224   ModuleEnvironmentObject* env = GetModuleEnvironmentForScript(script);
 225   MOZ_ASSERT(env);
 226
 227   mozilla::Maybe<PropertyInfo> prop;
 228   ModuleEnvironmentObject* targetEnv;
 229   MOZ_ALWAYS_TRUE(env->lookupImport(NameToId(name), &targetEnv, &prop));
 230
 231   uint32_t numFixedSlots = targetEnv->numFixedSlots();
 232   uint32_t slot = prop->slot();
 233
 234   // In the rare case where this import hasn't been initialized already (we have
 235   // an import cycle where modules reference each other's imports), we need a
 236   // check.
 237   bool needsLexicalCheck =
 238       targetEnv->getSlot(slot).isMagic(JS_UNINITIALIZED_LEXICAL);
 239
 240   return AddOpSnapshot<WarpGetImport>(alloc, snapshots, offset, targetEnv,
 241                                       numFixedSlots, slot, needsLexicalCheck);
 242 }
 243
 244 ICEntry& WarpScriptOracle::getICEntryAndFallback(BytecodeLocation loc,
 245                                                  ICFallbackStub** fallback) {
 246   const uint32_t offset = loc.bytecodeToOffset(script_);
 247
 248   do {
 249     *fallback = icScript_->fallbackStub(icEntryIndex_);
 250     icEntryIndex_++;
 251   } while ((*fallback)->pcOffset() < offset);
 252
 253   MOZ_ASSERT((*fallback)->pcOffset() == offset);
 254   return icScript_->icEntry(icEntryIndex_ - 1);
 255 }
 256
 257 WarpEnvironment WarpScriptOracle::createEnvironment() {
 258   // Don't do anything if the script doesn't use the environment chain.
 259   // Always make an environment chain if the script needs an arguments object
 260   // because ArgumentsObject construction requires the environment chain to be
 261   // passed in.
 262   if (!script_->jitScript()->usesEnvironmentChain() &&
 263       !script_->needsArgsObj()) {
 264     return WarpEnvironment(NoEnvironment());
 265   }
 266
 267   if (script_->isModule()) {
 268     ModuleObject* module = script_->module();
 269     JSObject* obj = &module->initialEnvironment();
 270     return WarpEnvironment(ConstantObjectEnvironment(obj));
 271   }
 272
 273   JSFunction* fun = script_->function();
 274   if (!fun) {
 275     // For global scripts without a non-syntactic global scope, the environment
 276     // chain is the global lexical environment.
 277     MOZ_ASSERT(!script_->isForEval());
 278     MOZ_ASSERT(!script_->hasNonSyntacticScope());
 279     JSObject* obj = &script_->global().lexicalEnvironment();
 280     return WarpEnvironment(ConstantObjectEnvironment(obj));
 281   }
 282
 283   JSObject* templateEnv = script_->jitScript()->templateEnvironment();
 284
 285   CallObject* callObjectTemplate = nullptr;
 286   if (fun->needsCallObject()) {
 287     callObjectTemplate = &templateEnv->as<CallObject>();
 288   }
 289
 290   NamedLambdaObject* namedLambdaTemplate = nullptr;
 291   if (fun->needsNamedLambdaEnvironment()) {
 292     if (callObjectTemplate) {
 293       templateEnv = templateEnv->enclosingEnvironment();
 294     }
 295     namedLambdaTemplate = &templateEnv->as<NamedLambdaObject>();
 296   }
 297
 298   return WarpEnvironment(
 299       FunctionEnvironment(callObjectTemplate, namedLambdaTemplate));
 300 }
 301
 302 AbortReasonOr<WarpScriptSnapshot*> WarpScriptOracle::createScriptSnapshot() {
 303   MOZ_ASSERT(script_->hasJitScript());
 304
 305   if (!script_->jitScript()->ensureHasCachedIonData(cx_, script_)) {
 306     return abort(AbortReason::Error);
 307   }
 308
 309   if (script_->failedBoundsCheck()) {
 310     oracle_->bailoutInfo().setFailedBoundsCheck();
 311   }
 312   if (script_->failedLexicalCheck()) {
 313     oracle_->bailoutInfo().setFailedLexicalCheck();
 314   }
 315
 316   WarpEnvironment environment = createEnvironment();
 317
 318   // Unfortunately LinkedList<> asserts the list is empty in its destructor.
 319   // Clear the list if we abort compilation.
 320   WarpOpSnapshotList opSnapshots;
 321   auto autoClearOpSnapshots =
 322       mozilla::MakeScopeExit([&] { opSnapshots.clear(); });
 323
 324   ModuleObject* moduleObject = nullptr;
 325
 326   // Analyze the bytecode. Abort compilation for unsupported ops and create
 327   // WarpOpSnapshots.
 328   for (BytecodeLocation loc : AllBytecodesIterable(script_)) {
 329     JSOp op = loc.getOp();
 330     uint32_t offset = loc.bytecodeToOffset(script_);
 331     switch (op) {
 332       case JSOp::Arguments: {
 333         MOZ_ASSERT(script_->needsArgsObj());
 334         bool mapped = script_->hasMappedArgsObj();
 335         ArgumentsObject* templateObj =
 336             script_->global().maybeArgumentsTemplateObject(mapped);
 337         if (!AddOpSnapshot<WarpArguments>(alloc_, opSnapshots, offset,
 338                                           templateObj)) {
 339           return abort(AbortReason::Alloc);
 340         }
 341         break;
 342       }
 343       case JSOp::RegExp: {
 344         bool hasShared = loc.getRegExp(script_)->hasShared();
 345         if (!AddOpSnapshot<WarpRegExp>(alloc_, opSnapshots, offset,
 346                                        hasShared)) {
 347           return abort(AbortReason::Alloc);
 348         }
 349         break;
 350       }
 351
 352       case JSOp::FunctionThis:
 353         if (!script_->strict() && script_->hasNonSyntacticScope()) {
 354           // Abort because MBoxNonStrictThis doesn't support non-syntactic
 355           // scopes (a deprecated SpiderMonkey mechanism). If this becomes an
 356           // issue we could support it by refactoring GetFunctionThis to not
 357           // take a frame pointer and then call that.
 358           return abort(AbortReason::Disable,
 359                        "JSOp::FunctionThis with non-syntactic scope");
 360         }
 361         break;
 362
 363       case JSOp::GlobalThis:
 364         MOZ_ASSERT(!script_->hasNonSyntacticScope());
 365         break;
 366
 367       case JSOp::BuiltinObject: {
 368         // If we already resolved this built-in we can bake it in.
 369         auto kind = loc.getBuiltinObjectKind();
 370         if (JSObject* proto = MaybeGetBuiltinObject(cx_->global(), kind)) {
 371           if (!AddOpSnapshot<WarpBuiltinObject>(alloc_, opSnapshots, offset,
 372                                                 proto)) {
 373             return abort(AbortReason::Alloc);
 374           }
 375         }
 376         break;
 377       }
 378
 379       case JSOp::GetIntrinsic: {
 380         // If we already cloned this intrinsic we can bake it in.
 381         // NOTE: When the initializer runs in a content global, we also have to
 382         //       worry about nursery objects. These quickly tenure and stay that
 383         //       way so this is only a temporary problem.
 384         PropertyName* name = loc.getPropertyName(script_);
 385         Value val;
 386         if (cx_->global()->maybeGetIntrinsicValue(name, &val, cx_) &&
 387             JS::GCPolicy<Value>::isTenured(val)) {
 388           if (!AddOpSnapshot<WarpGetIntrinsic>(alloc_, opSnapshots, offset,
 389                                                val)) {
 390             return abort(AbortReason::Alloc);
 391           }
 392         }
 393         break;
 394       }
 395
 396       case JSOp::ImportMeta: {
 397         if (!moduleObject) {
 398           moduleObject = GetModuleObjectForScript(script_);
 399           MOZ_ASSERT(moduleObject->isTenured());
 400         }
 401         break;
 402       }
 403
 404       case JSOp::GetImport: {
 405         PropertyName* name = loc.getPropertyName(script_);
 406         if (!AddWarpGetImport(alloc_, opSnapshots, offset, script_, name)) {
 407           return abort(AbortReason::Alloc);
 408         }
 409         break;
 410       }
 411
 412       case JSOp::Lambda: {
 413         JSFunction* fun = loc.getFunction(script_);
 414         if (IsAsmJSModule(fun)) {
 415           return abort(AbortReason::Disable, "asm.js module function lambda");
 416         }
 417         break;
 418       }
 419
 420       case JSOp::GetElemSuper: {
 421 #if defined(JS_CODEGEN_X86)
 422         // x86 does not have enough registers.
 423         return abort(AbortReason::Disable,
 424                      "GetElemSuper is not supported on x86");
 425 #else
 426         MOZ_TRY(maybeInlineIC(opSnapshots, loc));
 427         break;
 428 #endif
 429       }
 430
 431       case JSOp::Rest: {
 432         if (Shape* shape =
 433                 script_->global().maybeArrayShapeWithDefaultProto()) {
 434           if (!AddOpSnapshot<WarpRest>(alloc_, opSnapshots, offset, shape)) {
 435             return abort(AbortReason::Alloc);
 436           }
 437         }
 438         break;
 439       }
 440
 441       case JSOp::BindGName: {
 442         Rooted<GlobalObject*> global(cx_, &script_->global());
 443         Rooted<PropertyName*> name(cx_, loc.getPropertyName(script_));
 444         if (JSObject* env = MaybeOptimizeBindGlobalName(cx_, global, name)) {
 445           MOZ_ASSERT(env->isTenured());
 446           if (!AddOpSnapshot<WarpBindGName>(alloc_, opSnapshots, offset, env)) {
 447             return abort(AbortReason::Alloc);
 448           }
 449         } else {
 450           MOZ_TRY(maybeInlineIC(opSnapshots, loc));
 451         }
 452         break;
 453       }
 454
 455       case JSOp::PushVarEnv: {
 456         Rooted<VarScope*> scope(cx_, &loc.getScope(script_)->as<VarScope>());
 457
 458         auto* templateObj =
 459             VarEnvironmentObject::createTemplateObject(cx_, scope);
 460         if (!templateObj) {
 461           return abort(AbortReason::Alloc);
 462         }
 463         MOZ_ASSERT(templateObj->isTenured());
 464
 465         if (!AddOpSnapshot<WarpVarEnvironment>(alloc_, opSnapshots, offset,
 466                                                templateObj)) {
 467           return abort(AbortReason::Alloc);
 468         }
 469         break;
 470       }
 471
 472       case JSOp::PushLexicalEnv:
 473       case JSOp::FreshenLexicalEnv:
 474       case JSOp::RecreateLexicalEnv: {
 475         Rooted<LexicalScope*> scope(cx_,
 476                                     &loc.getScope(script_)->as<LexicalScope>());
 477
 478         auto* templateObj =
 479             BlockLexicalEnvironmentObject::createTemplateObject(cx_, scope);
 480         if (!templateObj) {
 481           return abort(AbortReason::Alloc);
 482         }
 483         MOZ_ASSERT(templateObj->isTenured());
 484
 485         if (!AddOpSnapshot<WarpLexicalEnvironment>(alloc_, opSnapshots, offset,
 486                                                    templateObj)) {
 487           return abort(AbortReason::Alloc);
 488         }
 489         break;
 490       }
 491
 492       case JSOp::PushClassBodyEnv: {
 493         Rooted<ClassBodyScope*> scope(
 494             cx_, &loc.getScope(script_)->as<ClassBodyScope>());
 495
 496         auto* templateObj =
 497             ClassBodyLexicalEnvironmentObject::createTemplateObject(cx_, scope);
 498         if (!templateObj) {
 499           return abort(AbortReason::Alloc);
 500         }
 501         MOZ_ASSERT(templateObj->isTenured());
 502
 503         if (!AddOpSnapshot<WarpClassBodyEnvironment>(alloc_, opSnapshots,
 504                                                      offset, templateObj)) {
 505           return abort(AbortReason::Alloc);
 506         }
 507         break;
 508       }
 509
 510       case JSOp::GetName:
 511       case JSOp::GetGName:
 512       case JSOp::GetProp:
 513       case JSOp::GetElem:
 514       case JSOp::SetProp:
 515       case JSOp::StrictSetProp:
 516       case JSOp::Call:
 517       case JSOp::CallContent:
 518       case JSOp::CallIgnoresRv:
 519       case JSOp::CallIter:
 520       case JSOp::CallContentIter:
 521       case JSOp::New:
 522       case JSOp::NewContent:
 523       case JSOp::SuperCall:
 524       case JSOp::SpreadCall:
 525       case JSOp::SpreadNew:
 526       case JSOp::SpreadSuperCall:
 527       case JSOp::ToNumeric:
 528       case JSOp::Pos:
 529       case JSOp::Inc:
 530       case JSOp::Dec:
 531       case JSOp::Neg:
 532       case JSOp::BitNot:
 533       case JSOp::Iter:
 534       case JSOp::Eq:
 535       case JSOp::Ne:
 536       case JSOp::Lt:
 537       case JSOp::Le:
 538       case JSOp::Gt:
 539       case JSOp::Ge:
 540       case JSOp::StrictEq:
 541       case JSOp::StrictNe:
 542       case JSOp::BindName:
 543       case JSOp::Add:
 544       case JSOp::Sub:
 545       case JSOp::Mul:
 546       case JSOp::Div:
 547       case JSOp::Mod:
 548       case JSOp::Pow:
 549       case JSOp::BitAnd:
 550       case JSOp::BitOr:
 551       case JSOp::BitXor:
 552       case JSOp::Lsh:
 553       case JSOp::Rsh:
 554       case JSOp::Ursh:
 555       case JSOp::In:
 556       case JSOp::HasOwn:
 557       case JSOp::CheckPrivateField:
 558       case JSOp::Instanceof:
 559       case JSOp::GetPropSuper:
 560       case JSOp::InitProp:
 561       case JSOp::InitLockedProp:
 562       case JSOp::InitHiddenProp:
 563       case JSOp::InitElem:
 564       case JSOp::InitHiddenElem:
 565       case JSOp::InitLockedElem:
 566       case JSOp::InitElemInc:
 567       case JSOp::SetName:
 568       case JSOp::StrictSetName:
 569       case JSOp::SetGName:
 570       case JSOp::StrictSetGName:
 571       case JSOp::InitGLexical:
 572       case JSOp::SetElem:
 573       case JSOp::StrictSetElem:
 574       case JSOp::ToPropertyKey:
 575       case JSOp::OptimizeSpreadCall:
 576       case JSOp::Typeof:
 577       case JSOp::TypeofExpr:
 578       case JSOp::NewObject:
 579       case JSOp::NewInit:
 580       case JSOp::NewArray:
 581       case JSOp::JumpIfFalse:
 582       case JSOp::JumpIfTrue:
 583       case JSOp::And:
 584       case JSOp::Or:
 585       case JSOp::Not:
 586       case JSOp::CloseIter:
 587         MOZ_TRY(maybeInlineIC(opSnapshots, loc));
 588         break;
 589
 590       case JSOp::Nop:
 591       case JSOp::NopDestructuring:
 592       case JSOp::TryDestructuring:
 593       case JSOp::Lineno:
 594       case JSOp::DebugLeaveLexicalEnv:
 595       case JSOp::Undefined:
 596       case JSOp::Void:
 597       case JSOp::Null:
 598       case JSOp::Hole:
 599       case JSOp::Uninitialized:
 600       case JSOp::IsConstructing:
 601       case JSOp::False:
 602       case JSOp::True:
 603       case JSOp::Zero:
 604       case JSOp::One:
 605       case JSOp::Int8:
 606       case JSOp::Uint16:
 607       case JSOp::Uint24:
 608       case JSOp::Int32:
 609       case JSOp::Double:
 610       case JSOp::BigInt:
 611       case JSOp::String:
 612       case JSOp::Symbol:
 613       case JSOp::Pop:
 614       case JSOp::PopN:
 615       case JSOp::Dup:
 616       case JSOp::Dup2:
 617       case JSOp::DupAt:
 618       case JSOp::Swap:
 619       case JSOp::Pick:
 620       case JSOp::Unpick:
 621       case JSOp::GetLocal:
 622       case JSOp::SetLocal:
 623       case JSOp::InitLexical:
 624       case JSOp::GetArg:
 625       case JSOp::GetFrameArg:
 626       case JSOp::SetArg:
 627       case JSOp::ArgumentsLength:
 628       case JSOp::GetActualArg:
 629       case JSOp::JumpTarget:
 630       case JSOp::LoopHead:
 631       case JSOp::Case:
 632       case JSOp::Default:
 633       case JSOp::Coalesce:
 634       case JSOp::Goto:
 635       case JSOp::DebugCheckSelfHosted:
 636       case JSOp::DynamicImport:
 637       case JSOp::ToString:
 638       case JSOp::GlobalOrEvalDeclInstantiation:
 639       case JSOp::BindVar:
 640       case JSOp::MutateProto:
 641       case JSOp::Callee:
 642       case JSOp::ToAsyncIter:
 643       case JSOp::ObjWithProto:
 644       case JSOp::GetAliasedVar:
 645       case JSOp::SetAliasedVar:
 646       case JSOp::InitAliasedLexical:
 647       case JSOp::EnvCallee:
 648       case JSOp::MoreIter:
 649       case JSOp::EndIter:
 650       case JSOp::IsNoIter:
 651       case JSOp::IsNullOrUndefined:
 652       case JSOp::DelProp:
 653       case JSOp::StrictDelProp:
 654       case JSOp::DelElem:
 655       case JSOp::StrictDelElem:
 656       case JSOp::SetFunName:
 657       case JSOp::PopLexicalEnv:
 658       case JSOp::ImplicitThis:
 659       case JSOp::CheckClassHeritage:
 660       case JSOp::CheckThis:
 661       case JSOp::CheckThisReinit:
 662       case JSOp::Generator:
 663       case JSOp::AfterYield:
 664       case JSOp::FinalYieldRval:
 665       case JSOp::AsyncResolve:
 666       case JSOp::CheckResumeKind:
 667       case JSOp::CanSkipAwait:
 668       case JSOp::MaybeExtractAwaitValue:
 669       case JSOp::AsyncAwait:
 670       case JSOp::Await:
 671       case JSOp::CheckReturn:
 672       case JSOp::CheckLexical:
 673       case JSOp::CheckAliasedLexical:
 674       case JSOp::InitHomeObject:
 675       case JSOp::SuperBase:
 676       case JSOp::SuperFun:
 677       case JSOp::InitElemArray:
 678       case JSOp::InitPropGetter:
 679       case JSOp::InitPropSetter:
 680       case JSOp::InitHiddenPropGetter:
 681       case JSOp::InitHiddenPropSetter:
 682       case JSOp::InitElemGetter:
 683       case JSOp::InitElemSetter:
 684       case JSOp::InitHiddenElemGetter:
 685       case JSOp::InitHiddenElemSetter:
 686       case JSOp::NewTarget:
 687       case JSOp::Object:
 688       case JSOp::CallSiteObj:
 689       case JSOp::CheckIsObj:
 690       case JSOp::CheckObjCoercible:
 691       case JSOp::FunWithProto:
 692       case JSOp::Debugger:
 693       case JSOp::TableSwitch:
 694       case JSOp::Exception:
 695       case JSOp::Throw:
 696       case JSOp::ThrowSetConst:
 697       case JSOp::SetRval:
 698       case JSOp::GetRval:
 699       case JSOp::Return:
 700       case JSOp::RetRval:
 701       case JSOp::InitialYield:
 702       case JSOp::Yield:
 703       case JSOp::ResumeKind:
 704       case JSOp::ThrowMsg:
 705       case JSOp::Try:
 706       case JSOp::Finally:
 707       case JSOp::NewPrivateName:
 708         // Supported by WarpBuilder. Nothing to do.
 709         break;
 710
 711         // Unsupported ops. Don't use a 'default' here, we want to trigger a
 712         // compiler warning when adding a new JSOp.
 713 #define DEF_CASE(OP) case JSOp::OP:
 714         WARP_UNSUPPORTED_OPCODE_LIST(DEF_CASE)
 715 #undef DEF_CASE
 716 #ifdef DEBUG
 717         return abort(AbortReason::Disable, "Unsupported opcode: %s",
 718                      CodeName(op));
 719 #else
 720         return abort(AbortReason::Disable, "Unsupported opcode: %u",
 721                      uint8_t(op));
 722 #endif
 723     }
 724   }
 725
 726   auto* scriptSnapshot = new (alloc_.fallible()) WarpScriptSnapshot(
 727       script_, environment, std::move(opSnapshots), moduleObject);
 728   if (!scriptSnapshot) {
 729     return abort(AbortReason::Alloc);
 730   }
 731
 732   autoClearOpSnapshots.release();
 733   return scriptSnapshot;
 734 }
 735
 736 static void LineNumberAndColumn(HandleScript script, BytecodeLocation loc,
 737                                 unsigned* line, unsigned* column) {
 738 #ifdef DEBUG
 739   *line = PCToLineNumber(script, loc.toRawBytecode(), column);
 740 #else
 741   *line = script->lineno();
 742   *column = script->column();
 743 #endif
 744 }
 745
 746 AbortReasonOr<Ok> WarpScriptOracle::maybeInlineIC(WarpOpSnapshotList& snapshots,
 747                                                   BytecodeLocation loc) {
 748   // Do one of the following:
 749   //
 750   // * If the Baseline IC has a single ICStub we can inline, add a WarpCacheIR
 751   //   snapshot to transpile it to MIR.
 752   //
 753   // * If that single ICStub is a call IC with a known target, instead add a
 754   //   WarpInline snapshot to transpile the guards to MIR and inline the target.
 755   //
 756   // * If the Baseline IC is cold (never executed), add a WarpBailout snapshot
 757   //   so that we can collect information in Baseline.
 758   //
 759   // * Else, don't add a snapshot and rely on WarpBuilder adding an Ion IC.
 760
 761   MOZ_ASSERT(loc.opHasIC());
 762
 763   // Don't create snapshots when testing ICs.
 764   if (JitOptions.forceInlineCaches) {
 765     return Ok();
 766   }
 767
 768   ICFallbackStub* fallbackStub;
 769   const ICEntry& entry = getICEntryAndFallback(loc, &fallbackStub);
 770   ICStub* firstStub = entry.firstStub();
 771
 772   uint32_t offset = loc.bytecodeToOffset(script_);
 773
 774   // Clear the used-by-transpiler flag on the IC. It can still be set from a
 775   // previous compilation because we don't clear the flag on every IC when
 776   // invalidating.
 777   fallbackStub->clearUsedByTranspiler();
 778
 779   if (firstStub == fallbackStub) {
 780     [[maybe_unused]] unsigned line, column;
 781     LineNumberAndColumn(script_, loc, &line, &column);
 782
 783     // No optimized stubs.
 784     JitSpew(JitSpew_WarpTranspiler,
 785             "fallback stub (entered-count: %" PRIu32
 786             ") for JSOp::%s @ %s:%u:%u",
 787             fallbackStub->enteredCount(), CodeName(loc.getOp()),
 788             script_->filename(), line, column);
 789
 790     // If the fallback stub was used but there's no optimized stub, use an IC.
 791     if (fallbackStub->enteredCount() != 0) {
 792       return Ok();
 793     }
 794
 795     // Cold IC. Bailout to collect information.
 796     if (!AddOpSnapshot<WarpBailout>(alloc_, snapshots, offset)) {
 797       return abort(AbortReason::Alloc);
 798     }
 799     return Ok();
 800   }
 801
 802   ICCacheIRStub* stub = firstStub->toCacheIRStub();
 803
 804   // Don't transpile if there are other stubs with entered-count > 0. Counters
 805   // are reset when a new stub is attached so this means the stub that was added
 806   // most recently didn't handle all cases.
 807   // If this code is changed, ICScript::hash may also need changing.
 808   bool firstStubHandlesAllCases = true;
 809   for (ICStub* next = stub->next(); next; next = next->maybeNext()) {
 810     if (next->enteredCount() != 0) {
 811       firstStubHandlesAllCases = false;
 812       break;
 813     }
 814   }
 815
 816   if (!firstStubHandlesAllCases) {
 817     // In some polymorphic cases, we can generate better code than the
 818     // default fallback if we know the observed types of the operands
 819     // and their relative frequency.
 820     if (ICSupportsPolymorphicTypeData(loc.getOp()) &&
 821         fallbackStub->enteredCount() == 0) {
 822       bool inlinedPolymorphicTypes = false;
 823       MOZ_TRY_VAR(
 824           inlinedPolymorphicTypes,
 825           maybeInlinePolymorphicTypes(snapshots, loc, stub, fallbackStub));
 826       if (inlinedPolymorphicTypes) {
 827         return Ok();
 828       }
 829     }
 830
 831     [[maybe_unused]] unsigned line, column;
 832     LineNumberAndColumn(script_, loc, &line, &column);
 833
 834     JitSpew(JitSpew_WarpTranspiler,
 835             "multiple active stubs for JSOp::%s @ %s:%u:%u",
 836             CodeName(loc.getOp()), script_->filename(), line, column);
 837     return Ok();
 838   }
 839
 840   const CacheIRStubInfo* stubInfo = stub->stubInfo();
 841   const uint8_t* stubData = stub->stubDataStart();
 842
 843   // Only create a snapshot if all opcodes are supported by the transpiler.
 844   CacheIRReader reader(stubInfo);
 845   while (reader.more()) {
 846     CacheOp op = reader.readOp();
 847     CacheIROpInfo opInfo = CacheIROpInfos[size_t(op)];
 848     reader.skip(opInfo.argLength);
 849
 850     if (!opInfo.transpile) {
 851       [[maybe_unused]] unsigned line, column;
 852       LineNumberAndColumn(script_, loc, &line, &column);
 853
 854       MOZ_ASSERT(
 855           fallbackStub->trialInliningState() != TrialInliningState::Inlined,
 856           "Trial-inlined stub not supported by transpiler");
 857
 858       // Unsupported CacheIR opcode.
 859       JitSpew(JitSpew_WarpTranspiler,
 860               "unsupported CacheIR opcode %s for JSOp::%s @ %s:%u:%u",
 861               CacheIROpNames[size_t(op)], CodeName(loc.getOp()),
 862               script_->filename(), line, column);
 863       return Ok();
 864     }
 865
 866     // While on the main thread, ensure code stubs exist for ops that require
 867     // them.
 868     switch (op) {
 869       case CacheOp::CallRegExpMatcherResult:
 870         if (!cx_->realm()->jitRealm()->ensureRegExpMatcherStubExists(cx_)) {
 871           return abort(AbortReason::Error);
 872         }
 873         break;
 874       case CacheOp::CallRegExpSearcherResult:
 875         if (!cx_->realm()->jitRealm()->ensureRegExpSearcherStubExists(cx_)) {
 876           return abort(AbortReason::Error);
 877         }
 878         break;
 879       case CacheOp::RegExpBuiltinExecMatchResult:
 880         if (!cx_->realm()->jitRealm()->ensureRegExpExecMatchStubExists(cx_)) {
 881           return abort(AbortReason::Error);
 882         }
 883         break;
 884       case CacheOp::RegExpBuiltinExecTestResult:
 885         if (!cx_->realm()->jitRealm()->ensureRegExpExecTestStubExists(cx_)) {
 886           return abort(AbortReason::Error);
 887         }
 888         break;
 889       default:
 890         break;
 891     }
 892   }
 893
 894   // Copy the ICStub data to protect against the stub being unlinked or mutated.
 895   // We don't need to copy the CacheIRStubInfo: because we store and trace the
 896   // stub's JitCode*, the baselineCacheIRStubCodes_ map in JitZone will keep it
 897   // alive.
 898   uint8_t* stubDataCopy = nullptr;
 899   size_t bytesNeeded = stubInfo->stubDataSize();
 900   if (bytesNeeded > 0) {
 901     stubDataCopy = alloc_.allocateArray<uint8_t>(bytesNeeded);
 902     if (!stubDataCopy) {
 903       return abort(AbortReason::Alloc);
 904     }
 905
 906     // Note: nursery pointers are handled below so we don't need to trigger any
 907     // GC barriers and can do a bitwise copy.
 908     std::copy_n(stubData, bytesNeeded, stubDataCopy);
 909
 910     if (!replaceNurseryAndAllocSitePointers(stub, stubInfo, stubDataCopy)) {
 911       return abort(AbortReason::Alloc);
 912     }
 913   }
 914
 915   JitCode* jitCode = stub->jitCode();
 916
 917   if (fallbackStub->trialInliningState() == TrialInliningState::Inlined ||
 918       fallbackStub->trialInliningState() ==
 919           TrialInliningState::MonomorphicInlined) {
 920     bool inlinedCall;
 921     MOZ_TRY_VAR(inlinedCall, maybeInlineCall(snapshots, loc, stub, fallbackStub,
 922                                              stubDataCopy));
 923     if (inlinedCall) {
 924       return Ok();
 925     }
 926   }
 927
 928   if (!AddOpSnapshot<WarpCacheIR>(alloc_, snapshots, offset, jitCode, stubInfo,
 929                                   stubDataCopy)) {
 930     return abort(AbortReason::Alloc);
 931   }
 932
 933   fallbackStub->setUsedByTranspiler();
 934
 935   return Ok();
 936 }
 937
 938 AbortReasonOr<bool> WarpScriptOracle::maybeInlineCall(
 939     WarpOpSnapshotList& snapshots, BytecodeLocation loc, ICCacheIRStub* stub,
 940     ICFallbackStub* fallbackStub, uint8_t* stubDataCopy) {
 941   Maybe<InlinableOpData> inlineData = FindInlinableOpData(stub, loc);
 942   if (inlineData.isNothing()) {
 943     return false;
 944   }
 945
 946   RootedFunction targetFunction(cx_, inlineData->target);
 947   if (!TrialInliner::canInline(targetFunction, script_, loc)) {
 948     return false;
 949   }
 950
 951   bool isTrialInlined =
 952       fallbackStub->trialInliningState() == TrialInliningState::Inlined;
 953   MOZ_ASSERT_IF(!isTrialInlined, fallbackStub->trialInliningState() ==
 954                                      TrialInliningState::MonomorphicInlined);
 955
 956   RootedScript targetScript(cx_, targetFunction->nonLazyScript());
 957   ICScript* icScript = nullptr;
 958   if (isTrialInlined) {
 959     icScript = inlineData->icScript;
 960   } else {
 961     JitScript* jitScript = targetScript->jitScript();
 962     icScript = jitScript->icScript();
 963   }
 964
 965   if (!icScript) {
 966     return false;
 967   }
 968
 969   // This is just a cheap check to limit the damage we can do to ourselves if
 970   // we try to monomorphically inline an indirectly recursive call.
 971   const uint32_t maxInliningDepth = 8;
 972   if (!isTrialInlined &&
 973       info_->inlineScriptTree()->depth() > maxInliningDepth) {
 974     return false;
 975   }
 976
 977   // And this is a second cheap check to ensure monomorphic inlining doesn't
 978   // cause us to blow past our script size budget.
 979   if (oracle_->accumulatedBytecodeSize() + targetScript->length() >
 980       JitOptions.ionMaxScriptSize) {
 981     return false;
 982   }
 983
 984   // Add the inlined script to the inline script tree.
 985   LifoAlloc* lifoAlloc = alloc_.lifoAlloc();
 986   InlineScriptTree* inlineScriptTree = info_->inlineScriptTree()->addCallee(
 987       &alloc_, loc.toRawBytecode(), targetScript);
 988   if (!inlineScriptTree) {
 989     return abort(AbortReason::Alloc);
 990   }
 991
 992   // Create a CompileInfo for the inlined script.
 993   jsbytecode* osrPc = nullptr;
 994   bool needsArgsObj = targetScript->needsArgsObj();
 995   CompileInfo* info = lifoAlloc->new_<CompileInfo>(
 996       mirGen_.runtime, targetScript, targetFunction, osrPc, needsArgsObj,
 997       inlineScriptTree);
 998   if (!info) {
 999     return abort(AbortReason::Alloc);
1000   }
1001
1002   // Take a snapshot of the CacheIR.
1003   uint32_t offset = loc.bytecodeToOffset(script_);
1004   JitCode* jitCode = stub->jitCode();
1005   const CacheIRStubInfo* stubInfo = stub->stubInfo();
1006   WarpCacheIR* cacheIRSnapshot = new (alloc_.fallible())
1007       WarpCacheIR(offset, jitCode, stubInfo, stubDataCopy);
1008   if (!cacheIRSnapshot) {
1009     return abort(AbortReason::Alloc);
1010   }
1011
1012   // Take a snapshot of the inlined script (which may do more
1013   // inlining recursively).
1014   WarpScriptOracle scriptOracle(cx_, oracle_, targetScript, info, icScript);
1015
1016   AbortReasonOr<WarpScriptSnapshot*> maybeScriptSnapshot =
1017       scriptOracle.createScriptSnapshot();
1018
1019   if (maybeScriptSnapshot.isErr()) {
1020     JitSpew(JitSpew_WarpTranspiler, "Can't create snapshot for JSOp::%s",
1021             CodeName(loc.getOp()));
1022
1023     switch (maybeScriptSnapshot.unwrapErr()) {
1024       case AbortReason::Disable: {
1025         // If the target script can't be warp-compiled, mark it as
1026         // uninlineable, clean up, and fall through to the non-inlined path.
1027         ICEntry* entry = icScript_->icEntryForStub(fallbackStub);
1028         fallbackStub->unlinkStub(cx_->zone(), entry, /*prev=*/nullptr, stub);
1029         targetScript->setUninlineable();
1030         info_->inlineScriptTree()->removeCallee(inlineScriptTree);
1031         if (isTrialInlined) {
1032           icScript_->removeInlinedChild(loc.bytecodeToOffset(script_));
1033         }
1034         fallbackStub->setTrialInliningState(TrialInliningState::Failure);
1035         return false;
1036       }
1037       case AbortReason::Error:
1038       case AbortReason::Alloc:
1039         return Err(maybeScriptSnapshot.unwrapErr());
1040       default:
1041         MOZ_CRASH("Unexpected abort reason");
1042     }
1043   }
1044
1045   WarpScriptSnapshot* scriptSnapshot = maybeScriptSnapshot.unwrap();
1046   if (!isTrialInlined) {
1047     scriptSnapshot->markIsMonomorphicInlined();
1048   }
1049
1050   oracle_->addScriptSnapshot(scriptSnapshot, icScript, targetScript->length());
1051
1052   if (!AddOpSnapshot<WarpInlinedCall>(alloc_, snapshots, offset,
1053                                       cacheIRSnapshot, scriptSnapshot, info)) {
1054     return abort(AbortReason::Alloc);
1055   }
1056   fallbackStub->setUsedByTranspiler();
1057   return true;
1058 }
1059
1060 struct TypeFrequency {
1061   TypeData typeData_;
1062   uint32_t successCount_;
1063   TypeFrequency(TypeData typeData, uint32_t successCount)
1064       : typeData_(typeData), successCount_(successCount) {}
1065
1066   // Sort highest frequency first.
1067   bool operator<(const TypeFrequency& other) const {
1068     return other.successCount_ < successCount_;
1069   }
1070 };
1071
1072 AbortReasonOr<bool> WarpScriptOracle::maybeInlinePolymorphicTypes(
1073     WarpOpSnapshotList& snapshots, BytecodeLocation loc,
1074     ICCacheIRStub* firstStub, ICFallbackStub* fallbackStub) {
1075   MOZ_ASSERT(ICSupportsPolymorphicTypeData(loc.getOp()));
1076
1077   // We use polymorphic type data if there are multiple active stubs,
1078   // all of which have type data available.
1079   Vector<TypeFrequency, 6, SystemAllocPolicy> candidates;
1080   for (ICStub* stub = firstStub; !stub->isFallback();
1081        stub = stub->maybeNext()) {
1082     ICCacheIRStub* cacheIRStub = stub->toCacheIRStub();
1083     uint32_t successCount =
1084         cacheIRStub->enteredCount() - cacheIRStub->next()->enteredCount();
1085     if (successCount == 0) {
1086       continue;
1087     }
1088     TypeData types = cacheIRStub->typeData();
1089     if (!types.hasData()) {
1090       return false;
1091     }
1092     if (!candidates.append(TypeFrequency(types, successCount))) {
1093       return abort(AbortReason::Alloc);
1094     }
1095   }
1096   if (candidates.length() < 2) {
1097     return false;
1098   }
1099
1100   // Sort candidates by success frequency.
1101   std::sort(candidates.begin(), candidates.end());
1102
1103   TypeDataList list;
1104   for (auto& candidate : candidates) {
1105     list.addTypeData(candidate.typeData_);
1106   }
1107
1108   uint32_t offset = loc.bytecodeToOffset(script_);
1109   if (!AddOpSnapshot<WarpPolymorphicTypes>(alloc_, snapshots, offset, list)) {
1110     return abort(AbortReason::Alloc);
1111   }
1112
1113   return true;
1114 }
1115
1116 bool WarpScriptOracle::replaceNurseryAndAllocSitePointers(
1117     ICCacheIRStub* stub, const CacheIRStubInfo* stubInfo,
1118     uint8_t* stubDataCopy) {
1119   // If the stub data contains nursery object pointers, replace them with the
1120   // corresponding nursery index. See WarpObjectField.
1121   //
1122   // If the stub data contains allocation site pointers replace them with the
1123   // initial heap to use, because the site's state may be mutated by the main
1124   // thread while we are compiling.
1125   //
1126   // Also asserts non-object fields don't contain nursery pointers.
1127
1128   uint32_t field = 0;
1129   size_t offset = 0;
1130   while (true) {
1131     StubField::Type fieldType = stubInfo->fieldType(field);
1132     switch (fieldType) {
1133       case StubField::Type::RawInt32:
1134       case StubField::Type::RawPointer:
1135       case StubField::Type::RawInt64:
1136       case StubField::Type::Double:
1137         break;
1138       case StubField::Type::Shape:
1139         static_assert(std::is_convertible_v<Shape*, gc::TenuredCell*>,
1140                       "Code assumes shapes are tenured");
1141         break;
1142       case StubField::Type::GetterSetter:
1143         static_assert(std::is_convertible_v<GetterSetter*, gc::TenuredCell*>,
1144                       "Code assumes GetterSetters are tenured");
1145         break;
1146       case StubField::Type::Symbol:
1147         static_assert(std::is_convertible_v<JS::Symbol*, gc::TenuredCell*>,
1148                       "Code assumes symbols are tenured");
1149         break;
1150       case StubField::Type::BaseScript:
1151         static_assert(std::is_convertible_v<BaseScript*, gc::TenuredCell*>,
1152                       "Code assumes scripts are tenured");
1153         break;
1154       case StubField::Type::JitCode:
1155         static_assert(std::is_convertible_v<JitCode*, gc::TenuredCell*>,
1156                       "Code assumes JitCodes are tenured");
1157         break;
1158       case StubField::Type::JSObject: {
1159         JSObject* obj =
1160             stubInfo->getStubField<ICCacheIRStub, JSObject*>(stub, offset);
1161         if (IsInsideNursery(obj)) {
1162           uint32_t nurseryIndex;
1163           if (!oracle_->registerNurseryObject(obj, &nurseryIndex)) {
1164             return false;
1165           }
1166           uintptr_t oldWord = WarpObjectField::fromObject(obj).rawData();
1167           uintptr_t newWord =
1168               WarpObjectField::fromNurseryIndex(nurseryIndex).rawData();
1169           stubInfo->replaceStubRawWord(stubDataCopy, offset, oldWord, newWord);
1170         }
1171         break;
1172       }
1173       case StubField::Type::String: {
1174 #ifdef DEBUG
1175         JSString* str =
1176             stubInfo->getStubField<ICCacheIRStub, JSString*>(stub, offset);
1177         MOZ_ASSERT(!IsInsideNursery(str));
1178 #endif
1179         break;
1180       }
1181       case StubField::Type::Id: {
1182 #ifdef DEBUG
1183         // jsid never contains nursery-allocated things.
1184         jsid id = stubInfo->getStubField<ICCacheIRStub, jsid>(stub, offset);
1185         MOZ_ASSERT_IF(id.isGCThing(),
1186                       !IsInsideNursery(id.toGCCellPtr().asCell()));
1187 #endif
1188         break;
1189       }
1190       case StubField::Type::Value: {
1191 #ifdef DEBUG
1192         Value v =
1193             stubInfo->getStubField<ICCacheIRStub, JS::Value>(stub, offset);
1194         MOZ_ASSERT_IF(v.isGCThing(), !IsInsideNursery(v.toGCThing()));
1195 #endif
1196         break;
1197       }
1198       case StubField::Type::AllocSite: {
1199         uintptr_t oldWord = stubInfo->getStubRawWord(stub, offset);
1200         auto* site = reinterpret_cast<gc::AllocSite*>(oldWord);
1201         gc::InitialHeap initialHeap = site->initialHeap();
1202         uintptr_t newWord = uintptr_t(initialHeap);
1203         stubInfo->replaceStubRawWord(stubDataCopy, offset, oldWord, newWord);
1204         break;
1205       }
1206       case StubField::Type::Limit:
1207         return true;  // Done.
1208     }
1209     field++;
1210     offset += StubField::sizeInBytes(fieldType);
1211   }
1212 }
1213
1214 bool WarpOracle::registerNurseryObject(JSObject* obj, uint32_t* nurseryIndex) {
1215   MOZ_ASSERT(IsInsideNursery(obj));
1216
1217   auto p = nurseryObjectsMap_.lookupForAdd(obj);
1218   if (p) {
1219     *nurseryIndex = p->value();
1220     return true;
1221   }
1222
1223   if (!nurseryObjects_.append(obj)) {
1224     return false;
1225   }
1226   *nurseryIndex = nurseryObjects_.length() - 1;
1227   return nurseryObjectsMap_.add(p, obj, *nurseryIndex);
1228 }