js/src/jit/JitScript.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "jit/JitScript-inl.h"
   8
   9 #include "mozilla/BinarySearch.h"
  10 #include "mozilla/CheckedInt.h"
  11
  12 #include <utility>
  13
  14 #include "jit/BaselineIC.h"
  15 #include "jit/BaselineJIT.h"
  16 #include "jit/BytecodeAnalysis.h"
  17 #include "jit/IonScript.h"
  18 #include "jit/JitFrames.h"
  19 #include "jit/JitSpewer.h"
  20 #include "jit/ScriptFromCalleeToken.h"
  21 #include "jit/TrialInlining.h"
  22 #include "js/ColumnNumber.h"  // JS::LimitedColumnNumberOneOrigin
  23 #include "vm/BytecodeUtil.h"
  24 #include "vm/Compartment.h"
  25 #include "vm/FrameIter.h"  // js::OnlyJSJitFrameIter
  26 #include "vm/JitActivation.h"
  27 #include "vm/JSScript.h"
  28
  29 #include "gc/GCContext-inl.h"
  30 #include "jit/JSJitFrameIter-inl.h"
  31 #include "vm/JSContext-inl.h"
  32 #include "vm/JSScript-inl.h"
  33
  34 using namespace js;
  35 using namespace js::jit;
  36
  37 using mozilla::CheckedInt;
  38
  39 JitScript::JitScript(JSScript* script, Offset fallbackStubsOffset,
  40                      Offset endOffset, const char* profileString)
  41     : profileString_(profileString),
  42       owningScript_(script),
  43       endOffset_(endOffset),
  44       icScript_(script->getWarmUpCount(),
  45                 fallbackStubsOffset - offsetOfICScript(),
  46                 endOffset - offsetOfICScript(),
  47                 /*depth=*/0, script->length()) {
  48   // Ensure the baselineScript_ and ionScript_ fields match the BaselineDisabled
  49   // and IonDisabled script flags.
  50   if (!script->canBaselineCompile()) {
  51     setBaselineScriptImpl(script, BaselineDisabledScriptPtr);
  52   }
  53   if (!script->canIonCompile()) {
  54     setIonScriptImpl(script, IonDisabledScriptPtr);
  55   }
  56 }
  57
  58 ICScript::~ICScript() {
  59   // The contents of the AllocSite LifoAlloc are removed and freed separately
  60   // after the next minor GC. See prepareForDestruction.
  61   MOZ_ASSERT(allocSitesSpace_.isEmpty());
  62 }
  63
  64 #ifdef DEBUG
  65 JitScript::~JitScript() {
  66   // BaselineScript and IonScript must have been destroyed at this point.
  67   MOZ_ASSERT(!hasBaselineScript());
  68   MOZ_ASSERT(!hasIonScript());
  69
  70   MOZ_ASSERT(!isInList());
  71 }
  72 #else
  73 JitScript::~JitScript() = default;
  74 #endif
  75
  76 bool JSScript::createJitScript(JSContext* cx) {
  77   MOZ_ASSERT(!hasJitScript());
  78   cx->check(this);
  79
  80   // Scripts with a JitScript can run in the Baseline Interpreter. Make sure
  81   // we don't create a JitScript for scripts we shouldn't Baseline interpret.
  82   MOZ_ASSERT_IF(IsBaselineInterpreterEnabled(),
  83                 CanBaselineInterpretScript(this));
  84
  85   // Store the profile string in the JitScript if the profiler is enabled.
  86   const char* profileString = nullptr;
  87   if (cx->runtime()->geckoProfiler().enabled()) {
  88     profileString = cx->runtime()->geckoProfiler().profileString(cx, this);
  89     if (!profileString) {
  90       return false;
  91     }
  92   }
  93
  94   static_assert(sizeof(JitScript) % sizeof(uintptr_t) == 0,
  95                 "Trailing arrays must be aligned properly");
  96   static_assert(sizeof(ICEntry) % sizeof(uintptr_t) == 0,
  97                 "Trailing arrays must be aligned properly");
  98
  99   static_assert(
 100       sizeof(JitScript) == offsetof(JitScript, icScript_) + sizeof(ICScript),
 101       "icScript_ must be the last field");
 102
 103   // Calculate allocation size.
 104   CheckedInt<uint32_t> allocSize = sizeof(JitScript);
 105   allocSize += CheckedInt<uint32_t>(numICEntries()) * sizeof(ICEntry);
 106   allocSize += CheckedInt<uint32_t>(numICEntries()) * sizeof(ICFallbackStub);
 107   if (!allocSize.isValid()) {
 108     ReportAllocationOverflow(cx);
 109     return false;
 110   }
 111
 112   void* raw = cx->pod_malloc<uint8_t>(allocSize.value());
 113   MOZ_ASSERT(uintptr_t(raw) % alignof(JitScript) == 0);
 114   if (!raw) {
 115     return false;
 116   }
 117
 118   size_t fallbackStubsOffset =
 119       sizeof(JitScript) + numICEntries() * sizeof(ICEntry);
 120
 121   UniquePtr<JitScript> jitScript(new (raw) JitScript(
 122       this, fallbackStubsOffset, allocSize.value(), profileString));
 123
 124   // Sanity check the length computation.
 125   MOZ_ASSERT(jitScript->numICEntries() == numICEntries());
 126
 127   jitScript->icScript()->initICEntries(cx, this);
 128
 129   cx->zone()->jitZone()->registerJitScript(jitScript.get());
 130
 131   warmUpData_.initJitScript(jitScript.release());
 132   AddCellMemory(this, allocSize.value(), MemoryUse::JitScript);
 133
 134   // We have a JitScript so we can set the script's jitCodeRaw pointer to the
 135   // Baseline Interpreter code.
 136   updateJitCodeRaw(cx->runtime());
 137
 138   return true;
 139 }
 140
 141 void JSScript::maybeReleaseJitScript(JS::GCContext* gcx) {
 142   MOZ_ASSERT(hasJitScript());
 143
 144   if (zone()->jitZone()->keepJitScripts() || jitScript()->hasBaselineScript() ||
 145       jitScript()->icScript()->active()) {
 146     return;
 147   }
 148
 149   releaseJitScript(gcx);
 150 }
 151
 152 void JSScript::releaseJitScript(JS::GCContext* gcx) {
 153   MOZ_ASSERT(hasJitScript());
 154   MOZ_ASSERT(!hasBaselineScript());
 155   MOZ_ASSERT(!hasIonScript());
 156
 157   gcx->removeCellMemory(this, jitScript()->allocBytes(), MemoryUse::JitScript);
 158
 159   JitScript::Destroy(zone(), jitScript());
 160   warmUpData_.clearJitScript();
 161   updateJitCodeRaw(gcx->runtime());
 162 }
 163
 164 void JSScript::releaseJitScriptOnFinalize(JS::GCContext* gcx) {
 165   MOZ_ASSERT(hasJitScript());
 166
 167   if (hasIonScript()) {
 168     IonScript* ion = jitScript()->clearIonScript(gcx, this);
 169     jit::IonScript::Destroy(gcx, ion);
 170   }
 171
 172   if (hasBaselineScript()) {
 173     BaselineScript* baseline = jitScript()->clearBaselineScript(gcx, this);
 174     jit::BaselineScript::Destroy(gcx, baseline);
 175   }
 176
 177   releaseJitScript(gcx);
 178 }
 179
 180 void JitScript::trace(JSTracer* trc) {
 181   TraceEdge(trc, &owningScript_, "JitScript::owningScript_");
 182
 183   icScript_.trace(trc);
 184
 185   if (hasBaselineScript()) {
 186     baselineScript()->trace(trc);
 187   }
 188
 189   if (hasIonScript()) {
 190     ionScript()->trace(trc);
 191   }
 192
 193   if (templateEnv_.isSome()) {
 194     TraceNullableEdge(trc, templateEnv_.ptr(), "jitscript-template-env");
 195   }
 196
 197   if (hasInliningRoot()) {
 198     inliningRoot()->trace(trc);
 199   }
 200 }
 201
 202 void JitScript::traceWeak(JSTracer* trc) {
 203   if (!icScript_.traceWeak(trc)) {
 204     notePurgedStubs();
 205   }
 206
 207   if (hasInliningRoot()) {
 208     if (!inliningRoot()->traceWeak(trc)) {
 209       notePurgedStubs();
 210     }
 211   }
 212
 213   if (hasIonScript()) {
 214     ionScript()->traceWeak(trc);
 215   }
 216 }
 217
 218 void ICScript::trace(JSTracer* trc) {
 219   // Mark all IC stub codes hanging off the IC stub entries.
 220   for (size_t i = 0; i < numICEntries(); i++) {
 221     ICEntry& ent = icEntry(i);
 222     ent.trace(trc);
 223   }
 224
 225   for (gc::AllocSite* site : allocSites_) {
 226     site->trace(trc);
 227   }
 228 }
 229
 230 bool ICScript::traceWeak(JSTracer* trc) {
 231   // Mark all IC stub codes hanging off the IC stub entries.
 232   bool allSurvived = true;
 233   for (size_t i = 0; i < numICEntries(); i++) {
 234     ICEntry& ent = icEntry(i);
 235     if (!ent.traceWeak(trc)) {
 236       allSurvived = false;
 237     }
 238   }
 239
 240   return allSurvived;
 241 }
 242
 243 bool ICScript::addInlinedChild(JSContext* cx, UniquePtr<ICScript> child,
 244                                uint32_t pcOffset) {
 245   MOZ_ASSERT(!hasInlinedChild(pcOffset));
 246
 247   if (!inlinedChildren_) {
 248     inlinedChildren_ = cx->make_unique<Vector<CallSite>>(cx);
 249     if (!inlinedChildren_) {
 250       return false;
 251     }
 252   }
 253
 254   // First reserve space in inlinedChildren_ to ensure that if the ICScript is
 255   // added to the inlining root, it can also be added to inlinedChildren_.
 256   CallSite callsite(child.get(), pcOffset);
 257   if (!inlinedChildren_->reserve(inlinedChildren_->length() + 1)) {
 258     return false;
 259   }
 260   if (!inliningRoot()->addInlinedScript(std::move(child))) {
 261     return false;
 262   }
 263   inlinedChildren_->infallibleAppend(callsite);
 264   return true;
 265 }
 266
 267 ICScript* ICScript::findInlinedChild(uint32_t pcOffset) {
 268   for (auto& callsite : *inlinedChildren_) {
 269     if (callsite.pcOffset_ == pcOffset) {
 270       return callsite.callee_;
 271     }
 272   }
 273   MOZ_CRASH("Inlined child expected at pcOffset");
 274 }
 275
 276 void ICScript::removeInlinedChild(uint32_t pcOffset) {
 277   MOZ_ASSERT(inliningRoot());
 278   inlinedChildren_->eraseIf([pcOffset](const CallSite& callsite) -> bool {
 279     return callsite.pcOffset_ == pcOffset;
 280   });
 281 }
 282
 283 bool ICScript::hasInlinedChild(uint32_t pcOffset) {
 284   if (!inlinedChildren_) {
 285     return false;
 286   }
 287   for (auto& callsite : *inlinedChildren_) {
 288     if (callsite.pcOffset_ == pcOffset) {
 289       return true;
 290     }
 291   }
 292   return false;
 293 }
 294
 295 void ICScript::purgeInactiveICScripts() {
 296   MOZ_ASSERT(inliningRoot());
 297
 298   if (!inlinedChildren_) {
 299     return;
 300   }
 301
 302   inlinedChildren_->eraseIf(
 303       [](const CallSite& callsite) { return !callsite.callee_->active(); });
 304
 305   if (inlinedChildren_->empty()) {
 306     inlinedChildren_.reset();
 307     return;
 308   }
 309
 310   // We have an active callee ICScript. This means the current ICScript must be
 311   // active too.
 312   MOZ_ASSERT(active());
 313 }
 314
 315 void JitScript::resetWarmUpCount(uint32_t count) {
 316   forEachICScript([&](ICScript* script) { script->resetWarmUpCount(count); });
 317 }
 318
 319 #ifdef DEBUG
 320 bool JitScript::hasActiveICScript() const {
 321   bool hasActive = false;
 322   forEachICScript([&](const ICScript* script) {
 323     if (script->active()) {
 324       hasActive = true;
 325     }
 326   });
 327   return hasActive;
 328 }
 329 #endif
 330
 331 void JitScript::resetAllActiveFlags() {
 332   forEachICScript([](ICScript* script) { script->resetActive(); });
 333 }
 334
 335 void JitScript::ensureProfileString(JSContext* cx, JSScript* script) {
 336   MOZ_ASSERT(cx->runtime()->geckoProfiler().enabled());
 337
 338   if (profileString_) {
 339     return;
 340   }
 341
 342   AutoEnterOOMUnsafeRegion oomUnsafe;
 343   profileString_ = cx->runtime()->geckoProfiler().profileString(cx, script);
 344   if (!profileString_) {
 345     oomUnsafe.crash("Failed to allocate profile string");
 346   }
 347 }
 348
 349 /* static */
 350 void JitScript::Destroy(Zone* zone, JitScript* script) {
 351   script->prepareForDestruction(zone);
 352
 353   // Remove from JitZone's linked list of JitScripts.
 354   script->remove();
 355
 356   js_delete(script);
 357 }
 358
 359 template <typename F>
 360 void JitScript::forEachICScript(const F& f) {
 361   f(&icScript_);
 362   if (hasInliningRoot()) {
 363     inliningRoot()->forEachInlinedScript(f);
 364   }
 365 }
 366
 367 template <typename F>
 368 void JitScript::forEachICScript(const F& f) const {
 369   f(&icScript_);
 370   if (hasInliningRoot()) {
 371     inliningRoot()->forEachInlinedScript(f);
 372   }
 373 }
 374
 375 void ICScript::prepareForDestruction(Zone* zone) {
 376   // Defer freeing AllocSite memory until after the next minor GC, because the
 377   // nursery can point to these alloc sites.
 378   JSRuntime* rt = zone->runtimeFromMainThread();
 379   rt->gc.queueAllLifoBlocksForFreeAfterMinorGC(&allocSitesSpace_);
 380
 381   // Trigger write barriers.
 382   PreWriteBarrier(zone, this);
 383 }
 384
 385 void JitScript::prepareForDestruction(Zone* zone) {
 386   forEachICScript(
 387       [&](ICScript* script) { script->prepareForDestruction(zone); });
 388
 389   // Trigger write barriers.
 390   owningScript_ = nullptr;
 391   baselineScript_.set(zone, nullptr);
 392   ionScript_.set(zone, nullptr);
 393 }
 394
 395 struct FallbackStubs {
 396   ICScript* const icScript_;
 397
 398   explicit FallbackStubs(ICScript* icScript) : icScript_(icScript) {}
 399
 400   size_t numEntries() const { return icScript_->numICEntries(); }
 401   ICFallbackStub* operator[](size_t index) const {
 402     return icScript_->fallbackStub(index);
 403   }
 404 };
 405
 406 static bool ComputeBinarySearchMid(FallbackStubs stubs, uint32_t pcOffset,
 407                                    size_t* loc) {
 408   return mozilla::BinarySearchIf(
 409       stubs, 0, stubs.numEntries(),
 410       [pcOffset](const ICFallbackStub* stub) {
 411         if (pcOffset < stub->pcOffset()) {
 412           return -1;
 413         }
 414         if (stub->pcOffset() < pcOffset) {
 415           return 1;
 416         }
 417         return 0;
 418       },
 419       loc);
 420 }
 421
 422 ICEntry& ICScript::icEntryFromPCOffset(uint32_t pcOffset) {
 423   size_t mid;
 424   bool success = ComputeBinarySearchMid(FallbackStubs(this), pcOffset, &mid);
 425 #ifdef MOZ_DIAGNOSTIC_ASSERT_ENABLED
 426   if (!success) {
 427     MOZ_CRASH_UNSAFE_PRINTF("Missing icEntry for offset %d (max offset: %d)",
 428                             int(pcOffset),
 429                             int(fallbackStub(numICEntries() - 1)->pcOffset()));
 430   }
 431 #endif
 432   MOZ_ALWAYS_TRUE(success);
 433
 434   MOZ_ASSERT(mid < numICEntries());
 435
 436   ICEntry& entry = icEntry(mid);
 437   MOZ_ASSERT(fallbackStubForICEntry(&entry)->pcOffset() == pcOffset);
 438   return entry;
 439 }
 440
 441 ICEntry* ICScript::interpreterICEntryFromPCOffset(uint32_t pcOffset) {
 442   // We have to return the entry to store in BaselineFrame::interpreterICEntry
 443   // when resuming in the Baseline Interpreter at pcOffset. The bytecode op at
 444   // pcOffset does not necessarily have an ICEntry, so we want to return the
 445   // first ICEntry for which the following is true:
 446   //
 447   //    entry.pcOffset() >= pcOffset
 448   //
 449   // Fortunately, ComputeBinarySearchMid returns exactly this entry.
 450
 451   size_t mid;
 452   ComputeBinarySearchMid(FallbackStubs(this), pcOffset, &mid);
 453
 454   if (mid < numICEntries()) {
 455     ICEntry& entry = icEntry(mid);
 456     MOZ_ASSERT(fallbackStubForICEntry(&entry)->pcOffset() >= pcOffset);
 457     return &entry;
 458   }
 459
 460   // Resuming at a pc after the last ICEntry. Just return nullptr:
 461   // BaselineFrame::interpreterICEntry will never be used in this case.
 462   return nullptr;
 463 }
 464
 465 void JitScript::purgeInactiveICScripts() {
 466   if (!hasInliningRoot()) {
 467     return;
 468   }
 469
 470   forEachICScript([](ICScript* script) { script->purgeInactiveICScripts(); });
 471
 472   inliningRoot()->purgeInactiveICScripts();
 473   if (inliningRoot()->numInlinedScripts() == 0) {
 474     inliningRoot_.reset();
 475     icScript()->inliningRoot_ = nullptr;
 476   } else {
 477     // If a callee script is active on the stack, the root script must be active
 478     // too.
 479     MOZ_ASSERT(icScript()->active());
 480   }
 481 }
 482
 483 void JitScript::purgeStubs(JSScript* script, ICStubSpace& newStubSpace) {
 484   MOZ_ASSERT(script->jitScript() == this);
 485
 486   Zone* zone = script->zone();
 487   if (IsAboutToBeFinalizedUnbarriered(script)) {
 488     // We're sweeping and the script is dead. Don't purge optimized stubs
 489     // because (1) accessing CacheIRStubInfo pointers in ICStubs is invalid
 490     // because we may have swept them already when we started (incremental)
 491     // sweeping and (2) it's unnecessary because this script will be finalized
 492     // soon anyway.
 493     return;
 494   }
 495
 496   JitSpew(JitSpew_BaselineIC, "Purging optimized stubs");
 497
 498   forEachICScript(
 499       [&](ICScript* script) { script->purgeStubs(zone, newStubSpace); });
 500
 501   notePurgedStubs();
 502 }
 503
 504 void ICScript::purgeStubs(Zone* zone, ICStubSpace& newStubSpace) {
 505   for (size_t i = 0; i < numICEntries(); i++) {
 506     ICEntry& entry = icEntry(i);
 507     ICFallbackStub* fallback = fallbackStub(i);
 508
 509     // If this is a trial inlining call site and the callee's ICScript hasn't
 510     // been discarded, clone the IC chain instead of purging stubs. In this case
 511     // both the current ICScript and the callee's inlined ICScript must be
 512     // active on the stack.
 513     //
 514     // We can't purge the IC stubs in this case because it'd confuse trial
 515     // inlining if we try to inline again later and we already have an ICScript
 516     // for this call site.
 517     if (fallback->trialInliningState() == TrialInliningState::Inlined &&
 518         hasInlinedChild(fallback->pcOffset())) {
 519       MOZ_ASSERT(active());
 520 #ifdef DEBUG
 521       // The callee script must be active. Also assert its bytecode size field
 522       // is valid, because this helps catch memory safety issues (bug 1871947).
 523       ICScript* callee = findInlinedChild(fallback->pcOffset());
 524       MOZ_ASSERT(callee->active());
 525       MOZ_ASSERT(callee->bytecodeSize() < inliningRoot()->totalBytecodeSize());
 526 #endif
 527
 528       JSRuntime* rt = zone->runtimeFromMainThread();
 529       ICCacheIRStub* prev = nullptr;
 530       ICStub* stub = entry.firstStub();
 531       while (stub != fallback) {
 532         ICCacheIRStub* clone = stub->toCacheIRStub()->clone(rt, newStubSpace);
 533         if (prev) {
 534           prev->setNext(clone);
 535         } else {
 536           entry.setFirstStub(clone);
 537         }
 538         MOZ_ASSERT(stub->toCacheIRStub()->next() == clone->next());
 539         prev = clone;
 540         stub = clone->next();
 541       }
 542       continue;
 543     }
 544
 545     MOZ_ASSERT(!hasInlinedChild(fallback->pcOffset()));
 546
 547     fallback->discardStubs(zone, &entry);
 548     fallback->state().reset();
 549   }
 550 }
 551
 552 bool JitScript::ensureHasCachedBaselineJitData(JSContext* cx,
 553                                                HandleScript script) {
 554   if (templateEnv_.isSome()) {
 555     return true;
 556   }
 557
 558   if (!script->function() ||
 559       !script->function()->needsFunctionEnvironmentObjects()) {
 560     templateEnv_.emplace();
 561     return true;
 562   }
 563
 564   Rooted<EnvironmentObject*> templateEnv(cx);
 565   Rooted<JSFunction*> fun(cx, script->function());
 566
 567   if (fun->needsNamedLambdaEnvironment()) {
 568     templateEnv = NamedLambdaObject::createTemplateObject(cx, fun);
 569     if (!templateEnv) {
 570       return false;
 571     }
 572   }
 573
 574   if (fun->needsCallObject()) {
 575     templateEnv = CallObject::createTemplateObject(cx, script, templateEnv);
 576     if (!templateEnv) {
 577       return false;
 578     }
 579   }
 580
 581   templateEnv_.emplace(templateEnv);
 582   return true;
 583 }
 584
 585 bool JitScript::ensureHasCachedIonData(JSContext* cx, HandleScript script) {
 586   MOZ_ASSERT(script->jitScript() == this);
 587
 588   if (usesEnvironmentChain_.isSome()) {
 589     return true;
 590   }
 591
 592   if (!ensureHasCachedBaselineJitData(cx, script)) {
 593     return false;
 594   }
 595
 596   usesEnvironmentChain_.emplace(ScriptUsesEnvironmentChain(script));
 597   return true;
 598 }
 599
 600 void JitScript::setBaselineScriptImpl(JSScript* script,
 601                                       BaselineScript* baselineScript) {
 602   JSRuntime* rt = script->runtimeFromMainThread();
 603   setBaselineScriptImpl(rt->gcContext(), script, baselineScript);
 604 }
 605
 606 void JitScript::setBaselineScriptImpl(JS::GCContext* gcx, JSScript* script,
 607                                       BaselineScript* baselineScript) {
 608   if (hasBaselineScript()) {
 609     gcx->removeCellMemory(script, baselineScript_->allocBytes(),
 610                           MemoryUse::BaselineScript);
 611     baselineScript_.set(script->zone(), nullptr);
 612   }
 613
 614   MOZ_ASSERT(ionScript_ == nullptr || ionScript_ == IonDisabledScriptPtr);
 615
 616   baselineScript_.set(script->zone(), baselineScript);
 617   if (hasBaselineScript()) {
 618     AddCellMemory(script, baselineScript_->allocBytes(),
 619                   MemoryUse::BaselineScript);
 620   }
 621
 622   script->resetWarmUpResetCounter();
 623   script->updateJitCodeRaw(gcx->runtime());
 624 }
 625
 626 void JitScript::setIonScriptImpl(JSScript* script, IonScript* ionScript) {
 627   JSRuntime* rt = script->runtimeFromMainThread();
 628   setIonScriptImpl(rt->gcContext(), script, ionScript);
 629 }
 630
 631 void JitScript::setIonScriptImpl(JS::GCContext* gcx, JSScript* script,
 632                                  IonScript* ionScript) {
 633   MOZ_ASSERT_IF(ionScript != IonDisabledScriptPtr,
 634                 !baselineScript()->hasPendingIonCompileTask());
 635
 636   JS::Zone* zone = script->zone();
 637   if (hasIonScript()) {
 638     gcx->removeCellMemory(script, ionScript_->allocBytes(),
 639                           MemoryUse::IonScript);
 640     ionScript_.set(zone, nullptr);
 641   }
 642
 643   ionScript_.set(zone, ionScript);
 644   MOZ_ASSERT_IF(hasIonScript(), hasBaselineScript());
 645   if (hasIonScript()) {
 646     AddCellMemory(script, ionScript_->allocBytes(), MemoryUse::IonScript);
 647   }
 648
 649   script->updateJitCodeRaw(gcx->runtime());
 650 }
 651
 652 #ifdef JS_STRUCTURED_SPEW
 653 static bool HasEnteredCounters(ICEntry& entry) {
 654   ICStub* stub = entry.firstStub();
 655   if (stub && !stub->isFallback()) {
 656     return true;
 657   }
 658   return false;
 659 }
 660
 661 void jit::JitSpewBaselineICStats(JSScript* script, const char* dumpReason) {
 662   MOZ_ASSERT(script->hasJitScript());
 663   JSContext* cx = TlsContext.get();
 664   AutoStructuredSpewer spew(cx, SpewChannel::BaselineICStats, script);
 665   if (!spew) {
 666     return;
 667   }
 668
 669   JitScript* jitScript = script->jitScript();
 670   spew->property("reason", dumpReason);
 671   spew->beginListProperty("entries");
 672   for (size_t i = 0; i < jitScript->numICEntries(); i++) {
 673     ICEntry& entry = jitScript->icEntry(i);
 674     ICFallbackStub* fallback = jitScript->fallbackStub(i);
 675     if (!HasEnteredCounters(entry)) {
 676       continue;
 677     }
 678
 679     uint32_t pcOffset = fallback->pcOffset();
 680     jsbytecode* pc = script->offsetToPC(pcOffset);
 681
 682     JS::LimitedColumnNumberOneOrigin column;
 683     unsigned int line = PCToLineNumber(script, pc, &column);
 684
 685     spew->beginObject();
 686     spew->property("op", CodeName(JSOp(*pc)));
 687     spew->property("pc", pcOffset);
 688     spew->property("line", line);
 689     spew->property("column", column.oneOriginValue());
 690
 691     spew->beginListProperty("counts");
 692     ICStub* stub = entry.firstStub();
 693     while (stub && !stub->isFallback()) {
 694       uint32_t count = stub->enteredCount();
 695       spew->value(count);
 696       stub = stub->toCacheIRStub()->next();
 697     }
 698     spew->endList();
 699     spew->property("fallback_count", fallback->enteredCount());
 700     spew->endObject();
 701   }
 702   spew->endList();
 703 }
 704 #endif
 705
 706 static void MarkActiveICScriptsAndCopyStubs(
 707     JSContext* cx, const JitActivationIterator& activation,
 708     ICStubSpace& newStubSpace) {
 709   for (OnlyJSJitFrameIter iter(activation); !iter.done(); ++iter) {
 710     const JSJitFrameIter& frame = iter.frame();
 711     switch (frame.type()) {
 712       case FrameType::BaselineJS:
 713         frame.script()->jitScript()->icScript()->setActive();
 714         // If the frame is using a trial-inlining ICScript, we have to preserve
 715         // it too.
 716         if (frame.baselineFrame()->icScript()->isInlined()) {
 717           frame.baselineFrame()->icScript()->setActive();
 718         }
 719         break;
 720       case FrameType::BaselineStub: {
 721         auto* layout = reinterpret_cast<BaselineStubFrameLayout*>(frame.fp());
 722         if (layout->maybeStubPtr() && !layout->maybeStubPtr()->isFallback()) {
 723           ICCacheIRStub* stub = layout->maybeStubPtr()->toCacheIRStub();
 724           ICCacheIRStub* newStub = stub->clone(cx->runtime(), newStubSpace);
 725           layout->setStubPtr(newStub);
 726
 727           // If this is a trial-inlining call site, also preserve the callee
 728           // ICScript. Inlined constructor calls invoke CreateThisFromIC (which
 729           // can trigger GC) before using the inlined ICScript.
 730           JSJitFrameIter parentFrame(frame);
 731           ++parentFrame;
 732           BaselineFrame* blFrame = parentFrame.baselineFrame();
 733           jsbytecode* pc;
 734           parentFrame.baselineScriptAndPc(nullptr, &pc);
 735           uint32_t pcOffset = blFrame->script()->pcToOffset(pc);
 736           if (blFrame->icScript()->hasInlinedChild(pcOffset)) {
 737             blFrame->icScript()->findInlinedChild(pcOffset)->setActive();
 738           }
 739         }
 740         break;
 741       }
 742       case FrameType::Exit:
 743         if (frame.exitFrame()->is<LazyLinkExitFrameLayout>()) {
 744           LazyLinkExitFrameLayout* ll =
 745               frame.exitFrame()->as<LazyLinkExitFrameLayout>();
 746           JSScript* script =
 747               ScriptFromCalleeToken(ll->jsFrame()->calleeToken());
 748           script->jitScript()->icScript()->setActive();
 749         }
 750         break;
 751       case FrameType::Bailout:
 752       case FrameType::IonJS: {
 753         // Keep the JitScript and BaselineScript around, since bailouts from
 754         // the ion jitcode need to re-enter into the Baseline code.
 755         frame.script()->jitScript()->icScript()->setActive();
 756         for (InlineFrameIterator inlineIter(cx, &frame); inlineIter.more();
 757              ++inlineIter) {
 758           inlineIter.script()->jitScript()->icScript()->setActive();
 759         }
 760         // Because we're purging ICScripts, the bailout machinery should use
 761         // the generic ICScript for inlined callees.
 762         frame.ionScript()->notePurgedICScripts();
 763         break;
 764       }
 765       default:;
 766     }
 767   }
 768 }
 769
 770 void jit::MarkActiveICScriptsAndCopyStubs(Zone* zone,
 771                                           ICStubSpace& newStubSpace) {
 772   if (zone->isAtomsZone()) {
 773     return;
 774   }
 775   JSContext* cx = TlsContext.get();
 776   for (JitActivationIterator iter(cx); !iter.done(); ++iter) {
 777     if (iter->compartment()->zone() == zone) {
 778       MarkActiveICScriptsAndCopyStubs(cx, iter, newStubSpace);
 779     }
 780   }
 781 }
 782
 783 InliningRoot* JitScript::getOrCreateInliningRoot(JSContext* cx,
 784                                                  JSScript* script) {
 785   if (!inliningRoot_) {
 786     inliningRoot_ = js::MakeUnique<InliningRoot>(cx, script);
 787     if (!inliningRoot_) {
 788       ReportOutOfMemory(cx);
 789       return nullptr;
 790     }
 791     icScript_.inliningRoot_ = inliningRoot_.get();
 792   }
 793   return inliningRoot_.get();
 794 }
 795
 796 gc::AllocSite* ICScript::getOrCreateAllocSite(JSScript* outerScript,
 797                                               uint32_t pcOffset) {
 798   // The script must be the outer script.
 799   MOZ_ASSERT(outerScript->jitScript()->icScript() == this ||
 800              (inliningRoot() && inliningRoot()->owningScript() == outerScript));
 801
 802   // The pcOffset must be for this (maybe inlined) script.
 803   MOZ_ASSERT(pcOffset < bytecodeSize());
 804
 805   for (gc::AllocSite* site : allocSites_) {
 806     if (site->pcOffset() == pcOffset) {
 807       MOZ_ASSERT(site->isNormal());
 808       MOZ_ASSERT(site->script() == outerScript);
 809       MOZ_ASSERT(site->traceKind() == JS::TraceKind::Object);
 810       return site;
 811     }
 812   }
 813
 814   Nursery& nursery = outerScript->runtimeFromMainThread()->gc.nursery();
 815   if (!nursery.canCreateAllocSite()) {
 816     // Don't block attaching an optimized stub, but don't process allocations
 817     // for this site.
 818     return outerScript->zone()->unknownAllocSite(JS::TraceKind::Object);
 819   }
 820
 821   if (!allocSites_.reserve(allocSites_.length() + 1)) {
 822     return nullptr;
 823   }
 824
 825   auto* site = allocSitesSpace_.new_<gc::AllocSite>(
 826       outerScript->zone(), outerScript, pcOffset, JS::TraceKind::Object);
 827   if (!site) {
 828     return nullptr;
 829   }
 830
 831   allocSites_.infallibleAppend(site);
 832
 833   nursery.noteAllocSiteCreated();
 834
 835   return site;
 836 }
 837
 838 bool JitScript::resetAllocSites(bool resetNurserySites,
 839                                 bool resetPretenuredSites) {
 840   MOZ_ASSERT(resetNurserySites || resetPretenuredSites);
 841
 842   bool anyReset = false;
 843
 844   forEachICScript([&](ICScript* script) {
 845     for (gc::AllocSite* site : script->allocSites_) {
 846       if ((resetNurserySites && site->initialHeap() == gc::Heap::Default) ||
 847           (resetPretenuredSites && site->initialHeap() == gc::Heap::Tenured)) {
 848         if (site->maybeResetState()) {
 849           anyReset = true;
 850         }
 851       }
 852     }
 853   });
 854
 855   return anyReset;
 856 }
 857
 858 void JitScript::addSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf,
 859                                        size_t* data, size_t* allocSites) const {
 860   *data += mallocSizeOf(this);
 861
 862   forEachICScript([=](const ICScript* script) {
 863     // |data| already includes the outer ICScript because it's part of the
 864     // JitScript.
 865     if (script != &icScript_) {
 866       *data += mallocSizeOf(script);
 867     }
 868
 869     // |data| already includes the LifoAlloc and Vector, so use
 870     // sizeOfExcludingThis.
 871     *allocSites += script->allocSitesSpace_.sizeOfExcludingThis(mallocSizeOf);
 872     *allocSites += script->allocSites_.sizeOfExcludingThis(mallocSizeOf);
 873   });
 874 }
 875
 876 JitScript* ICScript::outerJitScript() {
 877   MOZ_ASSERT(!isInlined());
 878   uint8_t* ptr = reinterpret_cast<uint8_t*>(this);
 879   return reinterpret_cast<JitScript*>(ptr - JitScript::offsetOfICScript());
 880 }
 881
 882 #ifdef DEBUG
 883 // This hash is used to verify that we do not recompile after a
 884 // TranspiledCacheIR invalidation with the exact same ICs.
 885 //
 886 // It should change iff an ICEntry in this ICScript (or an ICScript
 887 // inlined into this ICScript) is modified such that we will make a
 888 // different decision in WarpScriptOracle::maybeInlineIC. This means:
 889 //
 890 // 1. The hash will change if we attach a new stub.
 891 // 2. The hash will change if the entered count of any CacheIR stub
 892 //    other than the first changes from 0.
 893 // 3. The hash will change if the entered count of the fallback stub
 894 //    changes from 0.
 895 // 4. The hash will change if the failure count of the fallback stub
 896 //    changes from 0.
 897 HashNumber ICScript::hash() {
 898   HashNumber h = 0;
 899   for (size_t i = 0; i < numICEntries(); i++) {
 900     ICStub* stub = icEntry(i).firstStub();
 901
 902     // Hash the address of the first stub.
 903     h = mozilla::AddToHash(h, stub);
 904
 905     // Hash whether subsequent stubs have entry count 0.
 906     if (!stub->isFallback()) {
 907       stub = stub->toCacheIRStub()->next();
 908       while (!stub->isFallback()) {
 909         h = mozilla::AddToHash(h, stub->enteredCount() == 0);
 910         stub = stub->toCacheIRStub()->next();
 911       }
 912     }
 913
 914     // Hash whether the fallback has entry count 0 and failure count 0.
 915     MOZ_ASSERT(stub->isFallback());
 916     h = mozilla::AddToHash(h, stub->enteredCount() == 0);
 917     h = mozilla::AddToHash(h, stub->toFallbackStub()->state().hasFailures());
 918   }
 919
 920   return h;
 921 }
 922 #endif