js/src/vm/HelperThreads.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "vm/HelperThreads.h"
   8
   9 #include "mozilla/ReverseIterator.h"  // mozilla::Reversed(...)
  10 #include "mozilla/ScopeExit.h"
  11 #include "mozilla/Span.h"  // mozilla::Span<TaggedScriptThingIndex>
  12 #include "mozilla/Utf8.h"  // mozilla::Utf8Unit
  13
  14 #include <algorithm>
  15
  16 #include "frontend/CompilationStencil.h"  // frontend::CompilationStencil
  17 #include "gc/GC.h"
  18 #include "jit/Ion.h"
  19 #include "jit/IonCompileTask.h"
  20 #include "jit/JitRuntime.h"
  21 #include "jit/JitScript.h"
  22 #include "js/CompileOptions.h"  // JS::PrefableCompileOptions, JS::ReadOnlyCompileOptions
  23 #include "js/experimental/CompileScript.h"  // JS::ThreadStackQuotaForSize
  24 #include "js/friend/StackLimits.h"          // js::ReportOverRecursed
  25 #include "js/HelperThreadAPI.h"
  26 #include "js/Stack.h"
  27 #include "js/UniquePtr.h"
  28 #include "js/Utility.h"
  29 #include "threading/CpuCount.h"
  30 #include "vm/ErrorReporting.h"
  31 #include "vm/HelperThreadState.h"
  32 #include "vm/InternalThreadPool.h"
  33 #include "vm/MutexIDs.h"
  34 #include "wasm/WasmGenerator.h"
  35
  36 using namespace js;
  37
  38 using mozilla::TimeDuration;
  39 using mozilla::TimeStamp;
  40 using mozilla::Utf8Unit;
  41
  42 using JS::DispatchReason;
  43
  44 namespace js {
  45
  46 Mutex gHelperThreadLock(mutexid::GlobalHelperThreadState);
  47 GlobalHelperThreadState* gHelperThreadState = nullptr;
  48
  49 }  // namespace js
  50
  51 bool js::CreateHelperThreadsState() {
  52   MOZ_ASSERT(!gHelperThreadState);
  53   gHelperThreadState = js_new<GlobalHelperThreadState>();
  54   return gHelperThreadState;
  55 }
  56
  57 void js::DestroyHelperThreadsState() {
  58   AutoLockHelperThreadState lock;
  59
  60   if (!gHelperThreadState) {
  61     return;
  62   }
  63
  64   gHelperThreadState->finish(lock);
  65   js_delete(gHelperThreadState);
  66   gHelperThreadState = nullptr;
  67 }
  68
  69 bool js::EnsureHelperThreadsInitialized() {
  70   MOZ_ASSERT(gHelperThreadState);
  71   return gHelperThreadState->ensureInitialized();
  72 }
  73
  74 static size_t ClampDefaultCPUCount(size_t cpuCount) {
  75   // It's extremely rare for SpiderMonkey to have more than a few cores worth
  76   // of work. At higher core counts, performance can even decrease due to NUMA
  77   // (and SpiderMonkey's lack of NUMA-awareness), contention, and general lack
  78   // of optimization for high core counts. So to avoid wasting thread stack
  79   // resources (and cluttering gdb and core dumps), clamp to 8 cores for now.
  80   return std::min<size_t>(cpuCount, 8);
  81 }
  82
  83 static size_t ThreadCountForCPUCount(size_t cpuCount) {
  84   // We need at least two threads for tier-2 wasm compilations, because
  85   // there's a master task that holds a thread while other threads do the
  86   // compilation.
  87   return std::max<size_t>(cpuCount, 2);
  88 }
  89
  90 bool js::SetFakeCPUCount(size_t count) {
  91   HelperThreadState().setCpuCount(count);
  92   return true;
  93 }
  94
  95 void GlobalHelperThreadState::setCpuCount(size_t count) {
  96   // This must be called before any threads have been initialized.
  97   AutoLockHelperThreadState lock;
  98   MOZ_ASSERT(!isInitialized(lock));
  99
 100   // We can't do this if an external thread pool is in use.
 101   MOZ_ASSERT(!dispatchTaskCallback);
 102
 103   cpuCount = count;
 104   threadCount = ThreadCountForCPUCount(count);
 105 }
 106
 107 size_t js::GetHelperThreadCount() { return HelperThreadState().threadCount; }
 108
 109 size_t js::GetHelperThreadCPUCount() { return HelperThreadState().cpuCount; }
 110
 111 size_t js::GetMaxWasmCompilationThreads() {
 112   return HelperThreadState().maxWasmCompilationThreads();
 113 }
 114
 115 void JS::SetProfilingThreadCallbacks(
 116     JS::RegisterThreadCallback registerThread,
 117     JS::UnregisterThreadCallback unregisterThread) {
 118   HelperThreadState().registerThread = registerThread;
 119   HelperThreadState().unregisterThread = unregisterThread;
 120 }
 121
 122 // Bug 1630189: Without MOZ_NEVER_INLINE, Windows PGO builds have a linking
 123 // error for HelperThreadTaskCallback.
 124 JS_PUBLIC_API MOZ_NEVER_INLINE void JS::SetHelperThreadTaskCallback(
 125     HelperThreadTaskCallback callback, size_t threadCount, size_t stackSize) {
 126   AutoLockHelperThreadState lock;
 127   HelperThreadState().setDispatchTaskCallback(callback, threadCount, stackSize,
 128                                               lock);
 129 }
 130
 131 void GlobalHelperThreadState::setDispatchTaskCallback(
 132     JS::HelperThreadTaskCallback callback, size_t threadCount, size_t stackSize,
 133     const AutoLockHelperThreadState& lock) {
 134   MOZ_ASSERT(!isInitialized(lock));
 135   MOZ_ASSERT(!dispatchTaskCallback);
 136   MOZ_ASSERT(threadCount != 0);
 137   MOZ_ASSERT(stackSize >= 16 * 1024);
 138
 139   dispatchTaskCallback = callback;
 140   this->threadCount = threadCount;
 141   this->stackQuota = JS::ThreadStackQuotaForSize(stackSize);
 142 }
 143
 144 bool js::StartOffThreadWasmCompile(wasm::CompileTask* task,
 145                                    wasm::CompileMode mode) {
 146   return HelperThreadState().submitTask(task, mode);
 147 }
 148
 149 bool GlobalHelperThreadState::submitTask(wasm::CompileTask* task,
 150                                          wasm::CompileMode mode) {
 151   AutoLockHelperThreadState lock;
 152   if (!wasmWorklist(lock, mode).pushBack(task)) {
 153     return false;
 154   }
 155
 156   dispatch(DispatchReason::NewTask, lock);
 157   return true;
 158 }
 159
 160 size_t js::RemovePendingWasmCompileTasks(
 161     const wasm::CompileTaskState& taskState, wasm::CompileMode mode,
 162     const AutoLockHelperThreadState& lock) {
 163   wasm::CompileTaskPtrFifo& worklist =
 164       HelperThreadState().wasmWorklist(lock, mode);
 165   return worklist.eraseIf([&taskState](wasm::CompileTask* task) {
 166     return &task->state == &taskState;
 167   });
 168 }
 169
 170 void js::StartOffThreadWasmTier2Generator(wasm::UniqueTier2GeneratorTask task) {
 171   (void)HelperThreadState().submitTask(std::move(task));
 172 }
 173
 174 bool GlobalHelperThreadState::submitTask(wasm::UniqueTier2GeneratorTask task) {
 175   AutoLockHelperThreadState lock;
 176
 177   MOZ_ASSERT(isInitialized(lock));
 178
 179   if (!wasmTier2GeneratorWorklist(lock).append(task.get())) {
 180     return false;
 181   }
 182   (void)task.release();
 183
 184   dispatch(DispatchReason::NewTask, lock);
 185   return true;
 186 }
 187
 188 static void CancelOffThreadWasmTier2GeneratorLocked(
 189     AutoLockHelperThreadState& lock) {
 190   if (!HelperThreadState().isInitialized(lock)) {
 191     return;
 192   }
 193
 194   // Remove pending tasks from the tier2 generator worklist and cancel and
 195   // delete them.
 196   {
 197     wasm::Tier2GeneratorTaskPtrVector& worklist =
 198         HelperThreadState().wasmTier2GeneratorWorklist(lock);
 199     for (size_t i = 0; i < worklist.length(); i++) {
 200       wasm::Tier2GeneratorTask* task = worklist[i];
 201       HelperThreadState().remove(worklist, &i);
 202       js_delete(task);
 203     }
 204   }
 205
 206   // There is at most one running Tier2Generator task and we assume that
 207   // below.
 208   static_assert(GlobalHelperThreadState::MaxTier2GeneratorTasks == 1,
 209                 "code must be generalized");
 210
 211   // If there is a running Tier2 generator task, shut it down in a predictable
 212   // way.  The task will be deleted by the normal deletion logic.
 213   for (auto* helper : HelperThreadState().helperTasks(lock)) {
 214     if (helper->is<wasm::Tier2GeneratorTask>()) {
 215       // Set a flag that causes compilation to shortcut itself.
 216       helper->as<wasm::Tier2GeneratorTask>()->cancel();
 217
 218       // Wait for the generator task to finish.  This avoids a shutdown race
 219       // where the shutdown code is trying to shut down helper threads and the
 220       // ongoing tier2 compilation is trying to finish, which requires it to
 221       // have access to helper threads.
 222       uint32_t oldFinishedCount =
 223           HelperThreadState().wasmTier2GeneratorsFinished(lock);
 224       while (HelperThreadState().wasmTier2GeneratorsFinished(lock) ==
 225              oldFinishedCount) {
 226         HelperThreadState().wait(lock);
 227       }
 228
 229       // At most one of these tasks.
 230       break;
 231     }
 232   }
 233 }
 234
 235 void js::CancelOffThreadWasmTier2Generator() {
 236   AutoLockHelperThreadState lock;
 237   CancelOffThreadWasmTier2GeneratorLocked(lock);
 238 }
 239
 240 bool js::StartOffThreadIonCompile(jit::IonCompileTask* task,
 241                                   const AutoLockHelperThreadState& lock) {
 242   return HelperThreadState().submitTask(task, lock);
 243 }
 244
 245 bool GlobalHelperThreadState::submitTask(
 246     jit::IonCompileTask* task, const AutoLockHelperThreadState& locked) {
 247   MOZ_ASSERT(isInitialized(locked));
 248
 249   if (!ionWorklist(locked).append(task)) {
 250     return false;
 251   }
 252
 253   // The build is moving off-thread. Freeze the LifoAlloc to prevent any
 254   // unwanted mutations.
 255   task->alloc().lifoAlloc()->setReadOnly();
 256
 257   dispatch(DispatchReason::NewTask, locked);
 258   return true;
 259 }
 260
 261 bool js::StartOffThreadIonFree(jit::IonCompileTask* task,
 262                                const AutoLockHelperThreadState& lock) {
 263   js::UniquePtr<jit::IonFreeTask> freeTask =
 264       js::MakeUnique<jit::IonFreeTask>(task);
 265   if (!freeTask) {
 266     return false;
 267   }
 268
 269   return HelperThreadState().submitTask(std::move(freeTask), lock);
 270 }
 271
 272 bool GlobalHelperThreadState::submitTask(
 273     UniquePtr<jit::IonFreeTask> task, const AutoLockHelperThreadState& locked) {
 274   MOZ_ASSERT(isInitialized(locked));
 275
 276   if (!ionFreeList(locked).append(std::move(task))) {
 277     return false;
 278   }
 279
 280   dispatch(DispatchReason::NewTask, locked);
 281   return true;
 282 }
 283
 284 /*
 285  * Move an IonCompilationTask for which compilation has either finished, failed,
 286  * or been cancelled into the global finished compilation list. All off thread
 287  * compilations which are started must eventually be finished.
 288  */
 289 void js::FinishOffThreadIonCompile(jit::IonCompileTask* task,
 290                                    const AutoLockHelperThreadState& lock) {
 291   AutoEnterOOMUnsafeRegion oomUnsafe;
 292   if (!HelperThreadState().ionFinishedList(lock).append(task)) {
 293     oomUnsafe.crash("FinishOffThreadIonCompile");
 294   }
 295   task->script()
 296       ->runtimeFromAnyThread()
 297       ->jitRuntime()
 298       ->numFinishedOffThreadTasksRef(lock)++;
 299 }
 300
 301 static JSRuntime* GetSelectorRuntime(const CompilationSelector& selector) {
 302   struct Matcher {
 303     JSRuntime* operator()(JSScript* script) {
 304       return script->runtimeFromMainThread();
 305     }
 306     JSRuntime* operator()(Zone* zone) { return zone->runtimeFromMainThread(); }
 307     JSRuntime* operator()(ZonesInState zbs) { return zbs.runtime; }
 308     JSRuntime* operator()(JSRuntime* runtime) { return runtime; }
 309   };
 310
 311   return selector.match(Matcher());
 312 }
 313
 314 static bool JitDataStructuresExist(const CompilationSelector& selector) {
 315   struct Matcher {
 316     bool operator()(JSScript* script) { return !!script->zone()->jitZone(); }
 317     bool operator()(Zone* zone) { return !!zone->jitZone(); }
 318     bool operator()(ZonesInState zbs) { return zbs.runtime->hasJitRuntime(); }
 319     bool operator()(JSRuntime* runtime) { return runtime->hasJitRuntime(); }
 320   };
 321
 322   return selector.match(Matcher());
 323 }
 324
 325 static bool IonCompileTaskMatches(const CompilationSelector& selector,
 326                                   jit::IonCompileTask* task) {
 327   struct TaskMatches {
 328     jit::IonCompileTask* task_;
 329
 330     bool operator()(JSScript* script) { return script == task_->script(); }
 331     bool operator()(Zone* zone) {
 332       return zone == task_->script()->zoneFromAnyThread();
 333     }
 334     bool operator()(JSRuntime* runtime) {
 335       return runtime == task_->script()->runtimeFromAnyThread();
 336     }
 337     bool operator()(ZonesInState zbs) {
 338       return zbs.runtime == task_->script()->runtimeFromAnyThread() &&
 339              zbs.state == task_->script()->zoneFromAnyThread()->gcState();
 340     }
 341   };
 342
 343   return selector.match(TaskMatches{task});
 344 }
 345
 346 static void CancelOffThreadIonCompileLocked(const CompilationSelector& selector,
 347                                             AutoLockHelperThreadState& lock) {
 348   if (jit::IsPortableBaselineInterpreterEnabled()) {
 349     return;
 350   }
 351
 352   if (!HelperThreadState().isInitialized(lock)) {
 353     return;
 354   }
 355
 356   MOZ_ASSERT(GetSelectorRuntime(selector)->jitRuntime() != nullptr);
 357
 358   /* Cancel any pending entries for which processing hasn't started. */
 359   GlobalHelperThreadState::IonCompileTaskVector& worklist =
 360       HelperThreadState().ionWorklist(lock);
 361   for (size_t i = 0; i < worklist.length(); i++) {
 362     jit::IonCompileTask* task = worklist[i];
 363     if (IonCompileTaskMatches(selector, task)) {
 364       // Once finished, tasks are added to a Linked list which is
 365       // allocated with the IonCompileTask class. The IonCompileTask is
 366       // allocated in the LifoAlloc so we need the LifoAlloc to be mutable.
 367       worklist[i]->alloc().lifoAlloc()->setReadWrite();
 368
 369       FinishOffThreadIonCompile(task, lock);
 370       HelperThreadState().remove(worklist, &i);
 371     }
 372   }
 373
 374   /* Wait for in progress entries to finish up. */
 375   bool cancelled;
 376   do {
 377     cancelled = false;
 378     for (auto* helper : HelperThreadState().helperTasks(lock)) {
 379       if (!helper->is<jit::IonCompileTask>()) {
 380         continue;
 381       }
 382
 383       jit::IonCompileTask* ionCompileTask = helper->as<jit::IonCompileTask>();
 384       if (IonCompileTaskMatches(selector, ionCompileTask)) {
 385         ionCompileTask->mirGen().cancel();
 386         cancelled = true;
 387       }
 388     }
 389     if (cancelled) {
 390       HelperThreadState().wait(lock);
 391     }
 392   } while (cancelled);
 393
 394   /* Cancel code generation for any completed entries. */
 395   GlobalHelperThreadState::IonCompileTaskVector& finished =
 396       HelperThreadState().ionFinishedList(lock);
 397   for (size_t i = 0; i < finished.length(); i++) {
 398     jit::IonCompileTask* task = finished[i];
 399     if (IonCompileTaskMatches(selector, task)) {
 400       JSRuntime* rt = task->script()->runtimeFromAnyThread();
 401       rt->jitRuntime()->numFinishedOffThreadTasksRef(lock)--;
 402       jit::FinishOffThreadTask(rt, task, lock);
 403       HelperThreadState().remove(finished, &i);
 404     }
 405   }
 406
 407   /* Cancel lazy linking for pending tasks (attached to the ionScript). */
 408   JSRuntime* runtime = GetSelectorRuntime(selector);
 409   jit::IonCompileTask* task =
 410       runtime->jitRuntime()->ionLazyLinkList(runtime).getFirst();
 411   while (task) {
 412     jit::IonCompileTask* next = task->getNext();
 413     if (IonCompileTaskMatches(selector, task)) {
 414       jit::FinishOffThreadTask(runtime, task, lock);
 415     }
 416     task = next;
 417   }
 418 }
 419
 420 void js::CancelOffThreadIonCompile(const CompilationSelector& selector) {
 421   if (!JitDataStructuresExist(selector)) {
 422     return;
 423   }
 424
 425   AutoLockHelperThreadState lock;
 426   CancelOffThreadIonCompileLocked(selector, lock);
 427 }
 428
 429 #ifdef DEBUG
 430 bool js::HasOffThreadIonCompile(Zone* zone) {
 431   if (jit::IsPortableBaselineInterpreterEnabled()) {
 432     return false;
 433   }
 434
 435   AutoLockHelperThreadState lock;
 436
 437   if (!HelperThreadState().isInitialized(lock)) {
 438     return false;
 439   }
 440
 441   GlobalHelperThreadState::IonCompileTaskVector& worklist =
 442       HelperThreadState().ionWorklist(lock);
 443   for (size_t i = 0; i < worklist.length(); i++) {
 444     jit::IonCompileTask* task = worklist[i];
 445     if (task->script()->zoneFromAnyThread() == zone) {
 446       return true;
 447     }
 448   }
 449
 450   for (auto* helper : HelperThreadState().helperTasks(lock)) {
 451     if (!helper->is<jit::IonCompileTask>()) {
 452       continue;
 453     }
 454     JSScript* script = helper->as<jit::IonCompileTask>()->script();
 455     if (script->zoneFromAnyThread() == zone) {
 456       return true;
 457     }
 458   }
 459
 460   GlobalHelperThreadState::IonCompileTaskVector& finished =
 461       HelperThreadState().ionFinishedList(lock);
 462   for (size_t i = 0; i < finished.length(); i++) {
 463     jit::IonCompileTask* task = finished[i];
 464     if (task->script()->zoneFromAnyThread() == zone) {
 465       return true;
 466     }
 467   }
 468
 469   JSRuntime* rt = zone->runtimeFromMainThread();
 470   jit::IonCompileTask* task = rt->jitRuntime()->ionLazyLinkList(rt).getFirst();
 471   while (task) {
 472     if (task->script()->zone() == zone) {
 473       return true;
 474     }
 475     task = task->getNext();
 476   }
 477
 478   return false;
 479 }
 480 #endif
 481
 482 void js::StartOffThreadDelazification(
 483     JSContext* maybeCx, const JS::ReadOnlyCompileOptions& options,
 484     const frontend::CompilationStencil& stencil) {
 485   // Skip delazify tasks if we parse everything on-demand or ahead.
 486   auto strategy = options.eagerDelazificationStrategy();
 487   if (strategy == JS::DelazificationOption::OnDemandOnly ||
 488       strategy == JS::DelazificationOption::ParseEverythingEagerly) {
 489     return;
 490   }
 491
 492   // Skip delazify task if code coverage is enabled.
 493   if (maybeCx && maybeCx->realm()->collectCoverageForDebug()) {
 494     return;
 495   }
 496
 497   if (!CanUseExtraThreads()) {
 498     return;
 499   }
 500
 501   JSRuntime* maybeRuntime = maybeCx ? maybeCx->runtime() : nullptr;
 502   UniquePtr<DelazifyTask> task;
 503   task = DelazifyTask::Create(maybeRuntime, options, stencil);
 504   if (!task) {
 505     return;
 506   }
 507
 508   // Schedule delazification task if there is any function to delazify.
 509   if (!task->done()) {
 510     AutoLockHelperThreadState lock;
 511     HelperThreadState().submitTask(task.release(), lock);
 512   }
 513 }
 514
 515 UniquePtr<DelazifyTask> DelazifyTask::Create(
 516     JSRuntime* maybeRuntime, const JS::ReadOnlyCompileOptions& options,
 517     const frontend::CompilationStencil& stencil) {
 518   UniquePtr<DelazifyTask> task;
 519   task.reset(js_new<DelazifyTask>(maybeRuntime, options.prefableOptions()));
 520   if (!task) {
 521     return nullptr;
 522   }
 523
 524   if (!task->init(options, stencil)) {
 525     // In case of errors, skip this and delazify on-demand.
 526     return nullptr;
 527   }
 528
 529   return task;
 530 }
 531
 532 DelazifyTask::DelazifyTask(
 533     JSRuntime* maybeRuntime,
 534     const JS::PrefableCompileOptions& initialPrefableOptions)
 535     : maybeRuntime(maybeRuntime),
 536       delazificationCx(initialPrefableOptions, HelperThreadState().stackQuota) {
 537 }
 538
 539 DelazifyTask::~DelazifyTask() {
 540   // The LinkedListElement destructor will remove us from any list we are part
 541   // of without synchronization, so ensure that doesn't happen.
 542   MOZ_DIAGNOSTIC_ASSERT(!isInList());
 543 }
 544
 545 bool DelazifyTask::init(const JS::ReadOnlyCompileOptions& options,
 546                         const frontend::CompilationStencil& stencil) {
 547   return delazificationCx.init(options, stencil);
 548 }
 549
 550 size_t DelazifyTask::sizeOfExcludingThis(
 551     mozilla::MallocSizeOf mallocSizeOf) const {
 552   return delazificationCx.sizeOfExcludingThis(mallocSizeOf);
 553 }
 554
 555 void DelazifyTask::runHelperThreadTask(AutoLockHelperThreadState& lock) {
 556   {
 557     AutoUnlockHelperThreadState unlock(lock);
 558     // NOTE: We do not report errors beyond this scope, as there is no where
 559     // to report these errors to. In the mean time, prevent the eager
 560     // delazification from running after any kind of errors.
 561     (void)runTask();
 562   }
 563
 564   // If we should continue to delazify even more functions, then re-add this
 565   // task to the vector of delazification tasks. This might happen when the
 566   // DelazifyTask is interrupted by a higher priority task. (see
 567   // mozilla::TaskController & mozilla::Task)
 568   if (!delazificationCx.done()) {
 569     HelperThreadState().submitTask(this, lock);
 570   } else {
 571     UniquePtr<FreeDelazifyTask> freeTask(js_new<FreeDelazifyTask>(this));
 572     if (freeTask) {
 573       HelperThreadState().submitTask(std::move(freeTask), lock);
 574     }
 575   }
 576 }
 577
 578 bool DelazifyTask::runTask() { return delazificationCx.delazify(); }
 579
 580 bool DelazifyTask::done() const { return delazificationCx.done(); }
 581
 582 void FreeDelazifyTask::runHelperThreadTask(AutoLockHelperThreadState& locked) {
 583   {
 584     AutoUnlockHelperThreadState unlock(locked);
 585     js_delete(task);
 586     task = nullptr;
 587   }
 588
 589   js_delete(this);
 590 }
 591
 592 static void CancelPendingDelazifyTask(JSRuntime* rt,
 593                                       AutoLockHelperThreadState& lock) {
 594   auto& delazifyList = HelperThreadState().delazifyWorklist(lock);
 595
 596   auto end = delazifyList.end();
 597   for (auto iter = delazifyList.begin(); iter != end;) {
 598     DelazifyTask* task = *iter;
 599     ++iter;
 600     if (task->runtimeMatchesOrNoRuntime(rt)) {
 601       task->removeFrom(delazifyList);
 602       js_delete(task);
 603     }
 604   }
 605 }
 606
 607 static void WaitUntilCancelledDelazifyTasks(JSRuntime* rt,
 608                                             AutoLockHelperThreadState& lock) {
 609   if (!HelperThreadState().isInitialized(lock)) {
 610     return;
 611   }
 612
 613   while (true) {
 614     CancelPendingDelazifyTask(rt, lock);
 615
 616     // If running tasks are delazifying any functions, then we have to wait
 617     // until they complete to remove them from the pending list. DelazifyTask
 618     // are inserting themself back to be processed once more after delazifying a
 619     // function.
 620     bool inProgress = false;
 621     for (auto* helper : HelperThreadState().helperTasks(lock)) {
 622       if (helper->is<DelazifyTask>() &&
 623           helper->as<DelazifyTask>()->runtimeMatchesOrNoRuntime(rt)) {
 624         inProgress = true;
 625         break;
 626       }
 627     }
 628     if (!inProgress) {
 629       break;
 630     }
 631
 632     HelperThreadState().wait(lock);
 633   }
 634
 635 #ifdef DEBUG
 636   for (DelazifyTask* task : HelperThreadState().delazifyWorklist(lock)) {
 637     MOZ_ASSERT(!task->runtimeMatchesOrNoRuntime(rt));
 638   }
 639   for (auto* helper : HelperThreadState().helperTasks(lock)) {
 640     MOZ_ASSERT_IF(helper->is<DelazifyTask>(),
 641                   !helper->as<DelazifyTask>()->runtimeMatchesOrNoRuntime(rt));
 642   }
 643 #endif
 644 }
 645
 646 static void WaitUntilEmptyFreeDelazifyTaskVector(
 647     AutoLockHelperThreadState& lock) {
 648   if (!HelperThreadState().isInitialized(lock)) {
 649     return;
 650   }
 651
 652   while (true) {
 653     bool inProgress = false;
 654     auto& freeList = HelperThreadState().freeDelazifyTaskVector(lock);
 655     if (!freeList.empty()) {
 656       inProgress = true;
 657     }
 658
 659     // If running tasks are delazifying any functions, then we have to wait
 660     // until they complete to remove them from the pending list. DelazifyTask
 661     // are inserting themself back to be processed once more after delazifying a
 662     // function.
 663     for (auto* helper : HelperThreadState().helperTasks(lock)) {
 664       if (helper->is<FreeDelazifyTask>()) {
 665         inProgress = true;
 666         break;
 667       }
 668     }
 669     if (!inProgress) {
 670       break;
 671     }
 672
 673     HelperThreadState().wait(lock);
 674   }
 675 }
 676
 677 void js::CancelOffThreadDelazify(JSRuntime* runtime) {
 678   AutoLockHelperThreadState lock;
 679
 680   // Cancel all Delazify tasks from the given runtime, and wait if tasks are
 681   // from the given runtime are being executed.
 682   WaitUntilCancelledDelazifyTasks(runtime, lock);
 683
 684   // Empty the free list of delazify task, in case one of the delazify task
 685   // ended and therefore did not returned to the pending list of delazify tasks.
 686   WaitUntilEmptyFreeDelazifyTaskVector(lock);
 687 }
 688
 689 static bool HasAnyDelazifyTask(JSRuntime* rt, AutoLockHelperThreadState& lock) {
 690   auto& delazifyList = HelperThreadState().delazifyWorklist(lock);
 691   for (auto task : delazifyList) {
 692     if (task->runtimeMatchesOrNoRuntime(rt)) {
 693       return true;
 694     }
 695   }
 696
 697   for (auto* helper : HelperThreadState().helperTasks(lock)) {
 698     if (helper->is<DelazifyTask>() &&
 699         helper->as<DelazifyTask>()->runtimeMatchesOrNoRuntime(rt)) {
 700       return true;
 701     }
 702   }
 703
 704   return false;
 705 }
 706
 707 void js::WaitForAllDelazifyTasks(JSRuntime* rt) {
 708   AutoLockHelperThreadState lock;
 709   if (!HelperThreadState().isInitialized(lock)) {
 710     return;
 711   }
 712
 713   while (true) {
 714     if (!HasAnyDelazifyTask(rt, lock)) {
 715       break;
 716     }
 717
 718     HelperThreadState().wait(lock);
 719   }
 720 }
 721
 722 void GlobalHelperThreadState::submitTask(
 723     DelazifyTask* task, const AutoLockHelperThreadState& locked) {
 724   delazifyWorklist(locked).insertBack(task);
 725   dispatch(DispatchReason::NewTask, locked);
 726 }
 727
 728 bool GlobalHelperThreadState::submitTask(
 729     UniquePtr<FreeDelazifyTask> task, const AutoLockHelperThreadState& locked) {
 730   if (!freeDelazifyTaskVector(locked).append(std::move(task))) {
 731     return false;
 732   }
 733   dispatch(DispatchReason::NewTask, locked);
 734   return true;
 735 }
 736
 737 bool GlobalHelperThreadState::ensureInitialized() {
 738   MOZ_ASSERT(CanUseExtraThreads());
 739   MOZ_ASSERT(this == &HelperThreadState());
 740
 741   AutoLockHelperThreadState lock;
 742
 743   if (isInitialized(lock)) {
 744     return true;
 745   }
 746
 747   for (size_t& i : runningTaskCount) {
 748     i = 0;
 749   }
 750
 751   useInternalThreadPool_ = !dispatchTaskCallback;
 752   if (useInternalThreadPool(lock)) {
 753     if (!InternalThreadPool::Initialize(threadCount, lock)) {
 754       return false;
 755     }
 756   }
 757
 758   MOZ_ASSERT(dispatchTaskCallback);
 759
 760   if (!ensureThreadCount(threadCount, lock)) {
 761     finishThreads(lock);
 762     return false;
 763   }
 764
 765   MOZ_ASSERT(threadCount != 0);
 766   isInitialized_ = true;
 767   return true;
 768 }
 769
 770 bool GlobalHelperThreadState::ensureThreadCount(
 771     size_t count, AutoLockHelperThreadState& lock) {
 772   if (!helperTasks_.reserve(count)) {
 773     return false;
 774   }
 775
 776   if (useInternalThreadPool(lock)) {
 777     InternalThreadPool& pool = InternalThreadPool::Get();
 778     if (pool.threadCount(lock) < count) {
 779       if (!pool.ensureThreadCount(count, lock)) {
 780         return false;
 781       }
 782
 783       threadCount = pool.threadCount(lock);
 784     }
 785   }
 786
 787   return true;
 788 }
 789
 790 GlobalHelperThreadState::GlobalHelperThreadState()
 791     : cpuCount(0),
 792       threadCount(0),
 793       totalCountRunningTasks(0),
 794       registerThread(nullptr),
 795       unregisterThread(nullptr),
 796       wasmTier2GeneratorsFinished_(0) {
 797   MOZ_ASSERT(!gHelperThreadState);
 798
 799   cpuCount = ClampDefaultCPUCount(GetCPUCount());
 800   threadCount = ThreadCountForCPUCount(cpuCount);
 801   gcParallelThreadCount = threadCount;
 802
 803   MOZ_ASSERT(cpuCount > 0, "GetCPUCount() seems broken");
 804 }
 805
 806 void GlobalHelperThreadState::finish(AutoLockHelperThreadState& lock) {
 807   if (!isInitialized(lock)) {
 808     return;
 809   }
 810
 811   finishThreads(lock);
 812
 813   // Make sure there are no Ion free tasks left. We check this here because,
 814   // unlike the other tasks, we don't explicitly block on this when
 815   // destroying a runtime.
 816   auto& freeList = ionFreeList(lock);
 817   while (!freeList.empty()) {
 818     UniquePtr<jit::IonFreeTask> task = std::move(freeList.back());
 819     freeList.popBack();
 820     jit::FreeIonCompileTask(task->compileTask());
 821   }
 822 }
 823
 824 void GlobalHelperThreadState::finishThreads(AutoLockHelperThreadState& lock) {
 825   waitForAllTasksLocked(lock);
 826   terminating_ = true;
 827
 828   if (InternalThreadPool::IsInitialized()) {
 829     InternalThreadPool::ShutDown(lock);
 830   }
 831 }
 832
 833 #ifdef DEBUG
 834 void GlobalHelperThreadState::assertIsLockedByCurrentThread() const {
 835   gHelperThreadLock.assertOwnedByCurrentThread();
 836 }
 837 #endif  // DEBUG
 838
 839 void GlobalHelperThreadState::dispatch(
 840     DispatchReason reason, const AutoLockHelperThreadState& locked) {
 841   if (canStartTasks(locked) && tasksPending_ < threadCount) {
 842     // This doesn't guarantee that we don't dispatch more tasks to the external
 843     // pool than necessary if tasks are taking a long time to start, but it does
 844     // limit the number.
 845     tasksPending_++;
 846
 847     // The hazard analysis can't tell that the callback doesn't GC.
 848     JS::AutoSuppressGCAnalysis nogc;
 849
 850     dispatchTaskCallback(reason);
 851   }
 852 }
 853
 854 void GlobalHelperThreadState::wait(
 855     AutoLockHelperThreadState& locked,
 856     TimeDuration timeout /* = TimeDuration::Forever() */) {
 857   consumerWakeup.wait_for(locked, timeout);
 858 }
 859
 860 void GlobalHelperThreadState::notifyAll(const AutoLockHelperThreadState&) {
 861   consumerWakeup.notify_all();
 862 }
 863
 864 void GlobalHelperThreadState::notifyOne(const AutoLockHelperThreadState&) {
 865   consumerWakeup.notify_one();
 866 }
 867
 868 bool GlobalHelperThreadState::hasActiveThreads(
 869     const AutoLockHelperThreadState& lock) {
 870   return !helperTasks(lock).empty();
 871 }
 872
 873 void js::WaitForAllHelperThreads() { HelperThreadState().waitForAllTasks(); }
 874
 875 void js::WaitForAllHelperThreads(AutoLockHelperThreadState& lock) {
 876   HelperThreadState().waitForAllTasksLocked(lock);
 877 }
 878
 879 void GlobalHelperThreadState::waitForAllTasks() {
 880   AutoLockHelperThreadState lock;
 881   waitForAllTasksLocked(lock);
 882 }
 883
 884 void GlobalHelperThreadState::waitForAllTasksLocked(
 885     AutoLockHelperThreadState& lock) {
 886   CancelOffThreadWasmTier2GeneratorLocked(lock);
 887
 888   while (canStartTasks(lock) || tasksPending_ || hasActiveThreads(lock)) {
 889     wait(lock);
 890   }
 891
 892   MOZ_ASSERT(gcParallelWorklist().isEmpty(lock));
 893   MOZ_ASSERT(ionWorklist(lock).empty());
 894   MOZ_ASSERT(wasmWorklist(lock, wasm::CompileMode::Tier1).empty());
 895   MOZ_ASSERT(promiseHelperTasks(lock).empty());
 896   MOZ_ASSERT(compressionWorklist(lock).empty());
 897   MOZ_ASSERT(ionFreeList(lock).empty());
 898   MOZ_ASSERT(wasmWorklist(lock, wasm::CompileMode::Tier2).empty());
 899   MOZ_ASSERT(wasmTier2GeneratorWorklist(lock).empty());
 900   MOZ_ASSERT(!tasksPending_);
 901   MOZ_ASSERT(!hasActiveThreads(lock));
 902 }
 903
 904 // A task can be a "master" task, ie, it will block waiting for other worker
 905 // threads that perform work on its behalf.  If so it must not take the last
 906 // available thread; there must always be at least one worker thread able to do
 907 // the actual work.  (Or the system may deadlock.)
 908 //
 909 // If a task is a master task it *must* pass isMaster=true here, or perform a
 910 // similar calculation to avoid deadlock from starvation.
 911 //
 912 // isMaster should only be true if the thread calling checkTaskThreadLimit() is
 913 // a helper thread.
 914 //
 915 // NOTE: Calling checkTaskThreadLimit() from a helper thread in the dynamic
 916 // region after currentTask.emplace() and before currentTask.reset() may cause
 917 // it to return a different result than if it is called outside that dynamic
 918 // region, as the predicate inspects the values of the threads' currentTask
 919 // members.
 920
 921 bool GlobalHelperThreadState::checkTaskThreadLimit(
 922     ThreadType threadType, size_t maxThreads, bool isMaster,
 923     const AutoLockHelperThreadState& lock) const {
 924   MOZ_ASSERT(maxThreads > 0);
 925
 926   if (!isMaster && maxThreads >= threadCount) {
 927     return true;
 928   }
 929
 930   size_t count = runningTaskCount[threadType];
 931   if (count >= maxThreads) {
 932     return false;
 933   }
 934
 935   MOZ_ASSERT(threadCount >= totalCountRunningTasks);
 936   size_t idle = threadCount - totalCountRunningTasks;
 937
 938   // It is possible for the number of idle threads to be zero here, because
 939   // checkTaskThreadLimit() can be called from non-helper threads.  Notably,
 940   // the compression task scheduler invokes it, and runs off a helper thread.
 941   if (idle == 0) {
 942     return false;
 943   }
 944
 945   // A master thread that's the last available thread must not be allowed to
 946   // run.
 947   if (isMaster && idle == 1) {
 948     return false;
 949   }
 950
 951   return true;
 952 }
 953
 954 static inline bool IsHelperThreadSimulatingOOM(js::ThreadType threadType) {
 955 #if defined(DEBUG) || defined(JS_OOM_BREAKPOINT)
 956   return js::oom::simulator.targetThread() == threadType;
 957 #else
 958   return false;
 959 #endif
 960 }
 961
 962 void GlobalHelperThreadState::addSizeOfIncludingThis(
 963     JS::GlobalStats* stats, const AutoLockHelperThreadState& lock) const {
 964 #ifdef DEBUG
 965   assertIsLockedByCurrentThread();
 966 #endif
 967
 968   mozilla::MallocSizeOf mallocSizeOf = stats->mallocSizeOf_;
 969   JS::HelperThreadStats& htStats = stats->helperThread;
 970
 971   htStats.stateData += mallocSizeOf(this);
 972
 973   if (InternalThreadPool::IsInitialized()) {
 974     htStats.stateData +=
 975         InternalThreadPool::Get().sizeOfIncludingThis(mallocSizeOf, lock);
 976   }
 977
 978   // Report memory used by various containers
 979   htStats.stateData +=
 980       ionWorklist_.sizeOfExcludingThis(mallocSizeOf) +
 981       ionFinishedList_.sizeOfExcludingThis(mallocSizeOf) +
 982       ionFreeList_.sizeOfExcludingThis(mallocSizeOf) +
 983       wasmWorklist_tier1_.sizeOfExcludingThis(mallocSizeOf) +
 984       wasmWorklist_tier2_.sizeOfExcludingThis(mallocSizeOf) +
 985       wasmTier2GeneratorWorklist_.sizeOfExcludingThis(mallocSizeOf) +
 986       promiseHelperTasks_.sizeOfExcludingThis(mallocSizeOf) +
 987       compressionPendingList_.sizeOfExcludingThis(mallocSizeOf) +
 988       compressionWorklist_.sizeOfExcludingThis(mallocSizeOf) +
 989       compressionFinishedList_.sizeOfExcludingThis(mallocSizeOf) +
 990       gcParallelWorklist_.sizeOfExcludingThis(mallocSizeOf, lock) +
 991       helperTasks_.sizeOfExcludingThis(mallocSizeOf);
 992
 993   // Report IonCompileTasks on wait lists
 994   for (auto task : ionWorklist_) {
 995     htStats.ionCompileTask += task->sizeOfExcludingThis(mallocSizeOf);
 996   }
 997   for (auto task : ionFinishedList_) {
 998     htStats.ionCompileTask += task->sizeOfExcludingThis(mallocSizeOf);
 999   }
1000   for (const auto& task : ionFreeList_) {
1001     htStats.ionCompileTask +=
1002         task->compileTask()->sizeOfExcludingThis(mallocSizeOf);
1003   }
1004
1005   // Report wasm::CompileTasks on wait lists
1006   for (auto task : wasmWorklist_tier1_) {
1007     htStats.wasmCompile += task->sizeOfExcludingThis(mallocSizeOf);
1008   }
1009   for (auto task : wasmWorklist_tier2_) {
1010     htStats.wasmCompile += task->sizeOfExcludingThis(mallocSizeOf);
1011   }
1012
1013   // Report number of helper threads.
1014   MOZ_ASSERT(htStats.idleThreadCount == 0);
1015   MOZ_ASSERT(threadCount >= totalCountRunningTasks);
1016   htStats.activeThreadCount = totalCountRunningTasks;
1017   htStats.idleThreadCount = threadCount - totalCountRunningTasks;
1018 }
1019
1020 size_t GlobalHelperThreadState::maxIonCompilationThreads() const {
1021   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_ION)) {
1022     return 1;
1023   }
1024   return threadCount;
1025 }
1026
1027 size_t GlobalHelperThreadState::maxWasmCompilationThreads() const {
1028   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER1) ||
1029       IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER2)) {
1030     return 1;
1031   }
1032   return std::min(cpuCount, threadCount);
1033 }
1034
1035 size_t GlobalHelperThreadState::maxWasmTier2GeneratorThreads() const {
1036   return MaxTier2GeneratorTasks;
1037 }
1038
1039 size_t GlobalHelperThreadState::maxPromiseHelperThreads() const {
1040   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER1) ||
1041       IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER2)) {
1042     return 1;
1043   }
1044   return std::min(cpuCount, threadCount);
1045 }
1046
1047 size_t GlobalHelperThreadState::maxDelazifyThreads() const {
1048   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_DELAZIFY)) {
1049     return 1;
1050   }
1051   return std::min(cpuCount, threadCount);
1052 }
1053
1054 size_t GlobalHelperThreadState::maxCompressionThreads() const {
1055   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_COMPRESS)) {
1056     return 1;
1057   }
1058
1059   // Compression is triggered on major GCs to compress ScriptSources. It is
1060   // considered low priority work.
1061   return 1;
1062 }
1063
1064 size_t GlobalHelperThreadState::maxGCParallelThreads(
1065     const AutoLockHelperThreadState& lock) const {
1066   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_GCPARALLEL)) {
1067     return 1;
1068   }
1069   return gcParallelThreadCount;
1070 }
1071
1072 HelperThreadTask* GlobalHelperThreadState::maybeGetWasmTier1CompileTask(
1073     const AutoLockHelperThreadState& lock) {
1074   return maybeGetWasmCompile(lock, wasm::CompileMode::Tier1);
1075 }
1076
1077 HelperThreadTask* GlobalHelperThreadState::maybeGetWasmTier2CompileTask(
1078     const AutoLockHelperThreadState& lock) {
1079   return maybeGetWasmCompile(lock, wasm::CompileMode::Tier2);
1080 }
1081
1082 HelperThreadTask* GlobalHelperThreadState::maybeGetWasmCompile(
1083     const AutoLockHelperThreadState& lock, wasm::CompileMode mode) {
1084   if (!canStartWasmCompile(lock, mode)) {
1085     return nullptr;
1086   }
1087
1088   return wasmWorklist(lock, mode).popCopyFront();
1089 }
1090
1091 bool GlobalHelperThreadState::canStartWasmTier1CompileTask(
1092     const AutoLockHelperThreadState& lock) {
1093   return canStartWasmCompile(lock, wasm::CompileMode::Tier1);
1094 }
1095
1096 bool GlobalHelperThreadState::canStartWasmTier2CompileTask(
1097     const AutoLockHelperThreadState& lock) {
1098   return canStartWasmCompile(lock, wasm::CompileMode::Tier2);
1099 }
1100
1101 bool GlobalHelperThreadState::canStartWasmCompile(
1102     const AutoLockHelperThreadState& lock, wasm::CompileMode mode) {
1103   if (wasmWorklist(lock, mode).empty()) {
1104     return false;
1105   }
1106
1107   // Parallel compilation and background compilation should be disabled on
1108   // unicore systems.
1109
1110   MOZ_RELEASE_ASSERT(cpuCount > 1);
1111
1112   // If Tier2 is very backlogged we must give priority to it, since the Tier2
1113   // queue holds onto Tier1 tasks.  Indeed if Tier2 is backlogged we will
1114   // devote more resources to Tier2 and not start any Tier1 work at all.
1115
1116   bool tier2oversubscribed = wasmTier2GeneratorWorklist(lock).length() > 20;
1117
1118   // For Tier1 and Once compilation, honor the maximum allowed threads to
1119   // compile wasm jobs at once, to avoid oversaturating the machine.
1120   //
1121   // For Tier2 compilation we need to allow other things to happen too, so we
1122   // do not allow all logical cores to be used for background work; instead we
1123   // wish to use a fraction of the physical cores.  We can't directly compute
1124   // the physical cores from the logical cores, but 1/3 of the logical cores
1125   // is a safe estimate for the number of physical cores available for
1126   // background work.
1127
1128   size_t physCoresAvailable = size_t(ceil(cpuCount / 3.0));
1129
1130   size_t threads;
1131   ThreadType threadType;
1132   if (mode == wasm::CompileMode::Tier2) {
1133     if (tier2oversubscribed) {
1134       threads = maxWasmCompilationThreads();
1135     } else {
1136       threads = physCoresAvailable;
1137     }
1138     threadType = THREAD_TYPE_WASM_COMPILE_TIER2;
1139   } else {
1140     if (tier2oversubscribed) {
1141       threads = 0;
1142     } else {
1143       threads = maxWasmCompilationThreads();
1144     }
1145     threadType = THREAD_TYPE_WASM_COMPILE_TIER1;
1146   }
1147
1148   return threads != 0 && checkTaskThreadLimit(threadType, threads, lock);
1149 }
1150
1151 HelperThreadTask* GlobalHelperThreadState::maybeGetWasmTier2GeneratorTask(
1152     const AutoLockHelperThreadState& lock) {
1153   if (!canStartWasmTier2GeneratorTask(lock)) {
1154     return nullptr;
1155   }
1156
1157   return wasmTier2GeneratorWorklist(lock).popCopy();
1158 }
1159
1160 bool GlobalHelperThreadState::canStartWasmTier2GeneratorTask(
1161     const AutoLockHelperThreadState& lock) {
1162   return !wasmTier2GeneratorWorklist(lock).empty() &&
1163          checkTaskThreadLimit(THREAD_TYPE_WASM_GENERATOR_TIER2,
1164                               maxWasmTier2GeneratorThreads(),
1165                               /*isMaster=*/true, lock);
1166 }
1167
1168 HelperThreadTask* GlobalHelperThreadState::maybeGetPromiseHelperTask(
1169     const AutoLockHelperThreadState& lock) {
1170   if (!canStartPromiseHelperTask(lock)) {
1171     return nullptr;
1172   }
1173
1174   return promiseHelperTasks(lock).popCopy();
1175 }
1176
1177 bool GlobalHelperThreadState::canStartPromiseHelperTask(
1178     const AutoLockHelperThreadState& lock) {
1179   // PromiseHelperTasks can be wasm compilation tasks that in turn block on
1180   // wasm compilation so set isMaster = true.
1181   return !promiseHelperTasks(lock).empty() &&
1182          checkTaskThreadLimit(THREAD_TYPE_PROMISE_TASK,
1183                               maxPromiseHelperThreads(),
1184                               /*isMaster=*/true, lock);
1185 }
1186
1187 static bool IonCompileTaskHasHigherPriority(jit::IonCompileTask* first,
1188                                             jit::IonCompileTask* second) {
1189   // Return true if priority(first) > priority(second).
1190   //
1191   // This method can return whatever it wants, though it really ought to be a
1192   // total order. The ordering is allowed to race (change on the fly), however.
1193
1194   // A higher warm-up counter indicates a higher priority.
1195   jit::JitScript* firstJitScript = first->script()->jitScript();
1196   jit::JitScript* secondJitScript = second->script()->jitScript();
1197   return firstJitScript->warmUpCount() / first->script()->length() >
1198          secondJitScript->warmUpCount() / second->script()->length();
1199 }
1200
1201 HelperThreadTask* GlobalHelperThreadState::maybeGetIonCompileTask(
1202     const AutoLockHelperThreadState& lock) {
1203   if (!canStartIonCompileTask(lock)) {
1204     return nullptr;
1205   }
1206
1207   return highestPriorityPendingIonCompile(lock,
1208                                           /* checkExecutionStatus */ true);
1209 }
1210
1211 HelperThreadTask* GlobalHelperThreadState::maybeGetLowPrioIonCompileTask(
1212     const AutoLockHelperThreadState& lock) {
1213   if (!canStartIonCompileTask(lock)) {
1214     return nullptr;
1215   }
1216
1217   return highestPriorityPendingIonCompile(lock,
1218                                           /* checkExecutionStatus */ false);
1219 }
1220
1221 bool GlobalHelperThreadState::canStartIonCompileTask(
1222     const AutoLockHelperThreadState& lock) {
1223   return !ionWorklist(lock).empty() &&
1224          checkTaskThreadLimit(THREAD_TYPE_ION, maxIonCompilationThreads(),
1225                               lock);
1226 }
1227
1228 HelperThreadTask* GlobalHelperThreadState::maybeGetIonFreeTask(
1229     const AutoLockHelperThreadState& lock) {
1230   if (!canStartIonFreeTask(lock)) {
1231     return nullptr;
1232   }
1233
1234   UniquePtr<jit::IonFreeTask> task = std::move(ionFreeList(lock).back());
1235   ionFreeList(lock).popBack();
1236   return task.release();
1237 }
1238
1239 bool GlobalHelperThreadState::canStartIonFreeTask(
1240     const AutoLockHelperThreadState& lock) {
1241   return !ionFreeList(lock).empty();
1242 }
1243
1244 jit::IonCompileTask* GlobalHelperThreadState::highestPriorityPendingIonCompile(
1245     const AutoLockHelperThreadState& lock, bool checkExecutionStatus) {
1246   auto& worklist = ionWorklist(lock);
1247   MOZ_ASSERT(!worklist.empty());
1248
1249   // Get the highest priority IonCompileTask which has not started compilation
1250   // yet.
1251   size_t index = worklist.length();
1252   for (size_t i = 0; i < worklist.length(); i++) {
1253     if (checkExecutionStatus && !worklist[i]->isMainThreadRunningJS()) {
1254       continue;
1255     }
1256     if (i < index ||
1257         IonCompileTaskHasHigherPriority(worklist[i], worklist[index])) {
1258       index = i;
1259     }
1260   }
1261
1262   if (index == worklist.length()) {
1263     return nullptr;
1264   }
1265   jit::IonCompileTask* task = worklist[index];
1266   worklist.erase(&worklist[index]);
1267   return task;
1268 }
1269
1270 HelperThreadTask* GlobalHelperThreadState::maybeGetFreeDelazifyTask(
1271     const AutoLockHelperThreadState& lock) {
1272   auto& freeList = freeDelazifyTaskVector(lock);
1273   if (!freeList.empty()) {
1274     UniquePtr<FreeDelazifyTask> task = std::move(freeList.back());
1275     freeList.popBack();
1276     return task.release();
1277   }
1278   return nullptr;
1279 }
1280
1281 bool GlobalHelperThreadState::canStartFreeDelazifyTask(
1282     const AutoLockHelperThreadState& lock) {
1283   return !freeDelazifyTaskVector(lock).empty() &&
1284          checkTaskThreadLimit(THREAD_TYPE_DELAZIFY_FREE, maxDelazifyThreads(),
1285                               /*isMaster=*/true, lock);
1286 }
1287
1288 HelperThreadTask* GlobalHelperThreadState::maybeGetDelazifyTask(
1289     const AutoLockHelperThreadState& lock) {
1290   // NOTE: We want to span all cores availables with delazification tasks, in
1291   // order to parse a maximum number of functions ahead of their executions.
1292   // Thus, as opposed to parse task which have a higher priority, we are not
1293   // exclusively executing these task on parse threads.
1294   auto& worklist = delazifyWorklist(lock);
1295   if (worklist.isEmpty()) {
1296     return nullptr;
1297   }
1298   return worklist.popFirst();
1299 }
1300
1301 bool GlobalHelperThreadState::canStartDelazifyTask(
1302     const AutoLockHelperThreadState& lock) {
1303   return !delazifyWorklist(lock).isEmpty() &&
1304          checkTaskThreadLimit(THREAD_TYPE_DELAZIFY, maxDelazifyThreads(),
1305                               /*isMaster=*/true, lock);
1306 }
1307
1308 HelperThreadTask* GlobalHelperThreadState::maybeGetCompressionTask(
1309     const AutoLockHelperThreadState& lock) {
1310   if (!canStartCompressionTask(lock)) {
1311     return nullptr;
1312   }
1313
1314   auto& worklist = compressionWorklist(lock);
1315   UniquePtr<SourceCompressionTask> task = std::move(worklist.back());
1316   worklist.popBack();
1317   return task.release();
1318 }
1319
1320 bool GlobalHelperThreadState::canStartCompressionTask(
1321     const AutoLockHelperThreadState& lock) {
1322   return !compressionWorklist(lock).empty() &&
1323          checkTaskThreadLimit(THREAD_TYPE_COMPRESS, maxCompressionThreads(),
1324                               lock);
1325 }
1326
1327 void GlobalHelperThreadState::startHandlingCompressionTasks(
1328     ScheduleCompressionTask schedule, JSRuntime* maybeRuntime,
1329     const AutoLockHelperThreadState& lock) {
1330   MOZ_ASSERT((schedule == ScheduleCompressionTask::GC) ==
1331              (maybeRuntime != nullptr));
1332
1333   auto& pending = compressionPendingList(lock);
1334
1335   for (size_t i = 0; i < pending.length(); i++) {
1336     UniquePtr<SourceCompressionTask>& task = pending[i];
1337     if (schedule == ScheduleCompressionTask::API ||
1338         (task->runtimeMatches(maybeRuntime) && task->shouldStart())) {
1339       // OOMing during appending results in the task not being scheduled
1340       // and deleted.
1341       (void)submitTask(std::move(task), lock);
1342       remove(pending, &i);
1343     }
1344   }
1345 }
1346
1347 bool GlobalHelperThreadState::submitTask(
1348     UniquePtr<SourceCompressionTask> task,
1349     const AutoLockHelperThreadState& locked) {
1350   if (!compressionWorklist(locked).append(std::move(task))) {
1351     return false;
1352   }
1353
1354   dispatch(DispatchReason::NewTask, locked);
1355   return true;
1356 }
1357
1358 bool GlobalHelperThreadState::submitTask(
1359     GCParallelTask* task, const AutoLockHelperThreadState& locked) {
1360   gcParallelWorklist().insertBack(task, locked);
1361   dispatch(DispatchReason::NewTask, locked);
1362   return true;
1363 }
1364
1365 HelperThreadTask* GlobalHelperThreadState::maybeGetGCParallelTask(
1366     const AutoLockHelperThreadState& lock) {
1367   if (!canStartGCParallelTask(lock)) {
1368     return nullptr;
1369   }
1370
1371   return gcParallelWorklist().popFirst(lock);
1372 }
1373
1374 bool GlobalHelperThreadState::canStartGCParallelTask(
1375     const AutoLockHelperThreadState& lock) {
1376   return !gcParallelWorklist().isEmpty(lock) &&
1377          checkTaskThreadLimit(THREAD_TYPE_GCPARALLEL,
1378                               maxGCParallelThreads(lock), lock);
1379 }
1380
1381 bool js::EnqueueOffThreadCompression(JSContext* cx,
1382                                      UniquePtr<SourceCompressionTask> task) {
1383   AutoLockHelperThreadState lock;
1384
1385   auto& pending = HelperThreadState().compressionPendingList(lock);
1386   if (!pending.append(std::move(task))) {
1387     ReportOutOfMemory(cx);
1388     return false;
1389   }
1390
1391   return true;
1392 }
1393
1394 void js::StartHandlingCompressionsOnGC(JSRuntime* runtime) {
1395   AutoLockHelperThreadState lock;
1396   HelperThreadState().startHandlingCompressionTasks(
1397       GlobalHelperThreadState::ScheduleCompressionTask::GC, runtime, lock);
1398 }
1399
1400 template <typename T>
1401 static void ClearCompressionTaskList(T& list, JSRuntime* runtime) {
1402   for (size_t i = 0; i < list.length(); i++) {
1403     if (list[i]->runtimeMatches(runtime)) {
1404       HelperThreadState().remove(list, &i);
1405     }
1406   }
1407 }
1408
1409 void js::CancelOffThreadCompressions(JSRuntime* runtime) {
1410   if (!CanUseExtraThreads()) {
1411     return;
1412   }
1413
1414   AutoLockHelperThreadState lock;
1415
1416   // Cancel all pending compression tasks.
1417   ClearCompressionTaskList(HelperThreadState().compressionPendingList(lock),
1418                            runtime);
1419   ClearCompressionTaskList(HelperThreadState().compressionWorklist(lock),
1420                            runtime);
1421
1422   // Cancel all in-process compression tasks and wait for them to join so we
1423   // clean up the finished tasks.
1424   while (true) {
1425     bool inProgress = false;
1426     for (auto* helper : HelperThreadState().helperTasks(lock)) {
1427       if (!helper->is<SourceCompressionTask>()) {
1428         continue;
1429       }
1430
1431       if (helper->as<SourceCompressionTask>()->runtimeMatches(runtime)) {
1432         inProgress = true;
1433       }
1434     }
1435
1436     if (!inProgress) {
1437       break;
1438     }
1439
1440     HelperThreadState().wait(lock);
1441   }
1442
1443   // Clean up finished tasks.
1444   ClearCompressionTaskList(HelperThreadState().compressionFinishedList(lock),
1445                            runtime);
1446 }
1447
1448 void js::AttachFinishedCompressions(JSRuntime* runtime,
1449                                     AutoLockHelperThreadState& lock) {
1450   auto& finished = HelperThreadState().compressionFinishedList(lock);
1451   for (size_t i = 0; i < finished.length(); i++) {
1452     if (finished[i]->runtimeMatches(runtime)) {
1453       UniquePtr<SourceCompressionTask> compressionTask(std::move(finished[i]));
1454       HelperThreadState().remove(finished, &i);
1455       compressionTask->complete();
1456     }
1457   }
1458 }
1459
1460 void js::SweepPendingCompressions(AutoLockHelperThreadState& lock) {
1461   auto& pending = HelperThreadState().compressionPendingList(lock);
1462   for (size_t i = 0; i < pending.length(); i++) {
1463     if (pending[i]->shouldCancel()) {
1464       HelperThreadState().remove(pending, &i);
1465     }
1466   }
1467 }
1468
1469 void js::RunPendingSourceCompressions(JSRuntime* runtime) {
1470   if (!CanUseExtraThreads()) {
1471     return;
1472   }
1473
1474   AutoLockHelperThreadState lock;
1475
1476   HelperThreadState().startHandlingCompressionTasks(
1477       GlobalHelperThreadState::ScheduleCompressionTask::API, nullptr, lock);
1478
1479   // Wait until all tasks have started compression.
1480   while (!HelperThreadState().compressionWorklist(lock).empty()) {
1481     HelperThreadState().wait(lock);
1482   }
1483
1484   // Wait for all in-process compression tasks to complete.
1485   HelperThreadState().waitForAllTasksLocked(lock);
1486
1487   AttachFinishedCompressions(runtime, lock);
1488 }
1489
1490 void PromiseHelperTask::executeAndResolveAndDestroy(JSContext* cx) {
1491   execute();
1492   run(cx, JS::Dispatchable::NotShuttingDown);
1493 }
1494
1495 void PromiseHelperTask::runHelperThreadTask(AutoLockHelperThreadState& lock) {
1496   {
1497     AutoUnlockHelperThreadState unlock(lock);
1498     execute();
1499   }
1500
1501   // Don't release the lock between dispatching the resolve and destroy
1502   // operation (which may start immediately on another thread) and returning
1503   // from this method.
1504
1505   dispatchResolveAndDestroy(lock);
1506 }
1507
1508 bool js::StartOffThreadPromiseHelperTask(JSContext* cx,
1509                                          UniquePtr<PromiseHelperTask> task) {
1510   // Execute synchronously if there are no helper threads.
1511   if (!CanUseExtraThreads()) {
1512     task.release()->executeAndResolveAndDestroy(cx);
1513     return true;
1514   }
1515
1516   if (!HelperThreadState().submitTask(task.get())) {
1517     ReportOutOfMemory(cx);
1518     return false;
1519   }
1520
1521   (void)task.release();
1522   return true;
1523 }
1524
1525 bool js::StartOffThreadPromiseHelperTask(PromiseHelperTask* task) {
1526   MOZ_ASSERT(CanUseExtraThreads());
1527
1528   return HelperThreadState().submitTask(task);
1529 }
1530
1531 bool GlobalHelperThreadState::submitTask(PromiseHelperTask* task) {
1532   AutoLockHelperThreadState lock;
1533
1534   if (!promiseHelperTasks(lock).append(task)) {
1535     return false;
1536   }
1537
1538   dispatch(DispatchReason::NewTask, lock);
1539   return true;
1540 }
1541
1542 void GlobalHelperThreadState::trace(JSTracer* trc) {
1543   {
1544     AutoLockHelperThreadState lock;
1545
1546 #ifdef DEBUG
1547     // Since we hold the helper thread lock here we must disable GCMarker's
1548     // checking of the atom marking bitmap since that also relies on taking the
1549     // lock.
1550     GCMarker* marker = nullptr;
1551     if (trc->isMarkingTracer()) {
1552       marker = GCMarker::fromTracer(trc);
1553       marker->setCheckAtomMarking(false);
1554     }
1555     auto reenableAtomMarkingCheck = mozilla::MakeScopeExit([marker] {
1556       if (marker) {
1557         marker->setCheckAtomMarking(true);
1558       }
1559     });
1560 #endif
1561
1562     for (auto task : ionWorklist(lock)) {
1563       task->alloc().lifoAlloc()->setReadWrite();
1564       task->trace(trc);
1565       task->alloc().lifoAlloc()->setReadOnly();
1566     }
1567     for (auto task : ionFinishedList(lock)) {
1568       task->trace(trc);
1569     }
1570
1571     for (auto* helper : HelperThreadState().helperTasks(lock)) {
1572       if (helper->is<jit::IonCompileTask>()) {
1573         helper->as<jit::IonCompileTask>()->trace(trc);
1574       }
1575     }
1576   }
1577
1578   // The lazy link list is only accessed on the main thread, so trace it after
1579   // releasing the lock.
1580   JSRuntime* rt = trc->runtime();
1581   if (auto* jitRuntime = rt->jitRuntime()) {
1582     jit::IonCompileTask* task = jitRuntime->ionLazyLinkList(rt).getFirst();
1583     while (task) {
1584       task->trace(trc);
1585       task = task->getNext();
1586     }
1587   }
1588 }
1589
1590 // Definition of helper thread tasks.
1591 //
1592 // Priority is determined by the order they're listed here.
1593 const GlobalHelperThreadState::Selector GlobalHelperThreadState::selectors[] = {
1594     &GlobalHelperThreadState::maybeGetGCParallelTask,
1595     &GlobalHelperThreadState::maybeGetIonCompileTask,
1596     &GlobalHelperThreadState::maybeGetWasmTier1CompileTask,
1597     &GlobalHelperThreadState::maybeGetPromiseHelperTask,
1598     &GlobalHelperThreadState::maybeGetFreeDelazifyTask,
1599     &GlobalHelperThreadState::maybeGetDelazifyTask,
1600     &GlobalHelperThreadState::maybeGetCompressionTask,
1601     &GlobalHelperThreadState::maybeGetLowPrioIonCompileTask,
1602     &GlobalHelperThreadState::maybeGetIonFreeTask,
1603     &GlobalHelperThreadState::maybeGetWasmTier2CompileTask,
1604     &GlobalHelperThreadState::maybeGetWasmTier2GeneratorTask};
1605
1606 bool GlobalHelperThreadState::canStartTasks(
1607     const AutoLockHelperThreadState& lock) {
1608   return canStartGCParallelTask(lock) || canStartIonCompileTask(lock) ||
1609          canStartWasmTier1CompileTask(lock) ||
1610          canStartPromiseHelperTask(lock) || canStartFreeDelazifyTask(lock) ||
1611          canStartDelazifyTask(lock) || canStartCompressionTask(lock) ||
1612          canStartIonFreeTask(lock) || canStartWasmTier2CompileTask(lock) ||
1613          canStartWasmTier2GeneratorTask(lock);
1614 }
1615
1616 void JS::RunHelperThreadTask() {
1617   MOZ_ASSERT(CanUseExtraThreads());
1618
1619   AutoLockHelperThreadState lock;
1620
1621   if (!gHelperThreadState || HelperThreadState().isTerminating(lock)) {
1622     return;
1623   }
1624
1625   HelperThreadState().runOneTask(lock);
1626 }
1627
1628 void GlobalHelperThreadState::runOneTask(AutoLockHelperThreadState& lock) {
1629   MOZ_ASSERT(tasksPending_ > 0);
1630   tasksPending_--;
1631
1632   // The selectors may depend on the HelperThreadState not changing between task
1633   // selection and task execution, in particular, on new tasks not being added
1634   // (because of the lifo structure of the work lists). Unlocking the
1635   // HelperThreadState between task selection and execution is not well-defined.
1636   HelperThreadTask* task = findHighestPriorityTask(lock);
1637   if (task) {
1638     runTaskLocked(task, lock);
1639     dispatch(DispatchReason::FinishedTask, lock);
1640   }
1641
1642   notifyAll(lock);
1643 }
1644
1645 HelperThreadTask* GlobalHelperThreadState::findHighestPriorityTask(
1646     const AutoLockHelperThreadState& locked) {
1647   // Return the highest priority task that is ready to start, or nullptr.
1648
1649   for (const auto& selector : selectors) {
1650     if (auto* task = (this->*(selector))(locked)) {
1651       return task;
1652     }
1653   }
1654
1655   return nullptr;
1656 }
1657
1658 void GlobalHelperThreadState::runTaskLocked(HelperThreadTask* task,
1659                                             AutoLockHelperThreadState& locked) {
1660   JS::AutoSuppressGCAnalysis nogc;
1661
1662   HelperThreadState().helperTasks(locked).infallibleEmplaceBack(task);
1663
1664   ThreadType threadType = task->threadType();
1665   js::oom::SetThreadType(threadType);
1666   runningTaskCount[threadType]++;
1667   totalCountRunningTasks++;
1668
1669   task->runHelperThreadTask(locked);
1670
1671   // Delete task from helperTasks.
1672   HelperThreadState().helperTasks(locked).eraseIfEqual(task);
1673
1674   totalCountRunningTasks--;
1675   runningTaskCount[threadType]--;
1676
1677   js::oom::SetThreadType(js::THREAD_TYPE_NONE);
1678 }