js/src/vm/HelperThreads.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "vm/HelperThreads.h"
   8
   9 #include "mozilla/ReverseIterator.h"  // mozilla::Reversed(...)
  10 #include "mozilla/ScopeExit.h"
  11 #include "mozilla/Span.h"  // mozilla::Span<TaggedScriptThingIndex>
  12 #include "mozilla/Utf8.h"  // mozilla::Utf8Unit
  13
  14 #include <algorithm>
  15
  16 #include "frontend/CompilationStencil.h"  // frontend::CompilationStencil
  17 #include "gc/GC.h"
  18 #include "jit/Ion.h"
  19 #include "jit/IonCompileTask.h"
  20 #include "jit/JitRuntime.h"
  21 #include "jit/JitScript.h"
  22 #include "js/CompileOptions.h"  // JS::PrefableCompileOptions, JS::ReadOnlyCompileOptions
  23 #include "js/experimental/CompileScript.h"  // JS::ThreadStackQuotaForSize
  24 #include "js/friend/StackLimits.h"          // js::ReportOverRecursed
  25 #include "js/HelperThreadAPI.h"
  26 #include "js/Stack.h"
  27 #include "js/UniquePtr.h"
  28 #include "js/Utility.h"
  29 #include "threading/CpuCount.h"
  30 #include "vm/ErrorReporting.h"
  31 #include "vm/HelperThreadState.h"
  32 #include "vm/InternalThreadPool.h"
  33 #include "vm/MutexIDs.h"
  34 #include "wasm/WasmGenerator.h"
  35
  36 using namespace js;
  37
  38 using mozilla::TimeDuration;
  39 using mozilla::TimeStamp;
  40 using mozilla::Utf8Unit;
  41
  42 using JS::DispatchReason;
  43
  44 namespace js {
  45
  46 Mutex gHelperThreadLock(mutexid::GlobalHelperThreadState);
  47 GlobalHelperThreadState* gHelperThreadState = nullptr;
  48
  49 }  // namespace js
  50
  51 bool js::CreateHelperThreadsState() {
  52   MOZ_ASSERT(!gHelperThreadState);
  53   gHelperThreadState = js_new<GlobalHelperThreadState>();
  54   return gHelperThreadState;
  55 }
  56
  57 void js::DestroyHelperThreadsState() {
  58   AutoLockHelperThreadState lock;
  59
  60   if (!gHelperThreadState) {
  61     return;
  62   }
  63
  64   gHelperThreadState->finish(lock);
  65   js_delete(gHelperThreadState);
  66   gHelperThreadState = nullptr;
  67 }
  68
  69 bool js::EnsureHelperThreadsInitialized() {
  70   MOZ_ASSERT(gHelperThreadState);
  71   return gHelperThreadState->ensureInitialized();
  72 }
  73
  74 static size_t ClampDefaultCPUCount(size_t cpuCount) {
  75   // It's extremely rare for SpiderMonkey to have more than a few cores worth
  76   // of work. At higher core counts, performance can even decrease due to NUMA
  77   // (and SpiderMonkey's lack of NUMA-awareness), contention, and general lack
  78   // of optimization for high core counts. So to avoid wasting thread stack
  79   // resources (and cluttering gdb and core dumps), clamp to 8 cores for now.
  80   return std::min<size_t>(cpuCount, 8);
  81 }
  82
  83 static size_t ThreadCountForCPUCount(size_t cpuCount) {
  84   // We need at least two threads for tier-2 wasm compilations, because
  85   // there's a master task that holds a thread while other threads do the
  86   // compilation.
  87   return std::max<size_t>(cpuCount, 2);
  88 }
  89
  90 bool js::SetFakeCPUCount(size_t count) {
  91   HelperThreadState().setCpuCount(count);
  92   return true;
  93 }
  94
  95 void GlobalHelperThreadState::setCpuCount(size_t count) {
  96   // This must be called before any threads have been initialized.
  97   AutoLockHelperThreadState lock;
  98   MOZ_ASSERT(!isInitialized(lock));
  99
 100   // We can't do this if an external thread pool is in use.
 101   MOZ_ASSERT(!dispatchTaskCallback);
 102
 103   cpuCount = count;
 104   threadCount = ThreadCountForCPUCount(count);
 105 }
 106
 107 size_t js::GetHelperThreadCount() { return HelperThreadState().threadCount; }
 108
 109 size_t js::GetHelperThreadCPUCount() { return HelperThreadState().cpuCount; }
 110
 111 size_t js::GetMaxWasmCompilationThreads() {
 112   return HelperThreadState().maxWasmCompilationThreads();
 113 }
 114
 115 void JS::SetProfilingThreadCallbacks(
 116     JS::RegisterThreadCallback registerThread,
 117     JS::UnregisterThreadCallback unregisterThread) {
 118   HelperThreadState().registerThread = registerThread;
 119   HelperThreadState().unregisterThread = unregisterThread;
 120 }
 121
 122 // Bug 1630189: Without MOZ_NEVER_INLINE, Windows PGO builds have a linking
 123 // error for HelperThreadTaskCallback.
 124 JS_PUBLIC_API MOZ_NEVER_INLINE void JS::SetHelperThreadTaskCallback(
 125     HelperThreadTaskCallback callback, size_t threadCount, size_t stackSize) {
 126   AutoLockHelperThreadState lock;
 127   HelperThreadState().setDispatchTaskCallback(callback, threadCount, stackSize,
 128                                               lock);
 129 }
 130
 131 void GlobalHelperThreadState::setDispatchTaskCallback(
 132     JS::HelperThreadTaskCallback callback, size_t threadCount, size_t stackSize,
 133     const AutoLockHelperThreadState& lock) {
 134   MOZ_ASSERT(!isInitialized(lock));
 135   MOZ_ASSERT(!dispatchTaskCallback);
 136   MOZ_ASSERT(threadCount != 0);
 137   MOZ_ASSERT(stackSize >= 16 * 1024);
 138
 139   dispatchTaskCallback = callback;
 140   this->threadCount = threadCount;
 141   this->stackQuota = JS::ThreadStackQuotaForSize(stackSize);
 142 }
 143
 144 bool js::StartOffThreadWasmCompile(wasm::CompileTask* task,
 145                                    wasm::CompileMode mode) {
 146   return HelperThreadState().submitTask(task, mode);
 147 }
 148
 149 bool GlobalHelperThreadState::submitTask(wasm::CompileTask* task,
 150                                          wasm::CompileMode mode) {
 151   AutoLockHelperThreadState lock;
 152   if (!wasmWorklist(lock, mode).pushBack(task)) {
 153     return false;
 154   }
 155
 156   dispatch(DispatchReason::NewTask, lock);
 157   return true;
 158 }
 159
 160 size_t js::RemovePendingWasmCompileTasks(
 161     const wasm::CompileTaskState& taskState, wasm::CompileMode mode,
 162     const AutoLockHelperThreadState& lock) {
 163   wasm::CompileTaskPtrFifo& worklist =
 164       HelperThreadState().wasmWorklist(lock, mode);
 165   return worklist.eraseIf([&taskState](wasm::CompileTask* task) {
 166     return &task->state == &taskState;
 167   });
 168 }
 169
 170 void js::StartOffThreadWasmTier2Generator(wasm::UniqueTier2GeneratorTask task) {
 171   (void)HelperThreadState().submitTask(std::move(task));
 172 }
 173
 174 bool GlobalHelperThreadState::submitTask(wasm::UniqueTier2GeneratorTask task) {
 175   AutoLockHelperThreadState lock;
 176
 177   MOZ_ASSERT(isInitialized(lock));
 178
 179   if (!wasmTier2GeneratorWorklist(lock).append(task.get())) {
 180     return false;
 181   }
 182   (void)task.release();
 183
 184   dispatch(DispatchReason::NewTask, lock);
 185   return true;
 186 }
 187
 188 static void CancelOffThreadWasmTier2GeneratorLocked(
 189     AutoLockHelperThreadState& lock) {
 190   if (!HelperThreadState().isInitialized(lock)) {
 191     return;
 192   }
 193
 194   // Remove pending tasks from the tier2 generator worklist and cancel and
 195   // delete them.
 196   {
 197     wasm::Tier2GeneratorTaskPtrVector& worklist =
 198         HelperThreadState().wasmTier2GeneratorWorklist(lock);
 199     for (size_t i = 0; i < worklist.length(); i++) {
 200       wasm::Tier2GeneratorTask* task = worklist[i];
 201       HelperThreadState().remove(worklist, &i);
 202       js_delete(task);
 203     }
 204   }
 205
 206   // There is at most one running Tier2Generator task and we assume that
 207   // below.
 208   static_assert(GlobalHelperThreadState::MaxTier2GeneratorTasks == 1,
 209                 "code must be generalized");
 210
 211   // If there is a running Tier2 generator task, shut it down in a predictable
 212   // way.  The task will be deleted by the normal deletion logic.
 213   for (auto* helper : HelperThreadState().helperTasks(lock)) {
 214     if (helper->is<wasm::Tier2GeneratorTask>()) {
 215       // Set a flag that causes compilation to shortcut itself.
 216       helper->as<wasm::Tier2GeneratorTask>()->cancel();
 217
 218       // Wait for the generator task to finish.  This avoids a shutdown race
 219       // where the shutdown code is trying to shut down helper threads and the
 220       // ongoing tier2 compilation is trying to finish, which requires it to
 221       // have access to helper threads.
 222       uint32_t oldFinishedCount =
 223           HelperThreadState().wasmTier2GeneratorsFinished(lock);
 224       while (HelperThreadState().wasmTier2GeneratorsFinished(lock) ==
 225              oldFinishedCount) {
 226         HelperThreadState().wait(lock);
 227       }
 228
 229       // At most one of these tasks.
 230       break;
 231     }
 232   }
 233 }
 234
 235 void js::CancelOffThreadWasmTier2Generator() {
 236   AutoLockHelperThreadState lock;
 237   CancelOffThreadWasmTier2GeneratorLocked(lock);
 238 }
 239
 240 bool js::StartOffThreadIonCompile(jit::IonCompileTask* task,
 241                                   const AutoLockHelperThreadState& lock) {
 242   return HelperThreadState().submitTask(task, lock);
 243 }
 244
 245 bool GlobalHelperThreadState::submitTask(
 246     jit::IonCompileTask* task, const AutoLockHelperThreadState& locked) {
 247   MOZ_ASSERT(isInitialized(locked));
 248
 249   if (!ionWorklist(locked).append(task)) {
 250     return false;
 251   }
 252
 253   // The build is moving off-thread. Freeze the LifoAlloc to prevent any
 254   // unwanted mutations.
 255   task->alloc().lifoAlloc()->setReadOnly();
 256
 257   dispatch(DispatchReason::NewTask, locked);
 258   return true;
 259 }
 260
 261 bool js::AutoStartIonFreeTask::addIonCompileToFreeTaskBatch(
 262     jit::IonCompileTask* task) {
 263   return jitRuntime_->addIonCompileToFreeTaskBatch(task);
 264 }
 265
 266 js::AutoStartIonFreeTask::~AutoStartIonFreeTask() {
 267   jitRuntime_->maybeStartIonFreeTask(force_);
 268 }
 269
 270 void jit::JitRuntime::maybeStartIonFreeTask(bool force) {
 271   IonFreeCompileTasks& tasks = ionFreeTaskBatch_.ref();
 272   if (tasks.empty()) {
 273     return;
 274   }
 275
 276   // Start an IonFreeTask if we have at least eight tasks. If |force| is true we
 277   // always start an IonFreeTask.
 278   if (!force) {
 279     constexpr size_t MinBatchSize = 8;
 280     static_assert(IonFreeCompileTasks::InlineLength >= MinBatchSize,
 281                   "Minimum batch size shouldn't require malloc");
 282     if (tasks.length() < MinBatchSize) {
 283       return;
 284     }
 285   }
 286
 287   auto freeTask = js::MakeUnique<jit::IonFreeTask>(std::move(tasks));
 288   if (!freeTask) {
 289     // Free compilation data on the main thread instead.
 290     MOZ_ASSERT(!tasks.empty(), "shouldn't have moved tasks on OOM");
 291     jit::FreeIonCompileTasks(tasks);
 292     tasks.clearAndFree();
 293     return;
 294   }
 295
 296   AutoLockHelperThreadState lock;
 297   if (!HelperThreadState().submitTask(std::move(freeTask), lock)) {
 298     // If submitTask OOMs, then freeTask hasn't been moved so we can still use
 299     // its task list.
 300     jit::FreeIonCompileTasks(freeTask->compileTasks());
 301   }
 302
 303   tasks.clearAndFree();
 304 }
 305
 306 bool GlobalHelperThreadState::submitTask(
 307     UniquePtr<jit::IonFreeTask>&& task,
 308     const AutoLockHelperThreadState& locked) {
 309   MOZ_ASSERT(isInitialized(locked));
 310
 311   if (!ionFreeList(locked).append(std::move(task))) {
 312     return false;
 313   }
 314
 315   dispatch(DispatchReason::NewTask, locked);
 316   return true;
 317 }
 318
 319 /*
 320  * Move an IonCompilationTask for which compilation has either finished, failed,
 321  * or been cancelled into the global finished compilation list. All off thread
 322  * compilations which are started must eventually be finished.
 323  */
 324 void js::FinishOffThreadIonCompile(jit::IonCompileTask* task,
 325                                    const AutoLockHelperThreadState& lock) {
 326   AutoEnterOOMUnsafeRegion oomUnsafe;
 327   if (!HelperThreadState().ionFinishedList(lock).append(task)) {
 328     oomUnsafe.crash("FinishOffThreadIonCompile");
 329   }
 330   task->script()
 331       ->runtimeFromAnyThread()
 332       ->jitRuntime()
 333       ->numFinishedOffThreadTasksRef(lock)++;
 334 }
 335
 336 static JSRuntime* GetSelectorRuntime(const CompilationSelector& selector) {
 337   struct Matcher {
 338     JSRuntime* operator()(JSScript* script) {
 339       return script->runtimeFromMainThread();
 340     }
 341     JSRuntime* operator()(Zone* zone) { return zone->runtimeFromMainThread(); }
 342     JSRuntime* operator()(ZonesInState zbs) { return zbs.runtime; }
 343     JSRuntime* operator()(JSRuntime* runtime) { return runtime; }
 344   };
 345
 346   return selector.match(Matcher());
 347 }
 348
 349 static bool JitDataStructuresExist(const CompilationSelector& selector) {
 350   struct Matcher {
 351     bool operator()(JSScript* script) { return !!script->zone()->jitZone(); }
 352     bool operator()(Zone* zone) { return !!zone->jitZone(); }
 353     bool operator()(ZonesInState zbs) { return zbs.runtime->hasJitRuntime(); }
 354     bool operator()(JSRuntime* runtime) { return runtime->hasJitRuntime(); }
 355   };
 356
 357   return selector.match(Matcher());
 358 }
 359
 360 static bool IonCompileTaskMatches(const CompilationSelector& selector,
 361                                   jit::IonCompileTask* task) {
 362   struct TaskMatches {
 363     jit::IonCompileTask* task_;
 364
 365     bool operator()(JSScript* script) { return script == task_->script(); }
 366     bool operator()(Zone* zone) {
 367       return zone == task_->script()->zoneFromAnyThread();
 368     }
 369     bool operator()(JSRuntime* runtime) {
 370       return runtime == task_->script()->runtimeFromAnyThread();
 371     }
 372     bool operator()(ZonesInState zbs) {
 373       return zbs.runtime == task_->script()->runtimeFromAnyThread() &&
 374              zbs.state == task_->script()->zoneFromAnyThread()->gcState();
 375     }
 376   };
 377
 378   return selector.match(TaskMatches{task});
 379 }
 380
 381 // If we're canceling Ion compilations for a zone/runtime, force a new
 382 // IonFreeTask even if there are just a few tasks. This lets us free as much
 383 // memory as possible.
 384 static bool ShouldForceIonFreeTask(const CompilationSelector& selector) {
 385   struct Matcher {
 386     bool operator()(JSScript* script) { return false; }
 387     bool operator()(Zone* zone) { return true; }
 388     bool operator()(ZonesInState zbs) { return true; }
 389     bool operator()(JSRuntime* runtime) { return true; }
 390   };
 391
 392   return selector.match(Matcher());
 393 }
 394
 395 void js::CancelOffThreadIonCompile(const CompilationSelector& selector) {
 396   if (!JitDataStructuresExist(selector)) {
 397     return;
 398   }
 399
 400   if (jit::IsPortableBaselineInterpreterEnabled()) {
 401     return;
 402   }
 403
 404   jit::JitRuntime* jitRuntime = GetSelectorRuntime(selector)->jitRuntime();
 405   MOZ_ASSERT(jitRuntime);
 406
 407   AutoStartIonFreeTask freeTask(jitRuntime, ShouldForceIonFreeTask(selector));
 408
 409   {
 410     AutoLockHelperThreadState lock;
 411     if (!HelperThreadState().isInitialized(lock)) {
 412       return;
 413     }
 414
 415     /* Cancel any pending entries for which processing hasn't started. */
 416     GlobalHelperThreadState::IonCompileTaskVector& worklist =
 417         HelperThreadState().ionWorklist(lock);
 418     for (size_t i = 0; i < worklist.length(); i++) {
 419       jit::IonCompileTask* task = worklist[i];
 420       if (IonCompileTaskMatches(selector, task)) {
 421         // Once finished, tasks are added to a Linked list which is
 422         // allocated with the IonCompileTask class. The IonCompileTask is
 423         // allocated in the LifoAlloc so we need the LifoAlloc to be mutable.
 424         worklist[i]->alloc().lifoAlloc()->setReadWrite();
 425
 426         FinishOffThreadIonCompile(task, lock);
 427         HelperThreadState().remove(worklist, &i);
 428       }
 429     }
 430
 431     /* Wait for in progress entries to finish up. */
 432     bool cancelled;
 433     do {
 434       cancelled = false;
 435       for (auto* helper : HelperThreadState().helperTasks(lock)) {
 436         if (!helper->is<jit::IonCompileTask>()) {
 437           continue;
 438         }
 439
 440         jit::IonCompileTask* ionCompileTask = helper->as<jit::IonCompileTask>();
 441         if (IonCompileTaskMatches(selector, ionCompileTask)) {
 442           ionCompileTask->mirGen().cancel();
 443           cancelled = true;
 444         }
 445       }
 446       if (cancelled) {
 447         HelperThreadState().wait(lock);
 448       }
 449     } while (cancelled);
 450
 451     /* Cancel code generation for any completed entries. */
 452     GlobalHelperThreadState::IonCompileTaskVector& finished =
 453         HelperThreadState().ionFinishedList(lock);
 454     for (size_t i = 0; i < finished.length(); i++) {
 455       jit::IonCompileTask* task = finished[i];
 456       if (IonCompileTaskMatches(selector, task)) {
 457         JSRuntime* rt = task->script()->runtimeFromAnyThread();
 458         rt->jitRuntime()->numFinishedOffThreadTasksRef(lock)--;
 459         jit::FinishOffThreadTask(rt, freeTask, task);
 460         HelperThreadState().remove(finished, &i);
 461       }
 462     }
 463   }
 464
 465   /* Cancel lazy linking for pending tasks (attached to the ionScript). */
 466   JSRuntime* runtime = GetSelectorRuntime(selector);
 467   jit::IonCompileTask* task =
 468       runtime->jitRuntime()->ionLazyLinkList(runtime).getFirst();
 469   while (task) {
 470     jit::IonCompileTask* next = task->getNext();
 471     if (IonCompileTaskMatches(selector, task)) {
 472       jit::FinishOffThreadTask(runtime, freeTask, task);
 473     }
 474     task = next;
 475   }
 476 }
 477
 478 #ifdef DEBUG
 479 bool js::HasOffThreadIonCompile(Zone* zone) {
 480   if (jit::IsPortableBaselineInterpreterEnabled()) {
 481     return false;
 482   }
 483
 484   AutoLockHelperThreadState lock;
 485
 486   if (!HelperThreadState().isInitialized(lock)) {
 487     return false;
 488   }
 489
 490   GlobalHelperThreadState::IonCompileTaskVector& worklist =
 491       HelperThreadState().ionWorklist(lock);
 492   for (size_t i = 0; i < worklist.length(); i++) {
 493     jit::IonCompileTask* task = worklist[i];
 494     if (task->script()->zoneFromAnyThread() == zone) {
 495       return true;
 496     }
 497   }
 498
 499   for (auto* helper : HelperThreadState().helperTasks(lock)) {
 500     if (!helper->is<jit::IonCompileTask>()) {
 501       continue;
 502     }
 503     JSScript* script = helper->as<jit::IonCompileTask>()->script();
 504     if (script->zoneFromAnyThread() == zone) {
 505       return true;
 506     }
 507   }
 508
 509   GlobalHelperThreadState::IonCompileTaskVector& finished =
 510       HelperThreadState().ionFinishedList(lock);
 511   for (size_t i = 0; i < finished.length(); i++) {
 512     jit::IonCompileTask* task = finished[i];
 513     if (task->script()->zoneFromAnyThread() == zone) {
 514       return true;
 515     }
 516   }
 517
 518   JSRuntime* rt = zone->runtimeFromMainThread();
 519   if (rt->hasJitRuntime()) {
 520     jit::IonCompileTask* task =
 521         rt->jitRuntime()->ionLazyLinkList(rt).getFirst();
 522     while (task) {
 523       if (task->script()->zone() == zone) {
 524         return true;
 525       }
 526       task = task->getNext();
 527     }
 528   }
 529
 530   return false;
 531 }
 532 #endif
 533
 534 void js::StartOffThreadDelazification(
 535     JSContext* maybeCx, const JS::ReadOnlyCompileOptions& options,
 536     const frontend::CompilationStencil& stencil) {
 537   // Skip delazify tasks if we parse everything on-demand or ahead.
 538   auto strategy = options.eagerDelazificationStrategy();
 539   if (strategy == JS::DelazificationOption::OnDemandOnly ||
 540       strategy == JS::DelazificationOption::ParseEverythingEagerly) {
 541     return;
 542   }
 543
 544   // Skip delazify task if code coverage is enabled.
 545   if (maybeCx && maybeCx->realm()->collectCoverageForDebug()) {
 546     return;
 547   }
 548
 549   if (!CanUseExtraThreads()) {
 550     return;
 551   }
 552
 553   JSRuntime* maybeRuntime = maybeCx ? maybeCx->runtime() : nullptr;
 554   UniquePtr<DelazifyTask> task;
 555   task = DelazifyTask::Create(maybeRuntime, options, stencil);
 556   if (!task) {
 557     return;
 558   }
 559
 560   // Schedule delazification task if there is any function to delazify.
 561   if (!task->done()) {
 562     AutoLockHelperThreadState lock;
 563     HelperThreadState().submitTask(task.release(), lock);
 564   }
 565 }
 566
 567 UniquePtr<DelazifyTask> DelazifyTask::Create(
 568     JSRuntime* maybeRuntime, const JS::ReadOnlyCompileOptions& options,
 569     const frontend::CompilationStencil& stencil) {
 570   UniquePtr<DelazifyTask> task;
 571   task.reset(js_new<DelazifyTask>(maybeRuntime, options.prefableOptions()));
 572   if (!task) {
 573     return nullptr;
 574   }
 575
 576   if (!task->init(options, stencil)) {
 577     // In case of errors, skip this and delazify on-demand.
 578     return nullptr;
 579   }
 580
 581   return task;
 582 }
 583
 584 DelazifyTask::DelazifyTask(
 585     JSRuntime* maybeRuntime,
 586     const JS::PrefableCompileOptions& initialPrefableOptions)
 587     : maybeRuntime(maybeRuntime),
 588       delazificationCx(initialPrefableOptions, HelperThreadState().stackQuota) {
 589 }
 590
 591 DelazifyTask::~DelazifyTask() {
 592   // The LinkedListElement destructor will remove us from any list we are part
 593   // of without synchronization, so ensure that doesn't happen.
 594   MOZ_DIAGNOSTIC_ASSERT(!isInList());
 595 }
 596
 597 bool DelazifyTask::init(const JS::ReadOnlyCompileOptions& options,
 598                         const frontend::CompilationStencil& stencil) {
 599   return delazificationCx.init(options, stencil);
 600 }
 601
 602 size_t DelazifyTask::sizeOfExcludingThis(
 603     mozilla::MallocSizeOf mallocSizeOf) const {
 604   return delazificationCx.sizeOfExcludingThis(mallocSizeOf);
 605 }
 606
 607 void DelazifyTask::runHelperThreadTask(AutoLockHelperThreadState& lock) {
 608   {
 609     AutoUnlockHelperThreadState unlock(lock);
 610     // NOTE: We do not report errors beyond this scope, as there is no where
 611     // to report these errors to. In the mean time, prevent the eager
 612     // delazification from running after any kind of errors.
 613     (void)runTask();
 614   }
 615
 616   // If we should continue to delazify even more functions, then re-add this
 617   // task to the vector of delazification tasks. This might happen when the
 618   // DelazifyTask is interrupted by a higher priority task. (see
 619   // mozilla::TaskController & mozilla::Task)
 620   if (!delazificationCx.done()) {
 621     HelperThreadState().submitTask(this, lock);
 622   } else {
 623     UniquePtr<FreeDelazifyTask> freeTask(js_new<FreeDelazifyTask>(this));
 624     if (freeTask) {
 625       HelperThreadState().submitTask(std::move(freeTask), lock);
 626     }
 627   }
 628 }
 629
 630 bool DelazifyTask::runTask() { return delazificationCx.delazify(); }
 631
 632 bool DelazifyTask::done() const { return delazificationCx.done(); }
 633
 634 void FreeDelazifyTask::runHelperThreadTask(AutoLockHelperThreadState& locked) {
 635   {
 636     AutoUnlockHelperThreadState unlock(locked);
 637     js_delete(task);
 638     task = nullptr;
 639   }
 640
 641   js_delete(this);
 642 }
 643
 644 static void CancelPendingDelazifyTask(JSRuntime* rt,
 645                                       AutoLockHelperThreadState& lock) {
 646   auto& delazifyList = HelperThreadState().delazifyWorklist(lock);
 647
 648   auto end = delazifyList.end();
 649   for (auto iter = delazifyList.begin(); iter != end;) {
 650     DelazifyTask* task = *iter;
 651     ++iter;
 652     if (task->runtimeMatchesOrNoRuntime(rt)) {
 653       task->removeFrom(delazifyList);
 654       js_delete(task);
 655     }
 656   }
 657 }
 658
 659 static void WaitUntilCancelledDelazifyTasks(JSRuntime* rt,
 660                                             AutoLockHelperThreadState& lock) {
 661   if (!HelperThreadState().isInitialized(lock)) {
 662     return;
 663   }
 664
 665   while (true) {
 666     CancelPendingDelazifyTask(rt, lock);
 667
 668     // If running tasks are delazifying any functions, then we have to wait
 669     // until they complete to remove them from the pending list. DelazifyTask
 670     // are inserting themself back to be processed once more after delazifying a
 671     // function.
 672     bool inProgress = false;
 673     for (auto* helper : HelperThreadState().helperTasks(lock)) {
 674       if (helper->is<DelazifyTask>() &&
 675           helper->as<DelazifyTask>()->runtimeMatchesOrNoRuntime(rt)) {
 676         inProgress = true;
 677         break;
 678       }
 679     }
 680     if (!inProgress) {
 681       break;
 682     }
 683
 684     HelperThreadState().wait(lock);
 685   }
 686
 687 #ifdef DEBUG
 688   for (DelazifyTask* task : HelperThreadState().delazifyWorklist(lock)) {
 689     MOZ_ASSERT(!task->runtimeMatchesOrNoRuntime(rt));
 690   }
 691   for (auto* helper : HelperThreadState().helperTasks(lock)) {
 692     MOZ_ASSERT_IF(helper->is<DelazifyTask>(),
 693                   !helper->as<DelazifyTask>()->runtimeMatchesOrNoRuntime(rt));
 694   }
 695 #endif
 696 }
 697
 698 static void WaitUntilEmptyFreeDelazifyTaskVector(
 699     AutoLockHelperThreadState& lock) {
 700   if (!HelperThreadState().isInitialized(lock)) {
 701     return;
 702   }
 703
 704   while (true) {
 705     bool inProgress = false;
 706     auto& freeList = HelperThreadState().freeDelazifyTaskVector(lock);
 707     if (!freeList.empty()) {
 708       inProgress = true;
 709     }
 710
 711     // If running tasks are delazifying any functions, then we have to wait
 712     // until they complete to remove them from the pending list. DelazifyTask
 713     // are inserting themself back to be processed once more after delazifying a
 714     // function.
 715     for (auto* helper : HelperThreadState().helperTasks(lock)) {
 716       if (helper->is<FreeDelazifyTask>()) {
 717         inProgress = true;
 718         break;
 719       }
 720     }
 721     if (!inProgress) {
 722       break;
 723     }
 724
 725     HelperThreadState().wait(lock);
 726   }
 727 }
 728
 729 void js::CancelOffThreadDelazify(JSRuntime* runtime) {
 730   AutoLockHelperThreadState lock;
 731
 732   // Cancel all Delazify tasks from the given runtime, and wait if tasks are
 733   // from the given runtime are being executed.
 734   WaitUntilCancelledDelazifyTasks(runtime, lock);
 735
 736   // Empty the free list of delazify task, in case one of the delazify task
 737   // ended and therefore did not returned to the pending list of delazify tasks.
 738   WaitUntilEmptyFreeDelazifyTaskVector(lock);
 739 }
 740
 741 static bool HasAnyDelazifyTask(JSRuntime* rt, AutoLockHelperThreadState& lock) {
 742   auto& delazifyList = HelperThreadState().delazifyWorklist(lock);
 743   for (auto task : delazifyList) {
 744     if (task->runtimeMatchesOrNoRuntime(rt)) {
 745       return true;
 746     }
 747   }
 748
 749   for (auto* helper : HelperThreadState().helperTasks(lock)) {
 750     if (helper->is<DelazifyTask>() &&
 751         helper->as<DelazifyTask>()->runtimeMatchesOrNoRuntime(rt)) {
 752       return true;
 753     }
 754   }
 755
 756   return false;
 757 }
 758
 759 void js::WaitForAllDelazifyTasks(JSRuntime* rt) {
 760   AutoLockHelperThreadState lock;
 761   if (!HelperThreadState().isInitialized(lock)) {
 762     return;
 763   }
 764
 765   while (true) {
 766     if (!HasAnyDelazifyTask(rt, lock)) {
 767       break;
 768     }
 769
 770     HelperThreadState().wait(lock);
 771   }
 772 }
 773
 774 void GlobalHelperThreadState::submitTask(
 775     DelazifyTask* task, const AutoLockHelperThreadState& locked) {
 776   delazifyWorklist(locked).insertBack(task);
 777   dispatch(DispatchReason::NewTask, locked);
 778 }
 779
 780 bool GlobalHelperThreadState::submitTask(
 781     UniquePtr<FreeDelazifyTask> task, const AutoLockHelperThreadState& locked) {
 782   if (!freeDelazifyTaskVector(locked).append(std::move(task))) {
 783     return false;
 784   }
 785   dispatch(DispatchReason::NewTask, locked);
 786   return true;
 787 }
 788
 789 bool GlobalHelperThreadState::ensureInitialized() {
 790   MOZ_ASSERT(CanUseExtraThreads());
 791   MOZ_ASSERT(this == &HelperThreadState());
 792
 793   AutoLockHelperThreadState lock;
 794
 795   if (isInitialized(lock)) {
 796     return true;
 797   }
 798
 799   for (size_t& i : runningTaskCount) {
 800     i = 0;
 801   }
 802
 803   useInternalThreadPool_ = !dispatchTaskCallback;
 804   if (useInternalThreadPool(lock)) {
 805     if (!InternalThreadPool::Initialize(threadCount, lock)) {
 806       return false;
 807     }
 808   }
 809
 810   MOZ_ASSERT(dispatchTaskCallback);
 811
 812   if (!ensureThreadCount(threadCount, lock)) {
 813     finishThreads(lock);
 814     return false;
 815   }
 816
 817   MOZ_ASSERT(threadCount != 0);
 818   isInitialized_ = true;
 819   return true;
 820 }
 821
 822 bool GlobalHelperThreadState::ensureThreadCount(
 823     size_t count, AutoLockHelperThreadState& lock) {
 824   if (!helperTasks_.reserve(count)) {
 825     return false;
 826   }
 827
 828   if (useInternalThreadPool(lock)) {
 829     InternalThreadPool& pool = InternalThreadPool::Get();
 830     if (pool.threadCount(lock) < count) {
 831       if (!pool.ensureThreadCount(count, lock)) {
 832         return false;
 833       }
 834
 835       threadCount = pool.threadCount(lock);
 836     }
 837   }
 838
 839   return true;
 840 }
 841
 842 GlobalHelperThreadState::GlobalHelperThreadState()
 843     : cpuCount(0),
 844       threadCount(0),
 845       totalCountRunningTasks(0),
 846       registerThread(nullptr),
 847       unregisterThread(nullptr),
 848       wasmTier2GeneratorsFinished_(0) {
 849   MOZ_ASSERT(!gHelperThreadState);
 850
 851   cpuCount = ClampDefaultCPUCount(GetCPUCount());
 852   threadCount = ThreadCountForCPUCount(cpuCount);
 853
 854   MOZ_ASSERT(cpuCount > 0, "GetCPUCount() seems broken");
 855 }
 856
 857 void GlobalHelperThreadState::finish(AutoLockHelperThreadState& lock) {
 858   if (!isInitialized(lock)) {
 859     return;
 860   }
 861
 862   MOZ_ASSERT_IF(!JSRuntime::hasLiveRuntimes(), gcParallelMarkingThreads == 0);
 863
 864   finishThreads(lock);
 865
 866   // Make sure there are no Ion free tasks left. We check this here because,
 867   // unlike the other tasks, we don't explicitly block on this when
 868   // destroying a runtime.
 869   auto& freeList = ionFreeList(lock);
 870   while (!freeList.empty()) {
 871     UniquePtr<jit::IonFreeTask> task = std::move(freeList.back());
 872     freeList.popBack();
 873     jit::FreeIonCompileTasks(task->compileTasks());
 874   }
 875 }
 876
 877 void GlobalHelperThreadState::finishThreads(AutoLockHelperThreadState& lock) {
 878   waitForAllTasksLocked(lock);
 879   terminating_ = true;
 880
 881   if (InternalThreadPool::IsInitialized()) {
 882     InternalThreadPool::ShutDown(lock);
 883   }
 884 }
 885
 886 #ifdef DEBUG
 887 void GlobalHelperThreadState::assertIsLockedByCurrentThread() const {
 888   gHelperThreadLock.assertOwnedByCurrentThread();
 889 }
 890 #endif  // DEBUG
 891
 892 void GlobalHelperThreadState::dispatch(
 893     DispatchReason reason, const AutoLockHelperThreadState& locked) {
 894   if (canStartTasks(locked) && tasksPending_ < threadCount) {
 895     // This doesn't guarantee that we don't dispatch more tasks to the external
 896     // pool than necessary if tasks are taking a long time to start, but it does
 897     // limit the number.
 898     tasksPending_++;
 899
 900     // The hazard analysis can't tell that the callback doesn't GC.
 901     JS::AutoSuppressGCAnalysis nogc;
 902
 903     dispatchTaskCallback(reason);
 904   }
 905 }
 906
 907 void GlobalHelperThreadState::wait(
 908     AutoLockHelperThreadState& locked,
 909     TimeDuration timeout /* = TimeDuration::Forever() */) {
 910   consumerWakeup.wait_for(locked, timeout);
 911 }
 912
 913 void GlobalHelperThreadState::notifyAll(const AutoLockHelperThreadState&) {
 914   consumerWakeup.notify_all();
 915 }
 916
 917 void GlobalHelperThreadState::notifyOne(const AutoLockHelperThreadState&) {
 918   consumerWakeup.notify_one();
 919 }
 920
 921 bool GlobalHelperThreadState::hasActiveThreads(
 922     const AutoLockHelperThreadState& lock) {
 923   return !helperTasks(lock).empty();
 924 }
 925
 926 void js::WaitForAllHelperThreads() { HelperThreadState().waitForAllTasks(); }
 927
 928 void js::WaitForAllHelperThreads(AutoLockHelperThreadState& lock) {
 929   HelperThreadState().waitForAllTasksLocked(lock);
 930 }
 931
 932 void GlobalHelperThreadState::waitForAllTasks() {
 933   AutoLockHelperThreadState lock;
 934   waitForAllTasksLocked(lock);
 935 }
 936
 937 void GlobalHelperThreadState::waitForAllTasksLocked(
 938     AutoLockHelperThreadState& lock) {
 939   CancelOffThreadWasmTier2GeneratorLocked(lock);
 940
 941   while (canStartTasks(lock) || tasksPending_ || hasActiveThreads(lock)) {
 942     wait(lock);
 943   }
 944
 945   MOZ_ASSERT(gcParallelWorklist().isEmpty(lock));
 946   MOZ_ASSERT(ionWorklist(lock).empty());
 947   MOZ_ASSERT(wasmWorklist(lock, wasm::CompileMode::Tier1).empty());
 948   MOZ_ASSERT(promiseHelperTasks(lock).empty());
 949   MOZ_ASSERT(compressionWorklist(lock).empty());
 950   MOZ_ASSERT(ionFreeList(lock).empty());
 951   MOZ_ASSERT(wasmWorklist(lock, wasm::CompileMode::Tier2).empty());
 952   MOZ_ASSERT(wasmTier2GeneratorWorklist(lock).empty());
 953   MOZ_ASSERT(!tasksPending_);
 954   MOZ_ASSERT(!hasActiveThreads(lock));
 955 }
 956
 957 // A task can be a "master" task, ie, it will block waiting for other worker
 958 // threads that perform work on its behalf.  If so it must not take the last
 959 // available thread; there must always be at least one worker thread able to do
 960 // the actual work.  (Or the system may deadlock.)
 961 //
 962 // If a task is a master task it *must* pass isMaster=true here, or perform a
 963 // similar calculation to avoid deadlock from starvation.
 964 //
 965 // isMaster should only be true if the thread calling checkTaskThreadLimit() is
 966 // a helper thread.
 967 //
 968 // NOTE: Calling checkTaskThreadLimit() from a helper thread in the dynamic
 969 // region after currentTask.emplace() and before currentTask.reset() may cause
 970 // it to return a different result than if it is called outside that dynamic
 971 // region, as the predicate inspects the values of the threads' currentTask
 972 // members.
 973
 974 bool GlobalHelperThreadState::checkTaskThreadLimit(
 975     ThreadType threadType, size_t maxThreads, bool isMaster,
 976     const AutoLockHelperThreadState& lock) const {
 977   MOZ_ASSERT(maxThreads > 0);
 978
 979   if (!isMaster && maxThreads >= threadCount) {
 980     return true;
 981   }
 982
 983   size_t count = runningTaskCount[threadType];
 984   if (count >= maxThreads) {
 985     return false;
 986   }
 987
 988   MOZ_ASSERT(threadCount >= totalCountRunningTasks);
 989   size_t idle = threadCount - totalCountRunningTasks;
 990
 991   // It is possible for the number of idle threads to be zero here, because
 992   // checkTaskThreadLimit() can be called from non-helper threads.  Notably,
 993   // the compression task scheduler invokes it, and runs off a helper thread.
 994   if (idle == 0) {
 995     return false;
 996   }
 997
 998   // A master thread that's the last available thread must not be allowed to
 999   // run.
1000   if (isMaster && idle == 1) {
1001     return false;
1002   }
1003
1004   return true;
1005 }
1006
1007 static inline bool IsHelperThreadSimulatingOOM(js::ThreadType threadType) {
1008 #if defined(DEBUG) || defined(JS_OOM_BREAKPOINT)
1009   return js::oom::simulator.targetThread() == threadType;
1010 #else
1011   return false;
1012 #endif
1013 }
1014
1015 void GlobalHelperThreadState::addSizeOfIncludingThis(
1016     JS::GlobalStats* stats, const AutoLockHelperThreadState& lock) const {
1017 #ifdef DEBUG
1018   assertIsLockedByCurrentThread();
1019 #endif
1020
1021   mozilla::MallocSizeOf mallocSizeOf = stats->mallocSizeOf_;
1022   JS::HelperThreadStats& htStats = stats->helperThread;
1023
1024   htStats.stateData += mallocSizeOf(this);
1025
1026   if (InternalThreadPool::IsInitialized()) {
1027     htStats.stateData +=
1028         InternalThreadPool::Get().sizeOfIncludingThis(mallocSizeOf, lock);
1029   }
1030
1031   // Report memory used by various containers
1032   htStats.stateData +=
1033       ionWorklist_.sizeOfExcludingThis(mallocSizeOf) +
1034       ionFinishedList_.sizeOfExcludingThis(mallocSizeOf) +
1035       ionFreeList_.sizeOfExcludingThis(mallocSizeOf) +
1036       wasmWorklist_tier1_.sizeOfExcludingThis(mallocSizeOf) +
1037       wasmWorklist_tier2_.sizeOfExcludingThis(mallocSizeOf) +
1038       wasmTier2GeneratorWorklist_.sizeOfExcludingThis(mallocSizeOf) +
1039       promiseHelperTasks_.sizeOfExcludingThis(mallocSizeOf) +
1040       compressionPendingList_.sizeOfExcludingThis(mallocSizeOf) +
1041       compressionWorklist_.sizeOfExcludingThis(mallocSizeOf) +
1042       compressionFinishedList_.sizeOfExcludingThis(mallocSizeOf) +
1043       gcParallelWorklist_.sizeOfExcludingThis(mallocSizeOf, lock) +
1044       helperTasks_.sizeOfExcludingThis(mallocSizeOf);
1045
1046   // Report IonCompileTasks on wait lists
1047   for (auto task : ionWorklist_) {
1048     htStats.ionCompileTask += task->sizeOfExcludingThis(mallocSizeOf);
1049   }
1050   for (auto task : ionFinishedList_) {
1051     htStats.ionCompileTask += task->sizeOfExcludingThis(mallocSizeOf);
1052   }
1053   for (const auto& task : ionFreeList_) {
1054     for (auto* compileTask : task->compileTasks()) {
1055       htStats.ionCompileTask += compileTask->sizeOfExcludingThis(mallocSizeOf);
1056     }
1057   }
1058
1059   // Report wasm::CompileTasks on wait lists
1060   for (auto task : wasmWorklist_tier1_) {
1061     htStats.wasmCompile += task->sizeOfExcludingThis(mallocSizeOf);
1062   }
1063   for (auto task : wasmWorklist_tier2_) {
1064     htStats.wasmCompile += task->sizeOfExcludingThis(mallocSizeOf);
1065   }
1066
1067   // Report number of helper threads.
1068   MOZ_ASSERT(htStats.idleThreadCount == 0);
1069   MOZ_ASSERT(threadCount >= totalCountRunningTasks);
1070   htStats.activeThreadCount = totalCountRunningTasks;
1071   htStats.idleThreadCount = threadCount - totalCountRunningTasks;
1072 }
1073
1074 size_t GlobalHelperThreadState::maxIonCompilationThreads() const {
1075   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_ION)) {
1076     return 1;
1077   }
1078   return threadCount;
1079 }
1080
1081 size_t GlobalHelperThreadState::maxIonFreeThreads() const {
1082   // IonFree tasks are low priority. Limit to one thread to help avoid jemalloc
1083   // lock contention.
1084   return 1;
1085 }
1086
1087 size_t GlobalHelperThreadState::maxWasmCompilationThreads() const {
1088   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER1) ||
1089       IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER2)) {
1090     return 1;
1091   }
1092   return std::min(cpuCount, threadCount);
1093 }
1094
1095 size_t GlobalHelperThreadState::maxWasmTier2GeneratorThreads() const {
1096   return MaxTier2GeneratorTasks;
1097 }
1098
1099 size_t GlobalHelperThreadState::maxPromiseHelperThreads() const {
1100   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_PROMISE_TASK)) {
1101     return 1;
1102   }
1103   return std::min(cpuCount, threadCount);
1104 }
1105
1106 size_t GlobalHelperThreadState::maxDelazifyThreads() const {
1107   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_DELAZIFY)) {
1108     return 1;
1109   }
1110   return std::min(cpuCount, threadCount);
1111 }
1112
1113 size_t GlobalHelperThreadState::maxCompressionThreads() const {
1114   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_COMPRESS)) {
1115     return 1;
1116   }
1117
1118   // Compression is triggered on major GCs to compress ScriptSources. It is
1119   // considered low priority work.
1120   return 1;
1121 }
1122
1123 size_t GlobalHelperThreadState::maxGCParallelThreads() const {
1124   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_GCPARALLEL)) {
1125     return 1;
1126   }
1127   return threadCount;
1128 }
1129
1130 HelperThreadTask* GlobalHelperThreadState::maybeGetWasmTier1CompileTask(
1131     const AutoLockHelperThreadState& lock) {
1132   return maybeGetWasmCompile(lock, wasm::CompileMode::Tier1);
1133 }
1134
1135 HelperThreadTask* GlobalHelperThreadState::maybeGetWasmTier2CompileTask(
1136     const AutoLockHelperThreadState& lock) {
1137   return maybeGetWasmCompile(lock, wasm::CompileMode::Tier2);
1138 }
1139
1140 HelperThreadTask* GlobalHelperThreadState::maybeGetWasmCompile(
1141     const AutoLockHelperThreadState& lock, wasm::CompileMode mode) {
1142   if (!canStartWasmCompile(lock, mode)) {
1143     return nullptr;
1144   }
1145
1146   return wasmWorklist(lock, mode).popCopyFront();
1147 }
1148
1149 bool GlobalHelperThreadState::canStartWasmTier1CompileTask(
1150     const AutoLockHelperThreadState& lock) {
1151   return canStartWasmCompile(lock, wasm::CompileMode::Tier1);
1152 }
1153
1154 bool GlobalHelperThreadState::canStartWasmTier2CompileTask(
1155     const AutoLockHelperThreadState& lock) {
1156   return canStartWasmCompile(lock, wasm::CompileMode::Tier2);
1157 }
1158
1159 bool GlobalHelperThreadState::canStartWasmCompile(
1160     const AutoLockHelperThreadState& lock, wasm::CompileMode mode) {
1161   if (wasmWorklist(lock, mode).empty()) {
1162     return false;
1163   }
1164
1165   // Parallel compilation and background compilation should be disabled on
1166   // unicore systems.
1167
1168   MOZ_RELEASE_ASSERT(cpuCount > 1);
1169
1170   // If Tier2 is very backlogged we must give priority to it, since the Tier2
1171   // queue holds onto Tier1 tasks.  Indeed if Tier2 is backlogged we will
1172   // devote more resources to Tier2 and not start any Tier1 work at all.
1173
1174   bool tier2oversubscribed = wasmTier2GeneratorWorklist(lock).length() > 20;
1175
1176   // For Tier1 and Once compilation, honor the maximum allowed threads to
1177   // compile wasm jobs at once, to avoid oversaturating the machine.
1178   //
1179   // For Tier2 compilation we need to allow other things to happen too, so we
1180   // do not allow all logical cores to be used for background work; instead we
1181   // wish to use a fraction of the physical cores.  We can't directly compute
1182   // the physical cores from the logical cores, but 1/3 of the logical cores
1183   // is a safe estimate for the number of physical cores available for
1184   // background work.
1185
1186   size_t physCoresAvailable = size_t(ceil(cpuCount / 3.0));
1187
1188   size_t threads;
1189   ThreadType threadType;
1190   if (mode == wasm::CompileMode::Tier2) {
1191     if (tier2oversubscribed) {
1192       threads = maxWasmCompilationThreads();
1193     } else {
1194       threads = physCoresAvailable;
1195     }
1196     threadType = THREAD_TYPE_WASM_COMPILE_TIER2;
1197   } else {
1198     if (tier2oversubscribed) {
1199       threads = 0;
1200     } else {
1201       threads = maxWasmCompilationThreads();
1202     }
1203     threadType = THREAD_TYPE_WASM_COMPILE_TIER1;
1204   }
1205
1206   return threads != 0 && checkTaskThreadLimit(threadType, threads, lock);
1207 }
1208
1209 HelperThreadTask* GlobalHelperThreadState::maybeGetWasmTier2GeneratorTask(
1210     const AutoLockHelperThreadState& lock) {
1211   if (!canStartWasmTier2GeneratorTask(lock)) {
1212     return nullptr;
1213   }
1214
1215   return wasmTier2GeneratorWorklist(lock).popCopy();
1216 }
1217
1218 bool GlobalHelperThreadState::canStartWasmTier2GeneratorTask(
1219     const AutoLockHelperThreadState& lock) {
1220   return !wasmTier2GeneratorWorklist(lock).empty() &&
1221          checkTaskThreadLimit(THREAD_TYPE_WASM_GENERATOR_TIER2,
1222                               maxWasmTier2GeneratorThreads(),
1223                               /*isMaster=*/true, lock);
1224 }
1225
1226 HelperThreadTask* GlobalHelperThreadState::maybeGetPromiseHelperTask(
1227     const AutoLockHelperThreadState& lock) {
1228   if (!canStartPromiseHelperTask(lock)) {
1229     return nullptr;
1230   }
1231
1232   return promiseHelperTasks(lock).popCopy();
1233 }
1234
1235 bool GlobalHelperThreadState::canStartPromiseHelperTask(
1236     const AutoLockHelperThreadState& lock) {
1237   // PromiseHelperTasks can be wasm compilation tasks that in turn block on
1238   // wasm compilation so set isMaster = true.
1239   return !promiseHelperTasks(lock).empty() &&
1240          checkTaskThreadLimit(THREAD_TYPE_PROMISE_TASK,
1241                               maxPromiseHelperThreads(),
1242                               /*isMaster=*/true, lock);
1243 }
1244
1245 static bool IonCompileTaskHasHigherPriority(jit::IonCompileTask* first,
1246                                             jit::IonCompileTask* second) {
1247   // Return true if priority(first) > priority(second).
1248   //
1249   // This method can return whatever it wants, though it really ought to be a
1250   // total order. The ordering is allowed to race (change on the fly), however.
1251
1252   // A higher warm-up counter indicates a higher priority.
1253   jit::JitScript* firstJitScript = first->script()->jitScript();
1254   jit::JitScript* secondJitScript = second->script()->jitScript();
1255   return firstJitScript->warmUpCount() / first->script()->length() >
1256          secondJitScript->warmUpCount() / second->script()->length();
1257 }
1258
1259 HelperThreadTask* GlobalHelperThreadState::maybeGetIonCompileTask(
1260     const AutoLockHelperThreadState& lock) {
1261   if (!canStartIonCompileTask(lock)) {
1262     return nullptr;
1263   }
1264
1265   return highestPriorityPendingIonCompile(lock,
1266                                           /* checkExecutionStatus */ true);
1267 }
1268
1269 HelperThreadTask* GlobalHelperThreadState::maybeGetLowPrioIonCompileTask(
1270     const AutoLockHelperThreadState& lock) {
1271   if (!canStartIonCompileTask(lock)) {
1272     return nullptr;
1273   }
1274
1275   return highestPriorityPendingIonCompile(lock,
1276                                           /* checkExecutionStatus */ false);
1277 }
1278
1279 bool GlobalHelperThreadState::canStartIonCompileTask(
1280     const AutoLockHelperThreadState& lock) {
1281   return !ionWorklist(lock).empty() &&
1282          checkTaskThreadLimit(THREAD_TYPE_ION, maxIonCompilationThreads(),
1283                               lock);
1284 }
1285
1286 HelperThreadTask* GlobalHelperThreadState::maybeGetIonFreeTask(
1287     const AutoLockHelperThreadState& lock) {
1288   if (!canStartIonFreeTask(lock)) {
1289     return nullptr;
1290   }
1291
1292   UniquePtr<jit::IonFreeTask> task = std::move(ionFreeList(lock).back());
1293   ionFreeList(lock).popBack();
1294   return task.release();
1295 }
1296
1297 bool GlobalHelperThreadState::canStartIonFreeTask(
1298     const AutoLockHelperThreadState& lock) {
1299   return !ionFreeList(lock).empty() &&
1300          checkTaskThreadLimit(THREAD_TYPE_ION_FREE, maxIonFreeThreads(), lock);
1301 }
1302
1303 jit::IonCompileTask* GlobalHelperThreadState::highestPriorityPendingIonCompile(
1304     const AutoLockHelperThreadState& lock, bool checkExecutionStatus) {
1305   auto& worklist = ionWorklist(lock);
1306   MOZ_ASSERT(!worklist.empty());
1307
1308   // Get the highest priority IonCompileTask which has not started compilation
1309   // yet.
1310   size_t index = worklist.length();
1311   for (size_t i = 0; i < worklist.length(); i++) {
1312     if (checkExecutionStatus && !worklist[i]->isMainThreadRunningJS()) {
1313       continue;
1314     }
1315     if (i < index ||
1316         IonCompileTaskHasHigherPriority(worklist[i], worklist[index])) {
1317       index = i;
1318     }
1319   }
1320
1321   if (index == worklist.length()) {
1322     return nullptr;
1323   }
1324   jit::IonCompileTask* task = worklist[index];
1325   worklist.erase(&worklist[index]);
1326   return task;
1327 }
1328
1329 HelperThreadTask* GlobalHelperThreadState::maybeGetFreeDelazifyTask(
1330     const AutoLockHelperThreadState& lock) {
1331   auto& freeList = freeDelazifyTaskVector(lock);
1332   if (!freeList.empty()) {
1333     UniquePtr<FreeDelazifyTask> task = std::move(freeList.back());
1334     freeList.popBack();
1335     return task.release();
1336   }
1337   return nullptr;
1338 }
1339
1340 bool GlobalHelperThreadState::canStartFreeDelazifyTask(
1341     const AutoLockHelperThreadState& lock) {
1342   return !freeDelazifyTaskVector(lock).empty() &&
1343          checkTaskThreadLimit(THREAD_TYPE_DELAZIFY_FREE, maxDelazifyThreads(),
1344                               /*isMaster=*/true, lock);
1345 }
1346
1347 HelperThreadTask* GlobalHelperThreadState::maybeGetDelazifyTask(
1348     const AutoLockHelperThreadState& lock) {
1349   // NOTE: We want to span all cores availables with delazification tasks, in
1350   // order to parse a maximum number of functions ahead of their executions.
1351   // Thus, as opposed to parse task which have a higher priority, we are not
1352   // exclusively executing these task on parse threads.
1353   auto& worklist = delazifyWorklist(lock);
1354   if (worklist.isEmpty()) {
1355     return nullptr;
1356   }
1357   return worklist.popFirst();
1358 }
1359
1360 bool GlobalHelperThreadState::canStartDelazifyTask(
1361     const AutoLockHelperThreadState& lock) {
1362   return !delazifyWorklist(lock).isEmpty() &&
1363          checkTaskThreadLimit(THREAD_TYPE_DELAZIFY, maxDelazifyThreads(),
1364                               /*isMaster=*/true, lock);
1365 }
1366
1367 HelperThreadTask* GlobalHelperThreadState::maybeGetCompressionTask(
1368     const AutoLockHelperThreadState& lock) {
1369   if (!canStartCompressionTask(lock)) {
1370     return nullptr;
1371   }
1372
1373   auto& worklist = compressionWorklist(lock);
1374   UniquePtr<SourceCompressionTask> task = std::move(worklist.back());
1375   worklist.popBack();
1376   return task.release();
1377 }
1378
1379 bool GlobalHelperThreadState::canStartCompressionTask(
1380     const AutoLockHelperThreadState& lock) {
1381   return !compressionWorklist(lock).empty() &&
1382          checkTaskThreadLimit(THREAD_TYPE_COMPRESS, maxCompressionThreads(),
1383                               lock);
1384 }
1385
1386 void GlobalHelperThreadState::startHandlingCompressionTasks(
1387     ScheduleCompressionTask schedule, JSRuntime* maybeRuntime,
1388     const AutoLockHelperThreadState& lock) {
1389   MOZ_ASSERT((schedule == ScheduleCompressionTask::GC) ==
1390              (maybeRuntime != nullptr));
1391
1392   auto& pending = compressionPendingList(lock);
1393
1394   for (size_t i = 0; i < pending.length(); i++) {
1395     UniquePtr<SourceCompressionTask>& task = pending[i];
1396     if (schedule == ScheduleCompressionTask::API ||
1397         (task->runtimeMatches(maybeRuntime) && task->shouldStart())) {
1398       // OOMing during appending results in the task not being scheduled
1399       // and deleted.
1400       (void)submitTask(std::move(task), lock);
1401       remove(pending, &i);
1402     }
1403   }
1404 }
1405
1406 bool GlobalHelperThreadState::submitTask(
1407     UniquePtr<SourceCompressionTask> task,
1408     const AutoLockHelperThreadState& locked) {
1409   if (!compressionWorklist(locked).append(std::move(task))) {
1410     return false;
1411   }
1412
1413   dispatch(DispatchReason::NewTask, locked);
1414   return true;
1415 }
1416
1417 bool GlobalHelperThreadState::submitTask(
1418     GCParallelTask* task, const AutoLockHelperThreadState& locked) {
1419   gcParallelWorklist().insertBack(task, locked);
1420   dispatch(DispatchReason::NewTask, locked);
1421   return true;
1422 }
1423
1424 HelperThreadTask* GlobalHelperThreadState::maybeGetGCParallelTask(
1425     const AutoLockHelperThreadState& lock) {
1426   if (!canStartGCParallelTask(lock)) {
1427     return nullptr;
1428   }
1429
1430   return gcParallelWorklist().popFirst(lock);
1431 }
1432
1433 bool GlobalHelperThreadState::canStartGCParallelTask(
1434     const AutoLockHelperThreadState& lock) {
1435   return !gcParallelWorklist().isEmpty(lock) &&
1436          checkTaskThreadLimit(THREAD_TYPE_GCPARALLEL, maxGCParallelThreads(),
1437                               lock);
1438 }
1439
1440 bool js::EnqueueOffThreadCompression(JSContext* cx,
1441                                      UniquePtr<SourceCompressionTask> task) {
1442   AutoLockHelperThreadState lock;
1443
1444   auto& pending = HelperThreadState().compressionPendingList(lock);
1445   if (!pending.append(std::move(task))) {
1446     ReportOutOfMemory(cx);
1447     return false;
1448   }
1449
1450   return true;
1451 }
1452
1453 void js::StartHandlingCompressionsOnGC(JSRuntime* runtime) {
1454   AutoLockHelperThreadState lock;
1455   HelperThreadState().startHandlingCompressionTasks(
1456       GlobalHelperThreadState::ScheduleCompressionTask::GC, runtime, lock);
1457 }
1458
1459 template <typename T>
1460 static void ClearCompressionTaskList(T& list, JSRuntime* runtime) {
1461   for (size_t i = 0; i < list.length(); i++) {
1462     if (list[i]->runtimeMatches(runtime)) {
1463       HelperThreadState().remove(list, &i);
1464     }
1465   }
1466 }
1467
1468 void js::CancelOffThreadCompressions(JSRuntime* runtime) {
1469   if (!CanUseExtraThreads()) {
1470     return;
1471   }
1472
1473   AutoLockHelperThreadState lock;
1474
1475   // Cancel all pending compression tasks.
1476   ClearCompressionTaskList(HelperThreadState().compressionPendingList(lock),
1477                            runtime);
1478   ClearCompressionTaskList(HelperThreadState().compressionWorklist(lock),
1479                            runtime);
1480
1481   // Cancel all in-process compression tasks and wait for them to join so we
1482   // clean up the finished tasks.
1483   while (true) {
1484     bool inProgress = false;
1485     for (auto* helper : HelperThreadState().helperTasks(lock)) {
1486       if (!helper->is<SourceCompressionTask>()) {
1487         continue;
1488       }
1489
1490       if (helper->as<SourceCompressionTask>()->runtimeMatches(runtime)) {
1491         inProgress = true;
1492       }
1493     }
1494
1495     if (!inProgress) {
1496       break;
1497     }
1498
1499     HelperThreadState().wait(lock);
1500   }
1501
1502   // Clean up finished tasks.
1503   ClearCompressionTaskList(HelperThreadState().compressionFinishedList(lock),
1504                            runtime);
1505 }
1506
1507 void js::AttachFinishedCompressions(JSRuntime* runtime,
1508                                     AutoLockHelperThreadState& lock) {
1509   auto& finished = HelperThreadState().compressionFinishedList(lock);
1510   for (size_t i = 0; i < finished.length(); i++) {
1511     if (finished[i]->runtimeMatches(runtime)) {
1512       UniquePtr<SourceCompressionTask> compressionTask(std::move(finished[i]));
1513       HelperThreadState().remove(finished, &i);
1514       compressionTask->complete();
1515     }
1516   }
1517 }
1518
1519 void js::SweepPendingCompressions(AutoLockHelperThreadState& lock) {
1520   auto& pending = HelperThreadState().compressionPendingList(lock);
1521   for (size_t i = 0; i < pending.length(); i++) {
1522     if (pending[i]->shouldCancel()) {
1523       HelperThreadState().remove(pending, &i);
1524     }
1525   }
1526 }
1527
1528 void js::RunPendingSourceCompressions(JSRuntime* runtime) {
1529   if (!CanUseExtraThreads()) {
1530     return;
1531   }
1532
1533   AutoLockHelperThreadState lock;
1534
1535   HelperThreadState().startHandlingCompressionTasks(
1536       GlobalHelperThreadState::ScheduleCompressionTask::API, nullptr, lock);
1537
1538   // Wait until all tasks have started compression.
1539   while (!HelperThreadState().compressionWorklist(lock).empty()) {
1540     HelperThreadState().wait(lock);
1541   }
1542
1543   // Wait for all in-process compression tasks to complete.
1544   HelperThreadState().waitForAllTasksLocked(lock);
1545
1546   AttachFinishedCompressions(runtime, lock);
1547 }
1548
1549 void PromiseHelperTask::executeAndResolveAndDestroy(JSContext* cx) {
1550   execute();
1551   run(cx, JS::Dispatchable::NotShuttingDown);
1552 }
1553
1554 void PromiseHelperTask::runHelperThreadTask(AutoLockHelperThreadState& lock) {
1555   {
1556     AutoUnlockHelperThreadState unlock(lock);
1557     execute();
1558   }
1559
1560   // Don't release the lock between dispatching the resolve and destroy
1561   // operation (which may start immediately on another thread) and returning
1562   // from this method.
1563
1564   dispatchResolveAndDestroy(lock);
1565 }
1566
1567 bool js::StartOffThreadPromiseHelperTask(JSContext* cx,
1568                                          UniquePtr<PromiseHelperTask> task) {
1569   // Execute synchronously if there are no helper threads.
1570   if (!CanUseExtraThreads()) {
1571     task.release()->executeAndResolveAndDestroy(cx);
1572     return true;
1573   }
1574
1575   if (!HelperThreadState().submitTask(task.get())) {
1576     ReportOutOfMemory(cx);
1577     return false;
1578   }
1579
1580   (void)task.release();
1581   return true;
1582 }
1583
1584 bool js::StartOffThreadPromiseHelperTask(PromiseHelperTask* task) {
1585   MOZ_ASSERT(CanUseExtraThreads());
1586
1587   return HelperThreadState().submitTask(task);
1588 }
1589
1590 bool GlobalHelperThreadState::submitTask(PromiseHelperTask* task) {
1591   AutoLockHelperThreadState lock;
1592
1593   if (!promiseHelperTasks(lock).append(task)) {
1594     return false;
1595   }
1596
1597   dispatch(DispatchReason::NewTask, lock);
1598   return true;
1599 }
1600
1601 void GlobalHelperThreadState::trace(JSTracer* trc) {
1602   {
1603     AutoLockHelperThreadState lock;
1604
1605 #ifdef DEBUG
1606     // Since we hold the helper thread lock here we must disable GCMarker's
1607     // checking of the atom marking bitmap since that also relies on taking the
1608     // lock.
1609     GCMarker* marker = nullptr;
1610     if (trc->isMarkingTracer()) {
1611       marker = GCMarker::fromTracer(trc);
1612       marker->setCheckAtomMarking(false);
1613     }
1614     auto reenableAtomMarkingCheck = mozilla::MakeScopeExit([marker] {
1615       if (marker) {
1616         marker->setCheckAtomMarking(true);
1617       }
1618     });
1619 #endif
1620
1621     for (auto task : ionWorklist(lock)) {
1622       task->alloc().lifoAlloc()->setReadWrite();
1623       task->trace(trc);
1624       task->alloc().lifoAlloc()->setReadOnly();
1625     }
1626     for (auto task : ionFinishedList(lock)) {
1627       task->trace(trc);
1628     }
1629
1630     for (auto* helper : HelperThreadState().helperTasks(lock)) {
1631       if (helper->is<jit::IonCompileTask>()) {
1632         helper->as<jit::IonCompileTask>()->trace(trc);
1633       }
1634     }
1635   }
1636
1637   // The lazy link list is only accessed on the main thread, so trace it after
1638   // releasing the lock.
1639   JSRuntime* rt = trc->runtime();
1640   if (auto* jitRuntime = rt->jitRuntime()) {
1641     jit::IonCompileTask* task = jitRuntime->ionLazyLinkList(rt).getFirst();
1642     while (task) {
1643       task->trace(trc);
1644       task = task->getNext();
1645     }
1646   }
1647 }
1648
1649 // Definition of helper thread tasks.
1650 //
1651 // Priority is determined by the order they're listed here.
1652 const GlobalHelperThreadState::Selector GlobalHelperThreadState::selectors[] = {
1653     &GlobalHelperThreadState::maybeGetGCParallelTask,
1654     &GlobalHelperThreadState::maybeGetIonCompileTask,
1655     &GlobalHelperThreadState::maybeGetWasmTier1CompileTask,
1656     &GlobalHelperThreadState::maybeGetPromiseHelperTask,
1657     &GlobalHelperThreadState::maybeGetFreeDelazifyTask,
1658     &GlobalHelperThreadState::maybeGetDelazifyTask,
1659     &GlobalHelperThreadState::maybeGetCompressionTask,
1660     &GlobalHelperThreadState::maybeGetLowPrioIonCompileTask,
1661     &GlobalHelperThreadState::maybeGetIonFreeTask,
1662     &GlobalHelperThreadState::maybeGetWasmTier2CompileTask,
1663     &GlobalHelperThreadState::maybeGetWasmTier2GeneratorTask};
1664
1665 bool GlobalHelperThreadState::canStartTasks(
1666     const AutoLockHelperThreadState& lock) {
1667   return canStartGCParallelTask(lock) || canStartIonCompileTask(lock) ||
1668          canStartWasmTier1CompileTask(lock) ||
1669          canStartPromiseHelperTask(lock) || canStartFreeDelazifyTask(lock) ||
1670          canStartDelazifyTask(lock) || canStartCompressionTask(lock) ||
1671          canStartIonFreeTask(lock) || canStartWasmTier2CompileTask(lock) ||
1672          canStartWasmTier2GeneratorTask(lock);
1673 }
1674
1675 void JS::RunHelperThreadTask() {
1676   MOZ_ASSERT(CanUseExtraThreads());
1677
1678   AutoLockHelperThreadState lock;
1679
1680   if (!gHelperThreadState || HelperThreadState().isTerminating(lock)) {
1681     return;
1682   }
1683
1684   HelperThreadState().runOneTask(lock);
1685 }
1686
1687 void GlobalHelperThreadState::runOneTask(AutoLockHelperThreadState& lock) {
1688   MOZ_ASSERT(tasksPending_ > 0);
1689   tasksPending_--;
1690
1691   // The selectors may depend on the HelperThreadState not changing between task
1692   // selection and task execution, in particular, on new tasks not being added
1693   // (because of the lifo structure of the work lists). Unlocking the
1694   // HelperThreadState between task selection and execution is not well-defined.
1695   HelperThreadTask* task = findHighestPriorityTask(lock);
1696   if (task) {
1697     runTaskLocked(task, lock);
1698     dispatch(DispatchReason::FinishedTask, lock);
1699   }
1700
1701   notifyAll(lock);
1702 }
1703
1704 HelperThreadTask* GlobalHelperThreadState::findHighestPriorityTask(
1705     const AutoLockHelperThreadState& locked) {
1706   // Return the highest priority task that is ready to start, or nullptr.
1707
1708   for (const auto& selector : selectors) {
1709     if (auto* task = (this->*(selector))(locked)) {
1710       return task;
1711     }
1712   }
1713
1714   return nullptr;
1715 }
1716
1717 void GlobalHelperThreadState::runTaskLocked(HelperThreadTask* task,
1718                                             AutoLockHelperThreadState& locked) {
1719   JS::AutoSuppressGCAnalysis nogc;
1720
1721   HelperThreadState().helperTasks(locked).infallibleEmplaceBack(task);
1722
1723   ThreadType threadType = task->threadType();
1724   js::oom::SetThreadType(threadType);
1725   runningTaskCount[threadType]++;
1726   totalCountRunningTasks++;
1727
1728   task->runHelperThreadTask(locked);
1729
1730   // Delete task from helperTasks.
1731   HelperThreadState().helperTasks(locked).eraseIfEqual(task);
1732
1733   totalCountRunningTasks--;
1734   runningTaskCount[threadType]--;
1735
1736   js::oom::SetThreadType(js::THREAD_TYPE_NONE);
1737 }