js/src/vm/HelperThreads.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*-
   2  * vim: set ts=8 sts=2 et sw=2 tw=80:
   3  * This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "vm/HelperThreads.h"
   8
   9 #include "mozilla/ReverseIterator.h"  // mozilla::Reversed(...)
  10 #include "mozilla/ScopeExit.h"
  11 #include "mozilla/Span.h"  // mozilla::Span<TaggedScriptThingIndex>
  12 #include "mozilla/Utf8.h"  // mozilla::Utf8Unit
  13
  14 #include <algorithm>
  15
  16 #include "frontend/CompilationStencil.h"  // frontend::CompilationStencil
  17 #include "gc/GC.h"
  18 #include "jit/Ion.h"
  19 #include "jit/IonCompileTask.h"
  20 #include "jit/JitRuntime.h"
  21 #include "jit/JitScript.h"
  22 #include "js/CompileOptions.h"  // JS::PrefableCompileOptions, JS::ReadOnlyCompileOptions
  23 #include "js/experimental/CompileScript.h"  // JS::ThreadStackQuotaForSize
  24 #include "js/friend/StackLimits.h"          // js::ReportOverRecursed
  25 #include "js/HelperThreadAPI.h"
  26 #include "js/Stack.h"
  27 #include "js/UniquePtr.h"
  28 #include "js/Utility.h"
  29 #include "threading/CpuCount.h"
  30 #include "vm/ErrorReporting.h"
  31 #include "vm/HelperThreadState.h"
  32 #include "vm/InternalThreadPool.h"
  33 #include "vm/MutexIDs.h"
  34 #include "wasm/WasmGenerator.h"
  35
  36 using namespace js;
  37
  38 using mozilla::TimeDuration;
  39 using mozilla::TimeStamp;
  40 using mozilla::Utf8Unit;
  41
  42 using JS::DispatchReason;
  43
  44 namespace js {
  45
  46 Mutex gHelperThreadLock(mutexid::GlobalHelperThreadState);
  47 GlobalHelperThreadState* gHelperThreadState = nullptr;
  48
  49 }  // namespace js
  50
  51 bool js::CreateHelperThreadsState() {
  52   MOZ_ASSERT(!gHelperThreadState);
  53   gHelperThreadState = js_new<GlobalHelperThreadState>();
  54   return gHelperThreadState;
  55 }
  56
  57 void js::DestroyHelperThreadsState() {
  58   AutoLockHelperThreadState lock;
  59
  60   if (!gHelperThreadState) {
  61     return;
  62   }
  63
  64   gHelperThreadState->finish(lock);
  65   js_delete(gHelperThreadState);
  66   gHelperThreadState = nullptr;
  67 }
  68
  69 bool js::EnsureHelperThreadsInitialized() {
  70   MOZ_ASSERT(gHelperThreadState);
  71   return gHelperThreadState->ensureInitialized();
  72 }
  73
  74 static size_t ClampDefaultCPUCount(size_t cpuCount) {
  75   // It's extremely rare for SpiderMonkey to have more than a few cores worth
  76   // of work. At higher core counts, performance can even decrease due to NUMA
  77   // (and SpiderMonkey's lack of NUMA-awareness), contention, and general lack
  78   // of optimization for high core counts. So to avoid wasting thread stack
  79   // resources (and cluttering gdb and core dumps), clamp to 8 cores for now.
  80   return std::min<size_t>(cpuCount, 8);
  81 }
  82
  83 static size_t ThreadCountForCPUCount(size_t cpuCount) {
  84   // We need at least two threads for tier-2 wasm compilations, because
  85   // there's a master task that holds a thread while other threads do the
  86   // compilation.
  87   return std::max<size_t>(cpuCount, 2);
  88 }
  89
  90 bool js::SetFakeCPUCount(size_t count) {
  91   HelperThreadState().setCpuCount(count);
  92   return true;
  93 }
  94
  95 void GlobalHelperThreadState::setCpuCount(size_t count) {
  96   // This must be called before any threads have been initialized.
  97   AutoLockHelperThreadState lock;
  98   MOZ_ASSERT(!isInitialized(lock));
  99
 100   // We can't do this if an external thread pool is in use.
 101   MOZ_ASSERT(!dispatchTaskCallback);
 102
 103   cpuCount = count;
 104   threadCount = ThreadCountForCPUCount(count);
 105 }
 106
 107 size_t js::GetHelperThreadCount() { return HelperThreadState().threadCount; }
 108
 109 size_t js::GetHelperThreadCPUCount() { return HelperThreadState().cpuCount; }
 110
 111 size_t js::GetMaxWasmCompilationThreads() {
 112   return HelperThreadState().maxWasmCompilationThreads();
 113 }
 114
 115 void JS::SetProfilingThreadCallbacks(
 116     JS::RegisterThreadCallback registerThread,
 117     JS::UnregisterThreadCallback unregisterThread) {
 118   HelperThreadState().registerThread = registerThread;
 119   HelperThreadState().unregisterThread = unregisterThread;
 120 }
 121
 122 // Bug 1630189: Without MOZ_NEVER_INLINE, Windows PGO builds have a linking
 123 // error for HelperThreadTaskCallback.
 124 JS_PUBLIC_API MOZ_NEVER_INLINE void JS::SetHelperThreadTaskCallback(
 125     HelperThreadTaskCallback callback, size_t threadCount, size_t stackSize) {
 126   AutoLockHelperThreadState lock;
 127   HelperThreadState().setDispatchTaskCallback(callback, threadCount, stackSize,
 128                                               lock);
 129 }
 130
 131 void GlobalHelperThreadState::setDispatchTaskCallback(
 132     JS::HelperThreadTaskCallback callback, size_t threadCount, size_t stackSize,
 133     const AutoLockHelperThreadState& lock) {
 134   MOZ_ASSERT(!isInitialized(lock));
 135   MOZ_ASSERT(!dispatchTaskCallback);
 136   MOZ_ASSERT(threadCount != 0);
 137   MOZ_ASSERT(stackSize >= 16 * 1024);
 138
 139   dispatchTaskCallback = callback;
 140   this->threadCount = threadCount;
 141   this->stackQuota = JS::ThreadStackQuotaForSize(stackSize);
 142 }
 143
 144 bool js::StartOffThreadWasmCompile(wasm::CompileTask* task,
 145                                    wasm::CompileMode mode) {
 146   return HelperThreadState().submitTask(task, mode);
 147 }
 148
 149 bool GlobalHelperThreadState::submitTask(wasm::CompileTask* task,
 150                                          wasm::CompileMode mode) {
 151   AutoLockHelperThreadState lock;
 152   if (!wasmWorklist(lock, mode).pushBack(task)) {
 153     return false;
 154   }
 155
 156   dispatch(DispatchReason::NewTask, lock);
 157   return true;
 158 }
 159
 160 size_t js::RemovePendingWasmCompileTasks(
 161     const wasm::CompileTaskState& taskState, wasm::CompileMode mode,
 162     const AutoLockHelperThreadState& lock) {
 163   wasm::CompileTaskPtrFifo& worklist =
 164       HelperThreadState().wasmWorklist(lock, mode);
 165   return worklist.eraseIf([&taskState](wasm::CompileTask* task) {
 166     return &task->state == &taskState;
 167   });
 168 }
 169
 170 void js::StartOffThreadWasmTier2Generator(wasm::UniqueTier2GeneratorTask task) {
 171   (void)HelperThreadState().submitTask(std::move(task));
 172 }
 173
 174 bool GlobalHelperThreadState::submitTask(wasm::UniqueTier2GeneratorTask task) {
 175   AutoLockHelperThreadState lock;
 176
 177   MOZ_ASSERT(isInitialized(lock));
 178
 179   if (!wasmTier2GeneratorWorklist(lock).append(task.get())) {
 180     return false;
 181   }
 182   (void)task.release();
 183
 184   dispatch(DispatchReason::NewTask, lock);
 185   return true;
 186 }
 187
 188 static void CancelOffThreadWasmTier2GeneratorLocked(
 189     AutoLockHelperThreadState& lock) {
 190   if (!HelperThreadState().isInitialized(lock)) {
 191     return;
 192   }
 193
 194   // Remove pending tasks from the tier2 generator worklist and cancel and
 195   // delete them.
 196   {
 197     wasm::Tier2GeneratorTaskPtrVector& worklist =
 198         HelperThreadState().wasmTier2GeneratorWorklist(lock);
 199     for (size_t i = 0; i < worklist.length(); i++) {
 200       wasm::Tier2GeneratorTask* task = worklist[i];
 201       HelperThreadState().remove(worklist, &i);
 202       js_delete(task);
 203     }
 204   }
 205
 206   // There is at most one running Tier2Generator task and we assume that
 207   // below.
 208   static_assert(GlobalHelperThreadState::MaxTier2GeneratorTasks == 1,
 209                 "code must be generalized");
 210
 211   // If there is a running Tier2 generator task, shut it down in a predictable
 212   // way.  The task will be deleted by the normal deletion logic.
 213   for (auto* helper : HelperThreadState().helperTasks(lock)) {
 214     if (helper->is<wasm::Tier2GeneratorTask>()) {
 215       // Set a flag that causes compilation to shortcut itself.
 216       helper->as<wasm::Tier2GeneratorTask>()->cancel();
 217
 218       // Wait for the generator task to finish.  This avoids a shutdown race
 219       // where the shutdown code is trying to shut down helper threads and the
 220       // ongoing tier2 compilation is trying to finish, which requires it to
 221       // have access to helper threads.
 222       uint32_t oldFinishedCount =
 223           HelperThreadState().wasmTier2GeneratorsFinished(lock);
 224       while (HelperThreadState().wasmTier2GeneratorsFinished(lock) ==
 225              oldFinishedCount) {
 226         HelperThreadState().wait(lock);
 227       }
 228
 229       // At most one of these tasks.
 230       break;
 231     }
 232   }
 233 }
 234
 235 void js::CancelOffThreadWasmTier2Generator() {
 236   AutoLockHelperThreadState lock;
 237   CancelOffThreadWasmTier2GeneratorLocked(lock);
 238 }
 239
 240 bool js::StartOffThreadIonCompile(jit::IonCompileTask* task,
 241                                   const AutoLockHelperThreadState& lock) {
 242   return HelperThreadState().submitTask(task, lock);
 243 }
 244
 245 bool GlobalHelperThreadState::submitTask(
 246     jit::IonCompileTask* task, const AutoLockHelperThreadState& locked) {
 247   MOZ_ASSERT(isInitialized(locked));
 248
 249   if (!ionWorklist(locked).append(task)) {
 250     return false;
 251   }
 252
 253   // The build is moving off-thread. Freeze the LifoAlloc to prevent any
 254   // unwanted mutations.
 255   task->alloc().lifoAlloc()->setReadOnly();
 256
 257   dispatch(DispatchReason::NewTask, locked);
 258   return true;
 259 }
 260
 261 bool js::AutoStartIonFreeTask::addIonCompileToFreeTaskBatch(
 262     jit::IonCompileTask* task) {
 263   return jitRuntime_->addIonCompileToFreeTaskBatch(task);
 264 }
 265
 266 js::AutoStartIonFreeTask::~AutoStartIonFreeTask() {
 267   jitRuntime_->maybeStartIonFreeTask(force_);
 268 }
 269
 270 void jit::JitRuntime::maybeStartIonFreeTask(bool force) {
 271   IonFreeCompileTasks& tasks = ionFreeTaskBatch_.ref();
 272   if (tasks.empty()) {
 273     return;
 274   }
 275
 276   // Start an IonFreeTask if we have at least eight tasks. If |force| is true we
 277   // always start an IonFreeTask.
 278   if (!force) {
 279     constexpr size_t MinBatchSize = 8;
 280     static_assert(IonFreeCompileTasks::InlineLength >= MinBatchSize,
 281                   "Minimum batch size shouldn't require malloc");
 282     if (tasks.length() < MinBatchSize) {
 283       return;
 284     }
 285   }
 286
 287   auto freeTask = js::MakeUnique<jit::IonFreeTask>(std::move(tasks));
 288   if (!freeTask) {
 289     // Free compilation data on the main thread instead.
 290     MOZ_ASSERT(!tasks.empty(), "shouldn't have moved tasks on OOM");
 291     jit::FreeIonCompileTasks(tasks);
 292     tasks.clearAndFree();
 293     return;
 294   }
 295
 296   AutoLockHelperThreadState lock;
 297   if (!HelperThreadState().submitTask(std::move(freeTask), lock)) {
 298     // If submitTask OOMs, then freeTask hasn't been moved so we can still use
 299     // its task list.
 300     jit::FreeIonCompileTasks(freeTask->compileTasks());
 301   }
 302
 303   tasks.clearAndFree();
 304 }
 305
 306 bool GlobalHelperThreadState::submitTask(
 307     UniquePtr<jit::IonFreeTask>&& task,
 308     const AutoLockHelperThreadState& locked) {
 309   MOZ_ASSERT(isInitialized(locked));
 310
 311   if (!ionFreeList(locked).append(std::move(task))) {
 312     return false;
 313   }
 314
 315   dispatch(DispatchReason::NewTask, locked);
 316   return true;
 317 }
 318
 319 /*
 320  * Move an IonCompilationTask for which compilation has either finished, failed,
 321  * or been cancelled into the global finished compilation list. All off thread
 322  * compilations which are started must eventually be finished.
 323  */
 324 void js::FinishOffThreadIonCompile(jit::IonCompileTask* task,
 325                                    const AutoLockHelperThreadState& lock) {
 326   AutoEnterOOMUnsafeRegion oomUnsafe;
 327   if (!HelperThreadState().ionFinishedList(lock).append(task)) {
 328     oomUnsafe.crash("FinishOffThreadIonCompile");
 329   }
 330   task->script()
 331       ->runtimeFromAnyThread()
 332       ->jitRuntime()
 333       ->numFinishedOffThreadTasksRef(lock)++;
 334 }
 335
 336 static JSRuntime* GetSelectorRuntime(const CompilationSelector& selector) {
 337   struct Matcher {
 338     JSRuntime* operator()(JSScript* script) {
 339       return script->runtimeFromMainThread();
 340     }
 341     JSRuntime* operator()(Zone* zone) { return zone->runtimeFromMainThread(); }
 342     JSRuntime* operator()(ZonesInState zbs) { return zbs.runtime; }
 343     JSRuntime* operator()(JSRuntime* runtime) { return runtime; }
 344   };
 345
 346   return selector.match(Matcher());
 347 }
 348
 349 static bool JitDataStructuresExist(const CompilationSelector& selector) {
 350   struct Matcher {
 351     bool operator()(JSScript* script) { return !!script->zone()->jitZone(); }
 352     bool operator()(Zone* zone) { return !!zone->jitZone(); }
 353     bool operator()(ZonesInState zbs) { return zbs.runtime->hasJitRuntime(); }
 354     bool operator()(JSRuntime* runtime) { return runtime->hasJitRuntime(); }
 355   };
 356
 357   return selector.match(Matcher());
 358 }
 359
 360 static bool IonCompileTaskMatches(const CompilationSelector& selector,
 361                                   jit::IonCompileTask* task) {
 362   struct TaskMatches {
 363     jit::IonCompileTask* task_;
 364
 365     bool operator()(JSScript* script) { return script == task_->script(); }
 366     bool operator()(Zone* zone) {
 367       return zone == task_->script()->zoneFromAnyThread();
 368     }
 369     bool operator()(JSRuntime* runtime) {
 370       return runtime == task_->script()->runtimeFromAnyThread();
 371     }
 372     bool operator()(ZonesInState zbs) {
 373       return zbs.runtime == task_->script()->runtimeFromAnyThread() &&
 374              zbs.state == task_->script()->zoneFromAnyThread()->gcState();
 375     }
 376   };
 377
 378   return selector.match(TaskMatches{task});
 379 }
 380
 381 // If we're canceling Ion compilations for a zone/runtime, force a new
 382 // IonFreeTask even if there are just a few tasks. This lets us free as much
 383 // memory as possible.
 384 static bool ShouldForceIonFreeTask(const CompilationSelector& selector) {
 385   struct Matcher {
 386     bool operator()(JSScript* script) { return false; }
 387     bool operator()(Zone* zone) { return true; }
 388     bool operator()(ZonesInState zbs) { return true; }
 389     bool operator()(JSRuntime* runtime) { return true; }
 390   };
 391
 392   return selector.match(Matcher());
 393 }
 394
 395 void js::CancelOffThreadIonCompile(const CompilationSelector& selector) {
 396   if (!JitDataStructuresExist(selector)) {
 397     return;
 398   }
 399
 400   if (jit::IsPortableBaselineInterpreterEnabled()) {
 401     return;
 402   }
 403
 404   jit::JitRuntime* jitRuntime = GetSelectorRuntime(selector)->jitRuntime();
 405   MOZ_ASSERT(jitRuntime);
 406
 407   AutoStartIonFreeTask freeTask(jitRuntime, ShouldForceIonFreeTask(selector));
 408
 409   {
 410     AutoLockHelperThreadState lock;
 411     if (!HelperThreadState().isInitialized(lock)) {
 412       return;
 413     }
 414
 415     /* Cancel any pending entries for which processing hasn't started. */
 416     GlobalHelperThreadState::IonCompileTaskVector& worklist =
 417         HelperThreadState().ionWorklist(lock);
 418     for (size_t i = 0; i < worklist.length(); i++) {
 419       jit::IonCompileTask* task = worklist[i];
 420       if (IonCompileTaskMatches(selector, task)) {
 421         // Once finished, tasks are added to a Linked list which is
 422         // allocated with the IonCompileTask class. The IonCompileTask is
 423         // allocated in the LifoAlloc so we need the LifoAlloc to be mutable.
 424         worklist[i]->alloc().lifoAlloc()->setReadWrite();
 425
 426         FinishOffThreadIonCompile(task, lock);
 427         HelperThreadState().remove(worklist, &i);
 428       }
 429     }
 430
 431     /* Wait for in progress entries to finish up. */
 432     bool cancelled;
 433     do {
 434       cancelled = false;
 435       for (auto* helper : HelperThreadState().helperTasks(lock)) {
 436         if (!helper->is<jit::IonCompileTask>()) {
 437           continue;
 438         }
 439
 440         jit::IonCompileTask* ionCompileTask = helper->as<jit::IonCompileTask>();
 441         if (IonCompileTaskMatches(selector, ionCompileTask)) {
 442           ionCompileTask->mirGen().cancel();
 443           cancelled = true;
 444         }
 445       }
 446       if (cancelled) {
 447         HelperThreadState().wait(lock);
 448       }
 449     } while (cancelled);
 450
 451     /* Cancel code generation for any completed entries. */
 452     GlobalHelperThreadState::IonCompileTaskVector& finished =
 453         HelperThreadState().ionFinishedList(lock);
 454     for (size_t i = 0; i < finished.length(); i++) {
 455       jit::IonCompileTask* task = finished[i];
 456       if (IonCompileTaskMatches(selector, task)) {
 457         JSRuntime* rt = task->script()->runtimeFromAnyThread();
 458         rt->jitRuntime()->numFinishedOffThreadTasksRef(lock)--;
 459         jit::FinishOffThreadTask(rt, freeTask, task);
 460         HelperThreadState().remove(finished, &i);
 461       }
 462     }
 463   }
 464
 465   /* Cancel lazy linking for pending tasks (attached to the ionScript). */
 466   JSRuntime* runtime = GetSelectorRuntime(selector);
 467   jit::IonCompileTask* task =
 468       runtime->jitRuntime()->ionLazyLinkList(runtime).getFirst();
 469   while (task) {
 470     jit::IonCompileTask* next = task->getNext();
 471     if (IonCompileTaskMatches(selector, task)) {
 472       jit::FinishOffThreadTask(runtime, freeTask, task);
 473     }
 474     task = next;
 475   }
 476 }
 477
 478 #ifdef DEBUG
 479 bool js::HasOffThreadIonCompile(Zone* zone) {
 480   if (jit::IsPortableBaselineInterpreterEnabled()) {
 481     return false;
 482   }
 483
 484   AutoLockHelperThreadState lock;
 485
 486   if (!HelperThreadState().isInitialized(lock)) {
 487     return false;
 488   }
 489
 490   GlobalHelperThreadState::IonCompileTaskVector& worklist =
 491       HelperThreadState().ionWorklist(lock);
 492   for (size_t i = 0; i < worklist.length(); i++) {
 493     jit::IonCompileTask* task = worklist[i];
 494     if (task->script()->zoneFromAnyThread() == zone) {
 495       return true;
 496     }
 497   }
 498
 499   for (auto* helper : HelperThreadState().helperTasks(lock)) {
 500     if (!helper->is<jit::IonCompileTask>()) {
 501       continue;
 502     }
 503     JSScript* script = helper->as<jit::IonCompileTask>()->script();
 504     if (script->zoneFromAnyThread() == zone) {
 505       return true;
 506     }
 507   }
 508
 509   GlobalHelperThreadState::IonCompileTaskVector& finished =
 510       HelperThreadState().ionFinishedList(lock);
 511   for (size_t i = 0; i < finished.length(); i++) {
 512     jit::IonCompileTask* task = finished[i];
 513     if (task->script()->zoneFromAnyThread() == zone) {
 514       return true;
 515     }
 516   }
 517
 518   JSRuntime* rt = zone->runtimeFromMainThread();
 519   if (rt->hasJitRuntime()) {
 520     jit::IonCompileTask* task =
 521         rt->jitRuntime()->ionLazyLinkList(rt).getFirst();
 522     while (task) {
 523       if (task->script()->zone() == zone) {
 524         return true;
 525       }
 526       task = task->getNext();
 527     }
 528   }
 529
 530   return false;
 531 }
 532 #endif
 533
 534 void js::StartOffThreadDelazification(
 535     JSContext* maybeCx, const JS::ReadOnlyCompileOptions& options,
 536     const frontend::CompilationStencil& stencil) {
 537   // Skip delazify tasks if we parse everything on-demand or ahead.
 538   auto strategy = options.eagerDelazificationStrategy();
 539   if (strategy == JS::DelazificationOption::OnDemandOnly ||
 540       strategy == JS::DelazificationOption::ParseEverythingEagerly) {
 541     return;
 542   }
 543
 544   // Skip delazify task if code coverage is enabled.
 545   if (maybeCx && maybeCx->realm()->collectCoverageForDebug()) {
 546     return;
 547   }
 548
 549   if (!CanUseExtraThreads()) {
 550     return;
 551   }
 552
 553   JSRuntime* maybeRuntime = maybeCx ? maybeCx->runtime() : nullptr;
 554   UniquePtr<DelazifyTask> task;
 555   task = DelazifyTask::Create(maybeRuntime, options, stencil);
 556   if (!task) {
 557     return;
 558   }
 559
 560   // Schedule delazification task if there is any function to delazify.
 561   if (!task->done()) {
 562     AutoLockHelperThreadState lock;
 563     HelperThreadState().submitTask(task.release(), lock);
 564   }
 565 }
 566
 567 UniquePtr<DelazifyTask> DelazifyTask::Create(
 568     JSRuntime* maybeRuntime, const JS::ReadOnlyCompileOptions& options,
 569     const frontend::CompilationStencil& stencil) {
 570   UniquePtr<DelazifyTask> task;
 571   task.reset(js_new<DelazifyTask>(maybeRuntime, options.prefableOptions()));
 572   if (!task) {
 573     return nullptr;
 574   }
 575
 576   if (!task->init(options, stencil)) {
 577     // In case of errors, skip this and delazify on-demand.
 578     return nullptr;
 579   }
 580
 581   return task;
 582 }
 583
 584 DelazifyTask::DelazifyTask(
 585     JSRuntime* maybeRuntime,
 586     const JS::PrefableCompileOptions& initialPrefableOptions)
 587     : maybeRuntime(maybeRuntime),
 588       delazificationCx(initialPrefableOptions, HelperThreadState().stackQuota) {
 589 }
 590
 591 DelazifyTask::~DelazifyTask() {
 592   // The LinkedListElement destructor will remove us from any list we are part
 593   // of without synchronization, so ensure that doesn't happen.
 594   MOZ_DIAGNOSTIC_ASSERT(!isInList());
 595 }
 596
 597 bool DelazifyTask::init(const JS::ReadOnlyCompileOptions& options,
 598                         const frontend::CompilationStencil& stencil) {
 599   return delazificationCx.init(options, stencil);
 600 }
 601
 602 size_t DelazifyTask::sizeOfExcludingThis(
 603     mozilla::MallocSizeOf mallocSizeOf) const {
 604   return delazificationCx.sizeOfExcludingThis(mallocSizeOf);
 605 }
 606
 607 void DelazifyTask::runHelperThreadTask(AutoLockHelperThreadState& lock) {
 608   {
 609     AutoUnlockHelperThreadState unlock(lock);
 610     // NOTE: We do not report errors beyond this scope, as there is no where
 611     // to report these errors to. In the mean time, prevent the eager
 612     // delazification from running after any kind of errors.
 613     (void)runTask();
 614   }
 615
 616   // If we should continue to delazify even more functions, then re-add this
 617   // task to the vector of delazification tasks. This might happen when the
 618   // DelazifyTask is interrupted by a higher priority task. (see
 619   // mozilla::TaskController & mozilla::Task)
 620   if (!delazificationCx.done()) {
 621     HelperThreadState().submitTask(this, lock);
 622   } else {
 623     UniquePtr<FreeDelazifyTask> freeTask(js_new<FreeDelazifyTask>(this));
 624     if (freeTask) {
 625       HelperThreadState().submitTask(std::move(freeTask), lock);
 626     }
 627   }
 628 }
 629
 630 bool DelazifyTask::runTask() { return delazificationCx.delazify(); }
 631
 632 bool DelazifyTask::done() const { return delazificationCx.done(); }
 633
 634 void FreeDelazifyTask::runHelperThreadTask(AutoLockHelperThreadState& locked) {
 635   {
 636     AutoUnlockHelperThreadState unlock(locked);
 637     js_delete(task);
 638     task = nullptr;
 639   }
 640
 641   js_delete(this);
 642 }
 643
 644 static void CancelPendingDelazifyTask(JSRuntime* rt,
 645                                       AutoLockHelperThreadState& lock) {
 646   auto& delazifyList = HelperThreadState().delazifyWorklist(lock);
 647
 648   auto end = delazifyList.end();
 649   for (auto iter = delazifyList.begin(); iter != end;) {
 650     DelazifyTask* task = *iter;
 651     ++iter;
 652     if (task->runtimeMatchesOrNoRuntime(rt)) {
 653       task->removeFrom(delazifyList);
 654       js_delete(task);
 655     }
 656   }
 657 }
 658
 659 static void WaitUntilCancelledDelazifyTasks(JSRuntime* rt,
 660                                             AutoLockHelperThreadState& lock) {
 661   if (!HelperThreadState().isInitialized(lock)) {
 662     return;
 663   }
 664
 665   while (true) {
 666     CancelPendingDelazifyTask(rt, lock);
 667
 668     // If running tasks are delazifying any functions, then we have to wait
 669     // until they complete to remove them from the pending list. DelazifyTask
 670     // are inserting themself back to be processed once more after delazifying a
 671     // function.
 672     bool inProgress = false;
 673     for (auto* helper : HelperThreadState().helperTasks(lock)) {
 674       if (helper->is<DelazifyTask>() &&
 675           helper->as<DelazifyTask>()->runtimeMatchesOrNoRuntime(rt)) {
 676         inProgress = true;
 677         break;
 678       }
 679     }
 680     if (!inProgress) {
 681       break;
 682     }
 683
 684     HelperThreadState().wait(lock);
 685   }
 686
 687 #ifdef DEBUG
 688   for (DelazifyTask* task : HelperThreadState().delazifyWorklist(lock)) {
 689     MOZ_ASSERT(!task->runtimeMatchesOrNoRuntime(rt));
 690   }
 691   for (auto* helper : HelperThreadState().helperTasks(lock)) {
 692     MOZ_ASSERT_IF(helper->is<DelazifyTask>(),
 693                   !helper->as<DelazifyTask>()->runtimeMatchesOrNoRuntime(rt));
 694   }
 695 #endif
 696 }
 697
 698 static void WaitUntilEmptyFreeDelazifyTaskVector(
 699     AutoLockHelperThreadState& lock) {
 700   if (!HelperThreadState().isInitialized(lock)) {
 701     return;
 702   }
 703
 704   while (true) {
 705     bool inProgress = false;
 706     auto& freeList = HelperThreadState().freeDelazifyTaskVector(lock);
 707     if (!freeList.empty()) {
 708       inProgress = true;
 709     }
 710
 711     // If running tasks are delazifying any functions, then we have to wait
 712     // until they complete to remove them from the pending list. DelazifyTask
 713     // are inserting themself back to be processed once more after delazifying a
 714     // function.
 715     for (auto* helper : HelperThreadState().helperTasks(lock)) {
 716       if (helper->is<FreeDelazifyTask>()) {
 717         inProgress = true;
 718         break;
 719       }
 720     }
 721     if (!inProgress) {
 722       break;
 723     }
 724
 725     HelperThreadState().wait(lock);
 726   }
 727 }
 728
 729 void js::CancelOffThreadDelazify(JSRuntime* runtime) {
 730   AutoLockHelperThreadState lock;
 731
 732   // Cancel all Delazify tasks from the given runtime, and wait if tasks are
 733   // from the given runtime are being executed.
 734   WaitUntilCancelledDelazifyTasks(runtime, lock);
 735
 736   // Empty the free list of delazify task, in case one of the delazify task
 737   // ended and therefore did not returned to the pending list of delazify tasks.
 738   WaitUntilEmptyFreeDelazifyTaskVector(lock);
 739 }
 740
 741 static bool HasAnyDelazifyTask(JSRuntime* rt, AutoLockHelperThreadState& lock) {
 742   auto& delazifyList = HelperThreadState().delazifyWorklist(lock);
 743   for (auto task : delazifyList) {
 744     if (task->runtimeMatchesOrNoRuntime(rt)) {
 745       return true;
 746     }
 747   }
 748
 749   for (auto* helper : HelperThreadState().helperTasks(lock)) {
 750     if (helper->is<DelazifyTask>() &&
 751         helper->as<DelazifyTask>()->runtimeMatchesOrNoRuntime(rt)) {
 752       return true;
 753     }
 754   }
 755
 756   return false;
 757 }
 758
 759 void js::WaitForAllDelazifyTasks(JSRuntime* rt) {
 760   AutoLockHelperThreadState lock;
 761   if (!HelperThreadState().isInitialized(lock)) {
 762     return;
 763   }
 764
 765   while (true) {
 766     if (!HasAnyDelazifyTask(rt, lock)) {
 767       break;
 768     }
 769
 770     HelperThreadState().wait(lock);
 771   }
 772 }
 773
 774 void GlobalHelperThreadState::submitTask(
 775     DelazifyTask* task, const AutoLockHelperThreadState& locked) {
 776   delazifyWorklist(locked).insertBack(task);
 777   dispatch(DispatchReason::NewTask, locked);
 778 }
 779
 780 bool GlobalHelperThreadState::submitTask(
 781     UniquePtr<FreeDelazifyTask> task, const AutoLockHelperThreadState& locked) {
 782   if (!freeDelazifyTaskVector(locked).append(std::move(task))) {
 783     return false;
 784   }
 785   dispatch(DispatchReason::NewTask, locked);
 786   return true;
 787 }
 788
 789 bool GlobalHelperThreadState::ensureInitialized() {
 790   MOZ_ASSERT(CanUseExtraThreads());
 791   MOZ_ASSERT(this == &HelperThreadState());
 792
 793   AutoLockHelperThreadState lock;
 794
 795   if (isInitialized(lock)) {
 796     return true;
 797   }
 798
 799   for (size_t& i : runningTaskCount) {
 800     i = 0;
 801   }
 802
 803   useInternalThreadPool_ = !dispatchTaskCallback;
 804   if (useInternalThreadPool(lock)) {
 805     if (!InternalThreadPool::Initialize(threadCount, lock)) {
 806       return false;
 807     }
 808   }
 809
 810   MOZ_ASSERT(dispatchTaskCallback);
 811
 812   if (!ensureThreadCount(threadCount, lock)) {
 813     finishThreads(lock);
 814     return false;
 815   }
 816
 817   MOZ_ASSERT(threadCount != 0);
 818   isInitialized_ = true;
 819   return true;
 820 }
 821
 822 bool GlobalHelperThreadState::ensureThreadCount(
 823     size_t count, AutoLockHelperThreadState& lock) {
 824   if (!helperTasks_.reserve(count)) {
 825     return false;
 826   }
 827
 828   if (useInternalThreadPool(lock)) {
 829     InternalThreadPool& pool = InternalThreadPool::Get();
 830     if (pool.threadCount(lock) < count) {
 831       if (!pool.ensureThreadCount(count, lock)) {
 832         return false;
 833       }
 834
 835       threadCount = pool.threadCount(lock);
 836     }
 837   }
 838
 839   return true;
 840 }
 841
 842 GlobalHelperThreadState::GlobalHelperThreadState()
 843     : cpuCount(0),
 844       threadCount(0),
 845       totalCountRunningTasks(0),
 846       registerThread(nullptr),
 847       unregisterThread(nullptr),
 848       wasmTier2GeneratorsFinished_(0) {
 849   MOZ_ASSERT(!gHelperThreadState);
 850
 851   cpuCount = ClampDefaultCPUCount(GetCPUCount());
 852   threadCount = ThreadCountForCPUCount(cpuCount);
 853   gcParallelThreadCount = threadCount;
 854
 855   MOZ_ASSERT(cpuCount > 0, "GetCPUCount() seems broken");
 856 }
 857
 858 void GlobalHelperThreadState::finish(AutoLockHelperThreadState& lock) {
 859   if (!isInitialized(lock)) {
 860     return;
 861   }
 862
 863   finishThreads(lock);
 864
 865   // Make sure there are no Ion free tasks left. We check this here because,
 866   // unlike the other tasks, we don't explicitly block on this when
 867   // destroying a runtime.
 868   auto& freeList = ionFreeList(lock);
 869   while (!freeList.empty()) {
 870     UniquePtr<jit::IonFreeTask> task = std::move(freeList.back());
 871     freeList.popBack();
 872     jit::FreeIonCompileTasks(task->compileTasks());
 873   }
 874 }
 875
 876 void GlobalHelperThreadState::finishThreads(AutoLockHelperThreadState& lock) {
 877   waitForAllTasksLocked(lock);
 878   terminating_ = true;
 879
 880   if (InternalThreadPool::IsInitialized()) {
 881     InternalThreadPool::ShutDown(lock);
 882   }
 883 }
 884
 885 #ifdef DEBUG
 886 void GlobalHelperThreadState::assertIsLockedByCurrentThread() const {
 887   gHelperThreadLock.assertOwnedByCurrentThread();
 888 }
 889 #endif  // DEBUG
 890
 891 void GlobalHelperThreadState::dispatch(
 892     DispatchReason reason, const AutoLockHelperThreadState& locked) {
 893   if (canStartTasks(locked) && tasksPending_ < threadCount) {
 894     // This doesn't guarantee that we don't dispatch more tasks to the external
 895     // pool than necessary if tasks are taking a long time to start, but it does
 896     // limit the number.
 897     tasksPending_++;
 898
 899     // The hazard analysis can't tell that the callback doesn't GC.
 900     JS::AutoSuppressGCAnalysis nogc;
 901
 902     dispatchTaskCallback(reason);
 903   }
 904 }
 905
 906 void GlobalHelperThreadState::wait(
 907     AutoLockHelperThreadState& locked,
 908     TimeDuration timeout /* = TimeDuration::Forever() */) {
 909   consumerWakeup.wait_for(locked, timeout);
 910 }
 911
 912 void GlobalHelperThreadState::notifyAll(const AutoLockHelperThreadState&) {
 913   consumerWakeup.notify_all();
 914 }
 915
 916 void GlobalHelperThreadState::notifyOne(const AutoLockHelperThreadState&) {
 917   consumerWakeup.notify_one();
 918 }
 919
 920 bool GlobalHelperThreadState::hasActiveThreads(
 921     const AutoLockHelperThreadState& lock) {
 922   return !helperTasks(lock).empty();
 923 }
 924
 925 void js::WaitForAllHelperThreads() { HelperThreadState().waitForAllTasks(); }
 926
 927 void js::WaitForAllHelperThreads(AutoLockHelperThreadState& lock) {
 928   HelperThreadState().waitForAllTasksLocked(lock);
 929 }
 930
 931 void GlobalHelperThreadState::waitForAllTasks() {
 932   AutoLockHelperThreadState lock;
 933   waitForAllTasksLocked(lock);
 934 }
 935
 936 void GlobalHelperThreadState::waitForAllTasksLocked(
 937     AutoLockHelperThreadState& lock) {
 938   CancelOffThreadWasmTier2GeneratorLocked(lock);
 939
 940   while (canStartTasks(lock) || tasksPending_ || hasActiveThreads(lock)) {
 941     wait(lock);
 942   }
 943
 944   MOZ_ASSERT(gcParallelWorklist().isEmpty(lock));
 945   MOZ_ASSERT(ionWorklist(lock).empty());
 946   MOZ_ASSERT(wasmWorklist(lock, wasm::CompileMode::Tier1).empty());
 947   MOZ_ASSERT(promiseHelperTasks(lock).empty());
 948   MOZ_ASSERT(compressionWorklist(lock).empty());
 949   MOZ_ASSERT(ionFreeList(lock).empty());
 950   MOZ_ASSERT(wasmWorklist(lock, wasm::CompileMode::Tier2).empty());
 951   MOZ_ASSERT(wasmTier2GeneratorWorklist(lock).empty());
 952   MOZ_ASSERT(!tasksPending_);
 953   MOZ_ASSERT(!hasActiveThreads(lock));
 954 }
 955
 956 // A task can be a "master" task, ie, it will block waiting for other worker
 957 // threads that perform work on its behalf.  If so it must not take the last
 958 // available thread; there must always be at least one worker thread able to do
 959 // the actual work.  (Or the system may deadlock.)
 960 //
 961 // If a task is a master task it *must* pass isMaster=true here, or perform a
 962 // similar calculation to avoid deadlock from starvation.
 963 //
 964 // isMaster should only be true if the thread calling checkTaskThreadLimit() is
 965 // a helper thread.
 966 //
 967 // NOTE: Calling checkTaskThreadLimit() from a helper thread in the dynamic
 968 // region after currentTask.emplace() and before currentTask.reset() may cause
 969 // it to return a different result than if it is called outside that dynamic
 970 // region, as the predicate inspects the values of the threads' currentTask
 971 // members.
 972
 973 bool GlobalHelperThreadState::checkTaskThreadLimit(
 974     ThreadType threadType, size_t maxThreads, bool isMaster,
 975     const AutoLockHelperThreadState& lock) const {
 976   MOZ_ASSERT(maxThreads > 0);
 977
 978   if (!isMaster && maxThreads >= threadCount) {
 979     return true;
 980   }
 981
 982   size_t count = runningTaskCount[threadType];
 983   if (count >= maxThreads) {
 984     return false;
 985   }
 986
 987   MOZ_ASSERT(threadCount >= totalCountRunningTasks);
 988   size_t idle = threadCount - totalCountRunningTasks;
 989
 990   // It is possible for the number of idle threads to be zero here, because
 991   // checkTaskThreadLimit() can be called from non-helper threads.  Notably,
 992   // the compression task scheduler invokes it, and runs off a helper thread.
 993   if (idle == 0) {
 994     return false;
 995   }
 996
 997   // A master thread that's the last available thread must not be allowed to
 998   // run.
 999   if (isMaster && idle == 1) {
1000     return false;
1001   }
1002
1003   return true;
1004 }
1005
1006 static inline bool IsHelperThreadSimulatingOOM(js::ThreadType threadType) {
1007 #if defined(DEBUG) || defined(JS_OOM_BREAKPOINT)
1008   return js::oom::simulator.targetThread() == threadType;
1009 #else
1010   return false;
1011 #endif
1012 }
1013
1014 void GlobalHelperThreadState::addSizeOfIncludingThis(
1015     JS::GlobalStats* stats, const AutoLockHelperThreadState& lock) const {
1016 #ifdef DEBUG
1017   assertIsLockedByCurrentThread();
1018 #endif
1019
1020   mozilla::MallocSizeOf mallocSizeOf = stats->mallocSizeOf_;
1021   JS::HelperThreadStats& htStats = stats->helperThread;
1022
1023   htStats.stateData += mallocSizeOf(this);
1024
1025   if (InternalThreadPool::IsInitialized()) {
1026     htStats.stateData +=
1027         InternalThreadPool::Get().sizeOfIncludingThis(mallocSizeOf, lock);
1028   }
1029
1030   // Report memory used by various containers
1031   htStats.stateData +=
1032       ionWorklist_.sizeOfExcludingThis(mallocSizeOf) +
1033       ionFinishedList_.sizeOfExcludingThis(mallocSizeOf) +
1034       ionFreeList_.sizeOfExcludingThis(mallocSizeOf) +
1035       wasmWorklist_tier1_.sizeOfExcludingThis(mallocSizeOf) +
1036       wasmWorklist_tier2_.sizeOfExcludingThis(mallocSizeOf) +
1037       wasmTier2GeneratorWorklist_.sizeOfExcludingThis(mallocSizeOf) +
1038       promiseHelperTasks_.sizeOfExcludingThis(mallocSizeOf) +
1039       compressionPendingList_.sizeOfExcludingThis(mallocSizeOf) +
1040       compressionWorklist_.sizeOfExcludingThis(mallocSizeOf) +
1041       compressionFinishedList_.sizeOfExcludingThis(mallocSizeOf) +
1042       gcParallelWorklist_.sizeOfExcludingThis(mallocSizeOf, lock) +
1043       helperTasks_.sizeOfExcludingThis(mallocSizeOf);
1044
1045   // Report IonCompileTasks on wait lists
1046   for (auto task : ionWorklist_) {
1047     htStats.ionCompileTask += task->sizeOfExcludingThis(mallocSizeOf);
1048   }
1049   for (auto task : ionFinishedList_) {
1050     htStats.ionCompileTask += task->sizeOfExcludingThis(mallocSizeOf);
1051   }
1052   for (const auto& task : ionFreeList_) {
1053     for (auto* compileTask : task->compileTasks()) {
1054       htStats.ionCompileTask += compileTask->sizeOfExcludingThis(mallocSizeOf);
1055     }
1056   }
1057
1058   // Report wasm::CompileTasks on wait lists
1059   for (auto task : wasmWorklist_tier1_) {
1060     htStats.wasmCompile += task->sizeOfExcludingThis(mallocSizeOf);
1061   }
1062   for (auto task : wasmWorklist_tier2_) {
1063     htStats.wasmCompile += task->sizeOfExcludingThis(mallocSizeOf);
1064   }
1065
1066   // Report number of helper threads.
1067   MOZ_ASSERT(htStats.idleThreadCount == 0);
1068   MOZ_ASSERT(threadCount >= totalCountRunningTasks);
1069   htStats.activeThreadCount = totalCountRunningTasks;
1070   htStats.idleThreadCount = threadCount - totalCountRunningTasks;
1071 }
1072
1073 size_t GlobalHelperThreadState::maxIonCompilationThreads() const {
1074   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_ION)) {
1075     return 1;
1076   }
1077   return threadCount;
1078 }
1079
1080 size_t GlobalHelperThreadState::maxIonFreeThreads() const {
1081   // IonFree tasks are low priority. Limit to one thread to help avoid jemalloc
1082   // lock contention.
1083   return 1;
1084 }
1085
1086 size_t GlobalHelperThreadState::maxWasmCompilationThreads() const {
1087   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER1) ||
1088       IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER2)) {
1089     return 1;
1090   }
1091   return std::min(cpuCount, threadCount);
1092 }
1093
1094 size_t GlobalHelperThreadState::maxWasmTier2GeneratorThreads() const {
1095   return MaxTier2GeneratorTasks;
1096 }
1097
1098 size_t GlobalHelperThreadState::maxPromiseHelperThreads() const {
1099   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER1) ||
1100       IsHelperThreadSimulatingOOM(js::THREAD_TYPE_WASM_COMPILE_TIER2)) {
1101     return 1;
1102   }
1103   return std::min(cpuCount, threadCount);
1104 }
1105
1106 size_t GlobalHelperThreadState::maxDelazifyThreads() const {
1107   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_DELAZIFY)) {
1108     return 1;
1109   }
1110   return std::min(cpuCount, threadCount);
1111 }
1112
1113 size_t GlobalHelperThreadState::maxCompressionThreads() const {
1114   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_COMPRESS)) {
1115     return 1;
1116   }
1117
1118   // Compression is triggered on major GCs to compress ScriptSources. It is
1119   // considered low priority work.
1120   return 1;
1121 }
1122
1123 size_t GlobalHelperThreadState::maxGCParallelThreads(
1124     const AutoLockHelperThreadState& lock) const {
1125   if (IsHelperThreadSimulatingOOM(js::THREAD_TYPE_GCPARALLEL)) {
1126     return 1;
1127   }
1128   return gcParallelThreadCount;
1129 }
1130
1131 HelperThreadTask* GlobalHelperThreadState::maybeGetWasmTier1CompileTask(
1132     const AutoLockHelperThreadState& lock) {
1133   return maybeGetWasmCompile(lock, wasm::CompileMode::Tier1);
1134 }
1135
1136 HelperThreadTask* GlobalHelperThreadState::maybeGetWasmTier2CompileTask(
1137     const AutoLockHelperThreadState& lock) {
1138   return maybeGetWasmCompile(lock, wasm::CompileMode::Tier2);
1139 }
1140
1141 HelperThreadTask* GlobalHelperThreadState::maybeGetWasmCompile(
1142     const AutoLockHelperThreadState& lock, wasm::CompileMode mode) {
1143   if (!canStartWasmCompile(lock, mode)) {
1144     return nullptr;
1145   }
1146
1147   return wasmWorklist(lock, mode).popCopyFront();
1148 }
1149
1150 bool GlobalHelperThreadState::canStartWasmTier1CompileTask(
1151     const AutoLockHelperThreadState& lock) {
1152   return canStartWasmCompile(lock, wasm::CompileMode::Tier1);
1153 }
1154
1155 bool GlobalHelperThreadState::canStartWasmTier2CompileTask(
1156     const AutoLockHelperThreadState& lock) {
1157   return canStartWasmCompile(lock, wasm::CompileMode::Tier2);
1158 }
1159
1160 bool GlobalHelperThreadState::canStartWasmCompile(
1161     const AutoLockHelperThreadState& lock, wasm::CompileMode mode) {
1162   if (wasmWorklist(lock, mode).empty()) {
1163     return false;
1164   }
1165
1166   // Parallel compilation and background compilation should be disabled on
1167   // unicore systems.
1168
1169   MOZ_RELEASE_ASSERT(cpuCount > 1);
1170
1171   // If Tier2 is very backlogged we must give priority to it, since the Tier2
1172   // queue holds onto Tier1 tasks.  Indeed if Tier2 is backlogged we will
1173   // devote more resources to Tier2 and not start any Tier1 work at all.
1174
1175   bool tier2oversubscribed = wasmTier2GeneratorWorklist(lock).length() > 20;
1176
1177   // For Tier1 and Once compilation, honor the maximum allowed threads to
1178   // compile wasm jobs at once, to avoid oversaturating the machine.
1179   //
1180   // For Tier2 compilation we need to allow other things to happen too, so we
1181   // do not allow all logical cores to be used for background work; instead we
1182   // wish to use a fraction of the physical cores.  We can't directly compute
1183   // the physical cores from the logical cores, but 1/3 of the logical cores
1184   // is a safe estimate for the number of physical cores available for
1185   // background work.
1186
1187   size_t physCoresAvailable = size_t(ceil(cpuCount / 3.0));
1188
1189   size_t threads;
1190   ThreadType threadType;
1191   if (mode == wasm::CompileMode::Tier2) {
1192     if (tier2oversubscribed) {
1193       threads = maxWasmCompilationThreads();
1194     } else {
1195       threads = physCoresAvailable;
1196     }
1197     threadType = THREAD_TYPE_WASM_COMPILE_TIER2;
1198   } else {
1199     if (tier2oversubscribed) {
1200       threads = 0;
1201     } else {
1202       threads = maxWasmCompilationThreads();
1203     }
1204     threadType = THREAD_TYPE_WASM_COMPILE_TIER1;
1205   }
1206
1207   return threads != 0 && checkTaskThreadLimit(threadType, threads, lock);
1208 }
1209
1210 HelperThreadTask* GlobalHelperThreadState::maybeGetWasmTier2GeneratorTask(
1211     const AutoLockHelperThreadState& lock) {
1212   if (!canStartWasmTier2GeneratorTask(lock)) {
1213     return nullptr;
1214   }
1215
1216   return wasmTier2GeneratorWorklist(lock).popCopy();
1217 }
1218
1219 bool GlobalHelperThreadState::canStartWasmTier2GeneratorTask(
1220     const AutoLockHelperThreadState& lock) {
1221   return !wasmTier2GeneratorWorklist(lock).empty() &&
1222          checkTaskThreadLimit(THREAD_TYPE_WASM_GENERATOR_TIER2,
1223                               maxWasmTier2GeneratorThreads(),
1224                               /*isMaster=*/true, lock);
1225 }
1226
1227 HelperThreadTask* GlobalHelperThreadState::maybeGetPromiseHelperTask(
1228     const AutoLockHelperThreadState& lock) {
1229   if (!canStartPromiseHelperTask(lock)) {
1230     return nullptr;
1231   }
1232
1233   return promiseHelperTasks(lock).popCopy();
1234 }
1235
1236 bool GlobalHelperThreadState::canStartPromiseHelperTask(
1237     const AutoLockHelperThreadState& lock) {
1238   // PromiseHelperTasks can be wasm compilation tasks that in turn block on
1239   // wasm compilation so set isMaster = true.
1240   return !promiseHelperTasks(lock).empty() &&
1241          checkTaskThreadLimit(THREAD_TYPE_PROMISE_TASK,
1242                               maxPromiseHelperThreads(),
1243                               /*isMaster=*/true, lock);
1244 }
1245
1246 static bool IonCompileTaskHasHigherPriority(jit::IonCompileTask* first,
1247                                             jit::IonCompileTask* second) {
1248   // Return true if priority(first) > priority(second).
1249   //
1250   // This method can return whatever it wants, though it really ought to be a
1251   // total order. The ordering is allowed to race (change on the fly), however.
1252
1253   // A higher warm-up counter indicates a higher priority.
1254   jit::JitScript* firstJitScript = first->script()->jitScript();
1255   jit::JitScript* secondJitScript = second->script()->jitScript();
1256   return firstJitScript->warmUpCount() / first->script()->length() >
1257          secondJitScript->warmUpCount() / second->script()->length();
1258 }
1259
1260 HelperThreadTask* GlobalHelperThreadState::maybeGetIonCompileTask(
1261     const AutoLockHelperThreadState& lock) {
1262   if (!canStartIonCompileTask(lock)) {
1263     return nullptr;
1264   }
1265
1266   return highestPriorityPendingIonCompile(lock,
1267                                           /* checkExecutionStatus */ true);
1268 }
1269
1270 HelperThreadTask* GlobalHelperThreadState::maybeGetLowPrioIonCompileTask(
1271     const AutoLockHelperThreadState& lock) {
1272   if (!canStartIonCompileTask(lock)) {
1273     return nullptr;
1274   }
1275
1276   return highestPriorityPendingIonCompile(lock,
1277                                           /* checkExecutionStatus */ false);
1278 }
1279
1280 bool GlobalHelperThreadState::canStartIonCompileTask(
1281     const AutoLockHelperThreadState& lock) {
1282   return !ionWorklist(lock).empty() &&
1283          checkTaskThreadLimit(THREAD_TYPE_ION, maxIonCompilationThreads(),
1284                               lock);
1285 }
1286
1287 HelperThreadTask* GlobalHelperThreadState::maybeGetIonFreeTask(
1288     const AutoLockHelperThreadState& lock) {
1289   if (!canStartIonFreeTask(lock)) {
1290     return nullptr;
1291   }
1292
1293   UniquePtr<jit::IonFreeTask> task = std::move(ionFreeList(lock).back());
1294   ionFreeList(lock).popBack();
1295   return task.release();
1296 }
1297
1298 bool GlobalHelperThreadState::canStartIonFreeTask(
1299     const AutoLockHelperThreadState& lock) {
1300   return !ionFreeList(lock).empty() &&
1301          checkTaskThreadLimit(THREAD_TYPE_ION_FREE, maxIonFreeThreads(), lock);
1302 }
1303
1304 jit::IonCompileTask* GlobalHelperThreadState::highestPriorityPendingIonCompile(
1305     const AutoLockHelperThreadState& lock, bool checkExecutionStatus) {
1306   auto& worklist = ionWorklist(lock);
1307   MOZ_ASSERT(!worklist.empty());
1308
1309   // Get the highest priority IonCompileTask which has not started compilation
1310   // yet.
1311   size_t index = worklist.length();
1312   for (size_t i = 0; i < worklist.length(); i++) {
1313     if (checkExecutionStatus && !worklist[i]->isMainThreadRunningJS()) {
1314       continue;
1315     }
1316     if (i < index ||
1317         IonCompileTaskHasHigherPriority(worklist[i], worklist[index])) {
1318       index = i;
1319     }
1320   }
1321
1322   if (index == worklist.length()) {
1323     return nullptr;
1324   }
1325   jit::IonCompileTask* task = worklist[index];
1326   worklist.erase(&worklist[index]);
1327   return task;
1328 }
1329
1330 HelperThreadTask* GlobalHelperThreadState::maybeGetFreeDelazifyTask(
1331     const AutoLockHelperThreadState& lock) {
1332   auto& freeList = freeDelazifyTaskVector(lock);
1333   if (!freeList.empty()) {
1334     UniquePtr<FreeDelazifyTask> task = std::move(freeList.back());
1335     freeList.popBack();
1336     return task.release();
1337   }
1338   return nullptr;
1339 }
1340
1341 bool GlobalHelperThreadState::canStartFreeDelazifyTask(
1342     const AutoLockHelperThreadState& lock) {
1343   return !freeDelazifyTaskVector(lock).empty() &&
1344          checkTaskThreadLimit(THREAD_TYPE_DELAZIFY_FREE, maxDelazifyThreads(),
1345                               /*isMaster=*/true, lock);
1346 }
1347
1348 HelperThreadTask* GlobalHelperThreadState::maybeGetDelazifyTask(
1349     const AutoLockHelperThreadState& lock) {
1350   // NOTE: We want to span all cores availables with delazification tasks, in
1351   // order to parse a maximum number of functions ahead of their executions.
1352   // Thus, as opposed to parse task which have a higher priority, we are not
1353   // exclusively executing these task on parse threads.
1354   auto& worklist = delazifyWorklist(lock);
1355   if (worklist.isEmpty()) {
1356     return nullptr;
1357   }
1358   return worklist.popFirst();
1359 }
1360
1361 bool GlobalHelperThreadState::canStartDelazifyTask(
1362     const AutoLockHelperThreadState& lock) {
1363   return !delazifyWorklist(lock).isEmpty() &&
1364          checkTaskThreadLimit(THREAD_TYPE_DELAZIFY, maxDelazifyThreads(),
1365                               /*isMaster=*/true, lock);
1366 }
1367
1368 HelperThreadTask* GlobalHelperThreadState::maybeGetCompressionTask(
1369     const AutoLockHelperThreadState& lock) {
1370   if (!canStartCompressionTask(lock)) {
1371     return nullptr;
1372   }
1373
1374   auto& worklist = compressionWorklist(lock);
1375   UniquePtr<SourceCompressionTask> task = std::move(worklist.back());
1376   worklist.popBack();
1377   return task.release();
1378 }
1379
1380 bool GlobalHelperThreadState::canStartCompressionTask(
1381     const AutoLockHelperThreadState& lock) {
1382   return !compressionWorklist(lock).empty() &&
1383          checkTaskThreadLimit(THREAD_TYPE_COMPRESS, maxCompressionThreads(),
1384                               lock);
1385 }
1386
1387 void GlobalHelperThreadState::startHandlingCompressionTasks(
1388     ScheduleCompressionTask schedule, JSRuntime* maybeRuntime,
1389     const AutoLockHelperThreadState& lock) {
1390   MOZ_ASSERT((schedule == ScheduleCompressionTask::GC) ==
1391              (maybeRuntime != nullptr));
1392
1393   auto& pending = compressionPendingList(lock);
1394
1395   for (size_t i = 0; i < pending.length(); i++) {
1396     UniquePtr<SourceCompressionTask>& task = pending[i];
1397     if (schedule == ScheduleCompressionTask::API ||
1398         (task->runtimeMatches(maybeRuntime) && task->shouldStart())) {
1399       // OOMing during appending results in the task not being scheduled
1400       // and deleted.
1401       (void)submitTask(std::move(task), lock);
1402       remove(pending, &i);
1403     }
1404   }
1405 }
1406
1407 bool GlobalHelperThreadState::submitTask(
1408     UniquePtr<SourceCompressionTask> task,
1409     const AutoLockHelperThreadState& locked) {
1410   if (!compressionWorklist(locked).append(std::move(task))) {
1411     return false;
1412   }
1413
1414   dispatch(DispatchReason::NewTask, locked);
1415   return true;
1416 }
1417
1418 bool GlobalHelperThreadState::submitTask(
1419     GCParallelTask* task, const AutoLockHelperThreadState& locked) {
1420   gcParallelWorklist().insertBack(task, locked);
1421   dispatch(DispatchReason::NewTask, locked);
1422   return true;
1423 }
1424
1425 HelperThreadTask* GlobalHelperThreadState::maybeGetGCParallelTask(
1426     const AutoLockHelperThreadState& lock) {
1427   if (!canStartGCParallelTask(lock)) {
1428     return nullptr;
1429   }
1430
1431   return gcParallelWorklist().popFirst(lock);
1432 }
1433
1434 bool GlobalHelperThreadState::canStartGCParallelTask(
1435     const AutoLockHelperThreadState& lock) {
1436   return !gcParallelWorklist().isEmpty(lock) &&
1437          checkTaskThreadLimit(THREAD_TYPE_GCPARALLEL,
1438                               maxGCParallelThreads(lock), lock);
1439 }
1440
1441 bool js::EnqueueOffThreadCompression(JSContext* cx,
1442                                      UniquePtr<SourceCompressionTask> task) {
1443   AutoLockHelperThreadState lock;
1444
1445   auto& pending = HelperThreadState().compressionPendingList(lock);
1446   if (!pending.append(std::move(task))) {
1447     ReportOutOfMemory(cx);
1448     return false;
1449   }
1450
1451   return true;
1452 }
1453
1454 void js::StartHandlingCompressionsOnGC(JSRuntime* runtime) {
1455   AutoLockHelperThreadState lock;
1456   HelperThreadState().startHandlingCompressionTasks(
1457       GlobalHelperThreadState::ScheduleCompressionTask::GC, runtime, lock);
1458 }
1459
1460 template <typename T>
1461 static void ClearCompressionTaskList(T& list, JSRuntime* runtime) {
1462   for (size_t i = 0; i < list.length(); i++) {
1463     if (list[i]->runtimeMatches(runtime)) {
1464       HelperThreadState().remove(list, &i);
1465     }
1466   }
1467 }
1468
1469 void js::CancelOffThreadCompressions(JSRuntime* runtime) {
1470   if (!CanUseExtraThreads()) {
1471     return;
1472   }
1473
1474   AutoLockHelperThreadState lock;
1475
1476   // Cancel all pending compression tasks.
1477   ClearCompressionTaskList(HelperThreadState().compressionPendingList(lock),
1478                            runtime);
1479   ClearCompressionTaskList(HelperThreadState().compressionWorklist(lock),
1480                            runtime);
1481
1482   // Cancel all in-process compression tasks and wait for them to join so we
1483   // clean up the finished tasks.
1484   while (true) {
1485     bool inProgress = false;
1486     for (auto* helper : HelperThreadState().helperTasks(lock)) {
1487       if (!helper->is<SourceCompressionTask>()) {
1488         continue;
1489       }
1490
1491       if (helper->as<SourceCompressionTask>()->runtimeMatches(runtime)) {
1492         inProgress = true;
1493       }
1494     }
1495
1496     if (!inProgress) {
1497       break;
1498     }
1499
1500     HelperThreadState().wait(lock);
1501   }
1502
1503   // Clean up finished tasks.
1504   ClearCompressionTaskList(HelperThreadState().compressionFinishedList(lock),
1505                            runtime);
1506 }
1507
1508 void js::AttachFinishedCompressions(JSRuntime* runtime,
1509                                     AutoLockHelperThreadState& lock) {
1510   auto& finished = HelperThreadState().compressionFinishedList(lock);
1511   for (size_t i = 0; i < finished.length(); i++) {
1512     if (finished[i]->runtimeMatches(runtime)) {
1513       UniquePtr<SourceCompressionTask> compressionTask(std::move(finished[i]));
1514       HelperThreadState().remove(finished, &i);
1515       compressionTask->complete();
1516     }
1517   }
1518 }
1519
1520 void js::SweepPendingCompressions(AutoLockHelperThreadState& lock) {
1521   auto& pending = HelperThreadState().compressionPendingList(lock);
1522   for (size_t i = 0; i < pending.length(); i++) {
1523     if (pending[i]->shouldCancel()) {
1524       HelperThreadState().remove(pending, &i);
1525     }
1526   }
1527 }
1528
1529 void js::RunPendingSourceCompressions(JSRuntime* runtime) {
1530   if (!CanUseExtraThreads()) {
1531     return;
1532   }
1533
1534   AutoLockHelperThreadState lock;
1535
1536   HelperThreadState().startHandlingCompressionTasks(
1537       GlobalHelperThreadState::ScheduleCompressionTask::API, nullptr, lock);
1538
1539   // Wait until all tasks have started compression.
1540   while (!HelperThreadState().compressionWorklist(lock).empty()) {
1541     HelperThreadState().wait(lock);
1542   }
1543
1544   // Wait for all in-process compression tasks to complete.
1545   HelperThreadState().waitForAllTasksLocked(lock);
1546
1547   AttachFinishedCompressions(runtime, lock);
1548 }
1549
1550 void PromiseHelperTask::executeAndResolveAndDestroy(JSContext* cx) {
1551   execute();
1552   run(cx, JS::Dispatchable::NotShuttingDown);
1553 }
1554
1555 void PromiseHelperTask::runHelperThreadTask(AutoLockHelperThreadState& lock) {
1556   {
1557     AutoUnlockHelperThreadState unlock(lock);
1558     execute();
1559   }
1560
1561   // Don't release the lock between dispatching the resolve and destroy
1562   // operation (which may start immediately on another thread) and returning
1563   // from this method.
1564
1565   dispatchResolveAndDestroy(lock);
1566 }
1567
1568 bool js::StartOffThreadPromiseHelperTask(JSContext* cx,
1569                                          UniquePtr<PromiseHelperTask> task) {
1570   // Execute synchronously if there are no helper threads.
1571   if (!CanUseExtraThreads()) {
1572     task.release()->executeAndResolveAndDestroy(cx);
1573     return true;
1574   }
1575
1576   if (!HelperThreadState().submitTask(task.get())) {
1577     ReportOutOfMemory(cx);
1578     return false;
1579   }
1580
1581   (void)task.release();
1582   return true;
1583 }
1584
1585 bool js::StartOffThreadPromiseHelperTask(PromiseHelperTask* task) {
1586   MOZ_ASSERT(CanUseExtraThreads());
1587
1588   return HelperThreadState().submitTask(task);
1589 }
1590
1591 bool GlobalHelperThreadState::submitTask(PromiseHelperTask* task) {
1592   AutoLockHelperThreadState lock;
1593
1594   if (!promiseHelperTasks(lock).append(task)) {
1595     return false;
1596   }
1597
1598   dispatch(DispatchReason::NewTask, lock);
1599   return true;
1600 }
1601
1602 void GlobalHelperThreadState::trace(JSTracer* trc) {
1603   {
1604     AutoLockHelperThreadState lock;
1605
1606 #ifdef DEBUG
1607     // Since we hold the helper thread lock here we must disable GCMarker's
1608     // checking of the atom marking bitmap since that also relies on taking the
1609     // lock.
1610     GCMarker* marker = nullptr;
1611     if (trc->isMarkingTracer()) {
1612       marker = GCMarker::fromTracer(trc);
1613       marker->setCheckAtomMarking(false);
1614     }
1615     auto reenableAtomMarkingCheck = mozilla::MakeScopeExit([marker] {
1616       if (marker) {
1617         marker->setCheckAtomMarking(true);
1618       }
1619     });
1620 #endif
1621
1622     for (auto task : ionWorklist(lock)) {
1623       task->alloc().lifoAlloc()->setReadWrite();
1624       task->trace(trc);
1625       task->alloc().lifoAlloc()->setReadOnly();
1626     }
1627     for (auto task : ionFinishedList(lock)) {
1628       task->trace(trc);
1629     }
1630
1631     for (auto* helper : HelperThreadState().helperTasks(lock)) {
1632       if (helper->is<jit::IonCompileTask>()) {
1633         helper->as<jit::IonCompileTask>()->trace(trc);
1634       }
1635     }
1636   }
1637
1638   // The lazy link list is only accessed on the main thread, so trace it after
1639   // releasing the lock.
1640   JSRuntime* rt = trc->runtime();
1641   if (auto* jitRuntime = rt->jitRuntime()) {
1642     jit::IonCompileTask* task = jitRuntime->ionLazyLinkList(rt).getFirst();
1643     while (task) {
1644       task->trace(trc);
1645       task = task->getNext();
1646     }
1647   }
1648 }
1649
1650 // Definition of helper thread tasks.
1651 //
1652 // Priority is determined by the order they're listed here.
1653 const GlobalHelperThreadState::Selector GlobalHelperThreadState::selectors[] = {
1654     &GlobalHelperThreadState::maybeGetGCParallelTask,
1655     &GlobalHelperThreadState::maybeGetIonCompileTask,
1656     &GlobalHelperThreadState::maybeGetWasmTier1CompileTask,
1657     &GlobalHelperThreadState::maybeGetPromiseHelperTask,
1658     &GlobalHelperThreadState::maybeGetFreeDelazifyTask,
1659     &GlobalHelperThreadState::maybeGetDelazifyTask,
1660     &GlobalHelperThreadState::maybeGetCompressionTask,
1661     &GlobalHelperThreadState::maybeGetLowPrioIonCompileTask,
1662     &GlobalHelperThreadState::maybeGetIonFreeTask,
1663     &GlobalHelperThreadState::maybeGetWasmTier2CompileTask,
1664     &GlobalHelperThreadState::maybeGetWasmTier2GeneratorTask};
1665
1666 bool GlobalHelperThreadState::canStartTasks(
1667     const AutoLockHelperThreadState& lock) {
1668   return canStartGCParallelTask(lock) || canStartIonCompileTask(lock) ||
1669          canStartWasmTier1CompileTask(lock) ||
1670          canStartPromiseHelperTask(lock) || canStartFreeDelazifyTask(lock) ||
1671          canStartDelazifyTask(lock) || canStartCompressionTask(lock) ||
1672          canStartIonFreeTask(lock) || canStartWasmTier2CompileTask(lock) ||
1673          canStartWasmTier2GeneratorTask(lock);
1674 }
1675
1676 void JS::RunHelperThreadTask() {
1677   MOZ_ASSERT(CanUseExtraThreads());
1678
1679   AutoLockHelperThreadState lock;
1680
1681   if (!gHelperThreadState || HelperThreadState().isTerminating(lock)) {
1682     return;
1683   }
1684
1685   HelperThreadState().runOneTask(lock);
1686 }
1687
1688 void GlobalHelperThreadState::runOneTask(AutoLockHelperThreadState& lock) {
1689   MOZ_ASSERT(tasksPending_ > 0);
1690   tasksPending_--;
1691
1692   // The selectors may depend on the HelperThreadState not changing between task
1693   // selection and task execution, in particular, on new tasks not being added
1694   // (because of the lifo structure of the work lists). Unlocking the
1695   // HelperThreadState between task selection and execution is not well-defined.
1696   HelperThreadTask* task = findHighestPriorityTask(lock);
1697   if (task) {
1698     runTaskLocked(task, lock);
1699     dispatch(DispatchReason::FinishedTask, lock);
1700   }
1701
1702   notifyAll(lock);
1703 }
1704
1705 HelperThreadTask* GlobalHelperThreadState::findHighestPriorityTask(
1706     const AutoLockHelperThreadState& locked) {
1707   // Return the highest priority task that is ready to start, or nullptr.
1708
1709   for (const auto& selector : selectors) {
1710     if (auto* task = (this->*(selector))(locked)) {
1711       return task;
1712     }
1713   }
1714
1715   return nullptr;
1716 }
1717
1718 void GlobalHelperThreadState::runTaskLocked(HelperThreadTask* task,
1719                                             AutoLockHelperThreadState& locked) {
1720   JS::AutoSuppressGCAnalysis nogc;
1721
1722   HelperThreadState().helperTasks(locked).infallibleEmplaceBack(task);
1723
1724   ThreadType threadType = task->threadType();
1725   js::oom::SetThreadType(threadType);
1726   runningTaskCount[threadType]++;
1727   totalCountRunningTasks++;
1728
1729   task->runHelperThreadTask(locked);
1730
1731   // Delete task from helperTasks.
1732   HelperThreadState().helperTasks(locked).eraseIfEqual(task);
1733
1734   totalCountRunningTasks--;
1735   runningTaskCount[threadType]--;
1736
1737   js::oom::SetThreadType(js::THREAD_TYPE_NONE);
1738 }