xpcom/threads/TaskController.cpp

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #include "TaskController.h"
   8 #include "nsIIdleRunnable.h"
   9 #include "nsIRunnable.h"
  10 #include "nsThreadUtils.h"
  11 #include <algorithm>
  12 #include <initializer_list>
  13 #include "GeckoProfiler.h"
  14 #include "mozilla/EventQueue.h"
  15 #include "mozilla/BackgroundHangMonitor.h"
  16 #include "mozilla/InputTaskManager.h"
  17 #include "mozilla/VsyncTaskManager.h"
  18 #include "mozilla/IOInterposer.h"
  19 #include "mozilla/StaticMutex.h"
  20 #include "mozilla/SchedulerGroup.h"
  21 #include "mozilla/ScopeExit.h"
  22 #include "mozilla/Unused.h"
  23 #include "nsIThreadInternal.h"
  24 #include "nsQueryObject.h"
  25 #include "nsThread.h"
  26 #include "prenv.h"
  27 #include "prsystem.h"
  28
  29 namespace mozilla {
  30
  31 std::unique_ptr<TaskController> TaskController::sSingleton;
  32 thread_local size_t mThreadPoolIndex = -1;
  33 std::atomic<uint64_t> Task::sCurrentTaskSeqNo = 0;
  34
  35 const int32_t kMinimumPoolThreadCount = 2;
  36 const int32_t kMaximumPoolThreadCount = 8;
  37
  38 /* static */
  39 int32_t TaskController::GetPoolThreadCount() {
  40   if (PR_GetEnv("MOZ_TASKCONTROLLER_THREADCOUNT")) {
  41     return strtol(PR_GetEnv("MOZ_TASKCONTROLLER_THREADCOUNT"), nullptr, 0);
  42   }
  43
  44   int32_t numCores = std::max<int32_t>(1, PR_GetNumberOfProcessors());
  45
  46   return std::clamp<int32_t>(numCores, kMinimumPoolThreadCount,
  47                              kMaximumPoolThreadCount);
  48 }
  49
  50 #if defined(MOZ_COLLECTING_RUNNABLE_TELEMETRY)
  51
  52 struct TaskMarker {
  53   static constexpr Span<const char> MarkerTypeName() {
  54     return MakeStringSpan("Task");
  55   }
  56   static void StreamJSONMarkerData(baseprofiler::SpliceableJSONWriter& aWriter,
  57                                    const nsCString& aName, uint32_t aPriority) {
  58     aWriter.StringProperty("name", aName);
  59     aWriter.IntProperty("priority", aPriority);
  60
  61 #  define EVENT_PRIORITY(NAME, VALUE)                \
  62     if (aPriority == (VALUE)) {                      \
  63       aWriter.StringProperty("priorityName", #NAME); \
  64     } else
  65     EVENT_QUEUE_PRIORITY_LIST(EVENT_PRIORITY)
  66 #  undef EVENT_PRIORITY
  67     {
  68       aWriter.StringProperty("priorityName", "Invalid Value");
  69     }
  70   }
  71   static MarkerSchema MarkerTypeDisplay() {
  72     using MS = MarkerSchema;
  73     MS schema{MS::Location::MarkerChart, MS::Location::MarkerTable};
  74     schema.SetChartLabel("{marker.data.name}");
  75     schema.SetTableLabel(
  76         "{marker.name} - {marker.data.name} - priority: "
  77         "{marker.data.priorityName} ({marker.data.priority})");
  78     schema.AddKeyLabelFormatSearchable("name", "Task Name", MS::Format::String,
  79                                        MS::Searchable::Searchable);
  80     schema.AddKeyLabelFormat("priorityName", "Priority Name",
  81                              MS::Format::String);
  82     schema.AddKeyLabelFormat("priority", "Priority level", MS::Format::Integer);
  83     return schema;
  84   }
  85 };
  86
  87 class MOZ_RAII AutoProfileTask {
  88  public:
  89   explicit AutoProfileTask(nsACString& aName, uint64_t aPriority)
  90       : mName(aName), mPriority(aPriority) {
  91     if (profiler_is_active()) {
  92       mStartTime = TimeStamp::Now();
  93     }
  94   }
  95
  96   ~AutoProfileTask() {
  97     if (!profiler_thread_is_being_profiled_for_markers()) {
  98       return;
  99     }
 100
 101     AUTO_PROFILER_LABEL("AutoProfileTask", PROFILER);
 102     AUTO_PROFILER_STATS(AUTO_PROFILE_TASK);
 103     profiler_add_marker("Runnable", ::mozilla::baseprofiler::category::OTHER,
 104                         mStartTime.IsNull()
 105                             ? MarkerTiming::IntervalEnd()
 106                             : MarkerTiming::IntervalUntilNowFrom(mStartTime),
 107                         TaskMarker{}, mName, mPriority);
 108   }
 109
 110  private:
 111   TimeStamp mStartTime;
 112   nsAutoCString mName;
 113   uint32_t mPriority;
 114 };
 115
 116 #  define AUTO_PROFILE_FOLLOWING_TASK(task)                                  \
 117     nsAutoCString name;                                                      \
 118     (task)->GetName(name);                                                   \
 119     AUTO_PROFILER_LABEL_DYNAMIC_NSCSTRING_NONSENSITIVE("Task", OTHER, name); \
 120     mozilla::AutoProfileTask PROFILER_RAII(name, (task)->GetPriority());
 121 #else
 122 #  define AUTO_PROFILE_FOLLOWING_TASK(task)
 123 #endif
 124
 125 bool TaskManager::
 126     UpdateCachesForCurrentIterationAndReportPriorityModifierChanged(
 127         const MutexAutoLock& aProofOfLock, IterationType aIterationType) {
 128   mCurrentSuspended = IsSuspended(aProofOfLock);
 129
 130   if (aIterationType == IterationType::EVENT_LOOP_TURN && !mCurrentSuspended) {
 131     int32_t oldModifier = mCurrentPriorityModifier;
 132     mCurrentPriorityModifier =
 133         GetPriorityModifierForEventLoopTurn(aProofOfLock);
 134
 135     if (mCurrentPriorityModifier != oldModifier) {
 136       return true;
 137     }
 138   }
 139   return false;
 140 }
 141
 142 #ifdef MOZ_COLLECTING_RUNNABLE_TELEMETRY
 143 class MOZ_RAII AutoSetMainThreadRunnableName {
 144  public:
 145   explicit AutoSetMainThreadRunnableName(const nsCString& aName) {
 146     MOZ_ASSERT(NS_IsMainThread());
 147     // We want to record our current runnable's name in a static so
 148     // that BHR can record it.
 149     mRestoreRunnableName = nsThread::sMainThreadRunnableName;
 150
 151     // Copy the name into sMainThreadRunnableName's buffer, and append a
 152     // terminating null.
 153     uint32_t length = std::min((uint32_t)nsThread::kRunnableNameBufSize - 1,
 154                                (uint32_t)aName.Length());
 155     memcpy(nsThread::sMainThreadRunnableName.begin(), aName.BeginReading(),
 156            length);
 157     nsThread::sMainThreadRunnableName[length] = '\0';
 158   }
 159
 160   ~AutoSetMainThreadRunnableName() {
 161     nsThread::sMainThreadRunnableName = mRestoreRunnableName;
 162   }
 163
 164  private:
 165   Array<char, nsThread::kRunnableNameBufSize> mRestoreRunnableName;
 166 };
 167 #endif
 168
 169 Task* Task::GetHighestPriorityDependency() {
 170   Task* currentTask = this;
 171
 172   while (!currentTask->mDependencies.empty()) {
 173     auto iter = currentTask->mDependencies.begin();
 174
 175     while (iter != currentTask->mDependencies.end()) {
 176       if ((*iter)->mCompleted) {
 177         auto oldIter = iter;
 178         iter++;
 179         // Completed tasks are removed here to prevent needlessly keeping them
 180         // alive or iterating over them in the future.
 181         currentTask->mDependencies.erase(oldIter);
 182         continue;
 183       }
 184
 185       currentTask = iter->get();
 186       break;
 187     }
 188   }
 189
 190   return currentTask == this ? nullptr : currentTask;
 191 }
 192
 193 TaskController* TaskController::Get() {
 194   MOZ_ASSERT(sSingleton.get());
 195   return sSingleton.get();
 196 }
 197
 198 void TaskController::Initialize() {
 199   MOZ_ASSERT(!sSingleton);
 200   sSingleton = std::make_unique<TaskController>();
 201 }
 202
 203 void ThreadFuncPoolThread(void* aIndex) {
 204   mThreadPoolIndex = *reinterpret_cast<int32_t*>(aIndex);
 205   delete reinterpret_cast<int32_t*>(aIndex);
 206   TaskController::Get()->RunPoolThread();
 207 }
 208
 209 TaskController::TaskController()
 210     : mGraphMutex("TaskController::mGraphMutex"),
 211       mThreadPoolCV(mGraphMutex, "TaskController::mThreadPoolCV"),
 212       mMainThreadCV(mGraphMutex, "TaskController::mMainThreadCV"),
 213       mRunOutOfMTTasksCounter(0) {
 214   InputTaskManager::Init();
 215   VsyncTaskManager::Init();
 216   mMTProcessingRunnable = NS_NewRunnableFunction(
 217       "TaskController::ExecutePendingMTTasks()",
 218       []() { TaskController::Get()->ProcessPendingMTTask(); });
 219   mMTBlockingProcessingRunnable = NS_NewRunnableFunction(
 220       "TaskController::ExecutePendingMTTasks()",
 221       []() { TaskController::Get()->ProcessPendingMTTask(true); });
 222 }
 223
 224 // We want our default stack size limit to be approximately 2MB, to be safe for
 225 // JS helper tasks that can use a lot of stack, but expect most threads to use
 226 // much less. On Linux, however, requesting a stack of 2MB or larger risks the
 227 // kernel allocating an entire 2MB huge page for it on first access, which we do
 228 // not want. To avoid this possibility, we subtract 2 standard VM page sizes
 229 // from our default.
 230 constexpr PRUint32 sBaseStackSize = 2048 * 1024 - 2 * 4096;
 231
 232 // TSan enforces a minimum stack size that's just slightly larger than our
 233 // default helper stack size.  It does this to store blobs of TSan-specific data
 234 // on each thread's stack.  Unfortunately, that means that even though we'll
 235 // actually receive a larger stack than we requested, the effective usable space
 236 // of that stack is significantly less than what we expect.  To offset TSan
 237 // stealing our stack space from underneath us, double the default.
 238 //
 239 // Similarly, ASan requires more stack space due to red-zones.
 240 #if defined(MOZ_TSAN) || defined(MOZ_ASAN)
 241 constexpr PRUint32 sStackSize = 2 * sBaseStackSize;
 242 #else
 243 constexpr PRUint32 sStackSize = sBaseStackSize;
 244 #endif
 245
 246 void TaskController::InitializeThreadPool() {
 247   mPoolInitializationMutex.AssertCurrentThreadOwns();
 248   MOZ_ASSERT(!mThreadPoolInitialized);
 249   mThreadPoolInitialized = true;
 250
 251   int32_t poolSize = GetPoolThreadCount();
 252   for (int32_t i = 0; i < poolSize; i++) {
 253     int32_t* index = new int32_t(i);
 254     mPoolThreads.push_back(
 255         {PR_CreateThread(PR_USER_THREAD, ThreadFuncPoolThread, index,
 256                          PR_PRIORITY_NORMAL, PR_GLOBAL_THREAD,
 257                          PR_JOINABLE_THREAD, sStackSize),
 258          nullptr});
 259   }
 260 }
 261
 262 /* static */
 263 size_t TaskController::GetThreadStackSize() { return sStackSize; }
 264
 265 void TaskController::SetPerformanceCounterState(
 266     PerformanceCounterState* aPerformanceCounterState) {
 267   mPerformanceCounterState = aPerformanceCounterState;
 268 }
 269
 270 /* static */
 271 void TaskController::Shutdown() {
 272   InputTaskManager::Cleanup();
 273   VsyncTaskManager::Cleanup();
 274   if (sSingleton) {
 275     sSingleton->ShutdownThreadPoolInternal();
 276     sSingleton->ShutdownInternal();
 277   }
 278   MOZ_ASSERT(!sSingleton);
 279 }
 280
 281 void TaskController::ShutdownThreadPoolInternal() {
 282   {
 283     // Prevent racecondition on mShuttingDown and wait.
 284     MutexAutoLock lock(mGraphMutex);
 285
 286     mShuttingDown = true;
 287     mThreadPoolCV.NotifyAll();
 288   }
 289   for (PoolThread& thread : mPoolThreads) {
 290     PR_JoinThread(thread.mThread);
 291   }
 292 }
 293
 294 void TaskController::ShutdownInternal() { sSingleton = nullptr; }
 295
 296 void TaskController::RunPoolThread() {
 297   IOInterposer::RegisterCurrentThread();
 298
 299   // This is used to hold on to a task to make sure it is released outside the
 300   // lock. This is required since it's perfectly feasible for task destructors
 301   // to post events themselves.
 302   RefPtr<Task> lastTask;
 303
 304   nsAutoCString threadName;
 305   threadName.AppendLiteral("TaskController #");
 306   threadName.AppendInt(static_cast<int64_t>(mThreadPoolIndex));
 307   AUTO_PROFILER_REGISTER_THREAD(threadName.BeginReading());
 308
 309   MutexAutoLock lock(mGraphMutex);
 310   while (true) {
 311     bool ranTask = false;
 312
 313     if (!mThreadableTasks.empty()) {
 314       for (auto iter = mThreadableTasks.begin(); iter != mThreadableTasks.end();
 315            ++iter) {
 316         // Search for the highest priority dependency of the highest priority
 317         // task.
 318
 319         // We work with rawptrs to avoid needless refcounting. All our tasks
 320         // are always kept alive by the graph. If one is removed from the graph
 321         // it is kept alive by mPoolThreads[mThreadPoolIndex].mCurrentTask.
 322         Task* task = iter->get();
 323
 324         MOZ_ASSERT(!task->mTaskManager);
 325
 326         mPoolThreads[mThreadPoolIndex].mEffectiveTaskPriority =
 327             task->GetPriority();
 328
 329         Task* nextTask;
 330         while ((nextTask = task->GetHighestPriorityDependency())) {
 331           task = nextTask;
 332         }
 333
 334         if (task->GetKind() == Task::Kind::MainThreadOnly ||
 335             task->mInProgress) {
 336           continue;
 337         }
 338
 339         mPoolThreads[mThreadPoolIndex].mCurrentTask = task;
 340         mThreadableTasks.erase(task->mIterator);
 341         task->mIterator = mThreadableTasks.end();
 342         task->mInProgress = true;
 343
 344         if (!mThreadableTasks.empty()) {
 345           // Ensure at least one additional thread is woken up if there are
 346           // more threadable tasks to process. Notifying all threads at once
 347           // isn't actually better for performance since they all need the
 348           // GraphMutex to proceed anyway.
 349           mThreadPoolCV.Notify();
 350         }
 351
 352         bool taskCompleted = false;
 353         {
 354           MutexAutoUnlock unlock(mGraphMutex);
 355           lastTask = nullptr;
 356           AUTO_PROFILE_FOLLOWING_TASK(task);
 357           taskCompleted = task->Run();
 358           ranTask = true;
 359         }
 360
 361         task->mInProgress = false;
 362
 363         if (!taskCompleted) {
 364           // Presumably this task was interrupted, leave its dependencies
 365           // unresolved and reinsert into the queue.
 366           auto insertion = mThreadableTasks.insert(
 367               mPoolThreads[mThreadPoolIndex].mCurrentTask);
 368           MOZ_ASSERT(insertion.second);
 369           task->mIterator = insertion.first;
 370         } else {
 371           task->mCompleted = true;
 372 #ifdef DEBUG
 373           task->mIsInGraph = false;
 374 #endif
 375           task->mDependencies.clear();
 376           // This may have unblocked a main thread task. We could do this only
 377           // if there was a main thread task before this one in the dependency
 378           // chain.
 379           mMayHaveMainThreadTask = true;
 380           // Since this could have multiple dependencies thare are restricted
 381           // to the main thread. Let's make sure that's awake.
 382           EnsureMainThreadTasksScheduled();
 383
 384           MaybeInterruptTask(GetHighestPriorityMTTask());
 385         }
 386
 387         // Store last task for release next time we release the lock or enter
 388         // wait state.
 389         lastTask = mPoolThreads[mThreadPoolIndex].mCurrentTask.forget();
 390         break;
 391       }
 392     }
 393
 394     // Ensure the last task is released before we enter the wait state.
 395     if (lastTask) {
 396       MutexAutoUnlock unlock(mGraphMutex);
 397       lastTask = nullptr;
 398
 399       // Run another loop iteration, while we were unlocked there was an
 400       // opportunity for another task to be posted or shutdown to be initiated.
 401       continue;
 402     }
 403
 404     if (!ranTask) {
 405       if (mShuttingDown) {
 406         IOInterposer::UnregisterCurrentThread();
 407         MOZ_ASSERT(mThreadableTasks.empty());
 408         return;
 409       }
 410
 411       AUTO_PROFILER_LABEL("TaskController::RunPoolThread", IDLE);
 412       mThreadPoolCV.Wait();
 413     }
 414   }
 415 }
 416
 417 void TaskController::AddTask(already_AddRefed<Task>&& aTask) {
 418   RefPtr<Task> task(aTask);
 419
 420   if (task->GetKind() == Task::Kind::OffMainThreadOnly) {
 421     MutexAutoLock lock(mPoolInitializationMutex);
 422     if (!mThreadPoolInitialized) {
 423       InitializeThreadPool();
 424     }
 425   }
 426
 427   MutexAutoLock lock(mGraphMutex);
 428
 429   if (TaskManager* manager = task->GetManager()) {
 430     if (manager->mTaskCount == 0) {
 431       mTaskManagers.insert(manager);
 432     }
 433     manager->DidQueueTask();
 434
 435     // Set this here since if this manager's priority modifier doesn't change
 436     // we will not reprioritize when iterating over the queue.
 437     task->mPriorityModifier = manager->mCurrentPriorityModifier;
 438   }
 439
 440   if (profiler_is_active_and_unpaused()) {
 441     task->mInsertionTime = TimeStamp::Now();
 442   }
 443
 444 #ifdef DEBUG
 445   task->mIsInGraph = true;
 446
 447   for (const RefPtr<Task>& otherTask : task->mDependencies) {
 448     MOZ_ASSERT(!otherTask->mTaskManager ||
 449                otherTask->mTaskManager == task->mTaskManager);
 450   }
 451 #endif
 452
 453   LogTask::LogDispatch(task);
 454
 455   std::pair<std::set<RefPtr<Task>, Task::PriorityCompare>::iterator, bool>
 456       insertion;
 457   switch (task->GetKind()) {
 458     case Task::Kind::MainThreadOnly:
 459       insertion = mMainThreadTasks.insert(std::move(task));
 460       break;
 461     case Task::Kind::OffMainThreadOnly:
 462       insertion = mThreadableTasks.insert(std::move(task));
 463       break;
 464   }
 465   (*insertion.first)->mIterator = insertion.first;
 466   MOZ_ASSERT(insertion.second);
 467
 468   MaybeInterruptTask(*insertion.first);
 469 }
 470
 471 void TaskController::WaitForTaskOrMessage() {
 472   MutexAutoLock lock(mGraphMutex);
 473   while (!mMayHaveMainThreadTask) {
 474     AUTO_PROFILER_LABEL("TaskController::WaitForTaskOrMessage", IDLE);
 475     mMainThreadCV.Wait();
 476   }
 477 }
 478
 479 void TaskController::ExecuteNextTaskOnlyMainThread() {
 480   MOZ_ASSERT(NS_IsMainThread());
 481   MutexAutoLock lock(mGraphMutex);
 482   ExecuteNextTaskOnlyMainThreadInternal(lock);
 483 }
 484
 485 void TaskController::ProcessPendingMTTask(bool aMayWait) {
 486   MOZ_ASSERT(NS_IsMainThread());
 487   MutexAutoLock lock(mGraphMutex);
 488
 489   for (;;) {
 490     // We only ever process one event here. However we may sometimes
 491     // not actually process a real event because of suspended tasks.
 492     // This loop allows us to wait until we've processed something
 493     // in that scenario.
 494
 495     mMTTaskRunnableProcessedTask = ExecuteNextTaskOnlyMainThreadInternal(lock);
 496
 497     if (mMTTaskRunnableProcessedTask || !aMayWait) {
 498       break;
 499     }
 500
 501 #ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
 502     // Unlock before calling into the BackgroundHangMonitor API as it uses
 503     // the timer API.
 504     {
 505       MutexAutoUnlock unlock(mGraphMutex);
 506       BackgroundHangMonitor().NotifyWait();
 507     }
 508 #endif
 509
 510     {
 511       // ProcessNextEvent will also have attempted to wait, however we may have
 512       // given it a Runnable when all the tasks in our task graph were suspended
 513       // but we weren't able to cheaply determine that.
 514       AUTO_PROFILER_LABEL("TaskController::ProcessPendingMTTask", IDLE);
 515       mMainThreadCV.Wait();
 516     }
 517
 518 #ifdef MOZ_ENABLE_BACKGROUND_HANG_MONITOR
 519     {
 520       MutexAutoUnlock unlock(mGraphMutex);
 521       BackgroundHangMonitor().NotifyActivity();
 522     }
 523 #endif
 524   }
 525
 526   if (mMayHaveMainThreadTask) {
 527     EnsureMainThreadTasksScheduled();
 528   }
 529 }
 530
 531 void TaskController::ReprioritizeTask(Task* aTask, uint32_t aPriority) {
 532   MutexAutoLock lock(mGraphMutex);
 533   std::set<RefPtr<Task>, Task::PriorityCompare>* queue = &mMainThreadTasks;
 534   if (aTask->GetKind() == Task::Kind::OffMainThreadOnly) {
 535     queue = &mThreadableTasks;
 536   }
 537
 538   MOZ_ASSERT(aTask->mIterator != queue->end());
 539   queue->erase(aTask->mIterator);
 540
 541   aTask->mPriority = aPriority;
 542
 543   auto insertion = queue->insert(aTask);
 544   MOZ_ASSERT(insertion.second);
 545   aTask->mIterator = insertion.first;
 546
 547   MaybeInterruptTask(aTask);
 548 }
 549
 550 // Code supporting runnable compatibility.
 551 // Task that wraps a runnable.
 552 class RunnableTask : public Task {
 553  public:
 554   RunnableTask(already_AddRefed<nsIRunnable>&& aRunnable, int32_t aPriority,
 555                Kind aKind)
 556       : Task(aKind, aPriority), mRunnable(aRunnable) {}
 557
 558   virtual bool Run() override {
 559     mRunnable->Run();
 560     mRunnable = nullptr;
 561     return true;
 562   }
 563
 564   void SetIdleDeadline(TimeStamp aDeadline) override {
 565     nsCOMPtr<nsIIdleRunnable> idleRunnable = do_QueryInterface(mRunnable);
 566     if (idleRunnable) {
 567       idleRunnable->SetDeadline(aDeadline);
 568     }
 569   }
 570
 571   virtual bool GetName(nsACString& aName) override {
 572 #ifdef MOZ_COLLECTING_RUNNABLE_TELEMETRY
 573     if (nsCOMPtr<nsINamed> named = do_QueryInterface(mRunnable)) {
 574       MOZ_ALWAYS_TRUE(NS_SUCCEEDED(named->GetName(aName)));
 575     } else {
 576       aName.AssignLiteral("non-nsINamed runnable");
 577     }
 578     if (aName.IsEmpty()) {
 579       aName.AssignLiteral("anonymous runnable");
 580     }
 581     return true;
 582 #else
 583     return false;
 584 #endif
 585   }
 586
 587  private:
 588   RefPtr<nsIRunnable> mRunnable;
 589 };
 590
 591 void TaskController::DispatchRunnable(already_AddRefed<nsIRunnable>&& aRunnable,
 592                                       uint32_t aPriority,
 593                                       TaskManager* aManager) {
 594   RefPtr<RunnableTask> task = new RunnableTask(std::move(aRunnable), aPriority,
 595                                                Task::Kind::MainThreadOnly);
 596
 597   task->SetManager(aManager);
 598   TaskController::Get()->AddTask(task.forget());
 599 }
 600
 601 nsIRunnable* TaskController::GetRunnableForMTTask(bool aReallyWait) {
 602   MutexAutoLock lock(mGraphMutex);
 603
 604   while (mMainThreadTasks.empty()) {
 605     if (!aReallyWait) {
 606       return nullptr;
 607     }
 608
 609     AUTO_PROFILER_LABEL("TaskController::GetRunnableForMTTask::Wait", IDLE);
 610     mMainThreadCV.Wait();
 611   }
 612
 613   return aReallyWait ? mMTBlockingProcessingRunnable : mMTProcessingRunnable;
 614 }
 615
 616 bool TaskController::HasMainThreadPendingTasks() {
 617   MOZ_ASSERT(NS_IsMainThread());
 618   auto resetIdleState = MakeScopeExit([&idleManager = mIdleTaskManager] {
 619     if (idleManager) {
 620       idleManager->State().ClearCachedIdleDeadline();
 621     }
 622   });
 623
 624   for (bool considerIdle : {false, true}) {
 625     if (considerIdle && !mIdleTaskManager) {
 626       continue;
 627     }
 628
 629     MutexAutoLock lock(mGraphMutex);
 630
 631     if (considerIdle) {
 632       mIdleTaskManager->State().ForgetPendingTaskGuarantee();
 633       // Temporarily unlock so we can peek our idle deadline.
 634       // XXX We could do this _before_ we take the lock if the API would let us.
 635       // We do want to do this before looking at mMainThreadTasks, in case
 636       // someone adds one while we're unlocked.
 637       {
 638         MutexAutoUnlock unlock(mGraphMutex);
 639         mIdleTaskManager->State().CachePeekedIdleDeadline(unlock);
 640       }
 641     }
 642
 643     // Return early if there's no tasks at all.
 644     if (mMainThreadTasks.empty()) {
 645       return false;
 646     }
 647
 648     // We can cheaply count how many tasks are suspended.
 649     uint64_t totalSuspended = 0;
 650     for (TaskManager* manager : mTaskManagers) {
 651       DebugOnly<bool> modifierChanged =
 652           manager
 653               ->UpdateCachesForCurrentIterationAndReportPriorityModifierChanged(
 654                   lock, TaskManager::IterationType::NOT_EVENT_LOOP_TURN);
 655       MOZ_ASSERT(!modifierChanged);
 656
 657       // The idle manager should be suspended unless we're doing the idle pass.
 658       MOZ_ASSERT(manager != mIdleTaskManager || manager->mCurrentSuspended ||
 659                      considerIdle,
 660                  "Why are idle tasks not suspended here?");
 661
 662       if (manager->mCurrentSuspended) {
 663         // XXX - If managers manage off-main-thread tasks this breaks! This
 664         // scenario is explicitly not supported.
 665         //
 666         // This is only incremented inside the lock -or- decremented on the main
 667         // thread so this is safe.
 668         totalSuspended += manager->mTaskCount;
 669       }
 670     }
 671
 672     // This would break down if we have a non-suspended task depending on a
 673     // suspended task. This is why for the moment we do not allow tasks
 674     // to be dependent on tasks managed by another taskmanager.
 675     if (mMainThreadTasks.size() > totalSuspended) {
 676       // If mIdleTaskManager->mTaskCount is 0, we never updated the suspended
 677       // state of mIdleTaskManager above, hence shouldn't even check it here.
 678       // But in that case idle tasks are not contributing to our suspended task
 679       // count anyway.
 680       if (mIdleTaskManager && mIdleTaskManager->mTaskCount &&
 681           !mIdleTaskManager->mCurrentSuspended) {
 682         MOZ_ASSERT(considerIdle, "Why is mIdleTaskManager not suspended?");
 683         // Check whether the idle tasks were really needed to make our "we have
 684         // an unsuspended task" decision.  If they were, we need to force-enable
 685         // idle tasks until we run our next task.
 686         if (mMainThreadTasks.size() - mIdleTaskManager->mTaskCount <=
 687             totalSuspended) {
 688           mIdleTaskManager->State().EnforcePendingTaskGuarantee();
 689         }
 690       }
 691       return true;
 692     }
 693   }
 694   return false;
 695 }
 696
 697 uint64_t TaskController::PendingMainthreadTaskCountIncludingSuspended() {
 698   MutexAutoLock lock(mGraphMutex);
 699   return mMainThreadTasks.size();
 700 }
 701
 702 bool TaskController::ExecuteNextTaskOnlyMainThreadInternal(
 703     const MutexAutoLock& aProofOfLock) {
 704   MOZ_ASSERT(NS_IsMainThread());
 705   mGraphMutex.AssertCurrentThreadOwns();
 706   // Block to make it easier to jump to our cleanup.
 707   bool taskRan = false;
 708   do {
 709     taskRan = DoExecuteNextTaskOnlyMainThreadInternal(aProofOfLock);
 710     if (taskRan) {
 711       if (mIdleTaskManager && mIdleTaskManager->mTaskCount &&
 712           mIdleTaskManager->IsSuspended(aProofOfLock)) {
 713         uint32_t activeTasks = mMainThreadTasks.size();
 714         for (TaskManager* manager : mTaskManagers) {
 715           if (manager->IsSuspended(aProofOfLock)) {
 716             activeTasks -= manager->mTaskCount;
 717           } else {
 718             break;
 719           }
 720         }
 721
 722         if (!activeTasks) {
 723           // We have only idle (and maybe other suspended) tasks left, so need
 724           // to update the idle state. We need to temporarily release the lock
 725           // while we do that.
 726           MutexAutoUnlock unlock(mGraphMutex);
 727           mIdleTaskManager->State().RequestIdleDeadlineIfNeeded(unlock);
 728         }
 729       }
 730       break;
 731     }
 732
 733     if (!mIdleTaskManager) {
 734       break;
 735     }
 736
 737     if (mIdleTaskManager->mTaskCount) {
 738       // We have idle tasks that we may not have gotten above because
 739       // our idle state is not up to date.  We need to update the idle state
 740       // and try again.  We need to temporarily release the lock while we do
 741       // that.
 742       MutexAutoUnlock unlock(mGraphMutex);
 743       mIdleTaskManager->State().UpdateCachedIdleDeadline(unlock);
 744     } else {
 745       MutexAutoUnlock unlock(mGraphMutex);
 746       mIdleTaskManager->State().RanOutOfTasks(unlock);
 747     }
 748
 749     // When we unlocked, someone may have queued a new task on us.  So try to
 750     // see whether we can run things again.
 751     taskRan = DoExecuteNextTaskOnlyMainThreadInternal(aProofOfLock);
 752   } while (false);
 753
 754   if (mIdleTaskManager) {
 755     // The pending task guarantee is not needed anymore, since we just tried
 756     // running a task
 757     mIdleTaskManager->State().ForgetPendingTaskGuarantee();
 758
 759     if (mMainThreadTasks.empty()) {
 760       ++mRunOutOfMTTasksCounter;
 761
 762       // XXX the IdlePeriodState API demands we have a MutexAutoUnlock for it.
 763       // Otherwise we could perhaps just do this after we exit the locked block,
 764       // by pushing the lock down into this method.  Though it's not clear that
 765       // we could check mMainThreadTasks.size() once we unlock, and whether we
 766       // could maybe substitute mMayHaveMainThreadTask for that check.
 767       MutexAutoUnlock unlock(mGraphMutex);
 768       mIdleTaskManager->State().RanOutOfTasks(unlock);
 769     }
 770   }
 771
 772   return taskRan;
 773 }
 774
 775 bool TaskController::DoExecuteNextTaskOnlyMainThreadInternal(
 776     const MutexAutoLock& aProofOfLock) {
 777   mGraphMutex.AssertCurrentThreadOwns();
 778
 779   nsCOMPtr<nsIThread> mainIThread;
 780   NS_GetMainThread(getter_AddRefs(mainIThread));
 781
 782   nsThread* mainThread = static_cast<nsThread*>(mainIThread.get());
 783   if (mainThread) {
 784     mainThread->SetRunningEventDelay(TimeDuration(), TimeStamp());
 785   }
 786
 787   uint32_t totalSuspended = 0;
 788   for (TaskManager* manager : mTaskManagers) {
 789     bool modifierChanged =
 790         manager
 791             ->UpdateCachesForCurrentIterationAndReportPriorityModifierChanged(
 792                 aProofOfLock, TaskManager::IterationType::EVENT_LOOP_TURN);
 793     if (modifierChanged) {
 794       ProcessUpdatedPriorityModifier(manager);
 795     }
 796     if (manager->mCurrentSuspended) {
 797       totalSuspended += manager->mTaskCount;
 798     }
 799   }
 800
 801   MOZ_ASSERT(mMainThreadTasks.size() >= totalSuspended);
 802
 803   // This would break down if we have a non-suspended task depending on a
 804   // suspended task. This is why for the moment we do not allow tasks
 805   // to be dependent on tasks managed by another taskmanager.
 806   if (mMainThreadTasks.size() > totalSuspended) {
 807     for (auto iter = mMainThreadTasks.begin(); iter != mMainThreadTasks.end();
 808          iter++) {
 809       Task* task = iter->get();
 810
 811       if (task->mTaskManager && task->mTaskManager->mCurrentSuspended) {
 812         // Even though we may want to run some dependencies of this task, we
 813         // will run them at their own priority level and not the priority
 814         // level of their dependents.
 815         continue;
 816       }
 817
 818       task = GetFinalDependency(task);
 819
 820       if (task->GetKind() == Task::Kind::OffMainThreadOnly ||
 821           task->mInProgress ||
 822           (task->mTaskManager && task->mTaskManager->mCurrentSuspended)) {
 823         continue;
 824       }
 825
 826       mCurrentTasksMT.push(task);
 827       mMainThreadTasks.erase(task->mIterator);
 828       task->mIterator = mMainThreadTasks.end();
 829       task->mInProgress = true;
 830       TaskManager* manager = task->GetManager();
 831       bool result = false;
 832
 833       {
 834         MutexAutoUnlock unlock(mGraphMutex);
 835         if (manager) {
 836           manager->WillRunTask();
 837           if (manager != mIdleTaskManager) {
 838             // Notify the idle period state that we're running a non-idle task.
 839             // This needs to happen while our mutex is not locked!
 840             mIdleTaskManager->State().FlagNotIdle();
 841           } else {
 842             TimeStamp idleDeadline =
 843                 mIdleTaskManager->State().GetCachedIdleDeadline();
 844             MOZ_ASSERT(
 845                 idleDeadline,
 846                 "How can we not have a deadline if our manager is enabled?");
 847             task->SetIdleDeadline(idleDeadline);
 848           }
 849         }
 850         if (mIdleTaskManager) {
 851           // We found a task to run; we can clear the idle deadline on our idle
 852           // task manager.  This _must_ be done before we actually run the task,
 853           // because running the task could reenter via spinning the event loop
 854           // and we want to make sure there's no cached idle deadline at that
 855           // point.  But we have to make sure we do it after out SetIdleDeadline
 856           // call above, in the case when the task is actually an idle task.
 857           mIdleTaskManager->State().ClearCachedIdleDeadline();
 858         }
 859
 860         TimeStamp now = TimeStamp::Now();
 861
 862         if (mainThread) {
 863           if (task->GetPriority() < uint32_t(EventQueuePriority::InputHigh) ||
 864               task->mInsertionTime.IsNull()) {
 865             mainThread->SetRunningEventDelay(TimeDuration(), now);
 866           } else {
 867             mainThread->SetRunningEventDelay(now - task->mInsertionTime, now);
 868           }
 869         }
 870
 871         nsAutoCString name;
 872 #ifdef MOZ_COLLECTING_RUNNABLE_TELEMETRY
 873         task->GetName(name);
 874 #endif
 875
 876         PerformanceCounterState::Snapshot snapshot =
 877             mPerformanceCounterState->RunnableWillRun(
 878                 now, manager == mIdleTaskManager);
 879
 880         {
 881           LogTask::Run log(task);
 882 #ifdef MOZ_COLLECTING_RUNNABLE_TELEMETRY
 883           AutoSetMainThreadRunnableName nameGuard(name);
 884 #endif
 885           AUTO_PROFILE_FOLLOWING_TASK(task);
 886           result = task->Run();
 887         }
 888
 889         // Task itself should keep manager alive.
 890         if (manager) {
 891           manager->DidRunTask();
 892         }
 893
 894         mPerformanceCounterState->RunnableDidRun(name, std::move(snapshot));
 895       }
 896
 897       // Task itself should keep manager alive.
 898       if (manager && result && manager->mTaskCount == 0) {
 899         mTaskManagers.erase(manager);
 900       }
 901
 902       task->mInProgress = false;
 903
 904       if (!result) {
 905         // Presumably this task was interrupted, leave its dependencies
 906         // unresolved and reinsert into the queue.
 907         auto insertion =
 908             mMainThreadTasks.insert(std::move(mCurrentTasksMT.top()));
 909         MOZ_ASSERT(insertion.second);
 910         task->mIterator = insertion.first;
 911         manager->WillRunTask();
 912       } else {
 913         task->mCompleted = true;
 914 #ifdef DEBUG
 915         task->mIsInGraph = false;
 916 #endif
 917         // Clear dependencies to release references.
 918         task->mDependencies.clear();
 919
 920         if (!mThreadableTasks.empty()) {
 921           // We're going to wake up a single thread in our pool. This thread
 922           // is responsible for waking up additional threads in the situation
 923           // where more than one task became available.
 924           mThreadPoolCV.Notify();
 925         }
 926       }
 927
 928       mCurrentTasksMT.pop();
 929       return true;
 930     }
 931   }
 932
 933   mMayHaveMainThreadTask = false;
 934   if (mIdleTaskManager) {
 935     // We did not find a task to run.  We still need to clear the cached idle
 936     // deadline on our idle state, because that deadline was only relevant to
 937     // the execution of this function.  Had we found a task, we would have
 938     // cleared the deadline before running that task.
 939     mIdleTaskManager->State().ClearCachedIdleDeadline();
 940   }
 941   return false;
 942 }
 943
 944 Task* TaskController::GetFinalDependency(Task* aTask) {
 945   Task* nextTask;
 946
 947   while ((nextTask = aTask->GetHighestPriorityDependency())) {
 948     aTask = nextTask;
 949   }
 950
 951   return aTask;
 952 }
 953
 954 void TaskController::MaybeInterruptTask(Task* aTask) {
 955   mGraphMutex.AssertCurrentThreadOwns();
 956
 957   if (!aTask) {
 958     return;
 959   }
 960
 961   // This optimization prevents many slow lookups in long chains of similar
 962   // priority.
 963   if (!aTask->mDependencies.empty()) {
 964     Task* firstDependency = aTask->mDependencies.begin()->get();
 965     if (aTask->GetPriority() <= firstDependency->GetPriority() &&
 966         !firstDependency->mCompleted &&
 967         aTask->GetKind() == firstDependency->GetKind()) {
 968       // This task has the same or a higher priority as one of its dependencies,
 969       // never any need to interrupt.
 970       return;
 971     }
 972   }
 973
 974   Task* finalDependency = GetFinalDependency(aTask);
 975
 976   if (finalDependency->mInProgress) {
 977     // No need to wake anything, we can't schedule this task right now anyway.
 978     return;
 979   }
 980
 981   if (aTask->GetKind() == Task::Kind::MainThreadOnly) {
 982     mMayHaveMainThreadTask = true;
 983
 984     EnsureMainThreadTasksScheduled();
 985
 986     if (mCurrentTasksMT.empty()) {
 987       return;
 988     }
 989
 990     // We could go through the steps above here and interrupt an off main
 991     // thread task in case it has a lower priority.
 992     if (finalDependency->GetKind() == Task::Kind::OffMainThreadOnly) {
 993       return;
 994     }
 995
 996     if (mCurrentTasksMT.top()->GetPriority() < aTask->GetPriority()) {
 997       mCurrentTasksMT.top()->RequestInterrupt(aTask->GetPriority());
 998     }
 999   } else {
1000     Task* lowestPriorityTask = nullptr;
1001     for (PoolThread& thread : mPoolThreads) {
1002       if (!thread.mCurrentTask) {
1003         mThreadPoolCV.Notify();
1004         // There's a free thread, no need to interrupt anything.
1005         return;
1006       }
1007
1008       if (!lowestPriorityTask) {
1009         lowestPriorityTask = thread.mCurrentTask.get();
1010         continue;
1011       }
1012
1013       // This should possibly select the lowest priority task which was started
1014       // the latest. But for now we ignore that optimization.
1015       // This also doesn't guarantee a task is interruptable, so that's an
1016       // avenue for improvements as well.
1017       if (lowestPriorityTask->GetPriority() > thread.mEffectiveTaskPriority) {
1018         lowestPriorityTask = thread.mCurrentTask.get();
1019       }
1020     }
1021
1022     if (lowestPriorityTask->GetPriority() < aTask->GetPriority()) {
1023       lowestPriorityTask->RequestInterrupt(aTask->GetPriority());
1024     }
1025
1026     // We choose not to interrupt main thread tasks for tasks which may be
1027     // executed off the main thread.
1028   }
1029 }
1030
1031 Task* TaskController::GetHighestPriorityMTTask() {
1032   mGraphMutex.AssertCurrentThreadOwns();
1033
1034   if (!mMainThreadTasks.empty()) {
1035     return mMainThreadTasks.begin()->get();
1036   }
1037   return nullptr;
1038 }
1039
1040 void TaskController::EnsureMainThreadTasksScheduled() {
1041   if (mObserver) {
1042     mObserver->OnDispatchedEvent();
1043   }
1044   if (mExternalCondVar) {
1045     mExternalCondVar->Notify();
1046   }
1047   mMainThreadCV.Notify();
1048 }
1049
1050 void TaskController::ProcessUpdatedPriorityModifier(TaskManager* aManager) {
1051   mGraphMutex.AssertCurrentThreadOwns();
1052
1053   MOZ_ASSERT(NS_IsMainThread());
1054
1055   int32_t modifier = aManager->mCurrentPriorityModifier;
1056
1057   std::vector<RefPtr<Task>> storedTasks;
1058   // Find all relevant tasks.
1059   for (auto iter = mMainThreadTasks.begin(); iter != mMainThreadTasks.end();) {
1060     if ((*iter)->mTaskManager == aManager) {
1061       storedTasks.push_back(*iter);
1062       iter = mMainThreadTasks.erase(iter);
1063     } else {
1064       iter++;
1065     }
1066   }
1067
1068   // Reinsert found tasks with their new priorities.
1069   for (RefPtr<Task>& ref : storedTasks) {
1070     // Kept alive at first by the vector and then by mMainThreadTasks.
1071     Task* task = ref;
1072     task->mPriorityModifier = modifier;
1073     auto insertion = mMainThreadTasks.insert(std::move(ref));
1074     MOZ_ASSERT(insertion.second);
1075     task->mIterator = insertion.first;
1076   }
1077 }
1078
1079 }  // namespace mozilla