chrome/browser/metrics/thread_watcher_unittest.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 #include <math.h>
   6
   7 #include "base/basictypes.h"
   8 #include "base/bind.h"
   9 #include "base/location.h"
  10 #include "base/logging.h"
  11 #include "base/memory/scoped_ptr.h"
  12 #include "base/message_loop/message_loop.h"
  13 #include "base/run_loop.h"
  14 #include "base/single_thread_task_runner.h"
  15 #include "base/strings/string_number_conversions.h"
  16 #include "base/strings/string_split.h"
  17 #include "base/strings/string_tokenizer.h"
  18 #include "base/synchronization/condition_variable.h"
  19 #include "base/synchronization/lock.h"
  20 #include "base/synchronization/spin_wait.h"
  21 #include "base/synchronization/waitable_event.h"
  22 #include "base/threading/platform_thread.h"
  23 #include "base/time/time.h"
  24 #include "build/build_config.h"
  25 #include "chrome/browser/metrics/thread_watcher.h"
  26 #include "chrome/common/chrome_switches.h"
  27 #include "content/public/test/test_browser_thread.h"
  28 #include "testing/gtest/include/gtest/gtest.h"
  29 #include "testing/platform_test.h"
  30
  31 using base::TimeDelta;
  32 using base::TimeTicks;
  33 using content::BrowserThread;
  34
  35 enum State {
  36   INITIALIZED,        // Created ThreadWatch object.
  37   ACTIVATED,          // Thread watching activated.
  38   SENT_PING,          // Sent ping message to watched thread.
  39   RECEIVED_PONG,      // Received Pong message.
  40   DEACTIVATED,        // Thread watching de-activated.
  41 };
  42
  43 enum WaitState {
  44   UNINITIALIZED,
  45   STARTED_WAITING,    // Start waiting for state_ to change to expected_state.
  46   STOPPED_WAITING,    // Done with the waiting.
  47   ALL_DONE,           // Done with waiting for STOPPED_WAITING.
  48 };
  49
  50 enum CheckResponseState {
  51   UNKNOWN,
  52   SUCCESSFUL,         // CheckResponse was successful.
  53   FAILED,             // CheckResponse has failed.
  54 };
  55
  56 // This class helps to track and manipulate thread state during tests. This
  57 // class also has utility method to simulate hanging of watched thread by making
  58 // the watched thread wait for a very long time by posting a task on watched
  59 // thread that keeps it busy. It also has an utility method to block running of
  60 // tests until ThreadWatcher object's post-condition state changes to an
  61 // expected state.
  62 class CustomThreadWatcher : public ThreadWatcher {
  63  public:
  64   base::Lock custom_lock_;
  65   base::ConditionVariable state_changed_;
  66   State thread_watcher_state_;
  67   WaitState wait_state_;
  68   CheckResponseState check_response_state_;
  69   uint64 ping_sent_;
  70   uint64 pong_received_;
  71   base::subtle::Atomic32 success_response_;
  72   base::subtle::Atomic32 failed_response_;
  73   base::TimeTicks saved_ping_time_;
  74   uint64 saved_ping_sequence_number_;
  75
  76   CustomThreadWatcher(const BrowserThread::ID thread_id,
  77                       const std::string thread_name,
  78                       const TimeDelta& sleep_time,
  79                       const TimeDelta& unresponsive_time)
  80       : ThreadWatcher(WatchingParams(thread_id, thread_name, sleep_time,
  81                       unresponsive_time, ThreadWatcherList::kUnresponsiveCount,
  82                       true, ThreadWatcherList::kLiveThreadsThreshold)),
  83         state_changed_(&custom_lock_),
  84         thread_watcher_state_(INITIALIZED),
  85         wait_state_(UNINITIALIZED),
  86         check_response_state_(UNKNOWN),
  87         ping_sent_(0),
  88         pong_received_(0),
  89         success_response_(0),
  90         failed_response_(0),
  91         saved_ping_time_(base::TimeTicks::Now()),
  92         saved_ping_sequence_number_(0) {
  93   }
  94
  95   State UpdateState(State new_state) {
  96     State old_state;
  97     {
  98       base::AutoLock auto_lock(custom_lock_);
  99       old_state = thread_watcher_state_;
 100       if (old_state != DEACTIVATED)
 101         thread_watcher_state_ = new_state;
 102       if (new_state == SENT_PING)
 103         ++ping_sent_;
 104       if (new_state == RECEIVED_PONG)
 105         ++pong_received_;
 106       saved_ping_time_ = ping_time();
 107       saved_ping_sequence_number_ = ping_sequence_number();
 108     }
 109     state_changed_.Broadcast();
 110     return old_state;
 111   }
 112
 113   WaitState UpdateWaitState(WaitState new_state) {
 114     WaitState old_state;
 115     {
 116       base::AutoLock auto_lock(custom_lock_);
 117       old_state = wait_state_;
 118       wait_state_ = new_state;
 119     }
 120     state_changed_.Broadcast();
 121     return old_state;
 122   }
 123
 124   void ActivateThreadWatching() override {
 125     State old_state = UpdateState(ACTIVATED);
 126     EXPECT_EQ(old_state, INITIALIZED);
 127     ThreadWatcher::ActivateThreadWatching();
 128   }
 129
 130   void DeActivateThreadWatching() override {
 131     State old_state = UpdateState(DEACTIVATED);
 132     EXPECT_TRUE(old_state == ACTIVATED || old_state == SENT_PING ||
 133                 old_state == RECEIVED_PONG);
 134     ThreadWatcher::DeActivateThreadWatching();
 135   }
 136
 137   void PostPingMessage() override {
 138     State old_state = UpdateState(SENT_PING);
 139     EXPECT_TRUE(old_state == ACTIVATED || old_state == RECEIVED_PONG);
 140     ThreadWatcher::PostPingMessage();
 141   }
 142
 143   void OnPongMessage(uint64 ping_sequence_number) override {
 144     State old_state = UpdateState(RECEIVED_PONG);
 145     EXPECT_TRUE(old_state == SENT_PING || old_state == DEACTIVATED);
 146     ThreadWatcher::OnPongMessage(ping_sequence_number);
 147   }
 148
 149   void OnCheckResponsiveness(uint64 ping_sequence_number) override {
 150     ThreadWatcher::OnCheckResponsiveness(ping_sequence_number);
 151     {
 152       base::AutoLock auto_lock(custom_lock_);
 153       if (responsive_) {
 154         base::subtle::Release_Store(&success_response_,
 155             base::subtle::Acquire_Load(&success_response_) + 1);
 156         check_response_state_ = SUCCESSFUL;
 157       } else {
 158         base::subtle::Release_Store(&failed_response_,
 159             base::subtle::Acquire_Load(&failed_response_) + 1);
 160         check_response_state_ = FAILED;
 161       }
 162     }
 163     // Broadcast to indicate we have checked responsiveness of the thread that
 164     // is watched.
 165     state_changed_.Broadcast();
 166   }
 167
 168   void WaitForWaitStateChange(TimeDelta wait_time, WaitState expected_state) {
 169     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 170     TimeTicks end_time = TimeTicks::Now() + wait_time;
 171     {
 172       base::AutoLock auto_lock(custom_lock_);
 173       while (wait_state_ != expected_state && TimeTicks::Now() < end_time)
 174         state_changed_.TimedWait(end_time - TimeTicks::Now());
 175     }
 176   }
 177
 178   void VeryLongMethod(TimeDelta wait_time) {
 179     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 180     WaitForWaitStateChange(wait_time, STOPPED_WAITING);
 181     UpdateWaitState(ALL_DONE);
 182   }
 183
 184   State WaitForStateChange(const TimeDelta& wait_time, State expected_state) {
 185     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 186     UpdateWaitState(STARTED_WAITING);
 187
 188     State exit_state = INITIALIZED;
 189     // Keep the thread that is running the tests waiting until ThreadWatcher
 190     // object's state changes to the expected_state or until wait_time elapses.
 191     for (uint32 i = 0; i < unresponsive_threshold_; ++i) {
 192         TimeTicks end_time = TimeTicks::Now() + wait_time;
 193         {
 194           base::AutoLock auto_lock(custom_lock_);
 195           while (thread_watcher_state_ != expected_state &&
 196                  TimeTicks::Now() < end_time) {
 197             TimeDelta state_change_wait_time = end_time - TimeTicks::Now();
 198             state_changed_.TimedWait(state_change_wait_time);
 199           }
 200           // Capture the thread_watcher_state_ before it changes and return it
 201           // to the caller.
 202           exit_state = thread_watcher_state_;
 203           if (exit_state == expected_state)
 204             break;
 205         }
 206     }
 207     UpdateWaitState(STOPPED_WAITING);
 208     return exit_state;
 209   }
 210
 211   CheckResponseState WaitForCheckResponse(const TimeDelta& wait_time,
 212                                           CheckResponseState expected_state) {
 213     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 214     UpdateWaitState(STARTED_WAITING);
 215
 216     CheckResponseState exit_state = UNKNOWN;
 217     // Keep the thread that is running the tests waiting until ThreadWatcher
 218     // object's check_response_state_ changes to the expected_state or until
 219     // wait_time elapses.
 220     for (uint32 i = 0; i < unresponsive_threshold_; ++i) {
 221         TimeTicks end_time = TimeTicks::Now() + wait_time;
 222         {
 223           base::AutoLock auto_lock(custom_lock_);
 224           while (check_response_state_ != expected_state &&
 225                  TimeTicks::Now() < end_time) {
 226             TimeDelta state_change_wait_time = end_time - TimeTicks::Now();
 227             state_changed_.TimedWait(state_change_wait_time);
 228           }
 229           // Capture the check_response_state_ before it changes and return it
 230           // to the caller.
 231           exit_state = check_response_state_;
 232           if (exit_state == expected_state)
 233             break;
 234         }
 235     }
 236     UpdateWaitState(STOPPED_WAITING);
 237     return exit_state;
 238   }
 239 };
 240
 241 class ThreadWatcherTest : public ::testing::Test {
 242  public:
 243   static const TimeDelta kSleepTime;
 244   static const TimeDelta kUnresponsiveTime;
 245   static const BrowserThread::ID io_thread_id;
 246   static const std::string io_thread_name;
 247   static const BrowserThread::ID db_thread_id;
 248   static const std::string db_thread_name;
 249   static const std::string crash_on_hang_seconds;
 250   static const std::string crash_on_hang_thread_names;
 251   static const std::string thread_names_and_live_threshold;
 252   static const std::string crash_on_hang_thread_data;
 253   CustomThreadWatcher* io_watcher_;
 254   CustomThreadWatcher* db_watcher_;
 255   ThreadWatcherList* thread_watcher_list_;
 256
 257   ThreadWatcherTest()
 258       : setup_complete_(&lock_),
 259         initialized_(false) {
 260     db_thread_.reset(new content::TestBrowserThread(BrowserThread::DB));
 261     io_thread_.reset(new content::TestBrowserThread(BrowserThread::IO));
 262     watchdog_thread_.reset(new WatchDogThread());
 263     db_thread_->StartAndWaitForTesting();
 264     io_thread_->StartAndWaitForTesting();
 265     watchdog_thread_->StartAndWaitForTesting();
 266
 267     WatchDogThread::PostTask(
 268         FROM_HERE,
 269         base::Bind(&ThreadWatcherTest::SetUpObjects, base::Unretained(this)));
 270
 271     WaitForSetUp(TimeDelta::FromMinutes(1));
 272   }
 273
 274   void SetUpObjects() {
 275     DCHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 276
 277     // Setup the registry for thread watchers.
 278     thread_watcher_list_ = new ThreadWatcherList();
 279
 280     // Create thread watcher object for the IO thread.
 281     io_watcher_ = new CustomThreadWatcher(io_thread_id, io_thread_name,
 282                                           kSleepTime, kUnresponsiveTime);
 283     EXPECT_EQ(io_watcher_, thread_watcher_list_->Find(io_thread_id));
 284
 285     // Create thread watcher object for the DB thread.
 286     db_watcher_ = new CustomThreadWatcher(
 287         db_thread_id, db_thread_name, kSleepTime, kUnresponsiveTime);
 288     EXPECT_EQ(db_watcher_, thread_watcher_list_->Find(db_thread_id));
 289
 290     {
 291       base::AutoLock lock(lock_);
 292       initialized_ = true;
 293     }
 294     setup_complete_.Signal();
 295   }
 296
 297   void WaitForSetUp(TimeDelta wait_time) {
 298     DCHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 299     TimeTicks end_time = TimeTicks::Now() + wait_time;
 300     {
 301       base::AutoLock auto_lock(lock_);
 302       while (!initialized_ && TimeTicks::Now() < end_time)
 303         setup_complete_.TimedWait(end_time - TimeTicks::Now());
 304     }
 305   }
 306
 307   ~ThreadWatcherTest() override {
 308     ThreadWatcherList::DeleteAll();
 309     io_watcher_ = nullptr;
 310     db_watcher_ = nullptr;
 311     io_thread_.reset();
 312     db_thread_.reset();
 313     watchdog_thread_.reset();
 314     thread_watcher_list_ = nullptr;
 315   }
 316
 317  private:
 318   base::MessageLoop message_loop_;
 319   base::Lock lock_;
 320   base::ConditionVariable setup_complete_;
 321   bool initialized_;
 322   scoped_ptr<content::TestBrowserThread> db_thread_;
 323   scoped_ptr<content::TestBrowserThread> io_thread_;
 324   scoped_ptr<WatchDogThread> watchdog_thread_;
 325 };
 326
 327 // Define static constants.
 328 const TimeDelta ThreadWatcherTest::kSleepTime =
 329     TimeDelta::FromMilliseconds(50);
 330 const TimeDelta ThreadWatcherTest::kUnresponsiveTime =
 331     TimeDelta::FromMilliseconds(500);
 332 const BrowserThread::ID ThreadWatcherTest::io_thread_id = BrowserThread::IO;
 333 const std::string ThreadWatcherTest::io_thread_name = "IO";
 334 const BrowserThread::ID ThreadWatcherTest::db_thread_id = BrowserThread::DB;
 335 const std::string ThreadWatcherTest::db_thread_name = "DB";
 336 const std::string ThreadWatcherTest::crash_on_hang_thread_names = "UI,IO";
 337 const std::string ThreadWatcherTest::thread_names_and_live_threshold =
 338     "UI:4,IO:4";
 339 const std::string ThreadWatcherTest::crash_on_hang_thread_data =
 340     "UI:5:12,IO:5:12,FILE:5:12";
 341
 342 TEST_F(ThreadWatcherTest, ThreadNamesOnlyArgs) {
 343   // Setup command_line arguments.
 344   base::CommandLine command_line(base::CommandLine::NO_PROGRAM);
 345   command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
 346                                  crash_on_hang_thread_names);
 347
 348   // Parse command_line arguments.
 349   ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
 350   uint32 unresponsive_threshold;
 351   ThreadWatcherList::ParseCommandLine(command_line,
 352                                       &unresponsive_threshold,
 353                                       &crash_on_hang_threads);
 354
 355   // Verify the data.
 356   base::StringTokenizer tokens(crash_on_hang_thread_names, ",");
 357   while (tokens.GetNext()) {
 358     std::vector<base::StringPiece> values = base::SplitStringPiece(
 359         tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
 360     std::string thread_name = values[0].as_string();
 361
 362     ThreadWatcherList::CrashOnHangThreadMap::iterator it =
 363         crash_on_hang_threads.find(thread_name);
 364     bool crash_on_hang = (it != crash_on_hang_threads.end());
 365     EXPECT_TRUE(crash_on_hang);
 366     EXPECT_LT(0u, it->second.live_threads_threshold);
 367     EXPECT_LT(0u, it->second.unresponsive_threshold);
 368   }
 369 }
 370
 371 TEST_F(ThreadWatcherTest, ThreadNamesAndLiveThresholdArgs) {
 372   // Setup command_line arguments.
 373   base::CommandLine command_line(base::CommandLine::NO_PROGRAM);
 374   command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
 375                                  thread_names_and_live_threshold);
 376
 377   // Parse command_line arguments.
 378   ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
 379   uint32 unresponsive_threshold;
 380   ThreadWatcherList::ParseCommandLine(command_line,
 381                                       &unresponsive_threshold,
 382                                       &crash_on_hang_threads);
 383
 384   // Verify the data.
 385   base::StringTokenizer tokens(thread_names_and_live_threshold, ",");
 386   while (tokens.GetNext()) {
 387     std::vector<base::StringPiece> values = base::SplitStringPiece(
 388         tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
 389     std::string thread_name = values[0].as_string();
 390
 391     ThreadWatcherList::CrashOnHangThreadMap::iterator it =
 392         crash_on_hang_threads.find(thread_name);
 393     bool crash_on_hang = (it != crash_on_hang_threads.end());
 394     EXPECT_TRUE(crash_on_hang);
 395     EXPECT_EQ(4u, it->second.live_threads_threshold);
 396     EXPECT_LT(0u, it->second.unresponsive_threshold);
 397   }
 398 }
 399
 400 TEST_F(ThreadWatcherTest, CrashOnHangThreadsAllArgs) {
 401   // Setup command_line arguments.
 402   base::CommandLine command_line(base::CommandLine::NO_PROGRAM);
 403   command_line.AppendSwitchASCII(switches::kCrashOnHangThreads,
 404                                  crash_on_hang_thread_data);
 405
 406   // Parse command_line arguments.
 407   ThreadWatcherList::CrashOnHangThreadMap crash_on_hang_threads;
 408   uint32 unresponsive_threshold;
 409   ThreadWatcherList::ParseCommandLine(command_line,
 410                                       &unresponsive_threshold,
 411                                       &crash_on_hang_threads);
 412
 413   // Verify the data.
 414   base::StringTokenizer tokens(crash_on_hang_thread_data, ",");
 415   while (tokens.GetNext()) {
 416     std::vector<base::StringPiece> values = base::SplitStringPiece(
 417         tokens.token_piece(), ":", base::TRIM_WHITESPACE, base::SPLIT_WANT_ALL);
 418     std::string thread_name = values[0].as_string();
 419
 420     ThreadWatcherList::CrashOnHangThreadMap::iterator it =
 421         crash_on_hang_threads.find(thread_name);
 422
 423     bool crash_on_hang = (it != crash_on_hang_threads.end());
 424     EXPECT_TRUE(crash_on_hang);
 425
 426     uint32 crash_live_threads_threshold = it->second.live_threads_threshold;
 427     EXPECT_EQ(5u, crash_live_threads_threshold);
 428
 429     uint32 crash_unresponsive_threshold = it->second.unresponsive_threshold;
 430     uint32 crash_on_unresponsive_seconds =
 431         ThreadWatcherList::kUnresponsiveSeconds * crash_unresponsive_threshold;
 432     EXPECT_EQ(12u, crash_on_unresponsive_seconds);
 433   }
 434 }
 435
 436 // Test registration. When thread_watcher_list_ goes out of scope after
 437 // TearDown, all thread watcher objects will be deleted.
 438 TEST_F(ThreadWatcherTest, Registration) {
 439   // Check ThreadWatcher object has all correct parameters.
 440   EXPECT_EQ(io_thread_id, io_watcher_->thread_id());
 441   EXPECT_EQ(io_thread_name, io_watcher_->thread_name());
 442   EXPECT_EQ(kSleepTime, io_watcher_->sleep_time());
 443   EXPECT_EQ(kUnresponsiveTime, io_watcher_->unresponsive_time());
 444   EXPECT_FALSE(io_watcher_->active());
 445
 446   // Check ThreadWatcher object of watched DB thread has correct data.
 447   EXPECT_EQ(db_thread_id, db_watcher_->thread_id());
 448   EXPECT_EQ(db_thread_name, db_watcher_->thread_name());
 449   EXPECT_EQ(kSleepTime, db_watcher_->sleep_time());
 450   EXPECT_EQ(kUnresponsiveTime, db_watcher_->unresponsive_time());
 451   EXPECT_FALSE(db_watcher_->active());
 452 }
 453
 454 // Test ActivateThreadWatching and DeActivateThreadWatching of IO thread. This
 455 // method also checks that pong message was sent by the watched thread and pong
 456 // message was received by the WatchDogThread. It also checks that
 457 // OnCheckResponsiveness has verified the ping-pong mechanism and the watched
 458 // thread is not hung.
 459 TEST_F(ThreadWatcherTest, ThreadResponding) {
 460   TimeTicks time_before_ping = TimeTicks::Now();
 461   // Activate watching IO thread.
 462   WatchDogThread::PostTask(
 463       FROM_HERE,
 464       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 465                  base::Unretained(io_watcher_)));
 466
 467   // Activate would have started ping/pong messaging. Expect atleast one
 468   // ping/pong messaging sequence to happen.
 469   io_watcher_->WaitForStateChange(kSleepTime + TimeDelta::FromMinutes(1),
 470                                   RECEIVED_PONG);
 471   EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0));
 472   EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0));
 473   EXPECT_TRUE(io_watcher_->active());
 474   EXPECT_GE(io_watcher_->saved_ping_time_, time_before_ping);
 475   EXPECT_GE(io_watcher_->saved_ping_sequence_number_, static_cast<uint64>(0));
 476
 477   // Verify watched thread is responding with ping/pong messaging.
 478   io_watcher_->WaitForCheckResponse(
 479       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
 480   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
 481       static_cast<base::subtle::Atomic32>(0));
 482   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
 483       static_cast<base::subtle::Atomic32>(0));
 484
 485   // DeActivate thread watching for shutdown.
 486   WatchDogThread::PostTask(
 487       FROM_HERE,
 488       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 489       base::Unretained(io_watcher_)));
 490 }
 491
 492 // This test posts a task on watched thread that takes very long time (this is
 493 // to simulate hanging of watched thread). It then checks for
 494 // OnCheckResponsiveness raising an alert (OnCheckResponsiveness returns false
 495 // if the watched thread is not responding).
 496 TEST_F(ThreadWatcherTest, ThreadNotResponding) {
 497   // Simulate hanging of watched thread by making the watched thread wait for a
 498   // very long time by posting a task on watched thread that keeps it busy.
 499   // It is safe to use base::Unretained because test is waiting for the method
 500   // to finish.
 501   BrowserThread::PostTask(
 502       io_thread_id,
 503       FROM_HERE,
 504       base::Bind(&CustomThreadWatcher::VeryLongMethod,
 505                  base::Unretained(io_watcher_),
 506                  kUnresponsiveTime * 10));
 507
 508   // Activate thread watching.
 509   WatchDogThread::PostTask(
 510       FROM_HERE,
 511       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 512                  base::Unretained(io_watcher_)));
 513
 514   // Verify watched thread is not responding for ping messages.
 515   io_watcher_->WaitForCheckResponse(
 516       kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
 517   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
 518       static_cast<base::subtle::Atomic32>(0));
 519   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
 520       static_cast<base::subtle::Atomic32>(0));
 521
 522   // DeActivate thread watching for shutdown.
 523   WatchDogThread::PostTask(
 524       FROM_HERE,
 525       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 526                  base::Unretained(io_watcher_)));
 527
 528   // Wait for the io_watcher_'s VeryLongMethod to finish.
 529   io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
 530 }
 531
 532 // Test watching of multiple threads with all threads not responding.
 533 TEST_F(ThreadWatcherTest, MultipleThreadsResponding) {
 534   // Check for DB thread to perform ping/pong messaging.
 535   WatchDogThread::PostTask(
 536       FROM_HERE,
 537       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 538                  base::Unretained(db_watcher_)));
 539
 540   // Check for IO thread to perform ping/pong messaging.
 541   WatchDogThread::PostTask(
 542       FROM_HERE,
 543       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 544                  base::Unretained(io_watcher_)));
 545
 546   // Verify DB thread is responding with ping/pong messaging.
 547   db_watcher_->WaitForCheckResponse(
 548       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
 549   EXPECT_GT(db_watcher_->ping_sent_, static_cast<uint64>(0));
 550   EXPECT_GT(db_watcher_->pong_received_, static_cast<uint64>(0));
 551   EXPECT_GE(db_watcher_->ping_sequence_number_, static_cast<uint64>(0));
 552   EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)),
 553       static_cast<base::subtle::Atomic32>(0));
 554   EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)),
 555       static_cast<base::subtle::Atomic32>(0));
 556
 557   // Verify IO thread is responding with ping/pong messaging.
 558   io_watcher_->WaitForCheckResponse(
 559       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
 560   EXPECT_GT(io_watcher_->ping_sent_, static_cast<uint64>(0));
 561   EXPECT_GT(io_watcher_->pong_received_, static_cast<uint64>(0));
 562   EXPECT_GE(io_watcher_->ping_sequence_number_, static_cast<uint64>(0));
 563   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
 564       static_cast<base::subtle::Atomic32>(0));
 565   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
 566       static_cast<base::subtle::Atomic32>(0));
 567
 568   // DeActivate thread watching for shutdown.
 569   WatchDogThread::PostTask(
 570       FROM_HERE,
 571       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 572                  base::Unretained(io_watcher_)));
 573
 574   WatchDogThread::PostTask(
 575       FROM_HERE,
 576       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 577                  base::Unretained(db_watcher_)));
 578 }
 579
 580 // Test watching of multiple threads with one of the threads not responding.
 581 TEST_F(ThreadWatcherTest, MultipleThreadsNotResponding) {
 582   // Simulate hanging of watched thread by making the watched thread wait for a
 583   // very long time by posting a task on watched thread that keeps it busy.
 584   // It is safe ot use base::Unretained because test is waiting for the method
 585   // to finish.
 586   BrowserThread::PostTask(
 587       io_thread_id,
 588       FROM_HERE,
 589       base::Bind(&CustomThreadWatcher::VeryLongMethod,
 590                  base::Unretained(io_watcher_),
 591                  kUnresponsiveTime * 10));
 592
 593   // Activate watching of DB thread.
 594   WatchDogThread::PostTask(
 595       FROM_HERE,
 596       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 597                  base::Unretained(db_watcher_)));
 598
 599   // Activate watching of IO thread.
 600   WatchDogThread::PostTask(
 601       FROM_HERE,
 602       base::Bind(&ThreadWatcher::ActivateThreadWatching,
 603                  base::Unretained(io_watcher_)));
 604
 605   // Verify DB thread is responding with ping/pong messaging.
 606   db_watcher_->WaitForCheckResponse(
 607       kUnresponsiveTime + TimeDelta::FromMinutes(1), SUCCESSFUL);
 608   EXPECT_GT(base::subtle::NoBarrier_Load(&(db_watcher_->success_response_)),
 609       static_cast<base::subtle::Atomic32>(0));
 610   EXPECT_EQ(base::subtle::NoBarrier_Load(&(db_watcher_->failed_response_)),
 611       static_cast<base::subtle::Atomic32>(0));
 612
 613   // Verify IO thread is not responding for ping messages.
 614   io_watcher_->WaitForCheckResponse(
 615       kUnresponsiveTime + TimeDelta::FromMinutes(1), FAILED);
 616   EXPECT_EQ(base::subtle::NoBarrier_Load(&(io_watcher_->success_response_)),
 617       static_cast<base::subtle::Atomic32>(0));
 618   EXPECT_GT(base::subtle::NoBarrier_Load(&(io_watcher_->failed_response_)),
 619       static_cast<base::subtle::Atomic32>(0));
 620
 621   // DeActivate thread watching for shutdown.
 622   WatchDogThread::PostTask(
 623       FROM_HERE,
 624       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 625                  base::Unretained(io_watcher_)));
 626   WatchDogThread::PostTask(
 627       FROM_HERE,
 628       base::Bind(&ThreadWatcher::DeActivateThreadWatching,
 629                  base::Unretained(db_watcher_)));
 630
 631   // Wait for the io_watcher_'s VeryLongMethod to finish.
 632   io_watcher_->WaitForWaitStateChange(kUnresponsiveTime * 10, ALL_DONE);
 633 }
 634
 635 class ThreadWatcherListTest : public ::testing::Test {
 636  protected:
 637   ThreadWatcherListTest()
 638       : done_(&lock_),
 639         state_available_(false),
 640         has_thread_watcher_list_(false),
 641         stopped_(false) {
 642   }
 643
 644   void ReadStateOnWatchDogThread() {
 645     CHECK(WatchDogThread::CurrentlyOnWatchDogThread());
 646     {
 647       base::AutoLock auto_lock(lock_);
 648       has_thread_watcher_list_ =
 649           ThreadWatcherList::g_thread_watcher_list_ != nullptr;
 650       stopped_ = ThreadWatcherList::g_stopped_;
 651       state_available_ = true;
 652     }
 653     done_.Signal();
 654   }
 655
 656   void CheckState(bool has_thread_watcher_list,
 657                   bool stopped,
 658                   const char* const msg) {
 659     CHECK(!WatchDogThread::CurrentlyOnWatchDogThread());
 660     {
 661       base::AutoLock auto_lock(lock_);
 662       state_available_ = false;
 663     }
 664
 665     WatchDogThread::PostTask(
 666         FROM_HERE,
 667         base::Bind(&ThreadWatcherListTest::ReadStateOnWatchDogThread,
 668                    base::Unretained(this)));
 669     {
 670       base::AutoLock auto_lock(lock_);
 671       while (!state_available_)
 672         done_.Wait();
 673
 674       EXPECT_EQ(has_thread_watcher_list, has_thread_watcher_list_) << msg;
 675       EXPECT_EQ(stopped, stopped_) << msg;
 676     }
 677   }
 678
 679   base::Lock lock_;
 680   base::ConditionVariable done_;
 681
 682   bool state_available_;
 683   bool has_thread_watcher_list_;
 684   bool stopped_;
 685 };
 686
 687 TEST_F(ThreadWatcherListTest, Restart) {
 688   ThreadWatcherList::g_initialize_delay_seconds = 1;
 689
 690   base::MessageLoopForUI message_loop_for_ui;
 691   content::TestBrowserThread ui_thread(BrowserThread::UI, &message_loop_for_ui);
 692
 693   scoped_ptr<WatchDogThread> watchdog_thread_(new WatchDogThread());
 694   watchdog_thread_->StartAndWaitForTesting();
 695
 696   // See http://crbug.com/347887.
 697   // StartWatchingAll() will PostDelayedTask to create g_thread_watcher_list_,
 698   // whilst StopWatchingAll() will just PostTask to destroy it.
 699   // Ensure that when Stop is called, Start will NOT create
 700   // g_thread_watcher_list_ later on.
 701   ThreadWatcherList::StartWatchingAll(*base::CommandLine::ForCurrentProcess());
 702   ThreadWatcherList::StopWatchingAll();
 703   message_loop_for_ui.task_runner()->PostDelayedTask(
 704       FROM_HERE, message_loop_for_ui.QuitClosure(),
 705       base::TimeDelta::FromSeconds(
 706           ThreadWatcherList::g_initialize_delay_seconds));
 707   message_loop_for_ui.Run();
 708
 709   CheckState(false /* has_thread_watcher_list */,
 710              true /* stopped */,
 711              "Start / Stopped");
 712
 713   // Proceed with just |StartWatchingAll| and ensure it'll be started.
 714   ThreadWatcherList::StartWatchingAll(*base::CommandLine::ForCurrentProcess());
 715   message_loop_for_ui.task_runner()->PostDelayedTask(
 716       FROM_HERE, message_loop_for_ui.QuitClosure(),
 717       base::TimeDelta::FromSeconds(
 718           ThreadWatcherList::g_initialize_delay_seconds + 1));
 719   message_loop_for_ui.Run();
 720
 721   CheckState(true /* has_thread_watcher_list */,
 722              false /* stopped */,
 723              "Started");
 724
 725   // Finally, StopWatchingAll() must stop.
 726   ThreadWatcherList::StopWatchingAll();
 727   message_loop_for_ui.task_runner()->PostDelayedTask(
 728       FROM_HERE, message_loop_for_ui.QuitClosure(),
 729       base::TimeDelta::FromSeconds(
 730           ThreadWatcherList::g_initialize_delay_seconds));
 731   message_loop_for_ui.Run();
 732
 733   CheckState(false /* has_thread_watcher_list */,
 734              true /* stopped */,
 735              "Stopped");
 736 }
 737
 738 class TestingJankTimeBomb : public JankTimeBomb {
 739  public:
 740   explicit TestingJankTimeBomb(base::TimeDelta duration)
 741       : JankTimeBomb(duration),
 742         thread_id_(base::PlatformThread::CurrentId()),
 743         alarm_invoked_(false) {
 744   }
 745
 746   ~TestingJankTimeBomb() override {}
 747
 748   void Alarm(base::PlatformThreadId thread_id) override {
 749     EXPECT_EQ(thread_id_, thread_id);
 750     alarm_invoked_ = true;
 751   }
 752
 753   bool alarm_invoked() const { return alarm_invoked_; }
 754
 755  private:
 756   const base::PlatformThreadId thread_id_;
 757   bool alarm_invoked_;
 758
 759   DISALLOW_COPY_AND_ASSIGN(TestingJankTimeBomb);
 760 };
 761
 762 class JankTimeBombTest : public ::testing::Test {
 763  public:
 764   JankTimeBombTest() {
 765     watchdog_thread_.reset(new WatchDogThread());
 766     watchdog_thread_->Start();
 767     EXPECT_TRUE(watchdog_thread_->IsRunning());
 768     SPIN_FOR_TIMEDELTA_OR_UNTIL_TRUE(TimeDelta::FromMinutes(1),
 769                                      watchdog_thread_->Started());
 770     WaitForWatchDogThreadPostTask();
 771   }
 772
 773   ~JankTimeBombTest() override {
 774     watchdog_thread_.reset();
 775   }
 776
 777   static void WaitForWatchDogThreadPostTask() {
 778     base::WaitableEvent watchdog_thread_event(false, false);
 779     PostAndWaitForWatchdogThread(&watchdog_thread_event);
 780   }
 781
 782  private:
 783   static void OnJankTimeBombTask(base::WaitableEvent* event) {
 784     event->Signal();
 785   }
 786
 787   static void PostAndWaitForWatchdogThread(base::WaitableEvent* event) {
 788     WatchDogThread::PostDelayedTask(
 789         FROM_HERE,
 790         base::Bind(&JankTimeBombTest::OnJankTimeBombTask, event),
 791         base::TimeDelta::FromSeconds(0));
 792
 793     event->Wait();
 794   }
 795
 796   scoped_ptr<WatchDogThread> watchdog_thread_;
 797
 798   DISALLOW_COPY_AND_ASSIGN(JankTimeBombTest);
 799 };
 800
 801 // JankTimeBomb minimal constructor/destructor test..
 802 TEST_F(JankTimeBombTest, StartShutdownTest) {
 803   // Disarm's itself when it goes out of scope.
 804   TestingJankTimeBomb timebomb1(TimeDelta::FromMinutes(5));
 805   TestingJankTimeBomb timebomb2(TimeDelta::FromMinutes(5));
 806   WaitForWatchDogThreadPostTask();
 807   EXPECT_FALSE(timebomb1.alarm_invoked());
 808   EXPECT_FALSE(timebomb2.alarm_invoked());
 809 }
 810
 811 TEST_F(JankTimeBombTest, ArmTest) {
 812   // Test firing of Alarm by passing empty delay.
 813   TestingJankTimeBomb timebomb((base::TimeDelta()));
 814   if (!timebomb.IsEnabled())
 815     return;
 816   WaitForWatchDogThreadPostTask();
 817   EXPECT_TRUE(timebomb.alarm_invoked());
 818 }