sdk/platform-tools/systrace/catapult/telemetry/telemetry/internal/story_runner_unittest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import json
   6 import math
   7 import os
   8 import shutil
   9 import StringIO
  10 import sys
  11 import tempfile
  12 import unittest
  13
  14 from py_utils import cloud_storage  # pylint: disable=import-error
  15
  16 from telemetry import benchmark
  17 from telemetry.core import exceptions
  18 from telemetry.core import util
  19 from telemetry import decorators
  20 from telemetry.internal.actions import page_action
  21 from telemetry.internal.results import page_test_results
  22 from telemetry.internal.results import results_options
  23 from telemetry.internal import story_runner
  24 from telemetry.internal.util import exception_formatter as ex_formatter_module
  25 from telemetry.page import page as page_module
  26 from telemetry.page import legacy_page_test
  27 from telemetry import story as story_module
  28 from telemetry.testing import fakes
  29 from telemetry.testing import options_for_unittests
  30 from telemetry.testing import system_stub
  31 import mock
  32 from telemetry.value import failure
  33 from telemetry.value import improvement_direction
  34 from telemetry.value import list_of_scalar_values
  35 from telemetry.value import scalar
  36 from telemetry.value import skip
  37 from telemetry.value import summary as summary_module
  38 from telemetry.web_perf import story_test
  39 from telemetry.web_perf import timeline_based_measurement
  40 from telemetry.wpr import archive_info
  41
  42 # This linter complains if we define classes nested inside functions.
  43 # pylint: disable=bad-super-call
  44
  45 # pylint: disable=too-many-lines
  46
  47 class FakePlatform(object):
  48   def CanMonitorThermalThrottling(self):
  49     return False
  50
  51   def GetOSName(self):
  52     pass
  53
  54   def WaitForTemperature(self, _):
  55     pass
  56
  57   def GetDeviceTypeName(self):
  58     return "GetDeviceTypeName"
  59
  60 class TestSharedState(story_module.SharedState):
  61
  62   _platform = FakePlatform()
  63
  64   @classmethod
  65   def SetTestPlatform(cls, platform):
  66     cls._platform = platform
  67
  68   def __init__(self, test, options, story_set):
  69     super(TestSharedState, self).__init__(
  70         test, options, story_set)
  71     self._test = test
  72     self._current_story = None
  73
  74   @property
  75   def platform(self):
  76     return self._platform
  77
  78   def WillRunStory(self, story):
  79     self._current_story = story
  80
  81   def CanRunStory(self, story):
  82     return True
  83
  84   def RunStory(self, results):
  85     raise NotImplementedError
  86
  87   def DidRunStory(self, results):
  88     pass
  89
  90   def TearDownState(self):
  91     pass
  92
  93   def DumpStateUponFailure(self, story, results):
  94     pass
  95
  96
  97 class TestSharedPageState(TestSharedState):
  98   def RunStory(self, results):
  99     self._test.RunPage(self._current_story, None, results)
 100
 101
 102 class FooStoryState(TestSharedPageState):
 103   pass
 104
 105
 106 class BarStoryState(TestSharedPageState):
 107   pass
 108
 109
 110 class DummyTest(legacy_page_test.LegacyPageTest):
 111   def RunPage(self, *_):
 112     pass
 113
 114   def ValidateAndMeasurePage(self, page, tab, results):
 115     pass
 116
 117
 118 class EmptyMetadataForTest(benchmark.BenchmarkMetadata):
 119   def __init__(self):
 120     super(EmptyMetadataForTest, self).__init__('')
 121
 122
 123 class DummyLocalStory(story_module.Story):
 124   def __init__(self, shared_state_class, name=''):
 125     super(DummyLocalStory, self).__init__(
 126         shared_state_class, name=name)
 127
 128   def Run(self, shared_state):
 129     pass
 130
 131   @property
 132   def is_local(self):
 133     return True
 134
 135   @property
 136   def url(self):
 137     return 'data:,'
 138
 139
 140 class MixedStateStorySet(story_module.StorySet):
 141   @property
 142   def allow_mixed_story_states(self):
 143     return True
 144
 145
 146 def SetupStorySet(allow_multiple_story_states, story_state_list):
 147   if allow_multiple_story_states:
 148     story_set = MixedStateStorySet()
 149   else:
 150     story_set = story_module.StorySet()
 151   for i, story_state in enumerate(story_state_list):
 152     story_set.AddStory(DummyLocalStory(story_state,
 153                                        name='story%d' % i))
 154   return story_set
 155
 156 class FakeBenchmark(benchmark.Benchmark):
 157   @classmethod
 158   def Name(cls):
 159     return 'fake'
 160
 161   test = DummyTest
 162
 163   def page_set(self):
 164     return story_module.StorySet()
 165
 166
 167 def _GetOptionForUnittest():
 168   options = options_for_unittests.GetCopy()
 169   options.output_formats = ['none']
 170   options.suppress_gtest_report = False
 171   parser = options.CreateParser()
 172   story_runner.AddCommandLineArgs(parser)
 173   options.MergeDefaultValues(parser.get_default_values())
 174   story_runner.ProcessCommandLineArgs(parser, options)
 175   return options
 176
 177
 178 class FakeExceptionFormatterModule(object):
 179   @staticmethod
 180   def PrintFormattedException(
 181       exception_class=None, exception=None, tb=None, msg=None):
 182     pass
 183
 184
 185 def GetNumberOfSuccessfulPageRuns(results):
 186   return len([run for run in results.all_page_runs if run.ok or run.skipped])
 187
 188
 189 class TestOnlyException(Exception):
 190   pass
 191
 192
 193 class FailureValueMatcher(object):
 194   def __init__(self, expected_exception_message):
 195     self._expected_exception_message = expected_exception_message
 196
 197   def __eq__(self, other):
 198     return (isinstance(other, failure.FailureValue) and
 199             other.exc_info[1].message == self._expected_exception_message)
 200
 201
 202 class SkipValueMatcher(object):
 203   def __eq__(self, other):
 204     return isinstance(other, skip.SkipValue)
 205
 206
 207 class StoryRunnerTest(unittest.TestCase):
 208   def setUp(self):
 209     self.fake_stdout = StringIO.StringIO()
 210     self.actual_stdout = sys.stdout
 211     sys.stdout = self.fake_stdout
 212     self.options = _GetOptionForUnittest()
 213     self.results = results_options.CreateResults(
 214         EmptyMetadataForTest(), self.options)
 215     self._story_runner_logging_stub = None
 216
 217   def SuppressExceptionFormatting(self):
 218     """Fake out exception formatter to avoid spamming the unittest stdout."""
 219     story_runner.exception_formatter = FakeExceptionFormatterModule
 220     self._story_runner_logging_stub = system_stub.Override(
 221       story_runner, ['logging'])
 222
 223   def RestoreExceptionFormatter(self):
 224     story_runner.exception_formatter = ex_formatter_module
 225     if self._story_runner_logging_stub:
 226       self._story_runner_logging_stub.Restore()
 227       self._story_runner_logging_stub = None
 228
 229   def tearDown(self):
 230     sys.stdout = self.actual_stdout
 231     self.RestoreExceptionFormatter()
 232
 233   def testStoriesGroupedByStateClass(self):
 234     foo_states = [FooStoryState, FooStoryState, FooStoryState,
 235                   FooStoryState, FooStoryState]
 236     mixed_states = [FooStoryState, FooStoryState, FooStoryState,
 237                     BarStoryState, FooStoryState]
 238     # StorySet's are only allowed to have one SharedState.
 239     story_set = SetupStorySet(False, foo_states)
 240     story_groups = (
 241         story_runner.StoriesGroupedByStateClass(
 242             story_set, False))
 243     self.assertEqual(len(story_groups), 1)
 244     story_set = SetupStorySet(False, mixed_states)
 245     self.assertRaises(
 246         ValueError,
 247         story_runner.StoriesGroupedByStateClass,
 248         story_set, False)
 249     # BaseStorySets are allowed to have multiple SharedStates.
 250     mixed_story_set = SetupStorySet(True, mixed_states)
 251     story_groups = (
 252         story_runner.StoriesGroupedByStateClass(
 253             mixed_story_set, True))
 254     self.assertEqual(len(story_groups), 3)
 255     self.assertEqual(story_groups[0].shared_state_class,
 256                      FooStoryState)
 257     self.assertEqual(story_groups[1].shared_state_class,
 258                      BarStoryState)
 259     self.assertEqual(story_groups[2].shared_state_class,
 260                      FooStoryState)
 261
 262   def RunStoryTest(self, s, expected_successes):
 263     test = DummyTest()
 264     story_runner.Run(
 265         test, s, self.options, self.results)
 266     self.assertEquals(0, len(self.results.failures))
 267     self.assertEquals(expected_successes,
 268                       GetNumberOfSuccessfulPageRuns(self.results))
 269
 270   def testRunStoryWithMissingArchiveFile(self):
 271     story_set = story_module.StorySet(archive_data_file='data/hi.json')
 272     story_set.AddStory(page_module.Page(
 273         'http://www.testurl.com', story_set, story_set.base_dir))
 274     test = DummyTest()
 275     self.assertRaises(story_runner.ArchiveError, story_runner.Run, test,
 276                       story_set, self.options, self.results)
 277
 278   def testStoryTest(self):
 279     all_foo = [FooStoryState, FooStoryState, FooStoryState]
 280     one_bar = [FooStoryState, FooStoryState, BarStoryState]
 281     story_set = SetupStorySet(True, one_bar)
 282     self.RunStoryTest(story_set, 3)
 283     story_set = SetupStorySet(True, all_foo)
 284     self.RunStoryTest(story_set, 6)
 285     story_set = SetupStorySet(False, all_foo)
 286     self.RunStoryTest(story_set, 9)
 287     story_set = SetupStorySet(False, one_bar)
 288     test = DummyTest()
 289     self.assertRaises(ValueError, story_runner.Run, test, story_set,
 290                       self.options, self.results)
 291
 292   def testRunStoryWithLongName(self):
 293     story_set = story_module.StorySet()
 294     story_set.AddStory(DummyLocalStory(FooStoryState, name='l' * 182))
 295     test = DummyTest()
 296     self.assertRaises(ValueError, story_runner.Run, test, story_set,
 297                       self.options, self.results)
 298
 299   def testRunStoryWithLongURLPage(self):
 300     story_set = story_module.StorySet()
 301     story_set.AddStory(page_module.Page('file://long' + 'g' * 180, story_set))
 302     test = DummyTest()
 303     self.assertRaises(ValueError, story_runner.Run, test, story_set,
 304                       self.options, self.results)
 305
 306   def testSuccessfulTimelineBasedMeasurementTest(self):
 307     """Check that PageTest is not required for story_runner.Run.
 308
 309     Any PageTest related calls or attributes need to only be called
 310     for PageTest tests.
 311     """
 312     class TestSharedTbmState(TestSharedState):
 313       def RunStory(self, results):
 314         pass
 315
 316     TEST_WILL_RUN_STORY = 'test.WillRunStory'
 317     TEST_MEASURE = 'test.Measure'
 318     TEST_DID_RUN_STORY = 'test.DidRunStory'
 319
 320     EXPECTED_CALLS_IN_ORDER = [TEST_WILL_RUN_STORY,
 321                                TEST_MEASURE,
 322                                TEST_DID_RUN_STORY]
 323
 324     test = timeline_based_measurement.TimelineBasedMeasurement(
 325         timeline_based_measurement.Options())
 326
 327     manager = mock.MagicMock()
 328     test.WillRunStory = mock.MagicMock()
 329     test.Measure = mock.MagicMock()
 330     test.DidRunStory = mock.MagicMock()
 331     manager.attach_mock(test.WillRunStory, TEST_WILL_RUN_STORY)
 332     manager.attach_mock(test.Measure, TEST_MEASURE)
 333     manager.attach_mock(test.DidRunStory, TEST_DID_RUN_STORY)
 334
 335     story_set = story_module.StorySet()
 336     story_set.AddStory(DummyLocalStory(TestSharedTbmState, name='foo'))
 337     story_set.AddStory(DummyLocalStory(TestSharedTbmState, name='bar'))
 338     story_set.AddStory(DummyLocalStory(TestSharedTbmState, name='baz'))
 339     story_runner.Run(
 340         test, story_set, self.options, self.results)
 341     self.assertEquals(0, len(self.results.failures))
 342     self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results))
 343
 344     self.assertEquals(3*EXPECTED_CALLS_IN_ORDER,
 345                       [call[0] for call in manager.mock_calls])
 346
 347   def testCallOrderBetweenStoryTestAndSharedState(self):
 348     """Check that the call order between StoryTest and SharedState is correct.
 349     """
 350     TEST_WILL_RUN_STORY = 'test.WillRunStory'
 351     TEST_MEASURE = 'test.Measure'
 352     TEST_DID_RUN_STORY = 'test.DidRunStory'
 353     STATE_WILL_RUN_STORY = 'state.WillRunStory'
 354     STATE_RUN_STORY = 'state.RunStory'
 355     STATE_DID_RUN_STORY = 'state.DidRunStory'
 356
 357     EXPECTED_CALLS_IN_ORDER = [TEST_WILL_RUN_STORY,
 358                                STATE_WILL_RUN_STORY,
 359                                STATE_RUN_STORY,
 360                                TEST_MEASURE,
 361                                STATE_DID_RUN_STORY,
 362                                TEST_DID_RUN_STORY]
 363
 364     class TestStoryTest(story_test.StoryTest):
 365       def WillRunStory(self, platform):
 366         pass
 367
 368       def Measure(self, platform, results):
 369         pass
 370
 371       def DidRunStory(self, platform):
 372         pass
 373
 374     class TestSharedStateForStoryTest(TestSharedState):
 375       def RunStory(self, results):
 376         pass
 377
 378     @mock.patch.object(TestStoryTest, 'WillRunStory')
 379     @mock.patch.object(TestStoryTest, 'Measure')
 380     @mock.patch.object(TestStoryTest, 'DidRunStory')
 381     @mock.patch.object(TestSharedStateForStoryTest, 'WillRunStory')
 382     @mock.patch.object(TestSharedStateForStoryTest, 'RunStory')
 383     @mock.patch.object(TestSharedStateForStoryTest, 'DidRunStory')
 384     def GetCallsInOrder(state_DidRunStory, state_RunStory, state_WillRunStory,
 385                         test_DidRunStory, test_Measure, test_WillRunStory):
 386       manager = mock.MagicMock()
 387       manager.attach_mock(test_WillRunStory, TEST_WILL_RUN_STORY)
 388       manager.attach_mock(test_Measure, TEST_MEASURE)
 389       manager.attach_mock(test_DidRunStory, TEST_DID_RUN_STORY)
 390       manager.attach_mock(state_WillRunStory, STATE_WILL_RUN_STORY)
 391       manager.attach_mock(state_RunStory, STATE_RUN_STORY)
 392       manager.attach_mock(state_DidRunStory, STATE_DID_RUN_STORY)
 393
 394       test = TestStoryTest()
 395       story_set = story_module.StorySet()
 396       story_set.AddStory(DummyLocalStory(TestSharedStateForStoryTest))
 397       story_runner.Run(test, story_set, self.options, self.results)
 398       return [call[0] for call in manager.mock_calls]
 399
 400     calls_in_order = GetCallsInOrder() # pylint: disable=no-value-for-parameter
 401     self.assertEquals(EXPECTED_CALLS_IN_ORDER, calls_in_order)
 402
 403   def testTearDownStateAfterEachStoryOrStorySetRun(self):
 404     class TestSharedStateForTearDown(TestSharedState):
 405       num_of_tear_downs = 0
 406
 407       def RunStory(self, results):
 408         pass
 409
 410       def TearDownState(self):
 411         TestSharedStateForTearDown.num_of_tear_downs += 1
 412
 413     story_set = story_module.StorySet()
 414     story_set.AddStory(DummyLocalStory(TestSharedStateForTearDown, name='foo'))
 415     story_set.AddStory(DummyLocalStory(TestSharedStateForTearDown, name='bar'))
 416     story_set.AddStory(DummyLocalStory(TestSharedStateForTearDown, name='baz'))
 417
 418     TestSharedStateForTearDown.num_of_tear_downs = 0
 419     story_runner.Run(mock.MagicMock(), story_set, self.options, self.results)
 420     self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 1)
 421
 422     TestSharedStateForTearDown.num_of_tear_downs = 0
 423     story_runner.Run(mock.MagicMock(), story_set, self.options, self.results,
 424                      tear_down_after_story=True)
 425     self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 3)
 426
 427     self.options.pageset_repeat = 5
 428     TestSharedStateForTearDown.num_of_tear_downs = 0
 429     story_runner.Run(mock.MagicMock(), story_set, self.options, self.results,
 430                      tear_down_after_story_set=True)
 431     self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 5)
 432
 433   def testTearDownIsCalledOnceForEachStoryGroupWithPageSetRepeat(self):
 434     self.options.pageset_repeat = 3
 435     fooz_init_call_counter = [0]
 436     fooz_tear_down_call_counter = [0]
 437     barz_init_call_counter = [0]
 438     barz_tear_down_call_counter = [0]
 439     class FoozStoryState(FooStoryState):
 440       def __init__(self, test, options, storyz):
 441         super(FoozStoryState, self).__init__(
 442           test, options, storyz)
 443         fooz_init_call_counter[0] += 1
 444       def TearDownState(self):
 445         fooz_tear_down_call_counter[0] += 1
 446
 447     class BarzStoryState(BarStoryState):
 448       def __init__(self, test, options, storyz):
 449         super(BarzStoryState, self).__init__(
 450           test, options, storyz)
 451         barz_init_call_counter[0] += 1
 452       def TearDownState(self):
 453         barz_tear_down_call_counter[0] += 1
 454     def AssertAndCleanUpFoo():
 455       self.assertEquals(1, fooz_init_call_counter[0])
 456       self.assertEquals(1, fooz_tear_down_call_counter[0])
 457       fooz_init_call_counter[0] = 0
 458       fooz_tear_down_call_counter[0] = 0
 459
 460     story_set1_list = [FoozStoryState, FoozStoryState, FoozStoryState,
 461                        BarzStoryState, BarzStoryState]
 462     story_set1 = SetupStorySet(True, story_set1_list)
 463     self.RunStoryTest(story_set1, 15)
 464     AssertAndCleanUpFoo()
 465     self.assertEquals(1, barz_init_call_counter[0])
 466     self.assertEquals(1, barz_tear_down_call_counter[0])
 467     barz_init_call_counter[0] = 0
 468     barz_tear_down_call_counter[0] = 0
 469
 470     story_set2_list = [FoozStoryState, FoozStoryState, FoozStoryState,
 471                        FoozStoryState]
 472     story_set2 = SetupStorySet(False, story_set2_list)
 473     self.RunStoryTest(story_set2, 27)
 474     AssertAndCleanUpFoo()
 475     self.assertEquals(0, barz_init_call_counter[0])
 476     self.assertEquals(0, barz_tear_down_call_counter[0])
 477
 478   def testAppCrashExceptionCausesFailureValue(self):
 479     self.SuppressExceptionFormatting()
 480     story_set = story_module.StorySet()
 481     class SharedStoryThatCausesAppCrash(TestSharedPageState):
 482       def WillRunStory(self, story):
 483         raise exceptions.AppCrashException(msg='App Foo crashes')
 484
 485     story_set.AddStory(DummyLocalStory(
 486           SharedStoryThatCausesAppCrash))
 487     story_runner.Run(
 488         DummyTest(), story_set, self.options, self.results)
 489     self.assertEquals(1, len(self.results.failures))
 490     self.assertEquals(0, GetNumberOfSuccessfulPageRuns(self.results))
 491     self.assertIn('App Foo crashes', self.fake_stdout.getvalue())
 492
 493   def testExceptionRaisedInSharedStateTearDown(self):
 494     self.SuppressExceptionFormatting()
 495     story_set = story_module.StorySet()
 496     class SharedStoryThatCausesAppCrash(TestSharedPageState):
 497       def TearDownState(self):
 498         raise TestOnlyException()
 499
 500     story_set.AddStory(DummyLocalStory(
 501           SharedStoryThatCausesAppCrash))
 502     with self.assertRaises(TestOnlyException):
 503       story_runner.Run(
 504           DummyTest(), story_set, self.options, self.results)
 505
 506   def testUnknownExceptionIsFatal(self):
 507     self.SuppressExceptionFormatting()
 508     story_set = story_module.StorySet()
 509
 510     class UnknownException(Exception):
 511       pass
 512
 513     # This erroneous test is set up to raise exception for the 2nd story
 514     # run.
 515     class Test(legacy_page_test.LegacyPageTest):
 516       def __init__(self, *args):
 517         super(Test, self).__init__(*args)
 518         self.run_count = 0
 519
 520       def RunPage(self, *_):
 521         old_run_count = self.run_count
 522         self.run_count += 1
 523         if old_run_count == 1:
 524           raise UnknownException('FooBarzException')
 525
 526       def ValidateAndMeasurePage(self, page, tab, results):
 527         pass
 528
 529     s1 = DummyLocalStory(TestSharedPageState, name='foo')
 530     s2 = DummyLocalStory(TestSharedPageState, name='bar')
 531     story_set.AddStory(s1)
 532     story_set.AddStory(s2)
 533     test = Test()
 534     with self.assertRaises(UnknownException):
 535       story_runner.Run(
 536           test, story_set, self.options, self.results)
 537     self.assertEqual(set([s2]), self.results.pages_that_failed)
 538     self.assertEqual(set([s1]), self.results.pages_that_succeeded)
 539     self.assertIn('FooBarzException', self.fake_stdout.getvalue())
 540
 541   def testRaiseBrowserGoneExceptionFromRunPage(self):
 542     self.SuppressExceptionFormatting()
 543     story_set = story_module.StorySet()
 544
 545     class Test(legacy_page_test.LegacyPageTest):
 546       def __init__(self, *args):
 547         super(Test, self).__init__(*args)
 548         self.run_count = 0
 549
 550       def RunPage(self, *_):
 551         old_run_count = self.run_count
 552         self.run_count += 1
 553         if old_run_count == 0:
 554           raise exceptions.BrowserGoneException(
 555               None, 'i am a browser crash message')
 556
 557       def ValidateAndMeasurePage(self, page, tab, results):
 558         pass
 559
 560     story_set.AddStory(DummyLocalStory(TestSharedPageState, name='foo'))
 561     story_set.AddStory(DummyLocalStory(TestSharedPageState, name='bar'))
 562     test = Test()
 563     story_runner.Run(
 564         test, story_set, self.options, self.results)
 565     self.assertEquals(2, test.run_count)
 566     self.assertEquals(1, len(self.results.failures))
 567     self.assertEquals(1, GetNumberOfSuccessfulPageRuns(self.results))
 568
 569   def testAppCrashThenRaiseInTearDownFatal(self):
 570     self.SuppressExceptionFormatting()
 571     story_set = story_module.StorySet()
 572
 573     unit_test_events = []  # track what was called when
 574     class DidRunTestError(Exception):
 575       pass
 576
 577     class TestTearDownSharedState(TestSharedPageState):
 578       def TearDownState(self):
 579         unit_test_events.append('tear-down-state')
 580         raise DidRunTestError
 581
 582       def DumpStateUponFailure(self, story, results):
 583         unit_test_events.append('dump-state')
 584
 585
 586     class Test(legacy_page_test.LegacyPageTest):
 587       def __init__(self, *args):
 588         super(Test, self).__init__(*args)
 589         self.run_count = 0
 590
 591       def RunPage(self, *_):
 592         old_run_count = self.run_count
 593         self.run_count += 1
 594         if old_run_count == 0:
 595           unit_test_events.append('app-crash')
 596           raise exceptions.AppCrashException
 597
 598       def ValidateAndMeasurePage(self, page, tab, results):
 599         pass
 600
 601     story_set.AddStory(DummyLocalStory(TestTearDownSharedState, name='foo'))
 602     story_set.AddStory(DummyLocalStory(TestTearDownSharedState, name='bar'))
 603     test = Test()
 604
 605     with self.assertRaises(DidRunTestError):
 606       story_runner.Run(
 607           test, story_set, self.options, self.results)
 608     self.assertEqual(['app-crash', 'dump-state', 'tear-down-state'],
 609                      unit_test_events)
 610     # The AppCrashException gets added as a failure.
 611     self.assertEquals(1, len(self.results.failures))
 612
 613   def testPagesetRepeat(self):
 614     story_set = story_module.StorySet()
 615
 616     # TODO(eakuefner): Factor this out after flattening page ref in Value
 617     blank_story = DummyLocalStory(TestSharedPageState, name='blank')
 618     green_story = DummyLocalStory(TestSharedPageState, name='green')
 619     story_set.AddStory(blank_story)
 620     story_set.AddStory(green_story)
 621
 622     class Measurement(legacy_page_test.LegacyPageTest):
 623       i = 0
 624       def RunPage(self, page, _, results):
 625         self.i += 1
 626         results.AddValue(scalar.ScalarValue(
 627             page, 'metric', 'unit', self.i,
 628             improvement_direction=improvement_direction.UP))
 629
 630       def ValidateAndMeasurePage(self, page, tab, results):
 631         pass
 632
 633     self.options.pageset_repeat = 2
 634     self.options.output_formats = []
 635     results = results_options.CreateResults(
 636       EmptyMetadataForTest(), self.options)
 637     story_runner.Run(
 638         Measurement(), story_set, self.options, results)
 639     summary = summary_module.Summary(results.all_page_specific_values)
 640     values = summary.interleaved_computed_per_page_values_and_summaries
 641
 642     blank_value = list_of_scalar_values.ListOfScalarValues(
 643         blank_story, 'metric', 'unit', [1, 3],
 644         improvement_direction=improvement_direction.UP)
 645     green_value = list_of_scalar_values.ListOfScalarValues(
 646         green_story, 'metric', 'unit', [2, 4],
 647         improvement_direction=improvement_direction.UP)
 648     merged_value = list_of_scalar_values.ListOfScalarValues(
 649         None, 'metric', 'unit',
 650         [1, 3, 2, 4], std=math.sqrt(2),  # Pooled standard deviation.
 651         improvement_direction=improvement_direction.UP)
 652
 653     self.assertEquals(4, GetNumberOfSuccessfulPageRuns(results))
 654     self.assertEquals(0, len(results.failures))
 655     self.assertEquals(3, len(values))
 656     self.assertIn(blank_value, values)
 657     self.assertIn(green_value, values)
 658     self.assertIn(merged_value, values)
 659
 660   @decorators.Disabled('chromeos')  # crbug.com/483212
 661   def testUpdateAndCheckArchives(self):
 662     usr_stub = system_stub.Override(story_runner, ['cloud_storage'])
 663     wpr_stub = system_stub.Override(archive_info, ['cloud_storage'])
 664     archive_data_dir = os.path.join(
 665         util.GetTelemetryDir(),
 666         'telemetry', 'internal', 'testing', 'archive_files')
 667     try:
 668       story_set = story_module.StorySet()
 669       story_set.AddStory(page_module.Page(
 670           'http://www.testurl.com', story_set, story_set.base_dir))
 671       # Page set missing archive_data_file.
 672       self.assertRaises(
 673           story_runner.ArchiveError,
 674           story_runner._UpdateAndCheckArchives,
 675           story_set.archive_data_file,
 676           story_set.wpr_archive_info,
 677           story_set.stories)
 678
 679       story_set = story_module.StorySet(
 680           archive_data_file='missing_archive_data_file.json')
 681       story_set.AddStory(page_module.Page(
 682           'http://www.testurl.com', story_set, story_set.base_dir))
 683       # Page set missing json file specified in archive_data_file.
 684       self.assertRaises(
 685           story_runner.ArchiveError,
 686           story_runner._UpdateAndCheckArchives,
 687           story_set.archive_data_file,
 688           story_set.wpr_archive_info,
 689           story_set.stories)
 690
 691       story_set = story_module.StorySet(
 692           archive_data_file=os.path.join(archive_data_dir, 'test.json'),
 693           cloud_storage_bucket=cloud_storage.PUBLIC_BUCKET)
 694       story_set.AddStory(page_module.Page(
 695           'http://www.testurl.com', story_set, story_set.base_dir))
 696       # Page set with valid archive_data_file.
 697       self.assertTrue(story_runner._UpdateAndCheckArchives(
 698             story_set.archive_data_file, story_set.wpr_archive_info,
 699             story_set.stories))
 700       story_set.AddStory(page_module.Page(
 701           'http://www.google.com', story_set, story_set.base_dir))
 702       # Page set with an archive_data_file which exists but is missing a page.
 703       self.assertRaises(
 704           story_runner.ArchiveError,
 705           story_runner._UpdateAndCheckArchives,
 706           story_set.archive_data_file,
 707           story_set.wpr_archive_info,
 708           story_set.stories)
 709
 710       story_set = story_module.StorySet(
 711           archive_data_file=
 712               os.path.join(archive_data_dir, 'test_missing_wpr_file.json'),
 713           cloud_storage_bucket=cloud_storage.PUBLIC_BUCKET)
 714       story_set.AddStory(page_module.Page(
 715           'http://www.testurl.com', story_set, story_set.base_dir))
 716       story_set.AddStory(page_module.Page(
 717           'http://www.google.com', story_set, story_set.base_dir))
 718       # Page set with an archive_data_file which exists and contains all pages
 719       # but fails to find a wpr file.
 720       self.assertRaises(
 721           story_runner.ArchiveError,
 722           story_runner._UpdateAndCheckArchives,
 723           story_set.archive_data_file,
 724           story_set.wpr_archive_info,
 725           story_set.stories)
 726     finally:
 727       usr_stub.Restore()
 728       wpr_stub.Restore()
 729
 730
 731   def _testMaxFailuresOptionIsRespectedAndOverridable(
 732       self, num_failing_stories, runner_max_failures, options_max_failures,
 733       expected_num_failures):
 734     class SimpleSharedState(story_module.SharedState):
 735       _fake_platform = FakePlatform()
 736       _current_story = None
 737
 738       @property
 739       def platform(self):
 740         return self._fake_platform
 741
 742       def WillRunStory(self, story):
 743         self._current_story = story
 744
 745       def RunStory(self, results):
 746         self._current_story.Run(self)
 747
 748       def DidRunStory(self, results):
 749         pass
 750
 751       def CanRunStory(self, story):
 752         return True
 753
 754       def TearDownState(self):
 755         pass
 756
 757       def DumpStateUponFailure(self, story, results):
 758         pass
 759
 760     class FailingStory(story_module.Story):
 761       def __init__(self, name):
 762         super(FailingStory, self).__init__(
 763             shared_state_class=SimpleSharedState,
 764             is_local=True, name=name)
 765         self.was_run = False
 766
 767       def Run(self, shared_state):
 768         self.was_run = True
 769         raise legacy_page_test.Failure
 770
 771       @property
 772       def url(self):
 773         return 'data:,'
 774
 775     self.SuppressExceptionFormatting()
 776
 777     story_set = story_module.StorySet()
 778     for i in range(num_failing_stories):
 779       story_set.AddStory(FailingStory(name='failing%d' % i))
 780
 781     options = _GetOptionForUnittest()
 782     options.output_formats = ['none']
 783     options.suppress_gtest_report = True
 784     if options_max_failures:
 785       options.max_failures = options_max_failures
 786
 787     results = results_options.CreateResults(EmptyMetadataForTest(), options)
 788     story_runner.Run(
 789         DummyTest(), story_set, options,
 790         results, max_failures=runner_max_failures)
 791     self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results))
 792     self.assertEquals(expected_num_failures, len(results.failures))
 793     for ii, story in enumerate(story_set.stories):
 794       self.assertEqual(story.was_run, ii < expected_num_failures)
 795
 796   def testMaxFailuresNotSpecified(self):
 797     self._testMaxFailuresOptionIsRespectedAndOverridable(
 798         num_failing_stories=5, runner_max_failures=None,
 799         options_max_failures=None, expected_num_failures=5)
 800
 801   def testMaxFailuresSpecifiedToRun(self):
 802     # Runs up to max_failures+1 failing tests before stopping, since
 803     # every tests after max_failures failures have been encountered
 804     # may all be passing.
 805     self._testMaxFailuresOptionIsRespectedAndOverridable(
 806         num_failing_stories=5, runner_max_failures=3,
 807         options_max_failures=None, expected_num_failures=4)
 808
 809   def testMaxFailuresOption(self):
 810     # Runs up to max_failures+1 failing tests before stopping, since
 811     # every tests after max_failures failures have been encountered
 812     # may all be passing.
 813     self._testMaxFailuresOptionIsRespectedAndOverridable(
 814         num_failing_stories=5, runner_max_failures=3,
 815         options_max_failures=1, expected_num_failures=2)
 816
 817   def _CreateErrorProcessingMock(self, method_exceptions=None,
 818                                  legacy_test=False):
 819     if legacy_test:
 820       test_class = legacy_page_test.LegacyPageTest
 821     else:
 822       test_class = story_test.StoryTest
 823
 824     root_mock = mock.NonCallableMock(
 825         story=mock.NonCallableMagicMock(story_module.Story),
 826         results=mock.NonCallableMagicMock(page_test_results.PageTestResults),
 827         test=mock.NonCallableMagicMock(test_class),
 828         state=mock.NonCallableMagicMock(
 829             story_module.SharedState,
 830             CanRunStory=mock.Mock(return_value=True)))
 831
 832     if method_exceptions:
 833       root_mock.configure_mock(**{
 834           path + '.side_effect': exception
 835           for path, exception in method_exceptions.iteritems()})
 836
 837     return root_mock
 838
 839   def testRunStoryAndProcessErrorIfNeeded_success(self):
 840     root_mock = self._CreateErrorProcessingMock()
 841
 842     story_runner._RunStoryAndProcessErrorIfNeeded(
 843         root_mock.story, root_mock.results, root_mock.state, root_mock.test)
 844
 845     self.assertEquals(root_mock.method_calls, [
 846       mock.call.state.platform.GetOSName(),
 847       mock.call.test.WillRunStory(root_mock.state.platform),
 848       mock.call.state.WillRunStory(root_mock.story),
 849       mock.call.state.CanRunStory(root_mock.story),
 850       mock.call.state.RunStory(root_mock.results),
 851       mock.call.test.Measure(root_mock.state.platform, root_mock.results),
 852       mock.call.state.DidRunStory(root_mock.results),
 853       mock.call.test.DidRunStory(root_mock.state.platform),
 854       mock.call.state.platform.GetOSName(),
 855     ])
 856
 857   def testRunStoryAndProcessErrorIfNeeded_successLegacy(self):
 858     root_mock = self._CreateErrorProcessingMock(legacy_test=True)
 859
 860     story_runner._RunStoryAndProcessErrorIfNeeded(
 861         root_mock.story, root_mock.results, root_mock.state, root_mock.test)
 862
 863     self.assertEquals(root_mock.method_calls, [
 864       mock.call.state.platform.GetOSName(),
 865       mock.call.state.WillRunStory(root_mock.story),
 866       mock.call.state.CanRunStory(root_mock.story),
 867       mock.call.state.RunStory(root_mock.results),
 868       mock.call.state.DidRunStory(root_mock.results),
 869       mock.call.test.DidRunPage(root_mock.state.platform),
 870       mock.call.state.platform.GetOSName(),
 871     ])
 872
 873   def testRunStoryAndProcessErrorIfNeeded_tryTimeout(self):
 874     root_mock = self._CreateErrorProcessingMock(method_exceptions={
 875       'state.WillRunStory': exceptions.TimeoutException('foo')
 876     })
 877
 878     story_runner._RunStoryAndProcessErrorIfNeeded(
 879         root_mock.story, root_mock.results, root_mock.state, root_mock.test)
 880
 881     self.assertEquals(root_mock.method_calls, [
 882       mock.call.state.platform.GetOSName(),
 883       mock.call.test.WillRunStory(root_mock.state.platform),
 884       mock.call.state.WillRunStory(root_mock.story),
 885       mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
 886       mock.call.results.AddValue(FailureValueMatcher('foo')),
 887       mock.call.state.DidRunStory(root_mock.results),
 888       mock.call.test.DidRunStory(root_mock.state.platform),
 889       mock.call.state.platform.GetOSName(),
 890     ])
 891
 892   def testRunStoryAndProcessErrorIfNeeded_tryError(self):
 893     root_mock = self._CreateErrorProcessingMock(method_exceptions={
 894       'state.CanRunStory': exceptions.Error('foo')
 895     })
 896
 897     with self.assertRaisesRegexp(exceptions.Error, 'foo'):
 898       story_runner._RunStoryAndProcessErrorIfNeeded(
 899           root_mock.story, root_mock.results, root_mock.state, root_mock.test)
 900
 901     self.assertEquals(root_mock.method_calls, [
 902       mock.call.state.platform.GetOSName(),
 903       mock.call.test.WillRunStory(root_mock.state.platform),
 904       mock.call.state.WillRunStory(root_mock.story),
 905       mock.call.state.CanRunStory(root_mock.story),
 906       mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
 907       mock.call.results.AddValue(FailureValueMatcher('foo')),
 908       mock.call.state.DidRunStory(root_mock.results),
 909       mock.call.test.DidRunStory(root_mock.state.platform),
 910       mock.call.state.platform.GetOSName(),
 911     ])
 912
 913   def testRunStoryAndProcessErrorIfNeeded_tryUnsupportedAction(self):
 914     root_mock = self._CreateErrorProcessingMock(method_exceptions={
 915       'state.RunStory': page_action.PageActionNotSupported('foo')
 916     })
 917
 918     story_runner._RunStoryAndProcessErrorIfNeeded(
 919         root_mock.story, root_mock.results, root_mock.state, root_mock.test)
 920
 921     self.assertEquals(root_mock.method_calls, [
 922       mock.call.state.platform.GetOSName(),
 923       mock.call.test.WillRunStory(root_mock.state.platform),
 924       mock.call.state.WillRunStory(root_mock.story),
 925       mock.call.state.CanRunStory(root_mock.story),
 926       mock.call.state.RunStory(root_mock.results),
 927       mock.call.results.AddValue(SkipValueMatcher()),
 928       mock.call.state.DidRunStory(root_mock.results),
 929       mock.call.test.DidRunStory(root_mock.state.platform),
 930       mock.call.state.platform.GetOSName(),
 931     ])
 932
 933   def testRunStoryAndProcessErrorIfNeeded_tryUnhandlable(self):
 934     root_mock = self._CreateErrorProcessingMock(method_exceptions={
 935       'test.WillRunStory': Exception('foo')
 936     })
 937
 938     with self.assertRaisesRegexp(Exception, 'foo'):
 939       story_runner._RunStoryAndProcessErrorIfNeeded(
 940           root_mock.story, root_mock.results, root_mock.state, root_mock.test)
 941
 942     self.assertEquals(root_mock.method_calls, [
 943       mock.call.state.platform.GetOSName(),
 944       mock.call.test.WillRunStory(root_mock.state.platform),
 945       mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
 946       mock.call.results.AddValue(FailureValueMatcher('foo')),
 947       mock.call.state.DidRunStory(root_mock.results),
 948       mock.call.test.DidRunStory(root_mock.state.platform),
 949       mock.call.state.platform.GetOSName(),
 950     ])
 951
 952   def testRunStoryAndProcessErrorIfNeeded_finallyException(self):
 953     root_mock = self._CreateErrorProcessingMock(method_exceptions={
 954       'state.DidRunStory': Exception('bar')
 955     })
 956
 957     with self.assertRaisesRegexp(Exception, 'bar'):
 958       story_runner._RunStoryAndProcessErrorIfNeeded(
 959           root_mock.story, root_mock.results, root_mock.state, root_mock.test)
 960
 961     self.assertEquals(root_mock.method_calls, [
 962       mock.call.state.platform.GetOSName(),
 963       mock.call.test.WillRunStory(root_mock.state.platform),
 964       mock.call.state.WillRunStory(root_mock.story),
 965       mock.call.state.CanRunStory(root_mock.story),
 966       mock.call.state.RunStory(root_mock.results),
 967       mock.call.test.Measure(root_mock.state.platform, root_mock.results),
 968       mock.call.state.DidRunStory(root_mock.results),
 969       mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results)
 970     ])
 971
 972   def testRunStoryAndProcessErrorIfNeeded_tryTimeout_finallyException(self):
 973     root_mock = self._CreateErrorProcessingMock(method_exceptions={
 974       'state.RunStory': exceptions.TimeoutException('foo'),
 975       'state.DidRunStory': Exception('bar')
 976     })
 977
 978     story_runner._RunStoryAndProcessErrorIfNeeded(
 979         root_mock.story, root_mock.results, root_mock.state, root_mock.test)
 980
 981     self.assertEquals(root_mock.method_calls, [
 982       mock.call.state.platform.GetOSName(),
 983       mock.call.test.WillRunStory(root_mock.state.platform),
 984       mock.call.state.WillRunStory(root_mock.story),
 985       mock.call.state.CanRunStory(root_mock.story),
 986       mock.call.state.RunStory(root_mock.results),
 987       mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
 988       mock.call.results.AddValue(FailureValueMatcher('foo')),
 989       mock.call.state.DidRunStory(root_mock.results)
 990     ])
 991
 992   def testRunStoryAndProcessErrorIfNeeded_tryError_finallyException(self):
 993     root_mock = self._CreateErrorProcessingMock(method_exceptions={
 994       'state.WillRunStory': exceptions.Error('foo'),
 995       'test.DidRunStory': Exception('bar')
 996     })
 997
 998     with self.assertRaisesRegexp(exceptions.Error, 'foo'):
 999       story_runner._RunStoryAndProcessErrorIfNeeded(
1000           root_mock.story, root_mock.results, root_mock.state, root_mock.test)
1001
1002     self.assertEquals(root_mock.method_calls, [
1003       mock.call.state.platform.GetOSName(),
1004       mock.call.test.WillRunStory(root_mock.state.platform),
1005       mock.call.state.WillRunStory(root_mock.story),
1006       mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
1007       mock.call.results.AddValue(FailureValueMatcher('foo')),
1008       mock.call.state.DidRunStory(root_mock.results),
1009       mock.call.test.DidRunStory(root_mock.state.platform)
1010     ])
1011
1012   def testRunStoryAndProcessErrorIfNeeded_tryUnsupportedAction_finallyException(
1013       self):
1014     root_mock = self._CreateErrorProcessingMock(method_exceptions={
1015       'test.WillRunStory': page_action.PageActionNotSupported('foo'),
1016       'state.DidRunStory': Exception('bar')
1017     })
1018
1019     story_runner._RunStoryAndProcessErrorIfNeeded(
1020         root_mock.story, root_mock.results, root_mock.state, root_mock.test)
1021
1022     self.assertEquals(root_mock.method_calls, [
1023       mock.call.state.platform.GetOSName(),
1024       mock.call.test.WillRunStory(root_mock.state.platform),
1025       mock.call.results.AddValue(SkipValueMatcher()),
1026       mock.call.state.DidRunStory(root_mock.results)
1027     ])
1028
1029   def testRunStoryAndProcessErrorIfNeeded_tryUnhandlable_finallyException(self):
1030     root_mock = self._CreateErrorProcessingMock(method_exceptions={
1031       'test.Measure': Exception('foo'),
1032       'test.DidRunStory': Exception('bar')
1033     })
1034
1035     with self.assertRaisesRegexp(Exception, 'foo'):
1036       story_runner._RunStoryAndProcessErrorIfNeeded(
1037           root_mock.story, root_mock.results, root_mock.state, root_mock.test)
1038
1039     self.assertEquals(root_mock.method_calls, [
1040       mock.call.state.platform.GetOSName(),
1041       mock.call.test.WillRunStory(root_mock.state.platform),
1042       mock.call.state.WillRunStory(root_mock.story),
1043       mock.call.state.CanRunStory(root_mock.story),
1044       mock.call.state.RunStory(root_mock.results),
1045       mock.call.test.Measure(root_mock.state.platform, root_mock.results),
1046       mock.call.state.DumpStateUponFailure(root_mock.story, root_mock.results),
1047       mock.call.results.AddValue(FailureValueMatcher('foo')),
1048       mock.call.state.DidRunStory(root_mock.results),
1049       mock.call.test.DidRunStory(root_mock.state.platform)
1050     ])
1051
1052   def testRunBenchmarkTimeDuration(self):
1053     fake_benchmark = FakeBenchmark()
1054     options = fakes.CreateBrowserFinderOptions()
1055     options.upload_results = None
1056     options.suppress_gtest_report = False
1057     options.results_label = None
1058     options.use_live_sites = False
1059     options.max_failures = 100
1060     options.pageset_repeat = 1
1061     options.output_formats = ['chartjson']
1062
1063     with mock.patch('telemetry.internal.story_runner.time.time') as time_patch:
1064       # 3, because telemetry code asks for the time at some point
1065       time_patch.side_effect = [1, 0, 61]
1066       tmp_path = tempfile.mkdtemp()
1067
1068       try:
1069         options.output_dir = tmp_path
1070         story_runner.RunBenchmark(fake_benchmark, options)
1071         with open(os.path.join(tmp_path, 'results-chart.json')) as f:
1072           data = json.load(f)
1073
1074         self.assertEqual(len(data['charts']), 1)
1075         charts = data['charts']
1076         self.assertIn('BenchmarkDuration', charts)
1077         duration = charts['BenchmarkDuration']
1078         self.assertIn("summary", duration)
1079         summary = duration['summary']
1080         duration = summary['value']
1081         self.assertAlmostEqual(duration, 1)
1082       finally:
1083         shutil.rmtree(tmp_path)