sdk/platform-tools/systrace/catapult/telemetry/telemetry/benchmark_run_unittest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import unittest
   6
   7 from telemetry import benchmark as benchmark_module
   8 from telemetry import page as page_module
   9 from telemetry.page import legacy_page_test
  10 from telemetry import story as story_module
  11 from telemetry.testing import fakes
  12 import mock
  13
  14
  15 # pylint: disable=abstract-method
  16 class DummyPageTest(legacy_page_test.LegacyPageTest):
  17   def __init__(self):
  18     super(DummyPageTest, self).__init__()
  19     # Without disabling the above warning, this complains that
  20     # ValidateAndMeasurePage is abstract; but defining it complains
  21     # that its definition is overridden here.
  22     self.ValidateAndMeasurePage = mock.Mock()
  23
  24
  25 # More end-to-end tests of Benchmark, shared_page_state and associated
  26 # classes using telemetry.testing.fakes, to avoid needing to construct
  27 # a real browser instance.
  28
  29 class FakePage(page_module.Page):
  30   def __init__(self, page_set):
  31     super(FakePage, self).__init__(
  32       url='http://nonexistentserver.com/nonexistentpage.html',
  33       page_set=page_set,
  34       shared_page_state_class=fakes.FakeSharedPageState)
  35     self.RunNavigateSteps = mock.Mock()
  36     self.RunPageInteractions = mock.Mock()
  37
  38 class FakeBenchmark(benchmark_module.Benchmark):
  39   def __init__(self, max_failures=None):
  40     super(FakeBenchmark, self).__init__(max_failures)
  41     self._fake_pages = []
  42     self._fake_story_set = story_module.StorySet()
  43     self._created_story_set = False
  44     self.validator = DummyPageTest()
  45
  46   def CreatePageTest(self, options):
  47     return self.validator
  48
  49   def GetFakeStorySet(self):
  50     return self._fake_story_set
  51
  52   def AddFakePage(self, page):
  53     if self._created_story_set:
  54       raise Exception('Can not add any more fake pages')
  55     self._fake_pages.append(page)
  56
  57   def CreateStorySet(self, options):
  58     if self._created_story_set:
  59       raise Exception('Can only create the story set once per FakeBenchmark')
  60     for page in self._fake_pages:
  61       self._fake_story_set.AddStory(page)
  62     self._created_story_set = True
  63     return self._fake_story_set
  64
  65
  66 class FailingPage(FakePage):
  67   def __init__(self, page_set):
  68     super(FailingPage, self).__init__(page_set)
  69     self.RunNavigateSteps.side_effect = Exception('Deliberate exception')
  70
  71
  72 class BenchmarkRunTest(unittest.TestCase):
  73   def setupBenchmark(self):
  74     finder_options = fakes.CreateBrowserFinderOptions()
  75     finder_options.browser_options.platform = fakes.FakeLinuxPlatform()
  76     finder_options.output_formats = ['none']
  77     finder_options.suppress_gtest_report = True
  78     finder_options.output_dir = None
  79     finder_options.upload_bucket = 'public'
  80     finder_options.upload_results = False
  81     benchmarkclass = FakeBenchmark
  82     parser = finder_options.CreateParser()
  83     benchmark_module.AddCommandLineArgs(parser)
  84     benchmarkclass.AddCommandLineArgs(parser)
  85     options, _ = parser.parse_args([])
  86     benchmark_module.ProcessCommandLineArgs(parser, options)
  87     benchmarkclass.ProcessCommandLineArgs(parser, options)
  88     benchmark = benchmarkclass()
  89     return benchmark, finder_options
  90
  91   def testPassingPage(self):
  92     benchmark, finder_options = self.setupBenchmark()
  93     manager = mock.Mock()
  94     page = FakePage(benchmark.GetFakeStorySet())
  95     page.RunNavigateSteps = manager.page.RunNavigateSteps
  96     page.RunPageInteractions = manager.page.RunPageInteractions
  97     benchmark.validator.ValidateAndMeasurePage = (
  98       manager.validator.ValidateAndMeasurePage)
  99     benchmark.AddFakePage(page)
 100     self.assertEqual(benchmark.Run(finder_options), 0,
 101                      'Test should run with no errors')
 102     expected = [mock.call.page.RunNavigateSteps(mock.ANY),
 103                 mock.call.page.RunPageInteractions(mock.ANY),
 104                 mock.call.validator.ValidateAndMeasurePage(
 105                   page, mock.ANY, mock.ANY)]
 106     self.assertTrue(manager.mock_calls == expected)
 107
 108
 109   def testFailingPage(self):
 110     benchmark, finder_options = self.setupBenchmark()
 111     page = FailingPage(benchmark.GetFakeStorySet())
 112     benchmark.AddFakePage(page)
 113     self.assertNotEqual(benchmark.Run(finder_options), 0, 'Test should fail')
 114     self.assertFalse(page.RunPageInteractions.called)