sdk/platform-tools/systrace/catapult/telemetry/telemetry/internal/results/gtest_progress_reporter_unittest.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6 import traceback
   7
   8 from telemetry import story
   9 from telemetry.internal.results import base_test_results_unittest
  10 from telemetry.internal.results import gtest_progress_reporter
  11 from telemetry.internal.results import page_test_results
  12 from telemetry import page as page_module
  13 from telemetry.testing import fakes
  14 from telemetry.testing import stream
  15 from telemetry.value import failure
  16 from telemetry.value import skip
  17
  18
  19 _GROUPING_KEY_DEFAULT = {'1': '2'}
  20
  21
  22 def _MakeStorySet():
  23   story_set = story.StorySet(base_dir=os.path.dirname(__file__))
  24   story_set.AddStory(
  25       page_module.Page('http://www.foo.com/', story_set, story_set.base_dir))
  26   story_set.AddStory(
  27       page_module.Page('http://www.bar.com/', story_set, story_set.base_dir))
  28   story_set.AddStory(
  29       page_module.Page('http://www.baz.com/', story_set, story_set.base_dir))
  30   story_set.AddStory(
  31       page_module.Page('http://www.roz.com/', story_set, story_set.base_dir))
  32   story_set.AddStory(
  33       page_module.Page('http://www.fus.com/', story_set, story_set.base_dir,
  34                        grouping_keys=_GROUPING_KEY_DEFAULT))
  35   story_set.AddStory(
  36       page_module.Page('http://www.ro.com/', story_set, story_set.base_dir,
  37                        grouping_keys=_GROUPING_KEY_DEFAULT))
  38   return story_set
  39
  40
  41 class GTestProgressReporterTest(
  42     base_test_results_unittest.BaseTestResultsUnittest):
  43
  44   def setUp(self):
  45     super(GTestProgressReporterTest, self).setUp()
  46     self._fake_timer = fakes.FakeTimer(gtest_progress_reporter)
  47
  48     self._output_stream = stream.TestOutputStream()
  49     self._reporter = gtest_progress_reporter.GTestProgressReporter(
  50         self._output_stream)
  51
  52   def tearDown(self):
  53     self._fake_timer.Restore()
  54
  55   def testSingleSuccessPage(self):
  56     test_story_set = _MakeStorySet()
  57
  58     results = page_test_results.PageTestResults(
  59         progress_reporter=self._reporter)
  60     results.WillRunPage(test_story_set.stories[0])
  61     self._fake_timer.SetTime(0.007)
  62     results.DidRunPage(test_story_set.stories[0])
  63
  64     results.PrintSummary()
  65     expected = ('[ RUN      ] http://www.foo.com/\n'
  66                 '[       OK ] http://www.foo.com/ (7 ms)\n'
  67                 '[  PASSED  ] 1 test.\n\n')
  68     self.assertEquals(expected, ''.join(self._output_stream.output_data))
  69
  70   def testSingleSuccessPageWithGroupingKeys(self):
  71     test_story_set = _MakeStorySet()
  72
  73     results = page_test_results.PageTestResults(
  74         progress_reporter=self._reporter)
  75     results.WillRunPage(test_story_set.stories[4])
  76     self._fake_timer.SetTime(0.007)
  77     results.DidRunPage(test_story_set.stories[4])
  78
  79     results.PrintSummary()
  80     expected = ("[ RUN      ] http://www.fus.com/@{'1': '2'}\n"
  81                 "[       OK ] http://www.fus.com/@{'1': '2'} (7 ms)\n"
  82                 "[  PASSED  ] 1 test.\n\n")
  83     self.assertEquals(expected, ''.join(self._output_stream.output_data))
  84
  85   def testSingleFailedPage(self):
  86     test_story_set = _MakeStorySet()
  87
  88     results = page_test_results.PageTestResults(
  89         progress_reporter=self._reporter)
  90     results.WillRunPage(test_story_set.stories[0])
  91     exc_info = self.CreateException()
  92     results.AddValue(failure.FailureValue(test_story_set.stories[0], exc_info))
  93     results.DidRunPage(test_story_set.stories[0])
  94
  95     results.PrintSummary()
  96     exception_trace = ''.join(traceback.format_exception(*exc_info))
  97     expected = ('[ RUN      ] http://www.foo.com/\n'
  98                 '%s\n'
  99                 '[  FAILED  ] http://www.foo.com/ (0 ms)\n'
 100                 '[  PASSED  ] 0 tests.\n'
 101                 '[  FAILED  ] 1 test, listed below:\n'
 102                 '[  FAILED  ]  http://www.foo.com/\n\n'
 103                 '1 FAILED TEST\n\n' % exception_trace)
 104     self.assertEquals(expected, ''.join(self._output_stream.output_data))
 105
 106   def testSingleFailedPageWithGroupingKeys(self):
 107     test_story_set = _MakeStorySet()
 108
 109     results = page_test_results.PageTestResults(
 110         progress_reporter=self._reporter)
 111     results.WillRunPage(test_story_set.stories[4])
 112     exc_info = self.CreateException()
 113     results.AddValue(failure.FailureValue(test_story_set.stories[4], exc_info))
 114     results.DidRunPage(test_story_set.stories[4])
 115
 116     results.PrintSummary()
 117     exception_trace = ''.join(traceback.format_exception(*exc_info))
 118     expected = ("[ RUN      ] http://www.fus.com/@{'1': '2'}\n"
 119                 "%s\n"
 120                 "[  FAILED  ] http://www.fus.com/@{'1': '2'} (0 ms)\n"
 121                 "[  PASSED  ] 0 tests.\n"
 122                 "[  FAILED  ] 1 test, listed below:\n"
 123                 "[  FAILED  ]  http://www.fus.com/@{'1': '2'}\n\n"
 124                 "1 FAILED TEST\n\n" % exception_trace)
 125     self.assertEquals(expected, ''.join(self._output_stream.output_data))
 126
 127   def testSingleSkippedPage(self):
 128     test_story_set = _MakeStorySet()
 129     results = page_test_results.PageTestResults(
 130         progress_reporter=self._reporter)
 131     results.WillRunPage(test_story_set.stories[0])
 132     self._fake_timer.SetTime(0.007)
 133     results.AddValue(skip.SkipValue(test_story_set.stories[0],
 134         'Page skipped for testing reason'))
 135     results.DidRunPage(test_story_set.stories[0])
 136
 137     results.PrintSummary()
 138     expected = ('[ RUN      ] http://www.foo.com/\n'
 139                 '===== SKIPPING TEST http://www.foo.com/:'
 140                 ' Page skipped for testing reason =====\n'
 141                 '[       OK ] http://www.foo.com/ (7 ms)\n'
 142                 '[  PASSED  ] 1 test.\n\n')
 143     self.assertEquals(expected, ''.join(self._output_stream.output_data))
 144
 145   def testPassAndFailedPages(self):
 146     test_story_set = _MakeStorySet()
 147     results = page_test_results.PageTestResults(
 148         progress_reporter=self._reporter)
 149     exc_info = self.CreateException()
 150
 151     results.WillRunPage(test_story_set.stories[0])
 152     self._fake_timer.SetTime(0.007)
 153     results.DidRunPage(test_story_set.stories[0])
 154
 155     results.WillRunPage(test_story_set.stories[1])
 156     self._fake_timer.SetTime(0.009)
 157     results.AddValue(failure.FailureValue(test_story_set.stories[1], exc_info))
 158     results.DidRunPage(test_story_set.stories[1])
 159
 160     results.WillRunPage(test_story_set.stories[2])
 161     self._fake_timer.SetTime(0.015)
 162     results.AddValue(failure.FailureValue(test_story_set.stories[2], exc_info))
 163     results.DidRunPage(test_story_set.stories[2])
 164
 165     results.WillRunPage(test_story_set.stories[3])
 166     self._fake_timer.SetTime(0.020)
 167     results.DidRunPage(test_story_set.stories[3])
 168
 169     results.WillRunPage(test_story_set.stories[4])
 170     self._fake_timer.SetTime(0.025)
 171     results.DidRunPage(test_story_set.stories[4])
 172
 173     results.WillRunPage(test_story_set.stories[5])
 174     self._fake_timer.SetTime(0.030)
 175     results.AddValue(failure.FailureValue(test_story_set.stories[5], exc_info))
 176     results.DidRunPage(test_story_set.stories[5])
 177
 178     results.PrintSummary()
 179     exception_trace = ''.join(traceback.format_exception(*exc_info))
 180     expected = ("[ RUN      ] http://www.foo.com/\n"
 181                 "[       OK ] http://www.foo.com/ (7 ms)\n"
 182                 "[ RUN      ] http://www.bar.com/\n"
 183                 "%s\n"
 184                 "[  FAILED  ] http://www.bar.com/ (2 ms)\n"
 185                 "[ RUN      ] http://www.baz.com/\n"
 186                 "%s\n"
 187                 "[  FAILED  ] http://www.baz.com/ (6 ms)\n"
 188                 "[ RUN      ] http://www.roz.com/\n"
 189                 "[       OK ] http://www.roz.com/ (5 ms)\n"
 190                 "[ RUN      ] http://www.fus.com/@{'1': '2'}\n"
 191                 "[       OK ] http://www.fus.com/@{'1': '2'} (5 ms)\n"
 192                 "[ RUN      ] http://www.ro.com/@{'1': '2'}\n"
 193                 "%s\n"
 194                 "[  FAILED  ] http://www.ro.com/@{'1': '2'} (5 ms)\n"
 195                 "[  PASSED  ] 3 tests.\n"
 196                 "[  FAILED  ] 3 tests, listed below:\n"
 197                 "[  FAILED  ]  http://www.bar.com/\n"
 198                 "[  FAILED  ]  http://www.baz.com/\n"
 199                 "[  FAILED  ]  http://www.ro.com/@{'1': '2'}\n\n"
 200                 "3 FAILED TESTS\n\n"
 201                 % (exception_trace, exception_trace, exception_trace))
 202     self.assertEquals(expected, ''.join(self._output_stream.output_data))
 203
 204   def testStreamingResults(self):
 205     test_story_set = _MakeStorySet()
 206     results = page_test_results.PageTestResults(
 207         progress_reporter=self._reporter)
 208     exc_info = self.CreateException()
 209
 210     results.WillRunPage(test_story_set.stories[0])
 211     self._fake_timer.SetTime(0.007)
 212     results.DidRunPage(test_story_set.stories[0])
 213     expected = ('[ RUN      ] http://www.foo.com/\n'
 214                 '[       OK ] http://www.foo.com/ (7 ms)\n')
 215     self.assertEquals(expected, ''.join(self._output_stream.output_data))
 216
 217     results.WillRunPage(test_story_set.stories[1])
 218     self._fake_timer.SetTime(0.009)
 219     exception_trace = ''.join(traceback.format_exception(*exc_info))
 220     results.AddValue(failure.FailureValue(test_story_set.stories[1], exc_info))
 221     results.DidRunPage(test_story_set.stories[1])
 222     expected = ('[ RUN      ] http://www.foo.com/\n'
 223                 '[       OK ] http://www.foo.com/ (7 ms)\n'
 224                 '[ RUN      ] http://www.bar.com/\n'
 225                 '%s\n'
 226                 '[  FAILED  ] http://www.bar.com/ (2 ms)\n' % exception_trace)
 227
 228   def testOutputSkipInformation(self):
 229     test_story_set = _MakeStorySet()
 230     self._reporter = gtest_progress_reporter.GTestProgressReporter(
 231         self._output_stream, output_skipped_tests_summary=True)
 232     results = page_test_results.PageTestResults(
 233         progress_reporter=self._reporter)
 234     results.WillRunPage(test_story_set.stories[0])
 235     self._fake_timer.SetTime(0.007)
 236     results.AddValue(skip.SkipValue(test_story_set.stories[0],
 237         'Page skipped for testing reason'))
 238     results.DidRunPage(test_story_set.stories[0])
 239
 240     results.PrintSummary()
 241     expected = ('[ RUN      ] http://www.foo.com/\n'
 242                 '===== SKIPPING TEST http://www.foo.com/:'
 243                 ' Page skipped for testing reason =====\n'
 244                 '[       OK ] http://www.foo.com/ (7 ms)\n'
 245                 '[  PASSED  ] 1 test.\n'
 246                 '\n'
 247                 'Skipped pages:\n'
 248                 'http://www.foo.com/\n'
 249                 '\n')
 250     self.assertEquals(expected, ''.join(self._output_stream.output_data))