sdk/platform-tools/systrace/catapult/telemetry/telemetry/testing/browser_test_runner_unittest.py

   1 # Copyright 2016 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 import os
   6 import string
   7 import sys
   8 import tempfile
   9 import unittest
  10 import json
  11
  12 from telemetry import decorators
  13 from telemetry import project_config
  14 from telemetry.core import util
  15 from telemetry.testing import browser_test_context
  16 from telemetry.testing import browser_test_runner
  17 from telemetry.testing import options_for_unittests
  18 from telemetry.testing import run_browser_tests
  19 from telemetry.testing import serially_executed_browser_test_case
  20
  21
  22 class BrowserTestRunnerTest(unittest.TestCase):
  23
  24   def _ExtractTestResults(self, test_result):
  25     delimiter = test_result['path_delimiter']
  26     failures = []
  27     successes = []
  28     def _IsLeafNode(node):
  29       test_dict = node[1]
  30       return ('expected' in test_dict and
  31               isinstance(test_dict['expected'], basestring))
  32     node_queues = []
  33     for t in test_result['tests']:
  34       node_queues.append((t, test_result['tests'][t]))
  35     while node_queues:
  36       node = node_queues.pop()
  37       full_test_name, test_dict = node
  38       if _IsLeafNode(node):
  39         if all(res not in test_dict['expected'].split() for res in
  40                test_dict['actual'].split()):
  41           failures.append(full_test_name)
  42         else:
  43           successes.append(full_test_name)
  44       else:
  45         for k in test_dict:
  46           node_queues.append(
  47             ('%s%s%s' % (full_test_name, delimiter, k),
  48              test_dict[k]))
  49     return successes, failures
  50
  51   def baseTest(self, test_filter,
  52                failures, successes, test_name='SimpleTest'):
  53     config = project_config.ProjectConfig(
  54         top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'),
  55         client_configs=[],
  56         benchmark_dirs=[
  57             os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests')]
  58     )
  59     temp_file = tempfile.NamedTemporaryFile(delete=False)
  60     temp_file.close()
  61     temp_file_name = temp_file.name
  62     try:
  63       browser_test_runner.Run(
  64           config,
  65           [test_name,
  66            '--write-full-results-to=%s' % temp_file_name,
  67            '--test-filter=%s' % test_filter])
  68       with open(temp_file_name) as f:
  69         test_result = json.load(f)
  70
  71       actual_successes, actual_failures = self._ExtractTestResults(test_result)
  72       self.assertEquals(set(actual_failures), set(failures))
  73       self.assertEquals(set(actual_successes), set(successes))
  74     finally:
  75       os.remove(temp_file_name)
  76
  77   @decorators.Disabled('chromeos')  # crbug.com/696553
  78   def testJsonOutputFormatNegativeFilter(self):
  79     self.baseTest(
  80       '^(add|multiplier).*',
  81       ['browser_tests.simple_numeric_test.SimpleTest.add_1_and_2',
  82        'browser_tests.simple_numeric_test.SimpleTest.add_7_and_3',
  83        'browser_tests.simple_numeric_test.SimpleTest.multiplier_simple_2'],
  84       ['browser_tests.simple_numeric_test.SimpleTest.add_2_and_3',
  85        'browser_tests.simple_numeric_test.SimpleTest.multiplier_simple',
  86        'browser_tests.simple_numeric_test.SimpleTest.multiplier_simple_3'])
  87
  88   @decorators.Disabled('chromeos')  # crbug.com/696553
  89   def testJsonOutputWhenSetupClassFailed(self):
  90     self.baseTest(
  91       '.*',
  92       ['browser_tests.failed_tests.SetUpClassFailedTest.dummy_test_0',
  93        'browser_tests.failed_tests.SetUpClassFailedTest.dummy_test_1',
  94        'browser_tests.failed_tests.SetUpClassFailedTest.dummy_test_2'],
  95       [], test_name='SetUpClassFailedTest')
  96
  97   @decorators.Disabled('chromeos')  # crbug.com/696553
  98   def testJsonOutputWhenTearDownClassFailed(self):
  99     self.baseTest(
 100       '.*',
 101       ['browser_tests.failed_tests.TearDownClassFailedTest.dummy_test_0',
 102        'browser_tests.failed_tests.TearDownClassFailedTest.dummy_test_1',
 103        'browser_tests.failed_tests.TearDownClassFailedTest.dummy_test_2'],
 104       [], test_name='TearDownClassFailedTest')
 105
 106   @decorators.Disabled('chromeos')  # crbug.com/696553
 107   def testSetUpProcessCalledOnce(self):
 108     self.baseTest(
 109       '.*',
 110       [],
 111       ['browser_tests.process_tests.FailIfSetUpProcessCalledTwice.Dummy_0',
 112        'browser_tests.process_tests.FailIfSetUpProcessCalledTwice.Dummy_1',
 113        'browser_tests.process_tests.FailIfSetUpProcessCalledTwice.Dummy_2'],
 114       test_name='FailIfSetUpProcessCalledTwice')
 115
 116   @decorators.Disabled('chromeos')  # crbug.com/696553
 117   def testTearDownProcessCalledOnce(self):
 118     self.baseTest(
 119       '.*',
 120       [],
 121       ['browser_tests.process_tests.FailIfTearDownProcessCalledTwice.Dummy_0',
 122        'browser_tests.process_tests.FailIfTearDownProcessCalledTwice.Dummy_1',
 123        'browser_tests.process_tests.FailIfTearDownProcessCalledTwice.Dummy_2'],
 124       test_name='FailIfTearDownProcessCalledTwice')
 125
 126   @decorators.Disabled('chromeos')  # crbug.com/696553
 127   def testJsonOutputFormatPositiveFilter(self):
 128     self.baseTest(
 129       '(TestSimple|TestException).*',
 130       ['browser_tests.simple_numeric_test.SimpleTest.TestException',
 131        'browser_tests.simple_numeric_test.SimpleTest.TestSimple'], [])
 132
 133   @decorators.Disabled('chromeos')  # crbug.com/696553
 134   def testExecutingTestsInSortedOrder(self):
 135     alphabetical_tests = []
 136     prefix = 'browser_tests.simple_numeric_test.SimpleTest.Alphabetical_'
 137     for i in xrange(20):
 138       alphabetical_tests.append(prefix + str(i))
 139     for c in string.uppercase[:26]:
 140       alphabetical_tests.append(prefix + c)
 141     for c in string.lowercase[:26]:
 142       alphabetical_tests.append(prefix + c)
 143     alphabetical_tests.sort()
 144     self.baseTest(
 145         'Alphabetical', [], alphabetical_tests)
 146
 147   def shardingRangeTestHelper(self, total_shards, num_tests):
 148     shard_ranges = []
 149     for shard_index in xrange(0, total_shards):
 150       shard_ranges.append(run_browser_tests._TestRangeForShard(
 151         total_shards, shard_index, num_tests))
 152     # Make assertions about ranges
 153     num_tests_run = 0
 154     for i in xrange(0, len(shard_ranges)):
 155       cur_range = shard_ranges[i]
 156       if i < num_tests:
 157         self.assertGreater(cur_range[1], cur_range[0])
 158         num_tests_run += (cur_range[1] - cur_range[0])
 159       else:
 160         # Not enough tests to go around all of the shards.
 161         self.assertEquals(cur_range[0], cur_range[1])
 162     # Make assertions about non-overlapping ranges
 163     for i in xrange(1, len(shard_ranges)):
 164       prev_range = shard_ranges[i - 1]
 165       cur_range = shard_ranges[i]
 166       self.assertEquals(prev_range[1], cur_range[0])
 167     # Assert that we run all of the tests (very important)
 168     self.assertEquals(num_tests_run, num_tests)
 169
 170   def testShardsWithPrimeNumTests(self):
 171     for total_shards in xrange(1, 20):
 172       # Nice non-prime number
 173       self.shardingRangeTestHelper(total_shards, 101)
 174
 175   def testShardsWithDivisibleNumTests(self):
 176     for total_shards in xrange(1, 6):
 177       self.shardingRangeTestHelper(total_shards, 8)
 178
 179   def testShardBoundaryConditions(self):
 180     self.shardingRangeTestHelper(1, 0)
 181     self.shardingRangeTestHelper(1, 1)
 182     self.shardingRangeTestHelper(2, 1)
 183
 184   def baseShardingTest(self, total_shards, shard_index, failures, successes,
 185                        opt_abbr_input_json_file=None,
 186                        opt_test_filter='',
 187                        opt_filter_tests_after_sharding=False):
 188     config = project_config.ProjectConfig(
 189         top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'),
 190         client_configs=[],
 191         benchmark_dirs=[
 192             os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests')]
 193     )
 194     temp_file = tempfile.NamedTemporaryFile(delete=False)
 195     temp_file.close()
 196     temp_file_name = temp_file.name
 197     opt_args = []
 198     if opt_abbr_input_json_file:
 199       opt_args += [
 200         '--read-abbreviated-json-results-from=%s' % opt_abbr_input_json_file]
 201     if opt_test_filter:
 202       opt_args += [
 203         '--test-filter=%s' % opt_test_filter]
 204     if opt_filter_tests_after_sharding:
 205       opt_args += ['--filter-tests-after-sharding']
 206     try:
 207       browser_test_runner.Run(
 208           config,
 209           ['SimpleShardingTest',
 210            '--write-full-results-to=%s' % temp_file_name,
 211            '--total-shards=%d' % total_shards,
 212            '--shard-index=%d' % shard_index] + opt_args)
 213       with open(temp_file_name) as f:
 214         test_result = json.load(f)
 215       actual_successes, actual_failures = self._ExtractTestResults(test_result)
 216       self.assertEquals(set(actual_failures), set(failures))
 217       self.assertEquals(set(actual_successes), set(successes))
 218     finally:
 219       os.remove(temp_file_name)
 220
 221   @decorators.Disabled('chromeos')  # crbug.com/696553
 222   def testShardedTestRun(self):
 223     self.baseShardingTest(3, 0, [], [
 224       'browser_tests.simple_sharding_test.SimpleShardingTest.Test1',
 225       'browser_tests.simple_sharding_test.SimpleShardingTest.Test2',
 226       'browser_tests.simple_sharding_test.SimpleShardingTest.Test3',
 227       'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_0',
 228       'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_1',
 229     ])
 230     self.baseShardingTest(3, 1, [], [
 231       'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_2',
 232       'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_3',
 233       'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_4',
 234       'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_5',
 235     ])
 236     self.baseShardingTest(3, 2, [], [
 237       'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_6',
 238       'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_7',
 239       'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_8',
 240       'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_9',
 241     ])
 242
 243   def writeMockTestResultsFile(self):
 244     mock_test_results = {
 245       'passes': [
 246         'Test1',
 247         'Test2',
 248         'Test3',
 249         'passing_test_0',
 250         'passing_test_1',
 251         'passing_test_2',
 252         'passing_test_3',
 253         'passing_test_4',
 254         'passing_test_5',
 255         'passing_test_6',
 256         'passing_test_7',
 257         'passing_test_8',
 258         'passing_test_9',
 259       ],
 260       'failures': [],
 261       'valid': True,
 262       'times': {
 263         'Test1': 3.0,
 264         'Test2': 3.0,
 265         'Test3': 3.0,
 266         'passing_test_0': 3.0,
 267         'passing_test_1': 2.0,
 268         'passing_test_2': 2.0,
 269         'passing_test_3': 2.0,
 270         'passing_test_4': 2.0,
 271         'passing_test_5': 1.0,
 272         'passing_test_6': 1.0,
 273         'passing_test_7': 1.0,
 274         'passing_test_8': 1.0,
 275         'passing_test_9': 0.5,
 276       }
 277     }
 278     temp_file = tempfile.NamedTemporaryFile(delete=False)
 279     temp_file.close()
 280     temp_file_name = temp_file.name
 281     with open(temp_file_name, 'w') as f:
 282       json.dump(mock_test_results, f)
 283     return temp_file_name
 284
 285   @decorators.Disabled('chromeos')  # crbug.com/696553
 286   def testSplittingShardsByTimes(self):
 287     temp_file_name = self.writeMockTestResultsFile()
 288     # It seems that the sorting order of the first four tests above is:
 289     #   passing_test_0, Test1, Test2, Test3
 290     # This is probably because the relative order of the "fixed" tests
 291     # (starting with "Test") and the generated ones ("passing_") is
 292     # not well defined, and the sorting is stable afterward.  The
 293     # expectations have been adjusted for this fact.
 294     try:
 295       self.baseShardingTest(
 296         4, 0, [],
 297         ['browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_0',
 298          'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_1',
 299          'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_5',
 300          'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_9'
 301         ], temp_file_name)
 302       self.baseShardingTest(
 303         4, 1, [],
 304         ['browser_tests.simple_sharding_test.SimpleShardingTest.Test1',
 305          'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_2',
 306          'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_6'
 307         ], temp_file_name)
 308       self.baseShardingTest(
 309         4, 2, [],
 310         ['browser_tests.simple_sharding_test.SimpleShardingTest.Test2',
 311          'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_3',
 312          'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_7'
 313         ], temp_file_name)
 314       self.baseShardingTest(
 315         4, 3, [],
 316         ['browser_tests.simple_sharding_test.SimpleShardingTest.Test3',
 317          'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_4',
 318          'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_8'
 319         ], temp_file_name)
 320     finally:
 321       os.remove(temp_file_name)
 322
 323   @decorators.Disabled('chromeos')  # crbug.com/696553
 324   def testFilteringAfterSharding(self):
 325     temp_file_name = self.writeMockTestResultsFile()
 326     try:
 327       self.baseShardingTest(
 328         4, 1, [],
 329         ['browser_tests.simple_sharding_test.SimpleShardingTest.Test1',
 330          'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_2',
 331          'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_6'
 332         ], temp_file_name,
 333         opt_test_filter='(Test1|passing_test_2|passing_test_6)',
 334         opt_filter_tests_after_sharding=True)
 335     finally:
 336       os.remove(temp_file_name)
 337
 338   def testMedianComputation(self):
 339     self.assertEquals(2.0, run_browser_tests._MedianTestTime(
 340       {'test1': 2.0, 'test2': 7.0, 'test3': 1.0}))
 341     self.assertEquals(2.0, run_browser_tests._MedianTestTime(
 342       {'test1': 2.0}))
 343     self.assertEquals(0.0, run_browser_tests._MedianTestTime({}))
 344     self.assertEqual(4.0, run_browser_tests._MedianTestTime(
 345       {'test1': 2.0, 'test2': 6.0, 'test3': 1.0, 'test4': 8.0}))
 346
 347
 348 class Algebra(
 349     serially_executed_browser_test_case.SeriallyExecutedBrowserTestCase):
 350
 351   @classmethod
 352   def GenerateTestCases_Simple(cls, options):
 353     del options  # Unused.
 354     yield 'testOne', (1, 2)
 355     yield 'testTwo', (3, 3)
 356
 357   def Simple(self, x, y):
 358     self.assertEquals(x, y)
 359
 360   def TestNumber(self):
 361     self.assertEquals(0, 1)
 362
 363
 364 class ErrorneousGeometric(
 365     serially_executed_browser_test_case.SeriallyExecutedBrowserTestCase):
 366
 367   @classmethod
 368   def GenerateTestCases_Compare(cls, options):
 369     del options  # Unused.
 370     assert False, 'I am a problematic generator'
 371     yield 'testBasic', ('square', 'circle')
 372
 373   def Compare(self, x, y):
 374     self.assertEquals(x, y)
 375
 376   def TestAngle(self):
 377     self.assertEquals(90, 450)
 378
 379 class TestLoadAllTestModules(unittest.TestCase):
 380   def testLoadAllTestsInModule(self):
 381     context = browser_test_context.TypTestContext()
 382     context.finder_options = options_for_unittests.GetCopy()
 383     context.test_class = Algebra
 384     context.test_case_ids_to_run.add(
 385       'telemetry.testing.browser_test_runner_unittest.Algebra.TestNumber')
 386     context.test_case_ids_to_run.add(
 387       'telemetry.testing.browser_test_runner_unittest.Algebra.testOne')
 388     context.Freeze()
 389     browser_test_context._global_test_context = context
 390     try:
 391       # This should not invoke GenerateTestCases of ErrorneousGeometric class,
 392       # otherwise that would throw Exception.
 393       tests = serially_executed_browser_test_case.LoadAllTestsInModule(
 394           sys.modules[__name__])
 395       self.assertEquals(sorted([t.id() for t in tests]),
 396           ['telemetry.testing.browser_test_runner_unittest.Algebra.TestNumber',
 397            'telemetry.testing.browser_test_runner_unittest.Algebra.testOne'])
 398     finally:
 399       browser_test_context._global_test_context = None