build/android/pylib/gtest/setup.py

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Generates test runner factory and tests for GTests."""
   6
   7 import fnmatch
   8 import glob
   9 import logging
  10 import os
  11 import shutil
  12 import sys
  13
  14 from pylib import android_commands
  15 from pylib import cmd_helper
  16 from pylib import constants
  17 from pylib import ports
  18
  19 import test_package_apk
  20 import test_package_exe
  21 import test_runner
  22
  23 sys.path.insert(0,
  24                 os.path.join(constants.DIR_SOURCE_ROOT, 'build', 'util', 'lib',
  25                              'common'))
  26 import unittest_util
  27
  28
  29 _ISOLATE_FILE_PATHS = {
  30     'base_unittests': 'base/base_unittests.isolate',
  31     'breakpad_unittests': 'breakpad/breakpad_unittests.isolate',
  32     'cc_perftests': 'cc/cc_perftests.isolate',
  33     'components_unittests': 'components/components_unittests.isolate',
  34     'content_browsertests': 'content/content_browsertests.isolate',
  35     'content_unittests': 'content/content_unittests.isolate',
  36     'media_perftests': 'media/media_perftests.isolate',
  37     'media_unittests': 'media/media_unittests.isolate',
  38     'net_unittests': 'net/net_unittests.isolate',
  39     'ui_unittests': 'ui/ui_unittests.isolate',
  40     'unit_tests': 'chrome/unit_tests.isolate',
  41     'webkit_unit_tests':
  42       'third_party/WebKit/Source/web/WebKitUnitTests.isolate',
  43 }
  44
  45 # Paths relative to third_party/webrtc/ (kept separate for readability).
  46 _WEBRTC_ISOLATE_FILE_PATHS = {
  47     'audio_decoder_unittests':
  48       'modules/audio_coding/neteq4/audio_decoder_unittests.isolate',
  49     'common_audio_unittests': 'common_audio/common_audio_unittests.isolate',
  50     'common_video_unittests': 'common_video/common_video_unittests.isolate',
  51     'metrics_unittests': 'test/metrics_unittests.isolate',
  52     'modules_tests': 'modules/modules_tests.isolate',
  53     'modules_unittests': 'modules/modules_unittests.isolate',
  54     'neteq_unittests': 'modules/audio_coding/neteq/neteq_unittests.isolate',
  55     'system_wrappers_unittests':
  56       'system_wrappers/source/system_wrappers_unittests.isolate',
  57     'test_support_unittests': 'test/test_support_unittests.isolate',
  58     'tools_unittests': 'tools/tools_unittests.isolate',
  59     'video_engine_core_unittests':
  60       'video_engine/video_engine_core_unittests.isolate',
  61     'voice_engine_unittests': 'voice_engine/voice_engine_unittests.isolate',
  62 }
  63
  64 # Append the WebRTC tests with the full path from Chromium's src/ root.
  65 for test, isolate_path in _WEBRTC_ISOLATE_FILE_PATHS.items():
  66   _ISOLATE_FILE_PATHS[test] = 'third_party/webrtc/%s' % isolate_path
  67
  68 # Used for filtering large data deps at a finer grain than what's allowed in
  69 # isolate files since pushing deps to devices is expensive.
  70 # Wildcards are allowed.
  71 _DEPS_EXCLUSION_LIST = [
  72     'chrome/test/data/extensions/api_test',
  73     'chrome/test/data/extensions/secure_shell',
  74     'chrome/test/data/firefox*',
  75     'chrome/test/data/gpu',
  76     'chrome/test/data/image_decoding',
  77     'chrome/test/data/import',
  78     'chrome/test/data/page_cycler',
  79     'chrome/test/data/perf',
  80     'chrome/test/data/pyauto_private',
  81     'chrome/test/data/safari_import',
  82     'chrome/test/data/scroll',
  83     'chrome/test/data/third_party',
  84     'third_party/hunspell_dictionaries/*.dic',
  85     # crbug.com/258690
  86     'webkit/data/bmp_decoder',
  87     'webkit/data/ico_decoder',
  88 ]
  89
  90 _ISOLATE_SCRIPT = os.path.join(
  91     constants.DIR_SOURCE_ROOT, 'tools', 'swarming_client', 'isolate.py')
  92
  93
  94 def _GenerateDepsDirUsingIsolate(suite_name):
  95   """Generate the dependency dir for the test suite using isolate.
  96
  97   Args:
  98     suite_name: Name of the test suite (e.g. base_unittests).
  99   """
 100   if os.path.isdir(constants.ISOLATE_DEPS_DIR):
 101     shutil.rmtree(constants.ISOLATE_DEPS_DIR)
 102
 103   isolate_rel_path = _ISOLATE_FILE_PATHS.get(suite_name)
 104   if not isolate_rel_path:
 105     logging.info('Did not find an isolate file for the test suite.')
 106     return
 107
 108   isolate_abs_path = os.path.join(constants.DIR_SOURCE_ROOT, isolate_rel_path)
 109   isolated_abs_path = os.path.join(
 110       constants.GetOutDirectory(), '%s.isolated' % suite_name)
 111   assert os.path.exists(isolate_abs_path)
 112   isolate_cmd = [
 113       'python', _ISOLATE_SCRIPT,
 114       'remap',
 115       '--isolate', isolate_abs_path,
 116       '--isolated', isolated_abs_path,
 117       '-V', 'PRODUCT_DIR=%s' % constants.GetOutDirectory(),
 118       '-V', 'OS=android',
 119       '--outdir', constants.ISOLATE_DEPS_DIR,
 120   ]
 121   assert not cmd_helper.RunCmd(isolate_cmd)
 122
 123   # We're relying on the fact that timestamps are preserved
 124   # by the remap command (hardlinked). Otherwise, all the data
 125   # will be pushed to the device once we move to using time diff
 126   # instead of md5sum. Perform a sanity check here.
 127   for root, _, filenames in os.walk(constants.ISOLATE_DEPS_DIR):
 128     if filenames:
 129       linked_file = os.path.join(root, filenames[0])
 130       orig_file = os.path.join(
 131           constants.DIR_SOURCE_ROOT,
 132           os.path.relpath(linked_file, constants.ISOLATE_DEPS_DIR))
 133       if os.stat(linked_file).st_ino == os.stat(orig_file).st_ino:
 134         break
 135       else:
 136         raise Exception('isolate remap command did not use hardlinks.')
 137
 138   # Delete excluded files as defined by _DEPS_EXCLUSION_LIST.
 139   old_cwd = os.getcwd()
 140   try:
 141     os.chdir(constants.ISOLATE_DEPS_DIR)
 142     excluded_paths = [x for y in _DEPS_EXCLUSION_LIST for x in glob.glob(y)]
 143     if excluded_paths:
 144       logging.info('Excluding the following from dependency list: %s',
 145                    excluded_paths)
 146     for p in excluded_paths:
 147       if os.path.isdir(p):
 148         shutil.rmtree(p)
 149       else:
 150         os.remove(p)
 151   finally:
 152     os.chdir(old_cwd)
 153
 154   # On Android, all pak files need to be in the top-level 'paks' directory.
 155   paks_dir = os.path.join(constants.ISOLATE_DEPS_DIR, 'paks')
 156   os.mkdir(paks_dir)
 157   for root, _, filenames in os.walk(os.path.join(constants.ISOLATE_DEPS_DIR,
 158                                                  'out')):
 159     for filename in fnmatch.filter(filenames, '*.pak'):
 160       shutil.move(os.path.join(root, filename), paks_dir)
 161
 162   # Move everything in PRODUCT_DIR to top level.
 163   deps_product_dir = os.path.join(constants.ISOLATE_DEPS_DIR, 'out',
 164       constants.GetBuildType())
 165   if os.path.isdir(deps_product_dir):
 166     for p in os.listdir(deps_product_dir):
 167       shutil.move(os.path.join(deps_product_dir, p), constants.ISOLATE_DEPS_DIR)
 168     os.rmdir(deps_product_dir)
 169     os.rmdir(os.path.join(constants.ISOLATE_DEPS_DIR, 'out'))
 170
 171
 172 def _GetDisabledTestsFilterFromFile(suite_name):
 173   """Returns a gtest filter based on the *_disabled file.
 174
 175   Args:
 176     suite_name: Name of the test suite (e.g. base_unittests).
 177
 178   Returns:
 179     A gtest filter which excludes disabled tests.
 180     Example: '*-StackTrace.*:StringPrintfTest.StringPrintfMisc'
 181   """
 182   filter_file_path = os.path.join(
 183       os.path.abspath(os.path.dirname(__file__)),
 184       'filter', '%s_disabled' % suite_name)
 185
 186   if not filter_file_path or not os.path.exists(filter_file_path):
 187     logging.info('No filter file found at %s', filter_file_path)
 188     return '*'
 189
 190   filters = [x for x in [x.strip() for x in file(filter_file_path).readlines()]
 191              if x and x[0] != '#']
 192   disabled_filter = '*-%s' % ':'.join(filters)
 193   logging.info('Applying filter "%s" obtained from %s',
 194                disabled_filter, filter_file_path)
 195   return disabled_filter
 196
 197
 198 def _GetTestsFromDevice(runner_factory, devices):
 199   """Get a list of tests from a device.
 200
 201   Args:
 202     runner_factory: Callable that takes device and shard_index and returns
 203         a TestRunner.
 204     devices: A list of device ids.
 205
 206   Returns:
 207     All the tests in the test suite.
 208   """
 209   for device in devices:
 210     try:
 211       logging.info('Obtaining tests from %s', device)
 212       return runner_factory(device, 0).GetAllTests()
 213     except (android_commands.errors.WaitForResponseTimedOutError,
 214             android_commands.errors.DeviceUnresponsiveError), e:
 215       logging.warning('Failed obtaining test list from %s with exception: %s',
 216                       device, e)
 217   raise Exception('Failed to obtain test list from devices.')
 218
 219
 220 def _FilterTestsUsingPrefixes(all_tests, pre=False, manual=False):
 221   """Removes tests with disabled prefixes.
 222
 223   Args:
 224     all_tests: List of tests to filter.
 225     pre: If True, include tests with PRE_ prefix.
 226     manual: If True, include tests with MANUAL_ prefix.
 227
 228   Returns:
 229     List of tests remaining.
 230   """
 231   filtered_tests = []
 232   filter_prefixes = ['DISABLED_', 'FLAKY_', 'FAILS_']
 233
 234   if not pre:
 235     filter_prefixes.append('PRE_')
 236
 237   if not manual:
 238     filter_prefixes.append('MANUAL_')
 239
 240   for t in all_tests:
 241     test_case, test = t.split('.', 1)
 242     if not any([test_case.startswith(prefix) or test.startswith(prefix) for
 243                 prefix in filter_prefixes]):
 244       filtered_tests.append(t)
 245   return filtered_tests
 246
 247
 248 def _FilterDisabledTests(tests, suite_name, has_gtest_filter):
 249   """Removes disabled tests from |tests|.
 250
 251   Applies the following filters in order:
 252     1. Remove tests with disabled prefixes.
 253     2. Remove tests specified in the *_disabled files in the 'filter' dir
 254
 255   Args:
 256     tests: List of tests.
 257     suite_name: Name of the test suite (e.g. base_unittests).
 258     has_gtest_filter: Whether a gtest_filter is provided.
 259
 260   Returns:
 261     List of tests remaining.
 262   """
 263   tests = _FilterTestsUsingPrefixes(
 264       tests, has_gtest_filter, has_gtest_filter)
 265   tests = unittest_util.FilterTestNames(
 266       tests, _GetDisabledTestsFilterFromFile(suite_name))
 267
 268   return tests
 269
 270
 271 def Setup(test_options, devices):
 272   """Create the test runner factory and tests.
 273
 274   Args:
 275     test_options: A GTestOptions object.
 276     devices: A list of attached devices.
 277
 278   Returns:
 279     A tuple of (TestRunnerFactory, tests).
 280   """
 281   test_package = test_package_apk.TestPackageApk(test_options.suite_name)
 282   if not os.path.exists(test_package.suite_path):
 283     test_package = test_package_exe.TestPackageExecutable(
 284         test_options.suite_name)
 285     if not os.path.exists(test_package.suite_path):
 286       raise Exception(
 287           'Did not find %s target. Ensure it has been built.'
 288           % test_options.suite_name)
 289   logging.warning('Found target %s', test_package.suite_path)
 290
 291   _GenerateDepsDirUsingIsolate(test_options.suite_name)
 292
 293   # Constructs a new TestRunner with the current options.
 294   def TestRunnerFactory(device, shard_index):
 295     return test_runner.TestRunner(
 296         test_options,
 297         device,
 298         test_package)
 299
 300   tests = _GetTestsFromDevice(TestRunnerFactory, devices)
 301   if test_options.run_disabled:
 302     test_options = test_options._replace(
 303         test_arguments=('%s --gtest_also_run_disabled_tests' %
 304                         test_options.test_arguments))
 305   else:
 306     tests = _FilterDisabledTests(tests, test_options.suite_name,
 307                                  bool(test_options.gtest_filter))
 308   if test_options.gtest_filter:
 309     tests = unittest_util.FilterTestNames(tests, test_options.gtest_filter)
 310
 311   # Coalesce unit tests into a single test per device
 312   if test_options.suite_name != 'content_browsertests':
 313     num_devices = len(devices)
 314     tests = [':'.join(tests[i::num_devices]) for i in xrange(num_devices)]
 315     tests = [t for t in tests if t]
 316
 317   return (TestRunnerFactory, tests)