1 # Copyright 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """Generates test runner factory and tests for GTests."""
14 from pylib
import android_commands
15 from pylib
import cmd_helper
16 from pylib
import constants
17 from pylib
import ports
19 import test_package_apk
20 import test_package_exe
24 os
.path
.join(constants
.DIR_SOURCE_ROOT
, 'build', 'util', 'lib',
29 _ISOLATE_FILE_PATHS
= {
30 'base_unittests': 'base/base_unittests.isolate',
31 'breakpad_unittests': 'breakpad/breakpad_unittests.isolate',
32 'cc_perftests': 'cc/cc_perftests.isolate',
33 'components_unittests': 'components/components_unittests.isolate',
34 'content_browsertests': 'content/content_browsertests.isolate',
35 'content_unittests': 'content/content_unittests.isolate',
36 'media_perftests': 'media/media_perftests.isolate',
37 'media_unittests': 'media/media_unittests.isolate',
38 'net_unittests': 'net/net_unittests.isolate',
39 'ui_unittests': 'ui/ui_unittests.isolate',
40 'unit_tests': 'chrome/unit_tests.isolate',
42 'third_party/WebKit/Source/web/WebKitUnitTests.isolate',
45 # Paths relative to third_party/webrtc/ (kept separate for readability).
46 _WEBRTC_ISOLATE_FILE_PATHS
= {
47 'audio_decoder_unittests':
48 'modules/audio_coding/neteq4/audio_decoder_unittests.isolate',
49 'common_audio_unittests': 'common_audio/common_audio_unittests.isolate',
50 'common_video_unittests': 'common_video/common_video_unittests.isolate',
51 'metrics_unittests': 'test/metrics_unittests.isolate',
52 'modules_tests': 'modules/modules_tests.isolate',
53 'modules_unittests': 'modules/modules_unittests.isolate',
54 'neteq_unittests': 'modules/audio_coding/neteq/neteq_unittests.isolate',
55 'system_wrappers_unittests':
56 'system_wrappers/source/system_wrappers_unittests.isolate',
57 'test_support_unittests': 'test/test_support_unittests.isolate',
58 'tools_unittests': 'tools/tools_unittests.isolate',
59 'video_engine_core_unittests':
60 'video_engine/video_engine_core_unittests.isolate',
61 'voice_engine_unittests': 'voice_engine/voice_engine_unittests.isolate',
64 # Append the WebRTC tests with the full path from Chromium's src/ root.
65 for test
, isolate_path
in _WEBRTC_ISOLATE_FILE_PATHS
.items():
66 _ISOLATE_FILE_PATHS
[test
] = 'third_party/webrtc/%s' % isolate_path
68 # Used for filtering large data deps at a finer grain than what's allowed in
69 # isolate files since pushing deps to devices is expensive.
70 # Wildcards are allowed.
71 _DEPS_EXCLUSION_LIST
= [
72 'chrome/test/data/extensions/api_test',
73 'chrome/test/data/extensions/secure_shell',
74 'chrome/test/data/firefox*',
75 'chrome/test/data/gpu',
76 'chrome/test/data/image_decoding',
77 'chrome/test/data/import',
78 'chrome/test/data/page_cycler',
79 'chrome/test/data/perf',
80 'chrome/test/data/pyauto_private',
81 'chrome/test/data/safari_import',
82 'chrome/test/data/scroll',
83 'chrome/test/data/third_party',
84 'third_party/hunspell_dictionaries/*.dic',
86 'webkit/data/bmp_decoder',
87 'webkit/data/ico_decoder',
90 _ISOLATE_SCRIPT
= os
.path
.join(
91 constants
.DIR_SOURCE_ROOT
, 'tools', 'swarming_client', 'isolate.py')
94 def _GenerateDepsDirUsingIsolate(suite_name
):
95 """Generate the dependency dir for the test suite using isolate.
98 suite_name: Name of the test suite (e.g. base_unittests).
100 if os
.path
.isdir(constants
.ISOLATE_DEPS_DIR
):
101 shutil
.rmtree(constants
.ISOLATE_DEPS_DIR
)
103 isolate_rel_path
= _ISOLATE_FILE_PATHS
.get(suite_name
)
104 if not isolate_rel_path
:
105 logging
.info('Did not find an isolate file for the test suite.')
108 isolate_abs_path
= os
.path
.join(constants
.DIR_SOURCE_ROOT
, isolate_rel_path
)
109 isolated_abs_path
= os
.path
.join(
110 constants
.GetOutDirectory(), '%s.isolated' % suite_name
)
111 assert os
.path
.exists(isolate_abs_path
)
113 'python', _ISOLATE_SCRIPT
,
115 '--isolate', isolate_abs_path
,
116 '--isolated', isolated_abs_path
,
117 '-V', 'PRODUCT_DIR=%s' % constants
.GetOutDirectory(),
119 '--outdir', constants
.ISOLATE_DEPS_DIR
,
121 assert not cmd_helper
.RunCmd(isolate_cmd
)
123 # We're relying on the fact that timestamps are preserved
124 # by the remap command (hardlinked). Otherwise, all the data
125 # will be pushed to the device once we move to using time diff
126 # instead of md5sum. Perform a sanity check here.
127 for root
, _
, filenames
in os
.walk(constants
.ISOLATE_DEPS_DIR
):
129 linked_file
= os
.path
.join(root
, filenames
[0])
130 orig_file
= os
.path
.join(
131 constants
.DIR_SOURCE_ROOT
,
132 os
.path
.relpath(linked_file
, constants
.ISOLATE_DEPS_DIR
))
133 if os
.stat(linked_file
).st_ino
== os
.stat(orig_file
).st_ino
:
136 raise Exception('isolate remap command did not use hardlinks.')
138 # Delete excluded files as defined by _DEPS_EXCLUSION_LIST.
139 old_cwd
= os
.getcwd()
141 os
.chdir(constants
.ISOLATE_DEPS_DIR
)
142 excluded_paths
= [x
for y
in _DEPS_EXCLUSION_LIST
for x
in glob
.glob(y
)]
144 logging
.info('Excluding the following from dependency list: %s',
146 for p
in excluded_paths
:
154 # On Android, all pak files need to be in the top-level 'paks' directory.
155 paks_dir
= os
.path
.join(constants
.ISOLATE_DEPS_DIR
, 'paks')
157 for root
, _
, filenames
in os
.walk(os
.path
.join(constants
.ISOLATE_DEPS_DIR
,
159 for filename
in fnmatch
.filter(filenames
, '*.pak'):
160 shutil
.move(os
.path
.join(root
, filename
), paks_dir
)
162 # Move everything in PRODUCT_DIR to top level.
163 deps_product_dir
= os
.path
.join(constants
.ISOLATE_DEPS_DIR
, 'out',
164 constants
.GetBuildType())
165 if os
.path
.isdir(deps_product_dir
):
166 for p
in os
.listdir(deps_product_dir
):
167 shutil
.move(os
.path
.join(deps_product_dir
, p
), constants
.ISOLATE_DEPS_DIR
)
168 os
.rmdir(deps_product_dir
)
169 os
.rmdir(os
.path
.join(constants
.ISOLATE_DEPS_DIR
, 'out'))
172 def _GetDisabledTestsFilterFromFile(suite_name
):
173 """Returns a gtest filter based on the *_disabled file.
176 suite_name: Name of the test suite (e.g. base_unittests).
179 A gtest filter which excludes disabled tests.
180 Example: '*-StackTrace.*:StringPrintfTest.StringPrintfMisc'
182 filter_file_path
= os
.path
.join(
183 os
.path
.abspath(os
.path
.dirname(__file__
)),
184 'filter', '%s_disabled' % suite_name
)
186 if not filter_file_path
or not os
.path
.exists(filter_file_path
):
187 logging
.info('No filter file found at %s', filter_file_path
)
190 filters
= [x
for x
in [x
.strip() for x
in file(filter_file_path
).readlines()]
191 if x
and x
[0] != '#']
192 disabled_filter
= '*-%s' % ':'.join(filters
)
193 logging
.info('Applying filter "%s" obtained from %s',
194 disabled_filter
, filter_file_path
)
195 return disabled_filter
198 def _GetTestsFromDevice(runner_factory
, devices
):
199 """Get a list of tests from a device.
202 runner_factory: Callable that takes device and shard_index and returns
204 devices: A list of device ids.
207 All the tests in the test suite.
209 for device
in devices
:
211 logging
.info('Obtaining tests from %s', device
)
212 return runner_factory(device
, 0).GetAllTests()
213 except (android_commands
.errors
.WaitForResponseTimedOutError
,
214 android_commands
.errors
.DeviceUnresponsiveError
), e
:
215 logging
.warning('Failed obtaining test list from %s with exception: %s',
217 raise Exception('Failed to obtain test list from devices.')
220 def _FilterTestsUsingPrefixes(all_tests
, pre
=False, manual
=False):
221 """Removes tests with disabled prefixes.
224 all_tests: List of tests to filter.
225 pre: If True, include tests with PRE_ prefix.
226 manual: If True, include tests with MANUAL_ prefix.
229 List of tests remaining.
232 filter_prefixes
= ['DISABLED_', 'FLAKY_', 'FAILS_']
235 filter_prefixes
.append('PRE_')
238 filter_prefixes
.append('MANUAL_')
241 test_case
, test
= t
.split('.', 1)
242 if not any([test_case
.startswith(prefix
) or test
.startswith(prefix
) for
243 prefix
in filter_prefixes
]):
244 filtered_tests
.append(t
)
245 return filtered_tests
248 def _FilterDisabledTests(tests
, suite_name
, has_gtest_filter
):
249 """Removes disabled tests from |tests|.
251 Applies the following filters in order:
252 1. Remove tests with disabled prefixes.
253 2. Remove tests specified in the *_disabled files in the 'filter' dir
256 tests: List of tests.
257 suite_name: Name of the test suite (e.g. base_unittests).
258 has_gtest_filter: Whether a gtest_filter is provided.
261 List of tests remaining.
263 tests
= _FilterTestsUsingPrefixes(
264 tests
, has_gtest_filter
, has_gtest_filter
)
265 tests
= unittest_util
.FilterTestNames(
266 tests
, _GetDisabledTestsFilterFromFile(suite_name
))
271 def Setup(test_options
, devices
):
272 """Create the test runner factory and tests.
275 test_options: A GTestOptions object.
276 devices: A list of attached devices.
279 A tuple of (TestRunnerFactory, tests).
281 test_package
= test_package_apk
.TestPackageApk(test_options
.suite_name
)
282 if not os
.path
.exists(test_package
.suite_path
):
283 test_package
= test_package_exe
.TestPackageExecutable(
284 test_options
.suite_name
)
285 if not os
.path
.exists(test_package
.suite_path
):
287 'Did not find %s target. Ensure it has been built.'
288 % test_options
.suite_name
)
289 logging
.warning('Found target %s', test_package
.suite_path
)
291 _GenerateDepsDirUsingIsolate(test_options
.suite_name
)
293 # Constructs a new TestRunner with the current options.
294 def TestRunnerFactory(device
, shard_index
):
295 return test_runner
.TestRunner(
300 tests
= _GetTestsFromDevice(TestRunnerFactory
, devices
)
301 if test_options
.run_disabled
:
302 test_options
= test_options
._replace
(
303 test_arguments
=('%s --gtest_also_run_disabled_tests' %
304 test_options
.test_arguments
))
306 tests
= _FilterDisabledTests(tests
, test_options
.suite_name
,
307 bool(test_options
.gtest_filter
))
308 if test_options
.gtest_filter
:
309 tests
= unittest_util
.FilterTestNames(tests
, test_options
.gtest_filter
)
311 # Coalesce unit tests into a single test per device
312 if test_options
.suite_name
!= 'content_browsertests':
313 num_devices
= len(devices
)
314 tests
= [':'.join(tests
[i
::num_devices
]) for i
in xrange(num_devices
)]
315 tests
= [t
for t
in tests
if t
]
317 return (TestRunnerFactory
, tests
)