1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
13 # TODO(dpranke): This code is largely cloned from, and redundant with,
14 # src/mojo/tools/run_mojo_python_tests.py, and also duplicates logic
15 # in test-webkitpy and run-webkit-tests. We should consolidate the
16 # python TestResult parsing/converting/uploading code as much as possible.
19 def AddOptions(parser
):
20 parser
.add_option('--metadata', action
='append', default
=[],
21 help=('optional key=value metadata that will be stored '
22 'in the results files (can be used for revision '
24 parser
.add_option('--write-full-results-to', metavar
='FILENAME',
26 help='The path to write the list of full results to.')
27 parser
.add_option('--builder-name',
28 help='The name of the builder as shown on the waterfall.')
29 parser
.add_option('--master-name',
30 help='The name of the buildbot master.')
31 parser
.add_option("--test-results-server", default
="",
32 help=('If specified, upload full_results.json file to '
34 parser
.add_option('--test-type',
35 help=('Name of test type / step on the waterfall '
36 '(e.g., "telemetry_unittests").'))
39 def ValidateArgs(parser
, args
):
40 for val
in args
.metadata
:
42 parser
.error('Error: malformed metadata "%s"' % val
)
44 if (args
.test_results_server
and
45 (not args
.builder_name
or not args
.master_name
or not args
.test_type
)):
46 parser
.error('Error: --builder-name, --master-name, and --test-type '
47 'must be specified along with --test-result-server.')
50 def WriteFullResultsIfNecessary(args
, full_results
):
51 if not args
.write_full_results_to
:
54 with
open(args
.write_full_results_to
, 'w') as fp
:
55 json
.dump(full_results
, fp
, indent
=2)
59 def UploadFullResultsIfNecessary(args
, full_results
):
60 if not args
.test_results_server
:
63 url
= 'http://%s/testfile/upload' % args
.test_results_server
64 attrs
= [('builder', args
.builder_name
),
65 ('master', args
.master_name
),
66 ('testtype', args
.test_type
)]
67 content_type
, data
= _EncodeMultiPartFormData(attrs
, full_results
)
68 return _UploadData(url
, data
, content_type
)
74 def FullResults(args
, suite
, results
):
75 """Convert the unittest results to the Chromium JSON test result format.
77 This matches run-webkit-tests (the layout tests) and the flakiness dashboard.
81 full_results
['interrupted'] = False
82 full_results
['path_delimiter'] = TEST_SEPARATOR
83 full_results
['version'] = 3
84 full_results
['seconds_since_epoch'] = time
.time()
85 full_results
['builder_name'] = args
.builder_name
or ''
86 for md
in args
.metadata
:
87 key
, val
= md
.split('=', 1)
88 full_results
[key
] = val
90 all_test_names
= AllTestNames(suite
)
91 sets_of_passing_test_names
= map(PassingTestNames
, results
)
92 sets_of_failing_test_names
= map(functools
.partial(FailedTestNames
, suite
),
95 # TODO(crbug.com/405379): This handles tests that are skipped via the
96 # unittest skip decorators (like skipUnless). The tests that are skipped via
97 # telemetry's decorators package are not included in the test suite at all so
98 # we need those to be passed in in order to include them.
99 skipped_tests
= (set(all_test_names
) - sets_of_passing_test_names
[0]
100 - sets_of_failing_test_names
[0])
102 num_tests
= len(all_test_names
)
103 num_failures
= NumFailuresAfterRetries(suite
, results
)
104 num_skips
= len(skipped_tests
)
105 num_passes
= num_tests
- num_failures
- num_skips
106 full_results
['num_failures_by_type'] = {
107 'FAIL': num_failures
,
112 full_results
['tests'] = {}
114 for test_name
in all_test_names
:
115 if test_name
in skipped_tests
:
123 'actual': ActualResultsForTest(test_name
,
124 sets_of_failing_test_names
,
125 sets_of_passing_test_names
),
127 if value
['actual'].endswith('FAIL'):
128 value
['is_unexpected'] = True
129 _AddPathToTrie(full_results
['tests'], test_name
, value
)
134 def ActualResultsForTest(test_name
, sets_of_failing_test_names
,
135 sets_of_passing_test_names
):
137 for retry_num
in range(len(sets_of_failing_test_names
)):
138 if test_name
in sets_of_failing_test_names
[retry_num
]:
139 actuals
.append('FAIL')
140 elif test_name
in sets_of_passing_test_names
[retry_num
]:
141 assert ((retry_num
== 0) or
142 (test_name
in sets_of_failing_test_names
[retry_num
- 1])), (
143 'We should not have run a test that did not fail '
144 'on the previous run.')
145 actuals
.append('PASS')
147 assert actuals
, 'We did not find any result data for %s.' % test_name
148 return ' '.join(actuals
)
151 def ExitCodeFromFullResults(full_results
):
152 return 1 if full_results
['num_failures_by_type']['FAIL'] else 0
155 def AllTestNames(suite
):
157 # _tests is protected pylint: disable=W0212
158 for test
in suite
._tests
:
159 if isinstance(test
, unittest
.suite
.TestSuite
):
160 test_names
.extend(AllTestNames(test
))
162 test_names
.append(test
.id())
166 def NumFailuresAfterRetries(suite
, results
):
167 return len(FailedTestNames(suite
, results
[-1]))
170 def FailedTestNames(suite
, result
):
171 failed_test_names
= set()
172 for test
, error
in result
.failures
+ result
.errors
:
173 if isinstance(test
, unittest
.TestCase
):
174 failed_test_names
.add(test
.id())
175 elif isinstance(test
, unittest
.suite
._ErrorHolder
): # pylint: disable=W0212
176 # If there's an error in setUpClass or setUpModule, unittest gives us an
177 # _ErrorHolder object. We can parse the object's id for the class or
178 # module that failed, then find all tests in that class or module.
179 match
= re
.match('setUp[a-zA-Z]+ \\((.+)\\)', test
.id())
180 assert match
, "Don't know how to retry after this error:\n%s" % error
181 module_or_class
= match
.groups()[0]
182 failed_test_names |
= _FindChildren(module_or_class
, AllTestNames(suite
))
184 assert False, 'Unknown test type: %s' % test
.__class
__
185 return failed_test_names
188 def _FindChildren(parent
, potential_children
):
190 parent_name_parts
= parent
.split('.')
191 for potential_child
in potential_children
:
192 child_name_parts
= potential_child
.split('.')
193 if parent_name_parts
== child_name_parts
[:len(parent_name_parts
)]:
194 children
.add(potential_child
)
198 def PassingTestNames(result
):
199 return set(test
.id() for test
in result
.successes
)
202 def _AddPathToTrie(trie
, path
, value
):
203 if TEST_SEPARATOR
not in path
:
206 directory
, rest
= path
.split(TEST_SEPARATOR
, 1)
207 if directory
not in trie
:
209 _AddPathToTrie(trie
[directory
], rest
, value
)
212 def _EncodeMultiPartFormData(attrs
, full_results
):
213 # Cloned from webkitpy/common/net/file_uploader.py
214 BOUNDARY
= '-M-A-G-I-C---B-O-U-N-D-A-R-Y-'
218 for key
, value
in attrs
:
219 lines
.append('--' + BOUNDARY
)
220 lines
.append('Content-Disposition: form-data; name="%s"' % key
)
224 lines
.append('--' + BOUNDARY
)
225 lines
.append('Content-Disposition: form-data; name="file"; '
226 'filename="full_results.json"')
227 lines
.append('Content-Type: application/json')
229 lines
.append(json
.dumps(full_results
))
231 lines
.append('--' + BOUNDARY
+ '--')
233 body
= CRLF
.join(lines
)
234 content_type
= 'multipart/form-data; boundary=%s' % BOUNDARY
235 return content_type
, body
238 def _UploadData(url
, data
, content_type
):
239 request
= urllib2
.Request(url
, data
, {'Content-Type': content_type
})
241 response
= urllib2
.urlopen(request
)
242 if response
.code
== 200:
244 return True, ('Uploading the JSON results failed with %d: "%s"' %
245 (response
.code
, response
.read()))
246 except Exception as e
:
247 return True, 'Uploading the JSON results raised "%s"\n' % str(e
)