1 # Copyright 2014 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file.
5 """This module contains functionality for starting build try jobs via HTTP.
7 This includes both sending a request to start a job, and also related code
8 for querying the status of the job.
10 This module can be either run as a stand-alone script to send a request to a
11 builder, or imported and used by calling the public functions below.
22 # URL template for fetching JSON data about builds.
23 BUILDER_JSON_URL
= ('%(server_url)s/json/builders/%(bot_name)s/builds/'
24 '%(build_num)s?as_text=1&filter=0')
26 # URL template for displaying build steps.
27 BUILDER_HTML_URL
= ('%(server_url)s/builders/%(bot_name)s/builds/%(build_num)s')
29 # Try server status page for the perf try server.
30 PERF_TRY_SERVER_URL
= 'http://build.chromium.org/p/tryserver.chromium.perf'
32 # Hostname of the tryserver where perf bisect builders are hosted.
33 # This is used for posting build requests to the tryserver.
34 PERF_BISECT_BUILDER_HOST
= 'master4.golo.chromium.org'
36 # The default 'try_job_port' on tryserver to post build request.
37 PERF_BISECT_BUILDER_PORT
= 8341
39 # Status codes that can be returned by the GetBuildStatus method.
40 # From buildbot.status.builder.
41 # See: http://docs.buildbot.net/current/developer/results.html
42 SUCCESS
, WARNINGS
, FAILURE
, SKIPPED
, EXCEPTION
, RETRY
, TRYPENDING
= range(7)
44 OK
= (SUCCESS
, WARNINGS
) # These indicate build is complete.
45 FAILED
= (FAILURE
, EXCEPTION
, SKIPPED
) # These indicate build failure.
46 PENDING
= (RETRY
, TRYPENDING
) # These indicate in progress or in pending queue.
49 class ServerAccessError(Exception):
52 return '%s\nSorry, cannot connect to server.' % self
.args
[0]
55 def PostTryJob(host
, port
, params
):
56 """Sends a build request to the server using the HTTP protocol.
58 The required parameters are:
59 'revision': "src@rev", where rev is a git hash or SVN revision.
60 'bot': Name of builder bot to use, e.g. "win_perf_bisect_builder".
63 host: Hostname of the try server.
64 port: Port of the try server.
65 params: A dictionary of parameters to be sent in the POST request.
68 True if the request is posted successfully.
71 ServerAccessError: Failed to make a request to the try server.
72 ValueError: Not all of the expected inputs were given.
74 if not params
.get('revision'):
75 raise ValueError('Missing revision number.')
76 if not params
.get('bot'):
77 raise ValueError('Missing bot name.')
79 base_url
= 'http://%s:%s/send_try_patch' % (host
, port
)
80 print 'Posting build request by HTTP.'
81 print 'URL: %s, params: %s' % (base_url
, params
)
85 print 'Opening connection...'
86 connection
= urllib2
.urlopen(base_url
, urllib
.urlencode(params
))
87 print 'Done, request sent to server to produce build.'
89 raise ServerAccessError('%s is unaccessible. Reason: %s' % (base_url
, e
))
91 raise ServerAccessError('%s is unaccessible.' % base_url
)
93 response
= connection
.read()
94 print 'Received response from server: %s' % response
96 raise ServerAccessError('Response was not "OK".')
100 def _IsBuildRunning(build_data
):
101 """Checks whether the build is in progress on buildbot.
103 Presence of currentStep element in build JSON indicates build is in progress.
106 build_data: A dictionary with build data, loaded from buildbot JSON API.
109 True if build is in progress, otherwise False.
111 current_step
= build_data
.get('currentStep')
112 if (current_step
and current_step
.get('isStarted') and
113 current_step
.get('results') is None):
118 def _IsBuildFailed(build_data
):
119 """Checks whether the build failed on buildbot.
121 Sometime build status is marked as failed even though compile and packaging
122 steps are successful. This may happen due to some intermediate steps of less
123 importance such as gclient revert, generate_telemetry_profile are failed.
124 Therefore we do an addition check to confirm if build was successful by
125 calling _IsBuildSuccessful.
128 build_data: A dictionary with build data, loaded from buildbot JSON API.
131 True if revision is failed build, otherwise False.
133 if (build_data
.get('results') in FAILED
and
134 not _IsBuildSuccessful(build_data
)):
139 def _IsBuildSuccessful(build_data
):
140 """Checks whether the build succeeded on buildbot.
142 We treat build as successful if the package_build step is completed without
143 any error i.e., when results attribute of the this step has value 0 or 1
144 in its first element.
147 build_data: A dictionary with build data, loaded from buildbot JSON API.
150 True if revision is successfully build, otherwise False.
152 if build_data
.get('steps'):
153 for item
in build_data
.get('steps'):
154 # The 'results' attribute of each step consists of two elements,
155 # results[0]: This represents the status of build step.
156 # See: http://docs.buildbot.net/current/developer/results.html
157 # results[1]: List of items, contains text if step fails, otherwise empty.
158 if (item
.get('name') == 'package_build' and
159 item
.get('isFinished') and
160 item
.get('results')[0] in OK
):
165 def _FetchBuilderData(builder_url
):
166 """Fetches JSON data for the all the builds from the tryserver.
169 builder_url: A tryserver URL to fetch builds information.
172 A dictionary with information of all build on the tryserver.
176 url
= urllib2
.urlopen(builder_url
)
177 except urllib2
.URLError
as e
:
178 print ('urllib2.urlopen error %s, waterfall status page down.[%s]' % (
179 builder_url
, str(e
)))
185 print 'urllib2 file object read error %s, [%s].' % (builder_url
, str(e
))
189 def _GetBuildData(buildbot_url
):
190 """Gets build information for the given build id from the tryserver.
193 buildbot_url: A tryserver URL to fetch build information.
196 A dictionary with build information if build exists, otherwise None.
198 builds_json
= _FetchBuilderData(buildbot_url
)
200 return json
.loads(builds_json
)
204 def _GetBuildBotUrl(builder_host
, builder_port
):
205 """Gets build bot URL for fetching build info.
207 Bisect builder bots are hosted on tryserver.chromium.perf, though we cannot
208 access this tryserver using host and port number directly, so we use another
209 tryserver URL for the perf tryserver.
212 builder_host: Hostname of the server where the builder is hosted.
213 builder_port: Port number of ther server where the builder is hosted.
216 URL of the buildbot as a string.
218 if (builder_host
== PERF_BISECT_BUILDER_HOST
and
219 builder_port
== PERF_BISECT_BUILDER_PORT
):
220 return PERF_TRY_SERVER_URL
222 return 'http://%s:%s' % (builder_host
, builder_port
)
225 def GetBuildStatus(build_num
, bot_name
, builder_host
, builder_port
):
226 """Gets build status from the buildbot status page for a given build number.
229 build_num: A build number on tryserver to determine its status.
230 bot_name: Name of the bot where the build information is scanned.
231 builder_host: Hostname of the server where the builder is hosted.
232 builder_port: Port number of the server where the builder is hosted.
235 A pair which consists of build status (SUCCESS, FAILED or PENDING) and a
236 link to build status page on the waterfall.
240 # Get the URL for requesting JSON data with status information.
241 server_url
= _GetBuildBotUrl(builder_host
, builder_port
)
242 buildbot_url
= BUILDER_JSON_URL
% {
243 'server_url': server_url
,
244 'bot_name': bot_name
,
245 'build_num': build_num
,
247 build_data
= _GetBuildData(buildbot_url
)
249 # Link to build on the buildbot showing status of build steps.
250 results_url
= BUILDER_HTML_URL
% {
251 'server_url': server_url
,
252 'bot_name': bot_name
,
253 'build_num': build_num
,
255 if _IsBuildFailed(build_data
):
256 return (FAILED
, results_url
)
258 elif _IsBuildSuccessful(build_data
):
259 return (OK
, results_url
)
260 return (PENDING
, results_url
)
263 def GetBuildNumFromBuilder(build_reason
, bot_name
, builder_host
, builder_port
):
264 """Gets build number on build status page for a given 'build reason'.
266 This function parses the JSON data from buildbot page and collects basic
267 information about the all the builds, and then uniquely identifies the build
268 based on the 'reason' attribute in builds's JSON data.
270 The 'reason' attribute set is when a build request is posted, and it is used
271 to identify the build on status page.
274 build_reason: A unique build name set to build on tryserver.
275 bot_name: Name of the bot where the build information is scanned.
276 builder_host: Hostname of the server where the builder is hosted.
277 builder_port: Port number of ther server where the builder is hosted.
280 A build number as a string if found, otherwise None.
282 # Gets the buildbot url for the given host and port.
283 server_url
= _GetBuildBotUrl(builder_host
, builder_port
)
284 buildbot_url
= BUILDER_JSON_URL
% {
285 'server_url': server_url
,
286 'bot_name': bot_name
,
289 builds_json
= _FetchBuilderData(buildbot_url
)
291 builds_data
= json
.loads(builds_json
)
292 for current_build
in builds_data
:
293 if builds_data
[current_build
].get('reason') == build_reason
:
294 return builds_data
[current_build
].get('number')
298 def _GetRequestParams(options
):
299 """Extracts request parameters which will be passed to PostTryJob.
302 options: The options object parsed from the command line.
305 A dictionary with parameters to pass to PostTryJob.
308 'user': options
.user
,
309 'name': options
.name
,
311 # Add other parameters if they're available in the options object.
312 for key
in ['email', 'revision', 'root', 'bot', 'patch']:
313 option
= getattr(options
, key
)
320 """Returns a parser for getting command line arguments."""
321 usage
= ('%prog [options]\n'
322 'Post a build request to the try server for the given revision.')
323 parser
= optparse
.OptionParser(usage
=usage
)
324 parser
.add_option('-H', '--host',
325 help='Host address of the try server (required).')
326 parser
.add_option('-P', '--port', type='int',
327 help='HTTP port of the try server (required).')
328 parser
.add_option('-u', '--user', default
=getpass
.getuser(),
330 help='Owner user name [default: %default]')
331 parser
.add_option('-e', '--email',
332 default
=os
.environ
.get('TRYBOT_RESULTS_EMAIL_ADDRESS',
333 os
.environ
.get('EMAIL_ADDRESS')),
334 help=('Email address where to send the results. Use either '
335 'the TRYBOT_RESULTS_EMAIL_ADDRESS environment '
336 'variable or EMAIL_ADDRESS to set the email address '
337 'the try bots report results to [default: %default]'))
338 parser
.add_option('-n', '--name',
339 default
='try_job_http',
340 help='Descriptive name of the try job')
341 parser
.add_option('-b', '--bot',
342 help=('Only one builder per run may be specified; to post '
343 'jobs on on multiple builders, run script for each '
344 'builder separately.'))
345 parser
.add_option('-r', '--revision',
346 help=('Revision to use for the try job. The revision may '
347 'be determined by the try server; see its waterfall '
349 parser
.add_option('--root',
350 help=('Root to use for the patch; base subdirectory for '
351 'patch created in a subdirectory.'))
352 parser
.add_option('--patch',
353 help='Patch information.')
358 """Posts a try job based on command line parameters."""
359 parser
= _GenParser()
360 options
, _
= parser
.parse_args()
361 if not options
.host
or not options
.port
:
364 params
= _GetRequestParams(options
)
365 PostTryJob(options
.host
, options
.port
, params
)
368 if __name__
== '__main__':
369 sys
.exit(Main(sys
.argv
))