3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file.
7 """Updates the Chrome reference builds.
9 To update Chromium (unofficial) reference build, use the -r option and give a
10 Chromium SVN revision number, like 228977. To update a Chrome (official) build,
11 use the -v option and give a Chrome version number like 30.0.1595.0.
13 If you're getting a Chrome build, you can give the flag --gs to fetch from
14 Google Storage. Otherwise, it will be fetched from go/chrome_official_builds.
16 Before running this script, you should first verify that you are authenticated
17 for SVN. You can do this by running:
18 $ svn ls svn://svn.chromium.org/chrome/trunk/deps/reference_builds
19 You may need to get your SVN password from https://chromium-access.appspot.com/.
23 $ /path/to/update_reference_build.py --gs -v <version>
24 $ cd reference_builds/reference_builds
41 # Example chromium build location:
42 # gs://chromium-browser-snapshots/Linux/228977/chrome-linux.zip
43 CHROMIUM_URL_FMT
= ('http://commondatastorage.googleapis.com/'
44 'chromium-browser-snapshots/%s/%s/%s')
46 # Chrome official build storage
47 # https://wiki.corp.google.com/twiki/bin/view/Main/ChromeOfficialBuilds
49 # Internal Google archive of official Chrome builds, example:
50 # https://goto.google.com/chrome_official_builds/
51 # 32.0.1677.0/precise32bit/chrome-precise32bit.zip
52 CHROME_INTERNAL_URL_FMT
= ('http://master.chrome.corp.google.com/'
53 'official_builds/%s/%s/%s')
55 # Google storage location (no public web URL's), example:
56 # gs://chrome-archive/30/30.0.1595.0/precise32bit/chrome-precise32bit.zip
57 CHROME_GS_URL_FMT
= ('gs://chrome-archive/%s/%s/%s/%s')
60 class BuildUpdater(object):
61 _PLATFORM_FILES_MAP
= {
64 'chrome-win32-syms.zip',
77 _CHROME_PLATFORM_FILES_MAP
= {
80 'chrome-win32-syms.zip',
86 'chrome-precise32bit.zip',
89 'chrome-precise64bit.zip',
93 # Map of platform names to gs:// Chrome build names.
94 _BUILD_PLATFORM_MAP
= {
95 'Linux': 'precise32bit',
96 'Linux_x64': 'precise64bit',
101 _PLATFORM_DEST_MAP
= {
102 'Linux': 'chrome_linux',
103 'Linux_x64': 'chrome_linux64',
108 def __init__(self
, options
):
109 self
._platforms
= options
.platforms
.split(',')
110 self
._version
_or
_revision
= options
.version
or int(options
.revision
)
111 self
._use
_official
_version
= bool(options
.version
)
112 self
._use
_gs
= options
.use_gs
115 def _GetCmdStatusAndOutput(args
, cwd
=None, shell
=False):
116 """Executes a subprocess and returns its exit code and output.
119 args: A string or a sequence of program arguments.
120 cwd: If not None, the subprocess's current directory will be changed to
121 |cwd| before it's executed.
122 shell: Whether to execute args as a shell command.
125 The tuple (exit code, output).
127 logging
.info(str(args
) + ' ' + (cwd
or ''))
128 p
= subprocess
.Popen(args
=args
, cwd
=cwd
, stdout
=subprocess
.PIPE
,
129 stderr
=subprocess
.PIPE
, shell
=shell
)
130 stdout
, stderr
= p
.communicate()
131 exit_code
= p
.returncode
133 logging
.critical(stderr
)
135 return (exit_code
, stdout
)
137 def _GetBuildUrl(self
, platform
, version_or_revision
, filename
):
138 """Returns the URL for fetching one file.
141 platform: Platform name, must be a key in |self._BUILD_PLATFORM_MAP|.
142 version_or_revision: Either an SVN revision, e.g. 234567, or a Chrome
143 version number, e.g. 30.0.1600.1.
144 filename: Name of the file to fetch.
147 The URL for fetching a file. This may be a GS or HTTP URL.
149 if self
._use
_official
_version
:
150 # Chrome Google storage bucket.
151 version
= version_or_revision
153 release
= version
[:version
.find('.')]
154 return (CHROME_GS_URL_FMT
% (
157 self
._BUILD
_PLATFORM
_MAP
[platform
],
159 # Chrome internal archive.
160 return (CHROME_INTERNAL_URL_FMT
% (
162 self
._BUILD
_PLATFORM
_MAP
[platform
],
165 revision
= version_or_revision
166 return CHROMIUM_URL_FMT
% (urllib
.quote_plus(platform
), revision
, filename
)
168 def _FindBuildVersionOrRevision(
169 self
, platform
, version_or_revision
, filename
):
170 """Searches for a version or revision where a filename can be found.
173 platform: Platform name.
174 version_or_revision: Either Chrome version or Chromium revision.
175 filename: Filename to look for.
178 A version or revision where the file could be found, or None.
180 # TODO(shadi): Iterate over official versions to find a valid one.
181 if self
._use
_official
_version
:
182 version
= version_or_revision
184 if self
._DoesBuildExist
(platform
, version
, filename
) else None)
186 revision
= version_or_revision
187 MAX_REVISIONS_PER_BUILD
= 100
188 for revision_guess
in xrange(revision
, revision
+ MAX_REVISIONS_PER_BUILD
):
189 if self
._DoesBuildExist
(platform
, revision_guess
, filename
):
190 return revision_guess
195 def _DoesBuildExist(self
, platform
, version
, filename
):
196 """Checks whether a file can be found for the given Chrome version.
199 platform: Platform name.
200 version: Chrome version number, e.g. 30.0.1600.1.
201 filename: Filename to look for.
204 True if the file could be found, False otherwise.
206 url
= self
._GetBuildUrl
(platform
, version
, filename
)
208 return self
._DoesGSFileExist
(url
)
210 request
= urllib2
.Request(url
)
211 request
.get_method
= lambda: 'HEAD'
213 urllib2
.urlopen(request
)
215 except urllib2
.HTTPError
, err
:
219 def _DoesGSFileExist(self
, gs_file_name
):
220 """Returns True if the GS file can be found, False otherwise."""
221 exit_code
= BuildUpdater
._GetCmdStatusAndOutput
(
222 ['gsutil', 'ls', gs_file_name
])[0]
225 def _GetPlatformFiles(self
, platform
):
226 """Returns a list of filenames to fetch for the given platform."""
227 if self
._use
_official
_version
:
228 return BuildUpdater
._CHROME
_PLATFORM
_FILES
_MAP
[platform
]
229 return BuildUpdater
._PLATFORM
_FILES
_MAP
[platform
]
231 def _DownloadBuilds(self
):
232 for platform
in self
._platforms
:
233 for filename
in self
._GetPlatformFiles
(platform
):
234 output
= os
.path
.join('dl', platform
,
235 '%s_%s_%s' % (platform
,
236 self
._version
_or
_revision
,
238 if os
.path
.exists(output
):
239 logging
.info('%s alread exists, skipping download', output
)
241 version_or_revision
= self
._FindBuildVersionOrRevision
(
242 platform
, self
._version
_or
_revision
, filename
)
243 if not version_or_revision
:
244 logging
.critical('Failed to find %s build for r%s\n', platform
,
245 self
._version
_or
_revision
)
247 dirname
= os
.path
.dirname(output
)
248 if dirname
and not os
.path
.exists(dirname
):
250 url
= self
._GetBuildUrl
(platform
, version_or_revision
, filename
)
251 self
._DownloadFile
(url
, output
)
253 def _DownloadFile(self
, url
, output
):
254 logging
.info('Downloading %s, saving to %s', url
, output
)
255 if self
._use
_official
_version
and self
._use
_gs
:
256 BuildUpdater
._GetCmdStatusAndOutput
(['gsutil', 'cp', url
, output
])
258 response
= urllib2
.urlopen(url
)
259 with
file(output
, 'wb') as f
:
260 f
.write(response
.read())
262 def _FetchSvnRepos(self
):
263 if not os
.path
.exists('reference_builds'):
264 os
.makedirs('reference_builds')
265 BuildUpdater
._GetCmdStatusAndOutput
(
266 ['gclient', 'config',
267 'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
269 BuildUpdater
._GetCmdStatusAndOutput
(
270 ['gclient', 'sync'], 'reference_builds')
272 def _UnzipFile(self
, dl_file
, dest_dir
):
273 """Unzips a file if it is a zip file.
276 dl_file: The downloaded file to unzip.
277 dest_dir: The destination directory to unzip to.
280 True if the file was unzipped. False if it wasn't a zip file.
282 if not zipfile
.is_zipfile(dl_file
):
284 logging
.info('Opening %s', dl_file
)
285 with zipfile
.ZipFile(dl_file
, 'r') as z
:
286 for content
in z
.namelist():
287 dest
= os
.path
.join(dest_dir
, content
[content
.find('/')+1:])
288 # Create dest parent dir if it does not exist.
289 if not os
.path
.isdir(os
.path
.dirname(dest
)):
290 os
.makedirs(os
.path
.dirname(dest
))
291 # If dest is just a dir listing, do nothing.
292 if not os
.path
.basename(dest
):
294 if not os
.path
.isdir(os
.path
.dirname(dest
)):
295 os
.makedirs(os
.path
.dirname(dest
))
296 with z
.open(content
) as unzipped_content
:
297 logging
.info('Extracting %s to %s (%s)', content
, dest
, dl_file
)
298 with
file(dest
, 'wb') as dest_file
:
299 dest_file
.write(unzipped_content
.read())
300 permissions
= z
.getinfo(content
).external_attr
>> 16
302 os
.chmod(dest
, permissions
)
305 def _ClearDir(self
, dir):
306 """Clears all files in |dir| except for hidden files and folders."""
307 for root
, dirs
, files
in os
.walk(dir):
308 # Skip hidden files and folders (like .svn and .git).
309 files
= [f
for f
in files
if f
[0] != '.']
310 dirs
[:] = [d
for d
in dirs
if d
[0] != '.']
313 os
.remove(os
.path
.join(root
, f
))
315 def _ExtractBuilds(self
):
316 for platform
in self
._platforms
:
317 if os
.path
.exists('tmp_unzip'):
318 os
.path
.unlink('tmp_unzip')
319 dest_dir
= os
.path
.join('reference_builds', 'reference_builds',
320 BuildUpdater
._PLATFORM
_DEST
_MAP
[platform
])
321 self
._ClearDir
(dest_dir
)
322 for root
, _
, dl_files
in os
.walk(os
.path
.join('dl', platform
)):
323 for dl_file
in dl_files
:
324 dl_file
= os
.path
.join(root
, dl_file
)
325 if not self
._UnzipFile
(dl_file
, dest_dir
):
326 logging
.info('Copying %s to %s', dl_file
, dest_dir
)
327 shutil
.copy(dl_file
, dest_dir
)
329 def _SvnAddAndRemove(self
):
330 svn_dir
= os
.path
.join('reference_builds', 'reference_builds')
331 # List all changes without ignoring any files.
332 stat
= BuildUpdater
._GetCmdStatusAndOutput
(['svn', 'stat', '--no-ignore'],
334 for line
in stat
.splitlines():
335 action
, filename
= line
.split(None, 1)
336 # Add new and ignored files.
337 if action
== '?' or action
== 'I':
338 BuildUpdater
._GetCmdStatusAndOutput
(
339 ['svn', 'add', filename
], svn_dir
)
341 BuildUpdater
._GetCmdStatusAndOutput
(
342 ['svn', 'delete', filename
], svn_dir
)
343 filepath
= os
.path
.join(svn_dir
, filename
)
344 if not os
.path
.isdir(filepath
) and os
.access(filepath
, os
.X_OK
):
345 BuildUpdater
._GetCmdStatusAndOutput
(
346 ['svn', 'propset', 'svn:executable', 'true', filename
], svn_dir
)
348 def DownloadAndUpdateBuilds(self
):
349 self
._DownloadBuilds
()
350 self
._FetchSvnRepos
()
351 self
._ExtractBuilds
()
352 self
._SvnAddAndRemove
()
355 def ParseOptions(argv
):
356 parser
= optparse
.OptionParser()
357 usage
= 'usage: %prog <options>'
358 parser
.set_usage(usage
)
359 parser
.add_option('-v', dest
='version',
360 help='Chrome official version to pick up '
361 '(e.g. 30.0.1600.1).')
362 parser
.add_option('--gs', dest
='use_gs', action
='store_true', default
=False,
363 help='Use Google storage for official builds. Used with -b '
364 'option. Default is false (i.e. use internal storage.')
365 parser
.add_option('-p', dest
='platforms',
366 default
='Win,Mac,Linux,Linux_x64',
367 help='Comma separated list of platforms to download '
368 '(as defined by the chromium builders).')
369 parser
.add_option('-r', dest
='revision',
370 help='Chromium revision to pick up (e.g. 234567).')
372 (options
, _
) = parser
.parse_args(argv
)
373 if not options
.revision
and not options
.version
:
374 logging
.critical('Must specify either -r or -v.\n')
376 if options
.revision
and options
.version
:
377 logging
.critical('Must specify either -r or -v but not both.\n')
379 if options
.use_gs
and not options
.version
:
380 logging
.critical('Can only use --gs with -v option.\n')
387 logging
.getLogger().setLevel(logging
.DEBUG
)
388 options
= ParseOptions(argv
)
389 b
= BuildUpdater(options
)
390 b
.DownloadAndUpdateBuilds()
391 logging
.info('Successfully updated reference builds. Move to '
392 'reference_builds/reference_builds and make a change with gcl.')
394 if __name__
== '__main__':
395 sys
.exit(main(sys
.argv
))