GN: Make chrome/{browser,common,renderer} compile on mac
[chromium-blink-merge.git] / tools / update_reference_build.py
blob8224cd64fee0f35c56e04511d40421c8a47815d7
1 #!/usr/bin/env python
3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file.
7 """Updates the Chrome reference builds.
9 To update Chromium (unofficial) reference build, use the -r option and give a
10 Chromium SVN revision number, like 228977. To update a Chrome (official) build,
11 use the -v option and give a Chrome version number like 30.0.1595.0.
13 If you're getting a Chrome build, you can give the flag --gs to fetch from
14 Google Storage. Otherwise, it will be fetched from go/chrome_official_builds.
16 Before running this script, you should first verify that you are authenticated
17 for SVN. You can do this by running:
18 $ svn ls svn://svn.chromium.org/chrome/trunk/deps/reference_builds
19 You may need to get your SVN password from https://chromium-access.appspot.com/.
21 Usage:
22 $ cd /tmp
23 $ /path/to/update_reference_build.py --gs -v <version>
24 $ cd reference_builds/reference_builds
25 $ gcl change
26 $ gcl upload <change>
27 $ gcl commit <change>
28 """
30 import logging
31 import optparse
32 import os
33 import shutil
34 import subprocess
35 import sys
36 import time
37 import urllib
38 import urllib2
39 import zipfile
41 # Example chromium build location:
42 # gs://chromium-browser-snapshots/Linux/228977/chrome-linux.zip
43 CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/'
44 'chromium-browser-snapshots/%s/%s/%s')
46 # Chrome official build storage
47 # https://wiki.corp.google.com/twiki/bin/view/Main/ChromeOfficialBuilds
49 # Internal Google archive of official Chrome builds, example:
50 # https://goto.google.com/chrome_official_builds/
51 # 32.0.1677.0/precise32bit/chrome-precise32bit.zip
52 CHROME_INTERNAL_URL_FMT = ('http://master.chrome.corp.google.com/'
53 'official_builds/%s/%s/%s')
55 # Google storage location (no public web URL's), example:
56 # gs://chrome-archive/30/30.0.1595.0/precise32bit/chrome-precise32bit.zip
57 CHROME_GS_URL_FMT = ('gs://chrome-archive/%s/%s/%s/%s')
60 class BuildUpdater(object):
61 _PLATFORM_FILES_MAP = {
62 'Win': [
63 'chrome-win32.zip',
64 'chrome-win32-syms.zip',
66 'Mac': [
67 'chrome-mac.zip',
69 'Linux': [
70 'chrome-linux.zip',
72 'Linux_x64': [
73 'chrome-linux.zip',
77 _CHROME_PLATFORM_FILES_MAP = {
78 'Win': [
79 'chrome-win32.zip',
80 'chrome-win32-syms.zip',
82 'Mac': [
83 'chrome-mac.zip',
85 'Linux': [
86 'chrome-precise32bit.zip',
88 'Linux_x64': [
89 'chrome-precise64bit.zip',
93 # Map of platform names to gs:// Chrome build names.
94 _BUILD_PLATFORM_MAP = {
95 'Linux': 'precise32bit',
96 'Linux_x64': 'precise64bit',
97 'Win': 'win',
98 'Mac': 'mac',
101 _PLATFORM_DEST_MAP = {
102 'Linux': 'chrome_linux',
103 'Linux_x64': 'chrome_linux64',
104 'Win': 'chrome_win',
105 'Mac': 'chrome_mac',
108 def __init__(self, options):
109 self._platforms = options.platforms.split(',')
110 self._version_or_revision = options.version or int(options.revision)
111 self._use_official_version = bool(options.version)
112 self._use_gs = options.use_gs
114 @staticmethod
115 def _GetCmdStatusAndOutput(args, cwd=None, shell=False):
116 """Executes a subprocess and returns its exit code and output.
118 Args:
119 args: A string or a sequence of program arguments.
120 cwd: If not None, the subprocess's current directory will be changed to
121 |cwd| before it's executed.
122 shell: Whether to execute args as a shell command.
124 Returns:
125 The tuple (exit code, output).
127 logging.info(str(args) + ' ' + (cwd or ''))
128 p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
129 stderr=subprocess.PIPE, shell=shell)
130 stdout, stderr = p.communicate()
131 exit_code = p.returncode
132 if stderr:
133 logging.critical(stderr)
134 logging.info(stdout)
135 return (exit_code, stdout)
137 def _GetBuildUrl(self, platform, version_or_revision, filename):
138 """Returns the URL for fetching one file.
140 Args:
141 platform: Platform name, must be a key in |self._BUILD_PLATFORM_MAP|.
142 version_or_revision: Either an SVN revision, e.g. 234567, or a Chrome
143 version number, e.g. 30.0.1600.1.
144 filename: Name of the file to fetch.
146 Returns:
147 The URL for fetching a file. This may be a GS or HTTP URL.
149 if self._use_official_version:
150 # Chrome Google storage bucket.
151 version = version_or_revision
152 if self._use_gs:
153 release = version[:version.find('.')]
154 return (CHROME_GS_URL_FMT % (
155 release,
156 version,
157 self._BUILD_PLATFORM_MAP[platform],
158 filename))
159 # Chrome internal archive.
160 return (CHROME_INTERNAL_URL_FMT % (
161 version,
162 self._BUILD_PLATFORM_MAP[platform],
163 filename))
164 # Chromium archive.
165 revision = version_or_revision
166 return CHROMIUM_URL_FMT % (urllib.quote_plus(platform), revision, filename)
168 def _FindBuildVersionOrRevision(
169 self, platform, version_or_revision, filename):
170 """Searches for a version or revision where a filename can be found.
172 Args:
173 platform: Platform name.
174 version_or_revision: Either Chrome version or Chromium revision.
175 filename: Filename to look for.
177 Returns:
178 A version or revision where the file could be found, or None.
180 # TODO(shadi): Iterate over official versions to find a valid one.
181 if self._use_official_version:
182 version = version_or_revision
183 return (version
184 if self._DoesBuildExist(platform, version, filename) else None)
186 revision = version_or_revision
187 MAX_REVISIONS_PER_BUILD = 100
188 for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD):
189 if self._DoesBuildExist(platform, revision_guess, filename):
190 return revision_guess
191 else:
192 time.sleep(.1)
193 return None
195 def _DoesBuildExist(self, platform, version, filename):
196 """Checks whether a file can be found for the given Chrome version.
198 Args:
199 platform: Platform name.
200 version: Chrome version number, e.g. 30.0.1600.1.
201 filename: Filename to look for.
203 Returns:
204 True if the file could be found, False otherwise.
206 url = self._GetBuildUrl(platform, version, filename)
207 if self._use_gs:
208 return self._DoesGSFileExist(url)
210 request = urllib2.Request(url)
211 request.get_method = lambda: 'HEAD'
212 try:
213 urllib2.urlopen(request)
214 return True
215 except urllib2.HTTPError, err:
216 if err.code == 404:
217 return False
219 def _DoesGSFileExist(self, gs_file_name):
220 """Returns True if the GS file can be found, False otherwise."""
221 exit_code = BuildUpdater._GetCmdStatusAndOutput(
222 ['gsutil', 'ls', gs_file_name])[0]
223 return not exit_code
225 def _GetPlatformFiles(self, platform):
226 """Returns a list of filenames to fetch for the given platform."""
227 if self._use_official_version:
228 return BuildUpdater._CHROME_PLATFORM_FILES_MAP[platform]
229 return BuildUpdater._PLATFORM_FILES_MAP[platform]
231 def _DownloadBuilds(self):
232 for platform in self._platforms:
233 for filename in self._GetPlatformFiles(platform):
234 output = os.path.join('dl', platform,
235 '%s_%s_%s' % (platform,
236 self._version_or_revision,
237 filename))
238 if os.path.exists(output):
239 logging.info('%s alread exists, skipping download', output)
240 continue
241 version_or_revision = self._FindBuildVersionOrRevision(
242 platform, self._version_or_revision, filename)
243 if not version_or_revision:
244 logging.critical('Failed to find %s build for r%s\n', platform,
245 self._version_or_revision)
246 sys.exit(1)
247 dirname = os.path.dirname(output)
248 if dirname and not os.path.exists(dirname):
249 os.makedirs(dirname)
250 url = self._GetBuildUrl(platform, version_or_revision, filename)
251 self._DownloadFile(url, output)
253 def _DownloadFile(self, url, output):
254 logging.info('Downloading %s, saving to %s', url, output)
255 if self._use_official_version and self._use_gs:
256 BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output])
257 else:
258 response = urllib2.urlopen(url)
259 with file(output, 'wb') as f:
260 f.write(response.read())
262 def _FetchSvnRepos(self):
263 if not os.path.exists('reference_builds'):
264 os.makedirs('reference_builds')
265 BuildUpdater._GetCmdStatusAndOutput(
266 ['gclient', 'config',
267 'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
268 'reference_builds')
269 BuildUpdater._GetCmdStatusAndOutput(
270 ['gclient', 'sync'], 'reference_builds')
272 def _UnzipFile(self, dl_file, dest_dir):
273 """Unzips a file if it is a zip file.
275 Args:
276 dl_file: The downloaded file to unzip.
277 dest_dir: The destination directory to unzip to.
279 Returns:
280 True if the file was unzipped. False if it wasn't a zip file.
282 if not zipfile.is_zipfile(dl_file):
283 return False
284 logging.info('Opening %s', dl_file)
285 with zipfile.ZipFile(dl_file, 'r') as z:
286 for content in z.namelist():
287 dest = os.path.join(dest_dir, content[content.find('/')+1:])
288 # Create dest parent dir if it does not exist.
289 if not os.path.isdir(os.path.dirname(dest)):
290 os.makedirs(os.path.dirname(dest))
291 # If dest is just a dir listing, do nothing.
292 if not os.path.basename(dest):
293 continue
294 if not os.path.isdir(os.path.dirname(dest)):
295 os.makedirs(os.path.dirname(dest))
296 with z.open(content) as unzipped_content:
297 logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
298 with file(dest, 'wb') as dest_file:
299 dest_file.write(unzipped_content.read())
300 permissions = z.getinfo(content).external_attr >> 16
301 if permissions:
302 os.chmod(dest, permissions)
303 return True
305 def _ClearDir(self, dir):
306 """Clears all files in |dir| except for hidden files and folders."""
307 for root, dirs, files in os.walk(dir):
308 # Skip hidden files and folders (like .svn and .git).
309 files = [f for f in files if f[0] != '.']
310 dirs[:] = [d for d in dirs if d[0] != '.']
312 for f in files:
313 os.remove(os.path.join(root, f))
315 def _ExtractBuilds(self):
316 for platform in self._platforms:
317 if os.path.exists('tmp_unzip'):
318 os.path.unlink('tmp_unzip')
319 dest_dir = os.path.join('reference_builds', 'reference_builds',
320 BuildUpdater._PLATFORM_DEST_MAP[platform])
321 self._ClearDir(dest_dir)
322 for root, _, dl_files in os.walk(os.path.join('dl', platform)):
323 for dl_file in dl_files:
324 dl_file = os.path.join(root, dl_file)
325 if not self._UnzipFile(dl_file, dest_dir):
326 logging.info('Copying %s to %s', dl_file, dest_dir)
327 shutil.copy(dl_file, dest_dir)
329 def _SvnAddAndRemove(self):
330 svn_dir = os.path.join('reference_builds', 'reference_builds')
331 # List all changes without ignoring any files.
332 stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'],
333 svn_dir)[1]
334 for line in stat.splitlines():
335 action, filename = line.split(None, 1)
336 # Add new and ignored files.
337 if action == '?' or action == 'I':
338 BuildUpdater._GetCmdStatusAndOutput(
339 ['svn', 'add', filename], svn_dir)
340 elif action == '!':
341 BuildUpdater._GetCmdStatusAndOutput(
342 ['svn', 'delete', filename], svn_dir)
343 filepath = os.path.join(svn_dir, filename)
344 if not os.path.isdir(filepath) and os.access(filepath, os.X_OK):
345 BuildUpdater._GetCmdStatusAndOutput(
346 ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir)
348 def DownloadAndUpdateBuilds(self):
349 self._DownloadBuilds()
350 self._FetchSvnRepos()
351 self._ExtractBuilds()
352 self._SvnAddAndRemove()
355 def ParseOptions(argv):
356 parser = optparse.OptionParser()
357 usage = 'usage: %prog <options>'
358 parser.set_usage(usage)
359 parser.add_option('-v', dest='version',
360 help='Chrome official version to pick up '
361 '(e.g. 30.0.1600.1).')
362 parser.add_option('--gs', dest='use_gs', action='store_true', default=False,
363 help='Use Google storage for official builds. Used with -b '
364 'option. Default is false (i.e. use internal storage.')
365 parser.add_option('-p', dest='platforms',
366 default='Win,Mac,Linux,Linux_x64',
367 help='Comma separated list of platforms to download '
368 '(as defined by the chromium builders).')
369 parser.add_option('-r', dest='revision',
370 help='Chromium revision to pick up (e.g. 234567).')
372 (options, _) = parser.parse_args(argv)
373 if not options.revision and not options.version:
374 logging.critical('Must specify either -r or -v.\n')
375 sys.exit(1)
376 if options.revision and options.version:
377 logging.critical('Must specify either -r or -v but not both.\n')
378 sys.exit(1)
379 if options.use_gs and not options.version:
380 logging.critical('Can only use --gs with -v option.\n')
381 sys.exit(1)
383 return options
386 def main(argv):
387 logging.getLogger().setLevel(logging.DEBUG)
388 options = ParseOptions(argv)
389 b = BuildUpdater(options)
390 b.DownloadAndUpdateBuilds()
391 logging.info('Successfully updated reference builds. Move to '
392 'reference_builds/reference_builds and make a change with gcl.')
394 if __name__ == '__main__':
395 sys.exit(main(sys.argv))