tools/update_reference_build.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Updates the Chrome reference builds.
   8
   9 To update Chromium (unofficial) reference build, use the -r option and give a
  10 Chromium SVN revision number, like 228977. To update a Chrome (official) build,
  11 use the -v option and give a Chrome version number like 30.0.1595.0.
  12
  13 If you're getting a Chrome build, you can give the flag --gs to fetch from
  14 Google Storage. Otherwise, it will be fetched from go/chrome_official_builds.
  15
  16 Before running this script, you should first verify that you are authenticated
  17 for SVN. You can do this by running:
  18   $ svn ls svn://svn.chromium.org/chrome/trunk/deps/reference_builds
  19 You may need to get your SVN password from https://chromium-access.appspot.com/.
  20
  21 Usage:
  22   $ cd /tmp
  23   $ /path/to/update_reference_build.py --gs -v <version>
  24   $ cd reference_builds/reference_builds
  25   $ gcl change
  26   $ gcl upload <change>
  27   $ gcl commit <change>
  28 """
  29
  30 import logging
  31 import optparse
  32 import os
  33 import shutil
  34 import subprocess
  35 import sys
  36 import time
  37 import urllib
  38 import urllib2
  39 import zipfile
  40
  41 # Example chromium build location:
  42 # gs://chromium-browser-snapshots/Linux/228977/chrome-linux.zip
  43 CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/'
  44                     'chromium-browser-snapshots/%s/%s/%s')
  45
  46 # Chrome official build storage
  47 # https://wiki.corp.google.com/twiki/bin/view/Main/ChromeOfficialBuilds
  48
  49 # Internal Google archive of official Chrome builds, example:
  50 # https://goto.google.com/chrome_official_builds/
  51 # 32.0.1677.0/precise32bit/chrome-precise32bit.zip
  52 CHROME_INTERNAL_URL_FMT = ('http://master.chrome.corp.google.com/'
  53                            'official_builds/%s/%s/%s')
  54
  55 # Google storage location (no public web URL's), example:
  56 # gs://chrome-archive/30/30.0.1595.0/precise32bit/chrome-precise32bit.zip
  57 CHROME_GS_URL_FMT = ('gs://chrome-archive/%s/%s/%s/%s')
  58
  59
  60 class BuildUpdater(object):
  61   _PLATFORM_FILES_MAP = {
  62       'Win': [
  63           'chrome-win32.zip',
  64           'chrome-win32-syms.zip',
  65       ],
  66       'Mac': [
  67           'chrome-mac.zip',
  68       ],
  69       'Linux': [
  70           'chrome-linux.zip',
  71       ],
  72       'Linux_x64': [
  73           'chrome-linux.zip',
  74       ],
  75   }
  76
  77   _CHROME_PLATFORM_FILES_MAP = {
  78       'Win': [
  79           'chrome-win32.zip',
  80           'chrome-win32-syms.zip',
  81       ],
  82       'Mac': [
  83           'chrome-mac.zip',
  84       ],
  85       'Linux': [
  86           'chrome-precise32bit.zip',
  87       ],
  88       'Linux_x64': [
  89           'chrome-precise64bit.zip',
  90       ],
  91   }
  92
  93   # Map of platform names to gs:// Chrome build names.
  94   _BUILD_PLATFORM_MAP = {
  95       'Linux': 'precise32bit',
  96       'Linux_x64': 'precise64bit',
  97       'Win': 'win',
  98       'Mac': 'mac',
  99   }
 100
 101   _PLATFORM_DEST_MAP = {
 102       'Linux': 'chrome_linux',
 103       'Linux_x64': 'chrome_linux64',
 104       'Win': 'chrome_win',
 105       'Mac': 'chrome_mac',
 106   }
 107
 108   def __init__(self, options):
 109     self._platforms = options.platforms.split(',')
 110     self._version_or_revision = options.version or int(options.revision)
 111     self._use_official_version = bool(options.version)
 112     self._use_gs = options.use_gs
 113
 114   @staticmethod
 115   def _GetCmdStatusAndOutput(args, cwd=None, shell=False):
 116     """Executes a subprocess and returns its exit code and output.
 117
 118     Args:
 119       args: A string or a sequence of program arguments.
 120       cwd: If not None, the subprocess's current directory will be changed to
 121         |cwd| before it's executed.
 122       shell: Whether to execute args as a shell command.
 123
 124     Returns:
 125       The tuple (exit code, output).
 126     """
 127     logging.info(str(args) + ' ' + (cwd or ''))
 128     p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
 129                          stderr=subprocess.PIPE, shell=shell)
 130     stdout, stderr = p.communicate()
 131     exit_code = p.returncode
 132     if stderr:
 133       logging.critical(stderr)
 134     logging.info(stdout)
 135     return (exit_code, stdout)
 136
 137   def _GetBuildUrl(self, platform, version_or_revision, filename):
 138     """Returns the URL for fetching one file.
 139
 140     Args:
 141       platform: Platform name, must be a key in |self._BUILD_PLATFORM_MAP|.
 142       version_or_revision: Either an SVN revision, e.g. 234567, or a Chrome
 143           version number, e.g. 30.0.1600.1.
 144       filename: Name of the file to fetch.
 145
 146     Returns:
 147       The URL for fetching a file. This may be a GS or HTTP URL.
 148     """
 149     if self._use_official_version:
 150       # Chrome Google storage bucket.
 151       version = version_or_revision
 152       if self._use_gs:
 153         release = version[:version.find('.')]
 154         return (CHROME_GS_URL_FMT % (
 155             release,
 156             version,
 157             self._BUILD_PLATFORM_MAP[platform],
 158             filename))
 159       # Chrome internal archive.
 160       return (CHROME_INTERNAL_URL_FMT % (
 161           version,
 162           self._BUILD_PLATFORM_MAP[platform],
 163           filename))
 164     # Chromium archive.
 165     revision = version_or_revision
 166     return CHROMIUM_URL_FMT % (urllib.quote_plus(platform), revision, filename)
 167
 168   def _FindBuildVersionOrRevision(
 169       self, platform, version_or_revision, filename):
 170     """Searches for a version or revision where a filename can be found.
 171
 172     Args:
 173       platform: Platform name.
 174       version_or_revision: Either Chrome version or Chromium revision.
 175       filename: Filename to look for.
 176
 177     Returns:
 178       A version or revision where the file could be found, or None.
 179     """
 180     # TODO(shadi): Iterate over official versions to find a valid one.
 181     if self._use_official_version:
 182       version = version_or_revision
 183       return (version
 184               if self._DoesBuildExist(platform, version, filename) else None)
 185
 186     revision = version_or_revision
 187     MAX_REVISIONS_PER_BUILD = 100
 188     for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD):
 189       if self._DoesBuildExist(platform, revision_guess, filename):
 190         return revision_guess
 191       else:
 192         time.sleep(.1)
 193     return None
 194
 195   def _DoesBuildExist(self, platform, version, filename):
 196     """Checks whether a file can be found for the given Chrome version.
 197
 198     Args:
 199       platform: Platform name.
 200       version: Chrome version number, e.g. 30.0.1600.1.
 201       filename: Filename to look for.
 202
 203     Returns:
 204       True if the file could be found, False otherwise.
 205     """
 206     url = self._GetBuildUrl(platform, version, filename)
 207     if self._use_gs:
 208       return self._DoesGSFileExist(url)
 209
 210     request = urllib2.Request(url)
 211     request.get_method = lambda: 'HEAD'
 212     try:
 213       urllib2.urlopen(request)
 214       return True
 215     except urllib2.HTTPError, err:
 216       if err.code == 404:
 217         return False
 218
 219   def _DoesGSFileExist(self, gs_file_name):
 220     """Returns True if the GS file can be found, False otherwise."""
 221     exit_code = BuildUpdater._GetCmdStatusAndOutput(
 222         ['gsutil', 'ls', gs_file_name])[0]
 223     return not exit_code
 224
 225   def _GetPlatformFiles(self, platform):
 226     """Returns a list of filenames to fetch for the given platform."""
 227     if self._use_official_version:
 228       return BuildUpdater._CHROME_PLATFORM_FILES_MAP[platform]
 229     return BuildUpdater._PLATFORM_FILES_MAP[platform]
 230
 231   def _DownloadBuilds(self):
 232     for platform in self._platforms:
 233       for filename in self._GetPlatformFiles(platform):
 234         output = os.path.join('dl', platform,
 235                               '%s_%s_%s' % (platform,
 236                                             self._version_or_revision,
 237                                             filename))
 238         if os.path.exists(output):
 239           logging.info('%s alread exists, skipping download', output)
 240           continue
 241         version_or_revision = self._FindBuildVersionOrRevision(
 242             platform, self._version_or_revision, filename)
 243         if not version_or_revision:
 244           logging.critical('Failed to find %s build for r%s\n', platform,
 245                            self._version_or_revision)
 246           sys.exit(1)
 247         dirname = os.path.dirname(output)
 248         if dirname and not os.path.exists(dirname):
 249           os.makedirs(dirname)
 250         url = self._GetBuildUrl(platform, version_or_revision, filename)
 251         self._DownloadFile(url, output)
 252
 253   def _DownloadFile(self, url, output):
 254     logging.info('Downloading %s, saving to %s', url, output)
 255     if self._use_official_version and self._use_gs:
 256       BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output])
 257     else:
 258       response = urllib2.urlopen(url)
 259       with file(output, 'wb') as f:
 260         f.write(response.read())
 261
 262   def _FetchSvnRepos(self):
 263     if not os.path.exists('reference_builds'):
 264       os.makedirs('reference_builds')
 265     BuildUpdater._GetCmdStatusAndOutput(
 266         ['gclient', 'config',
 267          'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
 268         'reference_builds')
 269     BuildUpdater._GetCmdStatusAndOutput(
 270         ['gclient', 'sync'], 'reference_builds')
 271
 272   def _UnzipFile(self, dl_file, dest_dir):
 273     """Unzips a file if it is a zip file.
 274
 275     Args:
 276       dl_file: The downloaded file to unzip.
 277       dest_dir: The destination directory to unzip to.
 278
 279     Returns:
 280       True if the file was unzipped. False if it wasn't a zip file.
 281     """
 282     if not zipfile.is_zipfile(dl_file):
 283       return False
 284     logging.info('Opening %s', dl_file)
 285     with zipfile.ZipFile(dl_file, 'r') as z:
 286       for content in z.namelist():
 287         dest = os.path.join(dest_dir, content[content.find('/')+1:])
 288         # Create dest parent dir if it does not exist.
 289         if not os.path.isdir(os.path.dirname(dest)):
 290           os.makedirs(os.path.dirname(dest))
 291         # If dest is just a dir listing, do nothing.
 292         if not os.path.basename(dest):
 293           continue
 294         if not os.path.isdir(os.path.dirname(dest)):
 295           os.makedirs(os.path.dirname(dest))
 296         with z.open(content) as unzipped_content:
 297           logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
 298           with file(dest, 'wb') as dest_file:
 299             dest_file.write(unzipped_content.read())
 300           permissions = z.getinfo(content).external_attr >> 16
 301           if permissions:
 302             os.chmod(dest, permissions)
 303     return True
 304
 305   def _ClearDir(self, dir):
 306     """Clears all files in |dir| except for hidden files and folders."""
 307     for root, dirs, files in os.walk(dir):
 308       # Skip hidden files and folders (like .svn and .git).
 309       files = [f for f in files if f[0] != '.']
 310       dirs[:] = [d for d in dirs if d[0] != '.']
 311
 312       for f in files:
 313         os.remove(os.path.join(root, f))
 314
 315   def _ExtractBuilds(self):
 316     for platform in self._platforms:
 317       if os.path.exists('tmp_unzip'):
 318         os.path.unlink('tmp_unzip')
 319       dest_dir = os.path.join('reference_builds', 'reference_builds',
 320                               BuildUpdater._PLATFORM_DEST_MAP[platform])
 321       self._ClearDir(dest_dir)
 322       for root, _, dl_files in os.walk(os.path.join('dl', platform)):
 323         for dl_file in dl_files:
 324           dl_file = os.path.join(root, dl_file)
 325           if not self._UnzipFile(dl_file, dest_dir):
 326             logging.info('Copying %s to %s', dl_file, dest_dir)
 327             shutil.copy(dl_file, dest_dir)
 328
 329   def _SvnAddAndRemove(self):
 330     svn_dir = os.path.join('reference_builds', 'reference_builds')
 331     # List all changes without ignoring any files.
 332     stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'],
 333                                                svn_dir)[1]
 334     for line in stat.splitlines():
 335       action, filename = line.split(None, 1)
 336       # Add new and ignored files.
 337       if action == '?' or action == 'I':
 338         BuildUpdater._GetCmdStatusAndOutput(
 339             ['svn', 'add', filename], svn_dir)
 340       elif action == '!':
 341         BuildUpdater._GetCmdStatusAndOutput(
 342             ['svn', 'delete', filename], svn_dir)
 343       filepath = os.path.join(svn_dir, filename)
 344       if not os.path.isdir(filepath) and os.access(filepath, os.X_OK):
 345         BuildUpdater._GetCmdStatusAndOutput(
 346             ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir)
 347
 348   def DownloadAndUpdateBuilds(self):
 349     self._DownloadBuilds()
 350     self._FetchSvnRepos()
 351     self._ExtractBuilds()
 352     self._SvnAddAndRemove()
 353
 354
 355 def ParseOptions(argv):
 356   parser = optparse.OptionParser()
 357   usage = 'usage: %prog <options>'
 358   parser.set_usage(usage)
 359   parser.add_option('-v', dest='version',
 360                     help='Chrome official version to pick up '
 361                          '(e.g. 30.0.1600.1).')
 362   parser.add_option('--gs', dest='use_gs', action='store_true', default=False,
 363                     help='Use Google storage for official builds. Used with -b '
 364                          'option. Default is false (i.e. use internal storage.')
 365   parser.add_option('-p', dest='platforms',
 366                     default='Win,Mac,Linux,Linux_x64',
 367                     help='Comma separated list of platforms to download '
 368                          '(as defined by the chromium builders).')
 369   parser.add_option('-r', dest='revision',
 370                     help='Chromium revision to pick up (e.g. 234567).')
 371
 372   (options, _) = parser.parse_args(argv)
 373   if not options.revision and not options.version:
 374     logging.critical('Must specify either -r or -v.\n')
 375     sys.exit(1)
 376   if options.revision and options.version:
 377     logging.critical('Must specify either -r or -v but not both.\n')
 378     sys.exit(1)
 379   if options.use_gs and not options.version:
 380     logging.critical('Can only use --gs with -v option.\n')
 381     sys.exit(1)
 382
 383   return options
 384
 385
 386 def main(argv):
 387   logging.getLogger().setLevel(logging.DEBUG)
 388   options = ParseOptions(argv)
 389   b = BuildUpdater(options)
 390   b.DownloadAndUpdateBuilds()
 391   logging.info('Successfully updated reference builds. Move to '
 392                'reference_builds/reference_builds and make a change with gcl.')
 393
 394 if __name__ == '__main__':
 395   sys.exit(main(sys.argv))