tools/perf/update_reference_build.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 """Updates the Chrome reference builds.
   8
   9 Before running this script, you should first verify that you are authenticated
  10 for SVN. You can do this by running:
  11   $ svn ls svn://svn.chromium.org/chrome/trunk/deps/reference_builds
  12 You may need to get your SVN password from https://chromium-access.appspot.com/.
  13
  14 Usage:
  15   $ cd /tmp
  16   $ /path/to/update_reference_build.py
  17   $ cd reference_builds/reference_builds
  18   $ gcl change
  19   $ gcl upload <change>
  20   $ gcl commit <change>
  21 """
  22
  23 import collections
  24 import logging
  25 import os
  26 import shutil
  27 import subprocess
  28 import sys
  29 import urllib2
  30 import zipfile
  31
  32
  33 # Google storage location (no public web URL's), example:
  34 # gs://chrome-unsigned/desktop-*/30.0.1595.0/precise32/chrome-precise32.zip
  35 CHROME_GS_URL_FMT = ('gs://chrome-unsigned/desktop-*/%s/%s/%s')
  36
  37
  38 def _ReportValueError(error_string):
  39   #TODO(aiolos): alert sheriffs via email when an error is seen.
  40   #This should be added when alerts are added when updating the build.
  41   raise ValueError(error_string)
  42
  43
  44 class BuildUpdater(object):
  45   # Remove a platform name from this list to disable updating it.
  46   _REF_BUILD_PLATFORMS = ['Mac', 'Win', 'Linux', 'Linux_x64']
  47
  48   # Omaha is Chrome's autoupdate server. It reports the current versions used
  49   # by each platform on each channel.
  50   _OMAHA_PLATFORMS = ['mac', 'linux', 'win']
  51
  52   # All of the information we need to update each platform.
  53   #   omaha: name omaha uses for the plaftorms.
  54   #   zip_name: name of the zip file to be retrieved from cloud storage.
  55   #   gs_build: name of the Chrome build platform used in cloud storage.
  56   #   destination: Name of the folder to download the reference build to.
  57   UpdateInfo = collections.namedtuple('UpdateInfo',
  58       'omaha, gs_build, zip_name, destination')
  59   _PLATFORM_MAP = { 'Mac': UpdateInfo(omaha='mac',
  60                                       gs_build='mac',
  61                                       zip_name='chrome-mac.zip',
  62                                       destination='chrome_mac'),
  63                     'Win': UpdateInfo(omaha='win',
  64                                       gs_build='win',
  65                                       zip_name='chrome-win.zip',
  66                                       destination='chrome_win'),
  67                     'Linux': UpdateInfo(omaha='linux',
  68                                         gs_build='precise32',
  69                                         zip_name='chrome-precise32.zip',
  70                                         destination='chrome_linux'),
  71                     'Linux_x64': UpdateInfo(omaha='linux',
  72                                             gs_build='precise64',
  73                                             zip_name='chrome-precise65.zip',
  74                                             destination='chrome_linux64')}
  75
  76   def __init__(self):
  77     stable_versions = self._StableVersionsMap()
  78     current_versions = self._CurrentRefBuildsMap()
  79     self._platform_to_version_map = {}
  80     for platform in stable_versions:
  81       if (platform not in current_versions or
  82           stable_versions[platform] != current_versions[platform]):
  83         self._platform_to_version_map[platform] = stable_versions[platform]
  84
  85   @classmethod
  86   def _StableVersionsMap(cls):
  87     omaha_versions_map = cls._OmahaVersionsMap()
  88     versions_map = {}
  89     for platform in cls._REF_BUILD_PLATFORMS:
  90       omaha_platform = cls._PLATFORM_MAP[platform].omaha
  91       if omaha_platform in omaha_versions_map:
  92         versions_map[platform] = omaha_versions_map[omaha_platform]
  93     return versions_map
  94
  95   @classmethod
  96   def _OmahaReport(cls):
  97     url ='https://omahaproxy.appspot.com/all?channel=stable'
  98     lines = urllib2.urlopen(url).readlines()
  99     return [l.split(',') for l in lines]
 100
 101   @classmethod
 102   def _OmahaVersionsMap(cls):
 103     platforms = cls._OMAHA_PLATFORMS
 104     rows = cls._OmahaReport()
 105     if (len(rows) < 1 or
 106         not rows[0][0:3] == ['os', 'channel', 'current_version']):
 107       _ReportValueError('Omaha report is not in the expected form: %s.'
 108                         % rows)
 109     versions_map = {}
 110     for row in rows[1:]:
 111       if row[1] != 'stable':
 112         _ReportValueError('Omaha report contains a line with the channel %s'
 113                           % row[1])
 114       if row[0] in platforms:
 115         versions_map[row[0]] = row[2]
 116
 117     if not all(platform in versions_map for platform in platforms):
 118       _ReportValueError('Omaha report did not contain all desired platforms')
 119     return versions_map
 120
 121   @classmethod
 122   def _CurrentRefBuildsMap(cls):
 123     #TODO(aiolos): Add logic for pulling the current reference build versions.
 124     # Return an empty dictionary to force an update until we store the builds in
 125     # in cloud storage.
 126     return {}
 127
 128   @staticmethod
 129   def _GetCmdStatusAndOutput(args, cwd=None, shell=False):
 130     """Executes a subprocess and returns its exit code and output.
 131
 132     Args:
 133       args: A string or a sequence of program arguments.
 134       cwd: If not None, the subprocess's current directory will be changed to
 135         |cwd| before it's executed.
 136       shell: Whether to execute args as a shell command.
 137
 138     Returns:
 139       The tuple (exit code, output).
 140     """
 141     logging.info(str(args) + ' ' + (cwd or ''))
 142     p = subprocess.Popen(args=args, cwd=cwd, stdout=subprocess.PIPE,
 143                          stderr=subprocess.PIPE, shell=shell)
 144     stdout, stderr = p.communicate()
 145     exit_code = p.returncode
 146     if stderr:
 147       logging.critical(stderr)
 148     logging.info(stdout)
 149     return (exit_code, stdout)
 150
 151   def _GetBuildUrl(self, platform, version, filename):
 152     """Returns the URL for fetching one file.
 153
 154     Args:
 155       platform: Platform name, must be a key in |self._PLATFORM_MAP|.
 156       version: A Chrome version number, e.g. 30.0.1600.1.
 157       filename: Name of the file to fetch.
 158
 159     Returns:
 160       The URL for fetching a file. This may be a GS or HTTP URL.
 161     """
 162     return CHROME_GS_URL_FMT % (
 163         version, self._PLATFORM_MAP[platform].gs_build, filename)
 164
 165   def _FindBuildVersion(self, platform, version, filename):
 166     """Searches for a version where a filename can be found.
 167
 168     Args:
 169       platform: Platform name.
 170       version: A Chrome version number, e.g. 30.0.1600.1.
 171       filename: Filename to look for.
 172
 173     Returns:
 174       A version where the file could be found, or None.
 175     """
 176     # TODO(shadi): Iterate over official versions to find a valid one.
 177     return (version
 178             if self._DoesBuildExist(platform, version, filename) else None)
 179
 180   def _DoesBuildExist(self, platform, version, filename):
 181     """Checks whether a file can be found for the given Chrome version.
 182
 183     Args:
 184       platform: Platform name.
 185       version: Chrome version number, e.g. 30.0.1600.1.
 186       filename: Filename to look for.
 187
 188     Returns:
 189       True if the file could be found, False otherwise.
 190     """
 191     url = self._GetBuildUrl(platform, version, filename)
 192     return self._DoesGSFileExist(url)
 193
 194   def _DoesGSFileExist(self, gs_file_name):
 195     """Returns True if the GS file can be found, False otherwise."""
 196     exit_code = BuildUpdater._GetCmdStatusAndOutput(
 197         ['gsutil', 'ls', gs_file_name])[0]
 198     return not exit_code
 199
 200   def _GetPlatformFiles(self, platform):
 201     """Returns the name of the zip file to fetch for |platform|."""
 202     return BuildUpdater._PLATFORM_MAP[platform].zip_name
 203
 204   def _DownloadBuilds(self):
 205     for platform in self._platform_to_version_map:
 206       version = self._platform_to_version_map[platform]
 207       for filename in self._GetPlatformFiles(platform):
 208         output = os.path.join('dl', platform,
 209                               '%s_%s_%s' % (platform,
 210                                             version,
 211                                             filename))
 212         if os.path.exists(output):
 213           logging.info('%s alread exists, skipping download', output)
 214           continue
 215         build_version = self._FindBuildVersion(platform, version, filename)
 216         if not build_version:
 217           logging.critical('Failed to find %s build for r%s\n', platform,
 218                            version)
 219           sys.exit(1)
 220         dirname = os.path.dirname(output)
 221         if dirname and not os.path.exists(dirname):
 222           os.makedirs(dirname)
 223         url = self._GetBuildUrl(platform, build_version, filename)
 224         self._DownloadFile(url, output)
 225
 226   def _DownloadFile(self, url, output):
 227     logging.info('Downloading %s, saving to %s', url, output)
 228     BuildUpdater._GetCmdStatusAndOutput(['gsutil', 'cp', url, output])
 229
 230   def _FetchSvnRepos(self):
 231     if not os.path.exists('reference_builds'):
 232       os.makedirs('reference_builds')
 233     BuildUpdater._GetCmdStatusAndOutput(
 234         ['gclient', 'config',
 235          'svn://svn.chromium.org/chrome/trunk/deps/reference_builds'],
 236         'reference_builds')
 237     BuildUpdater._GetCmdStatusAndOutput(
 238         ['gclient', 'sync'], 'reference_builds')
 239
 240   def _UnzipFile(self, dl_file, dest_dir):
 241     """Unzips a file if it is a zip file.
 242
 243     Args:
 244       dl_file: The downloaded file to unzip.
 245       dest_dir: The destination directory to unzip to.
 246
 247     Returns:
 248       True if the file was unzipped. False if it wasn't a zip file.
 249     """
 250     if not zipfile.is_zipfile(dl_file):
 251       return False
 252     logging.info('Opening %s', dl_file)
 253     with zipfile.ZipFile(dl_file, 'r') as z:
 254       for content in z.namelist():
 255         dest = os.path.join(dest_dir, content[content.find('/')+1:])
 256         # Create dest parent dir if it does not exist.
 257         if not os.path.isdir(os.path.dirname(dest)):
 258           os.makedirs(os.path.dirname(dest))
 259         # If dest is just a dir listing, do nothing.
 260         if not os.path.basename(dest):
 261           continue
 262         if not os.path.isdir(os.path.dirname(dest)):
 263           os.makedirs(os.path.dirname(dest))
 264         with z.open(content) as unzipped_content:
 265           logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
 266           with file(dest, 'wb') as dest_file:
 267             dest_file.write(unzipped_content.read())
 268           permissions = z.getinfo(content).external_attr >> 16
 269           if permissions:
 270             os.chmod(dest, permissions)
 271     return True
 272
 273   def _ClearDir(self, directory):
 274     """Clears all files in |directory| except for hidden files and folders."""
 275     for root, dirs, files in os.walk(directory):
 276       # Skip hidden files and folders (like .svn and .git).
 277       files = [f for f in files if f[0] != '.']
 278       dirs[:] = [d for d in dirs if d[0] != '.']
 279
 280       for f in files:
 281         os.remove(os.path.join(root, f))
 282
 283   def _ExtractBuilds(self):
 284     for platform in self._platform_to_version_map:
 285       if os.path.exists('tmp_unzip'):
 286         os.path.unlink('tmp_unzip')
 287       dest_dir = os.path.join(
 288           'reference_builds', 'reference_builds',
 289           BuildUpdater._PLATFORM_MAP[platform].destination)
 290       self._ClearDir(dest_dir)
 291       for root, _, dl_files in os.walk(os.path.join('dl', platform)):
 292         for dl_file in dl_files:
 293           dl_file = os.path.join(root, dl_file)
 294           if not self._UnzipFile(dl_file, dest_dir):
 295             logging.info('Copying %s to %s', dl_file, dest_dir)
 296             shutil.copy(dl_file, dest_dir)
 297
 298   def _SvnAddAndRemove(self):
 299     svn_dir = os.path.join('reference_builds', 'reference_builds')
 300     # List all changes without ignoring any files.
 301     stat = BuildUpdater._GetCmdStatusAndOutput(['svn', 'stat', '--no-ignore'],
 302                                                svn_dir)[1]
 303     for line in stat.splitlines():
 304       action, filename = line.split(None, 1)
 305       # Add new and ignored files.
 306       if action == '?' or action == 'I':
 307         BuildUpdater._GetCmdStatusAndOutput(
 308             ['svn', 'add', filename], svn_dir)
 309       elif action == '!':
 310         BuildUpdater._GetCmdStatusAndOutput(
 311             ['svn', 'delete', filename], svn_dir)
 312       filepath = os.path.join(svn_dir, filename)
 313       if not os.path.isdir(filepath) and os.access(filepath, os.X_OK):
 314         BuildUpdater._GetCmdStatusAndOutput(
 315             ['svn', 'propset', 'svn:executable', 'true', filename], svn_dir)
 316
 317   def DownloadAndUpdateBuilds(self):
 318     self._DownloadBuilds()
 319     self._FetchSvnRepos()
 320     self._ExtractBuilds()
 321     self._SvnAddAndRemove()
 322
 323
 324 def main():
 325   logging.getLogger().setLevel(logging.DEBUG)
 326   #TODO(aiolos): check that there are no options passed (argparse).
 327   b = BuildUpdater()
 328   b.DownloadAndUpdateBuilds()
 329   logging.info('Successfully updated reference builds. Move to '
 330                'reference_builds/reference_builds and make a change with gcl.')
 331
 332 if __name__ == '__main__':
 333   main()