tools/auto_bisect/fetch_build.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This module contains functions for fetching and extracting archived builds.
   6
   7 The builds may be stored in different places by different types of builders;
   8 for example, builders on tryserver.chromium.perf stores builds in one place,
   9 while builders on chromium.linux store builds in another.
  10
  11 This module can be either imported or run as a stand-alone script to download
  12 and extract a build.
  13
  14 Usage: fetch_build.py <type> <revision> <output_dir> [options]
  15 """
  16
  17 import argparse
  18 import errno
  19 import logging
  20 import os
  21 import shutil
  22 import sys
  23 import zipfile
  24
  25 # Telemetry (src/tools/telemetry) is expected to be in the PYTHONPATH.
  26 from telemetry.util import cloud_storage
  27
  28 import bisect_utils
  29
  30 # Possible builder types.
  31 PERF_BUILDER = 'perf'
  32 FULL_BUILDER = 'full'
  33
  34
  35 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER,
  36                            target_arch='ia32', target_platform='chromium',
  37                            deps_patch_sha=None):
  38   """Returns the location where a build archive is expected to be.
  39
  40   Args:
  41     revision: Revision string, e.g. a git commit hash or SVN revision.
  42     builder_type: Type of build archive.
  43     target_arch: Architecture, e.g. "ia32".
  44     target_platform: Platform name, e.g. "chromium" or "android".
  45     deps_patch_sha: SHA1 hash which identifies a particular combination of
  46         custom revisions for dependency repositories.
  47
  48   Returns:
  49     A pair of strings (bucket, path), where the archive is expected to be.
  50   """
  51   build_archive = BuildArchive.Create(
  52       builder_type, target_arch=target_arch, target_platform=target_platform)
  53   bucket = build_archive.BucketName()
  54   remote_path = build_archive.FilePath(revision, deps_patch_sha=deps_patch_sha)
  55   return bucket, remote_path
  56
  57
  58 class BuildArchive(object):
  59   """Represents a place where builds of some type are stored.
  60
  61   There are two pieces of information required to locate a file in Google
  62   Cloud Storage, bucket name and file path. Subclasses of this class contain
  63   specific logic about which bucket names and paths should be used to fetch
  64   a build.
  65   """
  66
  67   @staticmethod
  68   def Create(builder_type, target_arch='ia32', target_platform='chromium'):
  69     if builder_type == PERF_BUILDER:
  70       return PerfBuildArchive(target_arch, target_platform)
  71     if builder_type == FULL_BUILDER:
  72       return FullBuildArchive(target_arch, target_platform)
  73     raise NotImplementedError('Builder type "%s" not supported.' % builder_type)
  74
  75   def __init__(self, target_arch='ia32', target_platform='chromium'):
  76     if bisect_utils.IsLinuxHost() and target_platform == 'android':
  77       self._platform = 'android'
  78     elif bisect_utils.IsLinuxHost():
  79       self._platform = 'linux'
  80     elif bisect_utils.IsMacHost():
  81       self._platform = 'mac'
  82     elif bisect_utils.Is64BitWindows() and target_arch == 'x64':
  83       self._platform = 'win64'
  84     elif bisect_utils.IsWindowsHost():
  85       self._platform = 'win'
  86     else:
  87       raise NotImplementedError('Unknown platform "%s".' % sys.platform)
  88
  89   def BucketName(self):
  90     raise NotImplementedError()
  91
  92   def FilePath(self, revision, deps_patch_sha=None):
  93     """Returns the remote file path to download a build from.
  94
  95     Args:
  96       revision: A Chromium revision; this could be a git commit hash or
  97           commit position or SVN revision number.
  98       deps_patch_sha: The SHA1 hash of a patch to the DEPS file, which
  99           uniquely identifies a change to use a particular revision of
 100           a dependency.
 101
 102     Returns:
 103       A file path, which not does not include a bucket name.
 104     """
 105     raise NotImplementedError()
 106
 107   def _ZipFileName(self, revision, deps_patch_sha=None):
 108     """Gets the file name of a zip archive for a particular revision.
 109
 110     This returns a file name of the form full-build-<platform>_<revision>.zip,
 111     which is a format used by multiple types of builders that store archives.
 112
 113     Args:
 114       revision: A git commit hash or other revision string.
 115       deps_patch_sha: SHA1 hash of a DEPS file patch.
 116
 117     Returns:
 118       The archive file name.
 119     """
 120     base_name = 'full-build-%s' % self._PlatformName()
 121     if deps_patch_sha:
 122       revision = '%s_%s' % (revision, deps_patch_sha)
 123     return '%s_%s.zip' % (base_name, revision)
 124
 125   def _PlatformName(self):
 126     """Return a string to be used in paths for the platform."""
 127     if self._platform in ('win', 'win64'):
 128       # Build archive for win64 is still stored with "win32" in the name.
 129       return 'win32'
 130     if self._platform in ('linux', 'android'):
 131       # Android builds are also stored with "linux" in the name.
 132       return 'linux'
 133     if self._platform == 'mac':
 134       return 'mac'
 135     raise NotImplementedError('Unknown platform "%s".' % sys.platform)
 136
 137
 138 class PerfBuildArchive(BuildArchive):
 139
 140   def BucketName(self):
 141     return 'chrome-perf'
 142
 143   def FilePath(self, revision, deps_patch_sha=None):
 144     return '%s/%s' % (self._ArchiveDirectory(),
 145                       self._ZipFileName(revision, deps_patch_sha))
 146
 147   def _ArchiveDirectory(self):
 148     """Returns the directory name to download builds from."""
 149     platform_to_directory = {
 150         'android': 'android_perf_rel',
 151         'linux': 'Linux Builder',
 152         'mac': 'Mac Builder',
 153         'win64': 'Win x64 Builder',
 154         'win': 'Win Builder',
 155     }
 156     assert self._platform in platform_to_directory
 157     return platform_to_directory.get(self._platform)
 158
 159
 160 class FullBuildArchive(BuildArchive):
 161
 162   def BucketName(self):
 163     platform_to_bucket = {
 164         'android': 'chromium-android',
 165         'linux': 'chromium-linux-archive',
 166         'mac': 'chromium-mac-archive',
 167         'win64': 'chromium-win-archive',
 168         'win': 'chromium-win-archive',
 169     }
 170     assert self._platform in platform_to_bucket
 171     return platform_to_bucket.get(self._platform)
 172
 173   def FilePath(self, revision, deps_patch_sha=None):
 174     return '%s/%s' % (self._ArchiveDirectory(),
 175                       self._ZipFileName(revision, deps_patch_sha))
 176
 177   def _ArchiveDirectory(self):
 178     """Returns the remote directory to download builds from."""
 179     platform_to_directory = {
 180         'android': 'android_main_rel',
 181         'linux': 'chromium.linux/Linux Builder',
 182         'mac': 'chromium.mac/Mac Builder',
 183         'win64': 'chromium.win/Win x64 Builder',
 184         'win': 'chromium.win/Win Builder',
 185     }
 186     assert self._platform in platform_to_directory
 187     return platform_to_directory.get(self._platform)
 188
 189
 190 def BuildIsAvailable(bucket_name, remote_path):
 191   """Checks whether a build is currently archived at some place."""
 192   logging.info('Checking existance: gs://%s/%s' % (bucket_name, remote_path))
 193   try:
 194     exists = cloud_storage.Exists(bucket_name, remote_path)
 195     logging.info('Exists? %s' % exists)
 196     return exists
 197   except cloud_storage.CloudStorageError:
 198     return False
 199
 200
 201 def FetchFromCloudStorage(bucket_name, source_path, destination_dir):
 202   """Fetches file(s) from the Google Cloud Storage.
 203
 204   As a side-effect, this prints messages to stdout about what's happening.
 205
 206   Args:
 207     bucket_name: Google Storage bucket name.
 208     source_path: Source file path.
 209     destination_dir: Destination file path.
 210
 211   Returns:
 212     Local file path of downloaded file if it was downloaded. If the file does
 213     not exist in the given bucket, or if there was an error while downloading,
 214     None is returned.
 215   """
 216   target_file = os.path.join(destination_dir, os.path.basename(source_path))
 217   gs_url = 'gs://%s/%s' % (bucket_name, source_path)
 218   try:
 219     if cloud_storage.Exists(bucket_name, source_path):
 220       logging.info('Fetching file from %s...', gs_url)
 221       cloud_storage.Get(bucket_name, source_path, target_file)
 222       if os.path.exists(target_file):
 223         return target_file
 224     else:
 225       logging.info('File %s not found in cloud storage.', gs_url)
 226       return None
 227   except Exception as e:
 228     logging.warn('Exception while fetching from cloud storage: %s', e)
 229     if os.path.exists(target_file):
 230       os.remove(target_file)
 231   return None
 232
 233
 234 def Unzip(file_path, output_dir, verbose=True):
 235   """Extracts a zip archive's contents into the given output directory.
 236
 237   This was based on ExtractZip from build/scripts/common/chromium_utils.py.
 238
 239   Args:
 240     file_path: Path of the zip file to extract.
 241     output_dir: Path to the destination directory.
 242     verbose: Whether to print out what is being extracted.
 243
 244   Raises:
 245     IOError: The unzip command had a non-zero exit code.
 246     RuntimeError: Failed to create the output directory.
 247   """
 248   _MakeDirectory(output_dir)
 249
 250   # On Linux and Mac, we use the unzip command because it handles links and
 251   # file permissions bits, so achieving this behavior is easier than with
 252   # ZipInfo options.
 253   #
 254   # The Mac Version of unzip unfortunately does not support Zip64, whereas
 255   # the python module does, so we have to fall back to the python zip module
 256   # on Mac if the file size is greater than 4GB.
 257   mac_zip_size_limit = 2 ** 32  # 4GB
 258   if (bisect_utils.IsLinuxHost() or
 259       (bisect_utils.IsMacHost()
 260        and os.path.getsize(file_path) < mac_zip_size_limit)):
 261     unzip_command = ['unzip', '-o']
 262     _UnzipUsingCommand(unzip_command, file_path, output_dir)
 263     return
 264
 265   # On Windows, try to use 7z if it is installed, otherwise fall back to the
 266   # Python zipfile module. If 7z is not installed, then this may fail if the
 267   # zip file is larger than 512MB.
 268   sevenzip_path = r'C:\Program Files\7-Zip\7z.exe'
 269   if bisect_utils.IsWindowsHost() and os.path.exists(sevenzip_path):
 270     unzip_command = [sevenzip_path, 'x', '-y']
 271     _UnzipUsingCommand(unzip_command, file_path, output_dir)
 272     return
 273
 274   _UnzipUsingZipFile(file_path, output_dir, verbose)
 275
 276
 277 def _UnzipUsingCommand(unzip_command, file_path, output_dir):
 278   """Extracts a zip file using an external command.
 279
 280   Args:
 281     unzip_command: An unzipping command, as a string list, without the filename.
 282     file_path: Path to the zip file.
 283     output_dir: The directory which the contents should be extracted to.
 284
 285   Raises:
 286     IOError: The command had a non-zero exit code.
 287   """
 288   absolute_filepath = os.path.abspath(file_path)
 289   command = unzip_command + [absolute_filepath]
 290   return_code = _RunCommandInDirectory(output_dir, command)
 291   if return_code:
 292     _RemoveDirectoryTree(output_dir)
 293     raise IOError('Unzip failed: %s => %s' % (str(command), return_code))
 294
 295
 296 def _RunCommandInDirectory(directory, command):
 297   """Changes to a directory, runs a command, then changes back."""
 298   saved_dir = os.getcwd()
 299   os.chdir(directory)
 300   return_code = bisect_utils.RunProcess(command)
 301   os.chdir(saved_dir)
 302   return return_code
 303
 304
 305 def _UnzipUsingZipFile(file_path, output_dir, verbose=True):
 306   """Extracts a zip file using the Python zipfile module."""
 307   assert bisect_utils.IsWindowsHost() or bisect_utils.IsMacHost()
 308   zf = zipfile.ZipFile(file_path)
 309   for name in zf.namelist():
 310     if verbose:
 311       print 'Extracting %s' % name
 312     zf.extract(name, output_dir)
 313     if bisect_utils.IsMacHost():
 314       # Restore file permission bits.
 315       mode = zf.getinfo(name).external_attr >> 16
 316       os.chmod(os.path.join(output_dir, name), mode)
 317
 318
 319 def _MakeDirectory(path):
 320   try:
 321     os.makedirs(path)
 322   except OSError as e:
 323     if e.errno != errno.EEXIST:
 324       raise
 325
 326
 327 def _RemoveDirectoryTree(path):
 328   try:
 329     if os.path.exists(path):
 330       shutil.rmtree(path)
 331   except OSError, e:
 332     if e.errno != errno.ENOENT:
 333       raise
 334
 335
 336 def Main(argv):
 337   """Downloads and extracts a build based on the command line arguments."""
 338   parser = argparse.ArgumentParser()
 339   parser.add_argument('builder_type')
 340   parser.add_argument('revision')
 341   parser.add_argument('output_dir')
 342   parser.add_argument('--target-arch', default='ia32')
 343   parser.add_argument('--target-platform', default='chromium')
 344   parser.add_argument('--deps-patch-sha')
 345   args = parser.parse_args(argv[1:])
 346
 347   bucket_name, remote_path = GetBucketAndRemotePath(
 348       args.revision, args.builder_type, target_arch=args.target_arch,
 349       target_platform=args.target_platform,
 350       deps_patch_sha=args.deps_patch_sha)
 351   print 'Bucket name: %s, remote path: %s' % (bucket_name, remote_path)
 352
 353   if not BuildIsAvailable(bucket_name, remote_path):
 354     print 'Build is not available.'
 355     return 1
 356
 357   FetchFromCloudStorage(bucket_name, remote_path, args.output_dir)
 358   print 'Build has been downloaded to and extracted in %s.' % args.output_dir
 359   return 0
 360
 361
 362 if __name__ == '__main__':
 363   sys.exit(Main(sys.argv))
 364