tools/auto_bisect/fetch_build.py

   1 # Copyright 2014 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """This module contains functions for fetching and extracting archived builds.
   6
   7 The builds may be stored in different places by different types of builders;
   8 for example, builders on tryserver.chromium.perf stores builds in one place,
   9 while builders on chromium.linux store builds in another.
  10
  11 This module can be either imported or run as a stand-alone script to download
  12 and extract a build.
  13
  14 Usage: fetch_build.py <type> <revision> <output_dir> [options]
  15 """
  16
  17 import argparse
  18 import errno
  19 import logging
  20 import os
  21 import shutil
  22 import sys
  23 import zipfile
  24
  25 # Telemetry (src/tools/telemetry) is expected to be in the PYTHONPATH.
  26 from telemetry.util import cloud_storage
  27
  28 import bisect_utils
  29
  30 # Possible builder types.
  31 PERF_BUILDER = 'perf'
  32 FULL_BUILDER = 'full'
  33 ANDROID_CHROME_PERF_BUILDER = 'android-chrome-perf'
  34
  35 # Maximum time in seconds to wait after posting build request to the try server.
  36 MAX_MAC_BUILD_TIME = 14400
  37 MAX_WIN_BUILD_TIME = 14400
  38 MAX_LINUX_BUILD_TIME = 14400
  39
  40 # Try server status page URLs, used to get build status.
  41 PERF_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.perf'
  42 LINUX_TRY_SERVER_URL = 'http://build.chromium.org/p/tryserver.chromium.linux'
  43
  44
  45 def GetBucketAndRemotePath(revision, builder_type=PERF_BUILDER,
  46                            target_arch='ia32', target_platform='chromium',
  47                            deps_patch_sha=None, extra_src=None):
  48   """Returns the location where a build archive is expected to be.
  49
  50   Args:
  51     revision: Revision string, e.g. a git commit hash or SVN revision.
  52     builder_type: Type of build archive.
  53     target_arch: Architecture, e.g. "ia32".
  54     target_platform: Platform name, e.g. "chromium" or "android".
  55     deps_patch_sha: SHA1 hash which identifies a particular combination of
  56         custom revisions for dependency repositories.
  57     extra_src: Path to a script which can be used to modify the bisect script's
  58         behavior.
  59
  60   Returns:
  61     A pair of strings (bucket, path), where the archive is expected to be.
  62   """
  63   logging.info('Getting GS URL for archive of builder "%s", "%s", "%s".',
  64                builder_type, target_arch, target_platform)
  65   build_archive = BuildArchive.Create(
  66       builder_type, target_arch=target_arch, target_platform=target_platform,
  67       extra_src=extra_src)
  68   bucket = build_archive.BucketName()
  69   remote_path = build_archive.FilePath(revision, deps_patch_sha=deps_patch_sha)
  70   return bucket, remote_path
  71
  72
  73 def GetBuilderNameAndBuildTime(builder_type=PERF_BUILDER, target_arch='ia32',
  74                                target_platform='chromium', extra_src=None):
  75   """Gets builder bot name and build time in seconds based on platform."""
  76   logging.info('Getting builder name for builder "%s", "%s", "%s".',
  77                builder_type, target_arch, target_platform)
  78   build_archive = BuildArchive.Create(
  79       builder_type, target_arch=target_arch, target_platform=target_platform,
  80       extra_src=extra_src)
  81   return build_archive.GetBuilderName(), build_archive.GetBuilderBuildTime()
  82
  83
  84 def GetBuildBotUrl(builder_type=PERF_BUILDER, target_arch='ia32',
  85                    target_platform='chromium', extra_src=None):
  86   """Gets buildbot URL for a given builder type."""
  87   logging.info('Getting buildbot URL for "%s", "%s", "%s".',
  88                builder_type, target_arch, target_platform)
  89   build_archive = BuildArchive.Create(
  90       builder_type, target_arch=target_arch, target_platform=target_platform,
  91       extra_src=extra_src)
  92   return build_archive.GetBuildBotUrl()
  93
  94
  95 class BuildArchive(object):
  96   """Represents a place where builds of some type are stored.
  97
  98   There are two pieces of information required to locate a file in Google
  99   Cloud Storage, bucket name and file path. Subclasses of this class contain
 100   specific logic about which bucket names and paths should be used to fetch
 101   a build.
 102   """
 103
 104   @staticmethod
 105   def Create(builder_type, target_arch='ia32', target_platform='chromium',
 106              extra_src=None):
 107     if builder_type == PERF_BUILDER:
 108       return PerfBuildArchive(target_arch, target_platform)
 109     if builder_type == FULL_BUILDER:
 110       return FullBuildArchive(target_arch, target_platform)
 111     if builder_type == ANDROID_CHROME_PERF_BUILDER:
 112       try:
 113         # Load and initialize a module in extra source file and
 114         # return its module object to access android-chrome specific data.
 115         loaded_extra_src = bisect_utils.LoadExtraSrc(extra_src)
 116         return AndroidChromeBuildArchive(
 117             target_arch, target_platform, loaded_extra_src)
 118       except (IOError, TypeError, ImportError):
 119         raise RuntimeError('Invalid or missing --extra_src. [%s]' % extra_src)
 120     raise NotImplementedError('Builder type "%s" not supported.' % builder_type)
 121
 122   def __init__(self, target_arch='ia32', target_platform='chromium',
 123                extra_src=None):
 124     self._extra_src = extra_src
 125     if bisect_utils.IsLinuxHost() and target_platform == 'android':
 126       self._platform = 'android'
 127     elif bisect_utils.IsLinuxHost() and target_platform == 'android-chrome':
 128       self._platform = 'android-chrome'
 129     elif bisect_utils.IsLinuxHost():
 130       self._platform = 'linux'
 131     elif bisect_utils.IsMacHost():
 132       self._platform = 'mac'
 133     elif bisect_utils.Is64BitWindows() and target_arch == 'x64':
 134       self._platform = 'win64'
 135     elif bisect_utils.IsWindowsHost():
 136       self._platform = 'win'
 137     else:
 138       raise NotImplementedError('Unknown platform "%s".' % sys.platform)
 139
 140   def BucketName(self):
 141     raise NotImplementedError()
 142
 143   def FilePath(self, revision, deps_patch_sha=None):
 144     """Returns the remote file path to download a build from.
 145
 146     Args:
 147       revision: A Chromium revision; this could be a git commit hash or
 148           commit position or SVN revision number.
 149       deps_patch_sha: The SHA1 hash of a patch to the DEPS file, which
 150           uniquely identifies a change to use a particular revision of
 151           a dependency.
 152
 153     Returns:
 154       A file path, which not does not include a bucket name.
 155     """
 156     raise NotImplementedError()
 157
 158   def _ZipFileName(self, revision, deps_patch_sha=None):
 159     """Gets the file name of a zip archive for a particular revision.
 160
 161     This returns a file name of the form full-build-<platform>_<revision>.zip,
 162     which is a format used by multiple types of builders that store archives.
 163
 164     Args:
 165       revision: A git commit hash or other revision string.
 166       deps_patch_sha: SHA1 hash of a DEPS file patch.
 167
 168     Returns:
 169       The archive file name.
 170     """
 171     base_name = 'full-build-%s' % self._PlatformName()
 172     if deps_patch_sha:
 173       revision = '%s_%s' % (revision, deps_patch_sha)
 174     return '%s_%s.zip' % (base_name, revision)
 175
 176   def _PlatformName(self):
 177     """Return a string to be used in paths for the platform."""
 178     if self._platform in ('win', 'win64'):
 179       # Build archive for win64 is still stored with "win32" in the name.
 180       return 'win32'
 181     if self._platform in ('linux', 'android'):
 182       # Android builds are also stored with "linux" in the name.
 183       return 'linux'
 184     if self._platform == 'mac':
 185       return 'mac'
 186     raise NotImplementedError('Unknown platform "%s".' % sys.platform)
 187
 188   def GetBuilderName(self):
 189     raise NotImplementedError()
 190
 191   def GetBuilderBuildTime(self):
 192     """Returns the time to wait for a build after requesting one."""
 193     if self._platform in ('win', 'win64'):
 194       return MAX_WIN_BUILD_TIME
 195     if self._platform in ('linux', 'android', 'android-chrome'):
 196       return MAX_LINUX_BUILD_TIME
 197     if self._platform == 'mac':
 198       return MAX_MAC_BUILD_TIME
 199     raise NotImplementedError('Unsupported Platform "%s".' % sys.platform)
 200
 201   def GetBuildBotUrl(self):
 202     raise NotImplementedError()
 203
 204
 205 class PerfBuildArchive(BuildArchive):
 206
 207   def BucketName(self):
 208     return 'chrome-perf'
 209
 210   def FilePath(self, revision, deps_patch_sha=None):
 211     return '%s/%s' % (self._ArchiveDirectory(),
 212                       self._ZipFileName(revision, deps_patch_sha))
 213
 214   def _ArchiveDirectory(self):
 215     """Returns the directory name to download builds from."""
 216     platform_to_directory = {
 217         'android': 'android_perf_rel',
 218         'linux': 'Linux Builder',
 219         'mac': 'Mac Builder',
 220         'win64': 'Win x64 Builder',
 221         'win': 'Win Builder',
 222     }
 223     assert self._platform in platform_to_directory
 224     return platform_to_directory.get(self._platform)
 225
 226   def GetBuilderName(self):
 227     """Gets builder bot name based on platform."""
 228     if self._platform == 'win64':
 229       return 'win_x64_perf_bisect_builder'
 230     elif self._platform == 'win':
 231       return 'win_perf_bisect_builder'
 232     elif self._platform == 'linux':
 233       return 'linux_perf_bisect_builder'
 234     elif self._platform == 'android':
 235       return 'android_perf_bisect_builder'
 236     elif self._platform == 'mac':
 237       return 'mac_perf_bisect_builder'
 238     raise NotImplementedError('Unsupported platform "%s".' % sys.platform)
 239
 240   def GetBuildBotUrl(self):
 241     """Returns buildbot URL for fetching build info."""
 242     return PERF_TRY_SERVER_URL
 243
 244
 245 class FullBuildArchive(BuildArchive):
 246
 247   def BucketName(self):
 248     platform_to_bucket = {
 249         'android': 'chromium-android',
 250         'linux': 'chromium-linux-archive',
 251         'mac': 'chromium-mac-archive',
 252         'win64': 'chromium-win-archive',
 253         'win': 'chromium-win-archive',
 254     }
 255     assert self._platform in platform_to_bucket
 256     return platform_to_bucket.get(self._platform)
 257
 258   def FilePath(self, revision, deps_patch_sha=None):
 259     return '%s/%s' % (self._ArchiveDirectory(),
 260                       self._ZipFileName(revision, deps_patch_sha))
 261
 262   def _ArchiveDirectory(self):
 263     """Returns the remote directory to download builds from."""
 264     platform_to_directory = {
 265         'android': 'android_main_rel',
 266         'linux': 'chromium.linux/Linux Builder',
 267         'mac': 'chromium.mac/Mac Builder',
 268         'win64': 'chromium.win/Win x64 Builder',
 269         'win': 'chromium.win/Win Builder',
 270     }
 271     assert self._platform in platform_to_directory
 272     return platform_to_directory.get(self._platform)
 273
 274   def GetBuilderName(self):
 275     """Gets builder bot name based on platform."""
 276     if self._platform == 'linux':
 277       return 'linux_full_bisect_builder'
 278     raise NotImplementedError('Unsupported platform "%s".' % sys.platform)
 279
 280   def GetBuildBotUrl(self):
 281     """Returns buildbot URL for fetching build info."""
 282     return LINUX_TRY_SERVER_URL
 283
 284
 285 class AndroidChromeBuildArchive(BuildArchive):
 286   """Represents a place where builds of android-chrome type are stored.
 287
 288   If AndroidChromeBuildArchive is used, it is assumed that the --extra_src
 289   is a valid Python module which contains the module-level functions
 290   GetBucketName and GetArchiveDirectory.
 291   """
 292
 293   def BucketName(self):
 294     return self._extra_src.GetBucketName()
 295
 296   def _ZipFileName(self, revision, deps_patch_sha=None):
 297     """Gets the file name of a zip archive on android-chrome.
 298
 299     This returns a file name of the form build_product_<revision>.zip,
 300     which is a format used by android-chrome.
 301
 302     Args:
 303       revision: A git commit hash or other revision string.
 304       deps_patch_sha: SHA1 hash of a DEPS file patch.
 305
 306     Returns:
 307       The archive file name.
 308     """
 309     if deps_patch_sha:
 310       revision = '%s_%s' % (revision, deps_patch_sha)
 311     return 'build_product_%s.zip' % revision
 312
 313   def FilePath(self, revision, deps_patch_sha=None):
 314     return '%s/%s' % (self._ArchiveDirectory(),
 315                       self._ZipFileName(revision, deps_patch_sha))
 316
 317   def _ArchiveDirectory(self):
 318     """Returns the directory name to download builds from."""
 319     return self._extra_src.GetArchiveDirectory()
 320
 321   def GetBuilderName(self):
 322     """Returns the builder name extra source."""
 323     return self._extra_src.GetBuilderName()
 324
 325   def GetBuildBotUrl(self):
 326     """Returns buildbot URL for fetching build info."""
 327     return self._extra_src.GetBuildBotUrl()
 328
 329
 330 def BuildIsAvailable(bucket_name, remote_path):
 331   """Checks whether a build is currently archived at some place."""
 332   logging.info('Checking existence: gs://%s/%s' % (bucket_name, remote_path))
 333   try:
 334     exists = cloud_storage.Exists(bucket_name, remote_path)
 335     logging.info('Exists? %s' % exists)
 336     return exists
 337   except cloud_storage.CloudStorageError:
 338     return False
 339
 340
 341 def FetchFromCloudStorage(bucket_name, source_path, destination_dir):
 342   """Fetches file(s) from the Google Cloud Storage.
 343
 344   As a side-effect, this prints messages to stdout about what's happening.
 345
 346   Args:
 347     bucket_name: Google Storage bucket name.
 348     source_path: Source file path.
 349     destination_dir: Destination file path.
 350
 351   Returns:
 352     Local file path of downloaded file if it was downloaded. If the file does
 353     not exist in the given bucket, or if there was an error while downloading,
 354     None is returned.
 355   """
 356   target_file = os.path.join(destination_dir, os.path.basename(source_path))
 357   gs_url = 'gs://%s/%s' % (bucket_name, source_path)
 358   try:
 359     if cloud_storage.Exists(bucket_name, source_path):
 360       logging.info('Fetching file from %s...', gs_url)
 361       cloud_storage.Get(bucket_name, source_path, target_file)
 362       if os.path.exists(target_file):
 363         return target_file
 364     else:
 365       logging.info('File %s not found in cloud storage.', gs_url)
 366       return None
 367   except Exception as e:
 368     logging.warn('Exception while fetching from cloud storage: %s', e)
 369     if os.path.exists(target_file):
 370       os.remove(target_file)
 371   return None
 372
 373
 374 def Unzip(file_path, output_dir, verbose=True):
 375   """Extracts a zip archive's contents into the given output directory.
 376
 377   This was based on ExtractZip from build/scripts/common/chromium_utils.py.
 378
 379   Args:
 380     file_path: Path of the zip file to extract.
 381     output_dir: Path to the destination directory.
 382     verbose: Whether to print out what is being extracted.
 383
 384   Raises:
 385     IOError: The unzip command had a non-zero exit code.
 386     RuntimeError: Failed to create the output directory.
 387   """
 388   _MakeDirectory(output_dir)
 389
 390   # On Linux and Mac, we use the unzip command because it handles links and
 391   # file permissions bits, so achieving this behavior is easier than with
 392   # ZipInfo options.
 393   #
 394   # The Mac Version of unzip unfortunately does not support Zip64, whereas
 395   # the python module does, so we have to fall back to the python zip module
 396   # on Mac if the file size is greater than 4GB.
 397   mac_zip_size_limit = 2 ** 32  # 4GB
 398   if (bisect_utils.IsLinuxHost() or
 399       (bisect_utils.IsMacHost()
 400        and os.path.getsize(file_path) < mac_zip_size_limit)):
 401     unzip_command = ['unzip', '-o']
 402     _UnzipUsingCommand(unzip_command, file_path, output_dir)
 403     return
 404
 405   # On Windows, try to use 7z if it is installed, otherwise fall back to the
 406   # Python zipfile module. If 7z is not installed, then this may fail if the
 407   # zip file is larger than 512MB.
 408   sevenzip_path = r'C:\Program Files\7-Zip\7z.exe'
 409   if bisect_utils.IsWindowsHost() and os.path.exists(sevenzip_path):
 410     unzip_command = [sevenzip_path, 'x', '-y']
 411     _UnzipUsingCommand(unzip_command, file_path, output_dir)
 412     return
 413
 414   _UnzipUsingZipFile(file_path, output_dir, verbose)
 415
 416
 417 def _UnzipUsingCommand(unzip_command, file_path, output_dir):
 418   """Extracts a zip file using an external command.
 419
 420   Args:
 421     unzip_command: An unzipping command, as a string list, without the filename.
 422     file_path: Path to the zip file.
 423     output_dir: The directory which the contents should be extracted to.
 424
 425   Raises:
 426     IOError: The command had a non-zero exit code.
 427   """
 428   absolute_filepath = os.path.abspath(file_path)
 429   command = unzip_command + [absolute_filepath]
 430   return_code = _RunCommandInDirectory(output_dir, command)
 431   if return_code:
 432     _RemoveDirectoryTree(output_dir)
 433     raise IOError('Unzip failed: %s => %s' % (str(command), return_code))
 434
 435
 436 def _RunCommandInDirectory(directory, command):
 437   """Changes to a directory, runs a command, then changes back."""
 438   saved_dir = os.getcwd()
 439   os.chdir(directory)
 440   return_code = bisect_utils.RunProcess(command)
 441   os.chdir(saved_dir)
 442   return return_code
 443
 444
 445 def _UnzipUsingZipFile(file_path, output_dir, verbose=True):
 446   """Extracts a zip file using the Python zipfile module."""
 447   assert bisect_utils.IsWindowsHost() or bisect_utils.IsMacHost()
 448   zf = zipfile.ZipFile(file_path)
 449   for name in zf.namelist():
 450     if verbose:
 451       print 'Extracting %s' % name
 452     zf.extract(name, output_dir)
 453     if bisect_utils.IsMacHost():
 454       # Restore file permission bits.
 455       mode = zf.getinfo(name).external_attr >> 16
 456       os.chmod(os.path.join(output_dir, name), mode)
 457
 458
 459 def _MakeDirectory(path):
 460   try:
 461     os.makedirs(path)
 462   except OSError as e:
 463     if e.errno != errno.EEXIST:
 464       raise
 465
 466
 467 def _RemoveDirectoryTree(path):
 468   try:
 469     if os.path.exists(path):
 470       shutil.rmtree(path)
 471   except OSError, e:
 472     if e.errno != errno.ENOENT:
 473       raise
 474
 475
 476 def Main(argv):
 477   """Downloads and extracts a build based on the command line arguments."""
 478   parser = argparse.ArgumentParser()
 479   parser.add_argument('builder_type')
 480   parser.add_argument('revision')
 481   parser.add_argument('output_dir')
 482   parser.add_argument('--target-arch', default='ia32')
 483   parser.add_argument('--target-platform', default='chromium')
 484   parser.add_argument('--deps-patch-sha')
 485   args = parser.parse_args(argv[1:])
 486
 487   bucket_name, remote_path = GetBucketAndRemotePath(
 488       args.revision, args.builder_type, target_arch=args.target_arch,
 489       target_platform=args.target_platform,
 490       deps_patch_sha=args.deps_patch_sha)
 491   print 'Bucket name: %s, remote path: %s' % (bucket_name, remote_path)
 492
 493   if not BuildIsAvailable(bucket_name, remote_path):
 494     print 'Build is not available.'
 495     return 1
 496
 497   FetchFromCloudStorage(bucket_name, remote_path, args.output_dir)
 498   print 'Build has been downloaded to and extracted in %s.' % args.output_dir
 499   return 0
 500
 501
 502 if __name__ == '__main__':
 503   sys.exit(Main(sys.argv))