build/get_syzygy_binaries.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """A utility script for downloading versioned Syzygy binaries."""
   7
   8 import cStringIO
   9 import hashlib
  10 import errno
  11 import json
  12 import logging
  13 import optparse
  14 import os
  15 import re
  16 import shutil
  17 import stat
  18 import sys
  19 import subprocess
  20 import urllib2
  21 import zipfile
  22
  23
  24 _LOGGER = logging.getLogger(os.path.basename(__file__))
  25
  26 # The URL where official builds are archived.
  27 _SYZYGY_ARCHIVE_URL = ('https://syzygy-archive.commondatastorage.googleapis.com'
  28     '/builds/official/%(revision)s')
  29
  30 # A JSON file containing the state of the download directory. If this file and
  31 # directory state do not agree, then the binaries will be downloaded and
  32 # installed again.
  33 _STATE = '.state'
  34
  35 # This matches an integer (an SVN revision number) or a SHA1 value (a GIT hash).
  36 # The archive exclusively uses lowercase GIT hashes.
  37 _REVISION_RE = re.compile('^(?:\d+|[a-f0-9]{40})$')
  38
  39 # This matches an MD5 hash.
  40 _MD5_RE = re.compile('^[a-f0-9]{32}$')
  41
  42 # List of reources to be downloaded and installed. These are tuples with the
  43 # following format:
  44 # (basename, logging name, relative installation path, extraction filter)
  45 _RESOURCES = [
  46   ('benchmark.zip', 'benchmark', '', None),
  47   ('binaries.zip', 'binaries', 'exe', None),
  48   ('symbols.zip', 'symbols', 'exe',
  49       lambda x: x.filename.endswith('.dll.pdb'))]
  50
  51
  52 def _Shell(*cmd, **kw):
  53   """Runs |cmd|, returns the results from Popen(cmd).communicate()."""
  54   _LOGGER.debug('Executing %s.', cmd)
  55   prog = subprocess.Popen(cmd, shell=True, **kw)
  56
  57   stdout, stderr = prog.communicate()
  58   if prog.returncode != 0:
  59     raise RuntimeError('Command "%s" returned %d.' % (cmd, prog.returncode))
  60   return (stdout, stderr)
  61
  62
  63 def _LoadState(output_dir):
  64   """Loads the contents of the state file for a given |output_dir|, returning
  65   None if it doesn't exist.
  66   """
  67   path = os.path.join(output_dir, _STATE)
  68   if not os.path.exists(path):
  69     _LOGGER.debug('No state file found.')
  70     return None
  71   with open(path, 'rb') as f:
  72     _LOGGER.debug('Reading state file: %s', path)
  73     try:
  74       return json.load(f)
  75     except ValueError:
  76       _LOGGER.debug('Invalid state file.')
  77       return None
  78
  79
  80 def _SaveState(output_dir, state, dry_run=False):
  81   """Saves the |state| dictionary to the given |output_dir| as a JSON file."""
  82   path = os.path.join(output_dir, _STATE)
  83   _LOGGER.debug('Writing state file: %s', path)
  84   if dry_run:
  85     return
  86   with open(path, 'wb') as f:
  87     f.write(json.dumps(state, sort_keys=True, indent=2))
  88
  89
  90 def _Md5(path):
  91   """Returns the MD5 hash of the file at |path|, which must exist."""
  92   return hashlib.md5(open(path, 'rb').read()).hexdigest()
  93
  94
  95 def _StateIsValid(state):
  96   """Returns true if the given state structure is valid."""
  97   if not isinstance(state, dict):
  98     _LOGGER.debug('State must be a dict.')
  99     return False
 100   r = state.get('revision', None)
 101   if not isinstance(r, basestring) or not _REVISION_RE.match(r):
 102     _LOGGER.debug('State contains an invalid revision.')
 103     return False
 104   c = state.get('contents', None)
 105   if not isinstance(c, dict):
 106     _LOGGER.debug('State must contain a contents dict.')
 107     return False
 108   for (relpath, md5) in c.iteritems():
 109     if not isinstance(relpath, basestring) or len(relpath) == 0:
 110       _LOGGER.debug('State contents dict contains an invalid path.')
 111       return False
 112     if not isinstance(md5, basestring) or not _MD5_RE.match(md5):
 113       _LOGGER.debug('State contents dict contains an invalid MD5 digest.')
 114       return False
 115   return True
 116
 117
 118 def _BuildActualState(stored, revision, output_dir):
 119   """Builds the actual state using the provided |stored| state as a template.
 120   Only examines files listed in the stored state, causing the script to ignore
 121   files that have been added to the directories locally. |stored| must be a
 122   valid state dictionary.
 123   """
 124   contents = {}
 125   state = { 'revision': revision, 'contents': contents }
 126   for relpath, md5 in stored['contents'].iteritems():
 127     abspath = os.path.abspath(os.path.join(output_dir, relpath))
 128     if os.path.isfile(abspath):
 129       m = _Md5(abspath)
 130       contents[relpath] = m
 131
 132   return state
 133
 134
 135 def _StatesAreConsistent(stored, actual):
 136   """Validates whether two state dictionaries are consistent. Both must be valid
 137   state dictionaries. Additional entries in |actual| are ignored.
 138   """
 139   if stored['revision'] != actual['revision']:
 140     _LOGGER.debug('Mismatched revision number.')
 141     return False
 142   cont_stored = stored['contents']
 143   cont_actual = actual['contents']
 144   for relpath, md5 in cont_stored.iteritems():
 145     if relpath not in cont_actual:
 146       _LOGGER.debug('Missing content: %s', relpath)
 147       return False
 148     if md5 != cont_actual[relpath]:
 149       _LOGGER.debug('Modified content: %s', relpath)
 150       return False
 151   return True
 152
 153
 154 def _GetCurrentState(revision, output_dir):
 155   """Loads the current state and checks to see if it is consistent. Returns
 156   a tuple (state, bool). The returned state will always be valid, even if an
 157   invalid state is present on disk.
 158   """
 159   stored = _LoadState(output_dir)
 160   if not _StateIsValid(stored):
 161     _LOGGER.debug('State is invalid.')
 162     # Return a valid but empty state.
 163     return ({'revision': '0', 'contents': {}}, False)
 164   actual = _BuildActualState(stored, revision, output_dir)
 165   # If the script has been modified consider the state invalid.
 166   path = os.path.join(output_dir, _STATE)
 167   if os.path.getmtime(__file__) > os.path.getmtime(path):
 168     return (stored, False)
 169   # Otherwise, explicitly validate the state.
 170   if not _StatesAreConsistent(stored, actual):
 171     return (stored, False)
 172   return (stored, True)
 173
 174
 175 def _DirIsEmpty(path):
 176   """Returns true if the given directory is empty, false otherwise."""
 177   for root, dirs, files in os.walk(path):
 178     return not dirs and not files
 179
 180
 181 def _RmTreeHandleReadOnly(func, path, exc):
 182   """An error handling function for use with shutil.rmtree. This will
 183   detect failures to remove read-only files, and will change their properties
 184   prior to removing them. This is necessary on Windows as os.remove will return
 185   an access error for read-only files, and git repos contain read-only
 186   pack/index files.
 187   """
 188   excvalue = exc[1]
 189   if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
 190     _LOGGER.debug('Removing read-only path: %s', path)
 191     os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
 192     func(path)
 193   else:
 194     raise
 195
 196
 197 def _RmTree(path):
 198   """A wrapper of shutil.rmtree that handles read-only files."""
 199   shutil.rmtree(path, ignore_errors=False, onerror=_RmTreeHandleReadOnly)
 200
 201
 202 def _CleanState(output_dir, state, dry_run=False):
 203   """Cleans up files/directories in |output_dir| that are referenced by
 204   the given |state|. Raises an error if there are local changes. Returns a
 205   dictionary of files that were deleted.
 206   """
 207   _LOGGER.debug('Deleting files from previous installation.')
 208   deleted = {}
 209
 210   # Generate a list of files to delete, relative to |output_dir|.
 211   contents = state['contents']
 212   files = sorted(contents.keys())
 213
 214   # Try to delete the files. Keep track of directories to delete as well.
 215   dirs = {}
 216   for relpath in files:
 217     fullpath = os.path.join(output_dir, relpath)
 218     fulldir = os.path.dirname(fullpath)
 219     dirs[fulldir] = True
 220     if os.path.exists(fullpath):
 221       # If somehow the file has become a directory complain about it.
 222       if os.path.isdir(fullpath):
 223         raise Exception('Directory exists where file expected: %s' % fullpath)
 224
 225       # Double check that the file doesn't have local changes. If it does
 226       # then refuse to delete it.
 227       if relpath in contents:
 228         stored_md5 = contents[relpath]
 229         actual_md5 = _Md5(fullpath)
 230         if actual_md5 != stored_md5:
 231           raise Exception('File has local changes: %s' % fullpath)
 232
 233       # The file is unchanged so it can safely be deleted.
 234       _LOGGER.debug('Deleting file "%s".', fullpath)
 235       deleted[relpath] = True
 236       if not dry_run:
 237         os.unlink(fullpath)
 238
 239   # Sort directories from longest name to shortest. This lets us remove empty
 240   # directories from the most nested paths first.
 241   dirs = sorted(dirs.keys(), key=lambda x: len(x), reverse=True)
 242   for p in dirs:
 243     if os.path.exists(p) and _DirIsEmpty(p):
 244       _LOGGER.debug('Deleting empty directory "%s".', p)
 245       if not dry_run:
 246         _RmTree(p)
 247
 248   return deleted
 249
 250
 251 def _Download(url):
 252   """Downloads the given URL and returns the contents as a string."""
 253   response = urllib2.urlopen(url)
 254   if response.code != 200:
 255     raise RuntimeError('Failed to download "%s".' % url)
 256   return response.read()
 257
 258
 259 def _InstallBinaries(options, deleted={}):
 260   """Installs Syzygy binaries. This assumes that the output directory has
 261   already been cleaned, as it will refuse to overwrite existing files."""
 262   contents = {}
 263   state = { 'revision': options.revision, 'contents': contents }
 264   archive_url = _SYZYGY_ARCHIVE_URL % { 'revision': options.revision }
 265   if options.resources:
 266     resources = [(resource, resource, '', None)
 267                  for resource in options.resources]
 268   else:
 269     resources = _RESOURCES
 270   for (base, name, subdir, filt) in resources:
 271     # Create the output directory if it doesn't exist.
 272     fulldir = os.path.join(options.output_dir, subdir)
 273     if os.path.isfile(fulldir):
 274       raise Exception('File exists where a directory needs to be created: %s' %
 275                       fulldir)
 276     if not os.path.exists(fulldir):
 277       _LOGGER.debug('Creating directory: %s', fulldir)
 278       if not options.dry_run:
 279         os.makedirs(fulldir)
 280
 281     # Download the archive.
 282     url = archive_url + '/' + base
 283     _LOGGER.debug('Retrieving %s archive at "%s".', name, url)
 284     data = _Download(url)
 285
 286     _LOGGER.debug('Unzipping %s archive.', name)
 287     archive = zipfile.ZipFile(cStringIO.StringIO(data))
 288     for entry in archive.infolist():
 289       if not filt or filt(entry):
 290         fullpath = os.path.normpath(os.path.join(fulldir, entry.filename))
 291         relpath = os.path.relpath(fullpath, options.output_dir)
 292         if os.path.exists(fullpath):
 293           # If in a dry-run take into account the fact that the file *would*
 294           # have been deleted.
 295           if options.dry_run and relpath in deleted:
 296             pass
 297           else:
 298             raise Exception('Path already exists: %s' % fullpath)
 299
 300         # Extract the file and update the state dictionary.
 301         _LOGGER.debug('Extracting "%s".', fullpath)
 302         if not options.dry_run:
 303           archive.extract(entry.filename, fulldir)
 304           md5 = _Md5(fullpath)
 305           contents[relpath] = md5
 306           if sys.platform == 'cygwin':
 307             os.chmod(fullpath, os.stat(fullpath).st_mode | stat.S_IXUSR)
 308
 309   return state
 310
 311
 312 def _ParseCommandLine():
 313   """Parses the command-line and returns an options structure."""
 314   option_parser = optparse.OptionParser()
 315   option_parser.add_option('--dry-run', action='store_true', default=False,
 316       help='If true then will simply list actions that would be performed.')
 317   option_parser.add_option('--force', action='store_true', default=False,
 318       help='Force an installation even if the binaries are up to date.')
 319   option_parser.add_option('--output-dir', type='string',
 320       help='The path where the binaries will be replaced. Existing binaries '
 321            'will only be overwritten if not up to date.')
 322   option_parser.add_option('--overwrite', action='store_true', default=False,
 323       help='If specified then the installation will happily delete and rewrite '
 324            'the entire output directory, blasting any local changes.')
 325   option_parser.add_option('--revision', type='string',
 326       help='The SVN revision or GIT hash associated with the required version.')
 327   option_parser.add_option('--revision-file', type='string',
 328       help='A text file containing an SVN revision or GIT hash.')
 329   option_parser.add_option('--resource', type='string', action='append',
 330       dest='resources', help='A resource to be downloaded.')
 331   option_parser.add_option('--verbose', dest='log_level', action='store_const',
 332       default=logging.INFO, const=logging.DEBUG,
 333       help='Enables verbose logging.')
 334   option_parser.add_option('--quiet', dest='log_level', action='store_const',
 335       default=logging.INFO, const=logging.ERROR,
 336       help='Disables all output except for errors.')
 337   options, args = option_parser.parse_args()
 338   if args:
 339     option_parser.error('Unexpected arguments: %s' % args)
 340   if not options.output_dir:
 341     option_parser.error('Must specify --output-dir.')
 342   if not options.revision and not options.revision_file:
 343     option_parser.error('Must specify one of --revision or --revision-file.')
 344   if options.revision and options.revision_file:
 345     option_parser.error('Must not specify both --revision and --revision-file.')
 346
 347   # Configure logging.
 348   logging.basicConfig(level=options.log_level)
 349
 350   # If a revision file has been specified then read it.
 351   if options.revision_file:
 352     options.revision = open(options.revision_file, 'rb').read().strip()
 353     _LOGGER.debug('Parsed revision "%s" from file "%s".',
 354                  options.revision, options.revision_file)
 355
 356   # Ensure that the specified SVN revision or GIT hash is valid.
 357   if not _REVISION_RE.match(options.revision):
 358     option_parser.error('Must specify a valid SVN or GIT revision.')
 359
 360   # This just makes output prettier to read.
 361   options.output_dir = os.path.normpath(options.output_dir)
 362
 363   return options
 364
 365
 366 def _RemoveOrphanedFiles(options):
 367   """This is run on non-Windows systems to remove orphaned files that may have
 368   been downloaded by a previous version of this script.
 369   """
 370   # Reconfigure logging to output info messages. This will allow inspection of
 371   # cleanup status on non-Windows buildbots.
 372   _LOGGER.setLevel(logging.INFO)
 373
 374   output_dir = os.path.abspath(options.output_dir)
 375
 376   # We only want to clean up the folder in 'src/third_party/syzygy', and we
 377   # expect to be called with that as an output directory. This is an attempt to
 378   # not start deleting random things if the script is run from an alternate
 379   # location, or not called from the gclient hooks.
 380   expected_syzygy_dir = os.path.abspath(os.path.join(
 381       os.path.dirname(__file__), '..', 'third_party', 'syzygy'))
 382   expected_output_dir = os.path.join(expected_syzygy_dir, 'binaries')
 383   if expected_output_dir != output_dir:
 384     _LOGGER.info('Unexpected output directory, skipping cleanup.')
 385     return
 386
 387   if not os.path.isdir(expected_syzygy_dir):
 388     _LOGGER.info('Output directory does not exist, skipping cleanup.')
 389     return
 390
 391   def OnError(function, path, excinfo):
 392     """Logs error encountered by shutil.rmtree."""
 393     _LOGGER.error('Error when running %s(%s)', function, path, exc_info=excinfo)
 394
 395   _LOGGER.info('Removing orphaned files from %s', expected_syzygy_dir)
 396   if not options.dry_run:
 397     shutil.rmtree(expected_syzygy_dir, True, OnError)
 398
 399
 400 def main():
 401   options = _ParseCommandLine()
 402
 403   if options.dry_run:
 404     _LOGGER.debug('Performing a dry-run.')
 405
 406   # We only care about Windows platforms, as the Syzygy binaries aren't used
 407   # elsewhere. However, there was a short period of time where this script
 408   # wasn't gated on OS types, and those OSes downloaded and installed binaries.
 409   # This will cleanup orphaned files on those operating systems.
 410   if sys.platform not in ('win32', 'cygwin'):
 411     return _RemoveOrphanedFiles(options)
 412
 413   # Load the current installation state, and validate it against the
 414   # requested installation.
 415   state, is_consistent = _GetCurrentState(options.revision, options.output_dir)
 416
 417   # Decide whether or not an install is necessary.
 418   if options.force:
 419     _LOGGER.debug('Forcing reinstall of binaries.')
 420   elif is_consistent:
 421     # Avoid doing any work if the contents of the directory are consistent.
 422     _LOGGER.debug('State unchanged, no reinstall necessary.')
 423     return
 424
 425   # Under normal logging this is the only only message that will be reported.
 426   _LOGGER.info('Installing revision %s Syzygy binaries.',
 427                options.revision[0:12])
 428
 429   # Clean up the old state to begin with.
 430   deleted = []
 431   if options.overwrite:
 432     if os.path.exists(options.output_dir):
 433       # If overwrite was specified then take a heavy-handed approach.
 434       _LOGGER.debug('Deleting entire installation directory.')
 435       if not options.dry_run:
 436         _RmTree(options.output_dir)
 437   else:
 438     # Otherwise only delete things that the previous installation put in place,
 439     # and take care to preserve any local changes.
 440     deleted = _CleanState(options.output_dir, state, options.dry_run)
 441
 442   # Install the new binaries. In a dry-run this will actually download the
 443   # archives, but it won't write anything to disk.
 444   state = _InstallBinaries(options, deleted)
 445
 446   # Build and save the state for the directory.
 447   _SaveState(options.output_dir, state, options.dry_run)
 448
 449
 450 if __name__ == '__main__':
 451   main()