tools/roll_angle.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import argparse
   7 import collections
   8 import logging
   9 import os
  10 import re
  11 import subprocess
  12 import sys
  13 import time
  14
  15 extra_trybots = [
  16   'win_clang_dbg',
  17 ]
  18
  19 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
  20 SRC_DIR = os.path.abspath(os.path.join(SCRIPT_DIR, os.pardir))
  21 import find_depot_tools
  22 find_depot_tools.add_depot_tools_to_path()
  23 import roll_dep_svn
  24 from gclient import GClientKeywords
  25 from third_party import upload
  26
  27 # Avoid depot_tools/third_party/upload.py print verbose messages.
  28 upload.verbosity = 0  # Errors only.
  29
  30 CHROMIUM_GIT_URL = 'https://chromium.googlesource.com/chromium/src.git'
  31 CL_ISSUE_RE = re.compile('^Issue number: ([0-9]+) \((.*)\)$')
  32 RIETVELD_URL_RE = re.compile('^https?://(.*)/(.*)')
  33 ROLL_BRANCH_NAME = 'special_angle_roll_branch'
  34 TRYJOB_STATUS_SLEEP_SECONDS = 30
  35
  36 # Use a shell for subcommands on Windows to get a PATH search.
  37 IS_WIN = sys.platform.startswith('win')
  38 ANGLE_PATH = os.path.join('third_party', 'angle')
  39
  40 CommitInfo = collections.namedtuple('CommitInfo', ['git_commit',
  41                                                    'git_repo_url'])
  42 CLInfo = collections.namedtuple('CLInfo', ['issue', 'url', 'rietveld_server'])
  43
  44 def _PosixPath(path):
  45   """Convert a possibly-Windows path to a posix-style path."""
  46   (_, path) = os.path.splitdrive(path)
  47   return path.replace(os.sep, '/')
  48
  49 def _ParseGitCommitHash(description):
  50   for line in description.splitlines():
  51     if line.startswith('commit '):
  52       return line.split()[1]
  53   logging.error('Failed to parse git commit id from:\n%s\n', description)
  54   sys.exit(-1)
  55   return None
  56
  57
  58 def _ParseDepsFile(filename):
  59   with open(filename, 'rb') as f:
  60     deps_content = f.read()
  61   return _ParseDepsDict(deps_content)
  62
  63
  64 def _ParseDepsDict(deps_content):
  65   local_scope = {}
  66   var = GClientKeywords.VarImpl({}, local_scope)
  67   global_scope = {
  68     'File': GClientKeywords.FileImpl,
  69     'From': GClientKeywords.FromImpl,
  70     'Var': var.Lookup,
  71     'deps_os': {},
  72   }
  73   exec(deps_content, global_scope, local_scope)
  74   return local_scope
  75
  76
  77 def _GenerateCLDescriptionCommand(angle_current, angle_new, bugs):
  78   def GetChangeString(current_hash, new_hash):
  79     return '%s..%s' % (current_hash[0:7], new_hash[0:7]);
  80
  81   def GetChangeLogURL(git_repo_url, change_string):
  82     return '%s/+log/%s' % (git_repo_url, change_string)
  83
  84   def GetBugString(bugs):
  85     bug_str = 'BUG='
  86     for bug in bugs:
  87       bug_str += str(bug) + ','
  88     return bug_str.rstrip(',')
  89
  90   if angle_current.git_commit != angle_new.git_commit:
  91     change_str = GetChangeString(angle_current.git_commit,
  92                                  angle_new.git_commit)
  93     changelog_url = GetChangeLogURL(angle_current.git_repo_url,
  94                                     change_str)
  95
  96   return [
  97     '-m', 'Roll ANGLE ' + change_str,
  98     '-m', '%s' % changelog_url,
  99     '-m', GetBugString(bugs),
 100     '-m', 'TEST=bots',
 101   ]
 102
 103
 104 class AutoRoller(object):
 105   def __init__(self, chromium_src):
 106     self._chromium_src = chromium_src
 107
 108   def _RunCommand(self, command, working_dir=None, ignore_exit_code=False,
 109                   extra_env=None):
 110     """Runs a command and returns the stdout from that command.
 111
 112     If the command fails (exit code != 0), the function will exit the process.
 113     """
 114     working_dir = working_dir or self._chromium_src
 115     logging.debug('cmd: %s cwd: %s', ' '.join(command), working_dir)
 116     env = os.environ.copy()
 117     if extra_env:
 118       logging.debug('extra env: %s', extra_env)
 119       env.update(extra_env)
 120     p = subprocess.Popen(command, stdout=subprocess.PIPE,
 121                          stderr=subprocess.PIPE, shell=IS_WIN, env=env,
 122                          cwd=working_dir, universal_newlines=True)
 123     output = p.stdout.read()
 124     p.wait()
 125     p.stdout.close()
 126     p.stderr.close()
 127
 128     if not ignore_exit_code and p.returncode != 0:
 129       logging.error('Command failed: %s\n%s', str(command), output)
 130       sys.exit(p.returncode)
 131     return output
 132
 133   def _GetCommitInfo(self, path_below_src, git_hash=None, git_repo_url=None):
 134     working_dir = os.path.join(self._chromium_src, path_below_src)
 135     self._RunCommand(['git', 'fetch', 'origin'], working_dir=working_dir)
 136     revision_range = git_hash or 'origin'
 137     ret = self._RunCommand(
 138         ['git', '--no-pager', 'log', revision_range, '--pretty=full', '-1'],
 139         working_dir=working_dir)
 140     return CommitInfo(_ParseGitCommitHash(ret), git_repo_url)
 141
 142   def _GetDepsCommitInfo(self, deps_dict, path_below_src):
 143     entry = deps_dict['deps'][_PosixPath('src/%s' % path_below_src)]
 144     at_index = entry.find('@')
 145     git_repo_url = entry[:at_index]
 146     git_hash = entry[at_index + 1:]
 147     return self._GetCommitInfo(path_below_src, git_hash, git_repo_url)
 148
 149   def _GetCLInfo(self):
 150     cl_output = self._RunCommand(['git', 'cl', 'issue'])
 151     m = CL_ISSUE_RE.match(cl_output.strip())
 152     if not m:
 153       logging.error('Cannot find any CL info. Output was:\n%s', cl_output)
 154       sys.exit(-1)
 155     issue_number = int(m.group(1))
 156     url = m.group(2)
 157
 158     # Parse the Rietveld host from the URL.
 159     m = RIETVELD_URL_RE.match(url)
 160     if not m:
 161       logging.error('Cannot parse Rietveld host from URL: %s', url)
 162       sys.exit(-1)
 163     rietveld_server = m.group(1)
 164     return CLInfo(issue_number, url, rietveld_server)
 165
 166   def _GetCurrentBranchName(self):
 167     return self._RunCommand(
 168         ['git', 'rev-parse', '--abbrev-ref', 'HEAD']).splitlines()[0]
 169
 170   def _IsTreeClean(self):
 171     lines = self._RunCommand(
 172       ['git', 'status', '--porcelain', '-uno']).splitlines()
 173     if len(lines) == 0:
 174       return True
 175
 176     logging.debug('Dirty/unversioned files:\n%s', '\n'.join(lines))
 177     return False
 178
 179   def _GetBugList(self, path_below_src, angle_current, angle_new):
 180     working_dir = os.path.join(self._chromium_src, path_below_src)
 181     lines = self._RunCommand(
 182         ['git','log',
 183             '%s..%s' % (angle_current.git_commit, angle_new.git_commit)],
 184         working_dir=working_dir).split('\n')
 185     bugs = set()
 186     for line in lines:
 187       line = line.strip()
 188       bug_prefix = 'BUG='
 189       if line.startswith(bug_prefix):
 190         bugs_strings = line[len(bug_prefix):].split(',')
 191         for bug_string in bugs_strings:
 192           try:
 193             bugs.add(int(bug_string))
 194           except:
 195             # skip this, it may be a project specific bug such as
 196             # "angleproject:X" or an ill-formed BUG= message
 197             pass
 198     return bugs
 199
 200   def _UpdateReadmeFile(self, readme_path, new_revision):
 201     readme = open(os.path.join(self._chromium_src, readme_path), 'r+')
 202     txt = readme.read()
 203     m = re.sub(re.compile('.*^Revision\: ([0-9]*).*', re.MULTILINE),
 204         ('Revision: %s' % new_revision), txt)
 205     readme.seek(0)
 206     readme.write(m)
 207     readme.truncate()
 208
 209   def PrepareRoll(self, ignore_checks):
 210     # TODO(kjellander): use os.path.normcase, os.path.join etc for all paths for
 211     # cross platform compatibility.
 212
 213     if not ignore_checks:
 214       if self._GetCurrentBranchName() != 'master':
 215         logging.error('Please checkout the master branch.')
 216         return -1
 217       if not self._IsTreeClean():
 218         logging.error('Please make sure you don\'t have any modified files.')
 219         return -1
 220
 221     # Always clean up any previous roll.
 222     self.Abort()
 223
 224     logging.debug('Pulling latest changes')
 225     if not ignore_checks:
 226       self._RunCommand(['git', 'pull'])
 227
 228     self._RunCommand(['git', 'checkout', '-b', ROLL_BRANCH_NAME])
 229
 230     # Modify Chromium's DEPS file.
 231
 232     # Parse current hashes.
 233     deps_filename = os.path.join(self._chromium_src, 'DEPS')
 234     deps = _ParseDepsFile(deps_filename)
 235     angle_current = self._GetDepsCommitInfo(deps, ANGLE_PATH)
 236
 237     # Find ToT revisions.
 238     angle_latest = self._GetCommitInfo(ANGLE_PATH)
 239
 240     if IS_WIN:
 241       # Make sure the roll script doesn't use windows line endings
 242       self._RunCommand(['git', 'config', 'core.autocrlf', 'true'])
 243
 244     self._UpdateDep(deps_filename, ANGLE_PATH, angle_latest)
 245
 246     if self._IsTreeClean():
 247       logging.debug('Tree is clean - no changes detected.')
 248       self._DeleteRollBranch()
 249     else:
 250       bugs = self._GetBugList(ANGLE_PATH, angle_current, angle_latest)
 251       description = _GenerateCLDescriptionCommand(
 252           angle_current, angle_latest, bugs)
 253       logging.debug('Committing changes locally.')
 254       self._RunCommand(['git', 'add', '--update', '.'])
 255       self._RunCommand(['git', 'commit'] + description)
 256       logging.debug('Uploading changes...')
 257       self._RunCommand(['git', 'cl', 'upload'],
 258                        extra_env={'EDITOR': 'true'})
 259
 260       # Run the default trybots
 261       base_try_cmd = ['git', 'cl', 'try']
 262       self._RunCommand(base_try_cmd)
 263
 264       if extra_trybots:
 265         # Run additional tryjobs
 266         extra_try_args = []
 267         for extra_trybot in extra_trybots:
 268           extra_try_args += ['-b', extra_trybot]
 269         self._RunCommand(base_try_cmd + extra_try_args)
 270
 271       cl_info = self._GetCLInfo()
 272       print 'Issue: %d URL: %s' % (cl_info.issue, cl_info.url)
 273
 274     # Checkout master again.
 275     self._RunCommand(['git', 'checkout', 'master'])
 276     print 'Roll branch left as ' + ROLL_BRANCH_NAME
 277     return 0
 278
 279   def _UpdateDep(self, deps_filename, dep_relative_to_src, commit_info):
 280     dep_name = _PosixPath(os.path.join('src', dep_relative_to_src))
 281
 282     # roll_dep_svn.py relies on cwd being the Chromium checkout, so let's
 283     # temporarily change the working directory and then change back.
 284     cwd = os.getcwd()
 285     os.chdir(os.path.dirname(deps_filename))
 286     roll_dep_svn.update_deps(deps_filename, dep_relative_to_src, dep_name,
 287                              commit_info.git_commit, '')
 288     os.chdir(cwd)
 289
 290   def _DeleteRollBranch(self):
 291     self._RunCommand(['git', 'checkout', 'master'])
 292     self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
 293     logging.debug('Deleted the local roll branch (%s)', ROLL_BRANCH_NAME)
 294
 295
 296   def _GetBranches(self):
 297     """Returns a tuple of active,branches.
 298
 299     The 'active' is the name of the currently active branch and 'branches' is a
 300     list of all branches.
 301     """
 302     lines = self._RunCommand(['git', 'branch']).split('\n')
 303     branches = []
 304     active = ''
 305     for l in lines:
 306       if '*' in l:
 307         # The assumption is that the first char will always be the '*'.
 308         active = l[1:].strip()
 309         branches.append(active)
 310       else:
 311         b = l.strip()
 312         if b:
 313           branches.append(b)
 314     return (active, branches)
 315
 316   def Abort(self):
 317     active_branch, branches = self._GetBranches()
 318     if active_branch == ROLL_BRANCH_NAME:
 319       active_branch = 'master'
 320     if ROLL_BRANCH_NAME in branches:
 321       print 'Aborting pending roll.'
 322       self._RunCommand(['git', 'checkout', ROLL_BRANCH_NAME])
 323       # Ignore an error here in case an issue wasn't created for some reason.
 324       self._RunCommand(['git', 'cl', 'set_close'], ignore_exit_code=True)
 325       self._RunCommand(['git', 'checkout', active_branch])
 326       self._RunCommand(['git', 'branch', '-D', ROLL_BRANCH_NAME])
 327     return 0
 328
 329
 330 def main():
 331   parser = argparse.ArgumentParser(
 332       description='Auto-generates a CL containing an ANGLE roll.')
 333   parser.add_argument('--abort',
 334     help=('Aborts a previously prepared roll. '
 335           'Closes any associated issues and deletes the roll branches'),
 336     action='store_true')
 337   parser.add_argument('--ignore-checks', action='store_true', default=False,
 338       help=('Skips checks for being on the master branch, dirty workspaces and '
 339             'the updating of the checkout. Will still delete and create local '
 340             'Git branches.'))
 341   parser.add_argument('-v', '--verbose', action='store_true', default=False,
 342       help='Be extra verbose in printing of log messages.')
 343   args = parser.parse_args()
 344
 345   if args.verbose:
 346     logging.basicConfig(level=logging.DEBUG)
 347   else:
 348     logging.basicConfig(level=logging.ERROR)
 349
 350   autoroller = AutoRoller(SRC_DIR)
 351   if args.abort:
 352     return autoroller.Abort()
 353   else:
 354     return autoroller.PrepareRoll(args.ignore_checks)
 355
 356 if __name__ == '__main__':
 357   sys.exit(main())