scripts/release/release.py

   1 #!/usr/bin/python2.5
   2 #
   3 # Copyright 2009 the Melange authors.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #   http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 from __future__ import with_statement
  18
  19 """Google Summer of Code Melange release script.
  20
  21 This script provides automation for the various tasks involved in
  22 pushing a new release of Melange to the official Google Summer of Code
  23 app engine instance.
  24
  25 It does not provide a turnkey autopilot solution. Notably, each stage
  26 of the release process must be started by a human operator, and some
  27 commands will request confirmation or extra details before
  28 proceeding. It is not a replacement for a cautious human
  29 operator.
  30
  31 Note that this script requires:
  32  - Python 2.5 or better (for various language features)
  33
  34  - Subversion 1.5.0 or better (for working copy depth control, which
  35      cuts down checkout/update times by several orders of
  36      magnitude).
  37 """
  38
  39 __authors__ = [
  40     # alphabetical order by last name, please
  41     '"David Anderson" <dave@natulte.net>',
  42     ]
  43
  44
  45 import functools
  46 import os
  47 import re
  48 import subprocess
  49 import sys
  50
  51 import error
  52 import log
  53 import subversion
  54 import util
  55
  56
  57 # Default repository URLs for Melange and the Google release
  58 # repository.
  59 MELANGE_REPOS = 'http://soc.googlecode.com/svn'
  60 GOOGLE_SOC_REPOS = 'https://soc-google.googlecode.com/svn'
  61
  62
  63 # Regular expression matching an apparently well formed Melange
  64 # release number.
  65 MELANGE_RELEASE_RE = re.compile(r'\d-\d-\d{8}p\d+')
  66
  67
  68 class Error(error.Error):
  69   pass
  70
  71
  72 class AbortedByUser(Error):
  73   """The operation was aborted by the user."""
  74   pass
  75
  76
  77 class FileAccessError(Error):
  78   """An error occured while accessing a file."""
  79   pass
  80
  81
  82 def getString(prompt):
  83   """Prompt for and return a string."""
  84   prompt += ' '
  85   log.stdout.write(prompt)
  86   log.stdout.flush()
  87
  88   response = sys.stdin.readline()
  89   log.terminal_echo(prompt + response.strip())
  90   if not response:
  91     raise AbortedByUser('Aborted by ctrl+D')
  92
  93   return response.strip()
  94
  95
  96 def confirm(prompt, default=False):
  97   """Ask a yes/no question and return the answer.
  98
  99   Will reprompt the user until one of "yes", "no", "y" or "n" is
 100   entered. The input is case insensitive.
 101
 102   Args:
 103     prompt: The question to ask the user.
 104     default: The answer to return if the user just hits enter.
 105
 106   Returns:
 107     True if the user answered affirmatively, False otherwise.
 108   """
 109   if default:
 110     question = prompt + ' [Yn]'
 111   else:
 112     question = prompt + ' [yN]'
 113   while True:
 114     answer = getString(question)
 115     if not answer:
 116       return default
 117     elif answer in ('y', 'yes'):
 118       return True
 119     elif answer in ('n', 'no'):
 120       return False
 121     else:
 122       log.error('Please answer yes or no.')
 123
 124
 125 def getNumber(prompt):
 126   """Prompt for and return a number.
 127
 128   Will reprompt the user until a number is entered.
 129   """
 130   while True:
 131     value_str = getString(prompt)
 132     try:
 133       return int(value_str)
 134     except ValueError:
 135       log.error('Please enter a number. You entered "%s".' % value_str)
 136
 137
 138 def getChoice(intro, prompt, choices, done=None, suggest=None):
 139   """Prompt for and return a choice from a menu.
 140
 141   Will reprompt the user until a valid menu entry is chosen.
 142
 143   Args:
 144     intro: Text to print verbatim before the choice menu.
 145     prompt: The prompt to print right before accepting input.
 146     choices: The list of string choices to display.
 147     done: If not None, the list of indices of previously
 148       selected/completed choices.
 149     suggest: If not None, the index of the choice to highlight as
 150       the suggested choice.
 151
 152   Returns:
 153     The index in the choices list of the selection the user made.
 154   """
 155   done = set(done or [])
 156   while True:
 157     print intro
 158     print
 159     for i, entry in enumerate(choices):
 160       done_text = ' (done)' if i in done else ''
 161       indent = '--> ' if i == suggest else '    '
 162       print '%s%2d. %s%s' % (indent, i+1, entry, done_text)
 163     print
 164     choice = getNumber(prompt)
 165     if 0 < choice <= len(choices):
 166       return choice-1
 167     log.error('%d is not a valid choice between %d and %d' %
 168               (choice, 1, len(choices)))
 169     print
 170
 171
 172 def fileToLines(path):
 173   """Read a file and return it as a list of lines."""
 174   try:
 175     with file(path) as f:
 176       return f.read().split('\n')
 177   except (IOError, OSError), e:
 178     raise FileAccessError(str(e))
 179
 180
 181 def linesToFile(path, lines):
 182   """Write a list of lines to a file."""
 183   try:
 184     with file(path, 'w') as f:
 185       f.write('\n'.join(lines))
 186   except (IOError, OSError), e:
 187     raise FileAccessError(str(e))
 188
 189
 190 #
 191 # Decorators for use in ReleaseEnvironment.
 192 #
 193 def pristine_wc(f):
 194   """A decorator that cleans up the release repository."""
 195   @functools.wraps(f)
 196   def revert_wc(self, *args, **kwargs):
 197     self.wc.revert()
 198     return f(self, *args, **kwargs)
 199   return revert_wc
 200
 201
 202 def requires_branch(f):
 203   """A decorator that checks that a release branch is active."""
 204   @functools.wraps(f)
 205   def check_branch(self, *args, **kwargs):
 206     if self.branch is None:
 207       raise error.ExpectationFailed(
 208         'This operation requires an active release branch')
 209     return f(self, *args, **kwargs)
 210   return check_branch
 211
 212
 213 class ReleaseEnvironment(util.Paths):
 214   """Encapsulates the state of a Melange release rolling environment.
 215
 216   This class contains the actual releasing logic, and makes use of
 217   the previously defined utility classes to carry out user commands.
 218
 219   Attributes:
 220     release_repos: The URL to the Google release repository root.
 221     upstream_repos: The URL to the Melange upstream repository root.
 222     wc: A Subversion object encapsulating a Google SoC working copy.
 223   """
 224
 225   BRANCH_FILE = 'BRANCH'
 226
 227   def __init__(self, root, release_repos, upstream_repos):
 228     """Initializer.
 229
 230     Args:
 231       root: The root of the release environment.
 232       release_repos: The URL to the Google release repository root.
 233       upstream_repos: The URL to the Melange upstream repository root.
 234     """
 235     util.Paths.__init__(self, root)
 236     self.wc = subversion.WorkingCopy(self.path('google-soc'))
 237     self.release_repos = release_repos.strip('/')
 238     self.upstream_repos = upstream_repos.strip('/')
 239
 240     if not self.wc.exists():
 241       self._InitializeWC()
 242     else:
 243       self.wc.revert()
 244
 245       if self.exists(self.BRANCH_FILE):
 246         branch = fileToLines(self.path(self.BRANCH_FILE))[0]
 247         self._switchBranch(branch)
 248       else:
 249         self._switchBranch(None)
 250
 251   def _InitializeWC(self):
 252     """Check out the initial release repository.
 253
 254     Will also select the latest release branch, if any, so that
 255     the end state is a fully ready to function release environment.
 256     """
 257     log.info('Checking out the release repository')
 258
 259     # Check out a sparse view of the relevant repository paths.
 260     self.wc.checkout(self.release_repos, depth='immediates')
 261     self.wc.update('vendor', depth='immediates')
 262     self.wc.update('branches', depth='immediates')
 263     self.wc.update('tags', depth='immediates')
 264
 265     # Locate the most recent release branch, if any, and switch
 266     # the release environment to it.
 267     branches = self._listBranches()
 268     if not branches:
 269       self._switchBranch(None)
 270     else:
 271       self._switchBranch(branches[-1])
 272
 273   def _listBranches(self):
 274     """Return a list of available Melange release branches.
 275
 276     Branches are returned in sorted order, from least recent to
 277     most recent in release number ordering.
 278     """
 279     assert self.wc.exists('branches')
 280     branches = self.wc.ls('branches')
 281
 282     # Some early release branches used a different naming scheme
 283     # that doesn't sort properly with new-style release names. We
 284     # filter those out here, along with empty lines.
 285     branches = [b.strip('/') for b in branches
 286           if MELANGE_RELEASE_RE.match(b.strip('/'))]
 287
 288     return sorted(branches)
 289
 290   def _switchBranch(self, release):
 291     """Activate the branch matching the given release.
 292
 293     Once activated, this branch is the target of future release
 294     operations.
 295
 296     None can be passed as the release. The result is that no
 297     branch is active, and all operations that require an active
 298     branch will fail until a branch is activated again. This is
 299     used only at initialization, when it is detected that there
 300     are no available release branches to activate.
 301
 302     Args:
 303       release: The version number of a Melange release already
 304         imported in the release repository, or None to activate
 305         no branch.
 306
 307     """
 308     if release is None:
 309       self.branch = None
 310       self.branch_dir = None
 311       log.info('No release branch available')
 312     else:
 313       self.wc.update()
 314       assert self.wc.exists('branches/' + release)
 315       linesToFile(self.path(self.BRANCH_FILE), [release])
 316       self.branch = release
 317       self.branch_dir = 'branches/' + release
 318       self.wc.update(self.branch_dir, depth='infinity')
 319       log.info('Working on branch ' + self.branch)
 320
 321   def _branchPath(self, path):
 322     """Return the given path with the release branch path prepended."""
 323     assert self.branch_dir is not None
 324     return os.path.join(self.branch_dir, path)
 325
 326   #
 327   # Release engineering commands. See further down for their
 328   # integration into a commandline interface.
 329   #
 330   @pristine_wc
 331   def update(self):
 332     """Update and clean the release repository"""
 333     self.wc.update()
 334
 335   @pristine_wc
 336   def switchToBranch(self):
 337     """Switch to another Melange release branch"""
 338     branches = self._listBranches()
 339     if not branches:
 340       raise error.ExpectationFailed(
 341         'No branches available. Please import one.')
 342
 343     choice = getChoice('Available release branches:',
 344                'Your choice?',
 345                branches,
 346                suggest=len(branches)-1)
 347     self._switchBranch(branches[choice])
 348
 349   def _addAppYaml(self):
 350     """Create a Google production app.yaml configuration.
 351
 352     The file is copied and modified from the upstream
 353     app.yaml.template, configure for Google's Summer of Code App
 354     Engine instance, and committed.
 355     """
 356     if self.wc.exists(self._branchPath('app/app.yaml')):
 357       raise ObstructionError('app/app.yaml exists already')
 358
 359     yaml_path = self._branchPath('app/app.yaml')
 360     self.wc.copy(yaml_path + '.template', yaml_path)
 361
 362     yaml = fileToLines(self.wc.path(yaml_path))
 363     out = []
 364     for i, line in enumerate(yaml):
 365       stripped_line = line.strip()
 366       if 'TODO' in stripped_line:
 367         continue
 368       elif stripped_line == '# application: FIXME':
 369         out.append('application: socghop')
 370       elif stripped_line.startswith('version:'):
 371         out.append(line.lstrip() + 'g0')
 372         out.append('# * initial Google fork of Melange ' + self.branch)
 373       else:
 374         out.append(line)
 375     linesToFile(self.wc.path(yaml_path), out)
 376
 377     self.wc.commit('Create app.yaml with Google patch version g0 '
 378              'in branch ' + self.branch)
 379
 380   def _applyGooglePatches(self):
 381     """Apply Google-specific patches to a vanilla Melange release.
 382
 383     Each patch is applied and committed in turn.
 384     """
 385     # Edit the base template to point users to the Google fork
 386     # of the Melange codebase instead of the vanilla release.
 387     tmpl_file = self.wc.path(
 388       self._branchPath('app/soc/templates/soc/base.html'))
 389     tmpl = fileToLines(tmpl_file)
 390     for i, line in enumerate(tmpl):
 391       if 'http://code.google.com/p/soc/source/browse/tags/' in line:
 392         tmpl[i] = line.replace('/p/soc/', '/p/soc-google/')
 393         break
 394     else:
 395       raise error.ExpectationFailed(
 396         'No source code link found in base.html')
 397     linesToFile(tmpl_file, tmpl)
 398
 399     self.wc.commit(
 400       'Customize the Melange release link in the sidebar menu')
 401
 402   @pristine_wc
 403   def importTag(self):
 404     """Import a new Melange release"""
 405     release = getString('Enter the Melange release to import:')
 406     if not release:
 407       AbortedByUser('No release provided, import aborted')
 408
 409     branch_dir = 'branches/' + release
 410     if self.wc.exists(branch_dir):
 411       raise ObstructionError('Release %s already imported' % release)
 412
 413     tag_url = '%s/tags/%s' % (self.upstream_repos, release)
 414     release_rev = subversion.find_tag_rev(tag_url)
 415
 416     if confirm('Confirm import of release %s, tagged at r%d?' %
 417            (release, release_rev)):
 418       # Add an entry to the vendor externals for the Melange
 419       # release.
 420       externals = self.wc.propget('svn:externals', 'vendor/soc')
 421       externals.append('%s -r %d %s' % (release, release_rev, tag_url))
 422       self.wc.propset('svn:externals', '\n'.join(externals), 'vendor/soc')
 423       self.wc.commit('Add svn:externals entry to pull in Melange '
 424           'release %s at r%d.' % (release, release_rev))
 425
 426       # Export the tag into the release repository's branches
 427       subversion.export(tag_url, release_rev, self.wc.path(branch_dir))
 428
 429       # Add and commit the branch add (very long operation!)
 430       self.wc.add([branch_dir])
 431       self.wc.commit('Branch of Melange release %s' % release, branch_dir)
 432       self._switchBranch(release)
 433
 434       # Commit the production GSoC configuration and
 435       # google-specific patches.
 436       self._addAppYaml()
 437       self._applyGooglePatches()
 438
 439       # All done!
 440       log.info('Melange release %s imported and googlified' % self.branch)
 441
 442   @requires_branch
 443   @pristine_wc
 444   def cherryPickChange(self):
 445     """Cherry-pick a change from the Melange trunk"""
 446     rev = getNumber('Revision number to cherry-pick:')
 447     bug = getNumber('Issue fixed by this change:')
 448
 449     diff = subversion.diff(self.upstream_repos + '/trunk', rev)
 450     if not diff.strip():
 451       raise error.ExpectationFailed(
 452         'Retrieved diff is empty. '
 453         'Did you accidentally cherry-pick a branch change?')
 454     util.run(['patch', '-p0'], cwd=self.wc.path(self.branch_dir), stdin=diff)
 455     self.wc.addRemove(self.branch_dir)
 456
 457     yaml_path = self.wc.path(self._branchPath('app/app.yaml'))
 458     out = []
 459     updated_patchlevel = False
 460     for line in fileToLines(yaml_path):
 461       if line.strip().startswith('version: '):
 462         version = line.strip().split()[-1]
 463         base, patch = line.rsplit('g', 1)
 464         new_version = '%sg%d' % (base, int(patch) + 1)
 465         message = ('Cherry-picked r%d from /p/soc/ to fix issue %d' %
 466             (rev, bug))
 467         out.append('version: ' + new_version)
 468         out.append('# * ' + message)
 469         updated_patchlevel = True
 470       else:
 471         out.append(line)
 472
 473     if not updated_patchlevel:
 474       log.error('Failed to update Google patch revision')
 475       log.error('Cherry-picking failed')
 476
 477     linesToFile(yaml_path, out)
 478
 479     log.info('Check the diff about to be committed with:')
 480     log.info('svn diff ' + self.wc.path(self.branch_dir))
 481     if not confirm('Commit this change?'):
 482       raise AbortedByUser('Cherry-pick aborted')
 483     self.wc.commit(message)
 484     log.info('Cherry-picked r%d from the Melange trunk.' % rev)
 485
 486   MENU_ORDER = [
 487     update,
 488     switchToBranch,
 489     importTag,
 490     cherryPickChange,
 491     ]
 492
 493   MENU_STRINGS = [d.__doc__ for d in MENU_ORDER]
 494
 495   MENU_SUGGESTIONS = {
 496     None: update,
 497     update: cherryPickChange,
 498     switchToBranch: cherryPickChange,
 499     importTag: cherryPickChange,
 500     cherryPickChange: None,
 501     }
 502
 503   def interactiveMenu(self):
 504     done = []
 505     last_choice = None
 506     while True:
 507       # Show the user their previously completed operations and
 508       # a suggested next op, to remind them where they are in
 509       # the release process (useful after long operations that
 510       # may have caused lunch or an extended context switch).
 511       if last_choice is not None:
 512         last_command = self.MENU_ORDER[last_choice]
 513       else:
 514         last_command = None
 515       suggested_next = self.MENU_ORDER.index(
 516         self.MENU_SUGGESTIONS[last_command])
 517
 518       try:
 519         choice = getChoice('Main menu:', 'Your choice?',
 520           self.MENU_STRINGS, done=done, suggest=suggested_next)
 521       except (KeyboardInterrupt, AbortedByUser):
 522         log.info('Exiting.')
 523         return
 524       try:
 525         self.MENU_ORDER[choice](self)
 526       except error.Error, e:
 527         log.error(str(e))
 528       else:
 529         done.append(choice)
 530         last_choice = choice
 531
 532
 533 def main(argv):
 534   if not (1 <= len(argv) <= 3):
 535     print ('Usage: gsoc-release.py [release repos root URL] '
 536            '[upstream repos root URL]')
 537     sys.exit(1)
 538
 539   release_repos, upstream_repos = GOOGLE_SOC_REPOS, MELANGE_REPOS
 540   if len(argv) >= 2:
 541     release_repos = argv[1]
 542   if len(argv) == 3:
 543     upstream_repos = argv[2]
 544
 545   log.init('release.log')
 546
 547   log.info('Release repository: ' + release_repos)
 548   log.info('Upstream repository: ' + upstream_repos)
 549
 550   r = ReleaseEnvironment(os.path.abspath('_release_'),
 551                          release_repos,
 552                          upstream_repos)
 553   r.interactiveMenu()
 554
 555
 556 if __name__ == '__main__':
 557   main(sys.argv)