release.py

   1 # Copyright (C) 2007, Thomas Leonard
   2 # See the README file for details, or visit http://0install.net.
   3
   4 import os, sys, subprocess, shutil, tarfile, tempfile
   5 from zeroinstall import SafeException
   6 from zeroinstall.injector import reader, model
   7 from logging import info
   8 from scm import GIT
   9
  10 XMLNS_RELEASE = 'http://zero-install.sourceforge.net/2007/namespaces/0release'
  11
  12 release_status_file = 'release-status'
  13
  14 valid_phases = ['commit-release']
  15
  16 def run_unit_tests(impl):
  17         self_test = impl.metadata.get('self-test', None)
  18         if self_test is None:
  19                 print "SKIPPING unit tests for %s (no 'self-test' attribute set)" % impl
  20                 return
  21         self_test = os.path.join(impl.id, self_test)
  22         print "Running self-test:", self_test
  23         exitstatus = subprocess.call([self_test], cwd = os.path.dirname(self_test))
  24         if exitstatus:
  25                 raise SafeException("Self-test failed with exit status %d" % exitstatus)
  26
  27 def show_and_run(cmd, args):
  28         print "Executing: %s %s" % (cmd, ' '.join("[%s]" % x for x in args))
  29         subprocess.check_call(['sh', '-c', cmd, '-'] + args)
  30
  31 def suggest_release_version(snapshot_version):
  32         """Given a snapshot version, suggest a suitable release version.
  33         >>> suggest_release_version('1.0-pre')
  34         '1.0'
  35         >>> suggest_release_version('0.9-post')
  36         '0.10'
  37         >>> suggest_release_version('3')
  38         Traceback (most recent call last):
  39                 ...
  40         SafeException: Version '3' is not a snapshot version (should end in -pre or -post)
  41         """
  42         version = model.parse_version(snapshot_version)
  43         mod = version[-1]
  44         if mod == 0:
  45                 raise SafeException("Version '%s' is not a snapshot version (should end in -pre or -post)" % snapshot_version)
  46         if mod > 0:
  47                 # -post, so increment the number
  48                 version[-2][-1] += 1
  49         version[-1] = 0 # Remove the modifier
  50         return model.format_version(version)
  51
  52 def publish(iface, **kwargs):
  53         args = [os.environ['0PUBLISH']]
  54         for k in kwargs:
  55                 value = kwargs[k]
  56                 if value is True:
  57                         args += ['--' + k.replace('_', '-')]
  58                 elif value is not None:
  59                         args += ['--' + k.replace('_', '-'), value]
  60         args.append(iface)
  61         info("Executing %s", args)
  62         subprocess.check_call(args)
  63
  64 def get_singleton_impl(iface):
  65         impls = iface.implementations
  66         if len(impls) != 1:
  67                 raise SafeException("Local feed '%s' contains %d versions! I need exactly one!" % (iface.uri, len(impls)))
  68         return impls.values()[0]
  69
  70 def backup_if_exists(name):
  71         if not os.path.exists(name):
  72                 return
  73         backup = name + '~'
  74         if os.path.exists(backup):
  75                 print "(deleting old backup %s)" % backup
  76                 if os.path.isdir(backup):
  77                         shutil.rmtree(backup)
  78                 else:
  79                         os.unlink(backup)
  80         os.rename(name, backup)
  81         print "(renamed old %s as %s; will delete on next run)" % (name, backup)
  82
  83 class Status(object):
  84         __slots__ = ['old_snapshot_version', 'release_version', 'head_before_release', 'new_snapshot_version', 'head_at_release', 'created_archive', 'tagged']
  85         def __init__(self):
  86                 for name in self.__slots__:
  87                         setattr(self, name, None)
  88
  89                 if os.path.isfile(release_status_file):
  90                         for line in file(release_status_file):
  91                                 assert line.endswith('\n')
  92                                 line = line[:-1]
  93                                 name, value = line.split('=')
  94                                 setattr(self, name, value)
  95                                 info("Loaded status %s=%s", name, value)
  96
  97         def save(self):
  98                 tmp_name = release_status_file + '.new'
  99                 tmp = file(tmp_name, 'w')
 100                 try:
 101                         lines = ["%s=%s\n" % (name, getattr(self, name)) for name in self.__slots__ if getattr(self, name)]
 102                         tmp.write(''.join(lines))
 103                         tmp.close()
 104                         os.rename(tmp_name, release_status_file)
 105                         info("Wrote status to %s", release_status_file)
 106                 except:
 107                         os.unlink(tmp_name)
 108                         raise
 109
 110 def get_choice(*options):
 111         while True:
 112                 choice = raw_input('/'.join(options) + ': ').lower()
 113                 if not choice: continue
 114                 for o in options:
 115                         if o.lower().startswith(choice):
 116                                 return o
 117
 118 def do_release(local_iface, options):
 119         status = Status()
 120         local_impl = get_singleton_impl(local_iface)
 121
 122         local_impl_dir = local_impl.id
 123         assert local_impl_dir.startswith('/')
 124         local_impl_dir = os.path.realpath(local_impl_dir)
 125         assert os.path.isdir(local_impl_dir)
 126         assert local_iface.uri.startswith(local_impl_dir + '/')
 127         local_iface_rel_path = local_iface.uri[len(local_impl_dir) + 1:]
 128         assert not local_iface_rel_path.startswith('/')
 129         assert os.path.isfile(os.path.join(local_impl_dir, local_iface_rel_path))
 130
 131         phase_actions = {}
 132         for phase in valid_phases:
 133                 phase_actions[phase] = []       # List of <release:action> elements
 134
 135         release_management = local_iface.get_metadata(XMLNS_RELEASE, 'management')
 136         if len(release_management) == 1:
 137                 info("Found <release:management> element.")
 138                 release_management = release_management[0]
 139                 for x in release_management.childNodes:
 140                         if x.uri == XMLNS_RELEASE and x.name == 'action':
 141                                 phase = x.getAttribute('phase')
 142                                 if phase not in valid_phases:
 143                                         raise SafeException("Invalid action phase '%s' in local feed %s. Valid actions are:\n%s" % (phase, local_iface.uri, '\n'.join(valid_phases)))
 144                                 phase_actions[phase].append(x.content)
 145                         else:
 146                                 warn("Unknown <release:management> element: %s", x)
 147         elif len(release_management) > 1:
 148                 raise SafeException("Multiple <release:management> sections in %s!" % local_iface)
 149         else:
 150                 info("No <release:management> element found in local feed.")
 151
 152         scm = GIT(local_iface, options)
 153
 154         def run_hooks(phase, cwd, env):
 155                 info("Running hooks for phase '%s'" % phase)
 156                 full_env = os.environ.copy()
 157                 full_env.update(env)
 158                 for x in phase_actions[phase]:
 159                         print "[%s]: %s" % (phase, x)
 160                         subprocess.check_call(x, shell = True, cwd = cwd, env = full_env)
 161
 162         def set_to_release():
 163                 print "Snapshot version is " + local_impl.get_version()
 164                 suggested = suggest_release_version(local_impl.get_version())
 165                 release_version = raw_input("Version number for new release [%s]: " % suggested)
 166                 if not release_version:
 167                         release_version = suggested
 168
 169                 scm.ensure_no_tag(release_version)
 170
 171                 status.head_before_release = scm.get_head_revision()
 172                 status.save()
 173
 174                 working_copy = local_impl.id
 175                 run_hooks('commit-release', cwd = working_copy, env = {'RELEASE_VERSION': release_version})
 176
 177                 print "Releasing version", release_version
 178                 publish(local_iface.uri, set_released = 'today', set_version = release_version)
 179
 180                 status.old_snapshot_version = local_impl.get_version()
 181                 status.release_version = release_version
 182                 scm.commit('Release %s' % release_version)
 183                 status.head_at_release = scm.get_head_revision()
 184                 status.save()
 185
 186                 return release_version
 187
 188         def set_to_snapshot(snapshot_version):
 189                 assert snapshot_version.endswith('-post')
 190                 publish(local_iface.uri, set_released = '', set_version = snapshot_version)
 191                 scm.commit('Start development series %s' % snapshot_version)
 192                 status.new_snapshot_version = scm.get_head_revision()
 193                 status.save()
 194
 195         def ensure_ready_to_release():
 196                 scm.ensure_committed()
 197                 info("No uncommitted changes. Good.")
 198                 # Not needed for GIT. For SCMs where tagging is expensive (e.g. svn) this might be useful.
 199                 #run_unit_tests(local_impl)
 200
 201         def create_feed(local_iface_stream, archive_file, archive_name, version):
 202                 tmp = tempfile.NamedTemporaryFile(prefix = '0release-')
 203                 shutil.copyfileobj(local_iface_stream, tmp)
 204                 tmp.flush()
 205
 206                 publish(tmp.name,
 207                         archive_url = options.archive_dir_public_url + '/' + os.path.basename(archive_file),
 208                         archive_file = archive_file,
 209                         archive_extract = archive_name)
 210                 return tmp
 211
 212         def unpack_tarball(archive_file, archive_name):
 213                 tar = tarfile.open(archive_file, 'r:bz2')
 214                 members = [m for m in tar.getmembers() if m.name != 'pax_global_header']
 215                 tar.extractall('.', members = members)
 216
 217         def fail_candidate(archive_file):
 218                 backup_if_exists(archive_file)
 219                 head = scm.get_head_revision()
 220                 if head != status.new_snapshot_version:
 221                         raise SafeException("There have been commits since starting the release! Please rebase them onto %s" % status.head_before_release)
 222                 # Check no uncommitted changes
 223                 scm.ensure_committed()
 224                 scm.reset_hard(status.head_before_release)
 225                 os.unlink(release_status_file)
 226                 print "Restored to state before starting release. Make your fixes and try again..."
 227
 228         def accept_and_publish(archive_file, archive_name, local_iface_rel_path):
 229                 assert options.master_feed_file
 230
 231                 if status.tagged:
 232                         print "Already tagged and added to master feed."
 233                 else:
 234                         tar = tarfile.open(archive_file, 'r:bz2')
 235                         stream = tar.extractfile(tar.getmember(archive_name + '/' + local_iface_rel_path))
 236                         remote_dl_iface = create_feed(stream, archive_file, archive_name, version)
 237                         stream.close()
 238
 239                         publish(options.master_feed_file, local = remote_dl_iface.name, xmlsign = True, key = options.key)
 240                         remote_dl_iface.close()
 241
 242                         scm.tag(status.release_version, status.head_at_release)
 243
 244                         status.tagged = 'true'
 245                         status.save()
 246
 247                 # Copy files...
 248                 print "Upload %s as %s" % (archive_file, options.archive_dir_public_url + '/' + os.path.basename(archive_file))
 249                 cmd = options.archive_upload_command.strip()
 250                 if cmd:
 251                         show_and_run(cmd, [archive_file])
 252                 else:
 253                         print "NOTE: No upload command set => you'll have to upload it yourself!"
 254
 255                 assert len(local_iface.feed_for) == 1
 256                 feed_base = os.path.dirname(local_iface.feed_for.keys()[0])
 257                 feed_files = [options.master_feed_file]
 258                 print "Upload %s into %s" % (', '.join(feed_files), feed_base)
 259                 cmd = options.master_feed_upload_command.strip()
 260                 if cmd:
 261                         show_and_run(cmd, feed_files)
 262                 else:
 263                         print "NOTE: No feed upload command set => you'll have to upload them yourself!"
 264
 265                 print "Push changes to public SCM repository..."
 266                 public_repos = options.public_scm_repository
 267                 if public_repos:
 268                         scm.push_head_and_release(status.release_version)
 269                 else:
 270                         print "NOTE: No public repository set => you'll have to push the tag and trunk yourself."
 271
 272                 os.unlink(release_status_file)
 273
 274         if status.head_before_release:
 275                 head = scm.get_head_revision()
 276                 if status.release_version:
 277                         print "RESUMING release of version %s" % status.release_version
 278                 elif head == status.head_before_release:
 279                         print "Restarting release (HEAD revision has not changed)"
 280                 else:
 281                         raise SafeException("Something went wrong with the last run:\n" +
 282                                             "HEAD revision for last run was " + status.head_before_release + "\n" +
 283                                             "HEAD revision now is " + head + "\n" +
 284                                             "You should revert your working copy to the previous head and try again.\n" +
 285                                             "If you're sure you want to release from the current head, delete '" + release_status_file + "'")
 286
 287         print "Releasing", local_iface.get_name()
 288
 289         ensure_ready_to_release()
 290
 291         if status.release_version:
 292                 version = status.release_version
 293                 need_set_snapshot = False
 294                 if status.new_snapshot_version:
 295                         head = scm.get_head_revision()
 296                         if head != status.new_snapshot_version:
 297                                 print "WARNING: there are more commits since we tagged; they will not be included in the release!"
 298                 else:
 299                         raise SafeException("Something went wrong previously when setting the new snapshot version.\n" +
 300                                             "Suggest you reset to the original HEAD of\n%s and delete '%s'." % (status.head_before_release, release_status_file))
 301         else:
 302                 version = set_to_release()
 303                 need_set_snapshot = True
 304
 305         archive_name = local_iface.get_name().lower().replace(' ', '-') + '-' + version
 306         archive_file = archive_name + '.tar.bz2'
 307
 308         if status.created_archive and os.path.isfile(archive_file):
 309                 print "Archive already created"
 310         else:
 311                 backup_if_exists(archive_file)
 312                 scm.export(archive_name, archive_file)
 313                 status.created_archive = 'true'
 314                 status.save()
 315
 316         if need_set_snapshot:
 317                 set_to_snapshot(version + '-post')
 318
 319         #backup_if_exists(archive_name)
 320         unpack_tarball(archive_file, archive_name)
 321         if local_impl.main:
 322                 main = os.path.join(archive_name, local_impl.main)
 323                 if not os.path.exists(main):
 324                         raise SafeException("Main executable '%s' not found after unpacking archive!" % main)
 325
 326         extracted_iface_path = os.path.abspath(os.path.join(archive_name, local_iface_rel_path))
 327         extracted_iface = model.Interface(extracted_iface_path)
 328         reader.update(extracted_iface, extracted_iface_path, local = True)
 329         extracted_impl = get_singleton_impl(extracted_iface)
 330
 331         try:
 332                 run_unit_tests(extracted_impl)
 333         except SafeException:
 334                 print "(leaving extracted directory for examination)"
 335                 fail_candidate(archive_file)
 336                 raise
 337         # Unpack it again in case the unit-tests changed anything
 338         shutil.rmtree(archive_name)
 339         unpack_tarball(archive_file, archive_name)
 340
 341         print "\nCandidate release archive:", archive_file
 342         print "(extracted to %s for inspection)" % os.path.abspath(archive_name)
 343
 344         print "\nPlease check candidate and select an action:"
 345         print "P) Publish candidate (accept)"
 346         print "F) Fail candidate (untag)"
 347         print "(you can also hit CTRL-C and resume this script when done)"
 348         choice = get_choice('Publish', 'Fail')
 349
 350         info("Deleting extracted archive %s", archive_name)
 351         shutil.rmtree(archive_name)
 352
 353         if choice == 'Publish':
 354                 accept_and_publish(archive_file, archive_name, local_iface_rel_path)
 355         else:
 356                 assert choice == 'Fail'
 357                 fail_candidate(archive_file)
 358
 359
 360 if __name__ == "__main__":
 361     import doctest
 362     doctest.testmod()