release.py

   1 # Copyright (C) 2007, Thomas Leonard
   2 # See the README file for details, or visit http://0install.net.
   3
   4 import os, sys, subprocess, shutil, tarfile, tempfile
   5 from zeroinstall import SafeException
   6 from zeroinstall.injector import reader, model
   7 from logging import info, warn
   8
   9 import support
  10 from scm import get_scm
  11
  12 XMLNS_RELEASE = 'http://zero-install.sourceforge.net/2007/namespaces/0release'
  13
  14 valid_phases = ['commit-release', 'generate-archive']
  15
  16 TMP_BRANCH_NAME = '0release-tmp'
  17
  18 def run_unit_tests(local_feed, impl):
  19         self_test = impl.metadata.get('self-test', None)
  20         if self_test is None:
  21                 print "SKIPPING unit tests for %s (no 'self-test' attribute set)" % impl
  22                 return
  23         self_test_dir = os.path.dirname(os.path.join(impl.id, self_test))
  24         print "Running self-test:", self_test
  25         exitstatus = subprocess.call(['0launch', '--main', self_test, local_feed], cwd = self_test_dir)
  26         if exitstatus:
  27                 raise SafeException("Self-test failed with exit status %d" % exitstatus)
  28
  29 def do_release(local_iface, options):
  30         status = support.Status()
  31         local_impl = support.get_singleton_impl(local_iface)
  32
  33         local_impl_dir = local_impl.id
  34         assert local_impl_dir.startswith('/')
  35         local_impl_dir = os.path.realpath(local_impl_dir)
  36         assert os.path.isdir(local_impl_dir)
  37         assert local_iface.uri.startswith(local_impl_dir + '/')
  38         local_iface_rel_path = local_iface.uri[len(local_impl_dir) + 1:]
  39         assert not local_iface_rel_path.startswith('/')
  40         assert os.path.isfile(os.path.join(local_impl_dir, local_iface_rel_path))
  41
  42         phase_actions = {}
  43         for phase in valid_phases:
  44                 phase_actions[phase] = []       # List of <release:action> elements
  45
  46         add_toplevel_dir = None
  47         release_management = local_iface.get_metadata(XMLNS_RELEASE, 'management')
  48         if len(release_management) == 1:
  49                 info("Found <release:management> element.")
  50                 release_management = release_management[0]
  51                 for x in release_management.childNodes:
  52                         if x.uri == XMLNS_RELEASE and x.name == 'action':
  53                                 phase = x.getAttribute('phase')
  54                                 if phase not in valid_phases:
  55                                         raise SafeException("Invalid action phase '%s' in local feed %s. Valid actions are:\n%s" % (phase, local_iface.uri, '\n'.join(valid_phases)))
  56                                 phase_actions[phase].append(x.content)
  57                         elif x.uri == XMLNS_RELEASE and x.name == 'add-toplevel-directory':
  58                                 add_toplevel_dir = local_iface.get_name()
  59                         else:
  60                                 warn("Unknown <release:management> element: %s", x)
  61         elif len(release_management) > 1:
  62                 raise SafeException("Multiple <release:management> sections in %s!" % local_iface)
  63         else:
  64                 info("No <release:management> element found in local feed.")
  65
  66         scm = get_scm(local_iface, options)
  67
  68         def run_hooks(phase, cwd, env):
  69                 info("Running hooks for phase '%s'" % phase)
  70                 full_env = os.environ.copy()
  71                 full_env.update(env)
  72                 for x in phase_actions[phase]:
  73                         print "[%s]: %s" % (phase, x)
  74                         support.check_call(x, shell = True, cwd = cwd, env = full_env)
  75
  76         def set_to_release():
  77                 print "Snapshot version is " + local_impl.get_version()
  78                 suggested = support.suggest_release_version(local_impl.get_version())
  79                 release_version = raw_input("Version number for new release [%s]: " % suggested)
  80                 if not release_version:
  81                         release_version = suggested
  82
  83                 scm.ensure_no_tag(release_version)
  84
  85                 status.head_before_release = scm.get_head_revision()
  86                 status.save()
  87
  88                 working_copy = local_impl.id
  89                 run_hooks('commit-release', cwd = working_copy, env = {'RELEASE_VERSION': release_version})
  90
  91                 print "Releasing version", release_version
  92                 support.publish(local_iface.uri, set_released = 'today', set_version = release_version)
  93
  94                 status.old_snapshot_version = local_impl.get_version()
  95                 status.release_version = release_version
  96                 status.head_at_release = scm.commit('Release %s' % release_version, branch = TMP_BRANCH_NAME, parent = 'HEAD')
  97                 status.save()
  98
  99         def set_to_snapshot(snapshot_version):
 100                 assert snapshot_version.endswith('-post')
 101                 support.publish(local_iface.uri, set_released = '', set_version = snapshot_version)
 102                 scm.commit('Start development series %s' % snapshot_version, branch = TMP_BRANCH_NAME, parent = TMP_BRANCH_NAME)
 103                 status.new_snapshot_version = scm.get_head_revision()
 104                 status.save()
 105
 106         def ensure_ready_to_release():
 107                 if not options.master_feed_file:
 108                         raise SafeException("Master feed file not set! Check your configuration")
 109
 110                 scm.ensure_committed()
 111                 scm.ensure_versioned(local_iface_rel_path)
 112                 info("No uncommitted changes. Good.")
 113                 # Not needed for GIT. For SCMs where tagging is expensive (e.g. svn) this might be useful.
 114                 #run_unit_tests(local_impl)
 115
 116                 scm.grep('\(^\\|[^=]\)\<\\(TODO\\|XXX\\|FIXME\\)\>')
 117
 118         def create_feed(local_iface_stream, archive_file, archive_name, main):
 119                 tmp = tempfile.NamedTemporaryFile(prefix = '0release-')
 120                 shutil.copyfileobj(local_iface_stream, tmp)
 121                 tmp.flush()
 122
 123                 support.publish(tmp.name,
 124                         set_main = main,
 125                         archive_url = options.archive_dir_public_url + '/' + os.path.basename(archive_file),
 126                         archive_file = archive_file,
 127                         archive_extract = archive_name)
 128                 return tmp
 129
 130         def get_previous_release(this_version):
 131                 """Return the highest numbered verison in the master feed before this_version.
 132                 @return: version, or None if there wasn't one"""
 133                 parsed_release_version = model.parse_version(this_version)
 134
 135                 if os.path.exists(options.master_feed_file):
 136                         master = model.Interface(os.path.realpath(options.master_feed_file))
 137                         reader.update(master, master.uri, local = True)
 138                         versions = [impl.version for impl in master.implementations.values() if impl.version < parsed_release_version]
 139                         if versions:
 140                                 return model.format_version(max(versions))
 141                 return None
 142
 143         def export_changelog(previous_release):
 144                 changelog = file('changelog-%s' % status.release_version, 'w')
 145                 try:
 146                         try:
 147                                 scm.export_changelog(previous_release, status.head_before_release, changelog)
 148                         except SafeException, ex:
 149                                 print "WARNING: Failed to generate changelog: " + str(ex)
 150                         else:
 151                                 print "Wrote changelog from %s to here as %s" % (previous_release or 'start', changelog.name)
 152                 finally:
 153                         changelog.close()
 154
 155         def fail_candidate(archive_file):
 156                 support.backup_if_exists(archive_file)
 157                 scm.delete_branch(TMP_BRANCH_NAME)
 158                 os.unlink(support.release_status_file)
 159                 print "Restored to state before starting release. Make your fixes and try again..."
 160
 161         def accept_and_publish(archive_file, archive_name, local_iface_rel_path, main):
 162                 assert options.master_feed_file
 163
 164                 if not options.archive_dir_public_url:
 165                         raise SafeException("Archive directory public URL is not set! Edit configuration and try again.")
 166
 167                 if status.tagged:
 168                         print "Already tagged in SCM. Not re-tagging."
 169                 else:
 170                         scm.ensure_committed()
 171                         head = scm.get_head_revision()
 172                         if head != status.head_before_release:
 173                                 raise SafeException("Changes committed since we started!\n" +
 174                                                     "HEAD was " + status.head_before_release + "\n"
 175                                                     "HEAD now " + head)
 176
 177                         scm.tag(status.release_version, status.head_at_release)
 178                         scm.reset_hard(TMP_BRANCH_NAME)
 179                         scm.delete_branch(TMP_BRANCH_NAME)
 180
 181                         status.tagged = 'true'
 182                         status.save()
 183
 184                 if status.updated_master_feed:
 185                         print "Already added to master feed. Not changing."
 186                 else:
 187                         if os.path.exists(options.master_feed_file):
 188                                 # Check we haven't already released this version
 189                                 master = model.Interface(os.path.realpath(options.master_feed_file))
 190                                 reader.update(master, master.uri, local = True)
 191                                 existing_releases = [impl for impl in master.implementations.values() if impl.get_version() == status.release_version]
 192                                 if len(existing_releases):
 193                                         raise SafeException("Master feed %s already contains an implementation with version number %s!" % (options.master_feed_file, status.release_version))
 194
 195                         tar = tarfile.open(archive_file, 'r:bz2')
 196                         stream = tar.extractfile(tar.getmember(export_prefix + '/' + local_iface_rel_path))
 197                         remote_dl_iface = create_feed(stream, archive_file, archive_name, main)
 198                         stream.close()
 199
 200                         support.publish(options.master_feed_file, local = remote_dl_iface.name, xmlsign = True, key = options.key)
 201                         remote_dl_iface.close()
 202
 203                         status.updated_master_feed = 'true'
 204                         status.save()
 205
 206                 def is_uploaded(url, size):
 207                         if url.startswith('http://TESTING/releases'):
 208                                 return True
 209
 210                         print "Testing URL %s..." % url
 211                         try:
 212                                 actual_size = int(support.get_size(url))
 213                         except Exception, ex:
 214                                 print "Can't get size of '%s': %s" % (url, ex)
 215                                 return False
 216                         else:
 217                                 if actual_size == size:
 218                                         return True
 219                                 print "WARNING: %s exists, but size is %d, not %d!" % (url, actual_size, size)
 220                                 return False
 221
 222                 # Copy files...
 223                 print
 224                 archive_url = options.archive_dir_public_url + '/' + os.path.basename(archive_file)
 225                 archive_size = os.path.getsize(archive_file)
 226                 if status.uploaded_archive and is_uploaded(archive_url, archive_size):
 227                         print "Archive already uploaded. Not uploading again."
 228                 else:
 229                         while True:
 230                                 print "Upload %s as %s" % (archive_file, archive_url)
 231                                 cmd = options.archive_upload_command.strip()
 232                                 if cmd:
 233                                         support.show_and_run(cmd, [archive_file])
 234                                 else:
 235                                         print "No upload command is set => please upload the archive manually now"
 236                                         raw_input('Press Return once archive is uploaded.')
 237                                 print
 238                                 if is_uploaded(archive_url, archive_size):
 239                                         print "OK, archive uploaded successfully"
 240                                         status.uploaded_archive = 'true'
 241                                         status.save()
 242                                         break
 243                                 print "** Archive still not uploaded! Try again..."
 244                                 if cmd:
 245                                         raw_input('Press Return to retry upload command.')
 246
 247                 assert len(local_iface.feed_for) == 1
 248                 feed_base = os.path.dirname(local_iface.feed_for.keys()[0])
 249                 feed_files = [options.master_feed_file]
 250                 print "Upload %s into %s" % (', '.join(feed_files), feed_base)
 251                 cmd = options.master_feed_upload_command.strip()
 252                 if cmd:
 253                         support.show_and_run(cmd, feed_files)
 254                 else:
 255                         print "NOTE: No feed upload command set => you'll have to upload them yourself!"
 256
 257                 print "Push changes to public SCM repository..."
 258                 public_repos = options.public_scm_repository
 259                 if public_repos:
 260                         scm.push_head_and_release(status.release_version)
 261                 else:
 262                         print "NOTE: No public repository set => you'll have to push the tag and trunk yourself."
 263
 264                 os.unlink(support.release_status_file)
 265
 266         if status.head_before_release:
 267                 head = scm.get_head_revision()
 268                 if status.release_version:
 269                         print "RESUMING release of %s %s" % (local_iface.get_name(), status.release_version)
 270                 elif head == status.head_before_release:
 271                         print "Restarting release of %s (HEAD revision has not changed)" % local_iface.get_name()
 272                 else:
 273                         raise SafeException("Something went wrong with the last run:\n" +
 274                                             "HEAD revision for last run was " + status.head_before_release + "\n" +
 275                                             "HEAD revision now is " + head + "\n" +
 276                                             "You should revert your working copy to the previous head and try again.\n" +
 277                                             "If you're sure you want to release from the current head, delete '" + support.release_status_file + "'")
 278         else:
 279                 print "Releasing", local_iface.get_name()
 280
 281         ensure_ready_to_release()
 282
 283         if status.release_version:
 284                 need_set_snapshot = False
 285                 if status.tagged:
 286                         print "Already tagged. Resuming the publishing process..."
 287                 elif status.new_snapshot_version:
 288                         head = scm.get_head_revision()
 289                         if head != status.head_before_release:
 290                                 raise SafeException("There are more commits since we started!\n"
 291                                                     "HEAD was " + status.head_before_release + "\n"
 292                                                     "HEAD now " + head + "\n"
 293                                                     "To include them, delete '" + support.release_status_file + "' and try again.\n"
 294                                                     "To leave them out, put them on a new branch and reset HEAD to the release version.")
 295                 else:
 296                         raise SafeException("Something went wrong previously when setting the new snapshot version.\n" +
 297                                             "Suggest you reset to the original HEAD of\n%s and delete '%s'." % (status.head_before_release, support.release_status_file))
 298         else:
 299                 set_to_release()
 300                 assert status.release_version
 301                 need_set_snapshot = True
 302
 303         archive_name = support.make_archive_name(local_iface.get_name(), status.release_version)
 304         archive_file = archive_name + '.tar.bz2'
 305
 306         export_prefix = archive_name
 307         if add_toplevel_dir is not None:
 308                 export_prefix += '/' + add_toplevel_dir
 309
 310         if status.created_archive and os.path.isfile(archive_file):
 311                 print "Archive already created"
 312         else:
 313                 support.backup_if_exists(archive_file)
 314                 scm.export(export_prefix, archive_file, status.head_at_release)
 315
 316                 has_submodules = scm.has_submodules()
 317
 318                 if phase_actions['generate-archive'] or has_submodules:
 319                         try:
 320                                 support.unpack_tarball(archive_file)
 321                                 if has_submodules:
 322                                         scm.export_submodules(archive_name)
 323                                 run_hooks('generate-archive', cwd = archive_name, env = {'RELEASE_VERSION': status.release_version})
 324                                 info("Regenerating archive (may have been modified by generate-archive hooks...")
 325                                 support.check_call(['tar', 'cjf', archive_file, archive_name])
 326                         except SafeException:
 327                                 scm.reset_hard(scm.get_current_branch())
 328                                 fail_candidate(archive_file)
 329                                 raise
 330
 331                 status.created_archive = 'true'
 332                 status.save()
 333
 334         if need_set_snapshot:
 335                 set_to_snapshot(status.release_version + '-post')
 336                 # Revert back to the original revision, so that any fixes the user makes
 337                 # will get applied before the tag
 338                 scm.reset_hard(scm.get_current_branch())
 339
 340         #backup_if_exists(archive_name)
 341         support.unpack_tarball(archive_file)
 342         if local_impl.main:
 343                 main = os.path.join(export_prefix, local_impl.main)
 344                 if not os.path.exists(main):
 345                         raise SafeException("Main executable '%s' not found after unpacking archive!" % main)
 346
 347         extracted_iface_path = os.path.abspath(os.path.join(export_prefix, local_iface_rel_path))
 348         assert os.path.isfile(extracted_iface_path), "Local feed not in archive! Is it under version control?"
 349         extracted_iface = model.Interface(extracted_iface_path)
 350         reader.update(extracted_iface, extracted_iface_path, local = True)
 351         extracted_impl = support.get_singleton_impl(extracted_iface)
 352
 353         try:
 354                 run_unit_tests(extracted_iface_path, extracted_impl)
 355         except SafeException:
 356                 print "(leaving extracted directory for examination)"
 357                 fail_candidate(archive_file)
 358                 raise
 359         # Unpack it again in case the unit-tests changed anything
 360         shutil.rmtree(archive_name)
 361         support.unpack_tarball(archive_file)
 362
 363         previous_release = get_previous_release(status.release_version)
 364         export_changelog(previous_release)
 365
 366         print "\nCandidate release archive:", archive_file
 367         print "(extracted to %s for inspection)" % os.path.abspath(archive_name)
 368
 369         print "\nPlease check candidate and select an action:"
 370         print "P) Publish candidate (accept)"
 371         print "F) Fail candidate (untag)"
 372         if previous_release:
 373                 print "D) Diff against release archive for %s" % previous_release
 374                 maybe_diff = ['Diff']
 375         else:
 376                 maybe_diff = []
 377         print "(you can also hit CTRL-C and resume this script when done)"
 378
 379         while True:
 380                 choice = support.get_choice(['Publish', 'Fail'] + maybe_diff)
 381                 if choice == 'Diff':
 382                         previous_archive_name = support.make_archive_name(local_iface.get_name(), previous_release)
 383                         previous_archive_file = previous_archive_name + '.tar.bz2'
 384                         if os.path.isfile(previous_archive_file):
 385                                 support.unpack_tarball(previous_archive_file)
 386                                 try:
 387                                         support.show_diff(previous_archive_name, archive_name)
 388                                 finally:
 389                                         shutil.rmtree(previous_archive_name)
 390                         else:
 391                                 # TODO: download it?
 392                                 print "Sorry, archive file %s not found! Can't show diff." % previous_archive_file
 393                 else:
 394                         break
 395
 396         info("Deleting extracted archive %s", archive_name)
 397         shutil.rmtree(archive_name)
 398
 399         if choice == 'Publish':
 400                 main = extracted_impl.main
 401                 if main and add_toplevel_dir:
 402                         main = os.path.join(add_toplevel_dir, main)
 403                 accept_and_publish(archive_file, archive_name, local_iface_rel_path, main)
 404         else:
 405                 assert choice == 'Fail'
 406                 fail_candidate(archive_file)