release.py

   1 # Copyright (C) 2009, Thomas Leonard
   2 # See the README file for details, or visit http://0install.net.
   3
   4 import os, subprocess, shutil, sys
   5 from zeroinstall import SafeException
   6 from zeroinstall.injector import model
   7 from zeroinstall.support import ro_rmtree
   8 from logging import info, warn
   9
  10 sys.path.insert(0, os.environ['RELEASE_0REPO'])
  11 from repo import registry
  12
  13 import support, compile
  14 from scm import get_scm
  15
  16 XMLNS_RELEASE = 'http://zero-install.sourceforge.net/2007/namespaces/0release'
  17
  18 valid_phases = ['commit-release', 'generate-archive']
  19
  20 TMP_BRANCH_NAME = '0release-tmp'
  21
  22 test_command = os.environ['0TEST']
  23
  24 def run_unit_tests(local_feed):
  25         print "Running self-tests..."
  26         exitstatus = subprocess.call([test_command, '--', local_feed])
  27         if exitstatus == 2:
  28                 print "SKIPPED unit tests for %s (no 'self-test' attribute set)" % local_feed
  29                 return
  30         if exitstatus:
  31                 raise SafeException("Self-test failed with exit status %d" % exitstatus)
  32
  33 def upload_archives(options, status, uploads):
  34         # For each binary or source archive in uploads, ensure it is available
  35         # from options.archive_dir_public_url
  36
  37         # We try to do all the uploads together first, and then verify them all
  38         # afterwards. This is because we may have to wait for them to be moved
  39         # from an incoming queue before we can test them.
  40
  41         def url(archive):
  42                 return support.get_archive_url(options, status.release_version, archive)
  43
  44         # Check that url exists and has the given size
  45         def is_uploaded(url, size):
  46                 if url.startswith('http://TESTING/releases'):
  47                         return True
  48
  49                 print "Testing URL %s..." % url
  50                 try:
  51                         actual_size = int(support.get_size(url))
  52                 except Exception, ex:
  53                         print "Can't get size of '%s': %s" % (url, ex)
  54                         return False
  55                 else:
  56                         if actual_size == size:
  57                                 return True
  58                         print "WARNING: %s exists, but size is %d, not %d!" % (url, actual_size, size)
  59                         return False
  60
  61         # status.verified_uploads is an array of status flags:
  62         description = {
  63                 'N': 'Upload required',
  64                 'A': 'Upload has been attempted, but we need to check whether it worked',
  65                 'V': 'Upload has been checked (exists and has correct size)',
  66         }
  67
  68         if status.verified_uploads is None:
  69                 # First time around; no point checking for existing uploads
  70                 status.verified_uploads = 'N' * len(uploads)
  71                 status.save()
  72
  73         while True:
  74                 print "\nUpload status:"
  75                 for i, stat in enumerate(status.verified_uploads):
  76                         print "- %s : %s" % (uploads[i], description[stat])
  77                 print
  78
  79                 # Break if finished
  80                 if status.verified_uploads == 'V' * len(uploads):
  81                         break
  82
  83                 # Find all New archives
  84                 to_upload = []
  85                 for i, stat in enumerate(status.verified_uploads):
  86                         assert stat in 'NAV'
  87                         if stat == 'N':
  88                                 to_upload.append(uploads[i])
  89                                 print "Upload %s/%s as %s" % (status.release_version, uploads[i], url(uploads[i]))
  90
  91                 cmd = options.archive_upload_command.strip()
  92
  93                 if to_upload:
  94                         # Mark all New items as Attempted
  95                         status.verified_uploads = status.verified_uploads.replace('N', 'A')
  96                         status.save()
  97
  98                         # Upload them...
  99                         if cmd:
 100                                 support.show_and_run(cmd, to_upload)
 101                         else:
 102                                 if len(to_upload) == 1:
 103                                         print "No upload command is set => please upload the archive manually now"
 104                                         raw_input('Press Return once the archive is uploaded.')
 105                                 else:
 106                                         print "No upload command is set => please upload the archives manually now"
 107                                         raw_input('Press Return once the %d archives are uploaded.' % len(to_upload))
 108
 109                 # Verify all Attempted uploads
 110                 new_stat = ''
 111                 for i, stat in enumerate(status.verified_uploads):
 112                         assert stat in 'AV', status.verified_uploads
 113                         if stat == 'A' :
 114                                 if not is_uploaded(url(uploads[i]), os.path.getsize(uploads[i])):
 115                                         print "** Archive '%s' still not uploaded! Try again..." % uploads[i]
 116                                         stat = 'N'
 117                                 else:
 118                                         stat = 'V'
 119                         new_stat += stat
 120
 121                 status.verified_uploads = new_stat
 122                 status.save()
 123
 124                 if 'N' in new_stat and cmd:
 125                         raw_input('Press Return to try again.')
 126
 127 def do_release(local_feed, options):
 128         if options.master_feed_file:
 129                 options.master_feed_file = os.path.abspath(options.master_feed_file)
 130
 131         if not local_feed.feed_for:
 132                 raise SafeException("Feed %s missing a <feed-for> element" % local_feed.local_path)
 133
 134         status = support.Status()
 135         local_impl = support.get_singleton_impl(local_feed)
 136
 137         local_impl_dir = local_impl.id
 138         assert local_impl_dir.startswith('/')
 139         local_impl_dir = os.path.realpath(local_impl_dir)
 140         assert os.path.isdir(local_impl_dir)
 141         assert local_feed.local_path.startswith(local_impl_dir + '/')
 142
 143         # From the impl directory to the feed
 144         # NOT relative to the archive root (in general)
 145         local_iface_rel_path = local_feed.local_path[len(local_impl_dir) + 1:]
 146         assert not local_iface_rel_path.startswith('/')
 147         assert os.path.isfile(os.path.join(local_impl_dir, local_iface_rel_path))
 148
 149         phase_actions = {}
 150         for phase in valid_phases:
 151                 phase_actions[phase] = []       # List of <release:action> elements
 152
 153         add_toplevel_dir = None
 154         release_management = local_feed.get_metadata(XMLNS_RELEASE, 'management')
 155         if len(release_management) == 1:
 156                 info("Found <release:management> element.")
 157                 release_management = release_management[0]
 158                 for x in release_management.childNodes:
 159                         if x.uri == XMLNS_RELEASE and x.name == 'action':
 160                                 phase = x.getAttribute('phase')
 161                                 if phase not in valid_phases:
 162                                         raise SafeException("Invalid action phase '%s' in local feed %s. Valid actions are:\n%s" % (phase, local_feed.local_path, '\n'.join(valid_phases)))
 163                                 phase_actions[phase].append(x.content)
 164                         elif x.uri == XMLNS_RELEASE and x.name == 'add-toplevel-directory':
 165                                 add_toplevel_dir = local_feed.get_name()
 166                         else:
 167                                 warn("Unknown <release:management> element: %s", x)
 168         elif len(release_management) > 1:
 169                 raise SafeException("Multiple <release:management> sections in %s!" % local_feed)
 170         else:
 171                 info("No <release:management> element found in local feed.")
 172
 173         scm = get_scm(local_feed, options)
 174
 175         # Path relative to the archive / SCM root
 176         local_iface_rel_root_path = local_feed.local_path[len(scm.root_dir) + 1:]
 177
 178         def run_hooks(phase, cwd, env):
 179                 info("Running hooks for phase '%s'" % phase)
 180                 full_env = os.environ.copy()
 181                 full_env.update(env)
 182                 for x in phase_actions[phase]:
 183                         print "[%s]: %s" % (phase, x)
 184                         support.check_call(x, shell = True, cwd = cwd, env = full_env)
 185
 186         def set_to_release():
 187                 print "Snapshot version is " + local_impl.get_version()
 188                 release_version = options.release_version
 189                 if release_version is None:
 190                         suggested = support.suggest_release_version(local_impl.get_version())
 191                         release_version = raw_input("Version number for new release [%s]: " % suggested)
 192                         if not release_version:
 193                                 release_version = suggested
 194
 195                 scm.ensure_no_tag(release_version)
 196
 197                 status.head_before_release = scm.get_head_revision()
 198                 status.save()
 199
 200                 working_copy = local_impl.id
 201                 run_hooks('commit-release', cwd = working_copy, env = {'RELEASE_VERSION': release_version})
 202
 203                 print "Releasing version", release_version
 204                 support.publish(local_feed.local_path, set_released = 'today', set_version = release_version)
 205
 206                 support.backup_if_exists(release_version)
 207                 os.mkdir(release_version)
 208                 os.chdir(release_version)
 209
 210                 status.old_snapshot_version = local_impl.get_version()
 211                 status.release_version = release_version
 212                 status.head_at_release = scm.commit('Release %s' % release_version, branch = TMP_BRANCH_NAME, parent = 'HEAD')
 213                 status.save()
 214
 215         def set_to_snapshot(snapshot_version):
 216                 assert snapshot_version.endswith('-post')
 217                 support.publish(local_feed.local_path, set_released = '', set_version = snapshot_version)
 218                 scm.commit('Start development series %s' % snapshot_version, branch = TMP_BRANCH_NAME, parent = TMP_BRANCH_NAME)
 219                 status.new_snapshot_version = scm.get_head_revision()
 220                 status.save()
 221
 222         def ensure_ready_to_release():
 223                 #if not options.master_feed_file:
 224                 #       raise SafeException("Master feed file not set! Check your configuration")
 225
 226                 scm.ensure_committed()
 227                 scm.ensure_versioned(os.path.abspath(local_feed.local_path))
 228                 info("No uncommitted changes. Good.")
 229                 # Not needed for GIT. For SCMs where tagging is expensive (e.g. svn) this might be useful.
 230                 #run_unit_tests(local_impl)
 231
 232                 scm.grep('\(^\\|[^=]\)\<\\(TODO\\|XXX\\|FIXME\\)\>')
 233
 234         def create_feed(target_feed, local_iface_path, archive_file, archive_name, main):
 235                 shutil.copyfile(local_iface_path, target_feed)
 236
 237                 support.publish(target_feed,
 238                         set_main = main,
 239                         archive_url = support.get_archive_url(options, status.release_version, os.path.basename(archive_file)),
 240                         archive_file = archive_file,
 241                         archive_extract = archive_name)
 242
 243         def get_previous_release(this_version):
 244                 """Return the highest numbered verison in the master feed before this_version.
 245                 @return: version, or None if there wasn't one"""
 246                 parsed_release_version = model.parse_version(this_version)
 247
 248                 versions = [model.parse_version(version) for version in scm.get_tagged_versions()]
 249                 versions = [version for version in versions if version < parsed_release_version]
 250
 251                 if versions:
 252                         return model.format_version(max(versions))
 253                 return None
 254
 255         def export_changelog(previous_release):
 256                 changelog = file('changelog-%s' % status.release_version, 'w')
 257                 try:
 258                         try:
 259                                 scm.export_changelog(previous_release, status.head_before_release, changelog)
 260                         except SafeException, ex:
 261                                 print "WARNING: Failed to generate changelog: " + str(ex)
 262                         else:
 263                                 print "Wrote changelog from %s to here as %s" % (previous_release or 'start', changelog.name)
 264                 finally:
 265                         changelog.close()
 266
 267         def fail_candidate():
 268                 cwd = os.getcwd()
 269                 assert cwd.endswith(status.release_version)
 270                 support.backup_if_exists(cwd)
 271                 scm.delete_branch(TMP_BRANCH_NAME)
 272                 os.unlink(support.release_status_file)
 273                 print "Restored to state before starting release. Make your fixes and try again..."
 274
 275         def release_via_0repo(new_impls_feed):
 276                 import repo.cmd
 277                 support.make_archives_relative(new_impls_feed)
 278                 oldcwd = os.getcwd()
 279                 try:
 280                         repo.cmd.main(['0repo', 'add', '--', new_impls_feed])
 281                 finally:
 282                         os.chdir(oldcwd)
 283
 284         def release_without_0repo(archive_file, new_impls_feed):
 285                 assert options.master_feed_file
 286
 287                 if not options.archive_dir_public_url:
 288                         raise SafeException("Archive directory public URL is not set! Edit configuration and try again.")
 289
 290                 if status.updated_master_feed:
 291                         print "Already added to master feed. Not changing."
 292                 else:
 293                         publish_opts = {}
 294                         if os.path.exists(options.master_feed_file):
 295                                 # Check we haven't already released this version
 296                                 master = support.load_feed(os.path.realpath(options.master_feed_file))
 297                                 existing_releases = [impl for impl in master.implementations.values() if impl.get_version() == status.release_version]
 298                                 if len(existing_releases):
 299                                         raise SafeException("Master feed %s already contains an implementation with version number %s!" % (options.master_feed_file, status.release_version))
 300
 301                                 previous_release = get_previous_release(status.release_version)
 302                                 previous_testing_releases = [impl for impl in master.implementations.values() if impl.get_version() == previous_release
 303                                                                                                              and impl.upstream_stability == model.stability_levels["testing"]]
 304                                 if previous_testing_releases:
 305                                         print "The previous release, version %s, is still marked as 'testing'. Set to stable?" % previous_release
 306                                         if support.get_choice(['Yes', 'No']) == 'Yes':
 307                                                 publish_opts['select_version'] = previous_release
 308                                                 publish_opts['set_stability'] = "stable"
 309
 310                         support.publish(options.master_feed_file, local = new_impls_feed, xmlsign = True, key = options.key, **publish_opts)
 311
 312                         status.updated_master_feed = 'true'
 313                         status.save()
 314
 315                 # Copy files...
 316                 uploads = [os.path.basename(archive_file)]
 317                 for b in compiler.get_binary_feeds():
 318                         binary_feed = support.load_feed(b)
 319                         impl, = binary_feed.implementations.values()
 320                         uploads.append(os.path.basename(impl.download_sources[0].url))
 321
 322                 upload_archives(options, status, uploads)
 323
 324                 feed_base = os.path.dirname(list(local_feed.feed_for)[0])
 325                 feed_files = [options.master_feed_file]
 326                 print "Upload %s into %s" % (', '.join(feed_files), feed_base)
 327                 cmd = options.master_feed_upload_command.strip()
 328                 if cmd:
 329                         support.show_and_run(cmd, feed_files)
 330                 else:
 331                         print "NOTE: No feed upload command set => you'll have to upload them yourself!"
 332
 333         def accept_and_publish(archive_file, src_feed_name):
 334                 if status.tagged:
 335                         print "Already tagged in SCM. Not re-tagging."
 336                 else:
 337                         scm.ensure_committed()
 338                         head = scm.get_head_revision()
 339                         if head != status.head_before_release:
 340                                 raise SafeException("Changes committed since we started!\n" +
 341                                                     "HEAD was " + status.head_before_release + "\n"
 342                                                     "HEAD now " + head)
 343
 344                         scm.tag(status.release_version, status.head_at_release)
 345                         scm.reset_hard(TMP_BRANCH_NAME)
 346                         scm.delete_branch(TMP_BRANCH_NAME)
 347
 348                         status.tagged = 'true'
 349                         status.save()
 350
 351                 assert len(local_feed.feed_for) == 1
 352
 353                 # Merge the source and binary feeds together first, so
 354                 # that we update the master feed atomically and only
 355                 # have to sign it once.
 356                 new_impls_feed = 'merged.xml'
 357                 shutil.copyfile(src_feed_name, new_impls_feed)
 358                 for b in compiler.get_binary_feeds():
 359                         support.publish(new_impls_feed, local = b)
 360
 361                 # TODO: support uploading to a sub-feed (requires support in 0repo too)
 362                 master_feed, = local_feed.feed_for
 363                 repository = registry.lookup(master_feed, missing_ok = True)
 364                 if repository:
 365                         release_via_0repo(new_impls_feed)
 366                 else:
 367                         release_without_0repo(archive_file, new_impls_feed)
 368
 369                 os.unlink(new_impls_feed)
 370
 371                 print "Push changes to public SCM repository..."
 372                 public_repos = options.public_scm_repository
 373                 if public_repos:
 374                         scm.push_head_and_release(status.release_version)
 375                 else:
 376                         print "NOTE: No public repository set => you'll have to push the tag and trunk yourself."
 377
 378                 os.unlink(support.release_status_file)
 379
 380         if status.head_before_release:
 381                 head = scm.get_head_revision()
 382                 if status.release_version:
 383                         print "RESUMING release of %s %s" % (local_feed.get_name(), status.release_version)
 384                         if options.release_version and options.release_version != status.release_version:
 385                                 raise SafeException("Can't start release of version %s; we are currently releasing %s.\nDelete the release-status file to abort the previous release." % (options.release_version, status.release_version))
 386                 elif head == status.head_before_release:
 387                         print "Restarting release of %s (HEAD revision has not changed)" % local_feed.get_name()
 388                 else:
 389                         raise SafeException("Something went wrong with the last run:\n" +
 390                                             "HEAD revision for last run was " + status.head_before_release + "\n" +
 391                                             "HEAD revision now is " + head + "\n" +
 392                                             "You should revert your working copy to the previous head and try again.\n" +
 393                                             "If you're sure you want to release from the current head, delete '" + support.release_status_file + "'")
 394         else:
 395                 print "Releasing", local_feed.get_name()
 396
 397         ensure_ready_to_release()
 398
 399         if status.release_version:
 400                 if not os.path.isdir(status.release_version):
 401                         raise SafeException("Can't resume; directory %s missing. Try deleting '%s'." % (status.release_version, support.release_status_file))
 402                 os.chdir(status.release_version)
 403                 need_set_snapshot = False
 404                 if status.tagged:
 405                         print "Already tagged. Resuming the publishing process..."
 406                 elif status.new_snapshot_version:
 407                         head = scm.get_head_revision()
 408                         if head != status.head_before_release:
 409                                 raise SafeException("There are more commits since we started!\n"
 410                                                     "HEAD was " + status.head_before_release + "\n"
 411                                                     "HEAD now " + head + "\n"
 412                                                     "To include them, delete '" + support.release_status_file + "' and try again.\n"
 413                                                     "To leave them out, put them on a new branch and reset HEAD to the release version.")
 414                 else:
 415                         raise SafeException("Something went wrong previously when setting the new snapshot version.\n" +
 416                                             "Suggest you reset to the original HEAD of\n%s and delete '%s'." % (status.head_before_release, support.release_status_file))
 417         else:
 418                 set_to_release()        # Changes directory
 419                 assert status.release_version
 420                 need_set_snapshot = True
 421
 422         # May be needed by the upload command
 423         os.environ['RELEASE_VERSION'] = status.release_version
 424
 425         archive_name = support.make_archive_name(local_feed.get_name(), status.release_version)
 426         archive_file = archive_name + '.tar.bz2'
 427
 428         export_prefix = archive_name
 429         if add_toplevel_dir is not None:
 430                 export_prefix += '/' + add_toplevel_dir
 431
 432         if status.created_archive and os.path.isfile(archive_file):
 433                 print "Archive already created"
 434         else:
 435                 support.backup_if_exists(archive_file)
 436                 scm.export(export_prefix, archive_file, status.head_at_release)
 437
 438                 has_submodules = scm.has_submodules()
 439
 440                 if phase_actions['generate-archive'] or has_submodules:
 441                         try:
 442                                 support.unpack_tarball(archive_file)
 443                                 if has_submodules:
 444                                         scm.export_submodules(archive_name)
 445                                 run_hooks('generate-archive', cwd = archive_name, env = {'RELEASE_VERSION': status.release_version})
 446                                 info("Regenerating archive (may have been modified by generate-archive hooks...")
 447                                 support.check_call(['tar', 'cjf', archive_file, archive_name])
 448                         except SafeException:
 449                                 scm.reset_hard(scm.get_current_branch())
 450                                 fail_candidate()
 451                                 raise
 452
 453                 status.created_archive = 'true'
 454                 status.save()
 455
 456         if need_set_snapshot:
 457                 set_to_snapshot(status.release_version + '-post')
 458                 # Revert back to the original revision, so that any fixes the user makes
 459                 # will get applied before the tag
 460                 scm.reset_hard(scm.get_current_branch())
 461
 462         #backup_if_exists(archive_name)
 463         support.unpack_tarball(archive_file)
 464
 465         extracted_feed_path = os.path.abspath(os.path.join(export_prefix, local_iface_rel_root_path))
 466         assert os.path.isfile(extracted_feed_path), "Local feed not in archive! Is it under version control?"
 467         extracted_feed = support.load_feed(extracted_feed_path)
 468         extracted_impl = support.get_singleton_impl(extracted_feed)
 469
 470         if extracted_impl.main:
 471                 # Find main executable, relative to the archive root
 472                 abs_main = os.path.join(os.path.dirname(extracted_feed_path), extracted_impl.id, extracted_impl.main)
 473                 main = support.relative_path(archive_name + '/', abs_main)
 474                 if main != extracted_impl.main:
 475                         print "(adjusting main: '%s' for the feed inside the archive, '%s' externally)" % (extracted_impl.main, main)
 476                         # XXX: this is going to fail if the feed uses the new <command> syntax
 477                 if not os.path.exists(abs_main):
 478                         raise SafeException("Main executable '%s' not found after unpacking archive!" % abs_main)
 479                 if main == extracted_impl.main:
 480                         main = None     # Don't change the main attribute
 481         else:
 482                 main = None
 483
 484         try:
 485                 if status.src_tests_passed:
 486                         print "Unit-tests already passed - not running again"
 487                 else:
 488                         # Make directories read-only (checks tests don't write)
 489                         support.make_readonly_recursive(archive_name)
 490
 491                         run_unit_tests(extracted_feed_path)
 492                         status.src_tests_passed = True
 493                         status.save()
 494         except SafeException:
 495                 print "(leaving extracted directory for examination)"
 496                 fail_candidate()
 497                 raise
 498         # Unpack it again in case the unit-tests changed anything
 499         ro_rmtree(archive_name)
 500         support.unpack_tarball(archive_file)
 501
 502         # Generate feed for source
 503         src_feed_name = '%s.xml' % archive_name
 504         create_feed(src_feed_name, extracted_feed_path, archive_file, archive_name, main)
 505         print "Wrote source feed as %s" % src_feed_name
 506
 507         # If it's a source package, compile the binaries now...
 508         compiler = compile.Compiler(options, os.path.abspath(src_feed_name), release_version = status.release_version)
 509         compiler.build_binaries()
 510
 511         previous_release = get_previous_release(status.release_version)
 512         export_changelog(previous_release)
 513
 514         if status.tagged:
 515                 raw_input('Already tagged. Press Return to resume publishing process...')
 516                 choice = 'Publish'
 517         else:
 518                 print "\nCandidate release archive:", archive_file
 519                 print "(extracted to %s for inspection)" % os.path.abspath(archive_name)
 520
 521                 print "\nPlease check candidate and select an action:"
 522                 print "P) Publish candidate (accept)"
 523                 print "F) Fail candidate (delete release-status file)"
 524                 if previous_release:
 525                         print "D) Diff against release archive for %s" % previous_release
 526                         maybe_diff = ['Diff']
 527                 else:
 528                         maybe_diff = []
 529                 print "(you can also hit CTRL-C and resume this script when done)"
 530
 531                 while True:
 532                         choice = support.get_choice(['Publish', 'Fail'] + maybe_diff)
 533                         if choice == 'Diff':
 534                                 previous_archive_name = support.make_archive_name(local_feed.get_name(), previous_release)
 535                                 previous_archive_file = '../%s/%s.tar.bz2' % (previous_release, previous_archive_name)
 536
 537                                 # For archives created by older versions of 0release
 538                                 if not os.path.isfile(previous_archive_file):
 539                                         old_previous_archive_file = '../%s.tar.bz2' % previous_archive_name
 540                                         if os.path.isfile(old_previous_archive_file):
 541                                                 previous_archive_file = old_previous_archive_file
 542
 543                                 if os.path.isfile(previous_archive_file):
 544                                         support.unpack_tarball(previous_archive_file)
 545                                         try:
 546                                                 support.show_diff(previous_archive_name, archive_name)
 547                                         finally:
 548                                                 shutil.rmtree(previous_archive_name)
 549                                 else:
 550                                         # TODO: download it?
 551                                         print "Sorry, archive file %s not found! Can't show diff." % previous_archive_file
 552                         else:
 553                                 break
 554
 555         info("Deleting extracted archive %s", archive_name)
 556         shutil.rmtree(archive_name)
 557
 558         if choice == 'Publish':
 559                 accept_and_publish(archive_file, src_feed_name)
 560         else:
 561                 assert choice == 'Fail'
 562                 fail_candidate()