From 896d9a187e82e341b29cd61cf8909449eba6772a Mon Sep 17 00:00:00 2001 From: Thomas Leonard Date: Thu, 2 May 2013 12:05:09 +0100 Subject: [PATCH] Use 0repo to release if a suitable repository is registered Supporting both old and new systems is a bit messy, but it allows a smoother change-over. --- 0release.xml | 4 ++ release.py | 119 ++++++++++++++++++++++++++++++--------------------- scm.py | 8 ++++ support.py | 18 +++++++- tests/testrelease.py | 75 +++++++++++++++++++++++++++++--- 5 files changed, 169 insertions(+), 55 deletions(-) diff --git a/0release.xml b/0release.xml index b3390c8..4dcb551 100644 --- a/0release.xml +++ b/0release.xml @@ -34,6 +34,10 @@ + + + + diff --git a/release.py b/release.py index 25f9ccf..d750648 100644 --- a/release.py +++ b/release.py @@ -1,12 +1,15 @@ # Copyright (C) 2009, Thomas Leonard # See the README file for details, or visit http://0install.net. -import os, subprocess, shutil +import os, subprocess, shutil, sys from zeroinstall import SafeException from zeroinstall.injector import model from zeroinstall.support import ro_rmtree from logging import info, warn +sys.path.insert(0, os.environ['RELEASE_0REPO']) +from repo import registry + import support, compile from scm import get_scm @@ -122,11 +125,8 @@ def upload_archives(options, status, uploads): raw_input('Press Return to try again.') def do_release(local_feed, options): - assert options.master_feed_file - options.master_feed_file = os.path.abspath(options.master_feed_file) - - if not options.archive_dir_public_url: - raise SafeException("Downloads directory not set. Edit the 'make-release' script and try again.") + if options.master_feed_file: + options.master_feed_file = os.path.abspath(options.master_feed_file) if not local_feed.feed_for: raise SafeException("Feed %s missing a element" % local_feed.local_path) @@ -211,17 +211,17 @@ def do_release(local_feed, options): status.release_version = release_version status.head_at_release = scm.commit('Release %s' % release_version, branch = TMP_BRANCH_NAME, parent = 'HEAD') status.save() - + def set_to_snapshot(snapshot_version): assert snapshot_version.endswith('-post') support.publish(local_feed.local_path, set_released = '', set_version = snapshot_version) scm.commit('Start development series %s' % snapshot_version, branch = TMP_BRANCH_NAME, parent = TMP_BRANCH_NAME) status.new_snapshot_version = scm.get_head_revision() status.save() - + def ensure_ready_to_release(): - if not options.master_feed_file: - raise SafeException("Master feed file not set! Check your configuration") + #if not options.master_feed_file: + # raise SafeException("Master feed file not set! Check your configuration") scm.ensure_committed() scm.ensure_versioned(os.path.abspath(local_feed.local_path)) @@ -230,7 +230,7 @@ def do_release(local_feed, options): #run_unit_tests(local_impl) scm.grep('\(^\\|[^=]\)\<\\(TODO\\|XXX\\|FIXME\\)\>') - + def create_feed(target_feed, local_iface_path, archive_file, archive_name, main): shutil.copyfile(local_iface_path, target_feed) @@ -239,17 +239,17 @@ def do_release(local_feed, options): archive_url = support.get_archive_url(options, status.release_version, os.path.basename(archive_file)), archive_file = archive_file, archive_extract = archive_name) - + def get_previous_release(this_version): """Return the highest numbered verison in the master feed before this_version. @return: version, or None if there wasn't one""" parsed_release_version = model.parse_version(this_version) - if os.path.exists(options.master_feed_file): - master = support.load_feed(options.master_feed_file) - versions = [impl.version for impl in master.implementations.values() if impl.version < parsed_release_version] - if versions: - return model.format_version(max(versions)) + versions = [model.parse_version(version) for version in scm.get_tagged_versions()] + versions = [version for version in versions if version < parsed_release_version] + + if versions: + return model.format_version(max(versions)) return None def export_changelog(previous_release): @@ -263,7 +263,7 @@ def do_release(local_feed, options): print "Wrote changelog from %s to here as %s" % (previous_release or 'start', changelog.name) finally: changelog.close() - + def fail_candidate(): cwd = os.getcwd() assert cwd.endswith(status.release_version) @@ -272,29 +272,21 @@ def do_release(local_feed, options): os.unlink(support.release_status_file) print "Restored to state before starting release. Make your fixes and try again..." - def accept_and_publish(archive_file, src_feed_name): + def release_via_0repo(new_impls_feed): + import repo.cmd + support.make_archives_relative(new_impls_feed) + oldcwd = os.getcwd() + try: + repo.cmd.main(['0repo', 'add', '--', new_impls_feed]) + finally: + os.chdir(oldcwd) + + def release_without_0repo(archive_file, new_impls_feed): assert options.master_feed_file if not options.archive_dir_public_url: raise SafeException("Archive directory public URL is not set! Edit configuration and try again.") - if status.tagged: - print "Already tagged in SCM. Not re-tagging." - else: - scm.ensure_committed() - head = scm.get_head_revision() - if head != status.head_before_release: - raise SafeException("Changes committed since we started!\n" + - "HEAD was " + status.head_before_release + "\n" - "HEAD now " + head) - - scm.tag(status.release_version, status.head_at_release) - scm.reset_hard(TMP_BRANCH_NAME) - scm.delete_branch(TMP_BRANCH_NAME) - - status.tagged = 'true' - status.save() - if status.updated_master_feed: print "Already added to master feed. Not changing." else: @@ -315,15 +307,7 @@ def do_release(local_feed, options): publish_opts['select_version'] = previous_release publish_opts['set_stability'] = "stable" - # Merge the source and binary feeds together first, so - # that we update the master feed atomically and only - # have to sign it once. - shutil.copyfile(src_feed_name, 'merged.xml') - for b in compiler.get_binary_feeds(): - support.publish('merged.xml', local = b) - - support.publish(options.master_feed_file, local = 'merged.xml', xmlsign = True, key = options.key, **publish_opts) - os.unlink('merged.xml') + support.publish(options.master_feed_file, local = new_impls_feed, xmlsign = True, key = options.key, **publish_opts) status.updated_master_feed = 'true' status.save() @@ -337,7 +321,6 @@ def do_release(local_feed, options): upload_archives(options, status, uploads) - assert len(local_feed.feed_for) == 1 feed_base = os.path.dirname(list(local_feed.feed_for)[0]) feed_files = [options.master_feed_file] print "Upload %s into %s" % (', '.join(feed_files), feed_base) @@ -347,6 +330,44 @@ def do_release(local_feed, options): else: print "NOTE: No feed upload command set => you'll have to upload them yourself!" + def accept_and_publish(archive_file, src_feed_name): + if status.tagged: + print "Already tagged in SCM. Not re-tagging." + else: + scm.ensure_committed() + head = scm.get_head_revision() + if head != status.head_before_release: + raise SafeException("Changes committed since we started!\n" + + "HEAD was " + status.head_before_release + "\n" + "HEAD now " + head) + + scm.tag(status.release_version, status.head_at_release) + scm.reset_hard(TMP_BRANCH_NAME) + scm.delete_branch(TMP_BRANCH_NAME) + + status.tagged = 'true' + status.save() + + assert len(local_feed.feed_for) == 1 + + # Merge the source and binary feeds together first, so + # that we update the master feed atomically and only + # have to sign it once. + new_impls_feed = 'merged.xml' + shutil.copyfile(src_feed_name, new_impls_feed) + for b in compiler.get_binary_feeds(): + support.publish(new_impls_feed, local = b) + + # TODO: support uploading to a sub-feed (requires support in 0repo too) + master_feed, = local_feed.feed_for + repository = registry.lookup(master_feed, missing_ok = True) + if repository: + release_via_0repo(new_impls_feed) + else: + release_without_0repo(archive_file, new_impls_feed) + + os.unlink(new_impls_feed) + print "Push changes to public SCM repository..." public_repos = options.public_scm_repository if public_repos: @@ -355,9 +376,9 @@ def do_release(local_feed, options): print "NOTE: No public repository set => you'll have to push the tag and trunk yourself." os.unlink(support.release_status_file) - + if status.head_before_release: - head = scm.get_head_revision() + head = scm.get_head_revision() if status.release_version: print "RESUMING release of %s %s" % (local_feed.get_name(), status.release_version) if options.release_version and options.release_version != status.release_version: @@ -383,7 +404,7 @@ def do_release(local_feed, options): if status.tagged: print "Already tagged. Resuming the publishing process..." elif status.new_snapshot_version: - head = scm.get_head_revision() + head = scm.get_head_revision() if head != status.head_before_release: raise SafeException("There are more commits since we started!\n" "HEAD was " + status.head_before_release + "\n" diff --git a/scm.py b/scm.py index 0532f22..6699478 100644 --- a/scm.py +++ b/scm.py @@ -83,6 +83,14 @@ class GIT(SCM): info("Current branch is %s", current_branch) return current_branch + def get_tagged_versions(self): + child = self._run(['tag', '-l', 'v*'], stdout = subprocess.PIPE) + stdout, unused = child.communicate() + status = child.wait() + if status: + raise SafeException("git tag failed with exit code %d" % status) + return [v[1:] for v in stdout.split('\n') if v] + def delete_branch(self, branch): self._run_check(['branch', '-D', branch]) diff --git a/support.py b/support.py index 4c8cbec..399129e 100644 --- a/support.py +++ b/support.py @@ -4,8 +4,10 @@ import copy import os, subprocess, tarfile import urlparse, ftplib, httplib +from xml.dom import minidom + from zeroinstall import SafeException -from zeroinstall.injector import model, qdom +from zeroinstall.injector import model, qdom, namespaces from zeroinstall.support import ro_rmtree from logging import info @@ -232,7 +234,21 @@ def make_readonly_recursive(path): os.chmod(full, mode & 0o555) def get_archive_url(options, release_version, archive): + if not options.archive_dir_public_url: + return archive # Not needed with 0repo + archive_dir_public_url = options.archive_dir_public_url.replace('$RELEASE_VERSION', release_version) if not archive_dir_public_url.endswith('/'): archive_dir_public_url += '/' return archive_dir_public_url + archive + +def make_archives_relative(feed): + with open(feed, 'rb') as stream: + doc = minidom.parse(stream) + for elem in doc.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'archive') + doc.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'file'): + href = elem.getAttribute('href') + assert href, 'Missing href on %r' % elem + if '/' in href: + elem.setAttribute('href', href.rsplit('/', 1)[1]) + with open(feed, 'wb') as stream: + doc.writexml(stream) diff --git a/tests/testrelease.py b/tests/testrelease.py index 542f091..3f00919 100755 --- a/tests/testrelease.py +++ b/tests/testrelease.py @@ -2,7 +2,9 @@ # Copyright (C) 2007, Thomas Leonard # See the README file for details, or visit http://0install.net. import sys, os, shutil, tempfile, subprocess, imp +from StringIO import StringIO import unittest + from zeroinstall.injector import model, qdom, writer from zeroinstall.injector.config import load_config from zeroinstall.support import basedir, ro_rmtree @@ -11,6 +13,8 @@ sys.path.insert(0, '..') os.environ['http_proxy'] = 'localhost:1111' # Prevent accidental network access import support +import release # (sets sys.path for 0repo) +import repo.cmd mydir = os.path.realpath(os.path.dirname(__file__)) release_feed = mydir + '/../0release.xml' @@ -27,6 +31,26 @@ help_with_testing = True network_use = full """ +CUSTOM_REPO_CONFIG = """ +REPOSITORY_BASE_URL = "http://0install.net/tests/" +ARCHIVES_BASE_URL = "http://TESTING/releases" + +def upload_archives(archives): + for dir_rel_url, files in paths.group_by_target_url_dir(archives): + target_dir = join('..', 'releases', 'archives') # hack: skip dir_rel_url + if not os.path.isdir(target_dir): + os.makedirs(target_dir) + subprocess.check_call(["cp"] + files + [target_dir]) + +def check_new_impl(impl): + pass + +def get_archive_rel_url(archive_basename, impl): + return "{version}/{archive}".format( + version = impl.get_version(), + archive = archive_basename) +""" + def call_with_output_suppressed(cmd, stdin, expect_failure = False, **kwargs): #cmd = [cmd[0], '-v'] + cmd[1:] if stdin: @@ -129,7 +153,7 @@ class TestRelease(unittest.TestCase): call_with_output_suppressed(['./make-release', '-k', 'Testing', '--builders=host'], '\nP\n\n') - feed = model.ZeroInstallFeed(qdom.parse(file('HelloWorld-in-C.xml'))) + feed = self.get_public_feed('HelloWorld-in-C.xml', 'c-prog.xml') assert len(feed.implementations) == 2 src_impl, = [x for x in feed.implementations.values() if x.arch == '*-src'] @@ -139,7 +163,7 @@ class TestRelease(unittest.TestCase): assert host_impl.main == 'hello' archives = os.listdir('archives') - assert os.path.basename(src_impl.download_sources[0].url) in archives + assert os.path.basename(src_impl.download_sources[0].url) in archives, src_impl.download_sources[0].url host_download = host_impl.download_sources[0] self.assertEqual('http://TESTING/releases/1.1/helloworld-in-c-linux-x86_64-1.1.tar.bz2', @@ -151,8 +175,49 @@ class TestRelease(unittest.TestCase): output, _ = c.communicate() self.assertEquals("Hello from C! (version 1.1)\n", output) - - -suite = unittest.makeSuite(TestRelease) + + def get_public_feed(self, name, uri_basename): + with open(name, 'rb') as stream: + return model.ZeroInstallFeed(qdom.parse(stream)) + +def run_repo(args): + oldcwd = os.getcwd() + + old_stdout = sys.stdout + sys.stdout = StringIO() + try: + sys.stdin = StringIO('\n') # (simulate a press of Return if needed) + repo.cmd.main(['0repo'] + args) + return sys.stdout.getvalue() + finally: + os.chdir(oldcwd) + sys.stdout = old_stdout + +class TestRepoRelease(TestRelease): + def setUp(self): + TestRelease.setUp(self) + + # Let GPG initialise (it's a bit verbose) + child = subprocess.Popen(['gpg', '-q', '--list-secret-keys'], stdout = subprocess.PIPE, stderr = subprocess.PIPE) + unused, unused = child.communicate() + child.wait() + + run_repo(['create', 'my-repo', 'Testing ']) + os.chdir('my-repo') + + if '0repo-config' in sys.modules: + del sys.modules['0repo-config'] + + with open('0repo-config.py', 'at') as stream: + stream.write(CUSTOM_REPO_CONFIG) + run_repo(['register']) + os.chdir('..') + + def get_public_feed(self, name, uri_basename): + with open(os.path.join(self.tmp, 'my-repo', 'public', uri_basename), 'rb') as stream: + return model.ZeroInstallFeed(qdom.parse(stream)) + +unittest.makeSuite(TestRelease) +unittest.makeSuite(TestRepoRelease) if __name__ == '__main__': unittest.main() -- 2.11.4.GIT