Cope with feeds that aren't at the top level
[0release.git] / release.py
blobc2ae2dde7b57cf09b17dbbcb48f2f6a725b8b9cc
1 # Copyright (C) 2009, Thomas Leonard
2 # See the README file for details, or visit http://0install.net.
4 import os, sys, subprocess, shutil, tempfile
5 from zeroinstall import SafeException
6 from zeroinstall.injector import reader, model
7 from logging import info, warn
9 import support, compile
10 from scm import get_scm
12 XMLNS_RELEASE = 'http://zero-install.sourceforge.net/2007/namespaces/0release'
14 valid_phases = ['commit-release', 'generate-archive']
16 TMP_BRANCH_NAME = '0release-tmp'
18 def run_unit_tests(local_feed, impl):
19 self_test = impl.metadata.get('self-test', None)
20 if self_test is None:
21 print "SKIPPING unit tests for %s (no 'self-test' attribute set)" % impl
22 return
23 self_test_dir = os.path.dirname(os.path.join(impl.id, self_test))
24 print "Running self-test:", self_test
25 exitstatus = subprocess.call(['0launch', '--main', self_test, local_feed], cwd = self_test_dir)
26 if exitstatus:
27 raise SafeException("Self-test failed with exit status %d" % exitstatus)
29 def do_release(local_iface, options):
30 assert options.master_feed_file
31 options.master_feed_file = os.path.abspath(options.master_feed_file)
33 status = support.Status()
34 local_impl = support.get_singleton_impl(local_iface)
36 local_impl_dir = local_impl.id
37 assert local_impl_dir.startswith('/')
38 local_impl_dir = os.path.realpath(local_impl_dir)
39 assert os.path.isdir(local_impl_dir)
40 assert local_iface.uri.startswith(local_impl_dir + '/')
42 # From the impl directory to the feed
43 # NOT relative to the archive root (in general)
44 local_iface_rel_path = local_iface.uri[len(local_impl_dir) + 1:]
45 assert not local_iface_rel_path.startswith('/')
46 assert os.path.isfile(os.path.join(local_impl_dir, local_iface_rel_path))
48 phase_actions = {}
49 for phase in valid_phases:
50 phase_actions[phase] = [] # List of <release:action> elements
52 add_toplevel_dir = None
53 release_management = local_iface.get_metadata(XMLNS_RELEASE, 'management')
54 if len(release_management) == 1:
55 info("Found <release:management> element.")
56 release_management = release_management[0]
57 for x in release_management.childNodes:
58 if x.uri == XMLNS_RELEASE and x.name == 'action':
59 phase = x.getAttribute('phase')
60 if phase not in valid_phases:
61 raise SafeException("Invalid action phase '%s' in local feed %s. Valid actions are:\n%s" % (phase, local_iface.uri, '\n'.join(valid_phases)))
62 phase_actions[phase].append(x.content)
63 elif x.uri == XMLNS_RELEASE and x.name == 'add-toplevel-directory':
64 add_toplevel_dir = local_iface.get_name()
65 else:
66 warn("Unknown <release:management> element: %s", x)
67 elif len(release_management) > 1:
68 raise SafeException("Multiple <release:management> sections in %s!" % local_iface)
69 else:
70 info("No <release:management> element found in local feed.")
72 scm = get_scm(local_iface, options)
74 # Path relative to the archive / SCM root
75 local_iface_rel_root_path = local_iface.uri[len(scm.root_dir) + 1:]
77 def run_hooks(phase, cwd, env):
78 info("Running hooks for phase '%s'" % phase)
79 full_env = os.environ.copy()
80 full_env.update(env)
81 for x in phase_actions[phase]:
82 print "[%s]: %s" % (phase, x)
83 support.check_call(x, shell = True, cwd = cwd, env = full_env)
85 def set_to_release():
86 print "Snapshot version is " + local_impl.get_version()
87 suggested = support.suggest_release_version(local_impl.get_version())
88 release_version = raw_input("Version number for new release [%s]: " % suggested)
89 if not release_version:
90 release_version = suggested
92 scm.ensure_no_tag(release_version)
94 status.head_before_release = scm.get_head_revision()
95 status.save()
97 working_copy = local_impl.id
98 run_hooks('commit-release', cwd = working_copy, env = {'RELEASE_VERSION': release_version})
100 print "Releasing version", release_version
101 support.publish(local_iface.uri, set_released = 'today', set_version = release_version)
103 support.backup_if_exists(release_version)
104 os.mkdir(release_version)
105 os.chdir(release_version)
107 status.old_snapshot_version = local_impl.get_version()
108 status.release_version = release_version
109 status.head_at_release = scm.commit('Release %s' % release_version, branch = TMP_BRANCH_NAME, parent = 'HEAD')
110 status.save()
112 def set_to_snapshot(snapshot_version):
113 assert snapshot_version.endswith('-post')
114 support.publish(local_iface.uri, set_released = '', set_version = snapshot_version)
115 scm.commit('Start development series %s' % snapshot_version, branch = TMP_BRANCH_NAME, parent = TMP_BRANCH_NAME)
116 status.new_snapshot_version = scm.get_head_revision()
117 status.save()
119 def ensure_ready_to_release():
120 if not options.master_feed_file:
121 raise SafeException("Master feed file not set! Check your configuration")
123 scm.ensure_committed()
124 scm.ensure_versioned(os.path.abspath(local_iface.uri))
125 info("No uncommitted changes. Good.")
126 # Not needed for GIT. For SCMs where tagging is expensive (e.g. svn) this might be useful.
127 #run_unit_tests(local_impl)
129 scm.grep('\(^\\|[^=]\)\<\\(TODO\\|XXX\\|FIXME\\)\>')
131 def create_feed(target_feed, local_iface_path, archive_file, archive_name, main):
132 shutil.copyfile(local_iface_path, target_feed)
134 support.publish(target_feed,
135 set_main = main,
136 archive_url = options.archive_dir_public_url + '/' + os.path.basename(archive_file),
137 archive_file = archive_file,
138 archive_extract = archive_name)
140 def get_previous_release(this_version):
141 """Return the highest numbered verison in the master feed before this_version.
142 @return: version, or None if there wasn't one"""
143 parsed_release_version = model.parse_version(this_version)
145 if os.path.exists(options.master_feed_file):
146 master = model.Interface(os.path.realpath(options.master_feed_file))
147 reader.update(master, master.uri, local = True)
148 versions = [impl.version for impl in master.implementations.values() if impl.version < parsed_release_version]
149 if versions:
150 return model.format_version(max(versions))
151 return None
153 def export_changelog(previous_release):
154 changelog = file('changelog-%s' % status.release_version, 'w')
155 try:
156 try:
157 scm.export_changelog(previous_release, status.head_before_release, changelog)
158 except SafeException, ex:
159 print "WARNING: Failed to generate changelog: " + str(ex)
160 else:
161 print "Wrote changelog from %s to here as %s" % (previous_release or 'start', changelog.name)
162 finally:
163 changelog.close()
165 def fail_candidate(archive_file):
166 cwd = os.getcwd()
167 assert cwd.endswith(status.release_version)
168 support.backup_if_exists(cwd)
169 scm.delete_branch(TMP_BRANCH_NAME)
170 os.unlink(support.release_status_file)
171 print "Restored to state before starting release. Make your fixes and try again..."
173 def accept_and_publish(archive_file, archive_name, src_feed_name):
174 assert options.master_feed_file
176 if not options.archive_dir_public_url:
177 raise SafeException("Archive directory public URL is not set! Edit configuration and try again.")
179 if status.tagged:
180 print "Already tagged in SCM. Not re-tagging."
181 else:
182 scm.ensure_committed()
183 head = scm.get_head_revision()
184 if head != status.head_before_release:
185 raise SafeException("Changes committed since we started!\n" +
186 "HEAD was " + status.head_before_release + "\n"
187 "HEAD now " + head)
189 scm.tag(status.release_version, status.head_at_release)
190 scm.reset_hard(TMP_BRANCH_NAME)
191 scm.delete_branch(TMP_BRANCH_NAME)
193 status.tagged = 'true'
194 status.save()
196 if status.updated_master_feed:
197 print "Already added to master feed. Not changing."
198 else:
199 if os.path.exists(options.master_feed_file):
200 # Check we haven't already released this version
201 master = model.Interface(os.path.realpath(options.master_feed_file))
202 reader.update(master, master.uri, local = True)
203 existing_releases = [impl for impl in master.implementations.values() if impl.get_version() == status.release_version]
204 if len(existing_releases):
205 raise SafeException("Master feed %s already contains an implementation with version number %s!" % (options.master_feed_file, status.release_version))
207 # Merge the source and binary feeds together first, so
208 # that we update the master feed atomically and only
209 # have to sign it once.
210 shutil.copyfile(src_feed_name, 'merged.xml')
211 for b in compiler.get_binary_feeds():
212 support.publish('merged.xml', local = b)
214 support.publish(options.master_feed_file, local = 'merged.xml', xmlsign = True, key = options.key)
215 os.unlink('merged.xml')
217 status.updated_master_feed = 'true'
218 status.save()
220 def is_uploaded(url, size):
221 if url.startswith('http://TESTING/releases'):
222 return True
224 print "Testing URL %s..." % url
225 try:
226 actual_size = int(support.get_size(url))
227 except Exception, ex:
228 print "Can't get size of '%s': %s" % (url, ex)
229 return False
230 else:
231 if actual_size == size:
232 return True
233 print "WARNING: %s exists, but size is %d, not %d!" % (url, actual_size, size)
234 return False
236 # Copy files...
237 print
238 archive_url = options.archive_dir_public_url + '/' + os.path.basename(archive_file)
239 archive_size = os.path.getsize(archive_file)
240 if status.uploaded_archive and is_uploaded(archive_url, archive_size):
241 print "Archive already uploaded. Not uploading again."
242 else:
243 while True:
244 print "Upload %s/%s as %s" % (status.release_version, archive_file, archive_url)
245 cmd = options.archive_upload_command.strip()
246 if cmd:
247 support.show_and_run(cmd, [archive_file])
248 else:
249 print "No upload command is set => please upload the archive manually now"
250 raw_input('Press Return once archive is uploaded.')
251 print
252 if is_uploaded(archive_url, archive_size):
253 print "OK, archive uploaded successfully"
254 status.uploaded_archive = 'true'
255 status.save()
256 break
257 print "** Archive still not uploaded! Try again..."
258 if cmd:
259 raw_input('Press Return to retry upload command.')
261 assert len(local_iface.feed_for) == 1
262 feed_base = os.path.dirname(local_iface.feed_for.keys()[0])
263 feed_files = [options.master_feed_file]
264 print "Upload %s into %s" % (', '.join(feed_files), feed_base)
265 cmd = options.master_feed_upload_command.strip()
266 if cmd:
267 support.show_and_run(cmd, feed_files)
268 else:
269 print "NOTE: No feed upload command set => you'll have to upload them yourself!"
271 print "Push changes to public SCM repository..."
272 public_repos = options.public_scm_repository
273 if public_repos:
274 scm.push_head_and_release(status.release_version)
275 else:
276 print "NOTE: No public repository set => you'll have to push the tag and trunk yourself."
278 os.unlink(support.release_status_file)
280 if status.head_before_release:
281 head = scm.get_head_revision()
282 if status.release_version:
283 print "RESUMING release of %s %s" % (local_iface.get_name(), status.release_version)
284 elif head == status.head_before_release:
285 print "Restarting release of %s (HEAD revision has not changed)" % local_iface.get_name()
286 else:
287 raise SafeException("Something went wrong with the last run:\n" +
288 "HEAD revision for last run was " + status.head_before_release + "\n" +
289 "HEAD revision now is " + head + "\n" +
290 "You should revert your working copy to the previous head and try again.\n" +
291 "If you're sure you want to release from the current head, delete '" + support.release_status_file + "'")
292 else:
293 print "Releasing", local_iface.get_name()
295 ensure_ready_to_release()
297 if status.release_version:
298 if not os.path.isdir(status.release_version):
299 raise SafeException("Can't resume; directory %s missing. Try deleting '%s'." % (status.release_version, support.release_status_file))
300 os.chdir(status.release_version)
301 need_set_snapshot = False
302 if status.tagged:
303 print "Already tagged. Resuming the publishing process..."
304 elif status.new_snapshot_version:
305 head = scm.get_head_revision()
306 if head != status.head_before_release:
307 raise SafeException("There are more commits since we started!\n"
308 "HEAD was " + status.head_before_release + "\n"
309 "HEAD now " + head + "\n"
310 "To include them, delete '" + support.release_status_file + "' and try again.\n"
311 "To leave them out, put them on a new branch and reset HEAD to the release version.")
312 else:
313 raise SafeException("Something went wrong previously when setting the new snapshot version.\n" +
314 "Suggest you reset to the original HEAD of\n%s and delete '%s'." % (status.head_before_release, support.release_status_file))
315 else:
316 set_to_release() # Changes directory
317 assert status.release_version
318 need_set_snapshot = True
320 archive_name = support.make_archive_name(local_iface.get_name(), status.release_version)
321 archive_file = archive_name + '.tar.bz2'
323 export_prefix = archive_name
324 if add_toplevel_dir is not None:
325 export_prefix += '/' + add_toplevel_dir
327 if status.created_archive and os.path.isfile(archive_file):
328 print "Archive already created"
329 else:
330 support.backup_if_exists(archive_file)
331 scm.export(export_prefix, archive_file, status.head_at_release)
333 has_submodules = scm.has_submodules()
335 if phase_actions['generate-archive'] or has_submodules:
336 try:
337 support.unpack_tarball(archive_file)
338 if has_submodules:
339 scm.export_submodules(archive_name)
340 run_hooks('generate-archive', cwd = archive_name, env = {'RELEASE_VERSION': status.release_version})
341 info("Regenerating archive (may have been modified by generate-archive hooks...")
342 support.check_call(['tar', 'cjf', archive_file, archive_name])
343 except SafeException:
344 scm.reset_hard(scm.get_current_branch())
345 fail_candidate(archive_file)
346 raise
348 status.created_archive = 'true'
349 status.save()
351 if need_set_snapshot:
352 set_to_snapshot(status.release_version + '-post')
353 # Revert back to the original revision, so that any fixes the user makes
354 # will get applied before the tag
355 scm.reset_hard(scm.get_current_branch())
357 #backup_if_exists(archive_name)
358 support.unpack_tarball(archive_file)
360 extracted_iface_path = os.path.abspath(os.path.join(export_prefix, local_iface_rel_root_path))
361 assert os.path.isfile(extracted_iface_path), "Local feed not in archive! Is it under version control?"
362 extracted_iface = model.Interface(extracted_iface_path)
363 reader.update(extracted_iface, extracted_iface_path, local = True)
364 extracted_impl = support.get_singleton_impl(extracted_iface)
366 if extracted_impl.main:
367 # Find main executable, relative to the archive root
368 abs_main = os.path.join(os.path.dirname(extracted_iface_path), extracted_impl.main)
369 main = support.relative_path(archive_name + '/', abs_main)
370 if main != extracted_impl.main:
371 print "(adjusting main: '%s' for the feed inside the archive, '%s' externally)" % (extracted_impl.main, main)
372 if not os.path.exists(abs_main):
373 raise SafeException("Main executable '%s' not found after unpacking archive!" % abs_main)
374 else:
375 main = None
377 try:
378 run_unit_tests(extracted_iface_path, extracted_impl)
379 except SafeException:
380 print "(leaving extracted directory for examination)"
381 fail_candidate(archive_file)
382 raise
383 # Unpack it again in case the unit-tests changed anything
384 shutil.rmtree(archive_name)
385 support.unpack_tarball(archive_file)
387 # Generate feed for source
388 stream = open(extracted_iface_path)
389 src_feed_name = '%s.xml' % archive_name
390 create_feed(src_feed_name, extracted_iface_path, archive_file, archive_name, main)
391 print "Wrote source feed as %s" % src_feed_name
393 # If it's a source package, compile the binaries now...
394 compiler = compile.Compiler(options, os.path.abspath(src_feed_name))
395 compiler.build_binaries()
397 previous_release = get_previous_release(status.release_version)
398 export_changelog(previous_release)
400 print "\nCandidate release archive:", archive_file
401 print "(extracted to %s for inspection)" % os.path.abspath(archive_name)
403 print "\nPlease check candidate and select an action:"
404 print "P) Publish candidate (accept)"
405 print "F) Fail candidate (untag)"
406 if previous_release:
407 print "D) Diff against release archive for %s" % previous_release
408 maybe_diff = ['Diff']
409 else:
410 maybe_diff = []
411 print "(you can also hit CTRL-C and resume this script when done)"
413 while True:
414 choice = support.get_choice(['Publish', 'Fail'] + maybe_diff)
415 if choice == 'Diff':
416 previous_archive_name = support.make_archive_name(local_iface.get_name(), previous_release)
417 previous_archive_file = '../%s/%s.tar.bz2' % (previous_release, previous_archive_name)
419 # For archives created by older versions of 0release
420 if not os.path.isfile(previous_archive_file):
421 old_previous_archive_file = '../%s.tar.bz2' % previous_archive_name
422 if os.path.isfile(old_previous_archive_file):
423 previous_archive_file = old_previous_archive_file
425 if os.path.isfile(previous_archive_file):
426 support.unpack_tarball(previous_archive_file)
427 try:
428 support.show_diff(previous_archive_name, archive_name)
429 finally:
430 shutil.rmtree(previous_archive_name)
431 else:
432 # TODO: download it?
433 print "Sorry, archive file %s not found! Can't show diff." % previous_archive_file
434 else:
435 break
437 info("Deleting extracted archive %s", archive_name)
438 shutil.rmtree(archive_name)
440 if choice == 'Publish':
441 accept_and_publish(archive_file, archive_name, src_feed_name)
442 else:
443 assert choice == 'Fail'
444 fail_candidate(archive_file)