Updated unit-tests and fixed some bugs they found.
[0release.git] / release.py
blobfc7ae5803e2eb1adfba2c759f655049758fec0ae
1 # Copyright (C) 2007, Thomas Leonard
2 # See the README file for details, or visit http://0install.net.
4 import os, sys, subprocess, shutil, tarfile, tempfile
5 from zeroinstall import SafeException
6 from zeroinstall.injector import reader, model
7 from logging import info
9 import support
10 from scm import GIT
12 XMLNS_RELEASE = 'http://zero-install.sourceforge.net/2007/namespaces/0release'
14 valid_phases = ['commit-release', 'generate-archive']
16 TMP_BRANCH_NAME = '0release-tmp'
18 def run_unit_tests(impl):
19 self_test = impl.metadata.get('self-test', None)
20 if self_test is None:
21 print "SKIPPING unit tests for %s (no 'self-test' attribute set)" % impl
22 return
23 self_test = os.path.join(impl.id, self_test)
24 print "Running self-test:", self_test
25 exitstatus = subprocess.call([self_test], cwd = os.path.dirname(self_test))
26 if exitstatus:
27 raise SafeException("Self-test failed with exit status %d" % exitstatus)
29 def do_release(local_iface, options):
30 status = support.Status()
31 local_impl = support.get_singleton_impl(local_iface)
33 local_impl_dir = local_impl.id
34 assert local_impl_dir.startswith('/')
35 local_impl_dir = os.path.realpath(local_impl_dir)
36 assert os.path.isdir(local_impl_dir)
37 assert local_iface.uri.startswith(local_impl_dir + '/')
38 local_iface_rel_path = local_iface.uri[len(local_impl_dir) + 1:]
39 assert not local_iface_rel_path.startswith('/')
40 assert os.path.isfile(os.path.join(local_impl_dir, local_iface_rel_path))
42 phase_actions = {}
43 for phase in valid_phases:
44 phase_actions[phase] = [] # List of <release:action> elements
46 release_management = local_iface.get_metadata(XMLNS_RELEASE, 'management')
47 if len(release_management) == 1:
48 info("Found <release:management> element.")
49 release_management = release_management[0]
50 for x in release_management.childNodes:
51 if x.uri == XMLNS_RELEASE and x.name == 'action':
52 phase = x.getAttribute('phase')
53 if phase not in valid_phases:
54 raise SafeException("Invalid action phase '%s' in local feed %s. Valid actions are:\n%s" % (phase, local_iface.uri, '\n'.join(valid_phases)))
55 phase_actions[phase].append(x.content)
56 else:
57 warn("Unknown <release:management> element: %s", x)
58 elif len(release_management) > 1:
59 raise SafeException("Multiple <release:management> sections in %s!" % local_iface)
60 else:
61 info("No <release:management> element found in local feed.")
63 scm = GIT(local_iface, options)
65 def run_hooks(phase, cwd, env):
66 info("Running hooks for phase '%s'" % phase)
67 full_env = os.environ.copy()
68 full_env.update(env)
69 for x in phase_actions[phase]:
70 print "[%s]: %s" % (phase, x)
71 support.check_call(x, shell = True, cwd = cwd, env = full_env)
73 def set_to_release():
74 print "Snapshot version is " + local_impl.get_version()
75 suggested = support.suggest_release_version(local_impl.get_version())
76 release_version = raw_input("Version number for new release [%s]: " % suggested)
77 if not release_version:
78 release_version = suggested
80 scm.ensure_no_tag(release_version)
82 status.head_before_release = scm.get_head_revision()
83 status.save()
85 working_copy = local_impl.id
86 run_hooks('commit-release', cwd = working_copy, env = {'RELEASE_VERSION': release_version})
88 print "Releasing version", release_version
89 support.publish(local_iface.uri, set_released = 'today', set_version = release_version)
91 status.old_snapshot_version = local_impl.get_version()
92 status.release_version = release_version
93 status.head_at_release = scm.commit('Release %s' % release_version, branch = TMP_BRANCH_NAME, parent = 'HEAD')
94 status.save()
96 def set_to_snapshot(snapshot_version):
97 assert snapshot_version.endswith('-post')
98 support.publish(local_iface.uri, set_released = '', set_version = snapshot_version)
99 scm.commit('Start development series %s' % snapshot_version, branch = TMP_BRANCH_NAME, parent = TMP_BRANCH_NAME)
100 status.new_snapshot_version = scm.get_head_revision()
101 status.save()
103 def ensure_ready_to_release():
104 if not options.master_feed_file:
105 raise SafeException("Master feed file not set! Check your configuration")
107 scm.ensure_committed()
108 scm.ensure_versioned(local_iface_rel_path)
109 info("No uncommitted changes. Good.")
110 # Not needed for GIT. For SCMs where tagging is expensive (e.g. svn) this might be useful.
111 #run_unit_tests(local_impl)
113 def create_feed(local_iface_stream, archive_file, archive_name):
114 tmp = tempfile.NamedTemporaryFile(prefix = '0release-')
115 shutil.copyfileobj(local_iface_stream, tmp)
116 tmp.flush()
118 support.publish(tmp.name,
119 archive_url = options.archive_dir_public_url + '/' + os.path.basename(archive_file),
120 archive_file = archive_file,
121 archive_extract = archive_name)
122 return tmp
124 def unpack_tarball(archive_file, archive_name):
125 tar = tarfile.open(archive_file, 'r:bz2')
126 members = [m for m in tar.getmembers() if m.name != 'pax_global_header']
127 tar.extractall('.', members = members)
129 def get_previous_release(this_version):
130 """Return the highest numbered verison in the master feed before this_version.
131 @return: version, or None if there wasn't one"""
132 parsed_release_version = model.parse_version(this_version)
134 if os.path.exists(options.master_feed_file):
135 master = model.Interface(os.path.realpath(options.master_feed_file))
136 reader.update(master, master.uri, local = True)
137 versions = [impl.version for impl in master.implementations.values() if impl.version < parsed_release_version]
138 if versions:
139 return model.format_version(max(versions))
140 return None
142 def export_changelog(previous_release):
143 changelog = file('changelog-%s' % status.release_version, 'w')
144 try:
145 try:
146 scm.export_changelog(previous_release, status.head_before_release, changelog)
147 except SafeException, ex:
148 print "WARNING: Failed to generate changelog: " + str(ex)
149 else:
150 print "Wrote changelog from %s to here as %s" % (previous_release or 'start', changelog.name)
151 finally:
152 changelog.close()
154 def fail_candidate(archive_file):
155 support.backup_if_exists(archive_file)
156 scm.delete_branch(TMP_BRANCH_NAME)
157 os.unlink(support.release_status_file)
158 print "Restored to state before starting release. Make your fixes and try again..."
160 def accept_and_publish(archive_file, archive_name, local_iface_rel_path):
161 assert options.master_feed_file
163 if not options.archive_dir_public_url:
164 raise SafeException("Archive directory public URL is not set! Edit configuration and try again.")
166 if status.tagged:
167 print "Already tagged in SCM. Not re-tagging."
168 else:
169 scm.ensure_committed()
170 head = scm.get_head_revision()
171 if head != status.head_before_release:
172 raise SafeException("Changes committed since we started!\n" +
173 "HEAD was " + status.head_before_release + "\n"
174 "HEAD now " + head)
176 scm.tag(status.release_version, status.head_at_release)
177 scm.reset_hard(TMP_BRANCH_NAME)
178 scm.delete_branch(TMP_BRANCH_NAME)
180 status.tagged = 'true'
181 status.save()
183 if status.updated_master_feed:
184 print "Already added to master feed. Not changing."
185 else:
186 if os.path.exists(options.master_feed_file):
187 # Check we haven't already released this version
188 master = model.Interface(os.path.realpath(options.master_feed_file))
189 reader.update(master, master.uri, local = True)
190 existing_releases = [impl for impl in master.implementations.values() if impl.get_version() == status.release_version]
191 if len(existing_releases):
192 raise SafeException("Master feed %s already contains an implementation with version number %s!" % (options.master_feed_file, status.release_version))
194 tar = tarfile.open(archive_file, 'r:bz2')
195 stream = tar.extractfile(tar.getmember(archive_name + '/' + local_iface_rel_path))
196 remote_dl_iface = create_feed(stream, archive_file, archive_name)
197 stream.close()
199 support.publish(options.master_feed_file, local = remote_dl_iface.name, xmlsign = True, key = options.key)
200 remote_dl_iface.close()
202 status.updated_master_feed = 'true'
203 status.save()
205 def is_uploaded(url, size):
206 if url.startswith('http://TESTING/releases'):
207 return True
209 try:
210 actual_size = support.get_size(url)
211 except Exception, ex:
212 print "Can't get size of '%s': %s" % (url, ex)
213 return False
214 else:
215 if int(actual_size) == size:
216 return True
217 print "WARNING: %s exists, but size is %d, not %d!" % (url, actual_size, size)
218 return False
220 # Copy files...
221 print
222 archive_url = options.archive_dir_public_url + '/' + os.path.basename(archive_file)
223 archive_size = os.path.getsize(archive_file)
224 if status.uploaded_archive and is_uploaded(archive_url, archive_size):
225 print "Archive already uploaded. Not uploading again."
226 else:
227 while True:
228 print "Upload %s as %s" % (archive_file, archive_url)
229 cmd = options.archive_upload_command.strip()
230 if cmd:
231 support.show_and_run(cmd, [archive_file])
232 else:
233 print "No upload command is set => please upload the archive manually now"
234 raw_input('Press Return once archive is uploaded.')
235 print
236 if is_uploaded(archive_url, archive_size):
237 print "OK, archive uploaded successfully"
238 status.uploaded_archive = 'true'
239 status.save()
240 break
241 print "** Archive still not uploaded! Try again..."
242 if cmd:
243 raw_input('Press Return to retry upload command.')
245 assert len(local_iface.feed_for) == 1
246 feed_base = os.path.dirname(local_iface.feed_for.keys()[0])
247 feed_files = [options.master_feed_file]
248 print "Upload %s into %s" % (', '.join(feed_files), feed_base)
249 cmd = options.master_feed_upload_command.strip()
250 if cmd:
251 support.show_and_run(cmd, feed_files)
252 else:
253 print "NOTE: No feed upload command set => you'll have to upload them yourself!"
255 print "Push changes to public SCM repository..."
256 public_repos = options.public_scm_repository
257 if public_repos:
258 scm.push_head_and_release(status.release_version)
259 else:
260 print "NOTE: No public repository set => you'll have to push the tag and trunk yourself."
262 os.unlink(support.release_status_file)
264 if status.head_before_release:
265 head = scm.get_head_revision()
266 if status.release_version:
267 print "RESUMING release of %s %s" % (local_iface.get_name(), status.release_version)
268 elif head == status.head_before_release:
269 print "Restarting release of %s (HEAD revision has not changed)" % local_iface.get_name()
270 else:
271 raise SafeException("Something went wrong with the last run:\n" +
272 "HEAD revision for last run was " + status.head_before_release + "\n" +
273 "HEAD revision now is " + head + "\n" +
274 "You should revert your working copy to the previous head and try again.\n" +
275 "If you're sure you want to release from the current head, delete '" + support.release_status_file + "'")
276 else:
277 print "Releasing", local_iface.get_name()
279 ensure_ready_to_release()
281 if status.release_version:
282 need_set_snapshot = False
283 if status.tagged:
284 print "Already tagged. Resuming the publishing process..."
285 elif status.new_snapshot_version:
286 head = scm.get_head_revision()
287 if head != status.head_before_release:
288 raise SafeException("There are more commits since we started!\n"
289 "HEAD was " + status.head_before_release + "\n"
290 "HEAD now " + head + "\n"
291 "To include them, delete '" + support.release_status_file + "' and try again.\n"
292 "To leave them out, put them on a new branch and reset HEAD to the release version.")
293 else:
294 raise SafeException("Something went wrong previously when setting the new snapshot version.\n" +
295 "Suggest you reset to the original HEAD of\n%s and delete '%s'." % (status.head_before_release, support.release_status_file))
296 else:
297 set_to_release()
298 assert status.release_version
299 need_set_snapshot = True
301 archive_name = support.make_archive_name(local_iface.get_name(), status.release_version)
302 archive_file = archive_name + '.tar.bz2'
304 if status.created_archive and os.path.isfile(archive_file):
305 print "Archive already created"
306 else:
307 support.backup_if_exists(archive_file)
308 scm.export(archive_name, archive_file, status.head_at_release)
310 if phase_actions['generate-archive']:
311 try:
312 unpack_tarball(archive_file, archive_name)
313 run_hooks('generate-archive', cwd = archive_name, env = {'RELEASE_VERSION': status.release_version})
314 info("Regenerating archive (may have been modified by generate-archive hooks...")
315 support.check_call(['tar', 'cjf', archive_file, archive_name])
316 except SafeException:
317 fail_candidate(archive_file)
318 raise
320 status.created_archive = 'true'
321 status.save()
323 if need_set_snapshot:
324 set_to_snapshot(status.release_version + '-post')
325 # Revert back to the original revision, so that any fixes the user makes
326 # will get applied before the tag
327 scm.reset_hard(scm.get_current_branch())
329 #backup_if_exists(archive_name)
330 unpack_tarball(archive_file, archive_name)
331 if local_impl.main:
332 main = os.path.join(archive_name, local_impl.main)
333 if not os.path.exists(main):
334 raise SafeException("Main executable '%s' not found after unpacking archive!" % main)
336 extracted_iface_path = os.path.abspath(os.path.join(archive_name, local_iface_rel_path))
337 assert os.path.isfile(extracted_iface_path), "Local feed not in archive! Is it under version control?"
338 extracted_iface = model.Interface(extracted_iface_path)
339 reader.update(extracted_iface, extracted_iface_path, local = True)
340 extracted_impl = support.get_singleton_impl(extracted_iface)
342 try:
343 run_unit_tests(extracted_impl)
344 except SafeException:
345 print "(leaving extracted directory for examination)"
346 fail_candidate(archive_file)
347 raise
348 # Unpack it again in case the unit-tests changed anything
349 shutil.rmtree(archive_name)
350 unpack_tarball(archive_file, archive_name)
352 previous_release = get_previous_release(status.release_version)
353 export_changelog(previous_release)
355 print "\nCandidate release archive:", archive_file
356 print "(extracted to %s for inspection)" % os.path.abspath(archive_name)
358 print "\nPlease check candidate and select an action:"
359 print "P) Publish candidate (accept)"
360 print "F) Fail candidate (untag)"
361 if previous_release:
362 print "D) Diff against release archive for %s" % previous_release
363 maybe_diff = ['Diff']
364 else:
365 maybe_diff = []
366 print "(you can also hit CTRL-C and resume this script when done)"
368 while True:
369 choice = support.get_choice(['Publish', 'Fail'] + maybe_diff)
370 if choice == 'Diff':
371 previous_archive_name = support.make_archive_name(local_iface.get_name(), previous_release)
372 previous_archive_file = previous_archive_name + '.tar.bz2'
373 if os.path.isfile(previous_archive_file):
374 unpack_tarball(previous_archive_file, previous_archive_name)
375 try:
376 support.show_diff(previous_archive_name, archive_name)
377 finally:
378 shutil.rmtree(previous_archive_name)
379 else:
380 # TODO: download it?
381 print "Sorry, archive file %s not found! Can't show diff." % previous_archive_file
382 else:
383 break
385 info("Deleting extracted archive %s", archive_name)
386 shutil.rmtree(archive_name)
388 if choice == 'Publish':
389 accept_and_publish(archive_file, archive_name, local_iface_rel_path)
390 else:
391 assert choice == 'Fail'
392 fail_candidate(archive_file)