1 # Copyright (C) 2009, Thomas Leonard
2 # See the README file for details, or visit http://0install.net.
4 import os
, sys
, subprocess
, shutil
, tempfile
5 from zeroinstall
import SafeException
6 from zeroinstall
.injector
import reader
, model
, qdom
7 from logging
import info
, warn
9 import support
, compile
10 from scm
import get_scm
12 XMLNS_RELEASE
= 'http://zero-install.sourceforge.net/2007/namespaces/0release'
14 valid_phases
= ['commit-release', 'generate-archive']
16 TMP_BRANCH_NAME
= '0release-tmp'
18 test_command
= os
.environ
['0TEST']
20 def run_unit_tests(local_feed
):
21 print "Running self-tests..."
22 exitstatus
= subprocess
.call([test_command
, '--', local_feed
])
24 print "SKIPPED unit tests for %s (no 'self-test' attribute set)" % local_feed
27 raise SafeException("Self-test failed with exit status %d" % exitstatus
)
29 def get_archive_url(options
, status
, archive
):
30 archive_dir_public_url
= options
.archive_dir_public_url
.replace('$RELEASE_VERSION', status
.release_version
)
31 if not archive_dir_public_url
.endswith('/'):
32 archive_dir_public_url
+= '/'
33 return archive_dir_public_url
+ archive
35 def upload_archives(options
, status
, uploads
):
36 # For each binary or source archive in uploads, ensure it is available
37 # from options.archive_dir_public_url
39 # We try to do all the uploads together first, and then verify them all
40 # afterwards. This is because we may have to wait for them to be moved
41 # from an incoming queue before we can test them.
44 return get_archive_url(options
, status
, archive
)
46 # Check that url exists and has the given size
47 def is_uploaded(url
, size
):
48 if url
.startswith('http://TESTING/releases'):
51 print "Testing URL %s..." % url
53 actual_size
= int(support
.get_size(url
))
55 print "Can't get size of '%s': %s" % (url
, ex
)
58 if actual_size
== size
:
60 print "WARNING: %s exists, but size is %d, not %d!" % (url
, actual_size
, size
)
63 # status.verified_uploads is an array of status flags:
65 'N': 'Upload required',
66 'A': 'Upload has been attempted, but we need to check whether it worked',
67 'V': 'Upload has been checked (exists and has correct size)',
70 if status
.verified_uploads
is None:
71 # First time around; no point checking for existing uploads
72 status
.verified_uploads
= 'N' * len(uploads
)
76 print "\nUpload status:"
77 for i
, stat
in enumerate(status
.verified_uploads
):
78 print "- %s : %s" % (uploads
[i
], description
[stat
])
82 if status
.verified_uploads
== 'V' * len(uploads
):
85 # Find all New archives
87 for i
, stat
in enumerate(status
.verified_uploads
):
90 to_upload
.append(uploads
[i
])
91 print "Upload %s/%s as %s" % (status
.release_version
, uploads
[i
], url(uploads
[i
]))
93 cmd
= options
.archive_upload_command
.strip()
96 # Mark all New items as Attempted
97 status
.verified_uploads
= status
.verified_uploads
.replace('N', 'A')
102 support
.show_and_run(cmd
, to_upload
)
104 if len(to_upload
) == 1:
105 print "No upload command is set => please upload the archive manually now"
106 raw_input('Press Return once the archive is uploaded.')
108 print "No upload command is set => please upload the archives manually now"
109 raw_input('Press Return once the %d archives are uploaded.' % len(to_upload
))
111 # Verify all Attempted uploads
113 for i
, stat
in enumerate(status
.verified_uploads
):
114 assert stat
in 'AV', status
.verified_uploads
116 if not is_uploaded(url(uploads
[i
]), os
.path
.getsize(uploads
[i
])):
117 print "** Archive '%s' still not uploaded! Try again..." % uploads
[i
]
123 status
.verified_uploads
= new_stat
126 if 'N' in new_stat
and cmd
:
127 raw_input('Press Return to try again.')
129 def do_release(local_iface
, options
):
130 assert options
.master_feed_file
131 options
.master_feed_file
= os
.path
.abspath(options
.master_feed_file
)
133 if not options
.archive_dir_public_url
:
134 raise SafeException("Downloads directory not set. Edit the 'make-release' script and try again.")
136 if not local_iface
.feed_for
:
137 raise SafeException("Feed %s missing a <feed-for> element" % local_iface
.uri
)
139 status
= support
.Status()
140 local_impl
= support
.get_singleton_impl(local_iface
)
142 local_impl_dir
= local_impl
.id
143 assert local_impl_dir
.startswith('/')
144 local_impl_dir
= os
.path
.realpath(local_impl_dir
)
145 assert os
.path
.isdir(local_impl_dir
)
146 assert local_iface
.uri
.startswith(local_impl_dir
+ '/')
148 # From the impl directory to the feed
149 # NOT relative to the archive root (in general)
150 local_iface_rel_path
= local_iface
.uri
[len(local_impl_dir
) + 1:]
151 assert not local_iface_rel_path
.startswith('/')
152 assert os
.path
.isfile(os
.path
.join(local_impl_dir
, local_iface_rel_path
))
155 for phase
in valid_phases
:
156 phase_actions
[phase
] = [] # List of <release:action> elements
158 add_toplevel_dir
= None
159 release_management
= local_iface
.get_metadata(XMLNS_RELEASE
, 'management')
160 if len(release_management
) == 1:
161 info("Found <release:management> element.")
162 release_management
= release_management
[0]
163 for x
in release_management
.childNodes
:
164 if x
.uri
== XMLNS_RELEASE
and x
.name
== 'action':
165 phase
= x
.getAttribute('phase')
166 if phase
not in valid_phases
:
167 raise SafeException("Invalid action phase '%s' in local feed %s. Valid actions are:\n%s" % (phase
, local_iface
.uri
, '\n'.join(valid_phases
)))
168 phase_actions
[phase
].append(x
.content
)
169 elif x
.uri
== XMLNS_RELEASE
and x
.name
== 'add-toplevel-directory':
170 add_toplevel_dir
= local_iface
.get_name()
172 warn("Unknown <release:management> element: %s", x
)
173 elif len(release_management
) > 1:
174 raise SafeException("Multiple <release:management> sections in %s!" % local_iface
)
176 info("No <release:management> element found in local feed.")
178 scm
= get_scm(local_iface
, options
)
180 # Path relative to the archive / SCM root
181 local_iface_rel_root_path
= local_iface
.uri
[len(scm
.root_dir
) + 1:]
183 def run_hooks(phase
, cwd
, env
):
184 info("Running hooks for phase '%s'" % phase
)
185 full_env
= os
.environ
.copy()
187 for x
in phase_actions
[phase
]:
188 print "[%s]: %s" % (phase
, x
)
189 support
.check_call(x
, shell
= True, cwd
= cwd
, env
= full_env
)
191 def set_to_release():
192 print "Snapshot version is " + local_impl
.get_version()
193 suggested
= support
.suggest_release_version(local_impl
.get_version())
194 release_version
= raw_input("Version number for new release [%s]: " % suggested
)
195 if not release_version
:
196 release_version
= suggested
198 scm
.ensure_no_tag(release_version
)
200 status
.head_before_release
= scm
.get_head_revision()
203 working_copy
= local_impl
.id
204 run_hooks('commit-release', cwd
= working_copy
, env
= {'RELEASE_VERSION': release_version
})
206 print "Releasing version", release_version
207 support
.publish(local_iface
.uri
, set_released
= 'today', set_version
= release_version
)
209 support
.backup_if_exists(release_version
)
210 os
.mkdir(release_version
)
211 os
.chdir(release_version
)
213 status
.old_snapshot_version
= local_impl
.get_version()
214 status
.release_version
= release_version
215 status
.head_at_release
= scm
.commit('Release %s' % release_version
, branch
= TMP_BRANCH_NAME
, parent
= 'HEAD')
218 def set_to_snapshot(snapshot_version
):
219 assert snapshot_version
.endswith('-post')
220 support
.publish(local_iface
.uri
, set_released
= '', set_version
= snapshot_version
)
221 scm
.commit('Start development series %s' % snapshot_version
, branch
= TMP_BRANCH_NAME
, parent
= TMP_BRANCH_NAME
)
222 status
.new_snapshot_version
= scm
.get_head_revision()
225 def ensure_ready_to_release():
226 if not options
.master_feed_file
:
227 raise SafeException("Master feed file not set! Check your configuration")
229 scm
.ensure_committed()
230 scm
.ensure_versioned(os
.path
.abspath(local_iface
.uri
))
231 info("No uncommitted changes. Good.")
232 # Not needed for GIT. For SCMs where tagging is expensive (e.g. svn) this might be useful.
233 #run_unit_tests(local_impl)
235 scm
.grep('\(^\\|[^=]\)\<\\(TODO\\|XXX\\|FIXME\\)\>')
237 def create_feed(target_feed
, local_iface_path
, archive_file
, archive_name
, main
):
238 shutil
.copyfile(local_iface_path
, target_feed
)
240 support
.publish(target_feed
,
242 archive_url
= get_archive_url(options
, status
, os
.path
.basename(archive_file
)),
243 archive_file
= archive_file
,
244 archive_extract
= archive_name
)
246 def get_previous_release(this_version
):
247 """Return the highest numbered verison in the master feed before this_version.
248 @return: version, or None if there wasn't one"""
249 parsed_release_version
= model
.parse_version(this_version
)
251 if os
.path
.exists(options
.master_feed_file
):
252 master
= model
.Interface(os
.path
.realpath(options
.master_feed_file
))
253 reader
.update(master
, master
.uri
, local
= True)
254 versions
= [impl
.version
for impl
in master
.implementations
.values() if impl
.version
< parsed_release_version
]
256 return model
.format_version(max(versions
))
259 def export_changelog(previous_release
):
260 changelog
= file('changelog-%s' % status
.release_version
, 'w')
263 scm
.export_changelog(previous_release
, status
.head_before_release
, changelog
)
264 except SafeException
, ex
:
265 print "WARNING: Failed to generate changelog: " + str(ex
)
267 print "Wrote changelog from %s to here as %s" % (previous_release
or 'start', changelog
.name
)
271 def fail_candidate(archive_file
):
273 assert cwd
.endswith(status
.release_version
)
274 support
.backup_if_exists(cwd
)
275 scm
.delete_branch(TMP_BRANCH_NAME
)
276 os
.unlink(support
.release_status_file
)
277 print "Restored to state before starting release. Make your fixes and try again..."
279 def accept_and_publish(archive_file
, archive_name
, src_feed_name
):
280 assert options
.master_feed_file
282 if not options
.archive_dir_public_url
:
283 raise SafeException("Archive directory public URL is not set! Edit configuration and try again.")
286 print "Already tagged in SCM. Not re-tagging."
288 scm
.ensure_committed()
289 head
= scm
.get_head_revision()
290 if head
!= status
.head_before_release
:
291 raise SafeException("Changes committed since we started!\n" +
292 "HEAD was " + status
.head_before_release
+ "\n"
295 scm
.tag(status
.release_version
, status
.head_at_release
)
296 scm
.reset_hard(TMP_BRANCH_NAME
)
297 scm
.delete_branch(TMP_BRANCH_NAME
)
299 status
.tagged
= 'true'
302 if status
.updated_master_feed
:
303 print "Already added to master feed. Not changing."
306 if os
.path
.exists(options
.master_feed_file
):
307 # Check we haven't already released this version
308 master
= model
.Interface(os
.path
.realpath(options
.master_feed_file
))
309 reader
.update(master
, master
.uri
, local
= True)
310 existing_releases
= [impl
for impl
in master
.implementations
.values() if impl
.get_version() == status
.release_version
]
311 if len(existing_releases
):
312 raise SafeException("Master feed %s already contains an implementation with version number %s!" % (options
.master_feed_file
, status
.release_version
))
314 previous_release
= get_previous_release(status
.release_version
)
315 previous_testing_releases
= [impl
for impl
in master
.implementations
.values() if impl
.get_version() == previous_release
316 and impl
.upstream_stability
== model
.stability_levels
["testing"]]
317 if previous_testing_releases
:
318 print "The previous release, version %s, is still marked as 'testing'. Set to stable?" % previous_release
319 if support
.get_choice(['Yes', 'No']) == 'Yes':
320 publish_opts
['select_version'] = previous_release
321 publish_opts
['set_stability'] = "stable"
323 # Merge the source and binary feeds together first, so
324 # that we update the master feed atomically and only
325 # have to sign it once.
326 shutil
.copyfile(src_feed_name
, 'merged.xml')
327 for b
in compiler
.get_binary_feeds():
328 support
.publish('merged.xml', local
= b
)
330 support
.publish(options
.master_feed_file
, local
= 'merged.xml', xmlsign
= True, key
= options
.key
, **publish_opts
)
331 os
.unlink('merged.xml')
333 status
.updated_master_feed
= 'true'
337 uploads
= [os
.path
.basename(archive_file
)]
338 for b
in compiler
.get_binary_feeds():
340 binary_feed
= model
.ZeroInstallFeed(qdom
.parse(stream
), local_path
= b
)
342 impl
, = binary_feed
.implementations
.values()
343 uploads
.append(os
.path
.basename(impl
.download_sources
[0].url
))
345 upload_archives(options
, status
, uploads
)
347 assert len(local_iface
.feed_for
) == 1
348 feed_base
= os
.path
.dirname(local_iface
.feed_for
.keys()[0])
349 feed_files
= [options
.master_feed_file
]
350 print "Upload %s into %s" % (', '.join(feed_files
), feed_base
)
351 cmd
= options
.master_feed_upload_command
.strip()
353 support
.show_and_run(cmd
, feed_files
)
355 print "NOTE: No feed upload command set => you'll have to upload them yourself!"
357 print "Push changes to public SCM repository..."
358 public_repos
= options
.public_scm_repository
360 scm
.push_head_and_release(status
.release_version
)
362 print "NOTE: No public repository set => you'll have to push the tag and trunk yourself."
364 os
.unlink(support
.release_status_file
)
366 if status
.head_before_release
:
367 head
= scm
.get_head_revision()
368 if status
.release_version
:
369 print "RESUMING release of %s %s" % (local_iface
.get_name(), status
.release_version
)
370 elif head
== status
.head_before_release
:
371 print "Restarting release of %s (HEAD revision has not changed)" % local_iface
.get_name()
373 raise SafeException("Something went wrong with the last run:\n" +
374 "HEAD revision for last run was " + status
.head_before_release
+ "\n" +
375 "HEAD revision now is " + head
+ "\n" +
376 "You should revert your working copy to the previous head and try again.\n" +
377 "If you're sure you want to release from the current head, delete '" + support
.release_status_file
+ "'")
379 print "Releasing", local_iface
.get_name()
381 ensure_ready_to_release()
383 if status
.release_version
:
384 if not os
.path
.isdir(status
.release_version
):
385 raise SafeException("Can't resume; directory %s missing. Try deleting '%s'." % (status
.release_version
, support
.release_status_file
))
386 os
.chdir(status
.release_version
)
387 need_set_snapshot
= False
389 print "Already tagged. Resuming the publishing process..."
390 elif status
.new_snapshot_version
:
391 head
= scm
.get_head_revision()
392 if head
!= status
.head_before_release
:
393 raise SafeException("There are more commits since we started!\n"
394 "HEAD was " + status
.head_before_release
+ "\n"
395 "HEAD now " + head
+ "\n"
396 "To include them, delete '" + support
.release_status_file
+ "' and try again.\n"
397 "To leave them out, put them on a new branch and reset HEAD to the release version.")
399 raise SafeException("Something went wrong previously when setting the new snapshot version.\n" +
400 "Suggest you reset to the original HEAD of\n%s and delete '%s'." % (status
.head_before_release
, support
.release_status_file
))
402 set_to_release() # Changes directory
403 assert status
.release_version
404 need_set_snapshot
= True
406 # May be needed by the upload command
407 os
.environ
['RELEASE_VERSION'] = status
.release_version
409 archive_name
= support
.make_archive_name(local_iface
.get_name(), status
.release_version
)
410 archive_file
= archive_name
+ '.tar.bz2'
412 export_prefix
= archive_name
413 if add_toplevel_dir
is not None:
414 export_prefix
+= '/' + add_toplevel_dir
416 if status
.created_archive
and os
.path
.isfile(archive_file
):
417 print "Archive already created"
419 support
.backup_if_exists(archive_file
)
420 scm
.export(export_prefix
, archive_file
, status
.head_at_release
)
422 has_submodules
= scm
.has_submodules()
424 if phase_actions
['generate-archive'] or has_submodules
:
426 support
.unpack_tarball(archive_file
)
428 scm
.export_submodules(archive_name
)
429 run_hooks('generate-archive', cwd
= archive_name
, env
= {'RELEASE_VERSION': status
.release_version
})
430 info("Regenerating archive (may have been modified by generate-archive hooks...")
431 support
.check_call(['tar', 'cjf', archive_file
, archive_name
])
432 except SafeException
:
433 scm
.reset_hard(scm
.get_current_branch())
434 fail_candidate(archive_file
)
437 status
.created_archive
= 'true'
440 if need_set_snapshot
:
441 set_to_snapshot(status
.release_version
+ '-post')
442 # Revert back to the original revision, so that any fixes the user makes
443 # will get applied before the tag
444 scm
.reset_hard(scm
.get_current_branch())
446 #backup_if_exists(archive_name)
447 support
.unpack_tarball(archive_file
)
449 extracted_iface_path
= os
.path
.abspath(os
.path
.join(export_prefix
, local_iface_rel_root_path
))
450 assert os
.path
.isfile(extracted_iface_path
), "Local feed not in archive! Is it under version control?"
451 extracted_iface
= model
.Interface(extracted_iface_path
)
452 reader
.update(extracted_iface
, extracted_iface_path
, local
= True)
453 extracted_impl
= support
.get_singleton_impl(extracted_iface
)
455 if extracted_impl
.main
:
456 # Find main executable, relative to the archive root
457 abs_main
= os
.path
.join(os
.path
.dirname(extracted_iface_path
), extracted_impl
.id, extracted_impl
.main
)
458 main
= support
.relative_path(archive_name
+ '/', abs_main
)
459 if main
!= extracted_impl
.main
:
460 print "(adjusting main: '%s' for the feed inside the archive, '%s' externally)" % (extracted_impl
.main
, main
)
461 # XXX: this is going to fail if the feed uses the new <command> syntax
462 if not os
.path
.exists(abs_main
):
463 raise SafeException("Main executable '%s' not found after unpacking archive!" % abs_main
)
464 if main
== extracted_impl
.main
:
465 main
= None # Don't change the main attribute
470 if status
.src_tests_passed
:
471 print "Unit-tests already passed - not running again"
473 run_unit_tests(extracted_iface_path
)
474 status
.src_tests_passed
= True
476 except SafeException
:
477 print "(leaving extracted directory for examination)"
478 fail_candidate(archive_file
)
480 # Unpack it again in case the unit-tests changed anything
481 shutil
.rmtree(archive_name
)
482 support
.unpack_tarball(archive_file
)
484 # Generate feed for source
485 stream
= open(extracted_iface_path
)
486 src_feed_name
= '%s.xml' % archive_name
487 create_feed(src_feed_name
, extracted_iface_path
, archive_file
, archive_name
, main
)
488 print "Wrote source feed as %s" % src_feed_name
490 # If it's a source package, compile the binaries now...
491 compiler
= compile.Compiler(options
, os
.path
.abspath(src_feed_name
))
492 compiler
.build_binaries()
494 previous_release
= get_previous_release(status
.release_version
)
495 export_changelog(previous_release
)
498 raw_input('Already tagged. Press Return to resume publishing process...')
501 print "\nCandidate release archive:", archive_file
502 print "(extracted to %s for inspection)" % os
.path
.abspath(archive_name
)
504 print "\nPlease check candidate and select an action:"
505 print "P) Publish candidate (accept)"
506 print "F) Fail candidate (untag)"
508 print "D) Diff against release archive for %s" % previous_release
509 maybe_diff
= ['Diff']
512 print "(you can also hit CTRL-C and resume this script when done)"
515 choice
= support
.get_choice(['Publish', 'Fail'] + maybe_diff
)
517 previous_archive_name
= support
.make_archive_name(local_iface
.get_name(), previous_release
)
518 previous_archive_file
= '../%s/%s.tar.bz2' % (previous_release
, previous_archive_name
)
520 # For archives created by older versions of 0release
521 if not os
.path
.isfile(previous_archive_file
):
522 old_previous_archive_file
= '../%s.tar.bz2' % previous_archive_name
523 if os
.path
.isfile(old_previous_archive_file
):
524 previous_archive_file
= old_previous_archive_file
526 if os
.path
.isfile(previous_archive_file
):
527 support
.unpack_tarball(previous_archive_file
)
529 support
.show_diff(previous_archive_name
, archive_name
)
531 shutil
.rmtree(previous_archive_name
)
534 print "Sorry, archive file %s not found! Can't show diff." % previous_archive_file
538 info("Deleting extracted archive %s", archive_name
)
539 shutil
.rmtree(archive_name
)
541 if choice
== 'Publish':
542 accept_and_publish(archive_file
, archive_name
, src_feed_name
)
544 assert choice
== 'Fail'
545 fail_candidate(archive_file
)