Only display selected dependencies in autocompile output
[0compile.git] / autocompile.py
blob650a0622e60b268c5586d0de9e06ab69b4598e85
1 # Copyright (C) 2009, Thomas Leonard
2 # See http://0install.net/0compile.html
4 import sys, os, __main__, tempfile, subprocess, signal, shutil
5 from xml.dom import minidom
6 from optparse import OptionParser
7 from logging import warn
9 from zeroinstall import SafeException
10 from zeroinstall.injector import arch, handler, driver, requirements, model, iface_cache, namespaces, writer, reader, qdom
11 from zeroinstall.injector.config import load_config
12 from zeroinstall.zerostore import manifest, NotStored
13 from zeroinstall.support import tasks, basedir, ro_rmtree
15 from support import BuildEnv, canonicalize_machine, XMLNS_0COMPILE
16 import support
18 build_target_machine_type = canonicalize_machine(support.uname[4])
19 assert build_target_machine_type in arch.machine_ranks, "Build target machine type '{build_target_machine_type}' is not supported on this platform; expected one of {types}".format(
20 build_target_machine_type = build_target_machine_type,
21 types = list(arch.machine_ranks.keys()))
23 # This is a bit hacky...
25 # We invent a new CPU type which is compatible with the host but worse than
26 # every existing type, and we use * for the OS type so that we don't beat 'Any'
27 # binaries either. This means that we always prefer an existing binary of the
28 # desired version to compiling a new one, but we'll compile a new version from source
29 # rather than use an older binary.
30 arch.machine_groups['newbuild'] = arch.machine_groups.get(build_target_machine_type, 0)
31 arch.machine_ranks['newbuild'] = max(arch.machine_ranks.values()) + 1
32 host_arch = '*-newbuild'
34 class ImplRestriction(model.Restriction):
35 reason = "Not the source we're trying to build"
37 def __init__(self, impl_id):
38 self.impl_id = impl_id
40 def meets_restriction(self, impl):
41 return impl.id == self.impl_id
43 def __str__(self):
44 return _("implementation {impl}").format(impl = self.impl_id)
46 class NewBuildImplementation(model.ZeroInstallImplementation):
47 # Assume that this (potential) binary is available so that we can select it as a
48 # dependency.
49 def is_available(self, stores):
50 return True
52 def get_commands(src_impl):
53 """Estimate the commands that the generated binary would have."""
54 cmd = src_impl.commands.get('compile', None)
55 if cmd is None:
56 warn("Source has no compile command! %s", src_impl)
57 return []
59 for elem in cmd.qdom.childNodes:
60 if elem.uri == XMLNS_0COMPILE and elem.name == 'implementation':
61 # Assume there's always a run command. Doesn't do any harm to have extra ones,
62 # and there are various ways this might get created.
63 commands = ['run']
64 for e in elem.childNodes:
65 if e.uri == namespaces.XMLNS_IFACE and e.name == 'command':
66 commands.append(e.getAttribute('name'))
67 return commands
68 return []
70 def add_binary_deps(src_impl, binary_impl):
71 # If src_impl contains a template, add those dependencies to the potential binary.
72 # Note: probably we should add "include-binary" dependencies here too...
74 compile_command = src_impl.commands['compile']
76 for elem in compile_command.qdom.childNodes:
77 if elem.uri == XMLNS_0COMPILE and elem.name == 'implementation':
78 template = elem
79 break
80 else:
81 return # No template
83 for elem in template.childNodes:
84 if elem.uri == namespaces.XMLNS_IFACE and elem.name in ('requires', 'restricts', 'runner'):
85 dep = model.process_depends(elem, local_feed_dir = None)
86 binary_impl.requires.append(dep)
88 class AutocompileCache(iface_cache.IfaceCache):
89 def __init__(self):
90 iface_cache.IfaceCache.__init__(self)
91 self.done = set()
93 def get_feed(self, url, force = False):
94 feed = iface_cache.IfaceCache.get_feed(self, url, force)
95 if not feed: return None
97 if feed not in self.done:
98 self.done.add(feed)
100 # For each source impl, add a corresponding binary
101 # (the binary has no dependencies as we can't predict them here,
102 # but they're not the same as the source's dependencies)
104 srcs = [x for x in feed.implementations.itervalues() if x.arch and x.arch.endswith('-src')]
105 for x in srcs:
106 new_id = '0compile=' + x.id
107 if not new_id in feed.implementations:
108 new = NewBuildImplementation(feed, new_id, None)
109 feed.implementations[new_id] = new
110 new.set_arch(host_arch)
111 new.version = x.version
113 # Give it some dummy commands in case we're using it as a <runner>, etc (otherwise it can't be selected)
114 for cmd_name in get_commands(x):
115 cmd = qdom.Element(namespaces.XMLNS_IFACE, 'command', {'path': 'new-build', 'name': cmd_name})
116 new.commands[cmd_name] = model.Command(cmd, None)
118 # Find the <command name='compile'/>
119 add_binary_deps(x, new)
121 return feed
123 class AutoCompiler:
124 # If (due to a bug) we get stuck in a loop, we use this to abort with a sensible error.
125 seen = None # ((iface, source_id) -> new_binary_id)
127 def __init__(self, config, iface_uri, options):
128 self.iface_uri = iface_uri
129 self.options = options
130 self.config = config
132 def pretty_print_plan(self, solver, root, indent = '- '):
133 """Display a tree showing the selected implementations."""
134 iface = self.config.iface_cache.get_interface(root)
135 impl = solver.selections[iface]
136 if impl is None:
137 msg = 'Failed to select any suitable version (source or binary)'
138 elif impl.id.startswith('0compile='):
139 real_impl_id = impl.id.split('=', 1)[1]
140 real_impl = impl.feed.implementations[real_impl_id]
141 msg = 'Compile %s (%s)' % (real_impl.get_version(), real_impl.id)
142 elif impl.arch and impl.arch.endswith('-src'):
143 msg = 'Compile %s (%s)' % (impl.get_version(), impl.id)
144 else:
145 if impl.arch:
146 msg = 'Use existing binary %s (%s)' % (impl.get_version(), impl.arch)
147 else:
148 msg = 'Use existing architecture-independent package %s' % impl.get_version()
149 self.note("%s%s: %s" % (indent, iface.get_name(), msg))
151 if impl:
152 indent = ' ' + indent
153 for x in solver.requires[iface]:
154 self.pretty_print_plan(solver, x.interface, indent)
156 def print_details(self, solver):
157 """Dump debugging details."""
158 self.note("\nFailed. Details of all components and versions considered:")
159 for iface in solver.details:
160 self.note('\n%s\n' % iface.get_name())
161 for impl, note in solver.details[iface]:
162 self.note('%s (%s) : %s' % (impl.get_version(), impl.arch or '*-*', note or 'OK'))
163 self.note("\nEnd details\n")
165 @tasks.async
166 def compile_and_register(self, sels, forced_iface_uri = None):
167 """If forced_iface_uri, register as an implementation of this interface,
168 ignoring the any <feed-for>, etc."""
170 buildenv = BuildEnv(need_config = False)
171 buildenv.config.set('compile', 'interface', sels.interface)
172 buildenv.config.set('compile', 'selections', 'selections.xml')
174 # Download any required packages now, so we can use the GUI to request confirmation, etc
175 download_missing = sels.download_missing(self.config, include_packages = True)
176 if download_missing:
177 yield download_missing
178 tasks.check(download_missing)
180 tmpdir = tempfile.mkdtemp(prefix = '0compile-')
181 try:
182 os.chdir(tmpdir)
184 # Write configuration for build...
186 buildenv.save()
188 sel_file = open('selections.xml', 'w')
189 try:
190 doc = sels.toDOM()
191 doc.writexml(sel_file)
192 sel_file.write('\n')
193 finally:
194 sel_file.close()
196 # Do the build...
198 build = self.spawn_build(buildenv.iface_name)
199 if build:
200 yield build
201 tasks.check(build)
203 # Register the result...
204 dom = minidom.parse(buildenv.local_iface_file)
206 feed_for_elem, = dom.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'feed-for')
207 claimed_iface = feed_for_elem.getAttribute('interface')
209 if forced_iface_uri is not None:
210 if forced_iface_uri != claimed_iface:
211 self.note("WARNING: registering as feed for {forced}, though feed claims to be for {claimed}".format(
212 forced = forced_iface_uri,
213 claimed = claimed_iface))
214 else:
215 forced_iface_uri = claimed_iface # (the top-level interface being built)
217 version = sels.selections[sels.interface].version
219 site_package_versions_dir = basedir.save_data_path('0install.net', 'site-packages',
220 *model.escape_interface_uri(forced_iface_uri))
221 leaf = '%s-%s' % (version, build_target_machine_type)
222 site_package_dir = os.path.join(site_package_versions_dir, leaf)
223 self.note("Storing build in %s" % site_package_dir)
225 # 1. Copy new version in under a temporary name. Names starting with '.' are ignored by 0install.
226 tmp_distdir = os.path.join(site_package_versions_dir, '.new-' + leaf)
227 shutil.copytree(buildenv.distdir, tmp_distdir, symlinks = True)
229 # 2. Rename the previous build to .old-VERSION (deleting that if it already existed)
230 if os.path.exists(site_package_dir):
231 self.note("(moving previous build out of the way)")
232 previous_build_dir = os.path.join(site_package_versions_dir, '.old-' + leaf)
233 if os.path.exists(previous_build_dir):
234 shutil.rmtree(previous_build_dir)
235 os.rename(site_package_dir, previous_build_dir)
236 else:
237 previous_build_dir = None
239 # 3. Rename the new version immediately after renaming away the old one to minimise time when there's
240 # no version.
241 os.rename(tmp_distdir, site_package_dir)
243 # 4. Delete the old version.
244 if previous_build_dir:
245 self.note("(deleting previous build)")
246 shutil.rmtree(previous_build_dir)
248 local_feed = os.path.join(site_package_dir, '0install', 'feed.xml')
249 assert os.path.exists(local_feed), "Feed %s not found!" % local_feed
251 # Reload - our 0install will detect the new feed automatically
252 iface = self.config.iface_cache.get_interface(forced_iface_uri)
253 reader.update_from_cache(iface, iface_cache = self.config.iface_cache)
254 self.config.iface_cache.get_feed(local_feed, force = True)
256 # Write it out - 0install will add the feed so that older 0install versions can find it
257 writer.save_interface(iface)
259 seen_key = (forced_iface_uri, sels.selections[sels.interface].id)
260 assert seen_key not in self.seen, seen_key
261 self.seen[seen_key] = site_package_dir
262 except:
263 self.note("\nBuild failed: leaving build directory %s for inspection...\n" % tmpdir)
264 raise
265 else:
266 # Can't delete current directory on Windows, so move to parent first
267 os.chdir(os.path.join(tmpdir, os.path.pardir))
269 ro_rmtree(tmpdir)
271 @tasks.async
272 def recursive_build(self, iface_uri, source_impl_id = None):
273 """Build an implementation of iface_uri and register it as a feed.
274 @param source_impl_id: the version to build, or None to build any version
275 @type source_impl_id: str
277 r = requirements.Requirements(iface_uri)
278 r.source = True
279 r.command = 'compile'
281 d = driver.Driver(self.config, r)
282 iface = self.config.iface_cache.get_interface(iface_uri)
283 d.solver.record_details = True
284 if source_impl_id is not None:
285 d.solver.extra_restrictions[iface] = [ImplRestriction(source_impl_id)]
287 # For testing...
288 #p.target_arch = arch.Architecture(os_ranks = {'FreeBSD': 0, None: 1}, machine_ranks = {'i386': 0, None: 1, 'newbuild': 2})
290 while True:
291 self.heading(iface_uri)
292 self.note("\nSelecting versions for %s..." % iface.get_name())
293 solved = d.solve_with_downloads()
294 if solved:
295 yield solved
296 tasks.check(solved)
298 if not d.solver.ready:
299 self.print_details(d.solver)
300 raise d.solver.get_failure_reason()
301 self.note("Selection done.")
303 self.note("\nPlan:\n")
304 self.pretty_print_plan(d.solver, r.interface_uri)
305 self.note('')
307 needed = []
308 for dep_iface_uri, dep_sel in d.solver.selections.selections.iteritems():
309 if dep_sel.id.startswith('0compile='):
310 if not needed:
311 self.note("Build dependencies that need to be compiled first:\n")
312 self.note("- {iface} {version}".format(iface = dep_iface_uri, version = dep_sel.version))
313 needed.append((dep_iface_uri, dep_sel))
315 if not needed:
316 self.note("No dependencies need compiling... compile %s itself..." % iface.get_name())
317 build = self.compile_and_register(d.solver.selections,
318 # force the interface in the recursive case
319 iface_uri if iface_uri != self.iface_uri else None)
320 yield build
321 tasks.check(build)
322 return
324 # Compile the first missing build dependency...
325 dep_iface_uri, dep_sel = needed[0]
327 self.note("")
329 #details = d.solver.details[self.config.iface_cache.get_interface(dep_iface.uri)]
330 #for de in details:
331 # print de
333 dep_source_id = dep_sel.id.split('=', 1)[1]
334 seen_key = (dep_iface_uri, dep_source_id)
335 if seen_key in self.seen:
336 self.note_error("BUG: Stuck in an auto-compile loop: already built {key}!".format(key = seen_key))
337 # Try to find out why the previous build couldn't be used...
338 dep_iface = self.config.iface_cache.get_interface(dep_iface_uri)
339 previous_build = self.seen[seen_key]
340 previous_build_feed = os.path.join(previous_build, '0install', 'feed.xml')
341 previous_feed = self.config.iface_cache.get_feed(previous_build_feed)
342 previous_binary_impl = previous_feed.implementations.values()[0]
343 raise SafeException("BUG: auto-compile loop: expected to select previously-build binary {binary}:\n\n{reason}".format(
344 binary = previous_binary_impl,
345 reason = d.solver.justify_decision(r, dep_iface, previous_binary_impl)))
347 build = self.recursive_build(dep_iface_uri, dep_source_id)
348 yield build
349 tasks.check(build)
351 assert seen_key in self.seen, (seen_key, self.seen) # Must have been built by now
353 # Try again with that dependency built...
355 def spawn_build(self, iface_name):
356 try:
357 subprocess.check_call([sys.executable, sys.argv[0], 'build'])
358 except subprocess.CalledProcessError as ex:
359 raise SafeException(str(ex))
361 def build(self):
362 self.seen = {}
363 tasks.wait_for_blocker(self.recursive_build(self.iface_uri))
365 def heading(self, msg):
366 self.note((' %s ' % msg).center(76, '='))
368 def note(self, msg):
369 print msg
371 def note_error(self, msg):
372 print msg
374 class GUIHandler(handler.Handler):
375 def downloads_changed(self):
376 self.compiler.downloads_changed()
378 def confirm_import_feed(self, pending, valid_sigs):
379 return handler.Handler.confirm_import_feed(self, pending, valid_sigs)
381 @tasks.async
382 def confirm_install(self, message):
383 from zeroinstall.injector.download import DownloadAborted
384 from zeroinstall.gtkui import gtkutils
385 import gtk
386 box = gtk.MessageDialog(self.compiler.dialog,
387 gtk.DIALOG_DESTROY_WITH_PARENT,
388 gtk.MESSAGE_QUESTION, gtk.BUTTONS_CANCEL,
389 message)
390 box.set_position(gtk.WIN_POS_CENTER)
392 install = gtkutils.MixedButton('Install', gtk.STOCK_OK)
393 install.set_flags(gtk.CAN_DEFAULT)
394 box.add_action_widget(install, gtk.RESPONSE_OK)
395 install.show_all()
396 box.set_default_response(gtk.RESPONSE_OK)
397 box.show()
399 response = gtkutils.DialogResponse(box)
400 yield response
401 box.destroy()
403 if response.response != gtk.RESPONSE_OK:
404 raise DownloadAborted()
406 class GTKAutoCompiler(AutoCompiler):
407 def __init__(self, config, iface_uri, options):
408 config.handler.compiler = self
410 AutoCompiler.__init__(self, config, iface_uri, options)
411 self.child = None
413 import pygtk; pygtk.require('2.0')
414 import gtk
416 w = gtk.Dialog('Autocompile %s' % iface_uri, None, 0,
417 (gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,
418 gtk.STOCK_OK, gtk.RESPONSE_OK))
419 self.dialog = w
421 w.set_default_size(int(gtk.gdk.screen_width() * 0.8),
422 int(gtk.gdk.screen_height() * 0.8))
424 vpaned = gtk.VPaned()
425 w.vbox.add(vpaned)
426 w.set_response_sensitive(gtk.RESPONSE_OK, False)
428 class AutoScroller:
429 def __init__(self):
430 tv = gtk.TextView()
431 tv.set_property('left-margin', 8)
432 tv.set_wrap_mode(gtk.WRAP_WORD_CHAR)
433 tv.set_editable(False)
434 swin = gtk.ScrolledWindow()
435 swin.add(tv)
436 swin.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC)
437 buffer = tv.get_buffer()
439 heading = buffer.create_tag('heading')
440 heading.set_property('scale', 1.5)
442 error = buffer.create_tag('error')
443 error.set_property('background', 'white')
444 error.set_property('foreground', 'red')
446 self.tv = tv
447 self.widget = swin
448 self.buffer = buffer
450 def insert_at_end_and_scroll(self, data, *tags):
451 vscroll = self.widget.get_vadjustment()
452 if not vscroll:
453 # Widget has been destroyed
454 print data,
455 return
456 near_end = vscroll.upper - vscroll.page_size * 1.5 < vscroll.value
457 end = self.buffer.get_end_iter()
458 self.buffer.insert_with_tags_by_name(end, data, *tags)
459 if near_end:
460 cursor = self.buffer.get_insert()
461 self.buffer.move_mark(cursor, end)
462 self.tv.scroll_to_mark(cursor, 0, False, 0, 0)
464 def set_text(self, text):
465 self.buffer.set_text(text)
467 self.overall = AutoScroller()
468 self.details = AutoScroller()
470 vpaned.pack1(self.overall.widget, True, False)
471 vpaned.pack2(self.details.widget, True, False)
473 self.closed = tasks.Blocker('Window closed')
475 w.show_all()
476 w.connect('destroy', lambda wd: self.closed.trigger())
478 def response(wd, resp):
479 if self.child is not None:
480 self.note_error('Sending TERM signal to build process group %d...' % self.child.pid)
481 os.kill(-self.child.pid, signal.SIGTERM)
482 else:
483 self.closed.trigger()
484 w.connect('response', response)
486 def downloads_changed(self):
487 if self.config.handler.monitored_downloads:
488 msg = 'Downloads in progress:\n'
489 for x in self.config.handler.monitored_downloads:
490 msg += '- {url}\n'.format(url = x.url)
491 else:
492 msg = ''
493 self.details.set_text(msg)
495 def heading(self, msg):
496 self.overall.insert_at_end_and_scroll(msg + '\n', 'heading')
498 def note(self, msg):
499 self.overall.insert_at_end_and_scroll(msg + '\n')
501 def note_error(self, msg):
502 self.overall.insert_at_end_and_scroll(msg + '\n', 'error')
504 def build(self):
505 self.seen = {}
506 import gtk
507 try:
508 tasks.wait_for_blocker(self.recursive_build(self.iface_uri))
509 except SafeException, ex:
510 self.note_error(str(ex))
511 else:
512 self.heading('All builds completed successfully!')
513 self.dialog.set_response_sensitive(gtk.RESPONSE_CANCEL, False)
514 self.dialog.set_response_sensitive(gtk.RESPONSE_OK, True)
516 tasks.wait_for_blocker(self.closed)
518 @tasks.async
519 def spawn_build(self, iface_name):
520 assert self.child is None
522 self.details.insert_at_end_and_scroll('Building %s\n' % iface_name, 'heading')
524 # Group all the child processes so we can kill them easily
525 def become_group_leader():
526 os.setpgid(0, 0)
527 devnull = os.open(os.devnull, os.O_RDONLY)
528 try:
529 self.child = subprocess.Popen([sys.executable, '-u', sys.argv[0], 'build'],
530 stdin = devnull,
531 stdout = subprocess.PIPE, stderr = subprocess.STDOUT,
532 preexec_fn = become_group_leader)
533 finally:
534 os.close(devnull)
536 import codecs
537 decoder = codecs.getincrementaldecoder('utf-8')(errors = 'replace')
539 while True:
540 yield tasks.InputBlocker(self.child.stdout, 'output from child')
541 got = os.read(self.child.stdout.fileno(), 100)
542 chars = decoder.decode(got, final = not got)
543 self.details.insert_at_end_and_scroll(chars)
544 if not got: break
546 self.child.wait()
547 code = self.child.returncode
548 self.child = None
549 if code:
550 self.details.insert_at_end_and_scroll('Build process exited with error status %d\n' % code, 'error')
551 raise SafeException('Build process exited with error status %d' % code)
552 self.details.insert_at_end_and_scroll('Build completed successfully\n', 'heading')
554 @tasks.async
555 def confirm_import_feed(self, pending, valid_sigs):
556 from zeroinstall.gtkui import trust_box
557 box = trust_box.TrustBox(pending, valid_sigs, parent = self.dialog)
558 box.show()
559 yield box.closed
561 def do_autocompile(args):
562 """autocompile [--gui] URI"""
564 parser = OptionParser(usage="usage: %prog autocompile [options]")
566 parser.add_option('', "--gui", help="graphical interface", action='store_true')
567 (options, args2) = parser.parse_args(args)
568 if len(args2) != 1:
569 raise __main__.UsageError()
571 if options.gui:
572 h = GUIHandler()
573 elif os.isatty(1):
574 h = handler.ConsoleHandler()
575 else:
576 h = handler.Handler()
577 config = load_config(handler = h)
578 config._iface_cache = AutocompileCache()
580 iface_uri = model.canonical_iface_uri(args2[0])
581 if options.gui:
582 compiler = GTKAutoCompiler(config, iface_uri, options)
583 else:
584 compiler = AutoCompiler(config, iface_uri, options)
586 compiler.build()
588 __main__.commands += [do_autocompile]