1 # Copyright (C) 2009, Thomas Leonard
2 # See http://0install.net/0compile.html
4 import sys
, os
, __main__
, tempfile
, subprocess
, signal
, shutil
5 from xml
.dom
import minidom
6 from optparse
import OptionParser
7 from logging
import warn
9 from zeroinstall
import SafeException
10 from zeroinstall
.injector
import arch
, handler
, driver
, requirements
, model
, iface_cache
, namespaces
, writer
, reader
, qdom
11 from zeroinstall
.injector
.config
import load_config
12 from zeroinstall
.zerostore
import manifest
, NotStored
13 from zeroinstall
.support
import tasks
, basedir
, ro_rmtree
15 from support
import BuildEnv
, canonicalize_machine
, XMLNS_0COMPILE
18 build_target_machine_type
= canonicalize_machine(support
.uname
[4])
19 assert build_target_machine_type
in arch
.machine_ranks
, "Build target machine type '{build_target_machine_type}' is not supported on this platform; expected one of {types}".format(
20 build_target_machine_type
= build_target_machine_type
,
21 types
= list(arch
.machine_ranks
.keys()))
23 # This is a bit hacky...
25 # We invent a new CPU type which is compatible with the host but worse than
26 # every existing type, and we use * for the OS type so that we don't beat 'Any'
27 # binaries either. This means that we always prefer an existing binary of the
28 # desired version to compiling a new one, but we'll compile a new version from source
29 # rather than use an older binary.
30 arch
.machine_groups
['newbuild'] = arch
.machine_groups
.get(build_target_machine_type
, 0)
31 arch
.machine_ranks
['newbuild'] = max(arch
.machine_ranks
.values()) + 1
32 host_arch
= '*-newbuild'
34 class ImplRestriction(model
.Restriction
):
35 reason
= "Not the source we're trying to build"
37 def __init__(self
, impl_id
):
38 self
.impl_id
= impl_id
40 def meets_restriction(self
, impl
):
41 return impl
.id == self
.impl_id
44 return _("implementation {impl}").format(impl
= self
.impl_id
)
46 class NewBuildImplementation(model
.ZeroInstallImplementation
):
47 # Assume that this (potential) binary is available so that we can select it as a
49 def is_available(self
, stores
):
52 def get_commands(src_impl
):
53 """Estimate the commands that the generated binary would have."""
54 cmd
= src_impl
.commands
.get('compile', None)
56 warn("Source has no compile command! %s", src_impl
)
59 for elem
in cmd
.qdom
.childNodes
:
60 if elem
.uri
== XMLNS_0COMPILE
and elem
.name
== 'implementation':
61 # Assume there's always a run command. Doesn't do any harm to have extra ones,
62 # and there are various ways this might get created.
64 for e
in elem
.childNodes
:
65 if e
.uri
== namespaces
.XMLNS_IFACE
and e
.name
== 'command':
66 commands
.append(e
.getAttribute('name'))
70 def add_binary_deps(src_impl
, binary_impl
):
71 # If src_impl contains a template, add those dependencies to the potential binary.
72 # Note: probably we should add "include-binary" dependencies here too...
74 compile_command
= src_impl
.commands
['compile']
76 for elem
in compile_command
.qdom
.childNodes
:
77 if elem
.uri
== XMLNS_0COMPILE
and elem
.name
== 'implementation':
83 for elem
in template
.childNodes
:
84 if elem
.uri
== namespaces
.XMLNS_IFACE
and elem
.name
in ('requires', 'restricts', 'runner'):
85 dep
= model
.process_depends(elem
, local_feed_dir
= None)
86 binary_impl
.requires
.append(dep
)
88 class AutocompileCache(iface_cache
.IfaceCache
):
90 iface_cache
.IfaceCache
.__init
__(self
)
93 def get_feed(self
, url
, force
= False):
94 feed
= iface_cache
.IfaceCache
.get_feed(self
, url
, force
)
95 if not feed
: return None
97 if feed
not in self
.done
:
100 # For each source impl, add a corresponding binary
101 # (the binary has no dependencies as we can't predict them here,
102 # but they're not the same as the source's dependencies)
104 srcs
= [x
for x
in feed
.implementations
.itervalues() if x
.arch
and x
.arch
.endswith('-src')]
106 new_id
= '0compile=' + x
.id
107 if not new_id
in feed
.implementations
:
108 new
= NewBuildImplementation(feed
, new_id
, None)
109 feed
.implementations
[new_id
] = new
110 new
.set_arch(host_arch
)
111 new
.version
= x
.version
113 # Give it some dummy commands in case we're using it as a <runner>, etc (otherwise it can't be selected)
114 for cmd_name
in get_commands(x
):
115 cmd
= qdom
.Element(namespaces
.XMLNS_IFACE
, 'command', {'path': 'new-build', 'name': cmd_name
})
116 new
.commands
[cmd_name
] = model
.Command(cmd
, None)
118 # Find the <command name='compile'/>
119 add_binary_deps(x
, new
)
124 # If (due to a bug) we get stuck in a loop, we use this to abort with a sensible error.
125 seen
= None # ((iface, source_id) -> new_binary_id)
127 def __init__(self
, config
, iface_uri
, options
):
128 self
.iface_uri
= iface_uri
129 self
.options
= options
132 def pretty_print_plan(self
, solver
, root
, indent
= '- '):
133 """Display a tree showing the selected implementations."""
134 iface
= self
.config
.iface_cache
.get_interface(root
)
135 impl
= solver
.selections
[iface
]
137 msg
= 'Failed to select any suitable version (source or binary)'
138 elif impl
.id.startswith('0compile='):
139 real_impl_id
= impl
.id.split('=', 1)[1]
140 real_impl
= impl
.feed
.implementations
[real_impl_id
]
141 msg
= 'Compile %s (%s)' % (real_impl
.get_version(), real_impl
.id)
142 elif impl
.arch
and impl
.arch
.endswith('-src'):
143 msg
= 'Compile %s (%s)' % (impl
.get_version(), impl
.id)
146 msg
= 'Use existing binary %s (%s)' % (impl
.get_version(), impl
.arch
)
148 msg
= 'Use existing architecture-independent package %s' % impl
.get_version()
149 self
.note("%s%s: %s" % (indent
, iface
.get_name(), msg
))
152 indent
= ' ' + indent
153 for x
in solver
.requires
[iface
]:
154 self
.pretty_print_plan(solver
, x
.interface
, indent
)
156 def print_details(self
, solver
):
157 """Dump debugging details."""
158 self
.note("\nFailed. Details of all components and versions considered:")
159 for iface
in solver
.details
:
160 self
.note('\n%s\n' % iface
.get_name())
161 for impl
, note
in solver
.details
[iface
]:
162 self
.note('%s (%s) : %s' % (impl
.get_version(), impl
.arch
or '*-*', note
or 'OK'))
163 self
.note("\nEnd details\n")
166 def compile_and_register(self
, sels
, forced_iface_uri
= None):
167 """If forced_iface_uri, register as an implementation of this interface,
168 ignoring the any <feed-for>, etc."""
170 buildenv
= BuildEnv(need_config
= False)
171 buildenv
.config
.set('compile', 'interface', sels
.interface
)
172 buildenv
.config
.set('compile', 'selections', 'selections.xml')
174 # Download any required packages now, so we can use the GUI to request confirmation, etc
175 download_missing
= sels
.download_missing(self
.config
, include_packages
= True)
177 yield download_missing
178 tasks
.check(download_missing
)
180 tmpdir
= tempfile
.mkdtemp(prefix
= '0compile-')
184 # Write configuration for build...
188 sel_file
= open('selections.xml', 'w')
191 doc
.writexml(sel_file
)
198 build
= self
.spawn_build(buildenv
.iface_name
)
203 # Register the result...
204 dom
= minidom
.parse(buildenv
.local_iface_file
)
206 feed_for_elem
, = dom
.getElementsByTagNameNS(namespaces
.XMLNS_IFACE
, 'feed-for')
207 claimed_iface
= feed_for_elem
.getAttribute('interface')
209 if forced_iface_uri
is not None:
210 if forced_iface_uri
!= claimed_iface
:
211 self
.note("WARNING: registering as feed for {forced}, though feed claims to be for {claimed}".format(
212 forced
= forced_iface_uri
,
213 claimed
= claimed_iface
))
215 forced_iface_uri
= claimed_iface
# (the top-level interface being built)
217 version
= sels
.selections
[sels
.interface
].version
219 site_package_versions_dir
= basedir
.save_data_path('0install.net', 'site-packages',
220 *model
.escape_interface_uri(forced_iface_uri
))
221 leaf
= '%s-%s' % (version
, build_target_machine_type
)
222 site_package_dir
= os
.path
.join(site_package_versions_dir
, leaf
)
223 self
.note("Storing build in %s" % site_package_dir
)
225 # 1. Copy new version in under a temporary name. Names starting with '.' are ignored by 0install.
226 tmp_distdir
= os
.path
.join(site_package_versions_dir
, '.new-' + leaf
)
227 shutil
.copytree(buildenv
.distdir
, tmp_distdir
, symlinks
= True)
229 # 2. Rename the previous build to .old-VERSION (deleting that if it already existed)
230 if os
.path
.exists(site_package_dir
):
231 self
.note("(moving previous build out of the way)")
232 previous_build_dir
= os
.path
.join(site_package_versions_dir
, '.old-' + leaf
)
233 if os
.path
.exists(previous_build_dir
):
234 shutil
.rmtree(previous_build_dir
)
235 os
.rename(site_package_dir
, previous_build_dir
)
237 previous_build_dir
= None
239 # 3. Rename the new version immediately after renaming away the old one to minimise time when there's
241 os
.rename(tmp_distdir
, site_package_dir
)
243 # 4. Delete the old version.
244 if previous_build_dir
:
245 self
.note("(deleting previous build)")
246 shutil
.rmtree(previous_build_dir
)
248 local_feed
= os
.path
.join(site_package_dir
, '0install', 'feed.xml')
249 assert os
.path
.exists(local_feed
), "Feed %s not found!" % local_feed
251 # Reload - our 0install will detect the new feed automatically
252 iface
= self
.config
.iface_cache
.get_interface(forced_iface_uri
)
253 reader
.update_from_cache(iface
, iface_cache
= self
.config
.iface_cache
)
254 self
.config
.iface_cache
.get_feed(local_feed
, force
= True)
256 # Write it out - 0install will add the feed so that older 0install versions can find it
257 writer
.save_interface(iface
)
259 seen_key
= (forced_iface_uri
, sels
.selections
[sels
.interface
].id)
260 assert seen_key
not in self
.seen
, seen_key
261 self
.seen
[seen_key
] = site_package_dir
263 self
.note("\nBuild failed: leaving build directory %s for inspection...\n" % tmpdir
)
266 # Can't delete current directory on Windows, so move to parent first
267 os
.chdir(os
.path
.join(tmpdir
, os
.path
.pardir
))
272 def recursive_build(self
, iface_uri
, source_impl_id
= None):
273 """Build an implementation of iface_uri and register it as a feed.
274 @param source_impl_id: the version to build, or None to build any version
275 @type source_impl_id: str
277 r
= requirements
.Requirements(iface_uri
)
279 r
.command
= 'compile'
281 d
= driver
.Driver(self
.config
, r
)
282 iface
= self
.config
.iface_cache
.get_interface(iface_uri
)
283 d
.solver
.record_details
= True
284 if source_impl_id
is not None:
285 d
.solver
.extra_restrictions
[iface
] = [ImplRestriction(source_impl_id
)]
288 #p.target_arch = arch.Architecture(os_ranks = {'FreeBSD': 0, None: 1}, machine_ranks = {'i386': 0, None: 1, 'newbuild': 2})
291 self
.heading(iface_uri
)
292 self
.note("\nSelecting versions for %s..." % iface
.get_name())
293 solved
= d
.solve_with_downloads()
298 if not d
.solver
.ready
:
299 self
.print_details(d
.solver
)
300 raise d
.solver
.get_failure_reason()
301 self
.note("Selection done.")
303 self
.note("\nPlan:\n")
304 self
.pretty_print_plan(d
.solver
, r
.interface_uri
)
308 for dep_iface_uri
, dep_sel
in d
.solver
.selections
.selections
.iteritems():
309 if dep_sel
.id.startswith('0compile='):
311 self
.note("Build dependencies that need to be compiled first:\n")
312 self
.note("- {iface} {version}".format(iface
= dep_iface_uri
, version
= dep_sel
.version
))
313 needed
.append((dep_iface_uri
, dep_sel
))
316 self
.note("No dependencies need compiling... compile %s itself..." % iface
.get_name())
317 build
= self
.compile_and_register(d
.solver
.selections
,
318 # force the interface in the recursive case
319 iface_uri
if iface_uri
!= self
.iface_uri
else None)
324 # Compile the first missing build dependency...
325 dep_iface_uri
, dep_sel
= needed
[0]
329 #details = d.solver.details[self.config.iface_cache.get_interface(dep_iface.uri)]
333 dep_source_id
= dep_sel
.id.split('=', 1)[1]
334 seen_key
= (dep_iface_uri
, dep_source_id
)
335 if seen_key
in self
.seen
:
336 self
.note_error("BUG: Stuck in an auto-compile loop: already built {key}!".format(key
= seen_key
))
337 # Try to find out why the previous build couldn't be used...
338 dep_iface
= self
.config
.iface_cache
.get_interface(dep_iface_uri
)
339 previous_build
= self
.seen
[seen_key
]
340 previous_build_feed
= os
.path
.join(previous_build
, '0install', 'feed.xml')
341 previous_feed
= self
.config
.iface_cache
.get_feed(previous_build_feed
)
342 previous_binary_impl
= previous_feed
.implementations
.values()[0]
343 raise SafeException("BUG: auto-compile loop: expected to select previously-build binary {binary}:\n\n{reason}".format(
344 binary
= previous_binary_impl
,
345 reason
= d
.solver
.justify_decision(r
, dep_iface
, previous_binary_impl
)))
347 build
= self
.recursive_build(dep_iface_uri
, dep_source_id
)
351 assert seen_key
in self
.seen
, (seen_key
, self
.seen
) # Must have been built by now
353 # Try again with that dependency built...
355 def spawn_build(self
, iface_name
):
357 subprocess
.check_call([sys
.executable
, sys
.argv
[0], 'build'])
358 except subprocess
.CalledProcessError
as ex
:
359 raise SafeException(str(ex
))
363 tasks
.wait_for_blocker(self
.recursive_build(self
.iface_uri
))
365 def heading(self
, msg
):
366 self
.note((' %s ' % msg
).center(76, '='))
371 def note_error(self
, msg
):
374 class GUIHandler(handler
.Handler
):
375 def downloads_changed(self
):
376 self
.compiler
.downloads_changed()
378 def confirm_import_feed(self
, pending
, valid_sigs
):
379 return handler
.Handler
.confirm_import_feed(self
, pending
, valid_sigs
)
382 def confirm_install(self
, message
):
383 from zeroinstall
.injector
.download
import DownloadAborted
384 from zeroinstall
.gtkui
import gtkutils
386 box
= gtk
.MessageDialog(self
.compiler
.dialog
,
387 gtk
.DIALOG_DESTROY_WITH_PARENT
,
388 gtk
.MESSAGE_QUESTION
, gtk
.BUTTONS_CANCEL
,
390 box
.set_position(gtk
.WIN_POS_CENTER
)
392 install
= gtkutils
.MixedButton('Install', gtk
.STOCK_OK
)
393 install
.set_flags(gtk
.CAN_DEFAULT
)
394 box
.add_action_widget(install
, gtk
.RESPONSE_OK
)
396 box
.set_default_response(gtk
.RESPONSE_OK
)
399 response
= gtkutils
.DialogResponse(box
)
403 if response
.response
!= gtk
.RESPONSE_OK
:
404 raise DownloadAborted()
406 class GTKAutoCompiler(AutoCompiler
):
407 def __init__(self
, config
, iface_uri
, options
):
408 config
.handler
.compiler
= self
410 AutoCompiler
.__init
__(self
, config
, iface_uri
, options
)
413 import pygtk
; pygtk
.require('2.0')
416 w
= gtk
.Dialog('Autocompile %s' % iface_uri
, None, 0,
417 (gtk
.STOCK_CANCEL
, gtk
.RESPONSE_CANCEL
,
418 gtk
.STOCK_OK
, gtk
.RESPONSE_OK
))
421 w
.set_default_size(int(gtk
.gdk
.screen_width() * 0.8),
422 int(gtk
.gdk
.screen_height() * 0.8))
424 vpaned
= gtk
.VPaned()
426 w
.set_response_sensitive(gtk
.RESPONSE_OK
, False)
431 tv
.set_property('left-margin', 8)
432 tv
.set_wrap_mode(gtk
.WRAP_WORD_CHAR
)
433 tv
.set_editable(False)
434 swin
= gtk
.ScrolledWindow()
436 swin
.set_policy(gtk
.POLICY_NEVER
, gtk
.POLICY_AUTOMATIC
)
437 buffer = tv
.get_buffer()
439 heading
= buffer.create_tag('heading')
440 heading
.set_property('scale', 1.5)
442 error
= buffer.create_tag('error')
443 error
.set_property('background', 'white')
444 error
.set_property('foreground', 'red')
450 def insert_at_end_and_scroll(self
, data
, *tags
):
451 vscroll
= self
.widget
.get_vadjustment()
453 # Widget has been destroyed
456 near_end
= vscroll
.upper
- vscroll
.page_size
* 1.5 < vscroll
.value
457 end
= self
.buffer.get_end_iter()
458 self
.buffer.insert_with_tags_by_name(end
, data
, *tags
)
460 cursor
= self
.buffer.get_insert()
461 self
.buffer.move_mark(cursor
, end
)
462 self
.tv
.scroll_to_mark(cursor
, 0, False, 0, 0)
464 def set_text(self
, text
):
465 self
.buffer.set_text(text
)
467 self
.overall
= AutoScroller()
468 self
.details
= AutoScroller()
470 vpaned
.pack1(self
.overall
.widget
, True, False)
471 vpaned
.pack2(self
.details
.widget
, True, False)
473 self
.closed
= tasks
.Blocker('Window closed')
476 w
.connect('destroy', lambda wd
: self
.closed
.trigger())
478 def response(wd
, resp
):
479 if self
.child
is not None:
480 self
.note_error('Sending TERM signal to build process group %d...' % self
.child
.pid
)
481 os
.kill(-self
.child
.pid
, signal
.SIGTERM
)
483 self
.closed
.trigger()
484 w
.connect('response', response
)
486 def downloads_changed(self
):
487 if self
.config
.handler
.monitored_downloads
:
488 msg
= 'Downloads in progress:\n'
489 for x
in self
.config
.handler
.monitored_downloads
:
490 msg
+= '- {url}\n'.format(url
= x
.url
)
493 self
.details
.set_text(msg
)
495 def heading(self
, msg
):
496 self
.overall
.insert_at_end_and_scroll(msg
+ '\n', 'heading')
499 self
.overall
.insert_at_end_and_scroll(msg
+ '\n')
501 def note_error(self
, msg
):
502 self
.overall
.insert_at_end_and_scroll(msg
+ '\n', 'error')
508 tasks
.wait_for_blocker(self
.recursive_build(self
.iface_uri
))
509 except SafeException
, ex
:
510 self
.note_error(str(ex
))
512 self
.heading('All builds completed successfully!')
513 self
.dialog
.set_response_sensitive(gtk
.RESPONSE_CANCEL
, False)
514 self
.dialog
.set_response_sensitive(gtk
.RESPONSE_OK
, True)
516 tasks
.wait_for_blocker(self
.closed
)
519 def spawn_build(self
, iface_name
):
520 assert self
.child
is None
522 self
.details
.insert_at_end_and_scroll('Building %s\n' % iface_name
, 'heading')
524 # Group all the child processes so we can kill them easily
525 def become_group_leader():
527 devnull
= os
.open(os
.devnull
, os
.O_RDONLY
)
529 self
.child
= subprocess
.Popen([sys
.executable
, '-u', sys
.argv
[0], 'build'],
531 stdout
= subprocess
.PIPE
, stderr
= subprocess
.STDOUT
,
532 preexec_fn
= become_group_leader
)
537 decoder
= codecs
.getincrementaldecoder('utf-8')(errors
= 'replace')
540 yield tasks
.InputBlocker(self
.child
.stdout
, 'output from child')
541 got
= os
.read(self
.child
.stdout
.fileno(), 100)
542 chars
= decoder
.decode(got
, final
= not got
)
543 self
.details
.insert_at_end_and_scroll(chars
)
547 code
= self
.child
.returncode
550 self
.details
.insert_at_end_and_scroll('Build process exited with error status %d\n' % code
, 'error')
551 raise SafeException('Build process exited with error status %d' % code
)
552 self
.details
.insert_at_end_and_scroll('Build completed successfully\n', 'heading')
555 def confirm_import_feed(self
, pending
, valid_sigs
):
556 from zeroinstall
.gtkui
import trust_box
557 box
= trust_box
.TrustBox(pending
, valid_sigs
, parent
= self
.dialog
)
561 def do_autocompile(args
):
562 """autocompile [--gui] URI"""
564 parser
= OptionParser(usage
="usage: %prog autocompile [options]")
566 parser
.add_option('', "--gui", help="graphical interface", action
='store_true')
567 (options
, args2
) = parser
.parse_args(args
)
569 raise __main__
.UsageError()
574 h
= handler
.ConsoleHandler()
576 h
= handler
.Handler()
577 config
= load_config(handler
= h
)
578 config
._iface
_cache
= AutocompileCache()
580 iface_uri
= model
.canonical_iface_uri(args2
[0])
582 compiler
= GTKAutoCompiler(config
, iface_uri
, options
)
584 compiler
= AutoCompiler(config
, iface_uri
, options
)
588 __main__
.commands
+= [do_autocompile
]