autocompile.py

   1 # Copyright (C) 2009, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import sys, os, __main__, tempfile, subprocess, signal
   5 from xml.dom import minidom
   6 from optparse import OptionParser
   7 from logging import warn
   8
   9 from zeroinstall import SafeException
  10 from zeroinstall.injector import arch, handler, driver, requirements, model, iface_cache, namespaces, writer, reader, qdom
  11 from zeroinstall.injector.config import load_config
  12 from zeroinstall.zerostore import manifest, NotStored
  13 from zeroinstall.support import tasks, basedir, ro_rmtree
  14
  15 from support import BuildEnv, uname, XMLNS_0COMPILE
  16
  17 # This is a bit hacky...
  18 #
  19 # We invent a new CPU type which is compatible with the host but worse than
  20 # every existing type, and we use * for the OS type so that we don't beat 'Any'
  21 # binaries either. This means that we always prefer an existing binary of the
  22 # desired version to compiling a new one, but we'll compile a new version from source
  23 # rather than use an older binary.
  24 arch.machine_groups['newbuild'] = arch.machine_groups.get(uname[4], 0)
  25 arch.machine_ranks['newbuild'] = max(arch.machine_ranks.values()) + 1
  26 host_arch = '*-newbuild'
  27
  28 class NewBuildImplementation(model.ZeroInstallImplementation):
  29         # Assume that this (potential) binary is available so that we can select it as a
  30         # dependency.
  31         def is_available(self, stores):
  32                 return True
  33
  34 def get_commands(src_impl):
  35         """Estimate the commands that the generated binary would have."""
  36         cmd = src_impl.commands.get('compile', None)
  37         if cmd is None:
  38                 warn("Source has no compile command! %s", src_impl)
  39                 return []
  40
  41         for elem in cmd.qdom.childNodes:
  42                 if elem.uri == XMLNS_0COMPILE and elem.name == 'implementation':
  43                         # Assume there's always a run command. Doesn't do any harm to have extra ones,
  44                         # and there are various ways this might get created.
  45                         commands = ['run']
  46                         for e in elem.childNodes:
  47                                 if e.uri == namespaces.XMLNS_IFACE and e.name == 'command':
  48                                         commands.append(e.getAttribute('name'))
  49                         return commands
  50         return []
  51
  52 class AutocompileCache(iface_cache.IfaceCache):
  53         def __init__(self):
  54                 iface_cache.IfaceCache.__init__(self)
  55                 self.done = set()
  56
  57         def get_feed(self, url, force = False):
  58                 feed = iface_cache.IfaceCache.get_feed(self, url, force)
  59                 if not feed: return None
  60
  61                 if feed not in self.done:
  62                         self.done.add(feed)
  63
  64                         # For each source impl, add a corresponding binary
  65                         # (the binary has no dependencies as we can't predict them here,
  66                         # but they're not the same as the source's dependencies)
  67
  68                         srcs = [x for x in feed.implementations.itervalues() if x.arch and x.arch.endswith('-src')]
  69                         for x in srcs:
  70                                 new_id = '0compile=' + x.id
  71                                 if not new_id in feed.implementations:
  72                                         new = NewBuildImplementation(feed, new_id, None)
  73                                         feed.implementations[new_id] = new
  74                                         new.set_arch(host_arch)
  75                                         new.version = x.version
  76
  77                                         # Give it some dummy commands in case we're using it as a <runner>, etc (otherwise it can't be selected)
  78                                         for cmd_name in get_commands(x):
  79                                                 cmd = qdom.Element(namespaces.XMLNS_IFACE, 'command', {'path': 'new-build', 'name': cmd_name})
  80                                                 new.commands[cmd_name] = model.Command(cmd, None)
  81
  82                 return feed
  83
  84 class AutoCompiler:
  85         def __init__(self, config, iface_uri, options):
  86                 self.iface_uri = iface_uri
  87                 self.options = options
  88                 self.config = config
  89
  90         def pretty_print_plan(self, solver, root, indent = '- '):
  91                 """Display a tree showing the selected implementations."""
  92                 iface = self.config.iface_cache.get_interface(root)
  93                 impl = solver.selections[iface]
  94                 if impl is None:
  95                         msg = 'Failed to select any suitable version (source or binary)'
  96                 elif impl.id.startswith('0compile='):
  97                         real_impl_id = impl.id.split('=', 1)[1]
  98                         real_impl = impl.feed.implementations[real_impl_id]
  99                         msg = 'Compile %s (%s)' % (real_impl.get_version(), real_impl.id)
 100                 elif impl.arch and impl.arch.endswith('-src'):
 101                         msg = 'Compile %s (%s)' % (impl.get_version(), impl.id)
 102                 else:
 103                         if impl.arch:
 104                                 msg = 'Use existing binary %s (%s)' % (impl.get_version(), impl.arch)
 105                         else:
 106                                 msg = 'Use existing architecture-independent package %s' % impl.get_version()
 107                 self.note("%s%s: %s" % (indent, iface.get_name(), msg))
 108
 109                 if impl:
 110                         indent = '  ' + indent
 111                         for x in impl.requires:
 112                                 self.pretty_print_plan(solver, x.interface, indent)
 113
 114         def print_details(self, solver):
 115                 """Dump debugging details."""
 116                 self.note("\nFailed. Details of all components and versions considered:")
 117                 for iface in solver.details:
 118                         self.note('\n%s\n' % iface.get_name())
 119                         for impl, note in solver.details[iface]:
 120                                 self.note('%s (%s) : %s' % (impl.get_version(), impl.arch or '*-*', note or 'OK'))
 121                 self.note("\nEnd details\n")
 122
 123         @tasks.async
 124         def compile_and_register(self, sels, forced_iface_uri = None):
 125                 """If forced_iface_uri, register as an implementation of this interface,
 126                 ignoring the any <feed-for>, etc."""
 127                 def valid_autocompile_feed(binary_feed):
 128                         cache = self.config.iface_cache
 129                         local_feed_impls = cache.get_feed(local_feed).implementations
 130                         if len(local_feed_impls) != 1:
 131                                 self.note("Invalid autocompile feed '%s'; expected exactly one implementation!" % binary_feed)
 132                                 return False
 133                         impl, = local_feed_impls.values()
 134                         try:
 135                                 cache.stores.lookup_any(impl.digests)
 136                                 return True
 137                         except NotStored, ex:
 138                                 self.note("Build metadata file '%s' exists but implementation is missing: %s" % (local_feed, ex))
 139                                 return False
 140
 141                 local_feed_dir = basedir.save_config_path('0install.net', '0compile', 'builds', model._pretty_escape(sels.interface))
 142
 143                 buildenv = BuildEnv(need_config = False)
 144                 buildenv.config.set('compile', 'interface', sels.interface)
 145                 buildenv.config.set('compile', 'selections', 'selections.xml')
 146
 147                 # Download any required packages now, so we can use the GUI to request confirmation, etc
 148                 download_missing = sels.download_missing(self.config, include_packages = True)
 149                 if download_missing:
 150                         yield download_missing
 151                         tasks.check(download_missing)
 152
 153                 version = sels.selections[sels.interface].version
 154                 local_feed = os.path.join(local_feed_dir, '%s-%s-%s.xml' % (buildenv.iface_name, version, uname[4]))
 155                 if os.path.exists(local_feed):
 156                         if not valid_autocompile_feed(local_feed):
 157                                 os.unlink(local_feed)
 158                         else:
 159                                 raise SafeException("Build metadata file '%s' already exists!" % local_feed)
 160
 161                 tmpdir = tempfile.mkdtemp(prefix = '0compile-')
 162                 try:
 163                         os.chdir(tmpdir)
 164
 165                         # Write configuration for build...
 166
 167                         buildenv.save()
 168
 169                         sel_file = open('selections.xml', 'w')
 170                         try:
 171                                 doc = sels.toDOM()
 172                                 doc.writexml(sel_file)
 173                                 sel_file.write('\n')
 174                         finally:
 175                                 sel_file.close()
 176
 177                         # Do the build...
 178
 179                         build = self.spawn_build(buildenv.iface_name)
 180                         if build:
 181                                 yield build
 182                                 tasks.check(build)
 183
 184                         # Register the result...
 185
 186                         alg = manifest.get_algorithm('sha1new')
 187                         digest = alg.new_digest()
 188                         lines = []
 189                         for line in alg.generate_manifest(buildenv.distdir):
 190                                 line += '\n'
 191                                 digest.update(line)
 192                                 lines.append(line)
 193                         actual_digest = alg.getID(digest)
 194
 195                         local_feed_file = file(local_feed, 'w')
 196                         try:
 197                                 dom = minidom.parse(buildenv.local_iface_file)
 198                                 impl, = dom.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'implementation')
 199                                 impl.setAttribute('id', actual_digest)
 200                                 dom.writexml(local_feed_file)
 201                                 local_feed_file.write('\n')
 202                         finally:
 203                                 local_feed_file.close()
 204
 205                         feed_for_elem, = dom.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'feed-for')
 206                         claimed_iface = feed_for_elem.getAttribute('interface')
 207
 208                         self.note("Implementation metadata written to %s" % local_feed)
 209
 210                         # No point adding it to the system store when only the user has the feed...
 211                         store = self.config.stores.stores[0]
 212                         self.note("Storing build in user cache %s..." % store.dir)
 213                         self.config.stores.add_dir_to_cache(actual_digest, buildenv.distdir)
 214
 215                         if forced_iface_uri is not None:
 216                                 if forced_iface_uri != claimed_iface:
 217                                         self.note("WARNING: registering as feed for {forced}, though feed claims to be for {claimed}".format(
 218                                                 forced = forced_iface_uri,
 219                                                 claimed = claimed_iface))
 220                         else:
 221                                 forced_iface_uri = claimed_iface                # (the top-level interface being built)
 222
 223                         iface = self.config.iface_cache.get_interface(forced_iface_uri)
 224                         self.note("Registering as feed for %s" % iface.uri)
 225                         feed = iface.get_feed(local_feed)
 226                         if feed:
 227                                 self.note("WARNING: feed %s already registered!" % local_feed)
 228                         else:
 229                                 iface.extra_feeds.append(model.Feed(local_feed, impl.getAttribute('arch'), user_override = True))
 230                         writer.save_interface(iface)
 231
 232                         # We might have cached an old version
 233                         new_feed = self.config.iface_cache.get_interface(local_feed)
 234                         reader.update_from_cache(new_feed)
 235                 except:
 236                         self.note("\nBuild failed: leaving build directory %s for inspection...\n" % tmpdir)
 237                         raise
 238                 else:
 239                         ro_rmtree(tmpdir)
 240
 241         @tasks.async
 242         def recursive_build(self, iface_uri, version = None):
 243                 """Build an implementation of iface_uri and register it as a feed.
 244                 @param version: the version to build, or None to build any version
 245                 @type version: str
 246                 """
 247                 r = requirements.Requirements(iface_uri)
 248                 r.source = True
 249                 r.command = 'compile'
 250
 251                 d = driver.Driver(self.config, r)
 252                 iface = self.config.iface_cache.get_interface(iface_uri)
 253                 d.solver.record_details = True
 254                 if version:
 255                         d.solver.extra_restrictions[iface] = [model.VersionRestriction(model.parse_version(version))]
 256
 257                 # For testing...
 258                 #p.target_arch = arch.Architecture(os_ranks = {'FreeBSD': 0, None: 1}, machine_ranks = {'i386': 0, None: 1, 'newbuild': 2})
 259
 260                 while True:
 261                         self.heading(iface_uri)
 262                         self.note("\nSelecting versions for %s..." % iface.get_name())
 263                         solved = d.solve_with_downloads()
 264                         if solved:
 265                                 yield solved
 266                                 tasks.check(solved)
 267
 268                         if not d.solver.ready:
 269                                 self.print_details(d.solver)
 270                                 raise SafeException("Can't find all required implementations (source or binary):\n" +
 271                                         '\n'.join(["- %s -> %s" % (iface, d.solver.selections[iface])
 272                                                    for iface in d.solver.selections]))
 273                         self.note("Selection done.")
 274
 275                         self.note("\nPlan:\n")
 276                         self.pretty_print_plan(d.solver, r.interface_uri)
 277                         self.note('')
 278
 279                         needed = []
 280                         for dep_iface, dep_impl in d.solver.selections.iteritems():
 281                                 if dep_impl.id.startswith('0compile='):
 282                                         if not needed:
 283                                                 self.note("Build dependencies that need to be compiled first:\n")
 284                                         self.note("- {iface} {version}".format(iface = dep_iface.uri, version = model.format_version(dep_impl.version)))
 285                                         needed.append((dep_iface, dep_impl))
 286
 287                         if not needed:
 288                                 self.note("No dependencies need compiling... compile %s itself..." % iface.get_name())
 289                                 build = self.compile_and_register(d.solver.selections,
 290                                                 # force the interface in the recursive case
 291                                                 iface_uri if iface_uri != self.iface_uri else None)
 292                                 yield build
 293                                 tasks.check(build)
 294                                 return
 295
 296                         # Compile the first missing build dependency...
 297                         dep_iface, dep_impl = needed[0]
 298
 299                         self.note("")
 300
 301                         #details = d.solver.details[self.config.iface_cache.get_interface(dep_iface.uri)]
 302                         #for de in details:
 303                         #       print de
 304
 305                         build = self.recursive_build(dep_iface.uri, dep_impl.get_version())
 306                         yield build
 307                         tasks.check(build)
 308                         # Try again with that dependency built...
 309
 310         def spawn_build(self, iface_name):
 311                 subprocess.check_call([sys.executable, sys.argv[0], 'build'])
 312
 313         def build(self):
 314                 tasks.wait_for_blocker(self.recursive_build(self.iface_uri))
 315
 316         def heading(self, msg):
 317                 self.note((' %s ' % msg).center(76, '='))
 318
 319         def note(self, msg):
 320                 print msg
 321
 322         def note_error(self, msg):
 323                 self.overall.insert_at_cursor(msg + '\n')
 324
 325 class GUIHandler(handler.Handler):
 326         def downloads_changed(self):
 327                 self.compiler.downloads_changed()
 328
 329         def confirm_import_feed(self, pending, valid_sigs):
 330                 return handler.Handler.confirm_import_feed(self, pending, valid_sigs)
 331
 332         @tasks.async
 333         def confirm_install(self, message):
 334                 from zeroinstall.injector.download import DownloadAborted
 335                 from zeroinstall.gtkui import gtkutils
 336                 import gtk
 337                 box = gtk.MessageDialog(self.compiler.dialog,
 338                                         gtk.DIALOG_DESTROY_WITH_PARENT,
 339                                         gtk.MESSAGE_QUESTION, gtk.BUTTONS_CANCEL,
 340                                         message)
 341                 box.set_position(gtk.WIN_POS_CENTER)
 342
 343                 install = gtkutils.MixedButton('Install', gtk.STOCK_OK)
 344                 install.set_flags(gtk.CAN_DEFAULT)
 345                 box.add_action_widget(install, gtk.RESPONSE_OK)
 346                 install.show_all()
 347                 box.set_default_response(gtk.RESPONSE_OK)
 348                 box.show()
 349
 350                 response = gtkutils.DialogResponse(box)
 351                 yield response
 352                 box.destroy()
 353
 354                 if response.response != gtk.RESPONSE_OK:
 355                         raise DownloadAborted()
 356
 357 class GTKAutoCompiler(AutoCompiler):
 358         def __init__(self, config, iface_uri, options):
 359                 config.handler.compiler = self
 360
 361                 AutoCompiler.__init__(self, config, iface_uri, options)
 362                 self.child = None
 363
 364                 import pygtk; pygtk.require('2.0')
 365                 import gtk
 366
 367                 w = gtk.Dialog('Autocompile %s' % iface_uri, None, 0,
 368                                                  (gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,
 369                                                   gtk.STOCK_OK, gtk.RESPONSE_OK))
 370                 self.dialog = w
 371
 372                 w.set_default_size(int(gtk.gdk.screen_width() * 0.8),
 373                                    int(gtk.gdk.screen_height() * 0.8))
 374
 375                 vpaned = gtk.VPaned()
 376                 w.vbox.add(vpaned)
 377                 w.set_response_sensitive(gtk.RESPONSE_OK, False)
 378
 379                 class AutoScroller:
 380                         def __init__(self):
 381                                 tv = gtk.TextView()
 382                                 tv.set_property('left-margin', 8)
 383                                 tv.set_wrap_mode(gtk.WRAP_WORD_CHAR)
 384                                 tv.set_editable(False)
 385                                 swin = gtk.ScrolledWindow()
 386                                 swin.add(tv)
 387                                 swin.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC)
 388                                 buffer = tv.get_buffer()
 389
 390                                 heading = buffer.create_tag('heading')
 391                                 heading.set_property('scale', 1.5)
 392
 393                                 error = buffer.create_tag('error')
 394                                 error.set_property('background', 'white')
 395                                 error.set_property('foreground', 'red')
 396
 397                                 self.tv = tv
 398                                 self.widget = swin
 399                                 self.buffer = buffer
 400
 401                         def insert_at_end_and_scroll(self, data, *tags):
 402                                 vscroll = self.widget.get_vadjustment()
 403                                 if not vscroll:
 404                                         # Widget has been destroyed
 405                                         print data,
 406                                         return
 407                                 near_end = vscroll.upper - vscroll.page_size * 1.5 < vscroll.value
 408                                 end = self.buffer.get_end_iter()
 409                                 self.buffer.insert_with_tags_by_name(end, data, *tags)
 410                                 if near_end:
 411                                         cursor = self.buffer.get_insert()
 412                                         self.buffer.move_mark(cursor, end)
 413                                         self.tv.scroll_to_mark(cursor, 0, False, 0, 0)
 414
 415                         def set_text(self, text):
 416                                 self.buffer.set_text(text)
 417
 418                 self.overall = AutoScroller()
 419                 self.details = AutoScroller()
 420
 421                 vpaned.pack1(self.overall.widget, True, False)
 422                 vpaned.pack2(self.details.widget, True, False)
 423
 424                 self.closed = tasks.Blocker('Window closed')
 425
 426                 w.show_all()
 427                 w.connect('destroy', lambda wd: self.closed.trigger())
 428
 429                 def response(wd, resp):
 430                         if self.child is not None:
 431                                 self.note_error('Sending TERM signal to build process group %d...' % self.child.pid)
 432                                 os.kill(-self.child.pid, signal.SIGTERM)
 433                         else:
 434                                 self.closed.trigger()
 435                 w.connect('response', response)
 436
 437         def downloads_changed(self):
 438                 if self.config.handler.monitored_downloads:
 439                         msg = 'Downloads in progress:\n'
 440                         for x in self.config.handler.monitored_downloads:
 441                                 msg += '- {url}\n'.format(url = x.url)
 442                 else:
 443                         msg = ''
 444                 self.details.set_text(msg)
 445
 446         def heading(self, msg):
 447                 self.overall.insert_at_end_and_scroll(msg + '\n', 'heading')
 448
 449         def note(self, msg):
 450                 self.overall.insert_at_end_and_scroll(msg + '\n')
 451
 452         def note_error(self, msg):
 453                 self.overall.insert_at_end_and_scroll(msg + '\n', 'error')
 454
 455         def build(self):
 456                 import gtk
 457                 try:
 458                         tasks.wait_for_blocker(self.recursive_build(self.iface_uri))
 459                 except SafeException, ex:
 460                         self.note_error(str(ex))
 461                 else:
 462                         self.heading('All builds completed successfully!')
 463                         self.dialog.set_response_sensitive(gtk.RESPONSE_CANCEL, False)
 464                         self.dialog.set_response_sensitive(gtk.RESPONSE_OK, True)
 465
 466                 tasks.wait_for_blocker(self.closed)
 467
 468         @tasks.async
 469         def spawn_build(self, iface_name):
 470                 assert self.child is None
 471
 472                 self.details.insert_at_end_and_scroll('Building %s\n' % iface_name, 'heading')
 473
 474                 # Group all the child processes so we can kill them easily
 475                 def become_group_leader():
 476                         os.setpgid(0, 0)
 477                 devnull = os.open(os.devnull, os.O_RDONLY)
 478                 try:
 479                         self.child = subprocess.Popen([sys.executable, '-u', sys.argv[0], 'build'],
 480                                                         stdin = devnull,
 481                                                         stdout = subprocess.PIPE, stderr = subprocess.STDOUT,
 482                                                         preexec_fn = become_group_leader)
 483                 finally:
 484                         os.close(devnull)
 485
 486                 import codecs
 487                 decoder = codecs.getincrementaldecoder('utf-8')(errors = 'replace')
 488
 489                 while True:
 490                         yield tasks.InputBlocker(self.child.stdout, 'output from child')
 491                         got = os.read(self.child.stdout.fileno(), 100)
 492                         chars = decoder.decode(got, final = not got)
 493                         self.details.insert_at_end_and_scroll(chars)
 494                         if not got: break
 495
 496                 self.child.wait()
 497                 code = self.child.returncode
 498                 self.child = None
 499                 if code:
 500                         self.details.insert_at_end_and_scroll('Build process exited with error status %d\n' % code, 'error')
 501                         raise SafeException('Build process exited with error status %d' % code)
 502                 self.details.insert_at_end_and_scroll('Build completed successfully\n', 'heading')
 503
 504         @tasks.async
 505         def confirm_import_feed(self, pending, valid_sigs):
 506                 from zeroinstall.gtkui import trust_box
 507                 box = trust_box.TrustBox(pending, valid_sigs, parent = self.dialog)
 508                 box.show()
 509                 yield box.closed
 510
 511 def do_autocompile(args):
 512         """autocompile [--gui] URI"""
 513
 514         parser = OptionParser(usage="usage: %prog autocompile [options]")
 515
 516         parser.add_option('', "--gui", help="graphical interface", action='store_true')
 517         (options, args2) = parser.parse_args(args)
 518         if len(args2) != 1:
 519                 raise __main__.UsageError()
 520
 521         if options.gui:
 522                 h = GUIHandler()
 523         elif os.isatty(1):
 524                 h = handler.ConsoleHandler()
 525         else:
 526                 h = handler.Handler()
 527         config = load_config(handler = h)
 528         config._iface_cache = AutocompileCache()
 529
 530         iface_uri = model.canonical_iface_uri(args2[0])
 531         if options.gui:
 532                 compiler = GTKAutoCompiler(config, iface_uri, options)
 533         else:
 534                 compiler = AutoCompiler(config, iface_uri, options)
 535
 536         compiler.build()
 537
 538 __main__.commands += [do_autocompile]