autocompile.py

   1 # Copyright (C) 2009, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import sys, os, __main__, tempfile, subprocess, signal
   5 from xml.dom import minidom
   6 from optparse import OptionParser
   7
   8 from zeroinstall import SafeException
   9 from zeroinstall.injector import arch, handler, policy, model, iface_cache, selections, namespaces, writer, reader
  10 from zeroinstall.zerostore import manifest, NotStored
  11 from zeroinstall.support import tasks, basedir, ro_rmtree
  12
  13 from support import BuildEnv
  14
  15 # This is a bit hacky...
  16 #
  17 # We invent a new CPU type which is compatible with the host but worse than
  18 # every existing type, and we use * for the OS type so that we don't beat 'Any'
  19 # binaries either. This means that we always prefer an existing binary of the
  20 # desired version to compiling a new one, but we'll compile a new version from source
  21 # rather than use an older binary.
  22 arch.machine_groups['newbuild'] = arch.machine_groups.get(arch._uname[-1], 0)
  23 arch.machine_ranks['newbuild'] = max(arch.machine_ranks.values()) + 1
  24 host_arch = '*-newbuild'
  25
  26 class DummyDownloadSource(model.RetrievalMethod):
  27         """0launch >= 0.46 won't select implementations without a download source."""
  28         pass
  29
  30 class AutocompileCache(iface_cache.IfaceCache):
  31         def __init__(self):
  32                 iface_cache.IfaceCache.__init__(self)
  33                 self.done = set()
  34
  35         def get_feed(self, url, force = False):
  36                 feed = iface_cache.IfaceCache.get_feed(self, url, force)
  37                 if not feed: return None
  38
  39                 if feed not in self.done:
  40                         self.done.add(feed)
  41
  42                         # For each source impl, add a corresponding binary
  43                         # (the binary has no dependencies as we can't predict them here,
  44                         # but they're not the same as the source's dependencies)
  45
  46                         srcs = [x for x in feed.implementations.itervalues() if x.arch and x.arch.endswith('-src')]
  47                         for x in srcs:
  48                                 new_id = '0compile=' + x.id
  49                                 if not new_id in feed.implementations:
  50                                         new = model.ZeroInstallImplementation(feed, new_id, None)
  51                                         new.download_sources.append(DummyDownloadSource())
  52                                         feed.implementations[new_id] = new
  53                                         new.digests.append(new_id)
  54                                         new.set_arch(host_arch)
  55                                         new.version = x.version
  56
  57                 return feed
  58
  59 class AutoCompiler:
  60         def __init__(self, config, iface_uri, options):
  61                 self.iface_uri = iface_uri
  62                 self.options = options
  63                 self.config = config
  64
  65         def pretty_print_plan(self, solver, root, indent = '- '):
  66                 """Display a tree showing the selected implementations."""
  67                 iface = self.config.iface_cache.get_interface(root)
  68                 impl = solver.selections[iface]
  69                 if impl is None:
  70                         msg = 'Failed to select any suitable version (source or binary)'
  71                 elif impl.id.startswith('0compile='):
  72                         real_impl_id = impl.id.split('=', 1)[1]
  73                         real_impl = impl.feed.implementations[real_impl_id]
  74                         msg = 'Compile %s (%s)' % (real_impl.get_version(), real_impl.id)
  75                 elif impl.arch and impl.arch.endswith('-src'):
  76                         msg = 'Compile %s (%s)' % (impl.get_version(), impl.id)
  77                 else:
  78                         if impl.arch:
  79                                 msg = 'Use existing binary %s (%s)' % (impl.get_version(), impl.arch)
  80                         else:
  81                                 msg = 'Use existing architecture-independent package %s' % impl.get_version()
  82                 self.note("%s%s: %s" % (indent, iface.get_name(), msg))
  83
  84                 if impl:
  85                         indent = '  ' + indent
  86                         for x in impl.requires:
  87                                 self.pretty_print_plan(solver, x.interface, indent)
  88
  89         def print_details(self, solver):
  90                 """Dump debugging details."""
  91                 self.note("\nFailed. Details of all components and versions considered:")
  92                 for iface in solver.details:
  93                         self.note('\n%s\n' % iface.get_name())
  94                         for impl, note in solver.details[iface]:
  95                                 self.note('%s (%s) : %s' % (impl.get_version(), impl.arch or '*-*', note or 'OK'))
  96                 self.note("\nEnd details\n")
  97
  98         @tasks.async
  99         def compile_and_register(self, policy):
 100                 def valid_autocompile_feed(binary_feed):
 101                         cache = policy.config.iface_cache
 102                         local_feed_impls = cache.get_feed(local_feed).implementations
 103                         if len(local_feed_impls) != 1:
 104                                 self.note("Invalid autocompile feed '%s'; expected exactly one implementation!" % binary_feed)
 105                                 return False
 106                         impl, = local_feed_impls.values()
 107                         try:
 108                                 cache.stores.lookup_any(impl.digests)
 109                                 return True
 110                         except NotStored, ex:
 111                                 self.note("Build metadata file '%s' exists but implementation is missing: %s" % (local_feed, ex))
 112                                 return False
 113
 114                 local_feed_dir = basedir.save_config_path('0install.net', '0compile', 'builds', model._pretty_escape(policy.root))
 115                 s = policy.solver.selections
 116
 117                 buildenv = BuildEnv(need_config = False)
 118                 buildenv.config.set('compile', 'interface', policy.root)
 119                 buildenv.config.set('compile', 'selections', 'selections.xml')
 120
 121                 # Download any required packages now, so we can use the GUI to request confirmation, etc
 122                 download_missing = s.download_missing(self.config, include_packages = True)
 123                 if download_missing:
 124                         yield download_missing
 125                         tasks.check(download_missing)
 126
 127                 version = s.selections[policy.root].version
 128                 local_feed = os.path.join(local_feed_dir, '%s-%s-%s.xml' % (buildenv.iface_name, version, arch._uname[-1]))
 129                 if os.path.exists(local_feed):
 130                         if not valid_autocompile_feed(local_feed):
 131                                 os.unlink(local_feed)
 132                         else:
 133                                 raise SafeException("Build metadata file '%s' already exists!" % local_feed)
 134
 135                 tmpdir = tempfile.mkdtemp(prefix = '0compile-')
 136                 try:
 137                         os.chdir(tmpdir)
 138
 139                         # Write configuration for build...
 140
 141                         buildenv.save()
 142
 143                         sel_file = open('selections.xml', 'w')
 144                         try:
 145                                 doc = s.toDOM()
 146                                 doc.writexml(sel_file)
 147                                 sel_file.write('\n')
 148                         finally:
 149                                 sel_file.close()
 150
 151                         # Do the build...
 152
 153                         build = self.spawn_build(buildenv.iface_name)
 154                         if build:
 155                                 yield build
 156                                 tasks.check(build)
 157
 158                         # Register the result...
 159
 160                         alg = manifest.get_algorithm('sha1new')
 161                         digest = alg.new_digest()
 162                         lines = []
 163                         for line in alg.generate_manifest(buildenv.distdir):
 164                                 line += '\n'
 165                                 digest.update(line)
 166                                 lines.append(line)
 167                         actual_digest = alg.getID(digest)
 168
 169                         local_feed_file = file(local_feed, 'w')
 170                         try:
 171                                 dom = minidom.parse(buildenv.local_iface_file)
 172                                 impl, = dom.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'implementation')
 173                                 impl.setAttribute('id', actual_digest)
 174                                 dom.writexml(local_feed_file)
 175                                 local_feed_file.write('\n')
 176                         finally:
 177                                 local_feed_file.close()
 178
 179                         feed_for_elem, = dom.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'feed-for')
 180
 181                         self.note("Implementation metadata written to %s" % local_feed)
 182
 183                         # No point adding it to the system store when only the user has the feed...
 184                         store = policy.config.stores.stores[0]
 185                         self.note("Storing build in user cache %s..." % store.dir)
 186                         policy.config.stores.add_dir_to_cache(actual_digest, buildenv.distdir)
 187
 188                         iface = policy.config.iface_cache.get_interface(feed_for_elem.getAttribute('interface'))
 189                         self.note("Registering as feed for %s" % iface.uri)
 190                         feed = iface.get_feed(local_feed)
 191                         if feed:
 192                                 self.note("WARNING: feed %s already registered!" % local_feed)
 193                         else:
 194                                 iface.extra_feeds.append(model.Feed(local_feed, impl.getAttribute('arch'), user_override = True))
 195                         writer.save_interface(iface)
 196
 197                         # We might have cached an old version
 198                         new_feed = policy.config.iface_cache.get_interface(local_feed)
 199                         reader.update_from_cache(new_feed)
 200                 except:
 201                         self.note("\nBuild failed: leaving build directory %s for inspection...\n" % tmpdir)
 202                         raise
 203                 else:
 204                         ro_rmtree(tmpdir)
 205
 206         @tasks.async
 207         def recursive_build(self, iface_uri, version = None):
 208                 p = policy.Policy(iface_uri, config = self.config, src = True)
 209                 iface = p.config.iface_cache.get_interface(iface_uri)
 210                 p.solver.record_details = True
 211                 if version:
 212                         p.solver.extra_restrictions[iface] = [model.VersionRestriction(model.parse_version(version))]
 213
 214                 # For testing...
 215                 #p.target_arch = arch.Architecture(os_ranks = {'FreeBSD': 0, None: 1}, machine_ranks = {'i386': 0, None: 1, 'newbuild': 2})
 216
 217                 while True:
 218                         self.heading(iface_uri)
 219                         self.note("\nSelecting versions for %s..." % iface.get_name())
 220                         solved = p.solve_with_downloads()
 221                         if solved:
 222                                 yield solved
 223                                 tasks.check(solved)
 224
 225                         if not p.solver.ready:
 226                                 self.print_details(p.solver)
 227                                 raise SafeException("Can't find all required implementations (source or binary):\n" +
 228                                         '\n'.join(["- %s -> %s" % (iface, p.solver.selections[iface])
 229                                                    for iface in p.solver.selections]))
 230                         self.note("Selection done.")
 231
 232                         self.note("\nPlan:\n")
 233                         self.pretty_print_plan(p.solver, p.root)
 234                         self.note('')
 235
 236                         for dep_iface, dep_impl in p.solver.selections.iteritems():
 237                                 if dep_impl.id.startswith('0compile='):
 238                                         build = self.recursive_build(dep_iface.uri, dep_impl.get_version())
 239                                         yield build
 240                                         tasks.check(build)
 241                                         break   # Try again with that dependency built...
 242                         else:
 243                                 self.note("No dependencies need compiling... compile %s itself..." % iface.get_name())
 244                                 build = self.compile_and_register(p)
 245                                 yield build
 246                                 tasks.check(build)
 247                                 return
 248
 249         def spawn_build(self, iface_name):
 250                 subprocess.check_call([sys.executable, sys.argv[0], 'build'])
 251
 252         def build(self):
 253                 tasks.wait_for_blocker(self.recursive_build(self.iface_uri))
 254
 255         def heading(self, msg):
 256                 self.note((' %s ' % msg).center(76, '='))
 257
 258         def note(self, msg):
 259                 print msg
 260
 261         def note_error(self, msg):
 262                 self.overall.insert_at_cursor(msg + '\n')
 263
 264 class GUIHandler(handler.Handler):
 265         def downloads_changed(self):
 266                 self.compiler.downloads_changed()
 267
 268         def confirm_import_feed(self, pending, valid_sigs):
 269                 return handler.Handler.confirm_import_feed(self, pending, valid_sigs)
 270
 271         @tasks.async
 272         def confirm_install(self, message):
 273                 from zeroinstall.injector.download import DownloadAborted
 274                 from zeroinstall.gtkui import gtkutils
 275                 import gtk
 276                 box = gtk.MessageDialog(self.compiler.dialog,
 277                                         gtk.DIALOG_DESTROY_WITH_PARENT,
 278                                         gtk.MESSAGE_QUESTION, gtk.BUTTONS_CANCEL,
 279                                         message)
 280                 box.set_position(gtk.WIN_POS_CENTER)
 281
 282                 install = gtkutils.MixedButton(_('Install'), gtk.STOCK_OK)
 283                 install.set_flags(gtk.CAN_DEFAULT)
 284                 box.add_action_widget(install, gtk.RESPONSE_OK)
 285                 install.show_all()
 286                 box.set_default_response(gtk.RESPONSE_OK)
 287                 box.show()
 288
 289                 response = gtkutils.DialogResponse(box)
 290                 yield response
 291                 box.destroy()
 292
 293                 if response.response != gtk.RESPONSE_OK:
 294                         raise DownloadAborted()
 295
 296 class GTKAutoCompiler(AutoCompiler):
 297         def __init__(self, config, iface_uri, options):
 298                 config.handler.compiler = self
 299
 300                 AutoCompiler.__init__(self, config, iface_uri, options)
 301                 self.child = None
 302
 303                 import pygtk; pygtk.require('2.0')
 304                 import gtk
 305
 306                 w = gtk.Dialog('Autocompile %s' % iface_uri, None, 0,
 307                                                  (gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,
 308                                                   gtk.STOCK_OK, gtk.RESPONSE_OK))
 309                 self.dialog = w
 310
 311                 w.set_default_size(int(gtk.gdk.screen_width() * 0.8),
 312                                    int(gtk.gdk.screen_height() * 0.8))
 313
 314                 vpaned = gtk.VPaned()
 315                 w.vbox.add(vpaned)
 316                 w.set_response_sensitive(gtk.RESPONSE_OK, False)
 317
 318                 class AutoScroller:
 319                         def __init__(self):
 320                                 tv = gtk.TextView()
 321                                 tv.set_property('left-margin', 8)
 322                                 tv.set_wrap_mode(gtk.WRAP_WORD_CHAR)
 323                                 tv.set_editable(False)
 324                                 swin = gtk.ScrolledWindow()
 325                                 swin.add(tv)
 326                                 swin.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC)
 327                                 buffer = tv.get_buffer()
 328
 329                                 heading = buffer.create_tag('heading')
 330                                 heading.set_property('scale', 1.5)
 331
 332                                 error = buffer.create_tag('error')
 333                                 error.set_property('background', 'white')
 334                                 error.set_property('foreground', 'red')
 335
 336                                 self.tv = tv
 337                                 self.widget = swin
 338                                 self.buffer = buffer
 339
 340                         def insert_at_end_and_scroll(self, data, *tags):
 341                                 vscroll = self.widget.get_vadjustment()
 342                                 if not vscroll:
 343                                         # Widget has been destroyed
 344                                         print data,
 345                                         return
 346                                 near_end = vscroll.upper - vscroll.page_size * 1.5 < vscroll.value
 347                                 end = self.buffer.get_end_iter()
 348                                 self.buffer.insert_with_tags_by_name(end, data, *tags)
 349                                 if near_end:
 350                                         cursor = self.buffer.get_insert()
 351                                         self.buffer.move_mark(cursor, end)
 352                                         self.tv.scroll_to_mark(cursor, 0, False, 0, 0)
 353
 354                 self.overall = AutoScroller()
 355                 self.details = AutoScroller()
 356
 357                 vpaned.pack1(self.overall.widget, True, False)
 358                 vpaned.pack2(self.details.widget, True, False)
 359
 360                 self.closed = tasks.Blocker('Window closed')
 361
 362                 w.show_all()
 363                 w.connect('destroy', lambda wd: self.closed.trigger())
 364
 365                 def response(wd, resp):
 366                         if self.child is not None:
 367                                 self.note_error('Sending TERM signal to build process group %d...' % self.child.pid)
 368                                 os.kill(-self.child.pid, signal.SIGTERM)
 369                         else:
 370                                 self.closed.trigger()
 371                 w.connect('response', response)
 372
 373         def downloads_changed(self):
 374                 if self.config.handler.monitored_downloads:
 375                         self.note('Downloads in progress:')
 376                         for x in self.config.handler.monitored_downloads:
 377                                 self.note('- %s' % x.url)
 378                 else:
 379                         self.note('No downloads remaining.')
 380
 381         def heading(self, msg):
 382                 self.overall.insert_at_end_and_scroll(msg + '\n', 'heading')
 383
 384         def note(self, msg):
 385                 self.overall.insert_at_end_and_scroll(msg + '\n')
 386
 387         def note_error(self, msg):
 388                 self.overall.insert_at_end_and_scroll(msg + '\n', 'error')
 389
 390         def build(self):
 391                 import gtk
 392                 try:
 393                         tasks.wait_for_blocker(self.recursive_build(self.iface_uri))
 394                 except SafeException, ex:
 395                         self.note_error(str(ex))
 396                 else:
 397                         self.heading('All builds completed successfully!')
 398                         self.dialog.set_response_sensitive(gtk.RESPONSE_CANCEL, False)
 399                         self.dialog.set_response_sensitive(gtk.RESPONSE_OK, True)
 400
 401                 tasks.wait_for_blocker(self.closed)
 402
 403         @tasks.async
 404         def spawn_build(self, iface_name):
 405                 assert self.child is None
 406
 407                 self.details.insert_at_end_and_scroll('Building %s\n' % iface_name, 'heading')
 408
 409                 # Group all the child processes so we can kill them easily
 410                 def become_group_leader():
 411                         os.setpgid(0, 0)
 412                 devnull = os.open(os.devnull, os.O_RDONLY)
 413                 try:
 414                         self.child = subprocess.Popen([sys.executable, '-u', sys.argv[0], 'build'],
 415                                                         stdin = devnull,
 416                                                         stdout = subprocess.PIPE, stderr = subprocess.STDOUT,
 417                                                         preexec_fn = become_group_leader)
 418                 finally:
 419                         os.close(devnull)
 420
 421                 import codecs
 422                 decoder = codecs.getincrementaldecoder('utf-8')(errors = 'replace')
 423
 424                 while True:
 425                         yield tasks.InputBlocker(self.child.stdout, 'output from child')
 426                         got = os.read(self.child.stdout.fileno(), 100)
 427                         chars = decoder.decode(got, final = not got)
 428                         self.details.insert_at_end_and_scroll(chars)
 429                         if not got: break
 430
 431                 self.child.wait()
 432                 code = self.child.returncode
 433                 self.child = None
 434                 if code:
 435                         self.details.insert_at_end_and_scroll('Build process exited with error status %d\n' % code, 'error')
 436                         raise SafeException('Build process exited with error status %d' % code)
 437                 self.details.insert_at_end_and_scroll('Build completed successfully\n', 'heading')
 438
 439         @tasks.async
 440         def confirm_import_feed(self, pending, valid_sigs):
 441                 from zeroinstall.gtkui import trust_box
 442                 box = trust_box.TrustBox(pending, valid_sigs, parent = self.dialog)
 443                 box.show()
 444                 yield box.closed
 445
 446 def do_autocompile(args):
 447         """autocompile [--gui] URI"""
 448
 449         parser = OptionParser(usage="usage: %prog autocompile [options]")
 450
 451         parser.add_option('', "--gui", help="graphical interface", action='store_true')
 452         (options, args2) = parser.parse_args(args)
 453         if len(args2) != 1:
 454                 raise __main__.UsageError()
 455
 456         if options.gui:
 457                 h = GUIHandler()
 458         elif os.isatty(1):
 459                 h = handler.ConsoleHandler()
 460         else:
 461                 h = handler.Handler()
 462         config = policy.load_config(handler = h)
 463         config._iface_cache = AutocompileCache()
 464
 465         iface_uri = model.canonical_iface_uri(args2[0])
 466         if options.gui:
 467                 compiler = GTKAutoCompiler(config, iface_uri, options)
 468         else:
 469                 compiler = AutoCompiler(config, iface_uri, options)
 470
 471         compiler.build()
 472
 473 __main__.commands += [do_autocompile]