autocompile.py

   1 # Copyright (C) 2009, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import sys, os, __main__, tempfile, subprocess, signal, shutil
   5 from xml.dom import minidom
   6 from optparse import OptionParser
   7 from logging import warn
   8
   9 from zeroinstall import SafeException
  10 from zeroinstall.injector import arch, handler, driver, requirements, model, iface_cache, namespaces, writer, reader, qdom
  11 from zeroinstall.injector.config import load_config
  12 from zeroinstall.zerostore import manifest, NotStored
  13 from zeroinstall.support import tasks, basedir, ro_rmtree
  14
  15 from support import BuildEnv, uname, XMLNS_0COMPILE
  16
  17 # This is a bit hacky...
  18 #
  19 # We invent a new CPU type which is compatible with the host but worse than
  20 # every existing type, and we use * for the OS type so that we don't beat 'Any'
  21 # binaries either. This means that we always prefer an existing binary of the
  22 # desired version to compiling a new one, but we'll compile a new version from source
  23 # rather than use an older binary.
  24 arch.machine_groups['newbuild'] = arch.machine_groups.get(uname[4], 0)
  25 arch.machine_ranks['newbuild'] = max(arch.machine_ranks.values()) + 1
  26 host_arch = '*-newbuild'
  27
  28 class ImplRestriction(model.Restriction):
  29         reason = "Not the source we're trying to build"
  30
  31         def __init__(self, impl_id):
  32                 self.impl_id = impl_id
  33
  34         def meets_restriction(self, impl):
  35                 return impl.id == self.impl_id
  36
  37         def __str__(self):
  38                 return _("implementation {impl}").format(impl = self.impl_id)
  39
  40 class NewBuildImplementation(model.ZeroInstallImplementation):
  41         # Assume that this (potential) binary is available so that we can select it as a
  42         # dependency.
  43         def is_available(self, stores):
  44                 return True
  45
  46 def get_commands(src_impl):
  47         """Estimate the commands that the generated binary would have."""
  48         cmd = src_impl.commands.get('compile', None)
  49         if cmd is None:
  50                 warn("Source has no compile command! %s", src_impl)
  51                 return []
  52
  53         for elem in cmd.qdom.childNodes:
  54                 if elem.uri == XMLNS_0COMPILE and elem.name == 'implementation':
  55                         # Assume there's always a run command. Doesn't do any harm to have extra ones,
  56                         # and there are various ways this might get created.
  57                         commands = ['run']
  58                         for e in elem.childNodes:
  59                                 if e.uri == namespaces.XMLNS_IFACE and e.name == 'command':
  60                                         commands.append(e.getAttribute('name'))
  61                         return commands
  62         return []
  63
  64 class AutocompileCache(iface_cache.IfaceCache):
  65         def __init__(self):
  66                 iface_cache.IfaceCache.__init__(self)
  67                 self.done = set()
  68
  69         def get_feed(self, url, force = False):
  70                 feed = iface_cache.IfaceCache.get_feed(self, url, force)
  71                 if not feed: return None
  72
  73                 if feed not in self.done:
  74                         self.done.add(feed)
  75
  76                         # For each source impl, add a corresponding binary
  77                         # (the binary has no dependencies as we can't predict them here,
  78                         # but they're not the same as the source's dependencies)
  79
  80                         srcs = [x for x in feed.implementations.itervalues() if x.arch and x.arch.endswith('-src')]
  81                         for x in srcs:
  82                                 new_id = '0compile=' + x.id
  83                                 if not new_id in feed.implementations:
  84                                         new = NewBuildImplementation(feed, new_id, None)
  85                                         feed.implementations[new_id] = new
  86                                         new.set_arch(host_arch)
  87                                         new.version = x.version
  88
  89                                         # Give it some dummy commands in case we're using it as a <runner>, etc (otherwise it can't be selected)
  90                                         for cmd_name in get_commands(x):
  91                                                 cmd = qdom.Element(namespaces.XMLNS_IFACE, 'command', {'path': 'new-build', 'name': cmd_name})
  92                                                 new.commands[cmd_name] = model.Command(cmd, None)
  93
  94                 return feed
  95
  96 class AutoCompiler:
  97         def __init__(self, config, iface_uri, options):
  98                 self.iface_uri = iface_uri
  99                 self.options = options
 100                 self.config = config
 101
 102         def pretty_print_plan(self, solver, root, indent = '- '):
 103                 """Display a tree showing the selected implementations."""
 104                 iface = self.config.iface_cache.get_interface(root)
 105                 impl = solver.selections[iface]
 106                 if impl is None:
 107                         msg = 'Failed to select any suitable version (source or binary)'
 108                 elif impl.id.startswith('0compile='):
 109                         real_impl_id = impl.id.split('=', 1)[1]
 110                         real_impl = impl.feed.implementations[real_impl_id]
 111                         msg = 'Compile %s (%s)' % (real_impl.get_version(), real_impl.id)
 112                 elif impl.arch and impl.arch.endswith('-src'):
 113                         msg = 'Compile %s (%s)' % (impl.get_version(), impl.id)
 114                 else:
 115                         if impl.arch:
 116                                 msg = 'Use existing binary %s (%s)' % (impl.get_version(), impl.arch)
 117                         else:
 118                                 msg = 'Use existing architecture-independent package %s' % impl.get_version()
 119                 self.note("%s%s: %s" % (indent, iface.get_name(), msg))
 120
 121                 if impl:
 122                         indent = '  ' + indent
 123                         for x in impl.requires:
 124                                 self.pretty_print_plan(solver, x.interface, indent)
 125
 126         def print_details(self, solver):
 127                 """Dump debugging details."""
 128                 self.note("\nFailed. Details of all components and versions considered:")
 129                 for iface in solver.details:
 130                         self.note('\n%s\n' % iface.get_name())
 131                         for impl, note in solver.details[iface]:
 132                                 self.note('%s (%s) : %s' % (impl.get_version(), impl.arch or '*-*', note or 'OK'))
 133                 self.note("\nEnd details\n")
 134
 135         @tasks.async
 136         def compile_and_register(self, sels, forced_iface_uri = None):
 137                 """If forced_iface_uri, register as an implementation of this interface,
 138                 ignoring the any <feed-for>, etc."""
 139
 140                 buildenv = BuildEnv(need_config = False)
 141                 buildenv.config.set('compile', 'interface', sels.interface)
 142                 buildenv.config.set('compile', 'selections', 'selections.xml')
 143
 144                 # Download any required packages now, so we can use the GUI to request confirmation, etc
 145                 download_missing = sels.download_missing(self.config, include_packages = True)
 146                 if download_missing:
 147                         yield download_missing
 148                         tasks.check(download_missing)
 149
 150                 tmpdir = tempfile.mkdtemp(prefix = '0compile-')
 151                 try:
 152                         os.chdir(tmpdir)
 153
 154                         # Write configuration for build...
 155
 156                         buildenv.save()
 157
 158                         sel_file = open('selections.xml', 'w')
 159                         try:
 160                                 doc = sels.toDOM()
 161                                 doc.writexml(sel_file)
 162                                 sel_file.write('\n')
 163                         finally:
 164                                 sel_file.close()
 165
 166                         # Do the build...
 167
 168                         build = self.spawn_build(buildenv.iface_name)
 169                         if build:
 170                                 yield build
 171                                 tasks.check(build)
 172
 173                         # Register the result...
 174                         dom = minidom.parse(buildenv.local_iface_file)
 175
 176                         feed_for_elem, = dom.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'feed-for')
 177                         claimed_iface = feed_for_elem.getAttribute('interface')
 178
 179                         if forced_iface_uri is not None:
 180                                 if forced_iface_uri != claimed_iface:
 181                                         self.note("WARNING: registering as feed for {forced}, though feed claims to be for {claimed}".format(
 182                                                 forced = forced_iface_uri,
 183                                                 claimed = claimed_iface))
 184                         else:
 185                                 forced_iface_uri = claimed_iface                # (the top-level interface being built)
 186
 187                         version = sels.selections[sels.interface].version
 188
 189                         site_package_versions_dir = basedir.save_data_path('0install.net', 'site-packages',
 190                                                 *model.escape_interface_uri(forced_iface_uri))
 191                         leaf =  '%s-%s' % (version, uname[4])
 192                         site_package_dir = os.path.join(site_package_versions_dir, leaf)
 193                         self.note("Storing build in %s" % site_package_dir)
 194
 195                         # 1. Copy new version in under a temporary name. Names starting with '.' are ignored by 0install.
 196                         tmp_distdir = os.path.join(site_package_versions_dir, '.new-' + leaf)
 197                         shutil.copytree(buildenv.distdir, tmp_distdir, symlinks = True)
 198
 199                         # 2. Rename the previous build to .old-VERSION (deleting that if it already existed)
 200                         if os.path.exists(site_package_dir):
 201                                 self.note("(moving previous build out of the way)")
 202                                 previous_build_dir = os.path.join(site_package_versions_dir, '.old-' + leaf)
 203                                 if os.path.exists(previous_build_dir):
 204                                         shutil.rmtree(previous_build_dir)
 205                                 os.rename(site_package_dir, previous_build_dir)
 206                         else:
 207                                 previous_build_dir = None
 208
 209                         # 3. Rename the new version immediately after renaming away the old one to minimise time when there's
 210                         # no version.
 211                         os.rename(tmp_distdir, site_package_dir)
 212
 213                         # 4. Delete the old version.
 214                         if previous_build_dir:
 215                                 self.note("(deleting previous build)")
 216                                 shutil.rmtree(previous_build_dir)
 217
 218                         local_feed = os.path.join(site_package_dir, '0install', 'feed.xml')
 219                         assert os.path.exists(local_feed), "Feed %s not found!" % local_feed
 220
 221                         # Reload - our 0install will detect the new feed automatically
 222                         iface = self.config.iface_cache.get_interface(forced_iface_uri)
 223                         reader.update_from_cache(iface, iface_cache = self.config.iface_cache)
 224                         self.config.iface_cache.get_feed(local_feed, force = True)
 225
 226                         # Write it out - 0install will add the feed so that older 0install versions can find it
 227                         writer.save_interface(iface)
 228                 except:
 229                         self.note("\nBuild failed: leaving build directory %s for inspection...\n" % tmpdir)
 230                         raise
 231                 else:
 232                         # Can't delete current directory on Windows, so move to parent first
 233                         os.chdir(os.path.join(tmpdir, os.path.pardir))
 234
 235                         ro_rmtree(tmpdir)
 236
 237         @tasks.async
 238         def recursive_build(self, iface_uri, source_impl_id = None):
 239                 """Build an implementation of iface_uri and register it as a feed.
 240                 @param source_impl_id: the version to build, or None to build any version
 241                 @type source_impl_id: str
 242                 """
 243                 r = requirements.Requirements(iface_uri)
 244                 r.source = True
 245                 r.command = 'compile'
 246
 247                 d = driver.Driver(self.config, r)
 248                 iface = self.config.iface_cache.get_interface(iface_uri)
 249                 d.solver.record_details = True
 250                 if source_impl_id is not None:
 251                         d.solver.extra_restrictions[iface] = [ImplRestriction(source_impl_id)]
 252
 253                 # For testing...
 254                 #p.target_arch = arch.Architecture(os_ranks = {'FreeBSD': 0, None: 1}, machine_ranks = {'i386': 0, None: 1, 'newbuild': 2})
 255
 256                 while True:
 257                         self.heading(iface_uri)
 258                         self.note("\nSelecting versions for %s..." % iface.get_name())
 259                         solved = d.solve_with_downloads()
 260                         if solved:
 261                                 yield solved
 262                                 tasks.check(solved)
 263
 264                         if not d.solver.ready:
 265                                 self.print_details(d.solver)
 266                                 raise d.solver.get_failure_reason()
 267                         self.note("Selection done.")
 268
 269                         self.note("\nPlan:\n")
 270                         self.pretty_print_plan(d.solver, r.interface_uri)
 271                         self.note('')
 272
 273                         needed = []
 274                         for dep_iface, dep_impl in d.solver.selections.iteritems():
 275                                 if dep_impl.id.startswith('0compile='):
 276                                         if not needed:
 277                                                 self.note("Build dependencies that need to be compiled first:\n")
 278                                         self.note("- {iface} {version}".format(iface = dep_iface.uri, version = model.format_version(dep_impl.version)))
 279                                         needed.append((dep_iface, dep_impl))
 280
 281                         if not needed:
 282                                 self.note("No dependencies need compiling... compile %s itself..." % iface.get_name())
 283                                 build = self.compile_and_register(d.solver.selections,
 284                                                 # force the interface in the recursive case
 285                                                 iface_uri if iface_uri != self.iface_uri else None)
 286                                 yield build
 287                                 tasks.check(build)
 288                                 return
 289
 290                         # Compile the first missing build dependency...
 291                         dep_iface, dep_impl = needed[0]
 292
 293                         self.note("")
 294
 295                         #details = d.solver.details[self.config.iface_cache.get_interface(dep_iface.uri)]
 296                         #for de in details:
 297                         #       print de
 298
 299                         dep_source_id = dep_impl.id.split('=', 1)[1]
 300                         build = self.recursive_build(dep_iface.uri, dep_source_id)
 301                         yield build
 302                         tasks.check(build)
 303                         # Try again with that dependency built...
 304
 305         def spawn_build(self, iface_name):
 306                 try:
 307                         subprocess.check_call([sys.executable, sys.argv[0], 'build'])
 308                 except subprocess.CalledProcessError as ex:
 309                         raise SafeException(str(ex))
 310
 311         def build(self):
 312                 tasks.wait_for_blocker(self.recursive_build(self.iface_uri))
 313
 314         def heading(self, msg):
 315                 self.note((' %s ' % msg).center(76, '='))
 316
 317         def note(self, msg):
 318                 print msg
 319
 320         def note_error(self, msg):
 321                 self.overall.insert_at_cursor(msg + '\n')
 322
 323 class GUIHandler(handler.Handler):
 324         def downloads_changed(self):
 325                 self.compiler.downloads_changed()
 326
 327         def confirm_import_feed(self, pending, valid_sigs):
 328                 return handler.Handler.confirm_import_feed(self, pending, valid_sigs)
 329
 330         @tasks.async
 331         def confirm_install(self, message):
 332                 from zeroinstall.injector.download import DownloadAborted
 333                 from zeroinstall.gtkui import gtkutils
 334                 import gtk
 335                 box = gtk.MessageDialog(self.compiler.dialog,
 336                                         gtk.DIALOG_DESTROY_WITH_PARENT,
 337                                         gtk.MESSAGE_QUESTION, gtk.BUTTONS_CANCEL,
 338                                         message)
 339                 box.set_position(gtk.WIN_POS_CENTER)
 340
 341                 install = gtkutils.MixedButton('Install', gtk.STOCK_OK)
 342                 install.set_flags(gtk.CAN_DEFAULT)
 343                 box.add_action_widget(install, gtk.RESPONSE_OK)
 344                 install.show_all()
 345                 box.set_default_response(gtk.RESPONSE_OK)
 346                 box.show()
 347
 348                 response = gtkutils.DialogResponse(box)
 349                 yield response
 350                 box.destroy()
 351
 352                 if response.response != gtk.RESPONSE_OK:
 353                         raise DownloadAborted()
 354
 355 class GTKAutoCompiler(AutoCompiler):
 356         def __init__(self, config, iface_uri, options):
 357                 config.handler.compiler = self
 358
 359                 AutoCompiler.__init__(self, config, iface_uri, options)
 360                 self.child = None
 361
 362                 import pygtk; pygtk.require('2.0')
 363                 import gtk
 364
 365                 w = gtk.Dialog('Autocompile %s' % iface_uri, None, 0,
 366                                                  (gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,
 367                                                   gtk.STOCK_OK, gtk.RESPONSE_OK))
 368                 self.dialog = w
 369
 370                 w.set_default_size(int(gtk.gdk.screen_width() * 0.8),
 371                                    int(gtk.gdk.screen_height() * 0.8))
 372
 373                 vpaned = gtk.VPaned()
 374                 w.vbox.add(vpaned)
 375                 w.set_response_sensitive(gtk.RESPONSE_OK, False)
 376
 377                 class AutoScroller:
 378                         def __init__(self):
 379                                 tv = gtk.TextView()
 380                                 tv.set_property('left-margin', 8)
 381                                 tv.set_wrap_mode(gtk.WRAP_WORD_CHAR)
 382                                 tv.set_editable(False)
 383                                 swin = gtk.ScrolledWindow()
 384                                 swin.add(tv)
 385                                 swin.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC)
 386                                 buffer = tv.get_buffer()
 387
 388                                 heading = buffer.create_tag('heading')
 389                                 heading.set_property('scale', 1.5)
 390
 391                                 error = buffer.create_tag('error')
 392                                 error.set_property('background', 'white')
 393                                 error.set_property('foreground', 'red')
 394
 395                                 self.tv = tv
 396                                 self.widget = swin
 397                                 self.buffer = buffer
 398
 399                         def insert_at_end_and_scroll(self, data, *tags):
 400                                 vscroll = self.widget.get_vadjustment()
 401                                 if not vscroll:
 402                                         # Widget has been destroyed
 403                                         print data,
 404                                         return
 405                                 near_end = vscroll.upper - vscroll.page_size * 1.5 < vscroll.value
 406                                 end = self.buffer.get_end_iter()
 407                                 self.buffer.insert_with_tags_by_name(end, data, *tags)
 408                                 if near_end:
 409                                         cursor = self.buffer.get_insert()
 410                                         self.buffer.move_mark(cursor, end)
 411                                         self.tv.scroll_to_mark(cursor, 0, False, 0, 0)
 412
 413                         def set_text(self, text):
 414                                 self.buffer.set_text(text)
 415
 416                 self.overall = AutoScroller()
 417                 self.details = AutoScroller()
 418
 419                 vpaned.pack1(self.overall.widget, True, False)
 420                 vpaned.pack2(self.details.widget, True, False)
 421
 422                 self.closed = tasks.Blocker('Window closed')
 423
 424                 w.show_all()
 425                 w.connect('destroy', lambda wd: self.closed.trigger())
 426
 427                 def response(wd, resp):
 428                         if self.child is not None:
 429                                 self.note_error('Sending TERM signal to build process group %d...' % self.child.pid)
 430                                 os.kill(-self.child.pid, signal.SIGTERM)
 431                         else:
 432                                 self.closed.trigger()
 433                 w.connect('response', response)
 434
 435         def downloads_changed(self):
 436                 if self.config.handler.monitored_downloads:
 437                         msg = 'Downloads in progress:\n'
 438                         for x in self.config.handler.monitored_downloads:
 439                                 msg += '- {url}\n'.format(url = x.url)
 440                 else:
 441                         msg = ''
 442                 self.details.set_text(msg)
 443
 444         def heading(self, msg):
 445                 self.overall.insert_at_end_and_scroll(msg + '\n', 'heading')
 446
 447         def note(self, msg):
 448                 self.overall.insert_at_end_and_scroll(msg + '\n')
 449
 450         def note_error(self, msg):
 451                 self.overall.insert_at_end_and_scroll(msg + '\n', 'error')
 452
 453         def build(self):
 454                 import gtk
 455                 try:
 456                         tasks.wait_for_blocker(self.recursive_build(self.iface_uri))
 457                 except SafeException, ex:
 458                         self.note_error(str(ex))
 459                 else:
 460                         self.heading('All builds completed successfully!')
 461                         self.dialog.set_response_sensitive(gtk.RESPONSE_CANCEL, False)
 462                         self.dialog.set_response_sensitive(gtk.RESPONSE_OK, True)
 463
 464                 tasks.wait_for_blocker(self.closed)
 465
 466         @tasks.async
 467         def spawn_build(self, iface_name):
 468                 assert self.child is None
 469
 470                 self.details.insert_at_end_and_scroll('Building %s\n' % iface_name, 'heading')
 471
 472                 # Group all the child processes so we can kill them easily
 473                 def become_group_leader():
 474                         os.setpgid(0, 0)
 475                 devnull = os.open(os.devnull, os.O_RDONLY)
 476                 try:
 477                         self.child = subprocess.Popen([sys.executable, '-u', sys.argv[0], 'build'],
 478                                                         stdin = devnull,
 479                                                         stdout = subprocess.PIPE, stderr = subprocess.STDOUT,
 480                                                         preexec_fn = become_group_leader)
 481                 finally:
 482                         os.close(devnull)
 483
 484                 import codecs
 485                 decoder = codecs.getincrementaldecoder('utf-8')(errors = 'replace')
 486
 487                 while True:
 488                         yield tasks.InputBlocker(self.child.stdout, 'output from child')
 489                         got = os.read(self.child.stdout.fileno(), 100)
 490                         chars = decoder.decode(got, final = not got)
 491                         self.details.insert_at_end_and_scroll(chars)
 492                         if not got: break
 493
 494                 self.child.wait()
 495                 code = self.child.returncode
 496                 self.child = None
 497                 if code:
 498                         self.details.insert_at_end_and_scroll('Build process exited with error status %d\n' % code, 'error')
 499                         raise SafeException('Build process exited with error status %d' % code)
 500                 self.details.insert_at_end_and_scroll('Build completed successfully\n', 'heading')
 501
 502         @tasks.async
 503         def confirm_import_feed(self, pending, valid_sigs):
 504                 from zeroinstall.gtkui import trust_box
 505                 box = trust_box.TrustBox(pending, valid_sigs, parent = self.dialog)
 506                 box.show()
 507                 yield box.closed
 508
 509 def do_autocompile(args):
 510         """autocompile [--gui] URI"""
 511
 512         parser = OptionParser(usage="usage: %prog autocompile [options]")
 513
 514         parser.add_option('', "--gui", help="graphical interface", action='store_true')
 515         (options, args2) = parser.parse_args(args)
 516         if len(args2) != 1:
 517                 raise __main__.UsageError()
 518
 519         if options.gui:
 520                 h = GUIHandler()
 521         elif os.isatty(1):
 522                 h = handler.ConsoleHandler()
 523         else:
 524                 h = handler.Handler()
 525         config = load_config(handler = h)
 526         config._iface_cache = AutocompileCache()
 527
 528         iface_uri = model.canonical_iface_uri(args2[0])
 529         if options.gui:
 530                 compiler = GTKAutoCompiler(config, iface_uri, options)
 531         else:
 532                 compiler = AutoCompiler(config, iface_uri, options)
 533
 534         compiler.build()
 535
 536 __main__.commands += [do_autocompile]