autocompile.py

   1 # Copyright (C) 2009, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import sys, os, __main__, tempfile, subprocess, signal, shutil
   5 from xml.dom import minidom
   6 from optparse import OptionParser
   7 from logging import warn
   8
   9 from zeroinstall import SafeException
  10 from zeroinstall.injector import arch, handler, driver, requirements, model, iface_cache, namespaces, writer, reader, qdom
  11 from zeroinstall.injector.config import load_config
  12 from zeroinstall.zerostore import manifest, NotStored
  13 from zeroinstall.support import tasks, basedir, ro_rmtree
  14
  15 from support import BuildEnv, uname, XMLNS_0COMPILE
  16
  17 # This is a bit hacky...
  18 #
  19 # We invent a new CPU type which is compatible with the host but worse than
  20 # every existing type, and we use * for the OS type so that we don't beat 'Any'
  21 # binaries either. This means that we always prefer an existing binary of the
  22 # desired version to compiling a new one, but we'll compile a new version from source
  23 # rather than use an older binary.
  24 arch.machine_groups['newbuild'] = arch.machine_groups.get(uname[4], 0)
  25 arch.machine_ranks['newbuild'] = max(arch.machine_ranks.values()) + 1
  26 host_arch = '*-newbuild'
  27
  28 class NewBuildImplementation(model.ZeroInstallImplementation):
  29         # Assume that this (potential) binary is available so that we can select it as a
  30         # dependency.
  31         def is_available(self, stores):
  32                 return True
  33
  34 def get_commands(src_impl):
  35         """Estimate the commands that the generated binary would have."""
  36         cmd = src_impl.commands.get('compile', None)
  37         if cmd is None:
  38                 warn("Source has no compile command! %s", src_impl)
  39                 return []
  40
  41         for elem in cmd.qdom.childNodes:
  42                 if elem.uri == XMLNS_0COMPILE and elem.name == 'implementation':
  43                         # Assume there's always a run command. Doesn't do any harm to have extra ones,
  44                         # and there are various ways this might get created.
  45                         commands = ['run']
  46                         for e in elem.childNodes:
  47                                 if e.uri == namespaces.XMLNS_IFACE and e.name == 'command':
  48                                         commands.append(e.getAttribute('name'))
  49                         return commands
  50         return []
  51
  52 class AutocompileCache(iface_cache.IfaceCache):
  53         def __init__(self):
  54                 iface_cache.IfaceCache.__init__(self)
  55                 self.done = set()
  56
  57         def get_feed(self, url, force = False):
  58                 feed = iface_cache.IfaceCache.get_feed(self, url, force)
  59                 if not feed: return None
  60
  61                 if feed not in self.done:
  62                         self.done.add(feed)
  63
  64                         # For each source impl, add a corresponding binary
  65                         # (the binary has no dependencies as we can't predict them here,
  66                         # but they're not the same as the source's dependencies)
  67
  68                         srcs = [x for x in feed.implementations.itervalues() if x.arch and x.arch.endswith('-src')]
  69                         for x in srcs:
  70                                 new_id = '0compile=' + x.id
  71                                 if not new_id in feed.implementations:
  72                                         new = NewBuildImplementation(feed, new_id, None)
  73                                         feed.implementations[new_id] = new
  74                                         new.set_arch(host_arch)
  75                                         new.version = x.version
  76
  77                                         # Give it some dummy commands in case we're using it as a <runner>, etc (otherwise it can't be selected)
  78                                         for cmd_name in get_commands(x):
  79                                                 cmd = qdom.Element(namespaces.XMLNS_IFACE, 'command', {'path': 'new-build', 'name': cmd_name})
  80                                                 new.commands[cmd_name] = model.Command(cmd, None)
  81
  82                 return feed
  83
  84 class AutoCompiler:
  85         def __init__(self, config, iface_uri, options):
  86                 self.iface_uri = iface_uri
  87                 self.options = options
  88                 self.config = config
  89
  90         def pretty_print_plan(self, solver, root, indent = '- '):
  91                 """Display a tree showing the selected implementations."""
  92                 iface = self.config.iface_cache.get_interface(root)
  93                 impl = solver.selections[iface]
  94                 if impl is None:
  95                         msg = 'Failed to select any suitable version (source or binary)'
  96                 elif impl.id.startswith('0compile='):
  97                         real_impl_id = impl.id.split('=', 1)[1]
  98                         real_impl = impl.feed.implementations[real_impl_id]
  99                         msg = 'Compile %s (%s)' % (real_impl.get_version(), real_impl.id)
 100                 elif impl.arch and impl.arch.endswith('-src'):
 101                         msg = 'Compile %s (%s)' % (impl.get_version(), impl.id)
 102                 else:
 103                         if impl.arch:
 104                                 msg = 'Use existing binary %s (%s)' % (impl.get_version(), impl.arch)
 105                         else:
 106                                 msg = 'Use existing architecture-independent package %s' % impl.get_version()
 107                 self.note("%s%s: %s" % (indent, iface.get_name(), msg))
 108
 109                 if impl:
 110                         indent = '  ' + indent
 111                         for x in impl.requires:
 112                                 self.pretty_print_plan(solver, x.interface, indent)
 113
 114         def print_details(self, solver):
 115                 """Dump debugging details."""
 116                 self.note("\nFailed. Details of all components and versions considered:")
 117                 for iface in solver.details:
 118                         self.note('\n%s\n' % iface.get_name())
 119                         for impl, note in solver.details[iface]:
 120                                 self.note('%s (%s) : %s' % (impl.get_version(), impl.arch or '*-*', note or 'OK'))
 121                 self.note("\nEnd details\n")
 122
 123         @tasks.async
 124         def compile_and_register(self, sels, forced_iface_uri = None):
 125                 """If forced_iface_uri, register as an implementation of this interface,
 126                 ignoring the any <feed-for>, etc."""
 127
 128                 buildenv = BuildEnv(need_config = False)
 129                 buildenv.config.set('compile', 'interface', sels.interface)
 130                 buildenv.config.set('compile', 'selections', 'selections.xml')
 131
 132                 # Download any required packages now, so we can use the GUI to request confirmation, etc
 133                 download_missing = sels.download_missing(self.config, include_packages = True)
 134                 if download_missing:
 135                         yield download_missing
 136                         tasks.check(download_missing)
 137
 138                 tmpdir = tempfile.mkdtemp(prefix = '0compile-')
 139                 try:
 140                         os.chdir(tmpdir)
 141
 142                         # Write configuration for build...
 143
 144                         buildenv.save()
 145
 146                         sel_file = open('selections.xml', 'w')
 147                         try:
 148                                 doc = sels.toDOM()
 149                                 doc.writexml(sel_file)
 150                                 sel_file.write('\n')
 151                         finally:
 152                                 sel_file.close()
 153
 154                         # Do the build...
 155
 156                         build = self.spawn_build(buildenv.iface_name)
 157                         if build:
 158                                 yield build
 159                                 tasks.check(build)
 160
 161                         # Register the result...
 162                         dom = minidom.parse(buildenv.local_iface_file)
 163
 164                         feed_for_elem, = dom.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'feed-for')
 165                         claimed_iface = feed_for_elem.getAttribute('interface')
 166
 167                         if forced_iface_uri is not None:
 168                                 if forced_iface_uri != claimed_iface:
 169                                         self.note("WARNING: registering as feed for {forced}, though feed claims to be for {claimed}".format(
 170                                                 forced = forced_iface_uri,
 171                                                 claimed = claimed_iface))
 172                         else:
 173                                 forced_iface_uri = claimed_iface                # (the top-level interface being built)
 174
 175                         version = sels.selections[sels.interface].version
 176
 177                         site_package_versions_dir = basedir.save_data_path('0install.net', 'site-packages',
 178                                                 model._pretty_escape(forced_iface_uri))
 179                         leaf =  '%s-%s' % (version, uname[4])
 180                         site_package_dir = os.path.join(site_package_versions_dir, leaf)
 181                         self.note("Storing build in %s" % site_package_dir)
 182
 183                         # 1. Copy new version in under a temporary name. Names starting with '.' are ignored by 0install.
 184                         tmp_distdir = os.path.join(site_package_versions_dir, '.new-' + leaf)
 185                         shutil.copytree(buildenv.distdir, tmp_distdir, symlinks = True)
 186
 187                         # 2. Rename the previous build to .old-VERSION (deleting that if it already existed)
 188                         if os.path.exists(site_package_dir):
 189                                 self.note("(moving previous build out of the way)")
 190                                 previous_build_dir = os.path.join(site_package_versions_dir, '.old-' + leaf)
 191                                 if os.path.exists(previous_build_dir):
 192                                         shutil.rmtree(previous_build_dir)
 193                                 os.rename(site_package_dir, previous_build_dir)
 194                         else:
 195                                 previous_build_dir = None
 196
 197                         # 3. Rename the new version immediately after renaming away the old one to minimise time when there's
 198                         # no version.
 199                         os.rename(tmp_distdir, site_package_dir)
 200
 201                         # 4. Delete the old version.
 202                         if previous_build_dir:
 203                                 self.note("(deleting previous build)")
 204                                 shutil.rmtree(previous_build_dir)
 205
 206                         local_feed = os.path.join(site_package_dir, '0install', 'feed.xml')
 207                         assert os.path.exists(local_feed), "Feed %s not found!" % local_feed
 208
 209                         # Reload - our 0install will detect the new feed automatically
 210                         iface = self.config.iface_cache.get_interface(forced_iface_uri)
 211                         reader.update_from_cache(iface)
 212                         self.config.iface_cache.get_feed(local_feed, force = True)
 213
 214                         # Write it out - 0install will add the feed so that older 0install versions can find it
 215                         writer.save_interface(iface)
 216                 except:
 217                         self.note("\nBuild failed: leaving build directory %s for inspection...\n" % tmpdir)
 218                         raise
 219                 else:
 220                         # Can't delete current directory on Windows, so move to parent first
 221                         os.chdir(os.path.join(tmpdir, os.path.pardir))
 222
 223                         ro_rmtree(tmpdir)
 224
 225         @tasks.async
 226         def recursive_build(self, iface_uri, version = None):
 227                 """Build an implementation of iface_uri and register it as a feed.
 228                 @param version: the version to build, or None to build any version
 229                 @type version: str
 230                 """
 231                 r = requirements.Requirements(iface_uri)
 232                 r.source = True
 233                 r.command = 'compile'
 234
 235                 d = driver.Driver(self.config, r)
 236                 iface = self.config.iface_cache.get_interface(iface_uri)
 237                 d.solver.record_details = True
 238                 if version:
 239                         d.solver.extra_restrictions[iface] = [model.VersionRestriction(model.parse_version(version))]
 240
 241                 # For testing...
 242                 #p.target_arch = arch.Architecture(os_ranks = {'FreeBSD': 0, None: 1}, machine_ranks = {'i386': 0, None: 1, 'newbuild': 2})
 243
 244                 while True:
 245                         self.heading(iface_uri)
 246                         self.note("\nSelecting versions for %s..." % iface.get_name())
 247                         solved = d.solve_with_downloads()
 248                         if solved:
 249                                 yield solved
 250                                 tasks.check(solved)
 251
 252                         if not d.solver.ready:
 253                                 self.print_details(d.solver)
 254                                 raise SafeException("Can't find all required implementations (source or binary):\n" +
 255                                         '\n'.join(["- %s -> %s" % (iface, d.solver.selections[iface])
 256                                                    for iface in d.solver.selections]))
 257                         self.note("Selection done.")
 258
 259                         self.note("\nPlan:\n")
 260                         self.pretty_print_plan(d.solver, r.interface_uri)
 261                         self.note('')
 262
 263                         needed = []
 264                         for dep_iface, dep_impl in d.solver.selections.iteritems():
 265                                 if dep_impl.id.startswith('0compile='):
 266                                         if not needed:
 267                                                 self.note("Build dependencies that need to be compiled first:\n")
 268                                         self.note("- {iface} {version}".format(iface = dep_iface.uri, version = model.format_version(dep_impl.version)))
 269                                         needed.append((dep_iface, dep_impl))
 270
 271                         if not needed:
 272                                 self.note("No dependencies need compiling... compile %s itself..." % iface.get_name())
 273                                 build = self.compile_and_register(d.solver.selections,
 274                                                 # force the interface in the recursive case
 275                                                 iface_uri if iface_uri != self.iface_uri else None)
 276                                 yield build
 277                                 tasks.check(build)
 278                                 return
 279
 280                         # Compile the first missing build dependency...
 281                         dep_iface, dep_impl = needed[0]
 282
 283                         self.note("")
 284
 285                         #details = d.solver.details[self.config.iface_cache.get_interface(dep_iface.uri)]
 286                         #for de in details:
 287                         #       print de
 288
 289                         build = self.recursive_build(dep_iface.uri, dep_impl.get_version())
 290                         yield build
 291                         tasks.check(build)
 292                         # Try again with that dependency built...
 293
 294         def spawn_build(self, iface_name):
 295                 subprocess.check_call([sys.executable, sys.argv[0], 'build'])
 296
 297         def build(self):
 298                 tasks.wait_for_blocker(self.recursive_build(self.iface_uri))
 299
 300         def heading(self, msg):
 301                 self.note((' %s ' % msg).center(76, '='))
 302
 303         def note(self, msg):
 304                 print msg
 305
 306         def note_error(self, msg):
 307                 self.overall.insert_at_cursor(msg + '\n')
 308
 309 class GUIHandler(handler.Handler):
 310         def downloads_changed(self):
 311                 self.compiler.downloads_changed()
 312
 313         def confirm_import_feed(self, pending, valid_sigs):
 314                 return handler.Handler.confirm_import_feed(self, pending, valid_sigs)
 315
 316         @tasks.async
 317         def confirm_install(self, message):
 318                 from zeroinstall.injector.download import DownloadAborted
 319                 from zeroinstall.gtkui import gtkutils
 320                 import gtk
 321                 box = gtk.MessageDialog(self.compiler.dialog,
 322                                         gtk.DIALOG_DESTROY_WITH_PARENT,
 323                                         gtk.MESSAGE_QUESTION, gtk.BUTTONS_CANCEL,
 324                                         message)
 325                 box.set_position(gtk.WIN_POS_CENTER)
 326
 327                 install = gtkutils.MixedButton('Install', gtk.STOCK_OK)
 328                 install.set_flags(gtk.CAN_DEFAULT)
 329                 box.add_action_widget(install, gtk.RESPONSE_OK)
 330                 install.show_all()
 331                 box.set_default_response(gtk.RESPONSE_OK)
 332                 box.show()
 333
 334                 response = gtkutils.DialogResponse(box)
 335                 yield response
 336                 box.destroy()
 337
 338                 if response.response != gtk.RESPONSE_OK:
 339                         raise DownloadAborted()
 340
 341 class GTKAutoCompiler(AutoCompiler):
 342         def __init__(self, config, iface_uri, options):
 343                 config.handler.compiler = self
 344
 345                 AutoCompiler.__init__(self, config, iface_uri, options)
 346                 self.child = None
 347
 348                 import pygtk; pygtk.require('2.0')
 349                 import gtk
 350
 351                 w = gtk.Dialog('Autocompile %s' % iface_uri, None, 0,
 352                                                  (gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,
 353                                                   gtk.STOCK_OK, gtk.RESPONSE_OK))
 354                 self.dialog = w
 355
 356                 w.set_default_size(int(gtk.gdk.screen_width() * 0.8),
 357                                    int(gtk.gdk.screen_height() * 0.8))
 358
 359                 vpaned = gtk.VPaned()
 360                 w.vbox.add(vpaned)
 361                 w.set_response_sensitive(gtk.RESPONSE_OK, False)
 362
 363                 class AutoScroller:
 364                         def __init__(self):
 365                                 tv = gtk.TextView()
 366                                 tv.set_property('left-margin', 8)
 367                                 tv.set_wrap_mode(gtk.WRAP_WORD_CHAR)
 368                                 tv.set_editable(False)
 369                                 swin = gtk.ScrolledWindow()
 370                                 swin.add(tv)
 371                                 swin.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC)
 372                                 buffer = tv.get_buffer()
 373
 374                                 heading = buffer.create_tag('heading')
 375                                 heading.set_property('scale', 1.5)
 376
 377                                 error = buffer.create_tag('error')
 378                                 error.set_property('background', 'white')
 379                                 error.set_property('foreground', 'red')
 380
 381                                 self.tv = tv
 382                                 self.widget = swin
 383                                 self.buffer = buffer
 384
 385                         def insert_at_end_and_scroll(self, data, *tags):
 386                                 vscroll = self.widget.get_vadjustment()
 387                                 if not vscroll:
 388                                         # Widget has been destroyed
 389                                         print data,
 390                                         return
 391                                 near_end = vscroll.upper - vscroll.page_size * 1.5 < vscroll.value
 392                                 end = self.buffer.get_end_iter()
 393                                 self.buffer.insert_with_tags_by_name(end, data, *tags)
 394                                 if near_end:
 395                                         cursor = self.buffer.get_insert()
 396                                         self.buffer.move_mark(cursor, end)
 397                                         self.tv.scroll_to_mark(cursor, 0, False, 0, 0)
 398
 399                         def set_text(self, text):
 400                                 self.buffer.set_text(text)
 401
 402                 self.overall = AutoScroller()
 403                 self.details = AutoScroller()
 404
 405                 vpaned.pack1(self.overall.widget, True, False)
 406                 vpaned.pack2(self.details.widget, True, False)
 407
 408                 self.closed = tasks.Blocker('Window closed')
 409
 410                 w.show_all()
 411                 w.connect('destroy', lambda wd: self.closed.trigger())
 412
 413                 def response(wd, resp):
 414                         if self.child is not None:
 415                                 self.note_error('Sending TERM signal to build process group %d...' % self.child.pid)
 416                                 os.kill(-self.child.pid, signal.SIGTERM)
 417                         else:
 418                                 self.closed.trigger()
 419                 w.connect('response', response)
 420
 421         def downloads_changed(self):
 422                 if self.config.handler.monitored_downloads:
 423                         msg = 'Downloads in progress:\n'
 424                         for x in self.config.handler.monitored_downloads:
 425                                 msg += '- {url}\n'.format(url = x.url)
 426                 else:
 427                         msg = ''
 428                 self.details.set_text(msg)
 429
 430         def heading(self, msg):
 431                 self.overall.insert_at_end_and_scroll(msg + '\n', 'heading')
 432
 433         def note(self, msg):
 434                 self.overall.insert_at_end_and_scroll(msg + '\n')
 435
 436         def note_error(self, msg):
 437                 self.overall.insert_at_end_and_scroll(msg + '\n', 'error')
 438
 439         def build(self):
 440                 import gtk
 441                 try:
 442                         tasks.wait_for_blocker(self.recursive_build(self.iface_uri))
 443                 except SafeException, ex:
 444                         self.note_error(str(ex))
 445                 else:
 446                         self.heading('All builds completed successfully!')
 447                         self.dialog.set_response_sensitive(gtk.RESPONSE_CANCEL, False)
 448                         self.dialog.set_response_sensitive(gtk.RESPONSE_OK, True)
 449
 450                 tasks.wait_for_blocker(self.closed)
 451
 452         @tasks.async
 453         def spawn_build(self, iface_name):
 454                 assert self.child is None
 455
 456                 self.details.insert_at_end_and_scroll('Building %s\n' % iface_name, 'heading')
 457
 458                 # Group all the child processes so we can kill them easily
 459                 def become_group_leader():
 460                         os.setpgid(0, 0)
 461                 devnull = os.open(os.devnull, os.O_RDONLY)
 462                 try:
 463                         self.child = subprocess.Popen([sys.executable, '-u', sys.argv[0], 'build'],
 464                                                         stdin = devnull,
 465                                                         stdout = subprocess.PIPE, stderr = subprocess.STDOUT,
 466                                                         preexec_fn = become_group_leader)
 467                 finally:
 468                         os.close(devnull)
 469
 470                 import codecs
 471                 decoder = codecs.getincrementaldecoder('utf-8')(errors = 'replace')
 472
 473                 while True:
 474                         yield tasks.InputBlocker(self.child.stdout, 'output from child')
 475                         got = os.read(self.child.stdout.fileno(), 100)
 476                         chars = decoder.decode(got, final = not got)
 477                         self.details.insert_at_end_and_scroll(chars)
 478                         if not got: break
 479
 480                 self.child.wait()
 481                 code = self.child.returncode
 482                 self.child = None
 483                 if code:
 484                         self.details.insert_at_end_and_scroll('Build process exited with error status %d\n' % code, 'error')
 485                         raise SafeException('Build process exited with error status %d' % code)
 486                 self.details.insert_at_end_and_scroll('Build completed successfully\n', 'heading')
 487
 488         @tasks.async
 489         def confirm_import_feed(self, pending, valid_sigs):
 490                 from zeroinstall.gtkui import trust_box
 491                 box = trust_box.TrustBox(pending, valid_sigs, parent = self.dialog)
 492                 box.show()
 493                 yield box.closed
 494
 495 def do_autocompile(args):
 496         """autocompile [--gui] URI"""
 497
 498         parser = OptionParser(usage="usage: %prog autocompile [options]")
 499
 500         parser.add_option('', "--gui", help="graphical interface", action='store_true')
 501         (options, args2) = parser.parse_args(args)
 502         if len(args2) != 1:
 503                 raise __main__.UsageError()
 504
 505         if options.gui:
 506                 h = GUIHandler()
 507         elif os.isatty(1):
 508                 h = handler.ConsoleHandler()
 509         else:
 510                 h = handler.Handler()
 511         config = load_config(handler = h)
 512         config._iface_cache = AutocompileCache()
 513
 514         iface_uri = model.canonical_iface_uri(args2[0])
 515         if options.gui:
 516                 compiler = GTKAutoCompiler(config, iface_uri, options)
 517         else:
 518                 compiler = AutoCompiler(config, iface_uri, options)
 519
 520         compiler.build()
 521
 522 __main__.commands += [do_autocompile]