autocompile.py

   1 # Copyright (C) 2009, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import sys, os, __main__, tempfile, subprocess, signal, shutil
   5 from xml.dom import minidom
   6 from optparse import OptionParser
   7 from logging import warn
   8
   9 from zeroinstall import SafeException
  10 from zeroinstall.injector import arch, handler, driver, requirements, model, iface_cache, namespaces, writer, reader, qdom
  11 from zeroinstall.injector.config import load_config
  12 from zeroinstall.zerostore import manifest, NotStored
  13 from zeroinstall.support import tasks, basedir, ro_rmtree
  14
  15 from support import BuildEnv, uname, XMLNS_0COMPILE
  16
  17 # This is a bit hacky...
  18 #
  19 # We invent a new CPU type which is compatible with the host but worse than
  20 # every existing type, and we use * for the OS type so that we don't beat 'Any'
  21 # binaries either. This means that we always prefer an existing binary of the
  22 # desired version to compiling a new one, but we'll compile a new version from source
  23 # rather than use an older binary.
  24 arch.machine_groups['newbuild'] = arch.machine_groups.get(uname[4], 0)
  25 arch.machine_ranks['newbuild'] = max(arch.machine_ranks.values()) + 1
  26 host_arch = '*-newbuild'
  27
  28 class NewBuildImplementation(model.ZeroInstallImplementation):
  29         # Assume that this (potential) binary is available so that we can select it as a
  30         # dependency.
  31         def is_available(self, stores):
  32                 return True
  33
  34 def get_commands(src_impl):
  35         """Estimate the commands that the generated binary would have."""
  36         cmd = src_impl.commands.get('compile', None)
  37         if cmd is None:
  38                 warn("Source has no compile command! %s", src_impl)
  39                 return []
  40
  41         for elem in cmd.qdom.childNodes:
  42                 if elem.uri == XMLNS_0COMPILE and elem.name == 'implementation':
  43                         # Assume there's always a run command. Doesn't do any harm to have extra ones,
  44                         # and there are various ways this might get created.
  45                         commands = ['run']
  46                         for e in elem.childNodes:
  47                                 if e.uri == namespaces.XMLNS_IFACE and e.name == 'command':
  48                                         commands.append(e.getAttribute('name'))
  49                         return commands
  50         return []
  51
  52 class AutocompileCache(iface_cache.IfaceCache):
  53         def __init__(self):
  54                 iface_cache.IfaceCache.__init__(self)
  55                 self.done = set()
  56
  57         def get_feed(self, url, force = False):
  58                 feed = iface_cache.IfaceCache.get_feed(self, url, force)
  59                 if not feed: return None
  60
  61                 if feed not in self.done:
  62                         self.done.add(feed)
  63
  64                         # For each source impl, add a corresponding binary
  65                         # (the binary has no dependencies as we can't predict them here,
  66                         # but they're not the same as the source's dependencies)
  67
  68                         srcs = [x for x in feed.implementations.itervalues() if x.arch and x.arch.endswith('-src')]
  69                         for x in srcs:
  70                                 new_id = '0compile=' + x.id
  71                                 if not new_id in feed.implementations:
  72                                         new = NewBuildImplementation(feed, new_id, None)
  73                                         feed.implementations[new_id] = new
  74                                         new.set_arch(host_arch)
  75                                         new.version = x.version
  76
  77                                         # Give it some dummy commands in case we're using it as a <runner>, etc (otherwise it can't be selected)
  78                                         for cmd_name in get_commands(x):
  79                                                 cmd = qdom.Element(namespaces.XMLNS_IFACE, 'command', {'path': 'new-build', 'name': cmd_name})
  80                                                 new.commands[cmd_name] = model.Command(cmd, None)
  81
  82                 return feed
  83
  84 class AutoCompiler:
  85         def __init__(self, config, iface_uri, options):
  86                 self.iface_uri = iface_uri
  87                 self.options = options
  88                 self.config = config
  89
  90         def pretty_print_plan(self, solver, root, indent = '- '):
  91                 """Display a tree showing the selected implementations."""
  92                 iface = self.config.iface_cache.get_interface(root)
  93                 impl = solver.selections[iface]
  94                 if impl is None:
  95                         msg = 'Failed to select any suitable version (source or binary)'
  96                 elif impl.id.startswith('0compile='):
  97                         real_impl_id = impl.id.split('=', 1)[1]
  98                         real_impl = impl.feed.implementations[real_impl_id]
  99                         msg = 'Compile %s (%s)' % (real_impl.get_version(), real_impl.id)
 100                 elif impl.arch and impl.arch.endswith('-src'):
 101                         msg = 'Compile %s (%s)' % (impl.get_version(), impl.id)
 102                 else:
 103                         if impl.arch:
 104                                 msg = 'Use existing binary %s (%s)' % (impl.get_version(), impl.arch)
 105                         else:
 106                                 msg = 'Use existing architecture-independent package %s' % impl.get_version()
 107                 self.note("%s%s: %s" % (indent, iface.get_name(), msg))
 108
 109                 if impl:
 110                         indent = '  ' + indent
 111                         for x in impl.requires:
 112                                 self.pretty_print_plan(solver, x.interface, indent)
 113
 114         def print_details(self, solver):
 115                 """Dump debugging details."""
 116                 self.note("\nFailed. Details of all components and versions considered:")
 117                 for iface in solver.details:
 118                         self.note('\n%s\n' % iface.get_name())
 119                         for impl, note in solver.details[iface]:
 120                                 self.note('%s (%s) : %s' % (impl.get_version(), impl.arch or '*-*', note or 'OK'))
 121                 self.note("\nEnd details\n")
 122
 123         @tasks.async
 124         def compile_and_register(self, sels, forced_iface_uri = None):
 125                 """If forced_iface_uri, register as an implementation of this interface,
 126                 ignoring the any <feed-for>, etc."""
 127
 128                 buildenv = BuildEnv(need_config = False)
 129                 buildenv.config.set('compile', 'interface', sels.interface)
 130                 buildenv.config.set('compile', 'selections', 'selections.xml')
 131
 132                 # Download any required packages now, so we can use the GUI to request confirmation, etc
 133                 download_missing = sels.download_missing(self.config, include_packages = True)
 134                 if download_missing:
 135                         yield download_missing
 136                         tasks.check(download_missing)
 137
 138                 tmpdir = tempfile.mkdtemp(prefix = '0compile-')
 139                 try:
 140                         os.chdir(tmpdir)
 141
 142                         # Write configuration for build...
 143
 144                         buildenv.save()
 145
 146                         sel_file = open('selections.xml', 'w')
 147                         try:
 148                                 doc = sels.toDOM()
 149                                 doc.writexml(sel_file)
 150                                 sel_file.write('\n')
 151                         finally:
 152                                 sel_file.close()
 153
 154                         # Do the build...
 155
 156                         build = self.spawn_build(buildenv.iface_name)
 157                         if build:
 158                                 yield build
 159                                 tasks.check(build)
 160
 161                         # Register the result...
 162                         dom = minidom.parse(buildenv.local_iface_file)
 163
 164                         feed_for_elem, = dom.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'feed-for')
 165                         claimed_iface = feed_for_elem.getAttribute('interface')
 166
 167                         if forced_iface_uri is not None:
 168                                 if forced_iface_uri != claimed_iface:
 169                                         self.note("WARNING: registering as feed for {forced}, though feed claims to be for {claimed}".format(
 170                                                 forced = forced_iface_uri,
 171                                                 claimed = claimed_iface))
 172                         else:
 173                                 forced_iface_uri = claimed_iface                # (the top-level interface being built)
 174
 175                         version = sels.selections[sels.interface].version
 176
 177                         site_package_versions_dir = basedir.save_data_path('0install.net', 'site-packages',
 178                                                 model._pretty_escape(forced_iface_uri))
 179                         leaf =  '%s-%s' % (version, uname[4])
 180                         site_package_dir = os.path.join(site_package_versions_dir, leaf)
 181                         self.note("Storing build in %s" % site_package_dir)
 182
 183                         # 1. Copy new version in under a temporary name. Names starting with '.' are ignored by 0install.
 184                         tmp_distdir = os.path.join(site_package_versions_dir, '.new-' + leaf)
 185                         shutil.copytree(buildenv.distdir, tmp_distdir, symlinks = True)
 186
 187                         # 2. Rename the previous build to .old-VERSION (deleting that if it already existed)
 188                         if os.path.exists(site_package_dir):
 189                                 self.note("(moving previous build out of the way)")
 190                                 previous_build_dir = os.path.join(site_package_versions_dir, '.old-' + leaf)
 191                                 if os.path.exists(previous_build_dir):
 192                                         shutil.rmtree(previous_build_dir)
 193                                 os.rename(site_package_dir, previous_build_dir)
 194                         else:
 195                                 previous_build_dir = None
 196
 197                         # 3. Rename the new version immediately after renaming away the old one to minimise time when there's
 198                         # no version.
 199                         os.rename(tmp_distdir, site_package_dir)
 200
 201                         # 4. Delete the old version.
 202                         if previous_build_dir:
 203                                 self.note("(deleting previous build)")
 204                                 shutil.rmtree(previous_build_dir)
 205
 206                         local_feed = os.path.join(site_package_dir, '0install', 'feed.xml')
 207                         assert os.path.exists(local_feed), "Feed %s not found!" % local_feed
 208
 209                         # Reload - our 0install will detect the new feed automatically
 210                         iface = self.config.iface_cache.get_interface(forced_iface_uri)
 211                         reader.update_from_cache(iface)
 212                         self.config.iface_cache.get_feed(local_feed, force = True)
 213
 214                         # Write it out - 0install will add the feed so that older 0install versions can find it
 215                         writer.save_interface(iface)
 216                 except:
 217                         self.note("\nBuild failed: leaving build directory %s for inspection...\n" % tmpdir)
 218                         raise
 219                 else:
 220                         ro_rmtree(tmpdir)
 221
 222         @tasks.async
 223         def recursive_build(self, iface_uri, version = None):
 224                 """Build an implementation of iface_uri and register it as a feed.
 225                 @param version: the version to build, or None to build any version
 226                 @type version: str
 227                 """
 228                 r = requirements.Requirements(iface_uri)
 229                 r.source = True
 230                 r.command = 'compile'
 231
 232                 d = driver.Driver(self.config, r)
 233                 iface = self.config.iface_cache.get_interface(iface_uri)
 234                 d.solver.record_details = True
 235                 if version:
 236                         d.solver.extra_restrictions[iface] = [model.VersionRestriction(model.parse_version(version))]
 237
 238                 # For testing...
 239                 #p.target_arch = arch.Architecture(os_ranks = {'FreeBSD': 0, None: 1}, machine_ranks = {'i386': 0, None: 1, 'newbuild': 2})
 240
 241                 while True:
 242                         self.heading(iface_uri)
 243                         self.note("\nSelecting versions for %s..." % iface.get_name())
 244                         solved = d.solve_with_downloads()
 245                         if solved:
 246                                 yield solved
 247                                 tasks.check(solved)
 248
 249                         if not d.solver.ready:
 250                                 self.print_details(d.solver)
 251                                 raise SafeException("Can't find all required implementations (source or binary):\n" +
 252                                         '\n'.join(["- %s -> %s" % (iface, d.solver.selections[iface])
 253                                                    for iface in d.solver.selections]))
 254                         self.note("Selection done.")
 255
 256                         self.note("\nPlan:\n")
 257                         self.pretty_print_plan(d.solver, r.interface_uri)
 258                         self.note('')
 259
 260                         needed = []
 261                         for dep_iface, dep_impl in d.solver.selections.iteritems():
 262                                 if dep_impl.id.startswith('0compile='):
 263                                         if not needed:
 264                                                 self.note("Build dependencies that need to be compiled first:\n")
 265                                         self.note("- {iface} {version}".format(iface = dep_iface.uri, version = model.format_version(dep_impl.version)))
 266                                         needed.append((dep_iface, dep_impl))
 267
 268                         if not needed:
 269                                 self.note("No dependencies need compiling... compile %s itself..." % iface.get_name())
 270                                 build = self.compile_and_register(d.solver.selections,
 271                                                 # force the interface in the recursive case
 272                                                 iface_uri if iface_uri != self.iface_uri else None)
 273                                 yield build
 274                                 tasks.check(build)
 275                                 return
 276
 277                         # Compile the first missing build dependency...
 278                         dep_iface, dep_impl = needed[0]
 279
 280                         self.note("")
 281
 282                         #details = d.solver.details[self.config.iface_cache.get_interface(dep_iface.uri)]
 283                         #for de in details:
 284                         #       print de
 285
 286                         build = self.recursive_build(dep_iface.uri, dep_impl.get_version())
 287                         yield build
 288                         tasks.check(build)
 289                         # Try again with that dependency built...
 290
 291         def spawn_build(self, iface_name):
 292                 subprocess.check_call([sys.executable, sys.argv[0], 'build'])
 293
 294         def build(self):
 295                 tasks.wait_for_blocker(self.recursive_build(self.iface_uri))
 296
 297         def heading(self, msg):
 298                 self.note((' %s ' % msg).center(76, '='))
 299
 300         def note(self, msg):
 301                 print msg
 302
 303         def note_error(self, msg):
 304                 self.overall.insert_at_cursor(msg + '\n')
 305
 306 class GUIHandler(handler.Handler):
 307         def downloads_changed(self):
 308                 self.compiler.downloads_changed()
 309
 310         def confirm_import_feed(self, pending, valid_sigs):
 311                 return handler.Handler.confirm_import_feed(self, pending, valid_sigs)
 312
 313         @tasks.async
 314         def confirm_install(self, message):
 315                 from zeroinstall.injector.download import DownloadAborted
 316                 from zeroinstall.gtkui import gtkutils
 317                 import gtk
 318                 box = gtk.MessageDialog(self.compiler.dialog,
 319                                         gtk.DIALOG_DESTROY_WITH_PARENT,
 320                                         gtk.MESSAGE_QUESTION, gtk.BUTTONS_CANCEL,
 321                                         message)
 322                 box.set_position(gtk.WIN_POS_CENTER)
 323
 324                 install = gtkutils.MixedButton('Install', gtk.STOCK_OK)
 325                 install.set_flags(gtk.CAN_DEFAULT)
 326                 box.add_action_widget(install, gtk.RESPONSE_OK)
 327                 install.show_all()
 328                 box.set_default_response(gtk.RESPONSE_OK)
 329                 box.show()
 330
 331                 response = gtkutils.DialogResponse(box)
 332                 yield response
 333                 box.destroy()
 334
 335                 if response.response != gtk.RESPONSE_OK:
 336                         raise DownloadAborted()
 337
 338 class GTKAutoCompiler(AutoCompiler):
 339         def __init__(self, config, iface_uri, options):
 340                 config.handler.compiler = self
 341
 342                 AutoCompiler.__init__(self, config, iface_uri, options)
 343                 self.child = None
 344
 345                 import pygtk; pygtk.require('2.0')
 346                 import gtk
 347
 348                 w = gtk.Dialog('Autocompile %s' % iface_uri, None, 0,
 349                                                  (gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,
 350                                                   gtk.STOCK_OK, gtk.RESPONSE_OK))
 351                 self.dialog = w
 352
 353                 w.set_default_size(int(gtk.gdk.screen_width() * 0.8),
 354                                    int(gtk.gdk.screen_height() * 0.8))
 355
 356                 vpaned = gtk.VPaned()
 357                 w.vbox.add(vpaned)
 358                 w.set_response_sensitive(gtk.RESPONSE_OK, False)
 359
 360                 class AutoScroller:
 361                         def __init__(self):
 362                                 tv = gtk.TextView()
 363                                 tv.set_property('left-margin', 8)
 364                                 tv.set_wrap_mode(gtk.WRAP_WORD_CHAR)
 365                                 tv.set_editable(False)
 366                                 swin = gtk.ScrolledWindow()
 367                                 swin.add(tv)
 368                                 swin.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC)
 369                                 buffer = tv.get_buffer()
 370
 371                                 heading = buffer.create_tag('heading')
 372                                 heading.set_property('scale', 1.5)
 373
 374                                 error = buffer.create_tag('error')
 375                                 error.set_property('background', 'white')
 376                                 error.set_property('foreground', 'red')
 377
 378                                 self.tv = tv
 379                                 self.widget = swin
 380                                 self.buffer = buffer
 381
 382                         def insert_at_end_and_scroll(self, data, *tags):
 383                                 vscroll = self.widget.get_vadjustment()
 384                                 if not vscroll:
 385                                         # Widget has been destroyed
 386                                         print data,
 387                                         return
 388                                 near_end = vscroll.upper - vscroll.page_size * 1.5 < vscroll.value
 389                                 end = self.buffer.get_end_iter()
 390                                 self.buffer.insert_with_tags_by_name(end, data, *tags)
 391                                 if near_end:
 392                                         cursor = self.buffer.get_insert()
 393                                         self.buffer.move_mark(cursor, end)
 394                                         self.tv.scroll_to_mark(cursor, 0, False, 0, 0)
 395
 396                         def set_text(self, text):
 397                                 self.buffer.set_text(text)
 398
 399                 self.overall = AutoScroller()
 400                 self.details = AutoScroller()
 401
 402                 vpaned.pack1(self.overall.widget, True, False)
 403                 vpaned.pack2(self.details.widget, True, False)
 404
 405                 self.closed = tasks.Blocker('Window closed')
 406
 407                 w.show_all()
 408                 w.connect('destroy', lambda wd: self.closed.trigger())
 409
 410                 def response(wd, resp):
 411                         if self.child is not None:
 412                                 self.note_error('Sending TERM signal to build process group %d...' % self.child.pid)
 413                                 os.kill(-self.child.pid, signal.SIGTERM)
 414                         else:
 415                                 self.closed.trigger()
 416                 w.connect('response', response)
 417
 418         def downloads_changed(self):
 419                 if self.config.handler.monitored_downloads:
 420                         msg = 'Downloads in progress:\n'
 421                         for x in self.config.handler.monitored_downloads:
 422                                 msg += '- {url}\n'.format(url = x.url)
 423                 else:
 424                         msg = ''
 425                 self.details.set_text(msg)
 426
 427         def heading(self, msg):
 428                 self.overall.insert_at_end_and_scroll(msg + '\n', 'heading')
 429
 430         def note(self, msg):
 431                 self.overall.insert_at_end_and_scroll(msg + '\n')
 432
 433         def note_error(self, msg):
 434                 self.overall.insert_at_end_and_scroll(msg + '\n', 'error')
 435
 436         def build(self):
 437                 import gtk
 438                 try:
 439                         tasks.wait_for_blocker(self.recursive_build(self.iface_uri))
 440                 except SafeException, ex:
 441                         self.note_error(str(ex))
 442                 else:
 443                         self.heading('All builds completed successfully!')
 444                         self.dialog.set_response_sensitive(gtk.RESPONSE_CANCEL, False)
 445                         self.dialog.set_response_sensitive(gtk.RESPONSE_OK, True)
 446
 447                 tasks.wait_for_blocker(self.closed)
 448
 449         @tasks.async
 450         def spawn_build(self, iface_name):
 451                 assert self.child is None
 452
 453                 self.details.insert_at_end_and_scroll('Building %s\n' % iface_name, 'heading')
 454
 455                 # Group all the child processes so we can kill them easily
 456                 def become_group_leader():
 457                         os.setpgid(0, 0)
 458                 devnull = os.open(os.devnull, os.O_RDONLY)
 459                 try:
 460                         self.child = subprocess.Popen([sys.executable, '-u', sys.argv[0], 'build'],
 461                                                         stdin = devnull,
 462                                                         stdout = subprocess.PIPE, stderr = subprocess.STDOUT,
 463                                                         preexec_fn = become_group_leader)
 464                 finally:
 465                         os.close(devnull)
 466
 467                 import codecs
 468                 decoder = codecs.getincrementaldecoder('utf-8')(errors = 'replace')
 469
 470                 while True:
 471                         yield tasks.InputBlocker(self.child.stdout, 'output from child')
 472                         got = os.read(self.child.stdout.fileno(), 100)
 473                         chars = decoder.decode(got, final = not got)
 474                         self.details.insert_at_end_and_scroll(chars)
 475                         if not got: break
 476
 477                 self.child.wait()
 478                 code = self.child.returncode
 479                 self.child = None
 480                 if code:
 481                         self.details.insert_at_end_and_scroll('Build process exited with error status %d\n' % code, 'error')
 482                         raise SafeException('Build process exited with error status %d' % code)
 483                 self.details.insert_at_end_and_scroll('Build completed successfully\n', 'heading')
 484
 485         @tasks.async
 486         def confirm_import_feed(self, pending, valid_sigs):
 487                 from zeroinstall.gtkui import trust_box
 488                 box = trust_box.TrustBox(pending, valid_sigs, parent = self.dialog)
 489                 box.show()
 490                 yield box.closed
 491
 492 def do_autocompile(args):
 493         """autocompile [--gui] URI"""
 494
 495         parser = OptionParser(usage="usage: %prog autocompile [options]")
 496
 497         parser.add_option('', "--gui", help="graphical interface", action='store_true')
 498         (options, args2) = parser.parse_args(args)
 499         if len(args2) != 1:
 500                 raise __main__.UsageError()
 501
 502         if options.gui:
 503                 h = GUIHandler()
 504         elif os.isatty(1):
 505                 h = handler.ConsoleHandler()
 506         else:
 507                 h = handler.Handler()
 508         config = load_config(handler = h)
 509         config._iface_cache = AutocompileCache()
 510
 511         iface_uri = model.canonical_iface_uri(args2[0])
 512         if options.gui:
 513                 compiler = GTKAutoCompiler(config, iface_uri, options)
 514         else:
 515                 compiler = AutoCompiler(config, iface_uri, options)
 516
 517         compiler.build()
 518
 519 __main__.commands += [do_autocompile]