autocompile.py

   1 # Copyright (C) 2009, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import sys, os, __main__, tempfile, shutil, subprocess, signal
   5 from xml.dom import minidom
   6 from optparse import OptionParser
   7
   8 from zeroinstall import SafeException
   9 from zeroinstall.injector import arch, handler, policy, model, iface_cache, selections, namespaces, writer, reader
  10 from zeroinstall.zerostore import manifest
  11 from zeroinstall.support import tasks, basedir
  12
  13 from support import BuildEnv
  14
  15 # This is a bit hacky...
  16 #
  17 # We invent a new CPU type which is compatible with the host but worse than
  18 # every existing type, and we use * for the OS type so that we don't beat 'Any'
  19 # binaries either. This means that we always prefer an existing binary of the
  20 # desired version to compiling a new one, but we'll compile a new version from source
  21 # rather than use an older binary.
  22 arch.machine_groups['newbuild'] = arch.machine_groups.get(arch._uname[-1], 0)
  23 arch.machine_ranks['newbuild'] = max(arch.machine_ranks.values()) + 1
  24 host_arch = '*-newbuild'
  25
  26 class AutocompileCache(iface_cache.IfaceCache):
  27         def __init__(self):
  28                 iface_cache.IfaceCache.__init__(self)
  29                 self.done = set()
  30
  31         def get_interface(self, uri):
  32                 iface = iface_cache.IfaceCache.get_interface(self, uri)
  33                 if not iface: return None
  34                 feed = iface._main_feed
  35
  36                 # Note: when a feed is updated, a new ZeroInstallFeed object is created,
  37                 # so record whether we've seen the feed, not the interface.
  38
  39                 if feed not in self.done:
  40                         self.done.add(feed)
  41
  42                         # For each source impl, add a corresponding binary
  43                         # (the binary has no dependencies as we can't predict them here,
  44                         # but they're not the same as the source's dependencies)
  45
  46                         srcs = [x for x in feed.implementations.itervalues() if x.arch and x.arch.endswith('-src')]
  47                         for x in srcs:
  48                                 new_id = '0compile=' + x.id
  49                                 if not new_id in feed.implementations:
  50                                         new = feed._get_impl(new_id)
  51                                         new.set_arch(host_arch)
  52                                         new.version = x.version
  53
  54                 return iface
  55
  56 policy.iface_cache = AutocompileCache()
  57
  58 class MyHandler(handler.Handler):
  59         def downloads_changed(self):
  60                 self.compiler.downloads_changed()
  61
  62 class AutoCompiler:
  63         def __init__(self, iface_uri, options):
  64                 self.iface_uri = iface_uri
  65                 self.options = options
  66                 self.handler = MyHandler()
  67                 self.handler.compiler = self
  68
  69         def downloads_changed(self):
  70                 if self.handler.monitored_downloads:
  71                         self.note('Downloads in progress:')
  72                         for x in self.handler.monitored_downloads:
  73                                 self.note('- %s' % x)
  74                 else:
  75                         self.note('No downloads remaining.')
  76
  77         def pretty_print_plan(self, solver, root, indent = '- '):
  78                 """Display a tree showing the selected implementations."""
  79                 iface = solver.iface_cache.get_interface(root)
  80                 impl = solver.selections[iface]
  81                 if impl is None:
  82                         msg = 'Failed to select any suitable version (source or binary)'
  83                 elif impl.id.startswith('0compile='):
  84                         real_impl_id = impl.id.split('=', 1)[1]
  85                         real_impl = impl.feed.implementations[real_impl_id]
  86                         msg = 'Compile %s (%s)' % (real_impl.get_version(), real_impl.id)
  87                 elif impl.arch and impl.arch.endswith('-src'):
  88                         msg = 'Compile %s (%s)' % (impl.get_version(), impl.id)
  89                 else:
  90                         if impl.arch:
  91                                 msg = 'Use existing binary %s (%s)' % (impl.get_version(), impl.arch)
  92                         else:
  93                                 msg = 'Use existing architecture-independent package %s' % impl.get_version()
  94                 self.note("%s%s: %s" % (indent, iface.get_name(), msg))
  95
  96                 if impl:
  97                         indent = '  ' + indent
  98                         for x in impl.requires:
  99                                 self.pretty_print_plan(solver, x.interface, indent)
 100
 101         def print_details(self, solver):
 102                 """Dump debugging details."""
 103                 self.note("\nDetails of all components and versions considered:")
 104                 for iface in solver.details:
 105                         self.note('\n%s\n' % iface.get_name())
 106                         for impl, note in solver.details[iface]:
 107                                 self.note('%s (%s) : %s' % (impl.get_version(), impl.arch or '*-*', note or 'OK'))
 108                 self.note("\nEnd details")
 109
 110         @tasks.async
 111         def compile_and_register(self, policy):
 112                 local_feed_dir = basedir.save_config_path('0install.net', '0compile', 'builds', model._pretty_escape(policy.root))
 113                 s = selections.Selections(policy)
 114
 115                 buildenv = BuildEnv(need_config = False)
 116                 buildenv.config.set('compile', 'interface', policy.root)
 117                 buildenv.config.set('compile', 'selections', 'selections.xml')
 118
 119                 version = s.selections[policy.root].version
 120                 local_feed = os.path.join(local_feed_dir, '%s-%s-%s.xml' % (buildenv.iface_name, version, arch._uname[-1]))
 121                 if os.path.exists(local_feed):
 122                         raise SafeException("Build metadata file '%s' already exists!" % local_feed)
 123
 124                 tmpdir = tempfile.mkdtemp(prefix = '0compile-')
 125                 try:
 126                         os.chdir(tmpdir)
 127
 128                         # Write configuration for build...
 129
 130                         buildenv.save()
 131
 132                         sel_file = open('selections.xml', 'w')
 133                         try:
 134                                 doc = s.toDOM()
 135                                 doc.writexml(sel_file)
 136                                 sel_file.write('\n')
 137                         finally:
 138                                 sel_file.close()
 139
 140                         # Do the build...
 141
 142                         build = self.spawn_build(buildenv.iface_name)
 143                         if build:
 144                                 yield build
 145                                 tasks.check(build)
 146
 147                         # Register the result...
 148
 149                         alg = manifest.get_algorithm('sha1new')
 150                         digest = alg.new_digest()
 151                         lines = []
 152                         for line in alg.generate_manifest(buildenv.distdir):
 153                                 line += '\n'
 154                                 digest.update(line)
 155                                 lines.append(line)
 156                         actual_digest = alg.getID(digest)
 157
 158                         local_feed_file = file(local_feed, 'w')
 159                         try:
 160                                 dom = minidom.parse(buildenv.local_iface_file)
 161                                 impl, = dom.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'implementation')
 162                                 impl.setAttribute('id', actual_digest)
 163                                 dom.writexml(local_feed_file)
 164                                 local_feed_file.write('\n')
 165                         finally:
 166                                 local_feed_file.close()
 167
 168                         self.note("Implementation metadata written to %s" % local_feed)
 169
 170                         store = policy.solver.iface_cache.stores
 171                         if self.options.local:
 172                                 store = store.stores[0]
 173                                 self.note("Storing build in local cache %s..." % store.dir)
 174                         else:
 175                                 self.note("Storing build in cache...")
 176                         policy.solver.iface_cache.stores.add_dir_to_cache(actual_digest, buildenv.distdir)
 177
 178                         self.note("Registering feed...")
 179                         iface = policy.solver.iface_cache.get_interface(policy.root)
 180                         feed = iface.get_feed(local_feed)
 181                         if feed:
 182                                 self.note("WARNING: feed %s already registered!" % local_feed)
 183                         else:
 184                                 iface.extra_feeds.append(model.Feed(local_feed, impl.getAttribute('arch'), user_override = True))
 185                         writer.save_interface(iface)
 186
 187                         # We might have cached an old version
 188                         new_feed = policy.solver.iface_cache.get_interface(local_feed)
 189                         reader.update_from_cache(new_feed)
 190                 except:
 191                         self.note("\nBuild failed: leaving build directory %s for inspection...\n" % tmpdir)
 192                         raise
 193                 else:
 194                         shutil.rmtree(tmpdir)
 195
 196         @tasks.async
 197         def recursive_build(self, iface_uri, version = None):
 198                 p = policy.Policy(iface_uri, handler = self.handler, src = True)
 199                 iface = p.solver.iface_cache.get_interface(iface_uri)
 200                 p.solver.record_details = True
 201                 if version:
 202                         p.solver.extra_restrictions[iface] = [model.VersionRestriction(model.parse_version(version))]
 203
 204                 # For testing...
 205                 #p.target_arch = arch.Architecture(os_ranks = {'FreeBSD': 0, None: 1}, machine_ranks = {'i386': 0, None: 1, 'newbuild': 2})
 206
 207                 while True:
 208                         self.heading(iface_uri)
 209                         self.note("\nSelecting versions for %s..." % iface.get_name())
 210                         solved = p.solve_with_downloads()
 211                         if solved:
 212                                 yield solved
 213                                 tasks.check(solved)
 214
 215                         if not p.solver.ready:
 216                                 self.print_details(p.solver)
 217                                 raise SafeException("Can't find all required implementations (source or binary):\n" +
 218                                         '\n'.join(["- %s -> %s" % (iface, p.solver.selections[iface])
 219                                                    for iface in p.solver.selections]))
 220                         self.note("Selection done.")
 221
 222                         self.note("\nPlan:\n")
 223                         self.pretty_print_plan(p.solver, p.root)
 224                         self.note('')
 225
 226                         for dep_iface, dep_impl in p.solver.selections.iteritems():
 227                                 if dep_impl.id.startswith('0compile='):
 228                                         build = self.recursive_build(dep_iface.uri, dep_impl.get_version())
 229                                         yield build
 230                                         tasks.check(build)
 231                                         break   # Try again with that dependency built...
 232                         else:
 233                                 self.note("No dependencies need compiling... compile %s itself..." % iface.get_name())
 234                                 build = self.compile_and_register(p)
 235                                 yield build
 236                                 tasks.check(build)
 237                                 return
 238
 239         def spawn_build(self, iface_name):
 240                 subprocess.check_call([sys.executable, sys.argv[0], 'build'])
 241
 242         def build(self):
 243                 self.handler.wait_for_blocker(self.recursive_build(self.iface_uri))
 244
 245         def heading(self, msg):
 246                 self.note((' %s ' % msg).center(76, '='))
 247
 248         def note(self, msg):
 249                 print msg
 250
 251         def note_error(self, msg):
 252                 self.overall.insert_at_cursor(msg + '\n')
 253
 254 class GTKAutoCompiler(AutoCompiler):
 255         def __init__(self, iface_uri, options):
 256                 AutoCompiler.__init__(self, iface_uri, options)
 257                 self.child = None
 258
 259                 import pygtk; pygtk.require('2.0')
 260                 import gtk
 261
 262                 w = gtk.Dialog('Autocompile %s' % iface_uri, None, 0,
 263                                                  (gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,
 264                                                   gtk.STOCK_OK, gtk.RESPONSE_OK))
 265                 self.dialog = w
 266
 267                 w.set_default_size(int(gtk.gdk.screen_width() * 0.8),
 268                                    int(gtk.gdk.screen_height() * 0.8))
 269
 270                 vpaned = gtk.VPaned()
 271                 w.vbox.add(vpaned)
 272                 w.set_response_sensitive(gtk.RESPONSE_OK, False)
 273
 274                 class AutoScroller:
 275                         def __init__(self):
 276                                 tv = gtk.TextView()
 277                                 tv.set_property('left-margin', 8)
 278                                 tv.set_wrap_mode(gtk.WRAP_WORD_CHAR)
 279                                 tv.set_editable(False)
 280                                 swin = gtk.ScrolledWindow()
 281                                 swin.add(tv)
 282                                 swin.set_policy(gtk.POLICY_NEVER, gtk.POLICY_AUTOMATIC)
 283                                 buffer = tv.get_buffer()
 284
 285                                 heading = buffer.create_tag('heading')
 286                                 heading.set_property('scale', 1.5)
 287
 288                                 error = buffer.create_tag('error')
 289                                 error.set_property('background', 'white')
 290                                 error.set_property('foreground', 'red')
 291
 292                                 self.tv = tv
 293                                 self.widget = swin
 294                                 self.buffer = buffer
 295
 296                         def insert_at_end_and_scroll(self, data, *tags):
 297                                 # TODO: data might not be complete UTF-8; buffer until EOL?
 298                                 self.vscroll = self.widget.get_vadjustment()
 299                                 near_end = self.vscroll.upper - self.vscroll.page_size * 1.5 < self.vscroll.value
 300                                 end = self.buffer.get_end_iter()
 301                                 self.buffer.insert_with_tags_by_name(end, data, *tags)
 302                                 if near_end:
 303                                         cursor = self.buffer.get_insert()
 304                                         self.buffer.move_mark(cursor, end)
 305                                         self.tv.scroll_to_mark(cursor, 0, False, 0, 0)
 306
 307                 self.overall = AutoScroller()
 308                 self.details = AutoScroller()
 309
 310                 vpaned.pack1(self.overall.widget, True, False)
 311                 vpaned.pack2(self.details.widget, True, False)
 312
 313                 self.closed = tasks.Blocker('Window closed')
 314
 315                 w.show_all()
 316                 w.connect('destroy', lambda wd: self.closed.trigger())
 317
 318                 def response(wd, resp):
 319                         if self.child is not None:
 320                                 self.note_error('Sending TERM signal to build process group %d...' % self.child.pid)
 321                                 os.kill(-self.child.pid, signal.SIGTERM)
 322                         else:
 323                                 self.closed.trigger()
 324                 w.connect('response', response)
 325
 326         def heading(self, msg):
 327                 self.overall.insert_at_end_and_scroll(msg + '\n', 'heading')
 328
 329         def note(self, msg):
 330                 self.overall.insert_at_end_and_scroll(msg + '\n')
 331
 332         def note_error(self, msg):
 333                 self.overall.insert_at_end_and_scroll(msg + '\n', 'error')
 334
 335         def build(self):
 336                 import gtk
 337                 try:
 338                         self.handler.wait_for_blocker(self.recursive_build(self.iface_uri))
 339                 except SafeException, ex:
 340                         self.note_error(str(ex))
 341                 else:
 342                         self.heading('All builds completed successfully!')
 343                         self.dialog.set_response_sensitive(gtk.RESPONSE_CANCEL, False)
 344                         self.dialog.set_response_sensitive(gtk.RESPONSE_OK, True)
 345
 346                 self.handler.wait_for_blocker(self.closed)
 347
 348         @tasks.async
 349         def spawn_build(self, iface_name):
 350                 assert self.child is None
 351
 352                 self.details.insert_at_end_and_scroll('Building %s\n' % iface_name, 'heading')
 353
 354                 # Group all the child processes so we can kill them easily
 355                 def become_group_leader():
 356                         os.setpgid(0, 0)
 357                 self.child = subprocess.Popen([sys.executable, sys.argv[0], 'build'],
 358                                                 stdout = subprocess.PIPE, stderr = subprocess.STDOUT,
 359                                                 preexec_fn = become_group_leader)
 360
 361                 while True:
 362                         yield tasks.InputBlocker(self.child.stdout, 'output from child')
 363                         got = os.read(self.child.stdout.fileno(), 100)
 364                         if not got: break
 365                         self.details.insert_at_end_and_scroll(got)
 366
 367                 self.child.wait()
 368                 code = self.child.returncode
 369                 self.child = None
 370                 if code:
 371                         self.details.insert_at_end_and_scroll('Build process exited with error status %d\n' % code, 'error')
 372                         raise SafeException('Build process exited with error status %d' % code)
 373                 self.details.insert_at_end_and_scroll('Build completed successfully\n', 'heading')
 374
 375 def do_autocompile(args):
 376         """autocompile [--gui] URI"""
 377
 378         parser = OptionParser(usage="usage: %prog autocompile [options]")
 379
 380         parser.add_option('', "--gui", help="graphical interface", action='store_true')
 381         parser.add_option('', "--local", help="store in user cache, not system cache", action='store_true')
 382         (options, args2) = parser.parse_args(args)
 383         if len(args2) != 1:
 384                 raise __main__.UsageError()
 385
 386         iface_uri = model.canonical_iface_uri(args2[0])
 387         if options.gui:
 388                 compiler = GTKAutoCompiler(iface_uri, options)
 389         else:
 390                 compiler = AutoCompiler(iface_uri, options)
 391
 392         compiler.build()
 393
 394 __main__.commands += [do_autocompile]