autocompile.py

   1 # Copyright (C) 2009, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import sys, os, __main__, tempfile, shutil, subprocess
   5 from xml.dom import minidom
   6
   7 from zeroinstall import SafeException
   8 from zeroinstall.injector import arch, handler, policy, model, iface_cache, selections, namespaces, writer, reader
   9 from zeroinstall.zerostore import manifest
  10 from zeroinstall.support import tasks, basedir
  11
  12 from support import BuildEnv
  13
  14 # This is a bit hacky...
  15 #
  16 # We invent a new CPU type which is compatible with the host but worse than
  17 # every existing type, and we use * for the OS type so that we don't beat 'Any'
  18 # binaries either. This means that we always prefer an existing binary of the
  19 # desired version to compiling a new one, but we'll compile a new version from source
  20 # rather than use an older binary.
  21 arch.machine_groups['newbuild'] = arch.machine_groups.get(arch._uname[-1], 0)
  22 arch.machine_ranks['newbuild'] = max(arch.machine_ranks.values()) + 1
  23 host_arch = '*-newbuild'
  24
  25 # 0launch 0.40 not released yet, so include a copy here
  26 class VersionRestriction(model.Restriction):
  27         """Only select implementations with a particular version number."""
  28
  29         def __init__(self, version):
  30                 """@param version: the required version number
  31                 @see: L{parse_version}; use this to pre-process the version number
  32                 """
  33                 self.version = version
  34
  35         def meets_restriction(self, impl):
  36                 return impl.version == self.version
  37
  38         def __str__(self):
  39                 return "(restriction: version = %s)" % model.format_version(self.version)
  40
  41 class AutocompileCache(iface_cache.IfaceCache):
  42         def __init__(self):
  43                 iface_cache.IfaceCache.__init__(self)
  44                 self.done = set()
  45
  46         def get_interface(self, uri):
  47                 iface = iface_cache.IfaceCache.get_interface(self, uri)
  48                 if not iface: return None
  49                 feed = iface._main_feed
  50
  51                 # Note: when a feed is updated, a new ZeroInstallFeed object is created,
  52                 # so record whether we've seen the feed, not the interface.
  53
  54                 if feed not in self.done:
  55                         self.done.add(feed)
  56
  57                         # For each source impl, add a corresponding binary
  58                         # (the binary has no dependencies as we can't predict them here,
  59                         # but they're not the same as the source's dependencies)
  60
  61                         srcs = [x for x in feed.implementations.itervalues() if x.arch and x.arch.endswith('-src')]
  62                         for x in srcs:
  63                                 new_id = '0compile=' + x.id
  64                                 if not new_id in feed.implementations:
  65                                         new = feed._get_impl(new_id)
  66                                         new.set_arch(host_arch)
  67                                         new.version = x.version
  68
  69                 return iface
  70
  71 policy.iface_cache = AutocompileCache()
  72
  73 def pretty_print_plan(solver, root, indent = '- '):
  74         """Display a tree showing the selected implementations."""
  75         iface = solver.iface_cache.get_interface(root)
  76         impl = solver.selections[iface]
  77         if impl is None:
  78                 msg = 'Failed to select any suitable version (source or binary)'
  79         elif impl.id.startswith('0compile='):
  80                 real_impl_id = impl.id.split('=', 1)[1]
  81                 real_impl = impl.feed.implementations[real_impl_id]
  82                 msg = 'Compile %s (%s)' % (real_impl.get_version(), real_impl.id)
  83         elif impl.arch and impl.arch.endswith('-src'):
  84                 msg = 'Compile %s (%s)' % (impl.get_version(), impl.id)
  85         else:
  86                 if impl.arch:
  87                         msg = 'Use existing binary %s (%s)' % (impl.get_version(), impl.arch)
  88                 else:
  89                         msg = 'Use existing architecture-independent package %s' % impl.get_version()
  90         print "%s%s: %s" % (indent, iface.get_name(), msg)
  91
  92         if impl:
  93                 indent = '  ' + indent
  94                 for x in impl.requires:
  95                         pretty_print_plan(solver, x.interface, indent)
  96
  97 def print_details(solver):
  98         """Dump debugging details."""
  99         print "\nDetails of all components and versions considered:"
 100         for iface in solver.details:
 101                 print '\n%s\n' % iface.get_name()
 102                 for impl, note in solver.details[iface]:
 103                         print '%s (%s) : %s' % (impl.get_version(), impl.arch or '*-*', note or 'OK')
 104         print "\nEnd details"
 105
 106 def compile_and_register(policy):
 107         local_feed_dir = basedir.save_config_path('0install.net', '0compile', 'builds', model._pretty_escape(policy.root))
 108         s = selections.Selections(policy)
 109
 110         buildenv = BuildEnv(need_config = False)
 111         buildenv.config.set('compile', 'interface', policy.root)
 112         buildenv.config.set('compile', 'selections', 'selections.xml')
 113
 114         version = s.selections[policy.root].version
 115         local_feed = os.path.join(local_feed_dir, '%s-%s-%s.xml' % (buildenv.iface_name, version, arch._uname[-1]))
 116         if os.path.exists(local_feed):
 117                 raise SafeException("Build metadata file '%s' already exists!" % local_feed)
 118
 119         tmpdir = tempfile.mkdtemp(prefix = '0compile-')
 120         try:
 121                 os.chdir(tmpdir)
 122
 123                 # Write configuration for build...
 124
 125                 buildenv.save()
 126
 127                 sel_file = open('selections.xml', 'w')
 128                 try:
 129                         doc = s.toDOM()
 130                         doc.writexml(sel_file)
 131                         sel_file.write('\n')
 132                 finally:
 133                         sel_file.close()
 134
 135                 # Do the build...
 136
 137                 subprocess.check_call([sys.executable, sys.argv[0], 'build'])
 138
 139                 # Register the result...
 140
 141                 alg = manifest.get_algorithm('sha1new')
 142                 digest = alg.new_digest()
 143                 lines = []
 144                 for line in alg.generate_manifest(buildenv.distdir):
 145                         line += '\n'
 146                         digest.update(line)
 147                         lines.append(line)
 148                 actual_digest = alg.getID(digest)
 149
 150                 local_feed_file = file(local_feed, 'w')
 151                 try:
 152                         dom = minidom.parse(buildenv.local_iface_file)
 153                         impl, = dom.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'implementation')
 154                         impl.setAttribute('id', actual_digest)
 155                         dom.writexml(local_feed_file)
 156                         local_feed_file.write('\n')
 157                 finally:
 158                         local_feed_file.close()
 159
 160                 print "Implementation metadata written to %s" % local_feed
 161
 162                 print "Storing build in cache..."
 163                 policy.solver.iface_cache.stores.add_dir_to_cache(actual_digest, buildenv.distdir)
 164
 165                 print "Registering feed..."
 166                 iface = policy.solver.iface_cache.get_interface(policy.root)
 167                 feed = iface.get_feed(local_feed)
 168                 if feed:
 169                         print "WARNING: feed %s already registered!" % local_feed
 170                 else:
 171                         iface.extra_feeds.append(model.Feed(local_feed, impl.getAttribute('arch'), user_override = True))
 172                 writer.save_interface(iface)
 173
 174                 # We might have cached an old version
 175                 new_feed = policy.solver.iface_cache.get_interface(local_feed)
 176                 reader.update_from_cache(new_feed)
 177         except:
 178                 print "\nBuild failed: leaving build directory %s for inspection...\n" % tmpdir
 179                 raise
 180         else:
 181                 shutil.rmtree(tmpdir)
 182
 183 def do_autocompile(args):
 184         """autocompile URI"""
 185         if len(args) != 1:
 186                 raise __main__.UsageError()
 187         iface_uri = model.canonical_iface_uri(args[0])
 188
 189         h = handler.Handler()
 190
 191         @tasks.async
 192         def recursive_build(iface_uri, version = None):
 193                 p = policy.Policy(iface_uri, handler = h, src = True)
 194                 iface = p.solver.iface_cache.get_interface(iface_uri)
 195                 p.solver.record_details = True
 196                 if version:
 197                         p.solver.extra_restrictions[iface] = [VersionRestriction(model.parse_version(version))]
 198
 199                 # For testing...
 200                 #p.target_arch = arch.Architecture(os_ranks = {'FreeBSD': 0, None: 1}, machine_ranks = {'i386': 0, None: 1, 'newbuild': 2})
 201
 202                 print (' %s ' % iface_uri).center(76, '=')
 203
 204                 while True:
 205                         print "\nSelecting versions for %s..." % iface.get_name()
 206                         solved = p.solve_with_downloads()
 207                         if solved:
 208                                 yield solved
 209                                 tasks.check(solved)
 210
 211                         if not p.solver.ready:
 212                                 print_details(p.solver)
 213                                 raise SafeException("Can't find all required implementations (source or binary):\n" +
 214                                         '\n'.join(["- %s -> %s" % (iface, p.solver.selections[iface])
 215                                                    for iface in p.solver.selections]))
 216                         print "Selection done."
 217
 218                         print "\nPlan:\n"
 219                         pretty_print_plan(p.solver, p.root)
 220                         print
 221
 222                         for dep_iface, dep_impl in p.solver.selections.iteritems():
 223                                 if dep_impl.id.startswith('0compile='):
 224                                         build = recursive_build(dep_iface.uri, dep_impl.get_version())
 225                                         yield build
 226                                         tasks.check(build)
 227                                         break   # Try again with that dependency built...
 228                         else:
 229                                 print "No dependencies need compiling... compile %s itself..." % iface.get_name()
 230                                 compile_and_register(p)
 231                                 return
 232
 233         h.wait_for_blocker(recursive_build(iface_uri))
 234
 235 __main__.commands += [do_autocompile]