support.py

   1 # Copyright (C) 2006, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import os, sys, shutil
   5 import subprocess
   6 from os.path import join
   7 from logging import info
   8 import ConfigParser
   9
  10 from zeroinstall.injector import model, selections, qdom, arch
  11 from zeroinstall.injector.arch import canonicalize_os, canonicalize_machine
  12
  13 from zeroinstall.injector.iface_cache import iface_cache
  14 from zeroinstall import SafeException
  15 from zeroinstall.zerostore import Store, NotStored
  16
  17 def _(x): return x
  18
  19
  20 # This is An os.uname() substitute that uses as much of ZI's
  21 # arch._uname as is available and yet has all four elements one
  22 # normally expects from os.uname() on Posix (on Windows, arch._uname
  23 # has only two elements).
  24 import platform
  25 uname = arch._uname + platform.uname()[len(arch._uname):]
  26
  27 ENV_FILE = '0compile.properties'
  28 XMLNS_0COMPILE = 'http://zero-install.sourceforge.net/2006/namespaces/0compile'
  29
  30 zeroinstall_dir = os.environ.get('0COMPILE_ZEROINSTALL', None)
  31 if zeroinstall_dir:
  32         # XXX: we're assuming that, if installed through 0install, 0launch requires
  33         # the same version of Python as 0compile. This is currently needed for Arch
  34         # Linux, but long-term we need to use the <runner>.
  35         install_prog = [sys.executable, os.path.join(zeroinstall_dir, '0install')]
  36         if not os.path.exists(install_prog[1]):
  37                 # For the Windows version...
  38                 install_prog[1] = os.path.join(zeroinstall_dir, 'zeroinstall', 'scripts', 'install.py')
  39 else:
  40         install_prog = ['0install']
  41
  42 if os.path.isdir('dependencies'):
  43         dep_dir = os.path.realpath('dependencies')
  44         iface_cache.stores.stores.append(Store(dep_dir))
  45         install_prog.append('--with-store='+ dep_dir)
  46
  47 class NoImpl:
  48         id = "none"
  49         version = "none"
  50 no_impl = NoImpl()
  51
  52 def is_package_impl(impl):
  53         return impl.id.startswith("package:")
  54
  55 def lookup(impl_or_sel):
  56         id = impl_or_sel.id
  57         if id.startswith('package:'):
  58                 return None
  59         local_path = impl_or_sel.local_path
  60         if local_path is not None:
  61                 if os.path.isdir(local_path):
  62                         return local_path
  63                 raise SafeException("Directory '%s' no longer exists. Try '0compile setup'" % local_path)
  64         try:
  65                 return iface_cache.stores.lookup_any(impl_or_sel.digests)
  66         except NotStored, ex:
  67                 raise NotStored(str(ex) + "\nHint: try '0compile setup'")
  68
  69 def ensure_dir(d, clean = False):
  70         if os.path.isdir(d):
  71                 if clean:
  72                         print "Removing", d
  73                         shutil.rmtree(d)
  74                 else:
  75                         return
  76         if os.path.exists(d):
  77                 raise SafeException("'%s' exists, but is not a directory!" % d)
  78         os.mkdir(d)
  79
  80 def find_in_path(prog):
  81         for d in os.environ['PATH'].split(':'):
  82                 path = os.path.join(d, prog)
  83                 if os.path.isfile(path):
  84                         return path
  85         return None
  86
  87 def spawn_and_check(prog, args):
  88         status = os.spawnv(os.P_WAIT, prog, [prog] + args)
  89         if status > 0:
  90                 raise SafeException("Program '%s' failed with exit code %d" % (prog, status))
  91         elif status < 0:
  92                 raise SafeException("Program '%s' failed with signal %d" % (prog, -status))
  93
  94 def spawn_and_check_maybe_sandboxed(readable, writable, tmpdir, prog, args):
  95         child = spawn_maybe_sandboxed(readable, writable, tmpdir, prog, args)
  96         status = child.wait()
  97         if status > 0:
  98                 raise SafeException('Command failed with exit status %d' %  status)
  99         elif status < 0:
 100                 raise SafeException('Command failed with signal %d' % -status)
 101
 102 def spawn_maybe_sandboxed(readable, writable, tmpdir, prog, args):
 103         """spawn prog, with (only) the 'writable' directories writable if sandboxing is available.
 104         The readable directories will be readable, as well as various standard locations.
 105         If no sandbox is available, run without a sandbox."""
 106
 107         USE_PLASH = 'USE_PLASH_0COMPILE'
 108
 109         assert os.path.isabs(prog)
 110         _pola_run = find_in_path('pola-run')
 111
 112         if _pola_run is None:
 113                 print "Not using sandbox (plash not installed)"
 114                 use_plash = False
 115         else:
 116                 use_plash = os.environ.get(USE_PLASH, '').lower() or 'not set'
 117                 if use_plash in ('not set', 'false'):
 118                         print "Not using plash: $%s is %s" % (USE_PLASH, use_plash)
 119                         use_plash = False
 120                 elif use_plash == 'true':
 121                         use_plash = True
 122                 else:
 123                         raise Exception('$%s must be "true" or "false", not "%s"' % (USE_PLASH, use_plash))
 124
 125         if not use_plash:
 126                 return subprocess.Popen([prog] + args)
 127
 128         print "Using plash to sandbox the build..."
 129
 130         # We have pola-shell :-)
 131         pola_args = ['--prog', prog, '-B']
 132         for a in args:
 133                 pola_args += ['-a', a]
 134         for r in readable:
 135                 pola_args += ['-f', r]
 136         for w in writable:
 137                 pola_args += ['-fw', w]
 138         pola_args += ['-tw', '/tmp', tmpdir]
 139         os.environ['TMPDIR'] = '/tmp'
 140         return subprocess.Popen([_pola_run] + pola_args)
 141
 142 def get_arch_name():
 143         target_os = canonicalize_os(uname[0])
 144         target_machine = canonicalize_machine(uname[4])
 145         if target_os == 'Darwin' and target_machine == 'i386':
 146                 # this system detection shell script comes from config.guess (20090918):
 147                 CC = os.getenv("CC_FOR_BUILD") or os.getenv("CC") or os.getenv("HOST_CC") or "cc"
 148                 process = subprocess.Popen("(echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | " +
 149                                            "(CCOPTS= %s -E - 2>/dev/null) | " % CC +
 150                                            "grep IS_64BIT_ARCH >/dev/null", stdout=subprocess.PIPE, shell=True)
 151                 output, error = process.communicate()
 152                 retcode = process.poll()
 153                 if retcode == 0:
 154                         target_machine='x86_64'
 155         if target_machine in ('i585', 'i686'):
 156                 target_machine = 'i486' # (sensible default)
 157         return target_os + '-' + target_machine
 158
 159 class BuildEnv:
 160         def __init__(self, need_config = True):
 161                 if need_config and not os.path.isfile(ENV_FILE):
 162                         raise SafeException("Run 0compile from a directory containing a '%s' file" % ENV_FILE)
 163
 164                 self.config = ConfigParser.RawConfigParser()
 165                 self.config.add_section('compile')
 166                 self.config.set('compile', 'download-base-url', '')
 167                 self.config.set('compile', 'version-modifier', '')
 168                 self.config.set('compile', 'interface', '')
 169                 self.config.set('compile', 'selections', '')
 170                 self.config.set('compile', 'metadir', '0install')
 171                 self.config.set('compile', 'distdir', '')
 172
 173                 self.config.read(ENV_FILE)
 174
 175                 self._selections = None
 176
 177                 return
 178
 179         @property
 180         def iface_name(self):
 181                 iface_name = os.path.basename(self.interface)
 182                 if iface_name.endswith('.xml'):
 183                         iface_name = iface_name[:-4]
 184                 iface_name = iface_name.replace(' ', '-')
 185                 if iface_name.endswith('-src'):
 186                         iface_name = iface_name[:-4]
 187                 return iface_name
 188
 189         interface = property(lambda self: model.canonical_iface_uri(self.config.get('compile', 'interface')))
 190
 191         @property
 192         def distdir(self):
 193                 distdir_name = self.config.get('compile', 'distdir')
 194                 if not distdir_name:
 195                         arch = self.target_arch.replace('*', 'any')
 196                         distdir_name = self.iface_name.lower()
 197                         distdir_name += '-' + arch.lower()
 198                 assert os.path.dirname(distdir_name) == ''
 199                 return os.path.realpath(distdir_name)
 200
 201         def get_binary_template(self):
 202                 """Find the <compile:implementation> element for the selected compile command, if any"""
 203                 sels = self.get_selections()
 204                 if sels.commands:
 205                         for elem in sels.commands[0].qdom.childNodes:
 206                                 if elem.uri == XMLNS_0COMPILE and elem.name == 'implementation':
 207                                         return elem
 208
 209                                 # XXX: hack for 0launch < 0.54 which messes up the namespace
 210                                 if elem.name == 'implementation':
 211                                         return elem
 212                 return None
 213
 214         @property
 215         def metadir(self):
 216                 metadir = self.config.get('compile', 'metadir')
 217                 assert not os.path.isabs(metadir)
 218                 return join(self.distdir, metadir)
 219
 220         @property
 221         def local_iface_file(self):
 222                 return join(self.metadir, self.iface_name + '.xml')
 223
 224         @property
 225         def target_arch(self):
 226                 temp = self.get_binary_template()
 227                 arch = temp and temp.getAttribute('arch')
 228                 return arch or get_arch_name()
 229
 230         @property
 231         def version_modifier(self):
 232                 vm = self.config.get('compile', 'version-modifier')
 233                 if vm: return vm
 234                 if self.user_srcdir:
 235                         return '-1'
 236                 return ''
 237
 238         @property
 239         def archive_stem(self):
 240                 # Use the version that we actually built, not the version we would build now
 241                 feed = self.load_built_feed()
 242                 assert len(feed.implementations) == 1
 243                 version = feed.implementations.values()[0].get_version()
 244
 245                 # Don't use the feed's name, as it may contain the version number
 246                 name = feed.get_name().lower().replace(' ', '-')
 247                 arch = self.target_arch.lower().replace('*-*', 'bin').replace('*', 'any')
 248
 249                 return '%s-%s-%s' % (name, arch, version)
 250
 251         def load_built_feed(self):
 252                 path = self.local_iface_file
 253                 stream = file(path)
 254                 try:
 255                         feed = model.ZeroInstallFeed(qdom.parse(stream), local_path = path)
 256                 finally:
 257                         stream.close()
 258                 return feed
 259
 260         def load_built_selections(self):
 261                 path = join(self.metadir, 'build-environment.xml')
 262                 if os.path.exists(path):
 263                         stream = file(path)
 264                         try:
 265                                 return selections.Selections(qdom.parse(stream))
 266                         finally:
 267                                 stream.close()
 268                 return None
 269
 270         @property
 271         def download_base_url(self):
 272                 return self.config.get('compile', 'download-base-url')
 273
 274         def chosen_impl(self, uri):
 275                 sels = self.get_selections()
 276                 assert uri in sels.selections
 277                 return sels.selections[uri]
 278
 279         @property
 280         def local_download_iface(self):
 281                 impl, = self.load_built_feed().implementations.values()
 282                 return '%s-%s.xml' % (self.iface_name, impl.get_version())
 283
 284         def save(self):
 285                 stream = file(ENV_FILE, 'w')
 286                 try:
 287                         self.config.write(stream)
 288                 finally:
 289                         stream.close()
 290
 291         def get_selections(self, prompt = False):
 292                 if self._selections:
 293                         assert not prompt
 294                         return self._selections
 295
 296                 selections_file = self.config.get('compile', 'selections')
 297                 if selections_file:
 298                         if prompt:
 299                                 raise SafeException("Selections are fixed by %s" % selections_file)
 300                         stream = file(selections_file)
 301                         try:
 302                                 self._selections = selections.Selections(qdom.parse(stream))
 303                         finally:
 304                                 stream.close()
 305                         from zeroinstall.injector import handler, policy
 306                         if os.isatty(1):
 307                                 h = handler.ConsoleHandler()
 308                         else:
 309                                 h = handler.Handler()
 310                         config = policy.load_config(h)
 311                         blocker = self._selections.download_missing(config)
 312                         if blocker:
 313                                 print "Waiting for selected implementations to be downloaded..."
 314                                 h.wait_for_blocker(blocker)
 315                 else:
 316                         command = install_prog + ['download', '--source', '--xml']
 317                         if prompt and '--console' not in install_prog:
 318                                 if os.name == 'nt':
 319                                         command[0] += '-win'
 320                                 command.append('--gui')
 321                         command.append(self.interface)
 322                         child = subprocess.Popen(command, stdout = subprocess.PIPE)
 323                         try:
 324                                 self._selections = selections.Selections(qdom.parse(child.stdout))
 325                         finally:
 326                                 if child.wait():
 327                                         raise SafeException(' '.join(repr(x) for x in command) + " failed (exit code %d)" % child.returncode)
 328
 329                 self.root_impl = self._selections.selections[self.interface]
 330
 331                 self.orig_srcdir = os.path.realpath(lookup(self.root_impl))
 332                 self.user_srcdir = None
 333
 334                 if os.path.isdir('src'):
 335                         self.user_srcdir = os.path.realpath('src')
 336                         if self.user_srcdir == self.orig_srcdir or \
 337                            self.user_srcdir.startswith(os.path.join(self.orig_srcdir, '')) or \
 338                            self.orig_srcdir.startswith(os.path.join(self.user_srcdir, '')):
 339                                 info("Ignoring 'src' directory because it coincides with %s",
 340                                         self.orig_srcdir)
 341                                 self.user_srcdir = None
 342
 343                 return self._selections
 344
 345         def get_build_changes(self):
 346                 sels = self.get_selections()
 347                 old_sels = self.load_built_selections()
 348                 changes = []
 349                 if old_sels:
 350                         # See if things have changed since the last build
 351                         all_ifaces = set(sels.selections) | set(old_sels.selections)
 352                         for x in all_ifaces:
 353                                 old_impl = old_sels.selections.get(x, no_impl)
 354                                 new_impl = sels.selections.get(x, no_impl)
 355                                 if old_impl.version != new_impl.version:
 356                                         changes.append("Version change for %s: %s -> %s" % (x, old_impl.version, new_impl.version))
 357                                 elif old_impl.id != new_impl.id:
 358                                         changes.append("Version change for %s: %s -> %s" % (x, old_impl.id, new_impl.id))
 359                 return changes
 360
 361 def depth(node):
 362         root = node.ownerDocument.documentElement
 363         depth = 0
 364         while node and node is not root:
 365                 node = node.parentNode
 366                 depth += 1
 367         return depth
 368
 369 def parse_bool(s):
 370         if s == 'true': return True
 371         if s == 'false': return False
 372         raise SafeException('Expected "true" or "false" but got "%s"' % s)
 373
 374 class Prefixes:
 375         # Copied from 0launch 0.54 (remove once 0.54 is released)
 376         def __init__(self, default_ns):
 377                 self.prefixes = {}
 378                 self.default_ns = default_ns
 379
 380         def get(self, ns):
 381                 prefix = self.prefixes.get(ns, None)
 382                 if prefix:
 383                         return prefix
 384                 prefix = 'ns%d' % len(self.prefixes)
 385                 self.prefixes[ns] = prefix
 386                 return prefix
 387
 388         def setAttributeNS(self, elem, uri, localName, value):
 389                 if uri is None:
 390                         elem.setAttributeNS(None, localName, value)
 391                 else:
 392                         elem.setAttributeNS(uri, self.get(uri) + ':' + localName, value)
 393
 394         def createElementNS(self, doc, uri, localName):
 395                 if uri == self.default_ns:
 396                         return doc.createElementNS(uri, localName)
 397                 else:
 398                         return doc.createElementNS(uri, self.get(uri) + ':' + localName)