support.py

   1 # Copyright (C) 2006, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import os, sys, shutil, traceback
   5 import subprocess
   6 from os.path import join
   7 from logging import info
   8 import ConfigParser
   9
  10 from zeroinstall.injector import model, selections, qdom
  11 from zeroinstall.injector.arch import canonicalize_os, canonicalize_machine
  12
  13 from zeroinstall.injector.iface_cache import iface_cache
  14 from zeroinstall import SafeException
  15 from zeroinstall.zerostore import Store, NotStored
  16
  17 def _(x): return x
  18
  19 ENV_FILE = '0compile.properties'
  20 XMLNS_0COMPILE = 'http://zero-install.sourceforge.net/2006/namespaces/0compile'
  21
  22 zeroinstall_dir = os.environ.get('0COMPILE_ZEROINSTALL', None)
  23 if zeroinstall_dir:
  24         # XXX: we're assuming that, if installed through 0install, 0launch requires
  25         # the same version of Python as 0compile. This is currently needed for Arch
  26         # Linux, but long-term we need to use the <runner>.
  27         launch_prog = [sys.executable, os.path.join(zeroinstall_dir, '0launch')]
  28 else:
  29         launch_prog = ['0launch']
  30
  31 if os.path.isdir('dependencies'):
  32         dep_dir = os.path.realpath('dependencies')
  33         iface_cache.stores.stores.append(Store(dep_dir))
  34         launch_prog += ['--with-store', dep_dir]
  35
  36 class NoImpl:
  37         id = "none"
  38         version = "none"
  39 no_impl = NoImpl()
  40
  41 def is_package_impl(impl):
  42         return impl.id.startswith("package:")
  43
  44 def lookup(impl_or_sel):
  45         id = impl_or_sel.id
  46         if id.startswith('/'):
  47                 if os.path.isdir(id):
  48                         return id
  49                 raise SafeException("Directory '%s' no longer exists. Try '0compile setup'" % id)
  50         try:
  51                 return iface_cache.stores.lookup_any(impl_or_sel.digests)
  52         except NotStored, ex:
  53                 raise NotStored(str(ex) + "\nHint: try '0compile setup'")
  54
  55 def ensure_dir(d, clean = False):
  56         if os.path.isdir(d):
  57                 if clean:
  58                         print "Removing", d
  59                         shutil.rmtree(d)
  60                 else:
  61                         return
  62         if os.path.exists(d):
  63                 raise SafeException("'%s' exists, but is not a directory!" % d)
  64         os.mkdir(d)
  65
  66 def find_in_path(prog):
  67         for d in os.environ['PATH'].split(':'):
  68                 path = os.path.join(d, prog)
  69                 if os.path.isfile(path):
  70                         return path
  71         return None
  72
  73 def spawn_and_check(prog, args):
  74         status = os.spawnv(os.P_WAIT, prog, [prog] + args)
  75         if status > 0:
  76                 raise SafeException("Program '%s' failed with exit code %d" % (prog, status))
  77         elif status < 0:
  78                 raise SafeException("Program '%s' failed with signal %d" % (prog, -status))
  79
  80 def wait_for_child(child):
  81         """Wait for child to exit and reap it. Throw an exception if it doesn't return success."""
  82         pid, status = os.waitpid(child, 0)
  83         assert pid == child
  84         if os.WIFEXITED(status):
  85                 exit_code = os.WEXITSTATUS(status)
  86                 if exit_code == 0:
  87                         return
  88                 else:
  89                         raise SafeException('Command failed with exit status %d' % exit_code)
  90         else:
  91                 raise SafeException('Command failed with signal %d' % os.WTERMSIG(status))
  92
  93 def spawn_maybe_sandboxed(readable, writable, tmpdir, prog, args):
  94         child = os.fork()
  95         if child == 0:
  96                 try:
  97                         try:
  98                                 exec_maybe_sandboxed(readable, writable, tmpdir, prog, args)
  99                         except:
 100                                 traceback.print_exc()
 101                 finally:
 102                         print >>sys.stderr, "Exec failed"
 103                         os._exit(1)
 104         wait_for_child(child)
 105
 106 def exec_maybe_sandboxed(readable, writable, tmpdir, prog, args):
 107         """execl prog, with (only) the 'writable' directories writable if sandboxing is available.
 108         The readable directories will be readable, as well as various standard locations.
 109         If no sandbox is available, run without a sandbox."""
 110
 111         USE_PLASH = 'USE_PLASH_0COMPILE'
 112
 113         assert prog.startswith('/')
 114         _pola_run = find_in_path('pola-run')
 115
 116         if _pola_run is None:
 117                 print "Not using sandbox (plash not installed)"
 118                 use_plash = False
 119         else:
 120                 use_plash = os.environ.get(USE_PLASH, '').lower() or 'not set'
 121                 if use_plash in ('not set', 'false'):
 122                         print "Not using plash: $%s is %s" % (USE_PLASH, use_plash)
 123                         use_plash = False
 124                 elif use_plash == 'true':
 125                         use_plash = True
 126                 else:
 127                         raise Exception('$%s must be "true" or "false", not "%s"' % (USE_PLASH, use_plash))
 128
 129         if not use_plash:
 130                 os.execlp(prog, prog, *args)
 131
 132         print "Using plash to sandbox the build..."
 133
 134         # We have pola-shell :-)
 135         pola_args = ['--prog', prog, '-B']
 136         for a in args:
 137                 pola_args += ['-a', a]
 138         for r in readable:
 139                 pola_args += ['-f', r]
 140         for w in writable:
 141                 pola_args += ['-fw', w]
 142         pola_args += ['-tw', '/tmp', tmpdir]
 143         os.environ['TMPDIR'] = '/tmp'
 144         os.execl(_pola_run, _pola_run, *pola_args)
 145
 146 def get_arch_name():
 147         uname = os.uname()
 148         target_os = canonicalize_os(uname[0])
 149         target_machine = canonicalize_machine(uname[4])
 150         if target_os == 'Darwin' and target_machine == 'i386':
 151                 # this system detection shell script comes from config.guess (20090918):
 152                 CC = os.getenv("CC_FOR_BUILD") or os.getenv("CC") or os.getenv("HOST_CC") or "cc"
 153                 process = subprocess.Popen("(echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | " +
 154                                            "(CCOPTS= %s -E - 2>/dev/null) | " % CC +
 155                                            "grep IS_64BIT_ARCH >/dev/null", stdout=subprocess.PIPE, shell=True)
 156                 output, error = process.communicate()
 157                 retcode = process.poll()
 158                 if retcode == 0:
 159                         target_machine='x86_64'
 160         if target_machine in ('i585', 'i686'):
 161                 target_machine = 'i486' # (sensible default)
 162         return target_os + '-' + target_machine
 163
 164 class BuildEnv:
 165         def __init__(self, need_config = True):
 166                 if need_config and not os.path.isfile(ENV_FILE):
 167                         raise SafeException("Run 0compile from a directory containing a '%s' file" % ENV_FILE)
 168
 169                 self.config = ConfigParser.RawConfigParser()
 170                 self.config.add_section('compile')
 171                 self.config.set('compile', 'download-base-url', '')
 172                 self.config.set('compile', 'version-modifier', '')
 173                 self.config.set('compile', 'interface', '')
 174                 self.config.set('compile', 'selections', '')
 175                 self.config.set('compile', 'metadir', '0install')
 176                 self.config.set('compile', 'distdir', '')
 177
 178                 self.config.read(ENV_FILE)
 179
 180                 self._selections = None
 181
 182                 return
 183
 184         @property
 185         def iface_name(self):
 186                 iface_name = os.path.basename(self.interface)
 187                 if iface_name.endswith('.xml'):
 188                         iface_name = iface_name[:-4]
 189                 iface_name = iface_name.replace(' ', '-')
 190                 if iface_name.endswith('-src'):
 191                         iface_name = iface_name[:-4]
 192                 return iface_name
 193
 194         interface = property(lambda self: model.canonical_iface_uri(self.config.get('compile', 'interface')))
 195
 196         @property
 197         def distdir(self):
 198                 distdir_name = self.config.get('compile', 'distdir')
 199                 if not distdir_name:
 200                         arch = self.target_arch.replace('*', 'any')
 201                         distdir_name = self.iface_name.lower()
 202                         distdir_name += '-' + arch.lower()
 203                 assert '/' not in distdir_name
 204                 return os.path.realpath(distdir_name)
 205
 206         def get_binary_template(self):
 207                 """Find the <compile:implementation> element for the selected compile command, if any"""
 208                 sels = self.get_selections()
 209                 if sels.commands:
 210                         for elem in sels.commands[0].qdom.childNodes:
 211                                 if elem.uri == XMLNS_0COMPILE and elem.name == 'implementation':
 212                                         return elem
 213
 214                                 # XXX: hack for 0launch < 0.54 which messes up the namespace
 215                                 if elem.name == 'implementation':
 216                                         return elem
 217                 return None
 218
 219         @property
 220         def metadir(self):
 221                 metadir = self.config.get('compile', 'metadir')
 222                 assert not metadir.startswith('/')
 223                 return join(self.distdir, metadir)
 224
 225         @property
 226         def local_iface_file(self):
 227                 return join(self.metadir, self.iface_name + '.xml')
 228
 229         @property
 230         def target_arch(self):
 231                 temp = self.get_binary_template()
 232                 arch = temp and temp.getAttribute('arch')
 233                 return arch or get_arch_name()
 234
 235         @property
 236         def version_modifier(self):
 237                 vm = self.config.get('compile', 'version-modifier')
 238                 if vm: return vm
 239                 if self.user_srcdir:
 240                         return '-1'
 241                 return ''
 242
 243         @property
 244         def archive_stem(self):
 245                 # Use the version that we actually built, not the version we would build now
 246                 feed = self.load_built_feed()
 247                 assert len(feed.implementations) == 1
 248                 version = feed.implementations.values()[0].get_version()
 249
 250                 # Don't use the feed's name, as it may contain the version number
 251                 name = feed.get_name().lower().replace(' ', '-')
 252                 arch = self.target_arch.lower().replace('*-*', 'bin').replace('*', 'any')
 253
 254                 return '%s-%s-%s' % (name, arch, version)
 255
 256         def load_built_feed(self):
 257                 path = self.local_iface_file
 258                 stream = file(path)
 259                 try:
 260                         feed = model.ZeroInstallFeed(qdom.parse(stream), local_path = path)
 261                 finally:
 262                         stream.close()
 263                 return feed
 264
 265         def load_built_selections(self):
 266                 path = join(self.metadir, 'build-environment.xml')
 267                 if os.path.exists(path):
 268                         stream = file(path)
 269                         try:
 270                                 return selections.Selections(qdom.parse(stream))
 271                         finally:
 272                                 stream.close()
 273                 return None
 274
 275         @property
 276         def download_base_url(self):
 277                 return self.config.get('compile', 'download-base-url')
 278
 279         def chosen_impl(self, uri):
 280                 sels = self.get_selections()
 281                 assert uri in sels.selections
 282                 return sels.selections[uri]
 283
 284         @property
 285         def local_download_iface(self):
 286                 impl, = self.load_built_feed().implementations.values()
 287                 return '%s-%s.xml' % (self.iface_name, impl.get_version())
 288
 289         def save(self):
 290                 stream = file(ENV_FILE, 'w')
 291                 try:
 292                         self.config.write(stream)
 293                 finally:
 294                         stream.close()
 295
 296         def get_selections(self, prompt = False):
 297                 if self._selections:
 298                         assert not prompt
 299                         return self._selections
 300
 301                 selections_file = self.config.get('compile', 'selections')
 302                 if selections_file:
 303                         if prompt:
 304                                 raise SafeException("Selections are fixed by %s" % selections_file)
 305                         stream = file(selections_file)
 306                         try:
 307                                 self._selections = selections.Selections(qdom.parse(stream))
 308                         finally:
 309                                 stream.close()
 310                         from zeroinstall.injector import handler, policy
 311                         if os.isatty(1):
 312                                 h = handler.ConsoleHandler()
 313                         else:
 314                                 h = handler.Handler()
 315                         config = policy.load_config(h)
 316                         blocker = self._selections.download_missing(config)
 317                         if blocker:
 318                                 print "Waiting for selected implementations to be downloaded..."
 319                                 h.wait_for_blocker(blocker)
 320                 else:
 321                         options = []
 322                         if prompt and '--console' not in launch_prog:
 323                                 options.append('--gui')
 324                         child = subprocess.Popen(launch_prog + ['--source', '--get-selections'] + options + [self.interface], stdout = subprocess.PIPE)
 325                         try:
 326                                 self._selections = selections.Selections(qdom.parse(child.stdout))
 327                         finally:
 328                                 if child.wait():
 329                                         raise SafeException("0launch --get-selections failed (exit code %d)" % child.returncode)
 330
 331                 self.root_impl = self._selections.selections[self.interface]
 332
 333                 self.orig_srcdir = os.path.realpath(lookup(self.root_impl))
 334                 self.user_srcdir = None
 335
 336                 if os.path.isdir('src'):
 337                         self.user_srcdir = os.path.realpath('src')
 338                         if self.user_srcdir == self.orig_srcdir or \
 339                            self.user_srcdir.startswith(self.orig_srcdir + '/') or \
 340                            self.orig_srcdir.startswith(self.user_srcdir + '/'):
 341                                 info("Ignoring 'src' directory because it coincides with %s",
 342                                         self.orig_srcdir)
 343                                 self.user_srcdir = None
 344
 345                 return self._selections
 346
 347         def get_build_changes(self):
 348                 sels = self.get_selections()
 349                 old_sels = self.load_built_selections()
 350                 changes = []
 351                 if old_sels:
 352                         # See if things have changed since the last build
 353                         all_ifaces = set(sels.selections) | set(old_sels.selections)
 354                         for x in all_ifaces:
 355                                 old_impl = old_sels.selections.get(x, no_impl)
 356                                 new_impl = sels.selections.get(x, no_impl)
 357                                 if old_impl.version != new_impl.version:
 358                                         changes.append("Version change for %s: %s -> %s" % (x, old_impl.version, new_impl.version))
 359                                 elif old_impl.id != new_impl.id:
 360                                         changes.append("Version change for %s: %s -> %s" % (x, old_impl.id, new_impl.id))
 361                 return changes
 362
 363 def depth(node):
 364         root = node.ownerDocument.documentElement
 365         depth = 0
 366         while node and node is not root:
 367                 node = node.parentNode
 368                 depth += 1
 369         return depth
 370
 371 def parse_bool(s):
 372         if s == 'true': return True
 373         if s == 'false': return False
 374         raise SafeException('Expected "true" or "false" but got "%s"' % s)
 375
 376 class Prefixes:
 377         # Copied from 0launch 0.54 (remove once 0.54 is released)
 378         def __init__(self, default_ns):
 379                 self.prefixes = {}
 380                 self.default_ns = default_ns
 381
 382         def get(self, ns):
 383                 prefix = self.prefixes.get(ns, None)
 384                 if prefix:
 385                         return prefix
 386                 prefix = 'ns%d' % len(self.prefixes)
 387                 self.prefixes[ns] = prefix
 388                 return prefix
 389
 390         def setAttributeNS(self, elem, uri, localName, value):
 391                 if uri is None:
 392                         elem.setAttributeNS(None, localName, value)
 393                 else:
 394                         elem.setAttributeNS(uri, self.get(uri) + ':' + localName, value)
 395
 396         def createElementNS(self, doc, uri, localName):
 397                 if uri == self.default_ns:
 398                         return doc.createElementNS(uri, localName)
 399                 else:
 400                         return doc.createElementNS(uri, self.get(uri) + ':' + localName)