support.py

   1 # Copyright (C) 2006, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import os, sys, shutil, traceback
   5 import subprocess
   6 from os.path import join
   7 from logging import info
   8 import ConfigParser
   9
  10 from zeroinstall.injector import model, selections, qdom
  11 from zeroinstall.injector.arch import canonicalize_os, canonicalize_machine
  12
  13 from zeroinstall.injector.iface_cache import iface_cache
  14 from zeroinstall import SafeException
  15 from zeroinstall.zerostore import Store, NotStored
  16
  17 def _(x): return x
  18
  19 ENV_FILE = '0compile.properties'
  20 XMLNS_0COMPILE = 'http://zero-install.sourceforge.net/2006/namespaces/0compile'
  21
  22 zeroinstall_dir = os.environ.get('0COMPILE_ZEROINSTALL', None)
  23 if zeroinstall_dir:
  24         # XXX: we're assuming that, if installed through 0install, 0launch requires
  25         # the same version of Python as 0compile. This is currently needed for Arch
  26         # Linux, but long-term we need to use the <runner>.
  27         launch_prog = [sys.executable, os.path.join(zeroinstall_dir, '0launch')]
  28 else:
  29         launch_prog = ['0launch']
  30
  31 if os.path.isdir('dependencies'):
  32         dep_dir = os.path.realpath('dependencies')
  33         iface_cache.stores.stores.append(Store(dep_dir))
  34         launch_prog += ['--with-store', dep_dir]
  35
  36 class NoImpl:
  37         id = "none"
  38         version = "none"
  39 no_impl = NoImpl()
  40
  41 def is_package_impl(impl):
  42         return impl.id.startswith("package:")
  43
  44 def lookup(impl_or_sel):
  45         id = impl_or_sel.id
  46         if id.startswith('package:'):
  47                 return None
  48         if id.startswith('/'):
  49                 if os.path.isdir(id):
  50                         return id
  51                 raise SafeException("Directory '%s' no longer exists. Try '0compile setup'" % id)
  52         try:
  53                 return iface_cache.stores.lookup_any(impl_or_sel.digests)
  54         except NotStored, ex:
  55                 raise NotStored(str(ex) + "\nHint: try '0compile setup'")
  56
  57 def ensure_dir(d, clean = False):
  58         if os.path.isdir(d):
  59                 if clean:
  60                         print "Removing", d
  61                         shutil.rmtree(d)
  62                 else:
  63                         return
  64         if os.path.exists(d):
  65                 raise SafeException("'%s' exists, but is not a directory!" % d)
  66         os.mkdir(d)
  67
  68 def find_in_path(prog):
  69         for d in os.environ['PATH'].split(':'):
  70                 path = os.path.join(d, prog)
  71                 if os.path.isfile(path):
  72                         return path
  73         return None
  74
  75 def spawn_and_check(prog, args):
  76         status = os.spawnv(os.P_WAIT, prog, [prog] + args)
  77         if status > 0:
  78                 raise SafeException("Program '%s' failed with exit code %d" % (prog, status))
  79         elif status < 0:
  80                 raise SafeException("Program '%s' failed with signal %d" % (prog, -status))
  81
  82 def wait_for_child(child):
  83         """Wait for child to exit and reap it. Throw an exception if it doesn't return success."""
  84         pid, status = os.waitpid(child, 0)
  85         assert pid == child
  86         if os.WIFEXITED(status):
  87                 exit_code = os.WEXITSTATUS(status)
  88                 if exit_code == 0:
  89                         return
  90                 else:
  91                         raise SafeException('Command failed with exit status %d' % exit_code)
  92         else:
  93                 raise SafeException('Command failed with signal %d' % os.WTERMSIG(status))
  94
  95 def spawn_maybe_sandboxed(readable, writable, tmpdir, prog, args):
  96         child = os.fork()
  97         if child == 0:
  98                 try:
  99                         try:
 100                                 exec_maybe_sandboxed(readable, writable, tmpdir, prog, args)
 101                         except:
 102                                 traceback.print_exc()
 103                 finally:
 104                         print >>sys.stderr, "Exec failed"
 105                         os._exit(1)
 106         wait_for_child(child)
 107
 108 def exec_maybe_sandboxed(readable, writable, tmpdir, prog, args):
 109         """execl prog, with (only) the 'writable' directories writable if sandboxing is available.
 110         The readable directories will be readable, as well as various standard locations.
 111         If no sandbox is available, run without a sandbox."""
 112
 113         USE_PLASH = 'USE_PLASH_0COMPILE'
 114
 115         assert prog.startswith('/')
 116         _pola_run = find_in_path('pola-run')
 117
 118         if _pola_run is None:
 119                 print "Not using sandbox (plash not installed)"
 120                 use_plash = False
 121         else:
 122                 use_plash = os.environ.get(USE_PLASH, '').lower() or 'not set'
 123                 if use_plash in ('not set', 'false'):
 124                         print "Not using plash: $%s is %s" % (USE_PLASH, use_plash)
 125                         use_plash = False
 126                 elif use_plash == 'true':
 127                         use_plash = True
 128                 else:
 129                         raise Exception('$%s must be "true" or "false", not "%s"' % (USE_PLASH, use_plash))
 130
 131         if not use_plash:
 132                 os.execlp(prog, prog, *args)
 133
 134         print "Using plash to sandbox the build..."
 135
 136         # We have pola-shell :-)
 137         pola_args = ['--prog', prog, '-B']
 138         for a in args:
 139                 pola_args += ['-a', a]
 140         for r in readable:
 141                 pola_args += ['-f', r]
 142         for w in writable:
 143                 pola_args += ['-fw', w]
 144         pola_args += ['-tw', '/tmp', tmpdir]
 145         os.environ['TMPDIR'] = '/tmp'
 146         os.execl(_pola_run, _pola_run, *pola_args)
 147
 148 def get_arch_name():
 149         uname = os.uname()
 150         target_os = canonicalize_os(uname[0])
 151         target_machine = canonicalize_machine(uname[4])
 152         if target_os == 'Darwin' and target_machine == 'i386':
 153                 # this system detection shell script comes from config.guess (20090918):
 154                 CC = os.getenv("CC_FOR_BUILD") or os.getenv("CC") or os.getenv("HOST_CC") or "cc"
 155                 process = subprocess.Popen("(echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | " +
 156                                            "(CCOPTS= %s -E - 2>/dev/null) | " % CC +
 157                                            "grep IS_64BIT_ARCH >/dev/null", stdout=subprocess.PIPE, shell=True)
 158                 output, error = process.communicate()
 159                 retcode = process.poll()
 160                 if retcode == 0:
 161                         target_machine='x86_64'
 162         if target_machine in ('i585', 'i686'):
 163                 target_machine = 'i486' # (sensible default)
 164         return target_os + '-' + target_machine
 165
 166 class BuildEnv:
 167         def __init__(self, need_config = True):
 168                 if need_config and not os.path.isfile(ENV_FILE):
 169                         raise SafeException("Run 0compile from a directory containing a '%s' file" % ENV_FILE)
 170
 171                 self.config = ConfigParser.RawConfigParser()
 172                 self.config.add_section('compile')
 173                 self.config.set('compile', 'download-base-url', '')
 174                 self.config.set('compile', 'version-modifier', '')
 175                 self.config.set('compile', 'interface', '')
 176                 self.config.set('compile', 'selections', '')
 177                 self.config.set('compile', 'metadir', '0install')
 178                 self.config.set('compile', 'distdir', '')
 179
 180                 self.config.read(ENV_FILE)
 181
 182                 self._selections = None
 183
 184                 return
 185
 186         @property
 187         def iface_name(self):
 188                 iface_name = os.path.basename(self.interface)
 189                 if iface_name.endswith('.xml'):
 190                         iface_name = iface_name[:-4]
 191                 iface_name = iface_name.replace(' ', '-')
 192                 if iface_name.endswith('-src'):
 193                         iface_name = iface_name[:-4]
 194                 return iface_name
 195
 196         interface = property(lambda self: model.canonical_iface_uri(self.config.get('compile', 'interface')))
 197
 198         @property
 199         def distdir(self):
 200                 distdir_name = self.config.get('compile', 'distdir')
 201                 if not distdir_name:
 202                         arch = self.target_arch.replace('*', 'any')
 203                         distdir_name = self.iface_name.lower()
 204                         distdir_name += '-' + arch.lower()
 205                 assert '/' not in distdir_name
 206                 return os.path.realpath(distdir_name)
 207
 208         def get_binary_template(self):
 209                 """Find the <compile:implementation> element for the selected compile command, if any"""
 210                 sels = self.get_selections()
 211                 if sels.commands:
 212                         for elem in sels.commands[0].qdom.childNodes:
 213                                 if elem.uri == XMLNS_0COMPILE and elem.name == 'implementation':
 214                                         return elem
 215
 216                                 # XXX: hack for 0launch < 0.54 which messes up the namespace
 217                                 if elem.name == 'implementation':
 218                                         return elem
 219                 return None
 220
 221         @property
 222         def metadir(self):
 223                 metadir = self.config.get('compile', 'metadir')
 224                 assert not metadir.startswith('/')
 225                 return join(self.distdir, metadir)
 226
 227         @property
 228         def local_iface_file(self):
 229                 return join(self.metadir, self.iface_name + '.xml')
 230
 231         @property
 232         def target_arch(self):
 233                 temp = self.get_binary_template()
 234                 arch = temp and temp.getAttribute('arch')
 235                 return arch or get_arch_name()
 236
 237         @property
 238         def version_modifier(self):
 239                 vm = self.config.get('compile', 'version-modifier')
 240                 if vm: return vm
 241                 if self.user_srcdir:
 242                         return '-1'
 243                 return ''
 244
 245         @property
 246         def archive_stem(self):
 247                 # Use the version that we actually built, not the version we would build now
 248                 feed = self.load_built_feed()
 249                 assert len(feed.implementations) == 1
 250                 version = feed.implementations.values()[0].get_version()
 251
 252                 # Don't use the feed's name, as it may contain the version number
 253                 name = feed.get_name().lower().replace(' ', '-')
 254                 arch = self.target_arch.lower().replace('*-*', 'bin').replace('*', 'any')
 255
 256                 return '%s-%s-%s' % (name, arch, version)
 257
 258         def load_built_feed(self):
 259                 path = self.local_iface_file
 260                 stream = file(path)
 261                 try:
 262                         feed = model.ZeroInstallFeed(qdom.parse(stream), local_path = path)
 263                 finally:
 264                         stream.close()
 265                 return feed
 266
 267         def load_built_selections(self):
 268                 path = join(self.metadir, 'build-environment.xml')
 269                 if os.path.exists(path):
 270                         stream = file(path)
 271                         try:
 272                                 return selections.Selections(qdom.parse(stream))
 273                         finally:
 274                                 stream.close()
 275                 return None
 276
 277         @property
 278         def download_base_url(self):
 279                 return self.config.get('compile', 'download-base-url')
 280
 281         def chosen_impl(self, uri):
 282                 sels = self.get_selections()
 283                 assert uri in sels.selections
 284                 return sels.selections[uri]
 285
 286         @property
 287         def local_download_iface(self):
 288                 impl, = self.load_built_feed().implementations.values()
 289                 return '%s-%s.xml' % (self.iface_name, impl.get_version())
 290
 291         def save(self):
 292                 stream = file(ENV_FILE, 'w')
 293                 try:
 294                         self.config.write(stream)
 295                 finally:
 296                         stream.close()
 297
 298         def get_selections(self, prompt = False):
 299                 if self._selections:
 300                         assert not prompt
 301                         return self._selections
 302
 303                 selections_file = self.config.get('compile', 'selections')
 304                 if selections_file:
 305                         if prompt:
 306                                 raise SafeException("Selections are fixed by %s" % selections_file)
 307                         stream = file(selections_file)
 308                         try:
 309                                 self._selections = selections.Selections(qdom.parse(stream))
 310                         finally:
 311                                 stream.close()
 312                         from zeroinstall.injector import handler, policy
 313                         if os.isatty(1):
 314                                 h = handler.ConsoleHandler()
 315                         else:
 316                                 h = handler.Handler()
 317                         config = policy.load_config(h)
 318                         blocker = self._selections.download_missing(config)
 319                         if blocker:
 320                                 print "Waiting for selected implementations to be downloaded..."
 321                                 h.wait_for_blocker(blocker)
 322                 else:
 323                         options = []
 324                         if prompt and '--console' not in launch_prog:
 325                                 options.append('--gui')
 326                         child = subprocess.Popen(launch_prog + ['--source', '--get-selections'] + options + [self.interface], stdout = subprocess.PIPE)
 327                         try:
 328                                 self._selections = selections.Selections(qdom.parse(child.stdout))
 329                         finally:
 330                                 if child.wait():
 331                                         raise SafeException("0launch --get-selections failed (exit code %d)" % child.returncode)
 332
 333                 self.root_impl = self._selections.selections[self.interface]
 334
 335                 self.orig_srcdir = os.path.realpath(lookup(self.root_impl))
 336                 self.user_srcdir = None
 337
 338                 if os.path.isdir('src'):
 339                         self.user_srcdir = os.path.realpath('src')
 340                         if self.user_srcdir == self.orig_srcdir or \
 341                            self.user_srcdir.startswith(self.orig_srcdir + '/') or \
 342                            self.orig_srcdir.startswith(self.user_srcdir + '/'):
 343                                 info("Ignoring 'src' directory because it coincides with %s",
 344                                         self.orig_srcdir)
 345                                 self.user_srcdir = None
 346
 347                 return self._selections
 348
 349         def get_build_changes(self):
 350                 sels = self.get_selections()
 351                 old_sels = self.load_built_selections()
 352                 changes = []
 353                 if old_sels:
 354                         # See if things have changed since the last build
 355                         all_ifaces = set(sels.selections) | set(old_sels.selections)
 356                         for x in all_ifaces:
 357                                 old_impl = old_sels.selections.get(x, no_impl)
 358                                 new_impl = sels.selections.get(x, no_impl)
 359                                 if old_impl.version != new_impl.version:
 360                                         changes.append("Version change for %s: %s -> %s" % (x, old_impl.version, new_impl.version))
 361                                 elif old_impl.id != new_impl.id:
 362                                         changes.append("Version change for %s: %s -> %s" % (x, old_impl.id, new_impl.id))
 363                 return changes
 364
 365 def depth(node):
 366         root = node.ownerDocument.documentElement
 367         depth = 0
 368         while node and node is not root:
 369                 node = node.parentNode
 370                 depth += 1
 371         return depth
 372
 373 def parse_bool(s):
 374         if s == 'true': return True
 375         if s == 'false': return False
 376         raise SafeException('Expected "true" or "false" but got "%s"' % s)
 377
 378 class Prefixes:
 379         # Copied from 0launch 0.54 (remove once 0.54 is released)
 380         def __init__(self, default_ns):
 381                 self.prefixes = {}
 382                 self.default_ns = default_ns
 383
 384         def get(self, ns):
 385                 prefix = self.prefixes.get(ns, None)
 386                 if prefix:
 387                         return prefix
 388                 prefix = 'ns%d' % len(self.prefixes)
 389                 self.prefixes[ns] = prefix
 390                 return prefix
 391
 392         def setAttributeNS(self, elem, uri, localName, value):
 393                 if uri is None:
 394                         elem.setAttributeNS(None, localName, value)
 395                 else:
 396                         elem.setAttributeNS(uri, self.get(uri) + ':' + localName, value)
 397
 398         def createElementNS(self, doc, uri, localName):
 399                 if uri == self.default_ns:
 400                         return doc.createElementNS(uri, localName)
 401                 else:
 402                         return doc.createElementNS(uri, self.get(uri) + ':' + localName)