support.py

   1 # Copyright (C) 2006, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import os, sys, shutil, traceback
   5 import subprocess
   6 from os.path import join
   7 from logging import info
   8 import ConfigParser
   9
  10 from zeroinstall.injector import model, selections, qdom
  11 from zeroinstall.injector.arch import canonicalize_os, canonicalize_machine
  12
  13 from zeroinstall.injector.iface_cache import iface_cache
  14 from zeroinstall import SafeException
  15 from zeroinstall.zerostore import Store, NotStored
  16
  17 def _(x): return x
  18
  19 ENV_FILE = '0compile.properties'
  20 XMLNS_0COMPILE = 'http://zero-install.sourceforge.net/2006/namespaces/0compile'
  21
  22 zeroinstall_dir = os.environ.get('0COMPILE_ZEROINSTALL', None)
  23 if zeroinstall_dir:
  24         launch_prog = [os.path.join(zeroinstall_dir, '0launch')]
  25 else:
  26         launch_prog = ['0launch']
  27
  28 if os.path.isdir('dependencies'):
  29         dep_dir = os.path.realpath('dependencies')
  30         iface_cache.stores.stores.append(Store(dep_dir))
  31         launch_prog += ['--with-store', dep_dir]
  32
  33 class NoImpl:
  34         id = "none"
  35         version = "none"
  36 no_impl = NoImpl()
  37
  38 def is_package_impl(impl):
  39         return impl.id.startswith("package:")
  40
  41 def lookup(impl_or_sel):
  42         id = impl_or_sel.id
  43         if id.startswith('/'):
  44                 if os.path.isdir(id):
  45                         return id
  46                 raise SafeException("Directory '%s' no longer exists. Try '0compile setup'" % id)
  47         try:
  48                 return iface_cache.stores.lookup_any(impl_or_sel.digests)
  49         except NotStored, ex:
  50                 raise NotStored(str(ex) + "\nHint: try '0compile setup'")
  51
  52 def ensure_dir(d, clean = False):
  53         if os.path.isdir(d):
  54                 if clean:
  55                         print "Removing", d
  56                         shutil.rmtree(d)
  57                 else:
  58                         return
  59         if os.path.exists(d):
  60                 raise SafeException("'%s' exists, but is not a directory!" % d)
  61         os.mkdir(d)
  62
  63 def find_in_path(prog):
  64         for d in os.environ['PATH'].split(':'):
  65                 path = os.path.join(d, prog)
  66                 if os.path.isfile(path):
  67                         return path
  68         return None
  69
  70 def spawn_and_check(prog, args):
  71         status = os.spawnv(os.P_WAIT, prog, [prog] + args)
  72         if status > 0:
  73                 raise SafeException("Program '%s' failed with exit code %d" % (prog, status))
  74         elif status < 0:
  75                 raise SafeException("Program '%s' failed with signal %d" % (prog, -status))
  76
  77 def wait_for_child(child):
  78         """Wait for child to exit and reap it. Throw an exception if it doesn't return success."""
  79         pid, status = os.waitpid(child, 0)
  80         assert pid == child
  81         if os.WIFEXITED(status):
  82                 exit_code = os.WEXITSTATUS(status)
  83                 if exit_code == 0:
  84                         return
  85                 else:
  86                         raise SafeException('Command failed with exit status %d' % exit_code)
  87         else:
  88                 raise SafeException('Command failed with signal %d' % os.WTERMSIG(status))
  89
  90 def spawn_maybe_sandboxed(readable, writable, tmpdir, prog, args):
  91         child = os.fork()
  92         if child == 0:
  93                 try:
  94                         try:
  95                                 exec_maybe_sandboxed(readable, writable, tmpdir, prog, args)
  96                         except:
  97                                 traceback.print_exc()
  98                 finally:
  99                         print >>sys.stderr, "Exec failed"
 100                         os._exit(1)
 101         wait_for_child(child)
 102
 103 def exec_maybe_sandboxed(readable, writable, tmpdir, prog, args):
 104         """execl prog, with (only) the 'writable' directories writable if sandboxing is available.
 105         The readable directories will be readable, as well as various standard locations.
 106         If no sandbox is available, run without a sandbox."""
 107
 108         USE_PLASH = 'USE_PLASH_0COMPILE'
 109
 110         assert prog.startswith('/')
 111         _pola_run = find_in_path('pola-run')
 112
 113         if _pola_run is None:
 114                 print "Not using sandbox (plash not installed)"
 115                 use_plash = False
 116         else:
 117                 use_plash = os.environ.get(USE_PLASH, '').lower() or 'not set'
 118                 if use_plash in ('not set', 'false'):
 119                         print "Not using plash: $%s is %s" % (USE_PLASH, use_plash)
 120                         use_plash = False
 121                 elif use_plash == 'true':
 122                         use_plash = True
 123                 else:
 124                         raise Exception('$%s must be "true" or "false", not "%s"' % (USE_PLASH, use_plash))
 125
 126         if not use_plash:
 127                 os.execlp(prog, prog, *args)
 128
 129         print "Using plash to sandbox the build..."
 130
 131         # We have pola-shell :-)
 132         pola_args = ['--prog', prog, '-B']
 133         for a in args:
 134                 pola_args += ['-a', a]
 135         for r in readable:
 136                 pola_args += ['-f', r]
 137         for w in writable:
 138                 pola_args += ['-fw', w]
 139         pola_args += ['-tw', '/tmp', tmpdir]
 140         os.environ['TMPDIR'] = '/tmp'
 141         os.execl(_pola_run, _pola_run, *pola_args)
 142
 143 def get_arch_name():
 144         uname = os.uname()
 145         target_os = canonicalize_os(uname[0])
 146         target_machine = canonicalize_machine(uname[4])
 147         if target_os == 'Darwin' and target_machine == 'i386':
 148                 # this system detection shell script comes from config.guess (20090918):
 149                 CC = os.getenv("CC_FOR_BUILD") or os.getenv("CC") or os.getenv("HOST_CC") or "cc"
 150                 process = subprocess.Popen("(echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | " +
 151                                            "(CCOPTS= %s -E - 2>/dev/null) | " % CC +
 152                                            "grep IS_64BIT_ARCH >/dev/null", stdout=subprocess.PIPE, shell=True)
 153                 output, error = process.communicate()
 154                 retcode = process.poll()
 155                 if retcode == 0:
 156                         target_machine='x86_64'
 157         if target_machine in ('i585', 'i686'):
 158                 target_machine = 'i486' # (sensible default)
 159         return target_os + '-' + target_machine
 160
 161 class BuildEnv:
 162         def __init__(self, need_config = True):
 163                 if need_config and not os.path.isfile(ENV_FILE):
 164                         raise SafeException("Run 0compile from a directory containing a '%s' file" % ENV_FILE)
 165
 166                 self.config = ConfigParser.RawConfigParser()
 167                 self.config.add_section('compile')
 168                 self.config.set('compile', 'download-base-url', '')
 169                 self.config.set('compile', 'version-modifier', '')
 170                 self.config.set('compile', 'interface', '')
 171                 self.config.set('compile', 'selections', '')
 172                 self.config.set('compile', 'metadir', '0install')
 173                 self.config.set('compile', 'distdir', '')
 174
 175                 self.config.read(ENV_FILE)
 176
 177                 self._selections = None
 178
 179                 return
 180
 181         @property
 182         def iface_name(self):
 183                 iface_name = os.path.basename(self.interface)
 184                 if iface_name.endswith('.xml'):
 185                         iface_name = iface_name[:-4]
 186                 iface_name = iface_name.replace(' ', '-')
 187                 if iface_name.endswith('-src'):
 188                         iface_name = iface_name[:-4]
 189                 return iface_name
 190
 191         interface = property(lambda self: model.canonical_iface_uri(self.config.get('compile', 'interface')))
 192
 193         @property
 194         def distdir(self):
 195                 distdir_name = self.config.get('compile', 'distdir')
 196                 if not distdir_name:
 197                         arch = self.target_arch.replace('*', 'any')
 198                         distdir_name = self.iface_name.lower()
 199                         distdir_name += '-' + arch.lower()
 200                 assert '/' not in distdir_name
 201                 return os.path.realpath(distdir_name)
 202
 203         def get_binary_template(self):
 204                 """Find the <compile:implementation> element for the selected compile command, if any"""
 205                 sels = self.get_selections()
 206                 if sels.commands:
 207                         for elem in sels.commands[0].qdom.childNodes:
 208                                 if elem.uri == XMLNS_0COMPILE and elem.name == 'implementation':
 209                                         return elem
 210
 211                                 # XXX: hack for 0launch < 0.54 which messes up the namespace
 212                                 if elem.name == 'implementation':
 213                                         return elem
 214                 return None
 215
 216         @property
 217         def metadir(self):
 218                 metadir = self.config.get('compile', 'metadir')
 219                 assert not metadir.startswith('/')
 220                 return join(self.distdir, metadir)
 221
 222         @property
 223         def local_iface_file(self):
 224                 return join(self.metadir, self.iface_name + '.xml')
 225
 226         @property
 227         def target_arch(self):
 228                 temp = self.get_binary_template()
 229                 arch = temp and temp.getAttribute('arch')
 230                 return arch or get_arch_name()
 231
 232         @property
 233         def version_modifier(self):
 234                 vm = self.config.get('compile', 'version-modifier')
 235                 if vm: return vm
 236                 if self.user_srcdir:
 237                         return '-1'
 238                 return ''
 239
 240         @property
 241         def archive_stem(self):
 242                 # Use the version that we actually built, not the version we would build now
 243                 feed = self.load_built_feed()
 244                 assert len(feed.implementations) == 1
 245                 version = feed.implementations.values()[0].get_version()
 246
 247                 # Don't use the feed's name, as it may contain the version number
 248                 name = feed.get_name().lower().replace(' ', '-')
 249
 250                 return '%s-%s-%s' % (name, self.target_arch.lower(), version)
 251
 252         def load_built_feed(self):
 253                 path = self.local_iface_file
 254                 stream = file(path)
 255                 try:
 256                         feed = model.ZeroInstallFeed(qdom.parse(stream), local_path = path)
 257                 finally:
 258                         stream.close()
 259                 return feed
 260
 261         def load_built_selections(self):
 262                 path = join(self.metadir, 'build-environment.xml')
 263                 if os.path.exists(path):
 264                         stream = file(path)
 265                         try:
 266                                 return selections.Selections(qdom.parse(stream))
 267                         finally:
 268                                 stream.close()
 269                 return None
 270
 271         @property
 272         def download_base_url(self):
 273                 return self.config.get('compile', 'download-base-url')
 274
 275         def chosen_impl(self, uri):
 276                 sels = self.get_selections()
 277                 assert uri in sels.selections
 278                 return sels.selections[uri]
 279
 280         @property
 281         def local_download_iface(self):
 282                 impl, = self.load_built_feed().implementations.values()
 283                 return '%s-%s.xml' % (self.iface_name, impl.get_version())
 284
 285         def save(self):
 286                 stream = file(ENV_FILE, 'w')
 287                 try:
 288                         self.config.write(stream)
 289                 finally:
 290                         stream.close()
 291
 292         def get_selections(self, prompt = False):
 293                 if self._selections:
 294                         assert not prompt
 295                         return self._selections
 296
 297                 selections_file = self.config.get('compile', 'selections')
 298                 if selections_file:
 299                         if prompt:
 300                                 raise SafeException("Selections are fixed by %s" % selections_file)
 301                         stream = file(selections_file)
 302                         try:
 303                                 self._selections = selections.Selections(qdom.parse(stream))
 304                         finally:
 305                                 stream.close()
 306                         from zeroinstall.injector import handler, policy
 307                         if os.isatty(1):
 308                                 h = handler.ConsoleHandler()
 309                         else:
 310                                 h = handler.Handler()
 311                         config = policy.load_config(h)
 312                         blocker = self._selections.download_missing(config)
 313                         if blocker:
 314                                 print "Waiting for selected implementations to be downloaded..."
 315                                 h.wait_for_blocker(blocker)
 316                 else:
 317                         options = []
 318                         if prompt and '--console' not in launch_prog:
 319                                 options.append('--gui')
 320                         child = subprocess.Popen(launch_prog + ['--source', '--get-selections'] + options + [self.interface], stdout = subprocess.PIPE)
 321                         try:
 322                                 self._selections = selections.Selections(qdom.parse(child.stdout))
 323                         finally:
 324                                 if child.wait():
 325                                         raise SafeException("0launch --get-selections failed (exit code %d)" % child.returncode)
 326
 327                 self.root_impl = self._selections.selections[self.interface]
 328
 329                 self.orig_srcdir = os.path.realpath(lookup(self.root_impl))
 330                 self.user_srcdir = None
 331
 332                 if os.path.isdir('src'):
 333                         self.user_srcdir = os.path.realpath('src')
 334                         if self.user_srcdir == self.orig_srcdir or \
 335                            self.user_srcdir.startswith(self.orig_srcdir + '/') or \
 336                            self.orig_srcdir.startswith(self.user_srcdir + '/'):
 337                                 info("Ignoring 'src' directory because it coincides with %s",
 338                                         self.orig_srcdir)
 339                                 self.user_srcdir = None
 340
 341                 return self._selections
 342
 343         def get_build_changes(self):
 344                 sels = self.get_selections()
 345                 old_sels = self.load_built_selections()
 346                 changes = []
 347                 if old_sels:
 348                         # See if things have changed since the last build
 349                         all_ifaces = set(sels.selections) | set(old_sels.selections)
 350                         for x in all_ifaces:
 351                                 old_impl = old_sels.selections.get(x, no_impl)
 352                                 new_impl = sels.selections.get(x, no_impl)
 353                                 if old_impl.version != new_impl.version:
 354                                         changes.append("Version change for %s: %s -> %s" % (x, old_impl.version, new_impl.version))
 355                                 elif old_impl.id != new_impl.id:
 356                                         changes.append("Version change for %s: %s -> %s" % (x, old_impl.id, new_impl.id))
 357                 return changes
 358
 359 def depth(node):
 360         root = node.ownerDocument.documentElement
 361         depth = 0
 362         while node and node is not root:
 363                 node = node.parentNode
 364                 depth += 1
 365         return depth
 366
 367 def parse_bool(s):
 368         if s == 'true': return True
 369         if s == 'false': return False
 370         raise SafeException('Expected "true" or "false" but got "%s"' % s)
 371
 372 class Prefixes:
 373         # Copied from 0launch 0.54 (remove once 0.54 is released)
 374         def __init__(self, default_ns):
 375                 self.prefixes = {}
 376                 self.default_ns = default_ns
 377
 378         def get(self, ns):
 379                 prefix = self.prefixes.get(ns, None)
 380                 if prefix:
 381                         return prefix
 382                 prefix = 'ns%d' % len(self.prefixes)
 383                 self.prefixes[ns] = prefix
 384                 return prefix
 385
 386         def setAttributeNS(self, elem, uri, localName, value):
 387                 if uri is None:
 388                         elem.setAttributeNS(None, localName, value)
 389                 else:
 390                         elem.setAttributeNS(uri, self.get(uri) + ':' + localName, value)
 391
 392         def createElementNS(self, doc, uri, localName):
 393                 if uri == self.default_ns:
 394                         return doc.createElementNS(uri, localName)
 395                 else:
 396                         return doc.createElementNS(uri, self.get(uri) + ':' + localName)