support.py

   1 # Copyright (C) 2006, Thomas Leonard
   2 # See http://0install.net/0compile.html
   3
   4 import os, sys, shutil
   5 import subprocess
   6 from os.path import join
   7 from logging import info
   8 import ConfigParser
   9
  10 from zeroinstall.injector import model, selections, qdom, arch
  11 from zeroinstall.injector.arch import canonicalize_os, canonicalize_machine
  12
  13 from zeroinstall.injector.iface_cache import iface_cache
  14 from zeroinstall import SafeException
  15 from zeroinstall.zerostore import Store, NotStored
  16 from zeroinstall.support import find_in_path
  17
  18 Prefixes = qdom.Prefixes
  19
  20 def _(x): return x
  21
  22
  23 # This is An os.uname() substitute that uses as much of ZI's
  24 # arch._uname as is available and yet has all four elements one
  25 # normally expects from os.uname() on Posix (on Windows, arch._uname
  26 # has only two elements).
  27 import platform
  28 uname = arch._uname + platform.uname()[len(arch._uname):]
  29
  30 ENV_FILE = '0compile.properties'
  31 XMLNS_0COMPILE = 'http://zero-install.sourceforge.net/2006/namespaces/0compile'
  32
  33 zeroinstall_dir = os.environ.get('0COMPILE_ZEROINSTALL', None)
  34 if zeroinstall_dir:
  35         # XXX: we're assuming that, if installed through 0install, 0launch requires
  36         # the same version of Python as 0compile. This is currently needed for Arch
  37         # Linux, but long-term we need to use the <runner>.
  38         install_prog = [sys.executable, os.path.join(zeroinstall_dir, '0install')]
  39         if not os.path.exists(install_prog[1]):
  40                 # For the Windows version...
  41                 install_prog[1] = os.path.join(zeroinstall_dir, 'zeroinstall', 'scripts', 'install.py')
  42 else:
  43         install_prog = ['0install']
  44
  45 if os.path.isdir('dependencies'):
  46         dep_dir = os.path.realpath('dependencies')
  47         iface_cache.stores.stores.append(Store(dep_dir))
  48         install_prog.append('--with-store='+ dep_dir)
  49
  50 class NoImpl:
  51         id = "none"
  52         version = "none"
  53 no_impl = NoImpl()
  54
  55 def is_package_impl(impl):
  56         return impl.id.startswith("package:")
  57
  58 def lookup(impl_or_sel):
  59         id = impl_or_sel.id
  60         if id.startswith('package:'):
  61                 return None
  62         local_path = impl_or_sel.local_path
  63         if local_path is not None:
  64                 if os.path.isdir(local_path):
  65                         return local_path
  66                 raise SafeException("Directory '%s' no longer exists. Try '0compile setup'" % local_path)
  67         try:
  68                 return iface_cache.stores.lookup_any(impl_or_sel.digests)
  69         except NotStored, ex:
  70                 raise NotStored(str(ex) + "\nHint: try '0compile setup'")
  71
  72 def ensure_dir(d, clean = False):
  73         if os.path.isdir(d):
  74                 if clean:
  75                         print "Removing", d
  76                         shutil.rmtree(d)
  77                 else:
  78                         return
  79         if os.path.exists(d):
  80                 raise SafeException("'%s' exists, but is not a directory!" % d)
  81         os.mkdir(d)
  82
  83 def spawn_and_check(prog, args):
  84         status = os.spawnv(os.P_WAIT, prog, [prog] + args)
  85         if status > 0:
  86                 raise SafeException("Program '%s' failed with exit code %d" % (prog, status))
  87         elif status < 0:
  88                 raise SafeException("Program '%s' failed with signal %d" % (prog, -status))
  89
  90 def spawn_and_check_maybe_sandboxed(readable, writable, tmpdir, prog, args):
  91         child = spawn_maybe_sandboxed(readable, writable, tmpdir, prog, args)
  92         status = child.wait()
  93         if status > 0:
  94                 raise SafeException('Command failed with exit status %d' %  status)
  95         elif status < 0:
  96                 raise SafeException('Command failed with signal %d' % -status)
  97
  98 def spawn_maybe_sandboxed(readable, writable, tmpdir, prog, args):
  99         """spawn prog, with (only) the 'writable' directories writable if sandboxing is available.
 100         The readable directories will be readable, as well as various standard locations.
 101         If no sandbox is available, run without a sandbox."""
 102
 103         USE_PLASH = 'USE_PLASH_0COMPILE'
 104
 105         assert os.path.isabs(prog)
 106         _pola_run = find_in_path('pola-run')
 107
 108         if _pola_run is None:
 109                 #print "Not using sandbox (plash not installed)"
 110                 use_plash = False
 111         else:
 112                 use_plash = os.environ.get(USE_PLASH, '').lower() or 'not set'
 113                 if use_plash in ('not set', 'false'):
 114                         print "Not using plash: $%s is %s" % (USE_PLASH, use_plash)
 115                         use_plash = False
 116                 elif use_plash == 'true':
 117                         use_plash = True
 118                 else:
 119                         raise Exception('$%s must be "true" or "false", not "%s"' % (USE_PLASH, use_plash))
 120
 121         if not use_plash:
 122                 return subprocess.Popen([prog] + args)
 123
 124         print "Using plash to sandbox the build..."
 125
 126         # We have pola-shell :-)
 127         pola_args = ['--prog', prog, '-B']
 128         for a in args:
 129                 pola_args += ['-a', a]
 130         for r in readable:
 131                 pola_args += ['-f', r]
 132         for w in writable:
 133                 pola_args += ['-fw', w]
 134         pola_args += ['-tw', '/tmp', tmpdir]
 135         os.environ['TMPDIR'] = '/tmp'
 136         return subprocess.Popen([_pola_run] + pola_args)
 137
 138 def get_arch_name():
 139         target_os = canonicalize_os(uname[0])
 140         target_machine = canonicalize_machine(uname[4])
 141         if target_os == 'Darwin' and target_machine == 'i386':
 142                 # this system detection shell script comes from config.guess (20090918):
 143                 CC = os.getenv("CC_FOR_BUILD") or os.getenv("CC") or os.getenv("HOST_CC") or "cc"
 144                 process = subprocess.Popen("(echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | " +
 145                                            "(CCOPTS= %s -E - 2>/dev/null) | " % CC +
 146                                            "grep IS_64BIT_ARCH >/dev/null", stdout=subprocess.PIPE, shell=True)
 147                 output, error = process.communicate()
 148                 retcode = process.poll()
 149                 if retcode == 0:
 150                         target_machine='x86_64'
 151         if target_machine in ('i585', 'i686'):
 152                 target_machine = 'i486' # (sensible default)
 153         return target_os + '-' + target_machine
 154
 155 class BuildEnv:
 156         def __init__(self, need_config = True):
 157                 if need_config and not os.path.isfile(ENV_FILE):
 158                         raise SafeException("Run 0compile from a directory containing a '%s' file" % ENV_FILE)
 159
 160                 self.config = ConfigParser.RawConfigParser()
 161                 self.config.add_section('compile')
 162                 self.config.set('compile', 'download-base-url', '')
 163                 self.config.set('compile', 'version-modifier', '')
 164                 self.config.set('compile', 'interface', '')
 165                 self.config.set('compile', 'selections', '')
 166                 self.config.set('compile', 'metadir', '0install')
 167                 self.config.set('compile', 'distdir', '')
 168
 169                 self.config.read(ENV_FILE)
 170
 171                 self._selections = None
 172
 173                 return
 174
 175         @property
 176         def iface_name(self):
 177                 iface_name = os.path.basename(self.interface)
 178                 if iface_name.endswith('.xml'):
 179                         iface_name = iface_name[:-4]
 180                 iface_name = iface_name.replace(' ', '-')
 181                 if iface_name.endswith('-src'):
 182                         iface_name = iface_name[:-4]
 183                 return iface_name
 184
 185         interface = property(lambda self: model.canonical_iface_uri(self.config.get('compile', 'interface')))
 186
 187         @property
 188         def distdir(self):
 189                 distdir_name = self.config.get('compile', 'distdir')
 190                 if not distdir_name:
 191                         arch = self.target_arch.replace('*', 'any')
 192                         distdir_name = self.iface_name.lower()
 193                         distdir_name += '-' + arch.lower()
 194                 assert os.path.dirname(distdir_name) == ''
 195                 return os.path.realpath(distdir_name)
 196
 197         def get_binary_template(self):
 198                 """Find the <compile:implementation> element for the selected compile command, if any"""
 199                 sels = self.get_selections()
 200                 if sels.commands:
 201                         for elem in sels.commands[0].qdom.childNodes:
 202                                 if elem.uri == XMLNS_0COMPILE and elem.name == 'implementation':
 203                                         return elem
 204                 return None
 205
 206         @property
 207         def metadir(self):
 208                 metadir = self.config.get('compile', 'metadir')
 209                 assert not os.path.isabs(metadir)
 210                 return join(self.distdir, metadir)
 211
 212         @property
 213         def local_iface_file(self):
 214                 return join(self.metadir, 'feed.xml')
 215
 216         @property
 217         def target_arch(self):
 218                 temp = self.get_binary_template()
 219                 arch = temp and temp.getAttribute('arch')
 220                 return arch or get_arch_name()
 221
 222         @property
 223         def version_modifier(self):
 224                 vm = self.config.get('compile', 'version-modifier')
 225                 if vm: return vm
 226                 if self.user_srcdir:
 227                         return '-1'
 228                 return ''
 229
 230         @property
 231         def archive_stem(self):
 232                 # Use the version that we actually built, not the version we would build now
 233                 feed = self.load_built_feed()
 234                 assert len(feed.implementations) == 1
 235                 version = feed.implementations.values()[0].get_version()
 236
 237                 # Don't use the feed's name, as it may contain the version number
 238                 name = feed.get_name().lower().replace(' ', '-')
 239                 arch = self.target_arch.lower().replace('*-*', 'bin').replace('*', 'any')
 240
 241                 return '%s-%s-%s' % (name, arch, version)
 242
 243         def load_built_feed(self):
 244                 path = self.local_iface_file
 245                 stream = file(path)
 246                 try:
 247                         feed = model.ZeroInstallFeed(qdom.parse(stream), local_path = path)
 248                 finally:
 249                         stream.close()
 250                 return feed
 251
 252         def load_built_selections(self):
 253                 path = join(self.metadir, 'build-environment.xml')
 254                 if os.path.exists(path):
 255                         stream = file(path)
 256                         try:
 257                                 return selections.Selections(qdom.parse(stream))
 258                         finally:
 259                                 stream.close()
 260                 return None
 261
 262         @property
 263         def download_base_url(self):
 264                 return self.config.get('compile', 'download-base-url')
 265
 266         def chosen_impl(self, uri):
 267                 sels = self.get_selections()
 268                 assert uri in sels.selections
 269                 return sels.selections[uri]
 270
 271         @property
 272         def local_download_iface(self):
 273                 impl, = self.load_built_feed().implementations.values()
 274                 return '%s-%s.xml' % (self.iface_name, impl.get_version())
 275
 276         def save(self):
 277                 stream = file(ENV_FILE, 'w')
 278                 try:
 279                         self.config.write(stream)
 280                 finally:
 281                         stream.close()
 282
 283         def get_selections(self, prompt = False):
 284                 if self._selections:
 285                         assert not prompt
 286                         return self._selections
 287
 288                 selections_file = self.config.get('compile', 'selections')
 289                 if selections_file:
 290                         if prompt:
 291                                 raise SafeException("Selections are fixed by %s" % selections_file)
 292                         stream = file(selections_file)
 293                         try:
 294                                 self._selections = selections.Selections(qdom.parse(stream))
 295                         finally:
 296                                 stream.close()
 297                         from zeroinstall.injector import handler
 298                         from zeroinstall.injector.config import load_config
 299                         if os.isatty(1):
 300                                 h = handler.ConsoleHandler()
 301                         else:
 302                                 h = handler.Handler()
 303                         config = load_config(h)
 304                         blocker = self._selections.download_missing(config)
 305                         if blocker:
 306                                 print "Waiting for selected implementations to be downloaded..."
 307                                 h.wait_for_blocker(blocker)
 308                 else:
 309                         command = install_prog + ['download', '--source', '--xml']
 310                         if prompt and '--console' not in install_prog:
 311                                 if os.name == 'nt':
 312                                         command[0] += '-win'
 313                                 command.append('--gui')
 314                         command.append(self.interface)
 315                         child = subprocess.Popen(command, stdout = subprocess.PIPE)
 316                         try:
 317                                 self._selections = selections.Selections(qdom.parse(child.stdout))
 318                         finally:
 319                                 if child.wait():
 320                                         raise SafeException(' '.join(repr(x) for x in command) + " failed (exit code %d)" % child.returncode)
 321
 322                 self.root_impl = self._selections.selections[self.interface]
 323
 324                 self.orig_srcdir = os.path.realpath(lookup(self.root_impl))
 325                 self.user_srcdir = None
 326
 327                 if os.path.isdir('src'):
 328                         self.user_srcdir = os.path.realpath('src')
 329                         if self.user_srcdir == self.orig_srcdir or \
 330                            self.user_srcdir.startswith(os.path.join(self.orig_srcdir, '')) or \
 331                            self.orig_srcdir.startswith(os.path.join(self.user_srcdir, '')):
 332                                 info("Ignoring 'src' directory because it coincides with %s",
 333                                         self.orig_srcdir)
 334                                 self.user_srcdir = None
 335
 336                 return self._selections
 337
 338         def get_build_changes(self):
 339                 sels = self.get_selections()
 340                 old_sels = self.load_built_selections()
 341                 changes = []
 342                 if old_sels:
 343                         # See if things have changed since the last build
 344                         all_ifaces = set(sels.selections) | set(old_sels.selections)
 345                         for x in all_ifaces:
 346                                 old_impl = old_sels.selections.get(x, no_impl)
 347                                 new_impl = sels.selections.get(x, no_impl)
 348                                 if old_impl.version != new_impl.version:
 349                                         changes.append("Version change for %s: %s -> %s" % (x, old_impl.version, new_impl.version))
 350                                 elif old_impl.id != new_impl.id:
 351                                         changes.append("Version change for %s: %s -> %s" % (x, old_impl.id, new_impl.id))
 352                 return changes
 353
 354 def depth(node):
 355         root = node.ownerDocument.documentElement
 356         depth = 0
 357         while node and node is not root:
 358                 node = node.parentNode
 359                 depth += 1
 360         return depth
 361
 362 def parse_bool(s):
 363         if s == 'true': return True
 364         if s == 'false': return False
 365         raise SafeException('Expected "true" or "false" but got "%s"' % s)