zeroinstall/injector/reader.py

   1 """
   2 Parses an XML interface into a Python representation.
   3 """
   4
   5 # Copyright (C) 2006, Thomas Leonard
   6 # See the README file for details, or visit http://0install.net.
   7
   8 import os
   9 import sys
  10 import shutil
  11 import time
  12 from logging import debug, warn, info
  13 from os.path import dirname
  14
  15 from zeroinstall.injector import basedir, qdom, distro
  16 from zeroinstall.injector.namespaces import *
  17 from zeroinstall.injector.model import *
  18 from zeroinstall.injector import model
  19 from zeroinstall import version, SafeException
  20
  21 class InvalidInterface(SafeException):
  22         """Raised when parsing an invalid interface."""
  23         def __init__(self, message, ex = None):
  24                 if ex:
  25                         message += "\n\n(exact error: %s)" % ex
  26                 SafeException.__init__(self, message)
  27
  28 _binding_names = frozenset(['environment'])
  29
  30 def _process_binding(e):
  31         if e.name == 'environment':
  32                 mode = {
  33                         None: EnvironmentBinding.PREPEND,
  34                         'prepend': EnvironmentBinding.PREPEND,
  35                         'append': EnvironmentBinding.APPEND,
  36                         'replace': EnvironmentBinding.REPLACE,
  37                 }[e.getAttribute('mode')]
  38
  39                 binding = EnvironmentBinding(e.getAttribute('name'),
  40                                              insert = e.getAttribute('insert'),
  41                                              default = e.getAttribute('default'),
  42                                              mode = mode)
  43                 if not binding.name: raise InvalidInterface("Missing 'name' in binding")
  44                 if binding.insert is None: raise InvalidInterface("Missing 'insert' in binding")
  45                 return binding
  46         else:
  47                 raise Exception("Unknown binding type '%s'" % e.name)
  48
  49 def _process_depends(item):
  50         # Note: also called from selections
  51         dep_iface = item.getAttribute('interface')
  52         if not dep_iface:
  53                 raise InvalidInterface("Missing 'interface' on <requires>")
  54         dependency = InterfaceDependency(dep_iface, metadata = item.attrs)
  55
  56         for e in item.childNodes:
  57                 if e.uri != XMLNS_IFACE: continue
  58                 if e.name in _binding_names:
  59                         dependency.bindings.append(_process_binding(e))
  60                 elif e.name == 'version':
  61                         dependency.restrictions.append(
  62                                 Restriction(not_before = parse_version(e.getAttribute('not-before')),
  63                                             before = parse_version(e.getAttribute('before'))))
  64         return dependency
  65
  66 def update_from_cache(interface):
  67         """Read a cached interface and any native feeds or user overrides.
  68         @param interface: the interface object to update
  69         @type interface: L{model.Interface}
  70         @return: True if cached version and user overrides loaded OK.
  71         False if upstream not cached. Local interfaces (starting with /) are
  72         always considered to be cached, although they are not actually stored in the cache.
  73         @rtype: bool"""
  74         interface.reset()
  75
  76         if interface.uri.startswith('/'):
  77                 debug("Loading local interface file '%s'", interface.uri)
  78                 update(interface, interface.uri, local = True)
  79                 interface.last_modified = int(os.stat(interface.uri).st_mtime)
  80                 cached = True
  81         else:
  82                 cached = basedir.load_first_cache(config_site, 'interfaces', escape(interface.uri))
  83                 if cached:
  84                         debug("Loading cached information for %s from %s", interface, cached)
  85                         update(interface, cached)
  86
  87         # Add the distribution package manager's version, if any
  88         path = basedir.load_first_data(config_site, 'native_feeds', model._pretty_escape(interface.uri))
  89         if path:
  90                 # Resolve any symlinks
  91                 info("Adding native packager feed '%s'", path)
  92                 interface.feeds.append(Feed(os.path.realpath(path), None, False))
  93
  94         update_user_overrides(interface)
  95
  96         # Special case: add our fall-back local copy of the injector as a feed
  97         if interface.uri == injector_gui_uri:
  98                 local_gui = os.path.join(os.path.abspath(dirname(dirname(__file__))), '0launch-gui', 'ZeroInstall-GUI.xml')
  99                 interface.feeds.append(Feed(local_gui, None, False))
 100
 101         return bool(cached)
 102
 103 def update_user_overrides(interface):
 104         """Update an interface with user-supplied information.
 105         @param interface: the interface object to update
 106         @type interface: L{model.Interface}"""
 107         user = basedir.load_first_config(config_site, config_prog,
 108                                            'user_overrides', escape(interface.uri))
 109         if not user:
 110                 return
 111
 112         root = qdom.parse(file(user))
 113
 114         last_checked = root.getAttribute('last-checked')
 115         if last_checked:
 116                 interface.last_checked = int(last_checked)
 117
 118         last_check_attempt = root.getAttribute('last-check-attempt')
 119         if last_check_attempt:
 120                 interface.last_check_attempt = int(last_check_attempt)
 121
 122         stability_policy = root.getAttribute('stability-policy')
 123         if stability_policy:
 124                 interface.set_stability_policy(stability_levels[str(stability_policy)])
 125
 126         for item in root.childNodes:
 127                 if item.uri != XMLNS_IFACE: continue
 128                 if item.name == 'implementation':
 129                         id = item.getAttribute('id')
 130                         assert id is not None
 131                         if not (id.startswith('/') or id.startswith('.') or id.startswith('package:')):
 132                                 assert '=' in id
 133                         impl = interface.implementations.get(id, None)
 134                         if not impl:
 135                                 debug("Ignoring user-override for unknown implementation %s in %s", id, interface)
 136                                 continue
 137
 138                         user_stability = item.getAttribute('user-stability')
 139                         if user_stability:
 140                                 impl.user_stability = stability_levels[str(user_stability)]
 141                 elif item.name == 'feed':
 142                         feed_src = item.getAttribute('src')
 143                         if not feed_src:
 144                                 raise InvalidInterface('Missing "src" attribute in <feed>')
 145                         interface.feeds.append(Feed(feed_src, item.getAttribute('arch'), True, langs = item.getAttribute('langs')))
 146
 147 def check_readable(interface_uri, source):
 148         """Test whether an interface file is valid.
 149         @param interface_uri: the interface's URI
 150         @type interface_uri: str
 151         @param source: the name of the file to test
 152         @type source: str
 153         @return: the modification time in src (usually just the mtime of the file)
 154         @rtype: int
 155         @raise InvalidInterface: If the source's syntax is incorrect,
 156         """
 157         tmp = Interface(interface_uri)
 158         try:
 159                 update(tmp, source)
 160         except InvalidInterface, ex:
 161                 info("Error loading interface:\n"
 162                         "Interface URI: %s\n"
 163                         "Local file: %s\n%s" %
 164                         (interface_uri, source, ex))
 165                 raise InvalidInterface("Error loading feed '%s':\n\n%s" % (interface_uri, ex))
 166         return tmp.last_modified
 167
 168 def _parse_time(t):
 169         try:
 170                 return long(t)
 171         except Exception, ex:
 172                 raise InvalidInterface("Date '%s' not in correct format (should be integer number "
 173                                         "of seconds since Unix epoch)\n%s" % (t, ex))
 174
 175 def _check_canonical_name(interface, root):
 176         "Ensure the uri= attribute in the interface file matches the interface we are trying to load"
 177         canonical_name = root.getAttribute('uri')
 178         if not canonical_name:
 179                 raise InvalidInterface("<interface> uri attribute missing")
 180         if canonical_name != interface.uri:
 181                 raise InvalidInterface("Incorrect URL used for feed.\n\n"
 182                                         "%s is given in the feed, but\n"
 183                                         "%s was requested" %
 184                                         (canonical_name, interface.uri))
 185
 186 def _get_long(elem, attr_name):
 187         val = elem.getAttribute(attr_name)
 188         if val is not None:
 189                 try:
 190                         val = long(val)
 191                 except ValueError, ex:
 192                         raise SafeException("Invalid value for integer attribute '%s': %s" % (attr_name, val))
 193         return val
 194
 195 def _merge_attrs(attrs, item):
 196         """Add each attribute of item to a copy of attrs and return the copy.
 197         @type attrs: {str: str}
 198         @type item: L{qdom.Element}
 199         @rtype: {str: str}
 200         """
 201         new = attrs.copy()
 202         for a in item.attrs:
 203                 new[str(a)] = item.attrs[a]
 204         return new
 205
 206 def update(interface, source, local = False):
 207         """Read in information about an interface.
 208         @param interface: the interface object to update
 209         @type interface: L{model.Interface}
 210         @param source: the name of the file to read
 211         @type source: str
 212         @param local: use file's mtime for last-modified, and uri attribute is ignored
 213         @raise InvalidInterface: if the source's syntax is incorrect
 214         @see: L{update_from_cache}, which calls this"""
 215         assert isinstance(interface, Interface)
 216
 217         try:
 218                 root = qdom.parse(file(source))
 219         except IOError, ex:
 220                 if ex.errno == 2:
 221                         raise InvalidInterface("Feed not found. Perhaps this is a local feed that no longer exists? You can remove it from the list of feeds in that case.", ex)
 222                 raise InvalidInterface("Can't read file", ex)
 223         except Exception, ex:
 224                 raise InvalidInterface("Invalid XML", ex)
 225
 226         if not local:
 227                 _check_canonical_name(interface, root)
 228                 time_str = root.getAttribute('last-modified')
 229                 if time_str:
 230                         # Old style cached items use an attribute
 231                         interface.last_modified = _parse_time(time_str)
 232                 else:
 233                         # New style items have the mtime in the signature,
 234                         # but for quick access we use the mtime of the file
 235                         interface.last_modified = int(os.stat(source).st_mtime)
 236         main = root.getAttribute('main')
 237         if main:
 238                 interface.main = main
 239
 240         min_injector_version = root.getAttribute('min-injector-version')
 241         if min_injector_version:
 242                 try:
 243                         min_ints = map(int, min_injector_version.split('.'))
 244                 except ValueError, ex:
 245                         raise InvalidInterface("Bad version number '%s'" % min_injector_version)
 246                 injector_version = map(int, version.split('.'))
 247                 if min_ints > injector_version:
 248                         raise InvalidInterface("This interface requires version %s or later of "
 249                                                 "the Zero Install injector, but I am only version %s. "
 250                                                 "You can get a newer version from http://0install.net" %
 251                                                 (min_injector_version, version))
 252
 253         if local:
 254                 iface_dir = os.path.dirname(source)
 255         else:
 256                 iface_dir = None        # Can't have relative paths
 257
 258         for x in root.childNodes:
 259                 if x.uri != XMLNS_IFACE:
 260                         interface.add_metadata(x)
 261                         continue
 262                 if x.name == 'name':
 263                         interface.name = interface.name or x.content
 264                 elif x.name == 'description':
 265                         interface.description = interface.description or x.content
 266                 elif x.name == 'summary':
 267                         interface.summary = interface.summary or x.content
 268                 elif x.name == 'feed-for':
 269                         feed_iface = x.getAttribute('interface')
 270                         if not feed_iface:
 271                                 raise InvalidInterface('Missing "interface" attribute in <feed-for>')
 272                         interface.feed_for[feed_iface] = True
 273                         # Bug report from a Debian/stable user that --feed gets the wrong value.
 274                         # Can't reproduce (even in a Debian/stable chroot), but add some logging here
 275                         # in case it happens again.
 276                         debug("Is feed-for %s", feed_iface)
 277                 elif x.name == 'feed':
 278                         feed_src = x.getAttribute('src')
 279                         if not feed_src:
 280                                 raise InvalidInterface('Missing "src" attribute in <feed>')
 281                         if feed_src.startswith('http:') or local:
 282                                 interface.feeds.append(Feed(feed_src, x.getAttribute('arch'), False, langs = x.getAttribute('langs')))
 283                         else:
 284                                 raise InvalidInterface("Invalid feed URL '%s'" % feed_src)
 285                 else:
 286                         interface.add_metadata(x)
 287
 288         def process_group(group, group_attrs, base_depends, base_bindings):
 289                 for item in group.childNodes:
 290                         if item.uri != XMLNS_IFACE: continue
 291
 292                         if item.name not in ('group', 'implementation', 'package-implementation'):
 293                                 continue
 294
 295                         depends = base_depends[:]
 296                         bindings = base_bindings[:]
 297
 298                         item_attrs = _merge_attrs(group_attrs, item)
 299
 300                         # We've found a group or implementation. Scan for dependencies
 301                         # and bindings. Doing this here means that:
 302                         # - We can share the code for groups and implementations here.
 303                         # - The order doesn't matter, because these get processed first.
 304                         # A side-effect is that the document root cannot contain
 305                         # these.
 306                         for child in item.childNodes:
 307                                 if child.uri != XMLNS_IFACE: continue
 308                                 if child.name == 'requires':
 309                                         dep = _process_depends(child)
 310                                         depends.append(dep)
 311                                 elif child.name in _binding_names:
 312                                         bindings.append(_process_binding(child))
 313
 314                         if item.name == 'group':
 315                                 process_group(item, item_attrs, depends, bindings)
 316                         elif item.name == 'implementation':
 317                                 process_impl(item, item_attrs, depends, bindings)
 318                         elif item.name == 'package-implementation':
 319                                 process_native_impl(item, item_attrs, depends)
 320                         else:
 321                                 assert 0
 322
 323         def process_impl(item, item_attrs, depends, bindings):
 324                 id = item.getAttribute('id')
 325                 if id is None:
 326                         raise InvalidInterface("Missing 'id' attribute on %s" % item)
 327                 if local and (id.startswith('/') or id.startswith('.')):
 328                         impl = interface.get_impl(os.path.abspath(os.path.join(iface_dir, id)))
 329                 else:
 330                         if '=' not in id:
 331                                 raise InvalidInterface('Invalid "id"; form is "alg=value" (got "%s")' % id)
 332                         alg, sha1 = id.split('=')
 333                         try:
 334                                 long(sha1, 16)
 335                         except Exception, ex:
 336                                 raise InvalidInterface('Bad SHA1 attribute: %s' % ex)
 337                         impl = interface.get_impl(id)
 338
 339                 impl.metadata = item_attrs
 340                 try:
 341                         version = item_attrs['version']
 342                         version_mod = item_attrs.get('version-modifier', None)
 343                         if version_mod: version += version_mod
 344                 except KeyError:
 345                         raise InvalidInterface("Missing version attribute")
 346                 impl.version = parse_version(version)
 347
 348                 item_main = item_attrs.get('main', None)
 349                 if item_main and item_main.startswith('/'):
 350                         raise InvalidInterface("'main' attribute must be relative, but '%s' starts with '/'!" %
 351                                                 item_main)
 352                 impl.main = item_main
 353
 354                 impl.released = item_attrs.get('released', None)
 355                 impl.langs = item_attrs.get('langs', None)
 356
 357                 size = item.getAttribute('size')
 358                 if size:
 359                         impl.size = long(size)
 360                 impl.arch = item_attrs.get('arch', None)
 361                 try:
 362                         stability = stability_levels[str(item_attrs['stability'])]
 363                 except KeyError:
 364                         stab = str(item_attrs['stability'])
 365                         if stab != stab.lower():
 366                                 raise InvalidInterface('Stability "%s" invalid - use lower case!' % item_attrs.stability)
 367                         raise InvalidInterface('Stability "%s" invalid' % item_attrs['stability'])
 368                 if stability >= preferred:
 369                         raise InvalidInterface("Upstream can't set stability to preferred!")
 370                 impl.upstream_stability = stability
 371
 372                 impl.bindings = bindings
 373                 impl.requires = depends
 374
 375                 for elem in item.childNodes:
 376                         if elem.uri != XMLNS_IFACE: continue
 377                         if elem.name == 'archive':
 378                                 url = elem.getAttribute('href')
 379                                 if not url:
 380                                         raise InvalidInterface("Missing href attribute on <archive>")
 381                                 size = elem.getAttribute('size')
 382                                 if not size:
 383                                         raise InvalidInterface("Missing size attribute on <archive>")
 384                                 impl.add_download_source(url = url, size = long(size),
 385                                                 extract = elem.getAttribute('extract'),
 386                                                 start_offset = _get_long(elem, 'start-offset'),
 387                                                 type = elem.getAttribute('type'))
 388                         elif elem.name == 'recipe':
 389                                 recipe = Recipe()
 390                                 for recipe_step in elem.childNodes:
 391                                         if recipe_step.uri == XMLNS_IFACE and recipe_step.name == 'archive':
 392                                                 url = recipe_step.getAttribute('href')
 393                                                 if not url:
 394                                                         raise InvalidInterface("Missing href attribute on <archive>")
 395                                                 size = recipe_step.getAttribute('size')
 396                                                 if not size:
 397                                                         raise InvalidInterface("Missing size attribute on <archive>")
 398                                                 recipe.steps.append(DownloadSource(None, url = url, size = long(size),
 399                                                                 extract = recipe_step.getAttribute('extract'),
 400                                                                 start_offset = _get_long(recipe_step, 'start-offset'),
 401                                                                 type = recipe_step.getAttribute('type')))
 402                                         else:
 403                                                 info("Unknown step '%s' in recipe; skipping recipe", recipe_step.name)
 404                                                 break
 405                                 else:
 406                                         impl.download_sources.append(recipe)
 407
 408         def process_native_impl(item, item_attrs, depends):
 409                 package = item_attrs.get('package', None)
 410                 if package is None:
 411                         raise InvalidInterface("Missing 'package' attribute on %s" % item)
 412
 413                 def factory(id):
 414                         assert id.startswith('package:')
 415                         impl = interface.get_impl(id)
 416
 417                         impl.metadata = item_attrs
 418
 419                         item_main = item_attrs.get('main', None)
 420                         if item_main and not item_main.startswith('/'):
 421                                 raise InvalidInterface("'main' attribute must be absolute, but '%s' doesn't start with '/'!" %
 422                                                         item_main)
 423                         impl.main = item_main
 424                         impl.upstream_stability = packaged
 425                         impl.requires = depends
 426
 427                         return impl
 428
 429                 distro.host_distribution.get_package_info(package, factory)
 430
 431
 432         process_group(root,
 433                 {'stability': 'testing',
 434                  'main' : root.getAttribute('main') or None,
 435                 },
 436                 [], [])