From d6f66c31953b5e7f25dce7c69085bcae955246d0 Mon Sep 17 00:00:00 2001 From: Thomas Leonard Date: Sun, 21 Feb 2010 16:45:38 +0000 Subject: [PATCH] Implementation.id doesn't have to be path or digest If the local-path attribute is present, this is the path and the ID can be any string. For non-local implementations, we keep a list of digests (for different algorithms). If the ID contains an '=' then it is added to this list, for backwards compatibility. The Selections XML format also supports and the local-path attribute. --- tests/selections.xml | 4 ++- tests/test-gui | 2 +- tests/testdistro.py | 5 +++- tests/testdownload.py | 4 +-- tests/testmodel.py | 20 ++----------- tests/testselections.py | 16 ++++++++++- zeroinstall/injector/fetch.py | 21 ++++++++------ zeroinstall/injector/model.py | 57 +++++++++++++++++++++++--------------- zeroinstall/injector/policy.py | 2 +- zeroinstall/injector/run.py | 2 +- zeroinstall/injector/selections.py | 40 ++++++++++++++++++++++---- zeroinstall/injector/solver.py | 4 +-- zeroinstall/zerostore/__init__.py | 23 +++++++++------ 13 files changed, 128 insertions(+), 72 deletions(-) diff --git a/tests/selections.xml b/tests/selections.xml index f540ea4..6161dc5 100644 --- a/tests/selections.xml +++ b/tests/selections.xml @@ -1 +1,3 @@ - + + + diff --git a/tests/test-gui b/tests/test-gui index fa04702..75d40a0 100755 --- a/tests/test-gui +++ b/tests/test-gui @@ -2,6 +2,6 @@ import os, sys mydir = os.path.dirname(__file__) output = """ -""" % mydir +""" % mydir sys.stdout.write('Length:%8x\n' % len(output)) sys.stdout.write(output) diff --git a/tests/testdistro.py b/tests/testdistro.py index bf436b6..16543c7 100755 --- a/tests/testdistro.py +++ b/tests/testdistro.py @@ -27,7 +27,10 @@ class TestDistro(BaseTest): BaseTest.tearDown(self) def factory(self, id): - return self.feed._get_impl(id) + impl = model.DistributionImplementation(self.feed, id) + assert id not in self.feed.implementations + self.feed.implementations[id] = impl + return impl def testDefault(self): host = distro.Distribution() diff --git a/tests/testdownload.py b/tests/testdownload.py index 934dd10..9be5322 100755 --- a/tests/testdownload.py +++ b/tests/testdownload.py @@ -168,7 +168,7 @@ class TestDownload(BaseTest): self.child = server.handle_requests('Hello.xml', '6FCF121BE2390E0B.gpg', '/key-info/key/DE937DD411906ACF7C263B396FCF121BE2390E0B', 'HelloWorld.tgz') sys.stdin = Reply("Y\n") _download_missing_selections(Options(), sels) - path = iface_cache.iface_cache.stores.lookup(sels.selections['http://example.com:8000/Hello.xml'].id) + path = iface_cache.iface_cache.stores.lookup_any(sels.selections['http://example.com:8000/Hello.xml'].digests) assert os.path.exists(os.path.join(path, 'HelloWorld', 'main')) assert sels.download_missing(iface_cache.iface_cache, None) is None @@ -189,7 +189,7 @@ class TestDownload(BaseTest): handler.wait_for_blocker(fetcher.download_and_import_feed('http://example.com:8000/Hello.xml', iface_cache.iface_cache)) _download_missing_selections(Options(), sels) - path = iface_cache.iface_cache.stores.lookup(sels.selections['http://example.com:8000/Hello.xml'].id) + path = iface_cache.iface_cache.stores.lookup_any(sels.selections['http://example.com:8000/Hello.xml'].digests) assert os.path.exists(os.path.join(path, 'HelloWorld', 'main')) assert sels.download_missing(iface_cache.iface_cache, None) is None diff --git a/tests/testmodel.py b/tests/testmodel.py index f87ebf1..409dcee 100755 --- a/tests/testmodel.py +++ b/tests/testmodel.py @@ -98,23 +98,9 @@ class TestModel(BaseTest): i.set_stability_policy(model.buggy) self.assertEquals(model.buggy, i.stability_policy) - def testGetImpl(self): - i = model.Interface('http://foo') - feed = model.ZeroInstallFeed(empty_feed, local_path = '/foo') - a = feed._get_impl('foo') - b = feed._get_impl('bar') - try: - c = feed._get_impl('foo') - assert 0 - except: - pass - assert a and b - assert a is not b - assert isinstance(a, model.ZeroInstallImplementation) - def testImpl(self): i = model.Interface('http://foo') - a = model.ZeroInstallImplementation(i, 'foo') + a = model.ZeroInstallImplementation(i, 'foo', None) assert a.id == 'foo' assert a.size == a.version == a.user_stability == None assert a.arch == a.upstream_stability == None @@ -131,13 +117,13 @@ class TestModel(BaseTest): self.assertEquals('1.2.3-rc2-post', a.get_version()) assert str(a) == 'foo' - b = model.ZeroInstallImplementation(i, 'foo') + b = model.ZeroInstallImplementation(i, 'foo', None) b.version = [1,2,1] assert b > a def testDownloadSource(self): f = model.ZeroInstallFeed(empty_feed, local_path = '/foo') - a = model.ZeroInstallImplementation(f, 'foo') + a = model.ZeroInstallImplementation(f, 'foo', None) a.add_download_source('ftp://foo', 1024, None) a.add_download_source('ftp://foo.tgz', 1025, 'foo') assert a.download_sources[0].url == 'ftp://foo' diff --git a/tests/testselections.py b/tests/testselections.py index b3cda33..25804cc 100755 --- a/tests/testselections.py +++ b/tests/testselections.py @@ -1,12 +1,14 @@ #!/usr/bin/env python2.5 from basetest import BaseTest from StringIO import StringIO -import sys +import sys, os import unittest sys.path.insert(0, '..') from zeroinstall.injector import selections, model, reader, policy, iface_cache, namespaces, qdom +mydir = os.path.dirname(os.path.abspath(__file__)) + class TestSelections(BaseTest): def testSelections(self): p = policy.Policy('http://foo/Source.xml', src = True) @@ -51,6 +53,7 @@ class TestSelections(BaseTest): dep = sels[1].dependencies[0] self.assertEquals('http://foo/Compiler.xml', dep.interface) self.assertEquals(1, len(dep.bindings)) + self.assertEquals(["sha1=345"], sels[0].digests) s1 = selections.Selections(p) s1.selections['http://foo/Source.xml'].attrs['http://namespace foo'] = 'bar' @@ -62,6 +65,17 @@ class TestSelections(BaseTest): s2 = selections.Selections(root) assertSel(s2) + + def testLocalPath(self): + iface = os.path.join(mydir, "Local.xml") + p = policy.Policy(iface) + p.recalculate() + s1 = selections.Selections(p) + xml = s1.toDOM().toxml("utf-8") + root = qdom.parse(StringIO(xml)) + s2 = selections.Selections(root) + local_path = s2.selections[iface].local_path + assert os.path.isdir(local_path), local_path suite = unittest.makeSuite(TestSelections) if __name__ == '__main__': diff --git a/zeroinstall/injector/fetch.py b/zeroinstall/injector/fetch.py index 34986af..b6013b9 100644 --- a/zeroinstall/injector/fetch.py +++ b/zeroinstall/injector/fetch.py @@ -300,10 +300,16 @@ class Fetcher(object): assert retrieval_method from zeroinstall.zerostore import manifest - alg = impl.id.split('=', 1)[0] - if alg not in manifest.algorithms: - raise SafeException(_("Unknown digest algorithm '%(algorithm)s' for '%(implementation)s' version %(version)s") % - {'algorithm': alg, 'implementation': impl.feed.get_name(), 'version': impl.get_version()}) + for required_digest in impl.digests: + alg = required_digest.split('=', 1)[0] + if alg in manifest.algorithms: + break + else: + if not impl.digests: + raise SafeException(_("No given for '%(implementation)s' version %(version)s") % + {'implementation': impl.feed.get_name(), 'version': impl.get_version()}) + raise SafeException(_("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s") % + {'algorithms': impl.digests, 'implementation': impl.feed.get_name(), 'version': impl.get_version()}) @tasks.async def download_impl(): @@ -313,9 +319,9 @@ class Fetcher(object): tasks.check(blocker) stream.seek(0) - self._add_to_cache(stores, retrieval_method, stream) + self._add_to_cache(required_digest, stores, retrieval_method, stream) elif isinstance(retrieval_method, Recipe): - blocker = self.cook(impl.id, retrieval_method, stores, force, impl_hint = impl) + blocker = self.cook(required_digest, retrieval_method, stores, force, impl_hint = impl) yield blocker tasks.check(blocker) else: @@ -324,9 +330,8 @@ class Fetcher(object): self.handler.impl_added_to_store(impl) return download_impl() - def _add_to_cache(self, stores, retrieval_method, stream): + def _add_to_cache(self, required_digest, stores, retrieval_method, stream): assert isinstance(retrieval_method, DownloadSource) - required_digest = retrieval_method.implementation.id url = retrieval_method.url stores.add_archive_to_cache(required_digest, stream, retrieval_method.url, retrieval_method.extract, type = retrieval_method.type, start_offset = retrieval_method.start_offset or 0) diff --git a/zeroinstall/injector/model.py b/zeroinstall/injector/model.py index 5e42078..9d27b06 100644 --- a/zeroinstall/injector/model.py +++ b/zeroinstall/injector/model.py @@ -15,7 +15,7 @@ well-known variables. from zeroinstall import _ import os, re -from logging import info, debug +from logging import info, debug, warn from zeroinstall import SafeException, version from zeroinstall.injector.namespaces import XMLNS_IFACE @@ -441,14 +441,19 @@ class DistributionImplementation(Implementation): class ZeroInstallImplementation(Implementation): """An implementation where all the information comes from Zero Install. + @ivar digests: a list of "algorith=value" strings (since 0.45) + @type digests: [str] @since: 0.28""" - __slots__ = ['os', 'size'] + __slots__ = ['os', 'size', 'digests', 'local_path'] - def __init__(self, feed, id): + def __init__(self, feed, id, local_path): """id can be a local path (string starting with /) or a manifest hash (eg "sha1=XXX")""" + assert not id.startswith('package:'), id Implementation.__init__(self, feed, id) self.size = None self.os = None + self.digests = [] + self.local_path = local_path # Deprecated dependencies = property(lambda self: dict([(x.interface, x) for x in self.requires @@ -461,12 +466,6 @@ class ZeroInstallImplementation(Implementation): def set_arch(self, arch): self.os, self.machine = _split_arch(arch) arch = property(lambda self: _join_arch(self.os, self.machine), set_arch) - - @property - def local_path(self): - if self.id.startswith('/'): - return self.id - return None class Interface(object): """An Interface represents some contract of behaviour. @@ -694,16 +693,16 @@ class ZeroInstallFeed(object): if id is None: raise InvalidInterface(_("Missing 'id' attribute on %s") % item) if local_dir and (id.startswith('/') or id.startswith('.')): - impl = self._get_impl(os.path.abspath(os.path.join(local_dir, id))) + id = os.path.abspath(os.path.join(local_dir, id)) + impl = ZeroInstallImplementation(self, id, id) else: - if '=' not in id: - raise InvalidInterface(_('Invalid "id"; form is "alg=value" (got "%s")') % id) - alg, sha1 = id.split('=') - try: - long(sha1, 16) - except Exception, ex: - raise InvalidInterface(_('Bad SHA1 attribute: %s') % ex) - impl = self._get_impl(id) + impl = ZeroInstallImplementation(self, id, None) + if '=' in id: + # In older feeds, the ID was the (single) digest + impl.digests.append(id) + if id in self.implementations: + warn(_("Duplicate ID '%s' in feed '%s'"), id, self) + self.implementations[id] = impl impl.metadata = item_attrs try: @@ -756,6 +755,11 @@ class ZeroInstallFeed(object): extract = elem.getAttribute('extract'), start_offset = _get_long(elem, 'start-offset'), type = elem.getAttribute('type')) + elif elem.name == 'manifest-digest': + for aname, avalue in elem.attrs.iteritems(): + if ' ' not in aname: + impl.digests.append('%s=%s' % (aname, avalue)) + impl.digests.sort() elif elem.name == 'recipe': recipe = Recipe() for recipe_step in elem.childNodes: @@ -783,7 +787,10 @@ class ZeroInstallFeed(object): def factory(id): assert id.startswith('package:') - impl = self._get_impl(id) + if id in self.implementations: + warn(_("Duplicate ID '%s' for DistributionImplementation"), id) + impl = DistributionImplementation(self, id) + self.implementations[id] = impl impl.metadata = item_attrs @@ -813,12 +820,18 @@ class ZeroInstallFeed(object): def __repr__(self): return _("") % self.url + """@deprecated""" def _get_impl(self, id): assert id not in self.implementations - if id.startswith('package:'): - impl = DistributionImplementation(self, id) + + if id.startswith('.') or id.startswith('/'): + id = os.path.abspath(os.path.join(self.url, id)) + local_path = id + impl = ZeroInstallImplementation(self, id, local_path) else: - impl = ZeroInstallImplementation(self, id) + impl = ZeroInstallImplementation(self, id, None) + impl.digests.append(id) + self.implementations[id] = impl return impl diff --git a/zeroinstall/injector/policy.py b/zeroinstall/injector/policy.py index e4e9804..59cef1a 100644 --- a/zeroinstall/injector/policy.py +++ b/zeroinstall/injector/policy.py @@ -221,7 +221,7 @@ class Policy(object): @rtype: str @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first.""" assert isinstance(impl, Implementation) - return impl.local_path or iface_cache.stores.lookup(impl.id) + return impl.local_path or iface_cache.stores.lookup_any(impl.digests) def get_implementation(self, interface): """Get the chosen implementation. diff --git a/zeroinstall/injector/run.py b/zeroinstall/injector/run.py index 2f08d1b..459e9fc 100644 --- a/zeroinstall/injector/run.py +++ b/zeroinstall/injector/run.py @@ -59,7 +59,7 @@ def _do_bindings(impl, bindings): do_env_binding(b, _get_implementation_path(impl)) def _get_implementation_path(impl): - return impl.local_path or iface_cache.stores.lookup(impl.id) + return impl.local_path or iface_cache.stores.lookup_any(impl.digests) def execute_selections(selections, prog_args, dry_run = False, main = None, wrapper = None): """Execute program. On success, doesn't return. On failure, raises an Exception. diff --git a/zeroinstall/injector/selections.py b/zeroinstall/injector/selections.py index c2a5f0c..b6456d3 100644 --- a/zeroinstall/injector/selections.py +++ b/zeroinstall/injector/selections.py @@ -19,15 +19,19 @@ class Selection(object): @type dependencies: [L{model.Dependency}] @ivar attrs: XML attributes map (name is in the format "{namespace} {localName}") @type attrs: {str: str} + @ivar digests: a list of manifest digests + @type digests: [str] @ivar version: the implementation's version number @type version: str""" - __slots__ = ['bindings', 'dependencies', 'attrs'] + __slots__ = ['bindings', 'dependencies', 'attrs', 'digests'] - def __init__(self, dependencies, bindings = None, attrs = None): + def __init__(self, dependencies, bindings = None, attrs = None, digests = None): if bindings is None: bindings = [] + if digests is None: digests = [] self.dependencies = dependencies self.bindings = bindings self.attrs = attrs + self.digests = digests assert self.interface assert self.id @@ -42,6 +46,9 @@ class Selection(object): @property def local_path(self): + local_path = self.attrs.get('local-path', None) + if local_path: + return local_path if self.id.startswith('/'): return self.id return None @@ -90,8 +97,10 @@ class Selections(object): attrs['version'] = impl.get_version() attrs['interface'] = needed_iface.uri attrs['from-feed'] = impl.feed.url + if impl.local_path: + attrs['local-path'] = impl.local_path - self.selections[needed_iface.uri] = Selection(solver_requires[needed_iface], impl.bindings, attrs) + self.selections[needed_iface.uri] = Selection(solver_requires[needed_iface], impl.bindings, attrs, impl.digests) def _init_from_qdom(self, root): """Parse and load a selections document. @@ -107,6 +116,7 @@ class Selections(object): requires = [] bindings = [] + digests = [] for dep_elem in selection.childNodes: if dep_elem.uri != XMLNS_IFACE: continue @@ -115,8 +125,16 @@ class Selections(object): elif dep_elem.name == 'requires': dep = process_depends(dep_elem) requires.append(dep) + elif dep_elem.name == 'manifest-digest': + for aname, avalue in dep_elem.attrs.iteritems(): + digests.append('%s=%s' % (aname, avalue)) - s = Selection(requires, bindings, selection.attrs) + # For backwards compatibility, allow getting the digest from the ID + sel_id = selection.attrs['id'] + if (not digests) and '=' in sel_id: + digests.append(sel_id) + + s = Selection(requires, bindings, selection.attrs, digests) self.selections[selection.attrs['interface']] = s def toDOM(self): @@ -163,6 +181,14 @@ class Selections(object): # Don't bother writing from-feed attr if it's the same as the interface selection_elem.setAttributeNS(None, name, value) + if selection.digests: + manifest_digest = doc.createElementNS(XMLNS_IFACE, 'manifest-digest') + for digest in selection.digests: + aname, avalue = digest.split('=', 1) + assert ':' not in aname + manifest_digest.setAttribute(aname, avalue) + selection_elem.appendChild(manifest_digest) + for b in selection.bindings: selection_elem.appendChild(b._toxml(doc)) @@ -193,7 +219,7 @@ class Selections(object): return "Selections for " + self.interface def download_missing(self, iface_cache, fetcher): - """Cache all selected implementations are available. + """Check all selected implementations are available. Download any that are not present. @param iface_cache: cache to find feeds with download information @param fetcher: used to download missing implementations @@ -205,7 +231,7 @@ class Selections(object): for sel in self.selections.values(): if not sel.local_path: try: - iface_cache.stores.lookup(sel.id) + iface_cache.stores.lookup_any(sel.digests) except NotStored, ex: needed_downloads.append(sel) if not needed_downloads: @@ -216,6 +242,8 @@ class Selections(object): # We're missing some. For each one, get the feed it came from # and find the corresponding in that. This will # tell us where to get it from. + # Note: we look for an implementation with the same ID. Maybe we + # should check it has the same digest(s) too? needed_impls = [] for sel in needed_downloads: feed_url = sel.attrs.get('from-feed', None) or sel.attrs['interface'] diff --git a/zeroinstall/injector/solver.py b/zeroinstall/injector/solver.py index 9361bd7..2d3d31c 100644 --- a/zeroinstall/injector/solver.py +++ b/zeroinstall/injector/solver.py @@ -273,10 +273,10 @@ class DefaultSolver(Solver): if isinstance(impl, model.DistributionImplementation): return impl.installed if impl.local_path: - return os.path.exists(impl.id) + return os.path.exists(impl.local_path) else: try: - path = self.stores.lookup(impl.id) + path = self.stores.lookup_any(impl.digests) assert path return True except BadDigest: diff --git a/zeroinstall/zerostore/__init__.py b/zeroinstall/zerostore/__init__.py index 0a608c1..5389441 100644 --- a/zeroinstall/zerostore/__init__.py +++ b/zeroinstall/zerostore/__init__.py @@ -239,16 +239,21 @@ class Stores(object): self.stores.append(Store(directory)) def lookup(self, digest): + return self.lookup_any([digest]) + + def lookup_any(self, digests): """Search for digest in all stores.""" - assert digest - if '/' in digest or '=' not in digest: - raise BadDigest(_('Syntax error in digest (use ALG=VALUE, not %s)') % digest) - for store in self.stores: - path = store.lookup(digest) - if path: - return path - raise NotStored(_("Item with digest '%(digest)s' not found in stores. Searched:\n- %(stores)s") % - {'digest': digest, 'stores': '\n- '.join([s.dir for s in self.stores])}) + assert digests + for digest in digests: + assert digest + if '/' in digest or '=' not in digest: + raise BadDigest(_('Syntax error in digest (use ALG=VALUE, not %s)') % digest) + for store in self.stores: + path = store.lookup(digest) + if path: + return path + raise NotStored(_("Item with digests '%(digests)s' not found in stores. Searched:\n- %(stores)s") % + {'digests': digests, 'stores': '\n- '.join([s.dir for s in self.stores])}) def add_dir_to_cache(self, required_digest, dir): """Add to the best writable cache. -- 2.11.4.GIT