From 6bee433ce77c7f26faab709a223d77f7e6d1e40e Mon Sep 17 00:00:00 2001 From: Thomas Leonard Date: Thu, 17 Jun 2010 21:34:24 +0100 Subject: [PATCH] Model information about distribution packages as separate feeds Before, reading in a feed containing a element would expand this to a list of packages provided by the distribution's package manager (installed or not) and include them in the feeds 'implementations'. Now, feed with such an element has a corresponding "distribution feed", which is the feed's URL prefixed with "distribuion:". This feed contains the distribution packages. Initially, it contains only installed packages. After updating it, it may also contain candidate (uninstalled) version. This has two advantages: - The loaded representation of feed no longer depends on the host distribution, which is cleaner. - Fetching candidates can be done asynchronously. --- tests/Native.xml | 12 ++++ tests/testdistro.py | 48 ++++++++++++--- tests/testdownload.py | 28 +++++++++ tests/testreader.py | 7 ++- tests/testsat.py | 3 + zeroinstall/injector/distro.py | 120 ++++++++++++++++++++++++++---------- zeroinstall/injector/fetch.py | 19 ++++++ zeroinstall/injector/iface_cache.py | 27 +++++++- zeroinstall/injector/model.py | 84 ++++++++++++------------- zeroinstall/injector/reader.py | 2 +- zeroinstall/injector/solver.py | 15 ++--- 11 files changed, 266 insertions(+), 99 deletions(-) create mode 100644 tests/Native.xml diff --git a/tests/Native.xml b/tests/Native.xml new file mode 100644 index 0000000..eacddae --- /dev/null +++ b/tests/Native.xml @@ -0,0 +1,12 @@ + + + + Native + Native + + + diff --git a/tests/testdistro.py b/tests/testdistro.py index d53c71e..cb56f42 100755 --- a/tests/testdistro.py +++ b/tests/testdistro.py @@ -5,9 +5,9 @@ from StringIO import StringIO import unittest sys.path.insert(0, '..') -from zeroinstall.injector import distro, model, qdom +from zeroinstall.injector import distro, model, qdom, iface_cache, handler -def parse_impls(impls, test_distro): +def parse_impls(impls): xml = """ Foo @@ -16,7 +16,7 @@ def parse_impls(impls, test_distro): %s """ % impls element = qdom.parse(StringIO(xml)) - return model.ZeroInstallFeed(element, "myfeed.xml", test_distro) + return model.ZeroInstallFeed(element, "myfeed.xml") class TestDistro(BaseTest): def setUp(self): @@ -78,8 +78,26 @@ class TestDistro(BaseTest): host.get_package_info('gimp', factory) self.assertEquals({}, self.feed.implementations) + # Initially, we only get information about the installed version... + host.get_package_info('python-bittorrent', factory) + self.assertEquals(1, len(self.feed.implementations)) + + # Tell distro to fetch information about candidates... + master_feed = parse_impls("""""") + h = handler.Handler() + candidates = host.fetch_candidates(master_feed) + # Async, so nothing yet... + self.feed = model.ZeroInstallFeed(empty_feed, local_path = '/empty.xml') + host.get_package_info('python-bittorrent', factory) + self.assertEquals(1, len(self.feed.implementations)) + + h.wait_for_blocker(candidates) + # Now we see the uninstalled package + self.feed = model.ZeroInstallFeed(empty_feed, local_path = '/empty.xml') host.get_package_info('python-bittorrent', factory) self.assertEquals(2, len(self.feed.implementations)) + + self.assertEquals(2, len(self.feed.implementations)) bittorrent_installed = self.feed.implementations['package:deb:python-bittorrent:3.4.2-10'] bittorrent_uninstalled = self.feed.implementations['package:deb:python-bittorrent:3.4.2-11.1'] self.assertEquals('3.4.2-10', bittorrent_installed.get_version()) @@ -108,24 +126,36 @@ class TestDistro(BaseTest): self.assertEquals('2.15.23-21', yast.get_version()) self.assertEquals('*-i586', yast.arch) - impls = parse_impls(""" + icache = iface_cache.IfaceCache(distro = rpm) + + feed = parse_impls(""" - """, rpm).implementations + """) + icache._feeds[feed.url] = feed + distro_feed_url = feed.get_distro_feed() + impls = icache.get_feed(distro_feed_url).implementations + self.assertEquals("distribution:myfeed.xml", distro_feed_url) assert len(impls) == 1, impls impl, = impls assert impl == 'package:rpm:yast2-update:2.15.23-21:i586' - impls = parse_impls(""" + feed = parse_impls(""" - """, rpm).implementations + """) + icache._feeds[feed.url] = feed + del icache._feeds['distribution:' + feed.url] + impls = icache.get_feed(feed.get_distro_feed()).implementations assert len(impls) == 2, impls - impls = parse_impls(""" + feed = parse_impls(""" - """, rpm).implementations + """) + icache._feeds[feed.url] = feed + del icache._feeds['distribution:' + feed.url] + impls = icache.get_feed(feed.get_distro_feed()).implementations assert len(impls) == 2, impls def testSlack(self): diff --git a/tests/testdownload.py b/tests/testdownload.py index ae60b39..049b505 100755 --- a/tests/testdownload.py +++ b/tests/testdownload.py @@ -208,6 +208,34 @@ class TestDownload(BaseTest): if "HelloWorld/Missing" not in str(ex): raise ex + def testDistro(self): + with output_suppressed(): + native_url = 'http://example.com:8000/Native.xml' + + # Initially, we don't have the feed at all... + master_feed = iface_cache.iface_cache.get_feed(native_url) + assert master_feed is None, master_feed + + trust.trust_db.trust_key('DE937DD411906ACF7C263B396FCF121BE2390E0B', 'example.com:8000') + self.child = server.handle_requests('Native.xml', '6FCF121BE2390E0B.gpg', '/key-info/key/DE937DD411906ACF7C263B396FCF121BE2390E0B') + h = DummyHandler() + policy = autopolicy.AutoPolicy(native_url, download_only = False, handler = h) + assert policy.need_download() + + solve = policy.solve_with_downloads() + h.wait_for_blocker(solve) + tasks.check(solve) + + master_feed = iface_cache.iface_cache.get_feed(native_url) + assert master_feed is not None + assert master_feed.implementations == {} + + distro_feed_url = master_feed.get_distro_feed() + assert distro_feed_url is not None + distro_feed = iface_cache.iface_cache.get_feed(distro_feed_url) + assert distro_feed is not None + assert len(distro_feed.implementations) == 2, distro_feed.implementations + def testWrongSize(self): with output_suppressed(): self.child = server.handle_requests('Hello-wrong-size', '6FCF121BE2390E0B.gpg', diff --git a/tests/testreader.py b/tests/testreader.py index 4b24ed0..b2d2a83 100755 --- a/tests/testreader.py +++ b/tests/testreader.py @@ -224,9 +224,12 @@ class TestReader(BaseTest): iface = model.Interface(foo_iface_uri) reader.update(iface, tmp.name, True) - feed = iface_cache.get_feed(foo_iface_uri) + master_feed = iface_cache.get_feed(foo_iface_uri) + assert len(master_feed.implementations) == 0 + distro_feed_url = master_feed.get_distro_feed() - assert len(feed.implementations) == 2 + feed = iface_cache.get_feed(distro_feed_url) + assert len(feed.implementations) == 1 impl = feed.implementations['package:deb:python-bittorrent:3.4.2-10'] assert impl.id == 'package:deb:python-bittorrent:3.4.2-10' diff --git a/tests/testsat.py b/tests/testsat.py index 720e12a..e6a5169 100755 --- a/tests/testsat.py +++ b/tests/testsat.py @@ -94,6 +94,9 @@ class TestCache: self.feeds[url] = feed return self.feeds[url] + def get_feed_imports(self, iface): + return [] + def assertSelection(expected, repo): cache = TestCache() diff --git a/zeroinstall/injector/distro.py b/zeroinstall/injector/distro.py index 404b21d..4578ada 100644 --- a/zeroinstall/injector/distro.py +++ b/zeroinstall/injector/distro.py @@ -10,7 +10,7 @@ from zeroinstall import _ import os, platform, re, glob, subprocess, sys from logging import warn, info from zeroinstall.injector import namespaces, model, arch -from zeroinstall.support import basedir +from zeroinstall.support import basedir, tasks _dotted_ints = '[0-9]+(?:\.[0-9]+)*' @@ -166,6 +166,52 @@ class Distribution(object): @return: True iff the package is currently installed""" return True + def get_feed(self, master_feed): + """Generate a feed containing information about distribution packages. + This should immediately return a feed containing an implementation for the + package if it's already installed. Information about versions that could be + installed using the distribution's package manager can be added asynchronously + later (see L{fetch_candidates}). + @param master_feed: feed containing the elements + @type master_feed: L{model.ZeroInstallFeed} + @rtype: L{model.ZeroInstallFeed}""" + + feed = model.ZeroInstallFeed(None) + feed.url = 'distribution:' + master_feed.url + + for item, item_attrs in master_feed.get_package_impls(self): + package = item_attrs.get('package', None) + if package is None: + raise model.InvalidInterface(_("Missing 'package' attribute on %s") % item) + + def factory(id): + assert id.startswith('package:') + if id in feed.implementations: + warn(_("Duplicate ID '%s' for DistributionImplementation"), id) + impl = model.DistributionImplementation(feed, id, self) + feed.implementations[id] = impl + + impl.metadata = item_attrs + + item_main = item_attrs.get('main', None) + if item_main and not item_main.startswith('/'): + raise model.InvalidInterface(_("'main' attribute must be absolute, but '%s' doesn't start with '/'!") % + item_main) + impl.main = item_main + impl.upstream_stability = model.packaged + + return impl + + self.get_package_info(package, factory) + return feed + + @tasks.async + def fetch_candidates(self, master_feed): + """Collect information about versions we could install using + the distribution's package manager. On success, the distribution + feed in iface_cache is updated.""" + yield + class CachedDistribution(Distribution): """For distributions where querying the package database is slow (e.g. requires running an external command), we cache the results. @@ -310,40 +356,8 @@ class DebianDistribution(Distribution): else: installed_version = None - # Check to see whether we could get a newer version using apt-get - cached = self.apt_cache.get(package) - if cached is None: - try: - null = os.open('/dev/null', os.O_WRONLY) - child = subprocess.Popen(['apt-cache', 'show', '--no-all-versions', '--', package], stdout = subprocess.PIPE, stderr = null) - os.close(null) - - arch = version = size = None - for line in child.stdout: - line = line.strip() - if line.startswith('Version: '): - version = line[9:] - if ':' in version: - # Debian's 'epoch' system - version = version.split(':', 1)[1] - version = try_cleanup_distro_version(version) - elif line.startswith('Architecture: '): - arch = canonical_machine(line[14:].strip()) - elif line.startswith('Size: '): - size = int(line[6:].strip()) - if version and arch: - cached = '%s\t%s\t%d' % (version, arch, size) - else: - cached = '-' - child.wait() - except Exception, ex: - warn("'apt-cache show %s' failed: %s", package, ex) - cached = '-' - # (multi-arch support? can there be multiple candidates?) - self.apt_cache.put(package, cached) - - if cached != '-': + if cached not in (None, '-'): candidate_version, candidate_arch, candidate_size = cached.split('\t') if candidate_version and candidate_version != installed_version: impl = factory('package:deb:%s:%s' % (package, candidate_version)) @@ -373,6 +387,44 @@ class DebianDistribution(Distribution): installed_id = 'package:deb:%s:%s' % (package, installed_version) return package_id == installed_id + @tasks.async + def fetch_candidates(self, master_feed): + package_names = [item.getAttribute("package") for item, item_attrs in master_feed.get_package_impls(self)] + yield + + for package in package_names: + # Check to see whether we could get a newer version using apt-get + cached = self.apt_cache.get(package) + if cached is None: + try: + null = os.open('/dev/null', os.O_WRONLY) + child = subprocess.Popen(['apt-cache', 'show', '--no-all-versions', '--', package], stdout = subprocess.PIPE, stderr = null) + os.close(null) + + arch = version = size = None + for line in child.stdout: + line = line.strip() + if line.startswith('Version: '): + version = line[9:] + if ':' in version: + # Debian's 'epoch' system + version = version.split(':', 1)[1] + version = try_cleanup_distro_version(version) + elif line.startswith('Architecture: '): + arch = canonical_machine(line[14:].strip()) + elif line.startswith('Size: '): + size = int(line[6:].strip()) + if version and arch: + cached = '%s\t%s\t%d' % (version, arch, size) + else: + cached = '-' + child.wait() + except Exception, ex: + warn("'apt-cache show %s' failed: %s", package, ex) + cached = '-' + # (multi-arch support? can there be multiple candidates?) + self.apt_cache.put(package, cached) + class RPMDistribution(CachedDistribution): """An RPM-based distribution.""" diff --git a/zeroinstall/injector/fetch.py b/zeroinstall/injector/fetch.py index ce3fc6d..d7f33f6 100644 --- a/zeroinstall/injector/fetch.py +++ b/zeroinstall/injector/fetch.py @@ -154,6 +154,22 @@ class Fetcher(object): return None return '%s/%s/latest.xml' % (self.feed_mirror, _get_feed_dir(url)) + @tasks.async + def get_packagekit_feed(self, iface_cache, feed_url): + """Send a query to PackageKit (if available) for information about this package. + On success, the result is added to iface_cache. + """ + assert feed_url.startswith('distribution:'), feed_url + master_feed = iface_cache.get_feed(feed_url.split(':', 1)[1]) + if master_feed: + fetch = iface_cache.distro.fetch_candidates(master_feed) + yield fetch + tasks.check(fetch) + + # Force feed to be regenerated with the new information + if feed_url in iface_cache._feeds: + del iface_cache._feeds[feed_url] + def download_and_import_feed(self, feed_url, iface_cache, force = False): """Download the feed, download any required keys, confirm trust if needed and import. @param feed_url: the feed to be downloaded @@ -166,6 +182,9 @@ class Fetcher(object): debug(_("download_and_import_feed %(url)s (force = %(force)d)"), {'url': feed_url, 'force': force}) assert not os.path.isabs(feed_url) + if feed_url.startswith('distribution:'): + return self.get_packagekit_feed(iface_cache, feed_url) + primary = self._download_and_import_feed(feed_url, iface_cache, force, use_mirror = False) @tasks.named_async("monitor feed downloads for " + feed_url) diff --git a/zeroinstall/injector/iface_cache.py b/zeroinstall/injector/iface_cache.py index 0de59cb..4ec192c 100644 --- a/zeroinstall/injector/iface_cache.py +++ b/zeroinstall/injector/iface_cache.py @@ -174,16 +174,31 @@ class IfaceCache(object): cache of L{model.Interface} objects, and an on-disk cache of L{model.ZeroInstallFeed}s. It will probably be split into two in future. + @ivar distro: the native distribution proxy + @type distro: L[distro.Distribution} + @see: L{iface_cache} - the singleton IfaceCache instance. """ - __slots__ = ['_interfaces', 'stores', '_feeds'] + __slots__ = ['_interfaces', 'stores', '_feeds', '_distro'] - def __init__(self): + def __init__(self, distro = None): + """@param distro: distribution used to fetch "distribution:" feeds (since 0.49) + @type distro: L[distro.Distribution}, or None to use the host distribution + """ self._interfaces = {} self._feeds = {} self.stores = zerostore.Stores() + + self._distro = distro + + @property + def distro(self): + if self._distro is None: + from zeroinstall.injector.distro import get_host_distribution + self._distro = get_host_distribution() + return self._distro def update_interface_if_trusted(self, interface, sigs, xml): import warnings @@ -320,7 +335,13 @@ class IfaceCache(object): if feed != False: return feed - feed = reader.load_feed_from_cache(url) + if url.startswith('distribution:'): + master_feed = self.get_feed(url.split(':', 1)[1]) + if not master_feed: + return None # Can't happen? + feed = self.distro.get_feed(master_feed) + else: + feed = reader.load_feed_from_cache(url) if feed: reader.update_user_feed_overrides(feed) self._feeds[url] = feed diff --git a/zeroinstall/injector/model.py b/zeroinstall/injector/model.py index 050cdc9..86a14eb 100644 --- a/zeroinstall/injector/model.py +++ b/zeroinstall/injector/model.py @@ -629,17 +629,14 @@ class ZeroInstallFeed(object): @ivar metadata: extra elements we didn't understand """ # _main is deprecated - __slots__ = ['url', 'implementations', 'name', 'descriptions', 'summaries', + __slots__ = ['url', 'implementations', 'name', 'descriptions', 'summaries', '_package_implementations', 'last_checked', 'last_modified', 'feeds', 'feed_for', 'metadata'] def __init__(self, feed_element, local_path = None, distro = None): """Create a feed object from a DOM. @param feed_element: the root element of a feed file @type feed_element: L{qdom.Element} - @param local_path: the pathname of this local feed, or None for remote feeds - @param distro: used to resolve distribution package references - @type distro: L{distro.Distribution} or None""" - assert feed_element + @param local_path: the pathname of this local feed, or None for remote feeds""" self.implementations = {} self.name = None self.summaries = {} # { lang: str } @@ -649,6 +646,14 @@ class ZeroInstallFeed(object): self.feed_for = set() self.metadata = [] self.last_checked = None + self._package_implementations = [] + + if distro is not None: + import warnings + warnings.warn("distro argument is now ignored", DeprecationWarning, 2) + + if feed_element is None: + return # XXX subclass? assert feed_element.name in ('interface', 'feed'), "Root element should be , not %s" % feed_element assert feed_element.uri == XMLNS_IFACE, "Wrong namespace on root element: %s" % feed_element.uri @@ -708,8 +713,6 @@ class ZeroInstallFeed(object): if not self.summary: raise InvalidInterface(_("Missing in feed")) - package_impls = [0, []] # Best score so far and packages with that score - def process_group(group, group_attrs, base_depends, base_bindings): for item in group.childNodes: if item.uri != XMLNS_IFACE: continue @@ -741,14 +744,8 @@ class ZeroInstallFeed(object): elif item.name == 'implementation': process_impl(item, item_attrs, depends, bindings) elif item.name == 'package-implementation': - distro_names = item_attrs.get('distributions', '') - for distro_name in distro_names.split(' '): - score = distro.get_score(distro_name) - if score > package_impls[0]: - package_impls[0] = score - package_impls[1] = [] - if score == package_impls[0]: - package_impls[1].append((item, item_attrs, depends)) + assert depends == [], "A with dependencies in %s!" % self.url + self._package_implementations.append((item, item_attrs)) else: assert 0 @@ -848,39 +845,40 @@ class ZeroInstallFeed(object): else: impl.download_sources.append(recipe) - def process_native_impl(item, item_attrs, depends): - package = item_attrs.get('package', None) - if package is None: - raise InvalidInterface(_("Missing 'package' attribute on %s") % item) - - def factory(id): - assert id.startswith('package:') - if id in self.implementations: - warn(_("Duplicate ID '%s' for DistributionImplementation"), id) - impl = DistributionImplementation(self, id, distro) - self.implementations[id] = impl - - impl.metadata = item_attrs - - item_main = item_attrs.get('main', None) - if item_main and not item_main.startswith('/'): - raise InvalidInterface(_("'main' attribute must be absolute, but '%s' doesn't start with '/'!") % - item_main) - impl.main = item_main - impl.upstream_stability = packaged - impl.requires = depends - - return impl - - distro.get_package_info(package, factory) - root_attrs = {'stability': 'testing'} if main: root_attrs['main'] = main process_group(feed_element, root_attrs, [], []) + + def get_distro_feed(self): + """Does this feed contain any elements? + i.e. is it worth asking the package manager for more information? + @return: the URL of the virtual feed, or None + @since: 0.49""" + if self._package_implementations: + return "distribution:" + self.url + return None - for args in package_impls[1]: - process_native_impl(*args) + def get_package_impls(self, distro): + """Find the best element(s) for the given distribution. + @param distro: the distribution to use to rate them + @type distro: L{distro.Distribution} + @return: a list of tuples for the best ranked elements + @rtype: [str] + @since: 0.49""" + best_score = 0 + best_impls = [] + + for item, item_attrs in self._package_implementations: + distro_names = item_attrs.get('distributions', '') + for distro_name in distro_names.split(' '): + score = distro.get_score(distro_name) + if score > best_score: + best_score = score + best_impls = [] + if score == best_score: + best_impls.append((item, item_attrs)) + return best_impls def get_name(self): return self.name or '(' + os.path.basename(self.url) + ')' diff --git a/zeroinstall/injector/reader.py b/zeroinstall/injector/reader.py index 9c903ca..ee6bdda 100644 --- a/zeroinstall/injector/reader.py +++ b/zeroinstall/injector/reader.py @@ -206,6 +206,6 @@ def load_feed(source, local = False): local_path = source else: local_path = None - feed = ZeroInstallFeed(root, local_path, distro.get_host_distribution()) + feed = ZeroInstallFeed(root, local_path) feed.last_modified = int(os.stat(source).st_mtime) return feed diff --git a/zeroinstall/injector/solver.py b/zeroinstall/injector/solver.py index 568eeee..baf8862 100644 --- a/zeroinstall/injector/solver.py +++ b/zeroinstall/injector/solver.py @@ -288,13 +288,7 @@ class SATSolver(Solver): @rtype: generator(ZeroInstallFeed)""" yield iface.uri - # Note: we only look one level deep here. Maybe we should recurse further? - feeds = iface.extra_feeds - main_feed = self.iface_cache.get_feed(iface.uri) - if main_feed: - feeds = feeds + main_feed.feeds - - for f in feeds: + for f in self.iface_cache.get_feed_imports(iface): # Note: when searching for src, None is not in machine_ranks if f.os in arch.os_ranks and \ (f.machine is None or f.machine in arch.machine_ranks): @@ -324,6 +318,13 @@ class SATSolver(Solver): if feed.implementations: impls.extend(feed.implementations.values()) + + distro_feed_url = feed.get_distro_feed() + if distro_feed_url: + self.feeds_used.add(distro_feed_url) + distro_feed = self.iface_cache.get_feed(distro_feed_url) + if distro_feed.implementations: + impls.extend(distro_feed.implementations.values()) except Exception, ex: warn(_("Failed to load feed %(feed)s for %(interface)s: %(exception)s"), {'feed': f, 'interface': iface, 'exception': ex}) -- 2.11.4.GIT