From 5e7088f6d3e00ce311e5127984ec6e6d9c536879 Mon Sep 17 00:00:00 2001 From: Thomas Leonard Date: Sat, 14 Jul 2012 15:49:23 +0100 Subject: [PATCH] Added support for implementation mirrors If there is an error downloading an archive, try the configured mirror site. Typically, the mirror will then redirect to the real mirror location. --- tests/basetest.py | 2 +- tests/server.py | 2 ++ tests/testdownload.py | 38 +++++++++++++++++++++----------------- zeroinstall/injector/config.py | 12 +++++++----- zeroinstall/injector/download.py | 11 ++++++++++- zeroinstall/injector/fetch.py | 32 ++++++++++++++++++++++++-------- zeroinstall/injector/scheduler.py | 23 +++++++++++++++++++++-- 7 files changed, 86 insertions(+), 34 deletions(-) diff --git a/tests/basetest.py b/tests/basetest.py index f90baaf..bd04256 100755 --- a/tests/basetest.py +++ b/tests/basetest.py @@ -155,7 +155,7 @@ class TestConfig: network_use = model.network_full key_info_server = None auto_approve_keys = False - feed_mirror = None + mirror = None def __init__(self): self.iface_cache = iface_cache.IfaceCache() diff --git a/tests/server.py b/tests/server.py index 4af55c7..abe9ed5 100644 --- a/tests/server.py +++ b/tests/server.py @@ -47,6 +47,8 @@ class MyHandler(server.BaseHTTPRequestHandler): # (don't use a symlink as they don't work on Windows) if leaf == 'latest.xml': leaf = 'Hello.xml' + elif parsed.path == '/0mirror/feeds/http/example.com:8000/Hello.xml/impl/sha1=3ce644dc725f1d21cfcf02562c76f375944b266a': + leaf = 'HelloWorld.tgz' if not resp: self.send_error(404, "Expected %s; got %s" % (next_step, parsed.path)) diff --git a/tests/testdownload.py b/tests/testdownload.py index ddaff4e..25431df 100755 --- a/tests/testdownload.py +++ b/tests/testdownload.py @@ -3,6 +3,7 @@ from __future__ import with_statement from basetest import BaseTest, StringIO import sys, tempfile, os import unittest +import logging from logging import getLogger, WARN, ERROR from contextlib import contextmanager @@ -410,22 +411,25 @@ class TestDownload(BaseTest): sys.stdout = old_out def testMirrors(self): - old_out = sys.stdout - try: - sys.stdout = StringIO() - getLogger().setLevel(ERROR) - trust.trust_db.trust_key('DE937DD411906ACF7C263B396FCF121BE2390E0B', 'example.com:8000') - run_server(server.Give404('/Hello.xml'), - '/0mirror/feeds/http/example.com:8000/Hello.xml/latest.xml', - '/0mirror/keys/6FCF121BE2390E0B.gpg') - driver = Driver(requirements = Requirements('http://example.com:8000/Hello.xml'), config = self.config) - self.config.feed_mirror = 'http://example.com:8000/0mirror' - - refreshed = driver.solve_with_downloads() - tasks.wait_for_blocker(refreshed) - assert driver.solver.ready - finally: - sys.stdout = old_out + getLogger().setLevel(logging.ERROR) + trust.trust_db.trust_key('DE937DD411906ACF7C263B396FCF121BE2390E0B', 'example.com:8000') + run_server(server.Give404('/Hello.xml'), + '/0mirror/feeds/http/example.com:8000/Hello.xml/latest.xml', + '/0mirror/keys/6FCF121BE2390E0B.gpg', + server.Give404('/HelloWorld.tgz'), + '/0mirror/feeds/http/example.com:8000/Hello.xml/impl/sha1=3ce644dc725f1d21cfcf02562c76f375944b266a') + driver = Driver(requirements = Requirements('http://example.com:8000/Hello.xml'), config = self.config) + self.config.mirror = 'http://example.com:8000/0mirror' + + refreshed = driver.solve_with_downloads() + tasks.wait_for_blocker(refreshed) + assert driver.solver.ready + + #getLogger().setLevel(logging.WARN) + downloaded = driver.download_uncached_implementations() + tasks.wait_for_blocker(downloaded) + path = self.config.stores.lookup_any(driver.solver.selections.selections['http://example.com:8000/Hello.xml'].digests) + assert os.path.exists(os.path.join(path, 'HelloWorld', 'main')) def testReplay(self): old_out = sys.stdout @@ -439,7 +443,7 @@ class TestDownload(BaseTest): trust.trust_db.trust_key('DE937DD411906ACF7C263B396FCF121BE2390E0B', 'example.com:8000') run_server(server.Give404('/Hello.xml'), 'latest.xml', '/0mirror/keys/6FCF121BE2390E0B.gpg', 'Hello.xml') - self.config.feed_mirror = 'http://example.com:8000/0mirror' + self.config.mirror = 'http://example.com:8000/0mirror' # Update from mirror (should ignore out-of-date timestamp) refreshed = self.config.fetcher.download_and_import_feed(iface.uri, self.config.iface_cache) diff --git a/zeroinstall/injector/config.py b/zeroinstall/injector/config.py index 4c081fb..4e6377a 100644 --- a/zeroinstall/injector/config.py +++ b/zeroinstall/injector/config.py @@ -20,7 +20,7 @@ from zeroinstall.injector.model import network_levels, network_full from zeroinstall.injector.namespaces import config_site, config_prog from zeroinstall.support import basedir -DEFAULT_FEED_MIRROR = "http://roscidus.com/0mirror" +DEFAULT_MIRROR = "http://roscidus.com/0mirror" DEFAULT_KEY_LOOKUP_SERVER = 'https://keylookup.appspot.com' class Config(object): @@ -31,13 +31,13 @@ class Config(object): @type handler: L{handler.Handler} @ivar key_info_server: the base URL of a key information server @type key_info_server: str - @ivar feed_mirror: the base URL of a mirror site for keys and feeds - @type feed_mirror: str | None + @ivar mirror: the base URL of a mirror site for feeds, keys and implementations (since 1.10) + @type mirror: str | None @ivar freshness: seconds since a feed was last checked before it is considered stale @type freshness: int """ - __slots__ = ['help_with_testing', 'freshness', 'network_use', 'feed_mirror', 'key_info_server', 'auto_approve_keys', + __slots__ = ['help_with_testing', 'freshness', 'network_use', 'mirror', 'key_info_server', 'auto_approve_keys', '_fetcher', '_stores', '_iface_cache', '_handler', '_trust_mgr', '_trust_db', '_app_mgr'] def __init__(self, handler = None): @@ -46,10 +46,12 @@ class Config(object): self.network_use = network_full self._handler = handler self._app_mgr = self._fetcher = self._stores = self._iface_cache = self._trust_mgr = self._trust_db = None - self.feed_mirror = DEFAULT_FEED_MIRROR + self.mirror = DEFAULT_MIRROR self.key_info_server = DEFAULT_KEY_LOOKUP_SERVER self.auto_approve_keys = True + feed_mirror = property(lambda self: self.mirror, lambda self, value: setattr(self, 'mirror', value)) + @property def stores(self): if not self._stores: diff --git a/zeroinstall/injector/download.py b/zeroinstall/injector/download.py index e90b4f5..54ad0c5 100644 --- a/zeroinstall/injector/download.py +++ b/zeroinstall/injector/download.py @@ -53,9 +53,11 @@ class Download(object): @type aborted_by_user: bool @ivar unmodified: whether the resource was not modified since the modification_time given at construction @type unmodified: bool + @ivar mirror: an alternative URL to try if this download fails + @type mirror: str | None """ __slots__ = ['url', 'tempfile', 'status', 'expected_size', 'downloaded', - 'hint', '_final_total_size', 'aborted_by_user', + 'hint', '_final_total_size', 'aborted_by_user', 'mirror', 'modification_time', 'unmodified', '_aborted'] def __init__(self, url, hint = None, modification_time = None, expected_size = None, auto_delete = True): @@ -73,6 +75,7 @@ class Download(object): self.tempfile = None # Stream for result self.downloaded = None + self.mirror = None self.expected_size = expected_size # Final size (excluding skipped bytes) self._final_total_size = None # Set when download is finished @@ -156,6 +159,12 @@ class Download(object): return os.fstat(self.tempfile.fileno()).st_size else: return self._final_total_size or 0 + + def get_next_mirror_url(self): + """Return an alternative download URL to try, or None if we're out of options.""" + mirror = self.mirror + self.mirror = None + return mirror def __str__(self): return _("") % self.url diff --git a/zeroinstall/injector/fetch.py b/zeroinstall/injector/fetch.py index 6625479..840ee00 100644 --- a/zeroinstall/injector/fetch.py +++ b/zeroinstall/injector/fetch.py @@ -6,7 +6,7 @@ Downloads feeds, keys, packages and icons. # See the README file for details, or visit http://0install.net. from zeroinstall import _, NeedDownload -import os +import os, sys from logging import info, debug, warn from zeroinstall import support @@ -150,13 +150,20 @@ class Fetcher(object): if tmpdir is not None: support.ro_rmtree(tmpdir) - def get_feed_mirror(self, url): + def _get_mirror_url(self, feed_url, resource): """Return the URL of a mirror for this feed.""" - if self.config.feed_mirror is None: + if self.config.mirror is None: return None - if support.urlparse(url).hostname == 'localhost': + if support.urlparse(feed_url).hostname == 'localhost': return None - return '%s/%s/latest.xml' % (self.config.feed_mirror, _get_feed_dir(url)) + return '%s/%s/%s' % (self.config.mirror, _get_feed_dir(feed_url), resource) + + def get_feed_mirror(self, url): + """Return the URL of a mirror for this feed.""" + return self._get_mirror_url(url, 'latest.xml') + + def _get_impl_mirror(self, impl): + return self._get_mirror_url(impl.feed.url, 'impl/' + _escape_slashes(impl.id)) @tasks.async def get_packagekit_feed(self, feed_url): @@ -288,7 +295,7 @@ class Fetcher(object): if use_mirror: # If we got the feed from a mirror, get the key from there too - key_mirror = self.config.feed_mirror + '/keys/' + key_mirror = self.config.mirror + '/keys/' else: key_mirror = None @@ -416,7 +423,13 @@ class Fetcher(object): raise SafeException(_("No 'type' attribute on archive, and I can't guess from the name (%s)") % download_source.url) if not self.external_store: unpack.check_type_ok(mime_type) - dl = self.download_url(download_source.url, hint = impl_hint) + + if impl_hint: + mirror = self._get_impl_mirror(impl_hint) + else: + mirror = None + + dl = self.download_url(download_source.url, hint = impl_hint, mirror_url = mirror) dl.expected_size = download_source.size + (download_source.start_offset or 0) return (dl.downloaded, dl.tempfile) @@ -541,11 +554,13 @@ class Fetcher(object): return impl.download_sources[0] return None - def download_url(self, url, hint = None, modification_time = None, expected_size = None): + def download_url(self, url, hint = None, modification_time = None, expected_size = None, mirror_url = None): """The most low-level method here; just download a raw URL. @param url: the location to download from @param hint: user-defined data to store on the Download (e.g. used by the GUI) @param modification_time: don't download unless newer than this + @param mirror_url: an altertive URL to try if this one fails + @type mirror_url: str @rtype: L{download.Download} @since: 1.5 """ @@ -553,6 +568,7 @@ class Fetcher(object): raise NeedDownload(url) dl = download.Download(url, hint = hint, modification_time = modification_time, expected_size = expected_size, auto_delete = not self.external_store) + dl.mirror = mirror_url self.handler.monitor_download(dl) dl.downloaded = self.scheduler.download(dl) return dl diff --git a/zeroinstall/injector/scheduler.py b/zeroinstall/injector/scheduler.py index de57613..b6c9de3 100644 --- a/zeroinstall/injector/scheduler.py +++ b/zeroinstall/injector/scheduler.py @@ -16,6 +16,7 @@ else: from collections import defaultdict import threading +import logging from zeroinstall import gobject from zeroinstall.support import tasks @@ -44,6 +45,8 @@ class DownloadScheduler: redirections_remaining = 10 + original_exception = None + # Assign the Download to a Site based on its scheme, host and port. If the result is a redirect, # reassign it to the appropriate new site. Note that proxy handling happens later; we want to group # and limit by the target site, not treat everything as going to a single site (the proxy). @@ -59,8 +62,24 @@ class DownloadScheduler: step.url = current_url blocker = self._sites[site_key].download(step) yield blocker - tasks.check(blocker) - + + try: + tasks.check(blocker) + except download.DownloadError as ex: + if original_exception is None: + original_exception = ex + else: + logging.warn("%s (while trying mirror)", ex) + mirror_url = step.dl.get_next_mirror_url() + if mirror_url is None: + raise original_exception + + # Try mirror + logging.warn("%s: trying mirror at %s", ex, mirror_url) + dl.expected_size = None + step.redirect = mirror_url + redirections_remaining = 10 + if not step.redirect: break -- 2.11.4.GIT