2 Downloads feeds, keys, packages and icons.
5 # Copyright (C) 2009, Thomas Leonard
6 # See the README file for details, or visit http://0install.net.
9 from logging
import info
, debug
, warn
11 from zeroinstall
.support
import tasks
, basedir
12 from zeroinstall
.injector
.namespaces
import XMLNS_IFACE
, config_site
13 from zeroinstall
.injector
.model
import DownloadSource
, Recipe
, SafeException
, escape
14 from zeroinstall
.injector
.iface_cache
import PendingFeed
, ReplayAttack
15 from zeroinstall
.injector
.handler
import NoTrustedKeys
17 def _escape_slashes(path
):
18 return path
.replace('/', '%23')
20 def _get_feed_dir(feed
):
21 """The algorithm from 0mirror."""
23 raise SafeException("Invalid URL '%s'" % feed
)
24 scheme
, rest
= feed
.split('://', 1)
25 domain
, rest
= rest
.split('/', 1)
26 for x
in [scheme
, domain
, rest
]:
27 if not x
or x
.startswith(','):
28 raise SafeException("Invalid URL '%s'" % feed
)
29 return os
.path
.join('feeds', scheme
, domain
, _escape_slashes(rest
))
31 class Fetcher(object):
32 """Downloads and stores various things.
33 @ivar handler: handler to use for user-interaction
34 @type handler: L{handler.Handler}
35 @ivar feed_mirror: the base URL of a mirror site for keys and feeds
36 @type feed_mirror: str
38 __slots__
= ['handler', 'feed_mirror']
40 def __init__(self
, handler
):
41 self
.handler
= handler
42 self
.feed_mirror
= "http://roscidus.com/0mirror"
45 def cook(self
, required_digest
, recipe
, stores
, force
= False, impl_hint
= None):
47 @param impl_hint: the Implementation this is for (if any) as a hint for the GUI
48 @see: L{download_impl} uses this method when appropriate"""
49 # Maybe we're taking this metaphor too far?
51 # Start downloading all the ingredients.
52 downloads
= {} # Downloads that are not yet successful
53 streams
= {} # Streams collected from successful downloads
55 # Start a download for each ingredient
57 for step
in recipe
.steps
:
58 blocker
, stream
= self
.download_archive(step
, force
= force
, impl_hint
= impl_hint
)
60 blockers
.append(blocker
)
61 streams
[step
] = stream
66 blockers
= [b
for b
in blockers
if not b
.happened
]
68 from zeroinstall
.zerostore
import unpack
70 # Create an empty directory for the new implementation
71 store
= stores
.stores
[0]
72 tmpdir
= store
.get_tmp_dir_for(required_digest
)
74 # Unpack each of the downloaded archives into it in turn
75 for step
in recipe
.steps
:
76 stream
= streams
[step
]
78 unpack
.unpack_archive_over(step
.url
, stream
, tmpdir
, step
.extract
)
79 # Check that the result is correct and store it in the cache
80 store
.check_manifest_and_rename(required_digest
, tmpdir
)
83 # If unpacking fails, remove the temporary directory
84 if tmpdir
is not None:
85 from zeroinstall
import support
86 support
.ro_rmtree(tmpdir
)
88 def get_feed_mirror(self
, url
):
89 """Return the URL of a mirror for this feed."""
91 if urlparse
.urlparse(url
).hostname
== 'localhost':
93 return '%s/%s/latest.xml' % (self
.feed_mirror
, _get_feed_dir(url
))
95 def download_and_import_feed(self
, feed_url
, iface_cache
, force
= False):
96 """Download the feed, download any required keys, confirm trust if needed and import.
97 @param feed_url: the feed to be downloaded
99 @param iface_cache: cache in which to store the feed
100 @type iface_cache: L{iface_cache.IfaceCache}
101 @param force: whether to abort and restart an existing download"""
102 from download
import DownloadAborted
104 debug("download_and_import_feed %s (force = %d)", feed_url
, force
)
105 assert not feed_url
.startswith('/')
107 primary
= self
._download
_and
_import
_feed
(feed_url
, iface_cache
, force
, use_mirror
= False)
109 @tasks.named_async("monitor feed downloads for " + feed_url
)
110 def wait_for_downloads(primary
):
111 # Download just the upstream feed, unless it takes too long...
112 timeout
= tasks
.TimeoutBlocker(5, 'Mirror timeout') # 5 seconds
114 yield primary
, timeout
120 return # OK, primary succeeded!
121 # OK, maybe it's just being slow...
122 info("Feed download from %s is taking a long time. Trying mirror too...", feed_url
)
124 except NoTrustedKeys
, ex
:
125 raise # Don't bother trying the mirror if we have a trust problem
126 except ReplayAttack
, ex
:
127 raise # Don't bother trying the mirror if we have a replay attack
128 except DownloadAborted
, ex
:
129 raise # Don't bother trying the mirror if the user cancelled
130 except SafeException
, ex
:
134 warn("Trying mirror, as feed download from %s failed: %s", feed_url
, ex
)
136 # Start downloading from mirror...
137 mirror
= self
._download
_and
_import
_feed
(feed_url
, iface_cache
, force
, use_mirror
= True)
139 # Wait until both mirror and primary tasks are complete...
141 blockers
= filter(None, [primary
, mirror
])
151 # No point carrying on with the mirror once the primary has succeeded
153 info("Primary feed download succeeded; aborting mirror download for " + feed_url
)
155 except SafeException
, ex
:
158 info("Feed download from %s failed; still trying mirror: %s", feed_url
, ex
)
166 # We already warned; no need to raise an exception too,
167 # as the mirror download succeeded.
169 except ReplayAttack
, ex
:
170 info("Version from mirror is older than cached version; ignoring it: %s", ex
)
173 except SafeException
, ex
:
174 info("Mirror download failed: %s", ex
)
180 return wait_for_downloads(primary
)
182 def _download_and_import_feed(self
, feed_url
, iface_cache
, force
, use_mirror
):
183 """Download and import a feed.
184 @param use_mirror: False to use primary location; True to use mirror."""
186 url
= self
.get_feed_mirror(feed_url
)
187 if url
is None: return None
191 dl
= self
.handler
.get_download(url
, force
= force
, hint
= feed_url
)
194 @tasks.named_async("fetch_feed " + url
)
197 tasks
.check(dl
.downloaded
)
199 pending
= PendingFeed(feed_url
, stream
)
202 # If we got the feed from a mirror, get the key from there too
203 key_mirror
= self
.feed_mirror
+ '/keys/'
207 keys_downloaded
= tasks
.Task(pending
.download_keys(self
.handler
, feed_hint
= feed_url
, key_mirror
= key_mirror
), "download keys for " + feed_url
)
208 yield keys_downloaded
.finished
209 tasks
.check(keys_downloaded
.finished
)
211 iface
= iface_cache
.get_interface(pending
.url
)
212 if not iface_cache
.update_interface_if_trusted(iface
, pending
.sigs
, pending
.new_xml
):
213 blocker
= self
.handler
.confirm_trust_keys(iface
, pending
.sigs
, pending
.new_xml
)
217 if not iface_cache
.update_interface_if_trusted(iface
, pending
.sigs
, pending
.new_xml
):
218 raise NoTrustedKeys("No signing keys trusted; not importing")
224 def download_impl(self
, impl
, retrieval_method
, stores
, force
= False):
225 """Download an implementation.
226 @param impl: the selected implementation
227 @type impl: L{model.ZeroInstallImplementation}
228 @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
229 @type retrieval_method: L{model.RetrievalMethod}
230 @param stores: where to store the downloaded implementation
231 @type stores: L{zerostore.Stores}
232 @param force: whether to abort and restart an existing download
233 @rtype: L{tasks.Blocker}"""
235 assert retrieval_method
237 from zeroinstall
.zerostore
import manifest
238 alg
= impl
.id.split('=', 1)[0]
239 if alg
not in manifest
.algorithms
:
240 raise SafeException("Unknown digest algorithm '%s' for '%s' version %s" %
241 (alg
, impl
.feed
.get_name(), impl
.get_version()))
245 if isinstance(retrieval_method
, DownloadSource
):
246 blocker
, stream
= self
.download_archive(retrieval_method
, force
= force
, impl_hint
= impl
)
251 self
._add
_to
_cache
(stores
, retrieval_method
, stream
)
252 elif isinstance(retrieval_method
, Recipe
):
253 blocker
= self
.cook(impl
.id, retrieval_method
, stores
, force
, impl_hint
= impl
)
257 raise Exception("Unknown download type for '%s'" % retrieval_method
)
259 self
.handler
.impl_added_to_store(impl
)
260 return download_impl()
262 def _add_to_cache(self
, stores
, retrieval_method
, stream
):
263 assert isinstance(retrieval_method
, DownloadSource
)
264 required_digest
= retrieval_method
.implementation
.id
265 url
= retrieval_method
.url
266 stores
.add_archive_to_cache(required_digest
, stream
, retrieval_method
.url
, retrieval_method
.extract
,
267 type = retrieval_method
.type, start_offset
= retrieval_method
.start_offset
or 0)
269 def download_archive(self
, download_source
, force
= False, impl_hint
= None):
270 """Fetch an archive. You should normally call L{download_impl}
271 instead, since it handles other kinds of retrieval method too."""
272 from zeroinstall
.zerostore
import unpack
274 url
= download_source
.url
275 if not (url
.startswith('http:') or url
.startswith('https:') or url
.startswith('ftp:')):
276 raise SafeException("Unknown scheme in download URL '%s'" % url
)
278 mime_type
= download_source
.type
280 mime_type
= unpack
.type_from_url(download_source
.url
)
282 raise SafeException("No 'type' attribute on archive, and I can't guess from the name (%s)" % download_source
.url
)
283 unpack
.check_type_ok(mime_type
)
284 dl
= self
.handler
.get_download(download_source
.url
, force
= force
, hint
= impl_hint
)
285 dl
.expected_size
= download_source
.size
+ (download_source
.start_offset
or 0)
286 return (dl
.downloaded
, dl
.tempfile
)
288 def download_icon(self
, interface
, force
= False):
289 """Download an icon for this interface and add it to the
290 icon cache. If the interface has no icon or we are offline, do nothing.
291 @return: the task doing the import, or None
292 @rtype: L{tasks.Task}"""
293 debug("download_icon %s (force = %d)", interface
, force
)
295 # Find a suitable icon to download
296 for icon
in interface
.get_metadata(XMLNS_IFACE
, 'icon'):
297 type = icon
.getAttribute('type')
298 if type != 'image/png':
299 debug('Skipping non-PNG icon')
301 source
= icon
.getAttribute('href')
304 warn('Missing "href" attribute on <icon> in %s', interface
)
306 info('No PNG icons found in %s', interface
)
309 dl
= self
.handler
.get_download(source
, force
= force
, hint
= interface
)
312 def download_and_add_icon():
316 tasks
.check(dl
.downloaded
)
320 icons_cache
= basedir
.save_cache_path(config_site
, 'interface_icons')
321 icon_file
= file(os
.path
.join(icons_cache
, escape(interface
.uri
)), 'w')
322 shutil
.copyfileobj(stream
, icon_file
)
323 except Exception, ex
:
324 self
.handler
.report_error(ex
)
326 return download_and_add_icon()
328 def download_impls(self
, implementations
, stores
):
329 """Download the given implementations, choosing a suitable retrieval method for each."""
333 for impl
in implementations
:
334 debug("start_downloading_impls: for %s get %s", impl
.feed
, impl
)
335 source
= self
.get_best_source(impl
)
337 raise SafeException("Implementation " + impl
.id + " of "
338 "interface " + impl
.feed
.get_name() + " cannot be "
339 "downloaded (no download locations given in "
341 to_download
.append((impl
, source
))
343 for impl
, source
in to_download
:
344 blockers
.append(self
.download_impl(impl
, source
, stores
))
350 def download_impls(blockers
):
351 # Record the first error log the rest
353 def dl_error(ex
, tb
= None):
355 self
.handler
.report_error(ex
)
360 tasks
.check(blockers
, dl_error
)
362 blockers
= [b
for b
in blockers
if not b
.happened
]
366 return download_impls(blockers
)
368 def get_best_source(self
, impl
):
369 """Return the best download source for this implementation.
370 @rtype: L{model.RetrievalMethod}"""
371 if impl
.download_sources
:
372 return impl
.download_sources
[0]