2 Downloads feeds, keys, packages and icons.
5 # Copyright (C) 2009, Thomas Leonard
6 # See the README file for details, or visit http://0install.net.
8 from zeroinstall
import _
, NeedDownload
10 from logging
import info
, debug
, warn
12 from zeroinstall
import support
13 from zeroinstall
.support
import tasks
, basedir
, portable_rename
14 from zeroinstall
.injector
.namespaces
import XMLNS_IFACE
, config_site
15 from zeroinstall
.injector
import model
16 from zeroinstall
.injector
.model
import DownloadSource
, Recipe
, SafeException
, escape
, DistributionSource
17 from zeroinstall
.injector
.iface_cache
import PendingFeed
, ReplayAttack
18 from zeroinstall
.injector
.handler
import NoTrustedKeys
19 from zeroinstall
.injector
import download
21 def _escape_slashes(path
):
22 return path
.replace('/', '%23')
24 def _get_feed_dir(feed
):
25 """The algorithm from 0mirror."""
27 raise SafeException(_("Invalid URL '%s'") % feed
)
28 scheme
, rest
= feed
.split('://', 1)
29 assert '/' in rest
, "Missing / in %s" % feed
30 domain
, rest
= rest
.split('/', 1)
31 for x
in [scheme
, domain
, rest
]:
32 if not x
or x
.startswith('.'):
33 raise SafeException(_("Invalid URL '%s'") % feed
)
34 return '/'.join(['feeds', scheme
, domain
, _escape_slashes(rest
)])
37 """Fetches information about a GPG key from a key-info server.
38 See L{Fetcher.fetch_key_info} for details.
43 >>> kf = KeyInfoFetcher(fetcher, 'https://server', fingerprint)
46 if kf.blocker is None: break
50 def __init__(self
, fetcher
, server
, fingerprint
):
51 self
.fingerprint
= fingerprint
55 if server
is None: return
57 self
.status
= _('Fetching key information from %s...') % server
59 dl
= fetcher
.download_url(server
+ '/key/' + fingerprint
)
61 from xml
.dom
import minidom
66 tempfile
= dl
.tempfile
69 tasks
.check(dl
.downloaded
)
71 doc
= minidom
.parse(tempfile
)
72 if doc
.documentElement
.localName
!= 'key-lookup':
73 raise SafeException(_('Expected <key-lookup>, not <%s>') % doc
.documentElement
.localName
)
74 self
.info
+= doc
.documentElement
.childNodes
75 except Exception as ex
:
76 doc
= minidom
.parseString('<item vote="bad"/>')
77 root
= doc
.documentElement
78 root
.appendChild(doc
.createTextNode(_('Error getting key information: %s') % ex
))
79 self
.info
.append(root
)
81 self
.blocker
= fetch_key_info()
83 class Fetcher(object):
84 """Downloads and stores various things.
85 @ivar config: used to get handler, iface_cache and stores
86 @type config: L{config.Config}
87 @ivar key_info: caches information about GPG keys
88 @type key_info: {str: L{KeyInfoFetcher}}
90 __slots__
= ['config', 'key_info', '_scheduler', 'external_store']
92 def __init__(self
, config
):
93 assert config
.handler
, "API change!"
96 self
._scheduler
= None
97 self
.external_store
= os
.environ
.get('ZEROINSTALL_EXTERNAL_STORE')
101 return self
.config
.handler
105 if self
._scheduler
is None:
106 from . import scheduler
107 self
._scheduler
= scheduler
.DownloadScheduler()
108 return self
._scheduler
110 # (force is deprecated and ignored)
112 def cook(self
, required_digest
, recipe
, stores
, force
= False, impl_hint
= None):
114 @param impl_hint: the Implementation this is for (if any) as a hint for the GUI
115 @see: L{download_impl} uses this method when appropriate"""
116 # Maybe we're taking this metaphor too far?
118 # Start a download for each ingredient
121 for stepdata
in recipe
.steps
:
122 cls
= StepRunner
.class_for(stepdata
)
123 step
= cls(stepdata
, impl_hint
=impl_hint
)
124 step
.prepare(self
, blockers
)
129 tasks
.check(blockers
)
130 blockers
= [b
for b
in blockers
if not b
.happened
]
133 if self
.external_store
:
134 # Note: external_store will not yet work with non-<archive> steps.
135 streams
= [step
.stream
for step
in steps
]
136 self
._add
_to
_external
_store
(required_digest
, recipe
.steps
, streams
)
138 # Create an empty directory for the new implementation
139 store
= stores
.stores
[0]
140 tmpdir
= store
.get_tmp_dir_for(required_digest
)
142 # Unpack each of the downloaded archives into it in turn
145 # Check that the result is correct and store it in the cache
146 store
.check_manifest_and_rename(required_digest
, tmpdir
)
149 # If unpacking fails, remove the temporary directory
150 if tmpdir
is not None:
151 support
.ro_rmtree(tmpdir
)
153 def get_feed_mirror(self
, url
):
154 """Return the URL of a mirror for this feed."""
155 if self
.config
.feed_mirror
is None:
157 if support
.urlparse(url
).hostname
== 'localhost':
159 return '%s/%s/latest.xml' % (self
.config
.feed_mirror
, _get_feed_dir(url
))
162 def get_packagekit_feed(self
, feed_url
):
163 """Send a query to PackageKit (if available) for information about this package.
164 On success, the result is added to iface_cache.
166 assert feed_url
.startswith('distribution:'), feed_url
167 master_feed
= self
.config
.iface_cache
.get_feed(feed_url
.split(':', 1)[1])
169 fetch
= self
.config
.iface_cache
.distro
.fetch_candidates(master_feed
)
174 # Force feed to be regenerated with the new information
175 self
.config
.iface_cache
.get_feed(feed_url
, force
= True)
177 def download_and_import_feed(self
, feed_url
, iface_cache
= None):
178 """Download the feed, download any required keys, confirm trust if needed and import.
179 @param feed_url: the feed to be downloaded
181 @param iface_cache: (deprecated)"""
182 from .download
import DownloadAborted
184 assert iface_cache
is None or iface_cache
is self
.config
.iface_cache
186 self
.config
.iface_cache
.mark_as_checking(feed_url
)
188 debug(_("download_and_import_feed %(url)s"), {'url': feed_url
})
189 assert not os
.path
.isabs(feed_url
)
191 if feed_url
.startswith('distribution:'):
192 return self
.get_packagekit_feed(feed_url
)
194 primary
= self
._download
_and
_import
_feed
(feed_url
, use_mirror
= False)
196 @tasks.named_async("monitor feed downloads for " + feed_url
)
197 def wait_for_downloads(primary
):
198 # Download just the upstream feed, unless it takes too long...
199 timeout
= tasks
.TimeoutBlocker(5, 'Mirror timeout') # 5 seconds
201 yield primary
, timeout
207 return # OK, primary succeeded!
208 # OK, maybe it's just being slow...
209 info("Feed download from %s is taking a long time.", feed_url
)
211 except NoTrustedKeys
as ex
:
212 raise # Don't bother trying the mirror if we have a trust problem
213 except ReplayAttack
as ex
:
214 raise # Don't bother trying the mirror if we have a replay attack
215 except DownloadAborted
as ex
:
216 raise # Don't bother trying the mirror if the user cancelled
217 except SafeException
as ex
:
221 warn(_("Feed download from %(url)s failed: %(exception)s"), {'url': feed_url
, 'exception': ex
})
223 # Start downloading from mirror...
224 mirror
= self
._download
_and
_import
_feed
(feed_url
, use_mirror
= True)
226 # Wait until both mirror and primary tasks are complete...
228 blockers
= list(filter(None, [primary
, mirror
]))
238 # No point carrying on with the mirror once the primary has succeeded
240 info(_("Primary feed download succeeded; aborting mirror download for %s") % feed_url
)
242 except SafeException
as ex
:
245 info(_("Feed download from %(url)s failed; still trying mirror: %(exception)s"), {'url': feed_url
, 'exception': ex
})
253 # We already warned; no need to raise an exception too,
254 # as the mirror download succeeded.
256 except ReplayAttack
as ex
:
257 info(_("Version from mirror is older than cached version; ignoring it: %s"), ex
)
260 except SafeException
as ex
:
261 info(_("Mirror download failed: %s"), ex
)
267 return wait_for_downloads(primary
)
269 def _download_and_import_feed(self
, feed_url
, use_mirror
):
270 """Download and import a feed.
271 @param use_mirror: False to use primary location; True to use mirror."""
273 url
= self
.get_feed_mirror(feed_url
)
274 if url
is None: return None
275 info(_("Trying mirror server for feed %s") % feed_url
)
279 dl
= self
.download_url(url
, hint
= feed_url
)
282 @tasks.named_async("fetch_feed " + url
)
285 tasks
.check(dl
.downloaded
)
287 pending
= PendingFeed(feed_url
, stream
)
290 # If we got the feed from a mirror, get the key from there too
291 key_mirror
= self
.config
.feed_mirror
+ '/keys/'
295 keys_downloaded
= tasks
.Task(pending
.download_keys(self
, feed_hint
= feed_url
, key_mirror
= key_mirror
), _("download keys for %s") % feed_url
)
296 yield keys_downloaded
.finished
297 tasks
.check(keys_downloaded
.finished
)
299 if not self
.config
.iface_cache
.update_feed_if_trusted(pending
.url
, pending
.sigs
, pending
.new_xml
):
300 blocker
= self
.config
.trust_mgr
.confirm_keys(pending
)
304 if not self
.config
.iface_cache
.update_feed_if_trusted(pending
.url
, pending
.sigs
, pending
.new_xml
):
305 raise NoTrustedKeys(_("No signing keys trusted; not importing"))
311 def fetch_key_info(self
, fingerprint
):
313 return self
.key_info
[fingerprint
]
315 self
.key_info
[fingerprint
] = key_info
= KeyInfoFetcher(self
,
316 self
.config
.key_info_server
, fingerprint
)
319 # (force is deprecated and ignored)
320 def download_impl(self
, impl
, retrieval_method
, stores
, force
= False):
321 """Download an implementation.
322 @param impl: the selected implementation
323 @type impl: L{model.ZeroInstallImplementation}
324 @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
325 @type retrieval_method: L{model.RetrievalMethod}
326 @param stores: where to store the downloaded implementation
327 @type stores: L{zerostore.Stores}
328 @rtype: L{tasks.Blocker}"""
330 assert retrieval_method
332 if isinstance(retrieval_method
, DistributionSource
):
333 return retrieval_method
.install(self
.handler
)
335 from zeroinstall
.zerostore
import manifest
, parse_algorithm_digest_pair
337 for digest
in impl
.digests
:
338 alg_name
, digest_value
= parse_algorithm_digest_pair(digest
)
339 alg
= manifest
.algorithms
.get(alg_name
, None)
340 if alg
and (best
is None or best
.rating
< alg
.rating
):
342 required_digest
= digest
346 raise SafeException(_("No <manifest-digest> given for '%(implementation)s' version %(version)s") %
347 {'implementation': impl
.feed
.get_name(), 'version': impl
.get_version()})
348 raise SafeException(_("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s") %
349 {'algorithms': impl
.digests
, 'implementation': impl
.feed
.get_name(), 'version': impl
.get_version()})
353 if isinstance(retrieval_method
, DownloadSource
):
354 blocker
, stream
= self
.download_archive(retrieval_method
, impl_hint
= impl
)
359 if self
.external_store
:
360 self
._add
_to
_external
_store
(required_digest
, [retrieval_method
], [stream
])
362 self
._add
_to
_cache
(required_digest
, stores
, retrieval_method
, stream
)
363 elif isinstance(retrieval_method
, Recipe
):
364 blocker
= self
.cook(required_digest
, retrieval_method
, stores
, impl_hint
= impl
)
368 raise Exception(_("Unknown download type for '%s'") % retrieval_method
)
370 self
.handler
.impl_added_to_store(impl
)
371 return download_impl()
373 def _add_to_cache(self
, required_digest
, stores
, retrieval_method
, stream
):
374 assert isinstance(retrieval_method
, DownloadSource
)
375 stores
.add_archive_to_cache(required_digest
, stream
, retrieval_method
.url
, retrieval_method
.extract
,
376 type = retrieval_method
.type, start_offset
= retrieval_method
.start_offset
or 0)
378 def _add_to_external_store(self
, required_digest
, steps
, streams
):
379 from zeroinstall
.zerostore
.unpack
import type_from_url
381 # combine archive path, extract directory and MIME type arguments in an alternating fashion
382 paths
= map(lambda stream
: stream
.name
, streams
)
383 extracts
= map(lambda step
: step
.extract
or "", steps
)
384 types
= map(lambda step
: step
.type or type_from_url(step
.url
), steps
)
385 args
= [None]*(len(paths
)+len(extracts
)+len(types
))
387 args
[1::3] = extracts
390 # close file handles to allow external processes access
391 for stream
in streams
:
394 # delegate extracting archives to external tool
396 subprocess
.call([self
.external_store
, "add", required_digest
] + args
)
402 # (force is deprecated and ignored)
403 def download_archive(self
, download_source
, force
= False, impl_hint
= None):
404 """Fetch an archive. You should normally call L{download_impl}
405 instead, since it handles other kinds of retrieval method too."""
406 from zeroinstall
.zerostore
import unpack
408 url
= download_source
.url
409 if not (url
.startswith('http:') or url
.startswith('https:') or url
.startswith('ftp:')):
410 raise SafeException(_("Unknown scheme in download URL '%s'") % url
)
412 mime_type
= download_source
.type
414 mime_type
= unpack
.type_from_url(download_source
.url
)
416 raise SafeException(_("No 'type' attribute on archive, and I can't guess from the name (%s)") % download_source
.url
)
417 if not self
.external_store
:
418 unpack
.check_type_ok(mime_type
)
419 dl
= self
.download_url(download_source
.url
, hint
= impl_hint
)
420 dl
.expected_size
= download_source
.size
+ (download_source
.start_offset
or 0)
421 return (dl
.downloaded
, dl
.tempfile
)
423 # (force is deprecated and ignored)
424 def download_icon(self
, interface
, force
= False):
425 """Download an icon for this interface and add it to the
426 icon cache. If the interface has no icon do nothing.
427 @return: the task doing the import, or None
428 @rtype: L{tasks.Task}"""
429 debug("download_icon %(interface)s", {'interface': interface
})
431 modification_time
= None
432 existing_icon
= self
.config
.iface_cache
.get_icon_path(interface
)
434 file_mtime
= os
.stat(existing_icon
).st_mtime
435 from email
.utils
import formatdate
436 modification_time
= formatdate(timeval
= file_mtime
, localtime
= False, usegmt
= True)
438 # Find a suitable icon to download
439 for icon
in interface
.get_metadata(XMLNS_IFACE
, 'icon'):
440 type = icon
.getAttribute('type')
441 if type != 'image/png':
442 debug(_('Skipping non-PNG icon'))
444 source
= icon
.getAttribute('href')
447 warn(_('Missing "href" attribute on <icon> in %s'), interface
)
449 info(_('No PNG icons found in %s'), interface
)
452 dl
= self
.download_url(source
, hint
= interface
, modification_time
= modification_time
)
455 def download_and_add_icon():
459 tasks
.check(dl
.downloaded
)
460 if dl
.unmodified
: return
463 import shutil
, tempfile
464 icons_cache
= basedir
.save_cache_path(config_site
, 'interface_icons')
466 tmp_file
= tempfile
.NamedTemporaryFile(dir = icons_cache
, delete
= False)
467 shutil
.copyfileobj(stream
, tmp_file
)
470 icon_file
= os
.path
.join(icons_cache
, escape(interface
.uri
))
471 portable_rename(tmp_file
.name
, icon_file
)
472 except Exception as ex
:
473 self
.handler
.report_error(ex
)
477 return download_and_add_icon()
479 def download_impls(self
, implementations
, stores
):
480 """Download the given implementations, choosing a suitable retrieval method for each.
481 If any of the retrieval methods are DistributionSources and
482 need confirmation, handler.confirm is called to check that the
483 installation should proceed.
488 for impl
in implementations
:
489 debug(_("start_downloading_impls: for %(feed)s get %(implementation)s"), {'feed': impl
.feed
, 'implementation': impl
})
490 source
= self
.get_best_source(impl
)
492 raise SafeException(_("Implementation %(implementation_id)s of interface %(interface)s"
493 " cannot be downloaded (no download locations given in "
494 "interface!)") % {'implementation_id': impl
.id, 'interface': impl
.feed
.get_name()})
495 to_download
.append((impl
, source
))
497 if isinstance(source
, DistributionSource
) and source
.needs_confirmation
:
498 unsafe_impls
.append(source
.package_id
)
501 def download_impls():
503 confirm
= self
.handler
.confirm_install(_('The following components need to be installed using native packages. '
504 'These come from your distribution, and should therefore be trustworthy, but they also '
505 'run with extra privileges. In particular, installing them may run extra services on your '
506 'computer or affect other users. You may be asked to enter a password to confirm. The '
507 'packages are:\n\n') + ('\n'.join('- ' + x
for x
in unsafe_impls
)))
513 for impl
, source
in to_download
:
514 blockers
.append(self
.download_impl(impl
, source
, stores
))
516 # Record the first error log the rest
518 def dl_error(ex
, tb
= None):
520 self
.handler
.report_error(ex
)
522 error
.append((ex
, tb
))
525 tasks
.check(blockers
, dl_error
)
527 blockers
= [b
for b
in blockers
if not b
.happened
]
529 from zeroinstall
import support
530 support
.raise_with_traceback(*error
[0])
535 return download_impls()
537 def get_best_source(self
, impl
):
538 """Return the best download source for this implementation.
539 @rtype: L{model.RetrievalMethod}"""
540 if impl
.download_sources
:
541 return impl
.download_sources
[0]
544 def download_url(self
, url
, hint
= None, modification_time
= None, expected_size
= None):
545 """The most low-level method here; just download a raw URL.
546 @param url: the location to download from
547 @param hint: user-defined data to store on the Download (e.g. used by the GUI)
548 @param modification_time: don't download unless newer than this
549 @rtype: L{download.Download}
552 if self
.handler
.dry_run
:
553 raise NeedDownload(url
)
555 dl
= download
.Download(url
, hint
= hint
, modification_time
= modification_time
, expected_size
= expected_size
, auto_delete
= not self
.external_store
)
556 self
.handler
.monitor_download(dl
)
557 dl
.downloaded
= self
.scheduler
.download(dl
)
560 class StepRunner(object):
561 """The base class of all step runners"""
562 def __init__(self
, stepdata
, impl_hint
):
563 self
.stepdata
= stepdata
564 self
.impl_hint
= impl_hint
566 def prepare(self
, fetcher
, blockers
):
570 def class_for(cls
, model
):
571 for subcls
in cls
.__subclasses
__():
572 if subcls
.model_type
== type(model
):
574 assert False, "Couldn't find step runner for %s" % (type(model
),)
576 class RenameStepRunner(StepRunner
):
577 """A step runner for the <rename> step"""
579 model_type
= model
.RenameStep
581 def apply(self
, basedir
):
582 source
= native_path_within_base(basedir
, self
.stepdata
.source
)
583 dest
= native_path_within_base(basedir
, self
.stepdata
.dest
)
584 os
.rename(source
, dest
)
586 class DownloadStepRunner(StepRunner
):
587 """A step runner for the <archive> step"""
589 model_type
= model
.DownloadSource
591 def prepare(self
, fetcher
, blockers
):
592 self
.blocker
, self
.stream
= fetcher
.download_archive(self
.stepdata
, impl_hint
= self
.impl_hint
)
594 blockers
.append(self
.blocker
)
596 def apply(self
, basedir
):
597 from zeroinstall
.zerostore
import unpack
598 assert self
.blocker
.happened
599 unpack
.unpack_archive_over(self
.stepdata
.url
, self
.stream
, basedir
,
600 extract
= self
.stepdata
.extract
,
601 type=self
.stepdata
.type,
602 start_offset
= self
.stepdata
.start_offset
or 0)
604 def native_path_within_base(base
, crossplatform_path
):
605 """Takes a cross-platform relative path (i.e using forward slashes, even on windows)
606 and returns the absolute, platform-native version of the path.
607 If the path does not resolve to a location within `base`, a SafeError is raised.
609 assert os
.path
.isabs(base
)
610 if crossplatform_path
.startswith("/"):
611 raise SafeException("path %r is not within the base directory" % (crossplatform_path
,))
612 native_path
= os
.path
.join(*crossplatform_path
.split("/"))
613 fullpath
= os
.path
.realpath(os
.path
.join(base
, native_path
))
614 base
= os
.path
.realpath(base
)
615 if not fullpath
.startswith(base
+ os
.path
.sep
):
616 raise SafeException("path %r is not within the base directory" % (crossplatform_path
,))