zeroinstall/injector/fetch.py

   1 """
   2 Downloads feeds, keys, packages and icons.
   3 """
   4
   5 # Copyright (C) 2009, Thomas Leonard
   6 # See the README file for details, or visit http://0install.net.
   7
   8 from zeroinstall import _
   9 import os
  10 from logging import info, debug, warn
  11
  12 from zeroinstall.support import tasks, basedir
  13 from zeroinstall.injector.namespaces import XMLNS_IFACE, config_site
  14 from zeroinstall.injector.model import SafeException, escape, DistributionSource
  15 from zeroinstall.injector.iface_cache import PendingFeed, ReplayAttack
  16 from zeroinstall.injector.handler import NoTrustedKeys
  17 from zeroinstall.injector import download
  18
  19 def _escape_slashes(path):
  20         return path.replace('/', '%23')
  21
  22 def _get_feed_dir(feed):
  23         """The algorithm from 0mirror."""
  24         if '#' in feed:
  25                 raise SafeException(_("Invalid URL '%s'") % feed)
  26         scheme, rest = feed.split('://', 1)
  27         assert '/' in rest, "Missing / in %s" % feed
  28         domain, rest = rest.split('/', 1)
  29         for x in [scheme, domain, rest]:
  30                 if not x or x.startswith(','):
  31                         raise SafeException(_("Invalid URL '%s'") % feed)
  32         return os.path.join('feeds', scheme, domain, _escape_slashes(rest))
  33
  34 class KeyInfoFetcher:
  35         """Fetches information about a GPG key from a key-info server.
  36         See L{Fetcher.fetch_key_info} for details.
  37         @since: 0.42
  38
  39         Example:
  40
  41         >>> kf = KeyInfoFetcher(handler, 'https://server', fingerprint)
  42         >>> while True:
  43                 print kf.info
  44                 if kf.blocker is None: break
  45                 print kf.status
  46                 yield kf.blocker
  47         """
  48         def __init__(self, handler, server, fingerprint):
  49                 self.fingerprint = fingerprint
  50                 self.info = []
  51                 self.blocker = None
  52
  53                 if server is None: return
  54
  55                 self.status = _('Fetching key information from %s...') % server
  56
  57                 dl = handler.get_download(server + '/key/' + fingerprint)
  58
  59                 from xml.dom import minidom
  60
  61                 @tasks.async
  62                 def fetch_key_info():
  63                         try:
  64                                 tempfile = dl.tempfile
  65                                 yield dl.downloaded
  66                                 self.blocker = None
  67                                 tasks.check(dl.downloaded)
  68                                 tempfile.seek(0)
  69                                 doc = minidom.parse(tempfile)
  70                                 if doc.documentElement.localName != 'key-lookup':
  71                                         raise SafeException(_('Expected <key-lookup>, not <%s>') % doc.documentElement.localName)
  72                                 self.info += doc.documentElement.childNodes
  73                         except Exception as ex:
  74                                 doc = minidom.parseString('<item vote="bad"/>')
  75                                 root = doc.documentElement
  76                                 root.appendChild(doc.createTextNode(_('Error getting key information: %s') % ex))
  77                                 self.info.append(root)
  78
  79                 self.blocker = fetch_key_info()
  80
  81 class Fetcher(object):
  82         """Downloads and stores various things.
  83         @ivar config: used to get handler, iface_cache and stores
  84         @type config: L{config.Config}
  85         @ivar key_info: caches information about GPG keys
  86         @type key_info: {str: L{KeyInfoFetcher}}
  87         """
  88         __slots__ = ['config', 'key_info']
  89
  90         def __init__(self, config):
  91                 assert config.handler, "API change!"
  92                 self.config = config
  93                 self.key_info = {}
  94
  95         @property
  96         def handler(self):
  97                 return self.config.handler
  98
  99         def cook(self, required_digest, recipe, stores, force = False, impl_hint = None):
 100                 """Follow a Recipe.
 101                 @deprecated: use impl.retrieve() instead
 102                 @param impl_hint: the Implementation this is for (if any) as a hint for the GUI
 103                 @see: L{download_impl} uses this method when appropriate"""
 104                 # Maybe we're taking this metaphor too far?
 105                 # Note: unused
 106                 return impl_hint.retrieve(self, recipe, stores, force)
 107
 108
 109         def get_feed_mirror(self, url):
 110                 """Return the URL of a mirror for this feed."""
 111                 if self.config.feed_mirror is None:
 112                         return None
 113                 import urlparse
 114                 if urlparse.urlparse(url).hostname == 'localhost':
 115                         return None
 116                 return '%s/%s/latest.xml' % (self.config.feed_mirror, _get_feed_dir(url))
 117
 118         @tasks.async
 119         def get_packagekit_feed(self, feed_url):
 120                 """Send a query to PackageKit (if available) for information about this package.
 121                 On success, the result is added to iface_cache.
 122                 """
 123                 assert feed_url.startswith('distribution:'), feed_url
 124                 master_feed = self.config.iface_cache.get_feed(feed_url.split(':', 1)[1])
 125                 if master_feed:
 126                         fetch = self.config.iface_cache.distro.fetch_candidates(master_feed)
 127                         if fetch:
 128                                 yield fetch
 129                                 tasks.check(fetch)
 130
 131                         # Force feed to be regenerated with the new information
 132                         self.config.iface_cache.get_feed(feed_url, force = True)
 133
 134         def download_and_import_feed(self, feed_url, iface_cache = None, force = False):
 135                 """Download the feed, download any required keys, confirm trust if needed and import.
 136                 @param feed_url: the feed to be downloaded
 137                 @type feed_url: str
 138                 @param iface_cache: (deprecated)
 139                 @param force: whether to abort and restart an existing download"""
 140                 from .download import DownloadAborted
 141
 142                 assert iface_cache is None or iface_cache is self.config.iface_cache
 143
 144                 debug(_("download_and_import_feed %(url)s (force = %(force)d)"), {'url': feed_url, 'force': force})
 145                 assert not os.path.isabs(feed_url)
 146
 147                 if feed_url.startswith('distribution:'):
 148                         return self.get_packagekit_feed(feed_url)
 149
 150                 primary = self._download_and_import_feed(feed_url, force, use_mirror = False)
 151
 152                 @tasks.named_async("monitor feed downloads for " + feed_url)
 153                 def wait_for_downloads(primary):
 154                         # Download just the upstream feed, unless it takes too long...
 155                         timeout = tasks.TimeoutBlocker(5, 'Mirror timeout')             # 5 seconds
 156
 157                         yield primary, timeout
 158                         tasks.check(timeout)
 159
 160                         try:
 161                                 tasks.check(primary)
 162                                 if primary.happened:
 163                                         return          # OK, primary succeeded!
 164                                 # OK, maybe it's just being slow...
 165                                 info("Feed download from %s is taking a long time.", feed_url)
 166                                 primary_ex = None
 167                         except NoTrustedKeys as ex:
 168                                 raise                   # Don't bother trying the mirror if we have a trust problem
 169                         except ReplayAttack as ex:
 170                                 raise                   # Don't bother trying the mirror if we have a replay attack
 171                         except DownloadAborted as ex:
 172                                 raise                   # Don't bother trying the mirror if the user cancelled
 173                         except SafeException as ex:
 174                                 # Primary failed
 175                                 primary = None
 176                                 primary_ex = ex
 177                                 warn(_("Feed download from %(url)s failed: %(exception)s"), {'url': feed_url, 'exception': ex})
 178
 179                         # Start downloading from mirror...
 180                         mirror = self._download_and_import_feed(feed_url, force, use_mirror = True)
 181
 182                         # Wait until both mirror and primary tasks are complete...
 183                         while True:
 184                                 blockers = filter(None, [primary, mirror])
 185                                 if not blockers:
 186                                         break
 187                                 yield blockers
 188
 189                                 if primary:
 190                                         try:
 191                                                 tasks.check(primary)
 192                                                 if primary.happened:
 193                                                         primary = None
 194                                                         # No point carrying on with the mirror once the primary has succeeded
 195                                                         if mirror:
 196                                                                 info(_("Primary feed download succeeded; aborting mirror download for %s") % feed_url)
 197                                                                 mirror.dl.abort()
 198                                         except SafeException as ex:
 199                                                 primary = None
 200                                                 primary_ex = ex
 201                                                 info(_("Feed download from %(url)s failed; still trying mirror: %(exception)s"), {'url': feed_url, 'exception': ex})
 202
 203                                 if mirror:
 204                                         try:
 205                                                 tasks.check(mirror)
 206                                                 if mirror.happened:
 207                                                         mirror = None
 208                                                         if primary_ex:
 209                                                                 # We already warned; no need to raise an exception too,
 210                                                                 # as the mirror download succeeded.
 211                                                                 primary_ex = None
 212                                         except ReplayAttack as ex:
 213                                                 info(_("Version from mirror is older than cached version; ignoring it: %s"), ex)
 214                                                 mirror = None
 215                                                 primary_ex = None
 216                                         except SafeException as ex:
 217                                                 info(_("Mirror download failed: %s"), ex)
 218                                                 mirror = None
 219
 220                         if primary_ex:
 221                                 raise primary_ex
 222
 223                 return wait_for_downloads(primary)
 224
 225         def _download_and_import_feed(self, feed_url, force, use_mirror):
 226                 """Download and import a feed.
 227                 @param use_mirror: False to use primary location; True to use mirror."""
 228                 if use_mirror:
 229                         url = self.get_feed_mirror(feed_url)
 230                         if url is None: return None
 231                         info(_("Trying mirror server for feed %s") % feed_url)
 232                 else:
 233                         url = feed_url
 234
 235                 dl = self.handler.get_download(url, force = force, hint = feed_url)
 236                 stream = dl.tempfile
 237
 238                 @tasks.named_async("fetch_feed " + url)
 239                 def fetch_feed():
 240                         yield dl.downloaded
 241                         tasks.check(dl.downloaded)
 242
 243                         pending = PendingFeed(feed_url, stream)
 244
 245                         if use_mirror:
 246                                 # If we got the feed from a mirror, get the key from there too
 247                                 key_mirror = self.config.feed_mirror + '/keys/'
 248                         else:
 249                                 key_mirror = None
 250
 251                         keys_downloaded = tasks.Task(pending.download_keys(self.handler, feed_hint = feed_url, key_mirror = key_mirror), _("download keys for %s") % feed_url)
 252                         yield keys_downloaded.finished
 253                         tasks.check(keys_downloaded.finished)
 254
 255                         if not self.config.iface_cache.update_feed_if_trusted(pending.url, pending.sigs, pending.new_xml):
 256                                 blocker = self.config.trust_mgr.confirm_keys(pending)
 257                                 if blocker:
 258                                         yield blocker
 259                                         tasks.check(blocker)
 260                                 if not self.config.iface_cache.update_feed_if_trusted(pending.url, pending.sigs, pending.new_xml):
 261                                         raise NoTrustedKeys(_("No signing keys trusted; not importing"))
 262
 263                 task = fetch_feed()
 264                 task.dl = dl
 265                 return task
 266
 267         def fetch_key_info(self, fingerprint):
 268                 try:
 269                         return self.key_info[fingerprint]
 270                 except KeyError:
 271                         self.key_info[fingerprint] = key_info = KeyInfoFetcher(self.handler,
 272                                                                         self.config.key_info_server, fingerprint)
 273                         return key_info
 274
 275         def download_impl(self, impl, retrieval_method, stores, force = False):
 276                 """Download an implementation.
 277                 @deprecated: use impl.retrieve(...) instead
 278                 @param impl: the selected implementation
 279                 @type impl: L{model.ZeroInstallImplementation}
 280                 @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
 281                 @type retrieval_method: L{model.RetrievalMethod}
 282                 @param stores: where to store the downloaded implementation
 283                 @type stores: L{zerostore.Stores}
 284                 @param force: whether to abort and restart an existing download
 285                 @rtype: L{tasks.Blocker}"""
 286                 assert impl
 287                 assert retrieval_method
 288                 return impl.retrieve(self, retrieval_method, stores, force)
 289
 290         def download_archive(self, download_source, force = False, impl_hint = None):
 291                 """Fetch an archive. You should normally call L{download_impl}
 292                 instead, since it handles other kinds of retrieval method too.
 293                 @deprecated: use download_source.download instead"""
 294                 # Note: unused
 295                 return download_source.download(self, force, impl_hint)
 296
 297         def download_icon(self, interface, force = False):
 298                 """Download an icon for this interface and add it to the
 299                 icon cache. If the interface has no icon do nothing.
 300                 @return: the task doing the import, or None
 301                 @rtype: L{tasks.Task}"""
 302                 debug("download_icon %(interface)s (force = %(force)d)", {'interface': interface, 'force': force})
 303
 304                 modification_time = None
 305                 existing_icon = self.config.iface_cache.get_icon_path(interface)
 306                 if existing_icon:
 307                         file_mtime = os.stat(existing_icon).st_mtime
 308                         from email.utils import formatdate
 309                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 310
 311                 # Find a suitable icon to download
 312                 for icon in interface.get_metadata(XMLNS_IFACE, 'icon'):
 313                         type = icon.getAttribute('type')
 314                         if type != 'image/png':
 315                                 debug(_('Skipping non-PNG icon'))
 316                                 continue
 317                         source = icon.getAttribute('href')
 318                         if source:
 319                                 break
 320                         warn(_('Missing "href" attribute on <icon> in %s'), interface)
 321                 else:
 322                         info(_('No PNG icons found in %s'), interface)
 323                         return
 324
 325                 try:
 326                         dl = self.handler.monitored_downloads[source]
 327                         if dl and force:
 328                                 dl.abort()
 329                                 raise KeyError
 330                 except KeyError:
 331                         dl = download.Download(source, hint = interface, modification_time = modification_time)
 332                         self.handler.monitor_download(dl)
 333
 334                 @tasks.async
 335                 def download_and_add_icon():
 336                         stream = dl.tempfile
 337                         yield dl.downloaded
 338                         try:
 339                                 tasks.check(dl.downloaded)
 340                                 if dl.unmodified: return
 341                                 stream.seek(0)
 342
 343                                 import shutil
 344                                 icons_cache = basedir.save_cache_path(config_site, 'interface_icons')
 345                                 icon_file = file(os.path.join(icons_cache, escape(interface.uri)), 'w')
 346                                 shutil.copyfileobj(stream, icon_file)
 347                         except Exception as ex:
 348                                 self.handler.report_error(ex)
 349
 350                 return download_and_add_icon()
 351
 352         def download_impls(self, implementations, stores):
 353                 """Download the given implementations, choosing a suitable retrieval method for each.
 354                 If any of the retrieval methods are DistributionSources and
 355                 need confirmation, handler.confirm is called to check that the
 356                 installation should proceed.
 357                 """
 358                 unsafe_impls = []
 359
 360                 to_download = []
 361                 for impl in implementations:
 362                         debug(_("start_downloading_impls: for %(feed)s get %(implementation)s"), {'feed': impl.feed, 'implementation': impl})
 363                         source = self.get_best_source(impl)
 364                         if not source:
 365                                 raise SafeException(_("Implementation %(implementation_id)s of interface %(interface)s"
 366                                         " cannot be downloaded (no download locations given in "
 367                                         "interface!)") % {'implementation_id': impl.id, 'interface': impl.feed.get_name()})
 368                         to_download.append((impl, source))
 369
 370                         if isinstance(source, DistributionSource) and source.needs_confirmation:
 371                                 unsafe_impls.append(source.package_id)
 372
 373                 @tasks.async
 374                 def download_impls():
 375                         if unsafe_impls:
 376                                 confirm = self.handler.confirm_install(_('The following components need to be installed using native packages. '
 377                                         'These come from your distribution, and should therefore be trustworthy, but they also '
 378                                         'run with extra privileges. In particular, installing them may run extra services on your '
 379                                         'computer or affect other users. You may be asked to enter a password to confirm. The '
 380                                         'packages are:\n\n') + ('\n'.join('- ' + x for x in unsafe_impls)))
 381                                 yield confirm
 382                                 tasks.check(confirm)
 383
 384                         blockers = []
 385
 386                         for impl, source in to_download:
 387                                 blockers.append(self.download_impl(impl, source, stores))
 388
 389                         # Record the first error log the rest
 390                         error = []
 391                         def dl_error(ex, tb = None):
 392                                 if error:
 393                                         self.handler.report_error(ex)
 394                                 else:
 395                                         error.append(ex)
 396                         while blockers:
 397                                 yield blockers
 398                                 tasks.check(blockers, dl_error)
 399
 400                                 blockers = [b for b in blockers if not b.happened]
 401                         if error:
 402                                 raise error[0]
 403
 404                 if not to_download:
 405                         return None
 406
 407                 return download_impls()
 408
 409         def get_best_source(self, impl):
 410                 """Return the best download source for this implementation.
 411                 @rtype: L{model.RetrievalMethod}
 412                 @deprecated: use impl.best_download_source instead
 413                 """
 414                 return impl.best_download_source
 415