zeroinstall/injector/fetch.py

   1 """
   2 Downloads feeds, keys, packages and icons.
   3 """
   4
   5 # Copyright (C) 2009, Thomas Leonard
   6 # See the README file for details, or visit http://0install.net.
   7
   8 from zeroinstall import _
   9 import os
  10 from logging import info, debug, warn
  11
  12 from zeroinstall.support import tasks, basedir
  13 from zeroinstall.injector.namespaces import XMLNS_IFACE, config_site
  14 from zeroinstall.injector.model import DownloadSource, Recipe, SafeException, escape, DistributionSource
  15 from zeroinstall.injector.iface_cache import PendingFeed, ReplayAttack
  16 from zeroinstall.injector.handler import NoTrustedKeys
  17 from zeroinstall.injector import download
  18
  19 def _escape_slashes(path):
  20         return path.replace('/', '%23')
  21
  22 def _get_feed_dir(feed):
  23         """The algorithm from 0mirror."""
  24         if '#' in feed:
  25                 raise SafeException(_("Invalid URL '%s'") % feed)
  26         scheme, rest = feed.split('://', 1)
  27         assert '/' in rest, "Missing / in %s" % feed
  28         domain, rest = rest.split('/', 1)
  29         for x in [scheme, domain, rest]:
  30                 if not x or x.startswith(','):
  31                         raise SafeException(_("Invalid URL '%s'") % feed)
  32         return os.path.join('feeds', scheme, domain, _escape_slashes(rest))
  33
  34 class KeyInfoFetcher:
  35         """Fetches information about a GPG key from a key-info server.
  36         See L{Fetcher.fetch_key_info} for details.
  37         @since: 0.42
  38
  39         Example:
  40
  41         >>> kf = KeyInfoFetcher(handler, 'https://server', fingerprint)
  42         >>> while True:
  43                 print kf.info
  44                 if kf.blocker is None: break
  45                 print kf.status
  46                 yield kf.blocker
  47         """
  48         def __init__(self, handler, server, fingerprint):
  49                 self.fingerprint = fingerprint
  50                 self.info = []
  51                 self.blocker = None
  52
  53                 if server is None: return
  54
  55                 self.status = _('Fetching key information from %s...') % server
  56
  57                 dl = handler.get_download(server + '/key/' + fingerprint)
  58
  59                 from xml.dom import minidom
  60
  61                 @tasks.async
  62                 def fetch_key_info():
  63                         try:
  64                                 tempfile = dl.tempfile
  65                                 yield dl.downloaded
  66                                 self.blocker = None
  67                                 tasks.check(dl.downloaded)
  68                                 tempfile.seek(0)
  69                                 doc = minidom.parse(tempfile)
  70                                 if doc.documentElement.localName != 'key-lookup':
  71                                         raise SafeException(_('Expected <key-lookup>, not <%s>') % doc.documentElement.localName)
  72                                 self.info += doc.documentElement.childNodes
  73                         except Exception as ex:
  74                                 doc = minidom.parseString('<item vote="bad"/>')
  75                                 root = doc.documentElement
  76                                 root.appendChild(doc.createTextNode(_('Error getting key information: %s') % ex))
  77                                 self.info.append(root)
  78
  79                 self.blocker = fetch_key_info()
  80
  81 class Fetcher(object):
  82         """Downloads and stores various things.
  83         @ivar config: used to get handler, iface_cache and stores
  84         @type config: L{config.Config}
  85         @ivar key_info: caches information about GPG keys
  86         @type key_info: {str: L{KeyInfoFetcher}}
  87         """
  88         __slots__ = ['config', 'key_info']
  89
  90         def __init__(self, config):
  91                 assert config.handler, "API change!"
  92                 self.config = config
  93                 self.key_info = {}
  94
  95         @property
  96         def handler(self):
  97                 return self.config.handler
  98
  99         @tasks.async
 100         def cook(self, required_digest, recipe, stores, force = False, impl_hint = None):
 101                 """Follow a Recipe.
 102                 @param impl_hint: the Implementation this is for (if any) as a hint for the GUI
 103                 @see: L{download_impl} uses this method when appropriate"""
 104                 # Maybe we're taking this metaphor too far?
 105
 106                 # Start downloading all the ingredients.
 107                 streams = {}    # Streams collected from successful downloads
 108
 109                 # Start a download for each ingredient
 110                 blockers = []
 111                 for step in recipe.steps:
 112                         blocker, stream = self.download_archive(step, force = force, impl_hint = impl_hint)
 113                         assert stream
 114                         blockers.append(blocker)
 115                         streams[step] = stream
 116
 117                 while blockers:
 118                         yield blockers
 119                         tasks.check(blockers)
 120                         blockers = [b for b in blockers if not b.happened]
 121
 122                 from zeroinstall.zerostore import unpack
 123
 124                 # Create an empty directory for the new implementation
 125                 store = stores.stores[0]
 126                 tmpdir = store.get_tmp_dir_for(required_digest)
 127                 try:
 128                         # Unpack each of the downloaded archives into it in turn
 129                         for step in recipe.steps:
 130                                 stream = streams[step]
 131                                 stream.seek(0)
 132                                 unpack.unpack_archive_over(step.url, stream, tmpdir,
 133                                                 extract = step.extract,
 134                                                 type = step.type,
 135                                                 start_offset = step.start_offset or 0)
 136                         # Check that the result is correct and store it in the cache
 137                         store.check_manifest_and_rename(required_digest, tmpdir)
 138                         tmpdir = None
 139                 finally:
 140                         # If unpacking fails, remove the temporary directory
 141                         if tmpdir is not None:
 142                                 from zeroinstall import support
 143                                 support.ro_rmtree(tmpdir)
 144
 145         def get_feed_mirror(self, url):
 146                 """Return the URL of a mirror for this feed."""
 147                 if self.config.feed_mirror is None:
 148                         return None
 149                 import urlparse
 150                 if urlparse.urlparse(url).hostname == 'localhost':
 151                         return None
 152                 return '%s/%s/latest.xml' % (self.config.feed_mirror, _get_feed_dir(url))
 153
 154         @tasks.async
 155         def get_packagekit_feed(self, feed_url):
 156                 """Send a query to PackageKit (if available) for information about this package.
 157                 On success, the result is added to iface_cache.
 158                 """
 159                 assert feed_url.startswith('distribution:'), feed_url
 160                 master_feed = self.config.iface_cache.get_feed(feed_url.split(':', 1)[1])
 161                 if master_feed:
 162                         fetch = self.config.iface_cache.distro.fetch_candidates(master_feed)
 163                         if fetch:
 164                                 yield fetch
 165                                 tasks.check(fetch)
 166
 167                         # Force feed to be regenerated with the new information
 168                         self.config.iface_cache.get_feed(feed_url, force = True)
 169
 170         def download_and_import_feed(self, feed_url, iface_cache = None, force = False):
 171                 """Download the feed, download any required keys, confirm trust if needed and import.
 172                 @param feed_url: the feed to be downloaded
 173                 @type feed_url: str
 174                 @param iface_cache: (deprecated)
 175                 @param force: whether to abort and restart an existing download"""
 176                 from .download import DownloadAborted
 177
 178                 assert iface_cache is None or iface_cache is self.config.iface_cache
 179
 180                 debug(_("download_and_import_feed %(url)s (force = %(force)d)"), {'url': feed_url, 'force': force})
 181                 assert not os.path.isabs(feed_url)
 182
 183                 if feed_url.startswith('distribution:'):
 184                         return self.get_packagekit_feed(feed_url)
 185
 186                 primary = self._download_and_import_feed(feed_url, force, use_mirror = False)
 187
 188                 @tasks.named_async("monitor feed downloads for " + feed_url)
 189                 def wait_for_downloads(primary):
 190                         # Download just the upstream feed, unless it takes too long...
 191                         timeout = tasks.TimeoutBlocker(5, 'Mirror timeout')             # 5 seconds
 192
 193                         yield primary, timeout
 194                         tasks.check(timeout)
 195
 196                         try:
 197                                 tasks.check(primary)
 198                                 if primary.happened:
 199                                         return          # OK, primary succeeded!
 200                                 # OK, maybe it's just being slow...
 201                                 info("Feed download from %s is taking a long time.", feed_url)
 202                                 primary_ex = None
 203                         except NoTrustedKeys as ex:
 204                                 raise                   # Don't bother trying the mirror if we have a trust problem
 205                         except ReplayAttack as ex:
 206                                 raise                   # Don't bother trying the mirror if we have a replay attack
 207                         except DownloadAborted as ex:
 208                                 raise                   # Don't bother trying the mirror if the user cancelled
 209                         except SafeException as ex:
 210                                 # Primary failed
 211                                 primary = None
 212                                 primary_ex = ex
 213                                 warn(_("Feed download from %(url)s failed: %(exception)s"), {'url': feed_url, 'exception': ex})
 214
 215                         # Start downloading from mirror...
 216                         mirror = self._download_and_import_feed(feed_url, force, use_mirror = True)
 217
 218                         # Wait until both mirror and primary tasks are complete...
 219                         while True:
 220                                 blockers = filter(None, [primary, mirror])
 221                                 if not blockers:
 222                                         break
 223                                 yield blockers
 224
 225                                 if primary:
 226                                         try:
 227                                                 tasks.check(primary)
 228                                                 if primary.happened:
 229                                                         primary = None
 230                                                         # No point carrying on with the mirror once the primary has succeeded
 231                                                         if mirror:
 232                                                                 info(_("Primary feed download succeeded; aborting mirror download for %s") % feed_url)
 233                                                                 mirror.dl.abort()
 234                                         except SafeException as ex:
 235                                                 primary = None
 236                                                 primary_ex = ex
 237                                                 info(_("Feed download from %(url)s failed; still trying mirror: %(exception)s"), {'url': feed_url, 'exception': ex})
 238
 239                                 if mirror:
 240                                         try:
 241                                                 tasks.check(mirror)
 242                                                 if mirror.happened:
 243                                                         mirror = None
 244                                                         if primary_ex:
 245                                                                 # We already warned; no need to raise an exception too,
 246                                                                 # as the mirror download succeeded.
 247                                                                 primary_ex = None
 248                                         except ReplayAttack as ex:
 249                                                 info(_("Version from mirror is older than cached version; ignoring it: %s"), ex)
 250                                                 mirror = None
 251                                                 primary_ex = None
 252                                         except SafeException as ex:
 253                                                 info(_("Mirror download failed: %s"), ex)
 254                                                 mirror = None
 255
 256                         if primary_ex:
 257                                 raise primary_ex
 258
 259                 return wait_for_downloads(primary)
 260
 261         def _download_and_import_feed(self, feed_url, force, use_mirror):
 262                 """Download and import a feed.
 263                 @param use_mirror: False to use primary location; True to use mirror."""
 264                 if use_mirror:
 265                         url = self.get_feed_mirror(feed_url)
 266                         if url is None: return None
 267                         info(_("Trying mirror server for feed %s") % feed_url)
 268                 else:
 269                         url = feed_url
 270
 271                 dl = self.handler.get_download(url, force = force, hint = feed_url)
 272                 stream = dl.tempfile
 273
 274                 @tasks.named_async("fetch_feed " + url)
 275                 def fetch_feed():
 276                         yield dl.downloaded
 277                         tasks.check(dl.downloaded)
 278
 279                         pending = PendingFeed(feed_url, stream)
 280
 281                         if use_mirror:
 282                                 # If we got the feed from a mirror, get the key from there too
 283                                 key_mirror = self.config.feed_mirror + '/keys/'
 284                         else:
 285                                 key_mirror = None
 286
 287                         keys_downloaded = tasks.Task(pending.download_keys(self.handler, feed_hint = feed_url, key_mirror = key_mirror), _("download keys for %s") % feed_url)
 288                         yield keys_downloaded.finished
 289                         tasks.check(keys_downloaded.finished)
 290
 291                         if not self.config.iface_cache.update_feed_if_trusted(pending.url, pending.sigs, pending.new_xml):
 292                                 blocker = self.config.trust_mgr.confirm_keys(pending)
 293                                 if blocker:
 294                                         yield blocker
 295                                         tasks.check(blocker)
 296                                 if not self.config.iface_cache.update_feed_if_trusted(pending.url, pending.sigs, pending.new_xml):
 297                                         raise NoTrustedKeys(_("No signing keys trusted; not importing"))
 298
 299                 task = fetch_feed()
 300                 task.dl = dl
 301                 return task
 302
 303         def fetch_key_info(self, fingerprint):
 304                 try:
 305                         return self.key_info[fingerprint]
 306                 except KeyError:
 307                         self.key_info[fingerprint] = key_info = KeyInfoFetcher(self.handler,
 308                                                                         self.config.key_info_server, fingerprint)
 309                         return key_info
 310
 311         def download_impl(self, impl, retrieval_method, stores, force = False):
 312                 """Download an implementation.
 313                 @param impl: the selected implementation
 314                 @type impl: L{model.ZeroInstallImplementation}
 315                 @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
 316                 @type retrieval_method: L{model.RetrievalMethod}
 317                 @param stores: where to store the downloaded implementation
 318                 @type stores: L{zerostore.Stores}
 319                 @param force: whether to abort and restart an existing download
 320                 @rtype: L{tasks.Blocker}"""
 321                 assert impl
 322                 assert retrieval_method
 323
 324                 if isinstance(retrieval_method, DistributionSource):
 325                         return retrieval_method.install(self.handler)
 326
 327                 from zeroinstall.zerostore import manifest
 328                 best = None
 329                 for digest in impl.digests:
 330                         alg_name = digest.split('=', 1)[0]
 331                         alg = manifest.algorithms.get(alg_name, None)
 332                         if alg and (best is None or best.rating < alg.rating):
 333                                 best = alg
 334                                 required_digest = digest
 335
 336                 if best is None:
 337                         if not impl.digests:
 338                                 raise SafeException(_("No <manifest-digest> given for '%(implementation)s' version %(version)s") %
 339                                                 {'implementation': impl.feed.get_name(), 'version': impl.get_version()})
 340                         raise SafeException(_("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s") %
 341                                         {'algorithms': impl.digests, 'implementation': impl.feed.get_name(), 'version': impl.get_version()})
 342
 343                 @tasks.async
 344                 def download_impl():
 345                         if isinstance(retrieval_method, DownloadSource):
 346                                 blocker, stream = self.download_archive(retrieval_method, force = force, impl_hint = impl)
 347                                 yield blocker
 348                                 tasks.check(blocker)
 349
 350                                 stream.seek(0)
 351                                 self._add_to_cache(required_digest, stores, retrieval_method, stream)
 352                         elif isinstance(retrieval_method, Recipe):
 353                                 blocker = self.cook(required_digest, retrieval_method, stores, force, impl_hint = impl)
 354                                 yield blocker
 355                                 tasks.check(blocker)
 356                         else:
 357                                 raise Exception(_("Unknown download type for '%s'") % retrieval_method)
 358
 359                         self.handler.impl_added_to_store(impl)
 360                 return download_impl()
 361
 362         def _add_to_cache(self, required_digest, stores, retrieval_method, stream):
 363                 assert isinstance(retrieval_method, DownloadSource)
 364                 stores.add_archive_to_cache(required_digest, stream, retrieval_method.url, retrieval_method.extract,
 365                                                  type = retrieval_method.type, start_offset = retrieval_method.start_offset or 0)
 366
 367         def download_archive(self, download_source, force = False, impl_hint = None):
 368                 """Fetch an archive. You should normally call L{download_impl}
 369                 instead, since it handles other kinds of retrieval method too."""
 370                 from zeroinstall.zerostore import unpack
 371
 372                 url = download_source.url
 373                 if not (url.startswith('http:') or url.startswith('https:') or url.startswith('ftp:')):
 374                         raise SafeException(_("Unknown scheme in download URL '%s'") % url)
 375
 376                 mime_type = download_source.type
 377                 if not mime_type:
 378                         mime_type = unpack.type_from_url(download_source.url)
 379                 if not mime_type:
 380                         raise SafeException(_("No 'type' attribute on archive, and I can't guess from the name (%s)") % download_source.url)
 381                 unpack.check_type_ok(mime_type)
 382                 dl = self.handler.get_download(download_source.url, force = force, hint = impl_hint)
 383                 dl.expected_size = download_source.size + (download_source.start_offset or 0)
 384                 return (dl.downloaded, dl.tempfile)
 385
 386         def download_icon(self, interface, force = False):
 387                 """Download an icon for this interface and add it to the
 388                 icon cache. If the interface has no icon do nothing.
 389                 @return: the task doing the import, or None
 390                 @rtype: L{tasks.Task}"""
 391                 debug("download_icon %(interface)s (force = %(force)d)", {'interface': interface, 'force': force})
 392
 393                 modification_time = None
 394                 existing_icon = self.config.iface_cache.get_icon_path(interface)
 395                 if existing_icon:
 396                         file_mtime = os.stat(existing_icon).st_mtime
 397                         from email.utils import formatdate
 398                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 399
 400                 # Find a suitable icon to download
 401                 for icon in interface.get_metadata(XMLNS_IFACE, 'icon'):
 402                         type = icon.getAttribute('type')
 403                         if type != 'image/png':
 404                                 debug(_('Skipping non-PNG icon'))
 405                                 continue
 406                         source = icon.getAttribute('href')
 407                         if source:
 408                                 break
 409                         warn(_('Missing "href" attribute on <icon> in %s'), interface)
 410                 else:
 411                         info(_('No PNG icons found in %s'), interface)
 412                         return
 413
 414                 try:
 415                         dl = self.handler.monitored_downloads[source]
 416                         if dl and force:
 417                                 dl.abort()
 418                                 raise KeyError
 419                 except KeyError:
 420                         dl = download.Download(source, hint = interface, modification_time = modification_time)
 421                         self.handler.monitor_download(dl)
 422
 423                 @tasks.async
 424                 def download_and_add_icon():
 425                         stream = dl.tempfile
 426                         yield dl.downloaded
 427                         try:
 428                                 tasks.check(dl.downloaded)
 429                                 if dl.unmodified: return
 430                                 stream.seek(0)
 431
 432                                 import shutil
 433                                 icons_cache = basedir.save_cache_path(config_site, 'interface_icons')
 434                                 icon_file = file(os.path.join(icons_cache, escape(interface.uri)), 'w')
 435                                 shutil.copyfileobj(stream, icon_file)
 436                         except Exception as ex:
 437                                 self.handler.report_error(ex)
 438
 439                 return download_and_add_icon()
 440
 441         def download_impls(self, implementations, stores):
 442                 """Download the given implementations, choosing a suitable retrieval method for each.
 443                 If any of the retrieval methods are DistributionSources and
 444                 need confirmation, handler.confirm is called to check that the
 445                 installation should proceed.
 446                 """
 447                 unsafe_impls = []
 448
 449                 to_download = []
 450                 for impl in implementations:
 451                         debug(_("start_downloading_impls: for %(feed)s get %(implementation)s"), {'feed': impl.feed, 'implementation': impl})
 452                         source = self.get_best_source(impl)
 453                         if not source:
 454                                 raise SafeException(_("Implementation %(implementation_id)s of interface %(interface)s"
 455                                         " cannot be downloaded (no download locations given in "
 456                                         "interface!)") % {'implementation_id': impl.id, 'interface': impl.feed.get_name()})
 457                         to_download.append((impl, source))
 458
 459                         if isinstance(source, DistributionSource) and source.needs_confirmation:
 460                                 unsafe_impls.append(source.package_id)
 461
 462                 @tasks.async
 463                 def download_impls():
 464                         if unsafe_impls:
 465                                 confirm = self.handler.confirm_install(_('The following components need to be installed using native packages. '
 466                                         'These come from your distribution, and should therefore be trustworthy, but they also '
 467                                         'run with extra privileges. In particular, installing them may run extra services on your '
 468                                         'computer or affect other users. You may be asked to enter a password to confirm. The '
 469                                         'packages are:\n\n') + ('\n'.join('- ' + x for x in unsafe_impls)))
 470                                 yield confirm
 471                                 tasks.check(confirm)
 472
 473                         blockers = []
 474
 475                         for impl, source in to_download:
 476                                 blockers.append(self.download_impl(impl, source, stores))
 477
 478                         # Record the first error log the rest
 479                         error = []
 480                         def dl_error(ex, tb = None):
 481                                 if error:
 482                                         self.handler.report_error(ex)
 483                                 else:
 484                                         error.append(ex)
 485                         while blockers:
 486                                 yield blockers
 487                                 tasks.check(blockers, dl_error)
 488
 489                                 blockers = [b for b in blockers if not b.happened]
 490                         if error:
 491                                 raise error[0]
 492
 493                 if not to_download:
 494                         return None
 495
 496                 return download_impls()
 497
 498         def get_best_source(self, impl):
 499                 """Return the best download source for this implementation.
 500                 @rtype: L{model.RetrievalMethod}"""
 501                 if impl.download_sources:
 502                         return impl.download_sources[0]
 503                 return None
 504