zeroinstall/injector/fetch.py

   1 """
   2 Downloads feeds, keys, packages and icons.
   3 """
   4
   5 # Copyright (C) 2009, Thomas Leonard
   6 # See the README file for details, or visit http://0install.net.
   7
   8 from zeroinstall import _
   9 import os
  10 from logging import info, debug, warn
  11
  12 from zeroinstall.support import tasks, basedir
  13 from zeroinstall.injector.namespaces import XMLNS_IFACE, config_site
  14 from zeroinstall.injector.model import DownloadSource, Recipe, SafeException, escape, DistributionSource
  15 from zeroinstall.injector.iface_cache import PendingFeed, ReplayAttack
  16 from zeroinstall.injector.handler import NoTrustedKeys
  17 from zeroinstall.injector import download
  18
  19 DEFAULT_FEED_MIRROR = "http://roscidus.com/0mirror"
  20 DEFAULT_KEY_LOOKUP_SERVER = 'https://keylookup.appspot.com'
  21
  22 def _escape_slashes(path):
  23         return path.replace('/', '%23')
  24
  25 def _get_feed_dir(feed):
  26         """The algorithm from 0mirror."""
  27         if '#' in feed:
  28                 raise SafeException(_("Invalid URL '%s'") % feed)
  29         scheme, rest = feed.split('://', 1)
  30         assert '/' in rest, "Missing / in %s" % feed
  31         domain, rest = rest.split('/', 1)
  32         for x in [scheme, domain, rest]:
  33                 if not x or x.startswith(','):
  34                         raise SafeException(_("Invalid URL '%s'") % feed)
  35         return os.path.join('feeds', scheme, domain, _escape_slashes(rest))
  36
  37 class KeyInfoFetcher:
  38         """Fetches information about a GPG key from a key-info server.
  39         See L{Fetcher.fetch_key_info} for details.
  40         @since: 0.42
  41
  42         Example:
  43
  44         >>> kf = KeyInfoFetcher(handler, 'https://server', fingerprint)
  45         >>> while True:
  46                 print kf.info
  47                 if kf.blocker is None: break
  48                 print kf.status
  49                 yield kf.blocker
  50         """
  51         def __init__(self, handler, server, fingerprint):
  52                 self.fingerprint = fingerprint
  53                 self.info = []
  54                 self.blocker = None
  55
  56                 if server is None: return
  57
  58                 self.status = _('Fetching key information from %s...') % server
  59
  60                 dl = handler.get_download(server + '/key/' + fingerprint)
  61
  62                 from xml.dom import minidom
  63
  64                 @tasks.async
  65                 def fetch_key_info():
  66                         try:
  67                                 tempfile = dl.tempfile
  68                                 yield dl.downloaded
  69                                 self.blocker = None
  70                                 tasks.check(dl.downloaded)
  71                                 tempfile.seek(0)
  72                                 doc = minidom.parse(tempfile)
  73                                 if doc.documentElement.localName != 'key-lookup':
  74                                         raise SafeException(_('Expected <key-lookup>, not <%s>') % doc.documentElement.localName)
  75                                 self.info += doc.documentElement.childNodes
  76                         except Exception, ex:
  77                                 doc = minidom.parseString('<item vote="bad"/>')
  78                                 root = doc.documentElement
  79                                 root.appendChild(doc.createTextNode(_('Error getting key information: %s') % ex))
  80                                 self.info.append(root)
  81
  82                 self.blocker = fetch_key_info()
  83
  84 class Fetcher(object):
  85         """Downloads and stores various things.
  86         @ivar config: used to get handler, iface_cache and stores
  87         @type config: L{config.Config}
  88         @ivar key_info: caches information about GPG keys
  89         @type key_info: {str: L{KeyInfoFetcher}}
  90         @ivar key_info_server: the base URL of a key information server
  91         @type key_info_server: str
  92         @ivar feed_mirror: the base URL of a mirror site for keys and feeds
  93         @type feed_mirror: str | None
  94         """
  95         __slots__ = ['config', 'feed_mirror', 'key_info_server', 'key_info']
  96
  97         def __init__(self, config):
  98                 assert config.handler, "API change!"
  99                 self.config = config
 100                 self.feed_mirror = DEFAULT_FEED_MIRROR
 101                 self.key_info_server = DEFAULT_KEY_LOOKUP_SERVER
 102                 self.key_info = {}
 103
 104         @property
 105         def handler(self):
 106                 return self.config.handler
 107
 108         @tasks.async
 109         def cook(self, required_digest, recipe, stores, force = False, impl_hint = None):
 110                 """Follow a Recipe.
 111                 @param impl_hint: the Implementation this is for (if any) as a hint for the GUI
 112                 @see: L{download_impl} uses this method when appropriate"""
 113                 # Maybe we're taking this metaphor too far?
 114
 115                 # Start downloading all the ingredients.
 116                 streams = {}    # Streams collected from successful downloads
 117
 118                 # Start a download for each ingredient
 119                 blockers = []
 120                 for step in recipe.steps:
 121                         blocker, stream = self.download_archive(step, force = force, impl_hint = impl_hint)
 122                         assert stream
 123                         blockers.append(blocker)
 124                         streams[step] = stream
 125
 126                 while blockers:
 127                         yield blockers
 128                         tasks.check(blockers)
 129                         blockers = [b for b in blockers if not b.happened]
 130
 131                 from zeroinstall.zerostore import unpack
 132
 133                 # Create an empty directory for the new implementation
 134                 store = stores.stores[0]
 135                 tmpdir = store.get_tmp_dir_for(required_digest)
 136                 try:
 137                         # Unpack each of the downloaded archives into it in turn
 138                         for step in recipe.steps:
 139                                 stream = streams[step]
 140                                 stream.seek(0)
 141                                 unpack.unpack_archive_over(step.url, stream, tmpdir, step.extract)
 142                         # Check that the result is correct and store it in the cache
 143                         store.check_manifest_and_rename(required_digest, tmpdir)
 144                         tmpdir = None
 145                 finally:
 146                         # If unpacking fails, remove the temporary directory
 147                         if tmpdir is not None:
 148                                 from zeroinstall import support
 149                                 support.ro_rmtree(tmpdir)
 150
 151         def get_feed_mirror(self, url):
 152                 """Return the URL of a mirror for this feed."""
 153                 if self.feed_mirror is None:
 154                         return None
 155                 import urlparse
 156                 if urlparse.urlparse(url).hostname == 'localhost':
 157                         return None
 158                 return '%s/%s/latest.xml' % (self.feed_mirror, _get_feed_dir(url))
 159
 160         @tasks.async
 161         def get_packagekit_feed(self, iface_cache, feed_url):
 162                 """Send a query to PackageKit (if available) for information about this package.
 163                 On success, the result is added to iface_cache.
 164                 """
 165                 assert feed_url.startswith('distribution:'), feed_url
 166                 master_feed = iface_cache.get_feed(feed_url.split(':', 1)[1])
 167                 if master_feed:
 168                         fetch = iface_cache.distro.fetch_candidates(master_feed)
 169                         if fetch:
 170                                 yield fetch
 171                                 tasks.check(fetch)
 172
 173                         # Force feed to be regenerated with the new information
 174                         iface_cache.get_feed(feed_url, force = True)
 175
 176         def download_and_import_feed(self, feed_url, iface_cache, force = False):
 177                 """Download the feed, download any required keys, confirm trust if needed and import.
 178                 @param feed_url: the feed to be downloaded
 179                 @type feed_url: str
 180                 @param iface_cache: cache in which to store the feed
 181                 @type iface_cache: L{iface_cache.IfaceCache}
 182                 @param force: whether to abort and restart an existing download"""
 183                 from download import DownloadAborted
 184
 185                 debug(_("download_and_import_feed %(url)s (force = %(force)d)"), {'url': feed_url, 'force': force})
 186                 assert not os.path.isabs(feed_url)
 187
 188                 if feed_url.startswith('distribution:'):
 189                         return self.get_packagekit_feed(iface_cache, feed_url)
 190
 191                 primary = self._download_and_import_feed(feed_url, iface_cache, force, use_mirror = False)
 192
 193                 @tasks.named_async("monitor feed downloads for " + feed_url)
 194                 def wait_for_downloads(primary):
 195                         # Download just the upstream feed, unless it takes too long...
 196                         timeout = tasks.TimeoutBlocker(5, 'Mirror timeout')             # 5 seconds
 197
 198                         yield primary, timeout
 199                         tasks.check(timeout)
 200
 201                         try:
 202                                 tasks.check(primary)
 203                                 if primary.happened:
 204                                         return          # OK, primary succeeded!
 205                                 # OK, maybe it's just being slow...
 206                                 info("Feed download from %s is taking a long time.", feed_url)
 207                                 primary_ex = None
 208                         except NoTrustedKeys, ex:
 209                                 raise                   # Don't bother trying the mirror if we have a trust problem
 210                         except ReplayAttack, ex:
 211                                 raise                   # Don't bother trying the mirror if we have a replay attack
 212                         except DownloadAborted, ex:
 213                                 raise                   # Don't bother trying the mirror if the user cancelled
 214                         except SafeException, ex:
 215                                 # Primary failed
 216                                 primary = None
 217                                 primary_ex = ex
 218                                 warn(_("Feed download from %(url)s failed: %(exception)s"), {'url': feed_url, 'exception': ex})
 219
 220                         # Start downloading from mirror...
 221                         mirror = self._download_and_import_feed(feed_url, iface_cache, force, use_mirror = True)
 222
 223                         # Wait until both mirror and primary tasks are complete...
 224                         while True:
 225                                 blockers = filter(None, [primary, mirror])
 226                                 if not blockers:
 227                                         break
 228                                 yield blockers
 229
 230                                 if primary:
 231                                         try:
 232                                                 tasks.check(primary)
 233                                                 if primary.happened:
 234                                                         primary = None
 235                                                         # No point carrying on with the mirror once the primary has succeeded
 236                                                         if mirror:
 237                                                                 info(_("Primary feed download succeeded; aborting mirror download for %s") % feed_url)
 238                                                                 mirror.dl.abort()
 239                                         except SafeException, ex:
 240                                                 primary = None
 241                                                 primary_ex = ex
 242                                                 info(_("Feed download from %(url)s failed; still trying mirror: %(exception)s"), {'url': feed_url, 'exception': ex})
 243
 244                                 if mirror:
 245                                         try:
 246                                                 tasks.check(mirror)
 247                                                 if mirror.happened:
 248                                                         mirror = None
 249                                                         if primary_ex:
 250                                                                 # We already warned; no need to raise an exception too,
 251                                                                 # as the mirror download succeeded.
 252                                                                 primary_ex = None
 253                                         except ReplayAttack, ex:
 254                                                 info(_("Version from mirror is older than cached version; ignoring it: %s"), ex)
 255                                                 mirror = None
 256                                                 primary_ex = None
 257                                         except SafeException, ex:
 258                                                 info(_("Mirror download failed: %s"), ex)
 259                                                 mirror = None
 260
 261                         if primary_ex:
 262                                 raise primary_ex
 263
 264                 return wait_for_downloads(primary)
 265
 266         def _download_and_import_feed(self, feed_url, iface_cache, force, use_mirror):
 267                 """Download and import a feed.
 268                 @param use_mirror: False to use primary location; True to use mirror."""
 269                 if use_mirror:
 270                         url = self.get_feed_mirror(feed_url)
 271                         if url is None: return None
 272                         warn(_("Trying mirror server for feed %s") % feed_url)
 273                 else:
 274                         url = feed_url
 275
 276                 dl = self.handler.get_download(url, force = force, hint = feed_url)
 277                 stream = dl.tempfile
 278
 279                 @tasks.named_async("fetch_feed " + url)
 280                 def fetch_feed():
 281                         yield dl.downloaded
 282                         tasks.check(dl.downloaded)
 283
 284                         pending = PendingFeed(feed_url, stream)
 285
 286                         if use_mirror:
 287                                 # If we got the feed from a mirror, get the key from there too
 288                                 key_mirror = self.feed_mirror + '/keys/'
 289                         else:
 290                                 key_mirror = None
 291
 292                         keys_downloaded = tasks.Task(pending.download_keys(self.handler, feed_hint = feed_url, key_mirror = key_mirror), _("download keys for %s") % feed_url)
 293                         yield keys_downloaded.finished
 294                         tasks.check(keys_downloaded.finished)
 295
 296                         if not iface_cache.update_feed_if_trusted(pending.url, pending.sigs, pending.new_xml):
 297                                 blocker = self.handler.confirm_keys(pending, self.fetch_key_info)
 298                                 if blocker:
 299                                         yield blocker
 300                                         tasks.check(blocker)
 301                                 if not iface_cache.update_feed_if_trusted(pending.url, pending.sigs, pending.new_xml):
 302                                         raise NoTrustedKeys(_("No signing keys trusted; not importing"))
 303
 304                 task = fetch_feed()
 305                 task.dl = dl
 306                 return task
 307
 308         def fetch_key_info(self, fingerprint):
 309                 try:
 310                         return self.key_info[fingerprint]
 311                 except KeyError:
 312                         self.key_info[fingerprint] = key_info = KeyInfoFetcher(self.handler,
 313                                                                         self.key_info_server, fingerprint)
 314                         return key_info
 315
 316         def download_impl(self, impl, retrieval_method, stores, force = False):
 317                 """Download an implementation.
 318                 @param impl: the selected implementation
 319                 @type impl: L{model.ZeroInstallImplementation}
 320                 @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
 321                 @type retrieval_method: L{model.RetrievalMethod}
 322                 @param stores: where to store the downloaded implementation
 323                 @type stores: L{zerostore.Stores}
 324                 @param force: whether to abort and restart an existing download
 325                 @rtype: L{tasks.Blocker}"""
 326                 assert impl
 327                 assert retrieval_method
 328
 329                 if isinstance(retrieval_method, DistributionSource):
 330                         return retrieval_method.install(self.handler)
 331
 332                 from zeroinstall.zerostore import manifest
 333                 best = None
 334                 for digest in impl.digests:
 335                         alg_name = digest.split('=', 1)[0]
 336                         alg = manifest.algorithms.get(alg_name, None)
 337                         if alg and (best is None or best.rating < alg.rating):
 338                                 best = alg
 339                                 required_digest = digest
 340
 341                 if best is None:
 342                         if not impl.digests:
 343                                 raise SafeException(_("No <manifest-digest> given for '%(implementation)s' version %(version)s") %
 344                                                 {'implementation': impl.feed.get_name(), 'version': impl.get_version()})
 345                         raise SafeException(_("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s") %
 346                                         {'algorithms': impl.digests, 'implementation': impl.feed.get_name(), 'version': impl.get_version()})
 347
 348                 @tasks.async
 349                 def download_impl():
 350                         if isinstance(retrieval_method, DownloadSource):
 351                                 blocker, stream = self.download_archive(retrieval_method, force = force, impl_hint = impl)
 352                                 yield blocker
 353                                 tasks.check(blocker)
 354
 355                                 stream.seek(0)
 356                                 self._add_to_cache(required_digest, stores, retrieval_method, stream)
 357                         elif isinstance(retrieval_method, Recipe):
 358                                 blocker = self.cook(required_digest, retrieval_method, stores, force, impl_hint = impl)
 359                                 yield blocker
 360                                 tasks.check(blocker)
 361                         else:
 362                                 raise Exception(_("Unknown download type for '%s'") % retrieval_method)
 363
 364                         self.handler.impl_added_to_store(impl)
 365                 return download_impl()
 366
 367         def _add_to_cache(self, required_digest, stores, retrieval_method, stream):
 368                 assert isinstance(retrieval_method, DownloadSource)
 369                 stores.add_archive_to_cache(required_digest, stream, retrieval_method.url, retrieval_method.extract,
 370                                                  type = retrieval_method.type, start_offset = retrieval_method.start_offset or 0)
 371
 372         def download_archive(self, download_source, force = False, impl_hint = None):
 373                 """Fetch an archive. You should normally call L{download_impl}
 374                 instead, since it handles other kinds of retrieval method too."""
 375                 from zeroinstall.zerostore import unpack
 376
 377                 url = download_source.url
 378                 if not (url.startswith('http:') or url.startswith('https:') or url.startswith('ftp:')):
 379                         raise SafeException(_("Unknown scheme in download URL '%s'") % url)
 380
 381                 mime_type = download_source.type
 382                 if not mime_type:
 383                         mime_type = unpack.type_from_url(download_source.url)
 384                 if not mime_type:
 385                         raise SafeException(_("No 'type' attribute on archive, and I can't guess from the name (%s)") % download_source.url)
 386                 unpack.check_type_ok(mime_type)
 387                 dl = self.handler.get_download(download_source.url, force = force, hint = impl_hint)
 388                 dl.expected_size = download_source.size + (download_source.start_offset or 0)
 389                 return (dl.downloaded, dl.tempfile)
 390
 391         def download_icon(self, interface, force = False):
 392                 """Download an icon for this interface and add it to the
 393                 icon cache. If the interface has no icon do nothing.
 394                 @return: the task doing the import, or None
 395                 @rtype: L{tasks.Task}"""
 396                 debug("download_icon %(interface)s (force = %(force)d)", {'interface': interface, 'force': force})
 397
 398                 modification_time = None
 399                 existing_icon = self.config.iface_cache.get_icon_path(interface)
 400                 if existing_icon:
 401                         file_mtime = os.stat(existing_icon).st_mtime
 402                         from email.utils import formatdate
 403                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 404
 405                 # Find a suitable icon to download
 406                 for icon in interface.get_metadata(XMLNS_IFACE, 'icon'):
 407                         type = icon.getAttribute('type')
 408                         if type != 'image/png':
 409                                 debug(_('Skipping non-PNG icon'))
 410                                 continue
 411                         source = icon.getAttribute('href')
 412                         if source:
 413                                 break
 414                         warn(_('Missing "href" attribute on <icon> in %s'), interface)
 415                 else:
 416                         info(_('No PNG icons found in %s'), interface)
 417                         return
 418
 419                 try:
 420                         dl = self.handler.monitored_downloads[source]
 421                         if dl and force:
 422                                 dl.abort()
 423                                 raise KeyError
 424                 except KeyError:
 425                         dl = download.Download(source, hint = interface, modification_time = modification_time)
 426                         self.handler.monitor_download(dl)
 427
 428                 @tasks.async
 429                 def download_and_add_icon():
 430                         stream = dl.tempfile
 431                         yield dl.downloaded
 432                         try:
 433                                 tasks.check(dl.downloaded)
 434                                 if dl.unmodified: return
 435                                 stream.seek(0)
 436
 437                                 import shutil
 438                                 icons_cache = basedir.save_cache_path(config_site, 'interface_icons')
 439                                 icon_file = file(os.path.join(icons_cache, escape(interface.uri)), 'w')
 440                                 shutil.copyfileobj(stream, icon_file)
 441                         except Exception, ex:
 442                                 self.handler.report_error(ex)
 443
 444                 return download_and_add_icon()
 445
 446         def download_impls(self, implementations, stores):
 447                 """Download the given implementations, choosing a suitable retrieval method for each.
 448                 If any of the retrieval methods are DistributionSources and
 449                 need confirmation, handler.confirm is called to check that the
 450                 installation should proceed.
 451                 """
 452                 unsafe_impls = []
 453
 454                 to_download = []
 455                 for impl in implementations:
 456                         debug(_("start_downloading_impls: for %(feed)s get %(implementation)s"), {'feed': impl.feed, 'implementation': impl})
 457                         source = self.get_best_source(impl)
 458                         if not source:
 459                                 raise SafeException(_("Implementation %(implementation_id)s of interface %(interface)s"
 460                                         " cannot be downloaded (no download locations given in "
 461                                         "interface!)") % {'implementation_id': impl.id, 'interface': impl.feed.get_name()})
 462                         to_download.append((impl, source))
 463
 464                         if isinstance(source, DistributionSource) and source.needs_confirmation:
 465                                 unsafe_impls.append(source.package_id)
 466
 467                 @tasks.async
 468                 def download_impls():
 469                         if unsafe_impls:
 470                                 confirm = self.handler.confirm_install(_('The following components need to be installed using native packages. '
 471                                         'These come from your distribution, and should therefore be trustworthy, but they also '
 472                                         'run with extra privileges. In particular, installing them may run extra services on your '
 473                                         'computer or affect other users. You may be asked to enter a password to confirm. The '
 474                                         'packages are:\n\n') + ('\n'.join('- ' + x for x in unsafe_impls)))
 475                                 yield confirm
 476                                 tasks.check(confirm)
 477
 478                         blockers = []
 479
 480                         for impl, source in to_download:
 481                                 blockers.append(self.download_impl(impl, source, stores))
 482
 483                         # Record the first error log the rest
 484                         error = []
 485                         def dl_error(ex, tb = None):
 486                                 if error:
 487                                         self.handler.report_error(ex)
 488                                 else:
 489                                         error.append(ex)
 490                         while blockers:
 491                                 yield blockers
 492                                 tasks.check(blockers, dl_error)
 493
 494                                 blockers = [b for b in blockers if not b.happened]
 495                         if error:
 496                                 raise error[0]
 497
 498                 if not to_download:
 499                         return None
 500
 501                 return download_impls()
 502
 503         def get_best_source(self, impl):
 504                 """Return the best download source for this implementation.
 505                 @rtype: L{model.RetrievalMethod}"""
 506                 if impl.download_sources:
 507                         return impl.download_sources[0]
 508                 return None
 509