zeroinstall/injector/fetch.py

   1 """
   2 Downloads feeds, keys, packages and icons.
   3 """
   4
   5 # Copyright (C) 2009, Thomas Leonard
   6 # See the README file for details, or visit http://0install.net.
   7
   8 from zeroinstall import _
   9 import os
  10 from logging import info, debug, warn
  11
  12 from zeroinstall.support import tasks, basedir
  13 from zeroinstall.injector.namespaces import XMLNS_IFACE, config_site
  14 from zeroinstall.injector.model import DownloadSource, Recipe, SafeException, escape, DistributionSource
  15 from zeroinstall.injector.iface_cache import PendingFeed, ReplayAttack
  16 from zeroinstall.injector.handler import NoTrustedKeys
  17 from zeroinstall.injector import download
  18
  19 DEFAULT_FEED_MIRROR = "http://roscidus.com/0mirror"
  20 DEFAULT_KEY_LOOKUP_SERVER = 'https://keylookup.appspot.com'
  21
  22 def _escape_slashes(path):
  23         return path.replace('/', '%23')
  24
  25 def _get_feed_dir(feed):
  26         """The algorithm from 0mirror."""
  27         if '#' in feed:
  28                 raise SafeException(_("Invalid URL '%s'") % feed)
  29         scheme, rest = feed.split('://', 1)
  30         assert '/' in rest, "Missing / in %s" % feed
  31         domain, rest = rest.split('/', 1)
  32         for x in [scheme, domain, rest]:
  33                 if not x or x.startswith(','):
  34                         raise SafeException(_("Invalid URL '%s'") % feed)
  35         return os.path.join('feeds', scheme, domain, _escape_slashes(rest))
  36
  37 class KeyInfoFetcher:
  38         """Fetches information about a GPG key from a key-info server.
  39         See L{Fetcher.fetch_key_info} for details.
  40         @since: 0.42
  41
  42         Example:
  43
  44         >>> kf = KeyInfoFetcher(handler, 'https://server', fingerprint)
  45         >>> while True:
  46                 print kf.info
  47                 if kf.blocker is None: break
  48                 print kf.status
  49                 yield kf.blocker
  50         """
  51         def __init__(self, handler, server, fingerprint):
  52                 self.fingerprint = fingerprint
  53                 self.info = []
  54                 self.blocker = None
  55
  56                 if server is None: return
  57
  58                 self.status = _('Fetching key information from %s...') % server
  59
  60                 dl = handler.get_download(server + '/key/' + fingerprint)
  61
  62                 from xml.dom import minidom
  63
  64                 @tasks.async
  65                 def fetch_key_info():
  66                         try:
  67                                 tempfile = dl.tempfile
  68                                 yield dl.downloaded
  69                                 self.blocker = None
  70                                 tasks.check(dl.downloaded)
  71                                 tempfile.seek(0)
  72                                 doc = minidom.parse(tempfile)
  73                                 if doc.documentElement.localName != 'key-lookup':
  74                                         raise SafeException(_('Expected <key-lookup>, not <%s>') % doc.documentElement.localName)
  75                                 self.info += doc.documentElement.childNodes
  76                         except Exception, ex:
  77                                 doc = minidom.parseString('<item vote="bad"/>')
  78                                 root = doc.documentElement
  79                                 root.appendChild(doc.createTextNode(_('Error getting key information: %s') % ex))
  80                                 self.info.append(root)
  81
  82                 self.blocker = fetch_key_info()
  83
  84 class Fetcher(object):
  85         """Downloads and stores various things.
  86         @ivar config: used to get handler, iface_cache and stores
  87         @type config: L{config.Config}
  88         @ivar key_info: caches information about GPG keys
  89         @type key_info: {str: L{KeyInfoFetcher}}
  90         @ivar key_info_server: the base URL of a key information server
  91         @type key_info_server: str
  92         @ivar feed_mirror: the base URL of a mirror site for keys and feeds
  93         @type feed_mirror: str | None
  94         """
  95         __slots__ = ['config', 'feed_mirror', 'key_info_server', 'key_info']
  96
  97         def __init__(self, config):
  98                 assert config.handler, "API change!"
  99                 self.config = config
 100                 self.feed_mirror = DEFAULT_FEED_MIRROR
 101                 self.key_info_server = DEFAULT_KEY_LOOKUP_SERVER
 102                 self.key_info = {}
 103
 104         @property
 105         def handler(self):
 106                 return self.config.handler
 107
 108         @tasks.async
 109         def cook(self, required_digest, recipe, stores, force = False, impl_hint = None):
 110                 """Follow a Recipe.
 111                 @param impl_hint: the Implementation this is for (if any) as a hint for the GUI
 112                 @see: L{download_impl} uses this method when appropriate"""
 113                 # Maybe we're taking this metaphor too far?
 114
 115                 # Start downloading all the ingredients.
 116                 streams = {}    # Streams collected from successful downloads
 117
 118                 # Start a download for each ingredient
 119                 blockers = []
 120                 for step in recipe.steps:
 121                         blocker, stream = self.download_archive(step, force = force, impl_hint = impl_hint)
 122                         assert stream
 123                         blockers.append(blocker)
 124                         streams[step] = stream
 125
 126                 while blockers:
 127                         yield blockers
 128                         tasks.check(blockers)
 129                         blockers = [b for b in blockers if not b.happened]
 130
 131                 from zeroinstall.zerostore import unpack
 132
 133                 # Create an empty directory for the new implementation
 134                 store = stores.stores[0]
 135                 tmpdir = store.get_tmp_dir_for(required_digest)
 136                 try:
 137                         # Unpack each of the downloaded archives into it in turn
 138                         for step in recipe.steps:
 139                                 stream = streams[step]
 140                                 stream.seek(0)
 141                                 unpack.unpack_archive_over(step.url, stream, tmpdir, step.extract)
 142                         # Check that the result is correct and store it in the cache
 143                         store.check_manifest_and_rename(required_digest, tmpdir)
 144                         tmpdir = None
 145                 finally:
 146                         # If unpacking fails, remove the temporary directory
 147                         if tmpdir is not None:
 148                                 from zeroinstall import support
 149                                 support.ro_rmtree(tmpdir)
 150
 151         def get_feed_mirror(self, url):
 152                 """Return the URL of a mirror for this feed."""
 153                 if self.feed_mirror is None:
 154                         return None
 155                 import urlparse
 156                 if urlparse.urlparse(url).hostname == 'localhost':
 157                         return None
 158                 return '%s/%s/latest.xml' % (self.feed_mirror, _get_feed_dir(url))
 159
 160         @tasks.async
 161         def get_packagekit_feed(self, feed_url):
 162                 """Send a query to PackageKit (if available) for information about this package.
 163                 On success, the result is added to iface_cache.
 164                 """
 165                 assert feed_url.startswith('distribution:'), feed_url
 166                 master_feed = self.config.iface_cache.get_feed(feed_url.split(':', 1)[1])
 167                 if master_feed:
 168                         fetch = self.config.iface_cache.distro.fetch_candidates(master_feed)
 169                         if fetch:
 170                                 yield fetch
 171                                 tasks.check(fetch)
 172
 173                         # Force feed to be regenerated with the new information
 174                         self.config.iface_cache.get_feed(feed_url, force = True)
 175
 176         def download_and_import_feed(self, feed_url, iface_cache = None, force = False):
 177                 """Download the feed, download any required keys, confirm trust if needed and import.
 178                 @param feed_url: the feed to be downloaded
 179                 @type feed_url: str
 180                 @param iface_cache: (deprecated)
 181                 @param force: whether to abort and restart an existing download"""
 182                 from .download import DownloadAborted
 183
 184                 assert iface_cache is None or iface_cache is self.config.iface_cache
 185
 186                 debug(_("download_and_import_feed %(url)s (force = %(force)d)"), {'url': feed_url, 'force': force})
 187                 assert not os.path.isabs(feed_url)
 188
 189                 if feed_url.startswith('distribution:'):
 190                         return self.get_packagekit_feed(feed_url)
 191
 192                 primary = self._download_and_import_feed(feed_url, force, use_mirror = False)
 193
 194                 @tasks.named_async("monitor feed downloads for " + feed_url)
 195                 def wait_for_downloads(primary):
 196                         # Download just the upstream feed, unless it takes too long...
 197                         timeout = tasks.TimeoutBlocker(5, 'Mirror timeout')             # 5 seconds
 198
 199                         yield primary, timeout
 200                         tasks.check(timeout)
 201
 202                         try:
 203                                 tasks.check(primary)
 204                                 if primary.happened:
 205                                         return          # OK, primary succeeded!
 206                                 # OK, maybe it's just being slow...
 207                                 info("Feed download from %s is taking a long time.", feed_url)
 208                                 primary_ex = None
 209                         except NoTrustedKeys, ex:
 210                                 raise                   # Don't bother trying the mirror if we have a trust problem
 211                         except ReplayAttack, ex:
 212                                 raise                   # Don't bother trying the mirror if we have a replay attack
 213                         except DownloadAborted, ex:
 214                                 raise                   # Don't bother trying the mirror if the user cancelled
 215                         except SafeException, ex:
 216                                 # Primary failed
 217                                 primary = None
 218                                 primary_ex = ex
 219                                 warn(_("Feed download from %(url)s failed: %(exception)s"), {'url': feed_url, 'exception': ex})
 220
 221                         # Start downloading from mirror...
 222                         mirror = self._download_and_import_feed(feed_url, force, use_mirror = True)
 223
 224                         # Wait until both mirror and primary tasks are complete...
 225                         while True:
 226                                 blockers = filter(None, [primary, mirror])
 227                                 if not blockers:
 228                                         break
 229                                 yield blockers
 230
 231                                 if primary:
 232                                         try:
 233                                                 tasks.check(primary)
 234                                                 if primary.happened:
 235                                                         primary = None
 236                                                         # No point carrying on with the mirror once the primary has succeeded
 237                                                         if mirror:
 238                                                                 info(_("Primary feed download succeeded; aborting mirror download for %s") % feed_url)
 239                                                                 mirror.dl.abort()
 240                                         except SafeException, ex:
 241                                                 primary = None
 242                                                 primary_ex = ex
 243                                                 info(_("Feed download from %(url)s failed; still trying mirror: %(exception)s"), {'url': feed_url, 'exception': ex})
 244
 245                                 if mirror:
 246                                         try:
 247                                                 tasks.check(mirror)
 248                                                 if mirror.happened:
 249                                                         mirror = None
 250                                                         if primary_ex:
 251                                                                 # We already warned; no need to raise an exception too,
 252                                                                 # as the mirror download succeeded.
 253                                                                 primary_ex = None
 254                                         except ReplayAttack, ex:
 255                                                 info(_("Version from mirror is older than cached version; ignoring it: %s"), ex)
 256                                                 mirror = None
 257                                                 primary_ex = None
 258                                         except SafeException, ex:
 259                                                 info(_("Mirror download failed: %s"), ex)
 260                                                 mirror = None
 261
 262                         if primary_ex:
 263                                 raise primary_ex
 264
 265                 return wait_for_downloads(primary)
 266
 267         def _download_and_import_feed(self, feed_url, force, use_mirror):
 268                 """Download and import a feed.
 269                 @param use_mirror: False to use primary location; True to use mirror."""
 270                 if use_mirror:
 271                         url = self.get_feed_mirror(feed_url)
 272                         if url is None: return None
 273                         warn(_("Trying mirror server for feed %s") % feed_url)
 274                 else:
 275                         url = feed_url
 276
 277                 dl = self.handler.get_download(url, force = force, hint = feed_url)
 278                 stream = dl.tempfile
 279
 280                 @tasks.named_async("fetch_feed " + url)
 281                 def fetch_feed():
 282                         yield dl.downloaded
 283                         tasks.check(dl.downloaded)
 284
 285                         pending = PendingFeed(feed_url, stream)
 286
 287                         if use_mirror:
 288                                 # If we got the feed from a mirror, get the key from there too
 289                                 key_mirror = self.feed_mirror + '/keys/'
 290                         else:
 291                                 key_mirror = None
 292
 293                         keys_downloaded = tasks.Task(pending.download_keys(self.handler, feed_hint = feed_url, key_mirror = key_mirror), _("download keys for %s") % feed_url)
 294                         yield keys_downloaded.finished
 295                         tasks.check(keys_downloaded.finished)
 296
 297                         if not self.config.iface_cache.update_feed_if_trusted(pending.url, pending.sigs, pending.new_xml):
 298                                 blocker = self.handler.confirm_keys(pending, self.fetch_key_info)
 299                                 if blocker:
 300                                         yield blocker
 301                                         tasks.check(blocker)
 302                                 if not self.config.iface_cache.update_feed_if_trusted(pending.url, pending.sigs, pending.new_xml):
 303                                         raise NoTrustedKeys(_("No signing keys trusted; not importing"))
 304
 305                 task = fetch_feed()
 306                 task.dl = dl
 307                 return task
 308
 309         def fetch_key_info(self, fingerprint):
 310                 try:
 311                         return self.key_info[fingerprint]
 312                 except KeyError:
 313                         self.key_info[fingerprint] = key_info = KeyInfoFetcher(self.handler,
 314                                                                         self.key_info_server, fingerprint)
 315                         return key_info
 316
 317         def download_impl(self, impl, retrieval_method, stores, force = False):
 318                 """Download an implementation.
 319                 @param impl: the selected implementation
 320                 @type impl: L{model.ZeroInstallImplementation}
 321                 @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
 322                 @type retrieval_method: L{model.RetrievalMethod}
 323                 @param stores: where to store the downloaded implementation
 324                 @type stores: L{zerostore.Stores}
 325                 @param force: whether to abort and restart an existing download
 326                 @rtype: L{tasks.Blocker}"""
 327                 assert impl
 328                 assert retrieval_method
 329
 330                 if isinstance(retrieval_method, DistributionSource):
 331                         return retrieval_method.install(self.handler)
 332
 333                 from zeroinstall.zerostore import manifest
 334                 best = None
 335                 for digest in impl.digests:
 336                         alg_name = digest.split('=', 1)[0]
 337                         alg = manifest.algorithms.get(alg_name, None)
 338                         if alg and (best is None or best.rating < alg.rating):
 339                                 best = alg
 340                                 required_digest = digest
 341
 342                 if best is None:
 343                         if not impl.digests:
 344                                 raise SafeException(_("No <manifest-digest> given for '%(implementation)s' version %(version)s") %
 345                                                 {'implementation': impl.feed.get_name(), 'version': impl.get_version()})
 346                         raise SafeException(_("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s") %
 347                                         {'algorithms': impl.digests, 'implementation': impl.feed.get_name(), 'version': impl.get_version()})
 348
 349                 @tasks.async
 350                 def download_impl():
 351                         if isinstance(retrieval_method, DownloadSource):
 352                                 blocker, stream = self.download_archive(retrieval_method, force = force, impl_hint = impl)
 353                                 yield blocker
 354                                 tasks.check(blocker)
 355
 356                                 stream.seek(0)
 357                                 self._add_to_cache(required_digest, stores, retrieval_method, stream)
 358                         elif isinstance(retrieval_method, Recipe):
 359                                 blocker = self.cook(required_digest, retrieval_method, stores, force, impl_hint = impl)
 360                                 yield blocker
 361                                 tasks.check(blocker)
 362                         else:
 363                                 raise Exception(_("Unknown download type for '%s'") % retrieval_method)
 364
 365                         self.handler.impl_added_to_store(impl)
 366                 return download_impl()
 367
 368         def _add_to_cache(self, required_digest, stores, retrieval_method, stream):
 369                 assert isinstance(retrieval_method, DownloadSource)
 370                 stores.add_archive_to_cache(required_digest, stream, retrieval_method.url, retrieval_method.extract,
 371                                                  type = retrieval_method.type, start_offset = retrieval_method.start_offset or 0)
 372
 373         def download_archive(self, download_source, force = False, impl_hint = None):
 374                 """Fetch an archive. You should normally call L{download_impl}
 375                 instead, since it handles other kinds of retrieval method too."""
 376                 from zeroinstall.zerostore import unpack
 377
 378                 url = download_source.url
 379                 if not (url.startswith('http:') or url.startswith('https:') or url.startswith('ftp:')):
 380                         raise SafeException(_("Unknown scheme in download URL '%s'") % url)
 381
 382                 mime_type = download_source.type
 383                 if not mime_type:
 384                         mime_type = unpack.type_from_url(download_source.url)
 385                 if not mime_type:
 386                         raise SafeException(_("No 'type' attribute on archive, and I can't guess from the name (%s)") % download_source.url)
 387                 unpack.check_type_ok(mime_type)
 388                 dl = self.handler.get_download(download_source.url, force = force, hint = impl_hint)
 389                 dl.expected_size = download_source.size + (download_source.start_offset or 0)
 390                 return (dl.downloaded, dl.tempfile)
 391
 392         def download_icon(self, interface, force = False):
 393                 """Download an icon for this interface and add it to the
 394                 icon cache. If the interface has no icon do nothing.
 395                 @return: the task doing the import, or None
 396                 @rtype: L{tasks.Task}"""
 397                 debug("download_icon %(interface)s (force = %(force)d)", {'interface': interface, 'force': force})
 398
 399                 modification_time = None
 400                 existing_icon = self.config.iface_cache.get_icon_path(interface)
 401                 if existing_icon:
 402                         file_mtime = os.stat(existing_icon).st_mtime
 403                         from email.utils import formatdate
 404                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 405
 406                 # Find a suitable icon to download
 407                 for icon in interface.get_metadata(XMLNS_IFACE, 'icon'):
 408                         type = icon.getAttribute('type')
 409                         if type != 'image/png':
 410                                 debug(_('Skipping non-PNG icon'))
 411                                 continue
 412                         source = icon.getAttribute('href')
 413                         if source:
 414                                 break
 415                         warn(_('Missing "href" attribute on <icon> in %s'), interface)
 416                 else:
 417                         info(_('No PNG icons found in %s'), interface)
 418                         return
 419
 420                 try:
 421                         dl = self.handler.monitored_downloads[source]
 422                         if dl and force:
 423                                 dl.abort()
 424                                 raise KeyError
 425                 except KeyError:
 426                         dl = download.Download(source, hint = interface, modification_time = modification_time)
 427                         self.handler.monitor_download(dl)
 428
 429                 @tasks.async
 430                 def download_and_add_icon():
 431                         stream = dl.tempfile
 432                         yield dl.downloaded
 433                         try:
 434                                 tasks.check(dl.downloaded)
 435                                 if dl.unmodified: return
 436                                 stream.seek(0)
 437
 438                                 import shutil
 439                                 icons_cache = basedir.save_cache_path(config_site, 'interface_icons')
 440                                 icon_file = file(os.path.join(icons_cache, escape(interface.uri)), 'w')
 441                                 shutil.copyfileobj(stream, icon_file)
 442                         except Exception, ex:
 443                                 self.handler.report_error(ex)
 444
 445                 return download_and_add_icon()
 446
 447         def download_impls(self, implementations, stores):
 448                 """Download the given implementations, choosing a suitable retrieval method for each.
 449                 If any of the retrieval methods are DistributionSources and
 450                 need confirmation, handler.confirm is called to check that the
 451                 installation should proceed.
 452                 """
 453                 unsafe_impls = []
 454
 455                 to_download = []
 456                 for impl in implementations:
 457                         debug(_("start_downloading_impls: for %(feed)s get %(implementation)s"), {'feed': impl.feed, 'implementation': impl})
 458                         source = self.get_best_source(impl)
 459                         if not source:
 460                                 raise SafeException(_("Implementation %(implementation_id)s of interface %(interface)s"
 461                                         " cannot be downloaded (no download locations given in "
 462                                         "interface!)") % {'implementation_id': impl.id, 'interface': impl.feed.get_name()})
 463                         to_download.append((impl, source))
 464
 465                         if isinstance(source, DistributionSource) and source.needs_confirmation:
 466                                 unsafe_impls.append(source.package_id)
 467
 468                 @tasks.async
 469                 def download_impls():
 470                         if unsafe_impls:
 471                                 confirm = self.handler.confirm_install(_('The following components need to be installed using native packages. '
 472                                         'These come from your distribution, and should therefore be trustworthy, but they also '
 473                                         'run with extra privileges. In particular, installing them may run extra services on your '
 474                                         'computer or affect other users. You may be asked to enter a password to confirm. The '
 475                                         'packages are:\n\n') + ('\n'.join('- ' + x for x in unsafe_impls)))
 476                                 yield confirm
 477                                 tasks.check(confirm)
 478
 479                         blockers = []
 480
 481                         for impl, source in to_download:
 482                                 blockers.append(self.download_impl(impl, source, stores))
 483
 484                         # Record the first error log the rest
 485                         error = []
 486                         def dl_error(ex, tb = None):
 487                                 if error:
 488                                         self.handler.report_error(ex)
 489                                 else:
 490                                         error.append(ex)
 491                         while blockers:
 492                                 yield blockers
 493                                 tasks.check(blockers, dl_error)
 494
 495                                 blockers = [b for b in blockers if not b.happened]
 496                         if error:
 497                                 raise error[0]
 498
 499                 if not to_download:
 500                         return None
 501
 502                 return download_impls()
 503
 504         def get_best_source(self, impl):
 505                 """Return the best download source for this implementation.
 506                 @rtype: L{model.RetrievalMethod}"""
 507                 if impl.download_sources:
 508                         return impl.download_sources[0]
 509                 return None
 510