zeroinstall/injector/fetch.py

   1 """
   2 Downloads feeds, keys, packages and icons.
   3 """
   4
   5 # Copyright (C) 2009, Thomas Leonard
   6 # See the README file for details, or visit http://0install.net.
   7
   8 from zeroinstall import _
   9 import os
  10 from logging import info, debug, warn
  11
  12 from zeroinstall.support import tasks, basedir
  13 from zeroinstall.injector.namespaces import XMLNS_IFACE, config_site
  14 from zeroinstall.injector.model import DownloadSource, Recipe, SafeException, escape, DistributionSource
  15 from zeroinstall.injector.iface_cache import PendingFeed, ReplayAttack
  16 from zeroinstall.injector.handler import NoTrustedKeys
  17 from zeroinstall.injector import download
  18
  19 def _escape_slashes(path):
  20         return path.replace('/', '%23')
  21
  22 def _get_feed_dir(feed):
  23         """The algorithm from 0mirror."""
  24         if '#' in feed:
  25                 raise SafeException(_("Invalid URL '%s'") % feed)
  26         scheme, rest = feed.split('://', 1)
  27         assert '/' in rest, "Missing / in %s" % feed
  28         domain, rest = rest.split('/', 1)
  29         for x in [scheme, domain, rest]:
  30                 if not x or x.startswith(','):
  31                         raise SafeException(_("Invalid URL '%s'") % feed)
  32         return os.path.join('feeds', scheme, domain, _escape_slashes(rest))
  33
  34 class KeyInfoFetcher:
  35         """Fetches information about a GPG key from a key-info server.
  36         See L{Fetcher.fetch_key_info} for details.
  37         @since: 0.42
  38
  39         Example:
  40
  41         >>> kf = KeyInfoFetcher(handler, 'https://server', fingerprint)
  42         >>> while True:
  43                 print kf.info
  44                 if kf.blocker is None: break
  45                 print kf.status
  46                 yield kf.blocker
  47         """
  48         def __init__(self, handler, server, fingerprint):
  49                 self.fingerprint = fingerprint
  50                 self.info = []
  51                 self.blocker = None
  52
  53                 if server is None: return
  54
  55                 self.status = _('Fetching key information from %s...') % server
  56
  57                 dl = handler.get_download(server + '/key/' + fingerprint)
  58
  59                 from xml.dom import minidom
  60
  61                 @tasks.async
  62                 def fetch_key_info():
  63                         try:
  64                                 tempfile = dl.tempfile
  65                                 yield dl.downloaded
  66                                 self.blocker = None
  67                                 tasks.check(dl.downloaded)
  68                                 tempfile.seek(0)
  69                                 doc = minidom.parse(tempfile)
  70                                 if doc.documentElement.localName != 'key-lookup':
  71                                         raise SafeException(_('Expected <key-lookup>, not <%s>') % doc.documentElement.localName)
  72                                 self.info += doc.documentElement.childNodes
  73                         except Exception as ex:
  74                                 doc = minidom.parseString('<item vote="bad"/>')
  75                                 root = doc.documentElement
  76                                 root.appendChild(doc.createTextNode(_('Error getting key information: %s') % ex))
  77                                 self.info.append(root)
  78
  79                 self.blocker = fetch_key_info()
  80
  81 class Fetcher(object):
  82         """Downloads and stores various things.
  83         @ivar config: used to get handler, iface_cache and stores
  84         @type config: L{config.Config}
  85         @ivar key_info: caches information about GPG keys
  86         @type key_info: {str: L{KeyInfoFetcher}}
  87         """
  88         __slots__ = ['config', 'key_info']
  89
  90         def __init__(self, config):
  91                 assert config.handler, "API change!"
  92                 self.config = config
  93                 self.key_info = {}
  94
  95         @property
  96         def handler(self):
  97                 return self.config.handler
  98
  99         @tasks.async
 100         def cook(self, required_digest, recipe, stores, force = False, impl_hint = None):
 101                 """Follow a Recipe.
 102                 @param impl_hint: the Implementation this is for (if any) as a hint for the GUI
 103                 @see: L{download_impl} uses this method when appropriate"""
 104                 # Maybe we're taking this metaphor too far?
 105
 106                 # Start downloading all the ingredients.
 107                 streams = {}    # Streams collected from successful downloads
 108
 109                 # Start a download for each ingredient
 110                 blockers = []
 111                 for step in recipe.steps:
 112                         blocker, stream = self.download_archive(step, force = force, impl_hint = impl_hint)
 113                         assert stream
 114                         blockers.append(blocker)
 115                         streams[step] = stream
 116
 117                 while blockers:
 118                         yield blockers
 119                         tasks.check(blockers)
 120                         blockers = [b for b in blockers if not b.happened]
 121
 122                 from zeroinstall.zerostore import unpack
 123
 124                 # Create an empty directory for the new implementation
 125                 store = stores.stores[0]
 126                 tmpdir = store.get_tmp_dir_for(required_digest)
 127                 try:
 128                         # Unpack each of the downloaded archives into it in turn
 129                         for step in recipe.steps:
 130                                 stream = streams[step]
 131                                 stream.seek(0)
 132                                 unpack.unpack_archive_over(step.url, stream, tmpdir,
 133                                                 extract = step.extract,
 134                                                 type = step.type,
 135                                                 start_offset = step.start_offset or 0)
 136                         # Check that the result is correct and store it in the cache
 137                         store.check_manifest_and_rename(required_digest, tmpdir)
 138                         tmpdir = None
 139                 finally:
 140                         # If unpacking fails, remove the temporary directory
 141                         if tmpdir is not None:
 142                                 from zeroinstall import support
 143                                 support.ro_rmtree(tmpdir)
 144
 145         def get_feed_mirror(self, url):
 146                 """Return the URL of a mirror for this feed."""
 147                 if self.config.feed_mirror is None:
 148                         return None
 149                 import urlparse
 150                 if urlparse.urlparse(url).hostname == 'localhost':
 151                         return None
 152                 return '%s/%s/latest.xml' % (self.config.feed_mirror, _get_feed_dir(url))
 153
 154         @tasks.async
 155         def get_packagekit_feed(self, feed_url):
 156                 """Send a query to PackageKit (if available) for information about this package.
 157                 On success, the result is added to iface_cache.
 158                 """
 159                 assert feed_url.startswith('distribution:'), feed_url
 160                 master_feed = self.config.iface_cache.get_feed(feed_url.split(':', 1)[1])
 161                 if master_feed:
 162                         fetch = self.config.iface_cache.distro.fetch_candidates(master_feed)
 163                         if fetch:
 164                                 yield fetch
 165                                 tasks.check(fetch)
 166
 167                         # Force feed to be regenerated with the new information
 168                         self.config.iface_cache.get_feed(feed_url, force = True)
 169
 170         def download_and_import_feed(self, feed_url, iface_cache = None, force = False):
 171                 """Download the feed, download any required keys, confirm trust if needed and import.
 172                 @param feed_url: the feed to be downloaded
 173                 @type feed_url: str
 174                 @param iface_cache: (deprecated)
 175                 @param force: whether to abort and restart an existing download"""
 176                 from .download import DownloadAborted
 177
 178                 assert iface_cache is None or iface_cache is self.config.iface_cache
 179
 180                 self.config.iface_cache.mark_as_checking(feed_url)
 181
 182                 debug(_("download_and_import_feed %(url)s (force = %(force)d)"), {'url': feed_url, 'force': force})
 183                 assert not os.path.isabs(feed_url)
 184
 185                 if feed_url.startswith('distribution:'):
 186                         return self.get_packagekit_feed(feed_url)
 187
 188                 primary = self._download_and_import_feed(feed_url, force, use_mirror = False)
 189
 190                 @tasks.named_async("monitor feed downloads for " + feed_url)
 191                 def wait_for_downloads(primary):
 192                         # Download just the upstream feed, unless it takes too long...
 193                         timeout = tasks.TimeoutBlocker(5, 'Mirror timeout')             # 5 seconds
 194
 195                         yield primary, timeout
 196                         tasks.check(timeout)
 197
 198                         try:
 199                                 tasks.check(primary)
 200                                 if primary.happened:
 201                                         return          # OK, primary succeeded!
 202                                 # OK, maybe it's just being slow...
 203                                 info("Feed download from %s is taking a long time.", feed_url)
 204                                 primary_ex = None
 205                         except NoTrustedKeys as ex:
 206                                 raise                   # Don't bother trying the mirror if we have a trust problem
 207                         except ReplayAttack as ex:
 208                                 raise                   # Don't bother trying the mirror if we have a replay attack
 209                         except DownloadAborted as ex:
 210                                 raise                   # Don't bother trying the mirror if the user cancelled
 211                         except SafeException as ex:
 212                                 # Primary failed
 213                                 primary = None
 214                                 primary_ex = ex
 215                                 warn(_("Feed download from %(url)s failed: %(exception)s"), {'url': feed_url, 'exception': ex})
 216
 217                         # Start downloading from mirror...
 218                         mirror = self._download_and_import_feed(feed_url, force, use_mirror = True)
 219
 220                         # Wait until both mirror and primary tasks are complete...
 221                         while True:
 222                                 blockers = filter(None, [primary, mirror])
 223                                 if not blockers:
 224                                         break
 225                                 yield blockers
 226
 227                                 if primary:
 228                                         try:
 229                                                 tasks.check(primary)
 230                                                 if primary.happened:
 231                                                         primary = None
 232                                                         # No point carrying on with the mirror once the primary has succeeded
 233                                                         if mirror:
 234                                                                 info(_("Primary feed download succeeded; aborting mirror download for %s") % feed_url)
 235                                                                 mirror.dl.abort()
 236                                         except SafeException as ex:
 237                                                 primary = None
 238                                                 primary_ex = ex
 239                                                 info(_("Feed download from %(url)s failed; still trying mirror: %(exception)s"), {'url': feed_url, 'exception': ex})
 240
 241                                 if mirror:
 242                                         try:
 243                                                 tasks.check(mirror)
 244                                                 if mirror.happened:
 245                                                         mirror = None
 246                                                         if primary_ex:
 247                                                                 # We already warned; no need to raise an exception too,
 248                                                                 # as the mirror download succeeded.
 249                                                                 primary_ex = None
 250                                         except ReplayAttack as ex:
 251                                                 info(_("Version from mirror is older than cached version; ignoring it: %s"), ex)
 252                                                 mirror = None
 253                                                 primary_ex = None
 254                                         except SafeException as ex:
 255                                                 info(_("Mirror download failed: %s"), ex)
 256                                                 mirror = None
 257
 258                         if primary_ex:
 259                                 raise primary_ex
 260
 261                 return wait_for_downloads(primary)
 262
 263         def _download_and_import_feed(self, feed_url, force, use_mirror):
 264                 """Download and import a feed.
 265                 @param use_mirror: False to use primary location; True to use mirror."""
 266                 if use_mirror:
 267                         url = self.get_feed_mirror(feed_url)
 268                         if url is None: return None
 269                         info(_("Trying mirror server for feed %s") % feed_url)
 270                 else:
 271                         url = feed_url
 272
 273                 dl = self.handler.get_download(url, force = force, hint = feed_url)
 274                 stream = dl.tempfile
 275
 276                 @tasks.named_async("fetch_feed " + url)
 277                 def fetch_feed():
 278                         yield dl.downloaded
 279                         tasks.check(dl.downloaded)
 280
 281                         pending = PendingFeed(feed_url, stream)
 282
 283                         if use_mirror:
 284                                 # If we got the feed from a mirror, get the key from there too
 285                                 key_mirror = self.config.feed_mirror + '/keys/'
 286                         else:
 287                                 key_mirror = None
 288
 289                         keys_downloaded = tasks.Task(pending.download_keys(self.handler, feed_hint = feed_url, key_mirror = key_mirror), _("download keys for %s") % feed_url)
 290                         yield keys_downloaded.finished
 291                         tasks.check(keys_downloaded.finished)
 292
 293                         if not self.config.iface_cache.update_feed_if_trusted(pending.url, pending.sigs, pending.new_xml):
 294                                 blocker = self.config.trust_mgr.confirm_keys(pending)
 295                                 if blocker:
 296                                         yield blocker
 297                                         tasks.check(blocker)
 298                                 if not self.config.iface_cache.update_feed_if_trusted(pending.url, pending.sigs, pending.new_xml):
 299                                         raise NoTrustedKeys(_("No signing keys trusted; not importing"))
 300
 301                 task = fetch_feed()
 302                 task.dl = dl
 303                 return task
 304
 305         def fetch_key_info(self, fingerprint):
 306                 try:
 307                         return self.key_info[fingerprint]
 308                 except KeyError:
 309                         self.key_info[fingerprint] = key_info = KeyInfoFetcher(self.handler,
 310                                                                         self.config.key_info_server, fingerprint)
 311                         return key_info
 312
 313         def download_impl(self, impl, retrieval_method, stores, force = False):
 314                 """Download an implementation.
 315                 @param impl: the selected implementation
 316                 @type impl: L{model.ZeroInstallImplementation}
 317                 @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
 318                 @type retrieval_method: L{model.RetrievalMethod}
 319                 @param stores: where to store the downloaded implementation
 320                 @type stores: L{zerostore.Stores}
 321                 @param force: whether to abort and restart an existing download
 322                 @rtype: L{tasks.Blocker}"""
 323                 assert impl
 324                 assert retrieval_method
 325
 326                 if isinstance(retrieval_method, DistributionSource):
 327                         return retrieval_method.install(self.handler)
 328
 329                 from zeroinstall.zerostore import manifest
 330                 best = None
 331                 for digest in impl.digests:
 332                         alg_name = digest.split('=', 1)[0]
 333                         alg = manifest.algorithms.get(alg_name, None)
 334                         if alg and (best is None or best.rating < alg.rating):
 335                                 best = alg
 336                                 required_digest = digest
 337
 338                 if best is None:
 339                         if not impl.digests:
 340                                 raise SafeException(_("No <manifest-digest> given for '%(implementation)s' version %(version)s") %
 341                                                 {'implementation': impl.feed.get_name(), 'version': impl.get_version()})
 342                         raise SafeException(_("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s") %
 343                                         {'algorithms': impl.digests, 'implementation': impl.feed.get_name(), 'version': impl.get_version()})
 344
 345                 @tasks.async
 346                 def download_impl():
 347                         if isinstance(retrieval_method, DownloadSource):
 348                                 blocker, stream = self.download_archive(retrieval_method, force = force, impl_hint = impl)
 349                                 yield blocker
 350                                 tasks.check(blocker)
 351
 352                                 stream.seek(0)
 353                                 self._add_to_cache(required_digest, stores, retrieval_method, stream)
 354                         elif isinstance(retrieval_method, Recipe):
 355                                 blocker = self.cook(required_digest, retrieval_method, stores, force, impl_hint = impl)
 356                                 yield blocker
 357                                 tasks.check(blocker)
 358                         else:
 359                                 raise Exception(_("Unknown download type for '%s'") % retrieval_method)
 360
 361                         self.handler.impl_added_to_store(impl)
 362                 return download_impl()
 363
 364         def _add_to_cache(self, required_digest, stores, retrieval_method, stream):
 365                 assert isinstance(retrieval_method, DownloadSource)
 366                 stores.add_archive_to_cache(required_digest, stream, retrieval_method.url, retrieval_method.extract,
 367                                                  type = retrieval_method.type, start_offset = retrieval_method.start_offset or 0)
 368
 369         def download_archive(self, download_source, force = False, impl_hint = None):
 370                 """Fetch an archive. You should normally call L{download_impl}
 371                 instead, since it handles other kinds of retrieval method too."""
 372                 from zeroinstall.zerostore import unpack
 373
 374                 url = download_source.url
 375                 if not (url.startswith('http:') or url.startswith('https:') or url.startswith('ftp:')):
 376                         raise SafeException(_("Unknown scheme in download URL '%s'") % url)
 377
 378                 mime_type = download_source.type
 379                 if not mime_type:
 380                         mime_type = unpack.type_from_url(download_source.url)
 381                 if not mime_type:
 382                         raise SafeException(_("No 'type' attribute on archive, and I can't guess from the name (%s)") % download_source.url)
 383                 unpack.check_type_ok(mime_type)
 384                 dl = self.handler.get_download(download_source.url, force = force, hint = impl_hint)
 385                 dl.expected_size = download_source.size + (download_source.start_offset or 0)
 386                 return (dl.downloaded, dl.tempfile)
 387
 388         def download_icon(self, interface, force = False):
 389                 """Download an icon for this interface and add it to the
 390                 icon cache. If the interface has no icon do nothing.
 391                 @return: the task doing the import, or None
 392                 @rtype: L{tasks.Task}"""
 393                 debug("download_icon %(interface)s (force = %(force)d)", {'interface': interface, 'force': force})
 394
 395                 modification_time = None
 396                 existing_icon = self.config.iface_cache.get_icon_path(interface)
 397                 if existing_icon:
 398                         file_mtime = os.stat(existing_icon).st_mtime
 399                         from email.utils import formatdate
 400                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 401
 402                 # Find a suitable icon to download
 403                 for icon in interface.get_metadata(XMLNS_IFACE, 'icon'):
 404                         type = icon.getAttribute('type')
 405                         if type != 'image/png':
 406                                 debug(_('Skipping non-PNG icon'))
 407                                 continue
 408                         source = icon.getAttribute('href')
 409                         if source:
 410                                 break
 411                         warn(_('Missing "href" attribute on <icon> in %s'), interface)
 412                 else:
 413                         info(_('No PNG icons found in %s'), interface)
 414                         return
 415
 416                 try:
 417                         dl = self.handler.monitored_downloads[source]
 418                         if dl and force:
 419                                 dl.abort()
 420                                 raise KeyError
 421                 except KeyError:
 422                         dl = download.Download(source, hint = interface, modification_time = modification_time)
 423                         self.handler.monitor_download(dl)
 424
 425                 @tasks.async
 426                 def download_and_add_icon():
 427                         stream = dl.tempfile
 428                         yield dl.downloaded
 429                         try:
 430                                 tasks.check(dl.downloaded)
 431                                 if dl.unmodified: return
 432                                 stream.seek(0)
 433
 434                                 import shutil
 435                                 icons_cache = basedir.save_cache_path(config_site, 'interface_icons')
 436                                 icon_file = file(os.path.join(icons_cache, escape(interface.uri)), 'w')
 437                                 shutil.copyfileobj(stream, icon_file)
 438                         except Exception as ex:
 439                                 self.handler.report_error(ex)
 440
 441                 return download_and_add_icon()
 442
 443         def download_impls(self, implementations, stores):
 444                 """Download the given implementations, choosing a suitable retrieval method for each.
 445                 If any of the retrieval methods are DistributionSources and
 446                 need confirmation, handler.confirm is called to check that the
 447                 installation should proceed.
 448                 """
 449                 unsafe_impls = []
 450
 451                 to_download = []
 452                 for impl in implementations:
 453                         debug(_("start_downloading_impls: for %(feed)s get %(implementation)s"), {'feed': impl.feed, 'implementation': impl})
 454                         source = self.get_best_source(impl)
 455                         if not source:
 456                                 raise SafeException(_("Implementation %(implementation_id)s of interface %(interface)s"
 457                                         " cannot be downloaded (no download locations given in "
 458                                         "interface!)") % {'implementation_id': impl.id, 'interface': impl.feed.get_name()})
 459                         to_download.append((impl, source))
 460
 461                         if isinstance(source, DistributionSource) and source.needs_confirmation:
 462                                 unsafe_impls.append(source.package_id)
 463
 464                 @tasks.async
 465                 def download_impls():
 466                         if unsafe_impls:
 467                                 confirm = self.handler.confirm_install(_('The following components need to be installed using native packages. '
 468                                         'These come from your distribution, and should therefore be trustworthy, but they also '
 469                                         'run with extra privileges. In particular, installing them may run extra services on your '
 470                                         'computer or affect other users. You may be asked to enter a password to confirm. The '
 471                                         'packages are:\n\n') + ('\n'.join('- ' + x for x in unsafe_impls)))
 472                                 yield confirm
 473                                 tasks.check(confirm)
 474
 475                         blockers = []
 476
 477                         for impl, source in to_download:
 478                                 blockers.append(self.download_impl(impl, source, stores))
 479
 480                         # Record the first error log the rest
 481                         error = []
 482                         def dl_error(ex, tb = None):
 483                                 if error:
 484                                         self.handler.report_error(ex)
 485                                 else:
 486                                         error.append(ex)
 487                         while blockers:
 488                                 yield blockers
 489                                 tasks.check(blockers, dl_error)
 490
 491                                 blockers = [b for b in blockers if not b.happened]
 492                         if error:
 493                                 raise error[0]
 494
 495                 if not to_download:
 496                         return None
 497
 498                 return download_impls()
 499
 500         def get_best_source(self, impl):
 501                 """Return the best download source for this implementation.
 502                 @rtype: L{model.RetrievalMethod}"""
 503                 if impl.download_sources:
 504                         return impl.download_sources[0]
 505                 return None
 506