zeroinstall/injector/fetch.py

   1 """
   2 Downloads feeds, keys, packages and icons.
   3 """
   4
   5 # Copyright (C) 2009, Thomas Leonard
   6 # See the README file for details, or visit http://0install.net.
   7
   8 import os
   9 from logging import info, debug, warn
  10
  11 from zeroinstall.support import tasks, basedir
  12 from zeroinstall.injector.namespaces import XMLNS_IFACE, config_site
  13 from zeroinstall.injector.model import DownloadSource, Recipe, SafeException, escape
  14 from zeroinstall.injector.iface_cache import PendingFeed, ReplayAttack
  15 from zeroinstall.injector.handler import NoTrustedKeys
  16 from zeroinstall.injector import download
  17
  18 def _escape_slashes(path):
  19         return path.replace('/', '%23')
  20
  21 def _get_feed_dir(feed):
  22         """The algorithm from 0mirror."""
  23         if '#' in feed:
  24                 raise SafeException("Invalid URL '%s'" % feed)
  25         scheme, rest = feed.split('://', 1)
  26         domain, rest = rest.split('/', 1)
  27         for x in [scheme, domain, rest]:
  28                 if not x or x.startswith(','):
  29                         raise SafeException("Invalid URL '%s'" % feed)
  30         return os.path.join('feeds', scheme, domain, _escape_slashes(rest))
  31
  32 class Fetcher(object):
  33         """Downloads and stores various things.
  34         @ivar handler: handler to use for user-interaction
  35         @type handler: L{handler.Handler}
  36         @ivar feed_mirror: the base URL of a mirror site for keys and feeds
  37         @type feed_mirror: str
  38         """
  39         __slots__ = ['handler', 'feed_mirror']
  40
  41         def __init__(self, handler):
  42                 self.handler = handler
  43                 self.feed_mirror = "http://roscidus.com/0mirror"
  44
  45         @tasks.async
  46         def cook(self, required_digest, recipe, stores, force = False, impl_hint = None):
  47                 """Follow a Recipe.
  48                 @param impl_hint: the Implementation this is for (if any) as a hint for the GUI
  49                 @see: L{download_impl} uses this method when appropriate"""
  50                 # Maybe we're taking this metaphor too far?
  51
  52                 # Start downloading all the ingredients.
  53                 downloads = {}  # Downloads that are not yet successful
  54                 streams = {}    # Streams collected from successful downloads
  55
  56                 # Start a download for each ingredient
  57                 blockers = []
  58                 for step in recipe.steps:
  59                         blocker, stream = self.download_archive(step, force = force, impl_hint = impl_hint)
  60                         assert stream
  61                         blockers.append(blocker)
  62                         streams[step] = stream
  63
  64                 while blockers:
  65                         yield blockers
  66                         tasks.check(blockers)
  67                         blockers = [b for b in blockers if not b.happened]
  68
  69                 from zeroinstall.zerostore import unpack
  70
  71                 # Create an empty directory for the new implementation
  72                 store = stores.stores[0]
  73                 tmpdir = store.get_tmp_dir_for(required_digest)
  74                 try:
  75                         # Unpack each of the downloaded archives into it in turn
  76                         for step in recipe.steps:
  77                                 stream = streams[step]
  78                                 stream.seek(0)
  79                                 unpack.unpack_archive_over(step.url, stream, tmpdir, step.extract)
  80                         # Check that the result is correct and store it in the cache
  81                         store.check_manifest_and_rename(required_digest, tmpdir)
  82                         tmpdir = None
  83                 finally:
  84                         # If unpacking fails, remove the temporary directory
  85                         if tmpdir is not None:
  86                                 from zeroinstall import support
  87                                 support.ro_rmtree(tmpdir)
  88
  89         def get_feed_mirror(self, url):
  90                 """Return the URL of a mirror for this feed."""
  91                 import urlparse
  92                 if urlparse.urlparse(url).hostname == 'localhost':
  93                         return None
  94                 return '%s/%s/latest.xml' % (self.feed_mirror, _get_feed_dir(url))
  95
  96         def download_and_import_feed(self, feed_url, iface_cache, force = False):
  97                 """Download the feed, download any required keys, confirm trust if needed and import.
  98                 @param feed_url: the feed to be downloaded
  99                 @type feed_url: str
 100                 @param iface_cache: cache in which to store the feed
 101                 @type iface_cache: L{iface_cache.IfaceCache}
 102                 @param force: whether to abort and restart an existing download"""
 103                 from download import DownloadAborted
 104
 105                 debug("download_and_import_feed %s (force = %d)", feed_url, force)
 106                 assert not feed_url.startswith('/')
 107
 108                 primary = self._download_and_import_feed(feed_url, iface_cache, force, use_mirror = False)
 109
 110                 @tasks.named_async("monitor feed downloads for " + feed_url)
 111                 def wait_for_downloads(primary):
 112                         # Download just the upstream feed, unless it takes too long...
 113                         timeout = tasks.TimeoutBlocker(5, 'Mirror timeout')             # 5 seconds
 114
 115                         yield primary, timeout
 116                         tasks.check(timeout)
 117
 118                         try:
 119                                 tasks.check(primary)
 120                                 if primary.happened:
 121                                         return          # OK, primary succeeded!
 122                                 # OK, maybe it's just being slow...
 123                                 info("Feed download from %s is taking a long time. Trying mirror too...", feed_url)
 124                                 primary_ex = None
 125                         except NoTrustedKeys, ex:
 126                                 raise                   # Don't bother trying the mirror if we have a trust problem
 127                         except ReplayAttack, ex:
 128                                 raise                   # Don't bother trying the mirror if we have a replay attack
 129                         except DownloadAborted, ex:
 130                                 raise                   # Don't bother trying the mirror if the user cancelled
 131                         except SafeException, ex:
 132                                 # Primary failed
 133                                 primary = None
 134                                 primary_ex = ex
 135                                 warn("Trying mirror, as feed download from %s failed: %s", feed_url, ex)
 136
 137                         # Start downloading from mirror...
 138                         mirror = self._download_and_import_feed(feed_url, iface_cache, force, use_mirror = True)
 139
 140                         # Wait until both mirror and primary tasks are complete...
 141                         while True:
 142                                 blockers = filter(None, [primary, mirror])
 143                                 if not blockers:
 144                                         break
 145                                 yield blockers
 146
 147                                 if primary:
 148                                         try:
 149                                                 tasks.check(primary)
 150                                                 if primary.happened:
 151                                                         primary = None
 152                                                         # No point carrying on with the mirror once the primary has succeeded
 153                                                         if mirror:
 154                                                                 info("Primary feed download succeeded; aborting mirror download for " + feed_url)
 155                                                                 mirror.dl.abort()
 156                                         except SafeException, ex:
 157                                                 primary = None
 158                                                 primary_ex = ex
 159                                                 info("Feed download from %s failed; still trying mirror: %s", feed_url, ex)
 160
 161                                 if mirror:
 162                                         try:
 163                                                 tasks.check(mirror)
 164                                                 if mirror.happened:
 165                                                         mirror = None
 166                                                         if primary_ex:
 167                                                                 # We already warned; no need to raise an exception too,
 168                                                                 # as the mirror download succeeded.
 169                                                                 primary_ex = None
 170                                         except ReplayAttack, ex:
 171                                                 info("Version from mirror is older than cached version; ignoring it: %s", ex)
 172                                                 mirror = None
 173                                                 primary_ex = None
 174                                         except SafeException, ex:
 175                                                 info("Mirror download failed: %s", ex)
 176                                                 mirror = None
 177
 178                         if primary_ex:
 179                                 raise primary_ex
 180
 181                 return wait_for_downloads(primary)
 182
 183         def _download_and_import_feed(self, feed_url, iface_cache, force, use_mirror):
 184                 """Download and import a feed.
 185                 @param use_mirror: False to use primary location; True to use mirror."""
 186                 if use_mirror:
 187                         url = self.get_feed_mirror(feed_url)
 188                         if url is None: return None
 189                 else:
 190                         url = feed_url
 191
 192                 dl = self.handler.get_download(url, force = force, hint = feed_url)
 193                 stream = dl.tempfile
 194
 195                 @tasks.named_async("fetch_feed " + url)
 196                 def fetch_feed():
 197                         yield dl.downloaded
 198                         tasks.check(dl.downloaded)
 199
 200                         pending = PendingFeed(feed_url, stream)
 201
 202                         if use_mirror:
 203                                 # If we got the feed from a mirror, get the key from there too
 204                                 key_mirror = self.feed_mirror + '/keys/'
 205                         else:
 206                                 key_mirror = None
 207
 208                         keys_downloaded = tasks.Task(pending.download_keys(self.handler, feed_hint = feed_url, key_mirror = key_mirror), "download keys for " + feed_url)
 209                         yield keys_downloaded.finished
 210                         tasks.check(keys_downloaded.finished)
 211
 212                         iface = iface_cache.get_interface(pending.url)
 213                         if not iface_cache.update_interface_if_trusted(iface, pending.sigs, pending.new_xml):
 214                                 blocker = self.handler.confirm_trust_keys(iface, pending.sigs, pending.new_xml)
 215                                 if blocker:
 216                                         yield blocker
 217                                         tasks.check(blocker)
 218                                 if not iface_cache.update_interface_if_trusted(iface, pending.sigs, pending.new_xml):
 219                                         raise NoTrustedKeys("No signing keys trusted; not importing")
 220
 221                 task = fetch_feed()
 222                 task.dl = dl
 223                 return task
 224
 225         def download_impl(self, impl, retrieval_method, stores, force = False):
 226                 """Download an implementation.
 227                 @param impl: the selected implementation
 228                 @type impl: L{model.ZeroInstallImplementation}
 229                 @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
 230                 @type retrieval_method: L{model.RetrievalMethod}
 231                 @param stores: where to store the downloaded implementation
 232                 @type stores: L{zerostore.Stores}
 233                 @param force: whether to abort and restart an existing download
 234                 @rtype: L{tasks.Blocker}"""
 235                 assert impl
 236                 assert retrieval_method
 237
 238                 from zeroinstall.zerostore import manifest
 239                 alg = impl.id.split('=', 1)[0]
 240                 if alg not in manifest.algorithms:
 241                         raise SafeException("Unknown digest algorithm '%s' for '%s' version %s" %
 242                                         (alg, impl.feed.get_name(), impl.get_version()))
 243
 244                 @tasks.async
 245                 def download_impl():
 246                         if isinstance(retrieval_method, DownloadSource):
 247                                 blocker, stream = self.download_archive(retrieval_method, force = force, impl_hint = impl)
 248                                 yield blocker
 249                                 tasks.check(blocker)
 250
 251                                 stream.seek(0)
 252                                 self._add_to_cache(stores, retrieval_method, stream)
 253                         elif isinstance(retrieval_method, Recipe):
 254                                 blocker = self.cook(impl.id, retrieval_method, stores, force, impl_hint = impl)
 255                                 yield blocker
 256                                 tasks.check(blocker)
 257                         else:
 258                                 raise Exception("Unknown download type for '%s'" % retrieval_method)
 259
 260                         self.handler.impl_added_to_store(impl)
 261                 return download_impl()
 262
 263         def _add_to_cache(self, stores, retrieval_method, stream):
 264                 assert isinstance(retrieval_method, DownloadSource)
 265                 required_digest = retrieval_method.implementation.id
 266                 url = retrieval_method.url
 267                 stores.add_archive_to_cache(required_digest, stream, retrieval_method.url, retrieval_method.extract,
 268                                                  type = retrieval_method.type, start_offset = retrieval_method.start_offset or 0)
 269
 270         def download_archive(self, download_source, force = False, impl_hint = None):
 271                 """Fetch an archive. You should normally call L{download_impl}
 272                 instead, since it handles other kinds of retrieval method too."""
 273                 from zeroinstall.zerostore import unpack
 274
 275                 url = download_source.url
 276                 if not (url.startswith('http:') or url.startswith('https:') or url.startswith('ftp:')):
 277                         raise SafeException("Unknown scheme in download URL '%s'" % url)
 278
 279                 mime_type = download_source.type
 280                 if not mime_type:
 281                         mime_type = unpack.type_from_url(download_source.url)
 282                 if not mime_type:
 283                         raise SafeException("No 'type' attribute on archive, and I can't guess from the name (%s)" % download_source.url)
 284                 unpack.check_type_ok(mime_type)
 285                 dl = self.handler.get_download(download_source.url, force = force, hint = impl_hint)
 286                 dl.expected_size = download_source.size + (download_source.start_offset or 0)
 287                 return (dl.downloaded, dl.tempfile)
 288
 289         def download_icon(self, interface, force = False, modification_time = None):
 290                 """Download an icon for this interface and add it to the
 291                 icon cache. If the interface has no icon or we are offline, do nothing.
 292                 @return: the task doing the import, or None
 293                 @rtype: L{tasks.Task}"""
 294                 debug("download_icon %s (force = %d)", interface, force)
 295
 296                 # Find a suitable icon to download
 297                 for icon in interface.get_metadata(XMLNS_IFACE, 'icon'):
 298                         type = icon.getAttribute('type')
 299                         if type != 'image/png':
 300                                 debug('Skipping non-PNG icon')
 301                                 continue
 302                         source = icon.getAttribute('href')
 303                         if source:
 304                                 break
 305                         warn('Missing "href" attribute on <icon> in %s', interface)
 306                 else:
 307                         info('No PNG icons found in %s', interface)
 308                         return
 309
 310                 try:
 311                         dl = self.handler.monitored_downloads[source]
 312                         if dl and force:
 313                                 dl.abort()
 314                                 raise KeyError
 315                 except KeyError:
 316                         dl = download.Download(source, hint = interface, modification_time = modification_time)
 317                         self.handler.monitor_download(dl)
 318
 319                 @tasks.async
 320                 def download_and_add_icon():
 321                         stream = dl.tempfile
 322                         yield dl.downloaded
 323                         try:
 324                                 tasks.check(dl.downloaded)
 325                                 if dl.unmodified: return
 326                                 stream.seek(0)
 327
 328                                 import shutil
 329                                 icons_cache = basedir.save_cache_path(config_site, 'interface_icons')
 330                                 icon_file = file(os.path.join(icons_cache, escape(interface.uri)), 'w')
 331                                 shutil.copyfileobj(stream, icon_file)
 332                         except Exception, ex:
 333                                 self.handler.report_error(ex)
 334
 335                 return download_and_add_icon()
 336
 337         def download_impls(self, implementations, stores):
 338                 """Download the given implementations, choosing a suitable retrieval method for each."""
 339                 blockers = []
 340
 341                 to_download = []
 342                 for impl in implementations:
 343                         debug("start_downloading_impls: for %s get %s", impl.feed, impl)
 344                         source = self.get_best_source(impl)
 345                         if not source:
 346                                 raise SafeException("Implementation " + impl.id + " of "
 347                                         "interface " + impl.feed.get_name() + " cannot be "
 348                                         "downloaded (no download locations given in "
 349                                         "interface!)")
 350                         to_download.append((impl, source))
 351
 352                 for impl, source in to_download:
 353                         blockers.append(self.download_impl(impl, source, stores))
 354
 355                 if not blockers:
 356                         return None
 357
 358                 @tasks.async
 359                 def download_impls(blockers):
 360                         # Record the first error log the rest
 361                         error = []
 362                         def dl_error(ex, tb = None):
 363                                 if error:
 364                                         self.handler.report_error(ex)
 365                                 else:
 366                                         error.append(ex)
 367                         while blockers:
 368                                 yield blockers
 369                                 tasks.check(blockers, dl_error)
 370
 371                                 blockers = [b for b in blockers if not b.happened]
 372                         if error:
 373                                 raise error[0]
 374
 375                 return download_impls(blockers)
 376
 377         def get_best_source(self, impl):
 378                 """Return the best download source for this implementation.
 379                 @rtype: L{model.RetrievalMethod}"""
 380                 if impl.download_sources:
 381                         return impl.download_sources[0]
 382                 return None
 383