zeroinstall/injector/fetch.py

   1 """
   2 Downloads feeds, keys, packages and icons.
   3 """
   4
   5 # Copyright (C) 2008, Thomas Leonard
   6 # See the README file for details, or visit http://0install.net.
   7
   8 import os
   9 from logging import info, debug, warn
  10
  11 from zeroinstall.support import tasks, basedir
  12 from zeroinstall.injector.namespaces import XMLNS_IFACE, config_site
  13 from zeroinstall.injector.model import DownloadSource, Recipe, SafeException, network_offline, escape
  14 from zeroinstall.injector.iface_cache import PendingFeed
  15
  16 class Fetcher(object):
  17         """Downloads and stores various things.
  18         @ivar handler: handler to use for user-interaction
  19         @type handler: L{handler.Handler}"""
  20         __slots__ = ['handler']
  21
  22         def __init__(self, handler):
  23                 self.handler = handler
  24
  25         @tasks.async
  26         def cook(self, required_digest, recipe, stores, force = False, impl_hint = None):
  27                 """Follow a Recipe.
  28                 @param impl_hint: the Implementation this is for (if any) as a hint for the GUI
  29                 @see: L{download_impl} uses this method when appropriate"""
  30                 # Maybe we're taking this metaphor too far?
  31
  32                 # Start downloading all the ingredients.
  33                 downloads = {}  # Downloads that are not yet successful
  34                 streams = {}    # Streams collected from successful downloads
  35
  36                 # Start a download for each ingredient
  37                 blockers = []
  38                 for step in recipe.steps:
  39                         blocker, stream = self.download_archive(step, force = force, impl_hint = impl_hint)
  40                         assert stream
  41                         blockers.append(blocker)
  42                         streams[step] = stream
  43
  44                 while blockers:
  45                         yield blockers
  46                         tasks.check(blockers)
  47                         blockers = [b for b in blockers if not b.happened]
  48
  49                 from zeroinstall.zerostore import unpack
  50
  51                 # Create an empty directory for the new implementation
  52                 store = stores.stores[0]
  53                 tmpdir = store.get_tmp_dir_for(required_digest)
  54                 try:
  55                         # Unpack each of the downloaded archives into it in turn
  56                         for step in recipe.steps:
  57                                 stream = streams[step]
  58                                 stream.seek(0)
  59                                 unpack.unpack_archive_over(step.url, stream, tmpdir, step.extract)
  60                         # Check that the result is correct and store it in the cache
  61                         store.check_manifest_and_rename(required_digest, tmpdir)
  62                         tmpdir = None
  63                 finally:
  64                         # If unpacking fails, remove the temporary directory
  65                         if tmpdir is not None:
  66                                 from zeroinstall import support
  67                                 support.ro_rmtree(tmpdir)
  68
  69         def download_and_import_feed(self, feed_url, iface_cache, force = False):
  70                 """Download the feed, download any required keys, confirm trust if needed and import.
  71                 @param feed_url: the feed to be downloaded
  72                 @type feed_url: str
  73                 @param iface_cache: cache in which to store the feed
  74                 @type iface_cache: L{iface_cache.IfaceCache}
  75                 @param force: whether to abort and restart an existing download"""
  76
  77                 debug("download_and_import_feed %s (force = %d)", feed_url, force)
  78                 assert not feed_url.startswith('/')
  79
  80                 dl = self.handler.get_download(feed_url, force = force, hint = feed_url)
  81
  82                 @tasks.named_async("fetch_feed " + feed_url)
  83                 def fetch_feed():
  84                         stream = dl.tempfile
  85
  86                         yield dl.downloaded
  87                         tasks.check(dl.downloaded)
  88
  89                         pending = PendingFeed(feed_url, stream)
  90                         iface_cache.add_pending(pending)
  91
  92                         keys_downloaded = tasks.Task(pending.download_keys(self.handler, feed_hint = feed_url), "download keys for " + feed_url)
  93                         yield keys_downloaded.finished
  94                         tasks.check(keys_downloaded.finished)
  95
  96                         iface = iface_cache.get_interface(pending.url)
  97                         if not iface_cache.update_interface_if_trusted(iface, pending.sigs, pending.new_xml):
  98                                 blocker = self.handler.confirm_trust_keys(iface, pending.sigs, pending.new_xml)
  99                                 if blocker:
 100                                         yield blocker
 101                                         tasks.check(blocker)
 102                                 if not iface_cache.update_interface_if_trusted(iface, pending.sigs, pending.new_xml):
 103                                         raise SafeException("No signing keys trusted; not importing")
 104
 105                 return fetch_feed()
 106
 107         def download_impl(self, impl, retrieval_method, stores, force = False):
 108                 """Download an implementation.
 109                 @param impl: the selected implementation
 110                 @type impl: L{model.ZeroInstallImplementation}
 111                 @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
 112                 @type retrieval_method: L{model.RetrievalMethod}
 113                 @param stores: where to store the downloaded implementation
 114                 @type stores: L{zerostore.Stores}
 115                 @param force: whether to abort and restart an existing download
 116                 @rtype: L{tasks.Blocker}"""
 117                 assert impl
 118                 assert retrieval_method
 119
 120                 from zeroinstall.zerostore import manifest
 121                 alg = impl.id.split('=', 1)[0]
 122                 if alg not in manifest.algorithms:
 123                         raise SafeException("Unknown digest algorithm '%s' for '%s' version %s" %
 124                                         (alg, impl.feed.get_name(), impl.get_version()))
 125
 126                 @tasks.async
 127                 def download_impl():
 128                         if isinstance(retrieval_method, DownloadSource):
 129                                 blocker, stream = self.download_archive(retrieval_method, force = force, impl_hint = impl)
 130                                 yield blocker
 131                                 tasks.check(blocker)
 132
 133                                 stream.seek(0)
 134                                 self._add_to_cache(stores, retrieval_method, stream)
 135                         elif isinstance(retrieval_method, Recipe):
 136                                 blocker = self.cook(impl.id, retrieval_method, stores, force, impl_hint = impl)
 137                                 yield blocker
 138                                 tasks.check(blocker)
 139                         else:
 140                                 raise Exception("Unknown download type for '%s'" % retrieval_method)
 141
 142                         self.handler.impl_added_to_store(impl)
 143                 return download_impl()
 144
 145         def _add_to_cache(self, stores, retrieval_method, stream):
 146                 assert isinstance(retrieval_method, DownloadSource)
 147                 required_digest = retrieval_method.implementation.id
 148                 url = retrieval_method.url
 149                 stores.add_archive_to_cache(required_digest, stream, retrieval_method.url, retrieval_method.extract,
 150                                                  type = retrieval_method.type, start_offset = retrieval_method.start_offset or 0)
 151
 152         def download_archive(self, download_source, force = False, impl_hint = None):
 153                 """Fetch an archive. You should normally call L{download_impl}
 154                 instead, since it handles other kinds of retrieval method too."""
 155                 from zeroinstall.zerostore import unpack
 156                 mime_type = download_source.type
 157                 if not mime_type:
 158                         mime_type = unpack.type_from_url(download_source.url)
 159                 if not mime_type:
 160                         raise SafeException("No 'type' attribute on archive, and I can't guess from the name (%s)" % download_source.url)
 161                 unpack.check_type_ok(mime_type)
 162                 dl = self.handler.get_download(download_source.url, force = force, hint = impl_hint)
 163                 dl.expected_size = download_source.size + (download_source.start_offset or 0)
 164                 return (dl.downloaded, dl.tempfile)
 165
 166         def download_icon(self, interface, force = False):
 167                 """Download an icon for this interface and add it to the
 168                 icon cache. If the interface has no icon or we are offline, do nothing.
 169                 @return: the task doing the import, or None
 170                 @rtype: L{tasks.Task}"""
 171                 debug("download_icon %s (force = %d)", interface, force)
 172
 173                 # Find a suitable icon to download
 174                 for icon in interface.get_metadata(XMLNS_IFACE, 'icon'):
 175                         type = icon.getAttribute('type')
 176                         if type != 'image/png':
 177                                 debug('Skipping non-PNG icon')
 178                                 continue
 179                         source = icon.getAttribute('href')
 180                         if source:
 181                                 break
 182                         warn('Missing "href" attribute on <icon> in %s', interface)
 183                 else:
 184                         info('No PNG icons found in %s', interface)
 185                         return
 186
 187                 dl = self.handler.get_download(source, force = force, hint = interface)
 188
 189                 @tasks.async
 190                 def download_and_add_icon():
 191                         stream = dl.tempfile
 192                         yield dl.downloaded
 193                         try:
 194                                 tasks.check(dl.downloaded)
 195                                 stream.seek(0)
 196
 197                                 import shutil
 198                                 icons_cache = basedir.save_cache_path(config_site, 'interface_icons')
 199                                 icon_file = file(os.path.join(icons_cache, escape(interface.uri)), 'w')
 200                                 shutil.copyfileobj(stream, icon_file)
 201                         except Exception, ex:
 202                                 self.handler.report_error(ex)
 203
 204                 return download_and_add_icon()
 205
 206         def download_impls(self, implementations, stores):
 207                 """Download the given implementations, choosing a suitable retrieval method for each."""
 208                 blockers = []
 209
 210                 to_download = []
 211                 for impl in implementations:
 212                         debug("start_downloading_impls: for %s get %s", impl.feed, impl)
 213                         source = self.get_best_source(impl)
 214                         if not source:
 215                                 raise SafeException("Implementation " + impl.id + " of "
 216                                         "interface " + impl.feed.get_name() + " cannot be "
 217                                         "downloaded (no download locations given in "
 218                                         "interface!)")
 219                         to_download.append((impl, source))
 220
 221                 for impl, source in to_download:
 222                         blockers.append(self.download_impl(impl, source, stores))
 223
 224                 if not blockers:
 225                         return None
 226
 227                 @tasks.async
 228                 def download_impls(blockers):
 229                         # Record the first error log the rest
 230                         error = []
 231                         def dl_error(ex, tb = None):
 232                                 if error:
 233                                         self.handler.report_error(ex)
 234                                 else:
 235                                         error.append(ex)
 236                         while blockers:
 237                                 yield blockers
 238                                 tasks.check(blockers, dl_error)
 239
 240                                 blockers = [b for b in blockers if not b.happened]
 241                         if error:
 242                                 raise error[0]
 243
 244                 return download_impls(blockers)
 245
 246         def get_best_source(self, impl):
 247                 """Return the best download source for this implementation.
 248                 @rtype: L{model.RetrievalMethod}"""
 249                 if impl.download_sources:
 250                         return impl.download_sources[0]
 251                 return None
 252