zeroinstall/injector/policy.py

   1 """
   2 Chooses a set of implementations based on a policy.
   3
   4 @deprecated: see L{solver}
   5 """
   6
   7 # Copyright (C) 2007, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 import time
  11 import sys, os, sets
  12 from logging import info, debug, warn
  13 import arch
  14
  15 from model import *
  16 from namespaces import *
  17 import ConfigParser
  18 from zeroinstall import NeedDownload
  19 from zeroinstall.support import tasks, basedir
  20 from zeroinstall.injector.iface_cache import iface_cache, PendingFeed
  21 from zeroinstall.injector.trust import trust_db
  22
  23 # If we started a check within this period, don't start another one:
  24 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  25
  26 def _cook(policy, required_digest, recipe, force = False):
  27         """A Cook follows a Recipe."""
  28         # Maybe we're taking this metaphor too far?
  29
  30         # Start downloading all the ingredients.
  31         downloads = {}  # Downloads that are not yet successful
  32         streams = {}    # Streams collected from successful downloads
  33
  34         # Start a download for each ingredient
  35         blockers = []
  36         for step in recipe.steps:
  37                 blocker, stream = policy.download_archive(step, force = force)
  38                 assert stream
  39                 blockers.append(blocker)
  40                 streams[step] = stream
  41
  42         while blockers:
  43                 yield blockers
  44                 tasks.check(blockers)
  45                 blockers = [b for b in blockers if not b.happened]
  46
  47         from zeroinstall.zerostore import unpack
  48
  49         # Create an empty directory for the new implementation
  50         store = iface_cache.stores.stores[0]
  51         tmpdir = store.get_tmp_dir_for(required_digest)
  52         try:
  53                 # Unpack each of the downloaded archives into it in turn
  54                 for step in recipe.steps:
  55                         stream = streams[step]
  56                         stream.seek(0)
  57                         unpack.unpack_archive_over(step.url, stream, tmpdir, step.extract)
  58                 # Check that the result is correct and store it in the cache
  59                 store.check_manifest_and_rename(required_digest, tmpdir)
  60                 tmpdir = None
  61         finally:
  62                 # If unpacking fails, remove the temporary directory
  63                 if tmpdir is not None:
  64                         from zeroinstall import support
  65                         support.ro_rmtree(tmpdir)
  66
  67 class Policy(object):
  68         """Chooses a set of implementations based on a policy.
  69         Typical use:
  70          1. Create a Policy object, giving it the URI of the program to be run and a handler.
  71          2. Call L{recalculate}. If more information is needed, the handler will be used to download it.
  72          3. When all downloads are complete, the L{implementation} map contains the chosen versions.
  73          4. Use L{get_uncached_implementations} to find where to get these versions and download them
  74             using L{begin_impl_download}.
  75
  76         @ivar root: URI of the root interface
  77         @ivar implementation: chosen implementations
  78         @type implementation: {model.Interface: model.Implementation or None}
  79         @ivar watchers: callbacks to invoke after recalculating
  80         @ivar help_with_testing: default stability policy
  81         @type help_with_testing: bool
  82         @ivar network_use: one of the model.network_* values
  83         @ivar freshness: seconds allowed since last update
  84         @type freshness: int
  85         @ivar ready: whether L{implementation} is complete enough to run the program
  86         @type ready: bool
  87         @ivar handler: handler for main-loop integration
  88         @type handler: L{handler.Handler}
  89         @ivar src: whether we are looking for source code
  90         @type src: bool
  91         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
  92         @type stale_feeds: set
  93         """
  94         __slots__ = ['root', 'watchers',
  95                      'freshness', 'handler', '_warned_offline',
  96                      'src', 'stale_feeds', 'solver']
  97
  98         help_with_testing = property(lambda self: self.solver.help_with_testing,
  99                                      lambda self, value: setattr(self.solver, 'help_with_testing', value))
 100
 101         network_use = property(lambda self: self.solver.network_use,
 102                                      lambda self, value: setattr(self.solver, 'network_use', value))
 103
 104         root_restrictions = property(lambda self: self.solver.root_restrictions,
 105                                      lambda self, value: setattr(self.solver, 'root_restrictions', value))
 106
 107         implementation = property(lambda self: self.solver.selections)
 108
 109         ready = property(lambda self: self.solver.ready)
 110
 111         def __init__(self, root, handler = None, src = False):
 112                 """
 113                 @param root: The URI of the root interface (the program we want to run).
 114                 @param handler: A handler for main-loop integration.
 115                 @type handler: L{zeroinstall.injector.handler.Handler}
 116                 @param src: Whether we are looking for source code.
 117                 @type src: bool
 118                 """
 119                 self.watchers = []
 120                 self.freshness = 60 * 60 * 24 * 30
 121                 self.src = src                          # Root impl must be a "src" machine type
 122                 self.stale_feeds = sets.Set()
 123
 124                 from zeroinstall.injector.solver import DefaultSolver
 125                 self.solver = DefaultSolver(network_full, iface_cache, iface_cache.stores, root_restrictions = [])
 126
 127                 # If we need to download something but can't because we are offline,
 128                 # warn the user. But only the first time.
 129                 self._warned_offline = False
 130
 131                 # (allow self for backwards compat)
 132                 self.handler = handler or self
 133
 134                 debug("Supported systems: '%s'", arch.os_ranks)
 135                 debug("Supported processors: '%s'", arch.machine_ranks)
 136
 137                 path = basedir.load_first_config(config_site, config_prog, 'global')
 138                 if path:
 139                         try:
 140                                 config = ConfigParser.ConfigParser()
 141                                 config.read(path)
 142                                 self.solver.help_with_testing = config.getboolean('global',
 143                                                                 'help_with_testing')
 144                                 self.solver.network_use = config.get('global', 'network_use')
 145                                 self.freshness = int(config.get('global', 'freshness'))
 146                                 assert self.solver.network_use in network_levels
 147                         except Exception, ex:
 148                                 warn("Error loading config: %s", ex)
 149
 150                 self.set_root(root)
 151
 152                 # Probably need weakrefs here...
 153                 iface_cache.add_watcher(self)
 154
 155         def set_root(self, root):
 156                 """Change the root interface URI."""
 157                 assert isinstance(root, (str, unicode))
 158                 self.root = root
 159                 for w in self.watchers: w()
 160
 161         def save_config(self):
 162                 """Write global settings."""
 163                 config = ConfigParser.ConfigParser()
 164                 config.add_section('global')
 165
 166                 config.set('global', 'help_with_testing', self.help_with_testing)
 167                 config.set('global', 'network_use', self.network_use)
 168                 config.set('global', 'freshness', self.freshness)
 169
 170                 path = basedir.save_config_path(config_site, config_prog)
 171                 path = os.path.join(path, 'global')
 172                 config.write(file(path + '.new', 'w'))
 173                 os.rename(path + '.new', path)
 174
 175         def recalculate(self, fetch_stale_interfaces = True):
 176                 """Try to choose a set of implementations.
 177                 This may start downloading more interfaces, but will return immediately.
 178                 @param fetch_stale_interfaces: whether to begin downloading interfaces which are present but haven't
 179                 been checked within the L{freshness} period
 180                 @type fetch_stale_interfaces: bool
 181                 @postcondition: L{ready} indicates whether a possible set of implementations was chosen
 182                 @note: A policy may be ready before all feeds have been downloaded. As new feeds
 183                 arrive, the chosen versions may change.
 184                 @return: a list of tasks which will require a recalculation when complete
 185                 """
 186
 187                 self.stale_feeds = sets.Set()
 188
 189                 host_arch = arch.get_host_architecture()
 190                 if self.src:
 191                         host_arch = arch.SourceArchitecture(host_arch)
 192                 self.solver.solve(self.root, host_arch)
 193
 194                 for f in self.solver.feeds_used:
 195                         self.get_interface(f)   # May start a download
 196
 197                 tasks = []
 198                 if fetch_stale_interfaces and self.network_use != network_offline:
 199                         for stale in self.stale_feeds:
 200                                 info("Checking for updates to stale feed %s", stale)
 201                                 tasks.append(self.download_and_import_feed(stale, False))
 202
 203                 for w in self.watchers: w()
 204
 205                 return tasks
 206
 207         def usable_feeds(self, iface):
 208                 """Generator for C{iface.feeds} that are valid for our architecture.
 209                 @rtype: generator
 210                 @see: L{arch}"""
 211                 if self.src and iface.uri == self.root:
 212                         # Note: when feeds are recursive, we'll need a better test for root here
 213                         machine_ranks = {'src': 1}
 214                 else:
 215                         machine_ranks = arch.machine_ranks
 216
 217                 for f in iface.feeds:
 218                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 219                                 yield f
 220                         else:
 221                                 debug("Skipping '%s'; unsupported architecture %s-%s",
 222                                         f, f.os, f.machine)
 223
 224         def get_interface(self, uri):
 225                 """Get an interface from the L{iface_cache}. If it is missing start a new download.
 226                 If it is present but stale, add it to L{stale_feeds}. This should only be called
 227                 from L{recalculate}.
 228                 @see: iface_cache.iface_cache.get_interface
 229                 @rtype: L{model.Interface}"""
 230                 iface = iface_cache.get_interface(uri)
 231
 232                 if uri in iface_cache.pending:
 233                         # Don't start another download while one is pending
 234                         # TODO: unless the pending version is very old
 235                         return iface
 236
 237                 if not uri.startswith('/'):
 238                         if iface.last_modified is None:
 239                                 if self.network_use != network_offline:
 240                                         debug("Feed not cached and not off-line. Downloading...")
 241                                         self.download_and_import_feed(iface.uri)
 242                                 else:
 243                                         if self._warned_offline:
 244                                                 debug("Nothing known about interface, but we are off-line.")
 245                                         else:
 246                                                 if iface.feeds:
 247                                                         info("Nothing known about interface '%s' and off-line. Trying feeds only.", uri)
 248                                                 else:
 249                                                         warn("Nothing known about interface '%s', but we are in off-line mode "
 250                                                                 "(so not fetching).", uri)
 251                                                         self._warned_offline = True
 252                         else:
 253                                 now = time.time()
 254                                 staleness = now - (iface.last_checked or 0)
 255                                 debug("Staleness for %s is %.2f hours", iface, staleness / 3600.0)
 256
 257                                 if self.freshness > 0 and staleness > self.freshness:
 258                                         last_check_attempt = iface_cache.get_last_check_attempt(iface.uri)
 259                                         if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 260                                                 debug("Stale, but tried to check recently (%s) so not rechecking now.", time.ctime(last_check_attempt))
 261                                         else:
 262                                                 debug("Adding %s to stale set", iface)
 263                                                 self.stale_feeds.add(iface)
 264                 #else: debug("Local interface, so not checking staleness.")
 265
 266                 return iface
 267
 268         def download_and_import_feed(self, feed_url, force = False):
 269                 """Download the feed, download any required keys, confirm trust if needed and import."""
 270
 271                 debug("download_and_import_feed %s (force = %d)", feed_url, force)
 272                 assert not feed_url.startswith('/')
 273
 274                 dl = self.handler.get_download(feed_url, force = force)
 275
 276                 def fetch_feed():
 277                         stream = dl.tempfile
 278
 279                         yield dl.downloaded
 280                         tasks.check(dl.downloaded)
 281
 282                         pending = PendingFeed(feed_url, stream)
 283                         iface_cache.add_pending(pending)
 284
 285                         keys_downloaded = tasks.Task(pending.download_keys(self.handler), "download keys for " + feed_url)
 286                         yield keys_downloaded.finished
 287                         tasks.check(keys_downloaded.finished)
 288
 289                         iface = iface_cache.get_interface(pending.url)
 290                         if not iface_cache.update_interface_if_trusted(iface, pending.sigs, pending.new_xml):
 291                                 blocker = self.handler.confirm_trust_keys(iface, pending.sigs, pending.new_xml)
 292                                 if blocker:
 293                                         yield blocker
 294                                         tasks.check(blocker)
 295                                 if not iface_cache.update_interface_if_trusted(iface, pending.sigs, pending.new_xml):
 296                                         raise SafeException("No signing keys trusted; not importing")
 297
 298                 return tasks.Task(fetch_feed(), "download_and_import_feed " + feed_url).finished
 299
 300         def download_impl(self, impl, retrieval_method, force = False):
 301                 """Download impl, using retrieval_method. See Task."""
 302                 assert impl
 303                 assert retrieval_method
 304
 305                 from zeroinstall.zerostore import manifest
 306                 alg = impl.id.split('=', 1)[0]
 307                 if alg not in manifest.algorithms:
 308                         raise SafeException("Unknown digest algorithm '%s' for '%s' version %s" %
 309                                         (alg, impl.feed.get_name(), impl.get_version()))
 310
 311                 if isinstance(retrieval_method, DownloadSource):
 312                         blocker, stream = self.download_archive(retrieval_method, force = force)
 313                         yield blocker
 314                         tasks.check(blocker)
 315
 316                         stream.seek(0)
 317                         iface_cache.add_to_cache(retrieval_method, stream)
 318                 elif isinstance(retrieval_method, Recipe):
 319                         blocker = tasks.Task(_cook(self, impl.id, retrieval_method, force), "cook").finished
 320                         yield blocker
 321                         tasks.check(blocker)
 322                 else:
 323                         raise Exception("Unknown download type for '%s'" % retrieval_method)
 324
 325         def download_archive(self, download_source, force = False):
 326                 """Fetch an archive. You should normally call L{begin_impl_download}
 327                 instead, since it handles other kinds of retrieval method too."""
 328                 from zeroinstall.zerostore import unpack
 329                 mime_type = download_source.type
 330                 if not mime_type:
 331                         mime_type = unpack.type_from_url(download_source.url)
 332                 if not mime_type:
 333                         raise SafeException("No 'type' attribute on archive, and I can't guess from the name (%s)" % download_source.url)
 334                 unpack.check_type_ok(mime_type)
 335                 dl = self.handler.get_download(download_source.url, force = force)
 336                 dl.expected_size = download_source.size + (download_source.start_offset or 0)
 337                 return (dl.downloaded, dl.tempfile)
 338
 339         def begin_icon_download(self, interface, force = False):
 340                 """Start downloading an icon for this interface. On success, add it to the
 341                 icon cache. If the interface has no icon, do nothing."""
 342                 debug("begin_icon_download %s (force = %d)", interface, force)
 343
 344                 # Find a suitable icon to download
 345                 for icon in interface.get_metadata(XMLNS_IFACE, 'icon'):
 346                         type = icon.getAttribute('type')
 347                         if type != 'image/png':
 348                                 debug('Skipping non-PNG icon')
 349                                 continue
 350                         source = icon.getAttribute('href')
 351                         if source:
 352                                 break
 353                         warn('Missing "href" attribute on <icon> in %s', interface)
 354                 else:
 355                         info('No PNG icons found in %s', interface)
 356                         return
 357
 358                 dl = self.handler.get_download(source, force = force)
 359                 if dl.on_success:
 360                         # Possibly we should handle this better, but it's unlikely anyone will need
 361                         # to use an icon as an interface or implementation as well, and some of the code
 362                         # may assume it's OK keep asking for the same icon to be downloaded.
 363                         info("Already have a handler for %s; not adding another", source)
 364                         return
 365                 dl.on_success.append(lambda stream: self.store_icon(interface, stream))
 366
 367         def store_icon(self, interface, stream):
 368                 """Called when an icon has been successfully downloaded.
 369                 Subclasses may wish to wrap this to repaint the display."""
 370                 from zeroinstall.injector import basedir
 371                 import shutil
 372                 icons_cache = basedir.save_cache_path(config_site, 'interface_icons')
 373                 icon_file = file(os.path.join(icons_cache, escape(interface.uri)), 'w')
 374                 shutil.copyfileobj(stream, icon_file)
 375
 376         def get_implementation_path(self, impl):
 377                 """Return the local path of impl.
 378                 @rtype: str
 379                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 380                 assert isinstance(impl, Implementation)
 381                 if impl.id.startswith('/'):
 382                         return impl.id
 383                 return iface_cache.stores.lookup(impl.id)
 384
 385         def get_implementation(self, interface):
 386                 """Get the chosen implementation.
 387                 @type interface: Interface
 388                 @rtype: L{model.Implementation}
 389                 @raise SafeException: if interface has not been fetched or no implementation could be
 390                 chosen."""
 391                 assert isinstance(interface, Interface)
 392
 393                 if not interface.name and not interface.feeds:
 394                         raise SafeException("We don't have enough information to "
 395                                             "run this program yet. "
 396                                             "Need to download:\n%s" % interface.uri)
 397                 try:
 398                         return self.implementation[interface]
 399                 except KeyError, ex:
 400                         if interface.implementations:
 401                                 offline = ""
 402                                 if self.network_use == network_offline:
 403                                         offline = "\nThis may be because 'Network Use' is set to Off-line."
 404                                 raise SafeException("No usable implementation found for '%s'.%s" %
 405                                                 (interface.name, offline))
 406                         raise ex
 407
 408         def get_cached(self, impl):
 409                 """Check whether an implementation is available locally.
 410                 @type impl: model.Implementation
 411                 @rtype: bool
 412                 """
 413                 if isinstance(impl, DistributionImplementation):
 414                         return impl.installed
 415                 if impl.id.startswith('/'):
 416                         return os.path.exists(impl.id)
 417                 else:
 418                         try:
 419                                 path = self.get_implementation_path(impl)
 420                                 assert path
 421                                 return True
 422                         except:
 423                                 pass # OK
 424                 return False
 425
 426         def add_to_cache(self, source, data):
 427                 """Wrapper for L{iface_cache.IfaceCache.add_to_cache}."""
 428                 iface_cache.add_to_cache(source, data)
 429
 430         def get_uncached_implementations(self):
 431                 """List all chosen implementations which aren't yet available locally.
 432                 @rtype: [(str, model.Implementation)]"""
 433                 uncached = []
 434                 for iface in self.solver.selections:
 435                         impl = self.solver.selections[iface]
 436                         assert impl, self.solver.selections
 437                         if not self.get_cached(impl):
 438                                 uncached.append((iface, impl))
 439                 return uncached
 440
 441         def refresh_all(self, force = True):
 442                 """Start downloading all feeds for all selected interfaces.
 443                 @param force: Whether to restart existing downloads."""
 444                 task = tasks.Task(self.solve_with_downloads(force = True), "refresh all")
 445                 self.handler.wait_for_blocker(task.finished)
 446
 447         def get_feed_targets(self, feed_iface_uri):
 448                 """Return a list of Interfaces for which feed_iface can be a feed.
 449                 This is used by B{0launch --feed}.
 450                 @rtype: [model.Interface]
 451                 @raise SafeException: If there are no known feeds."""
 452                 # TODO: what if it isn't cached yet?
 453                 feed_iface = iface_cache.get_interface(feed_iface_uri)
 454                 if not feed_iface.feed_for:
 455                         if not feed_iface.name:
 456                                 raise SafeException("Can't get feed targets for '%s'; failed to load interface." %
 457                                                 feed_iface_uri)
 458                         raise SafeException("Missing <feed-for> element in '%s'; "
 459                                         "this interface can't be used as a feed." % feed_iface_uri)
 460                 feed_targets = feed_iface.feed_for
 461                 debug("Feed targets: %s", feed_targets)
 462                 if not feed_iface.name:
 463                         warn("Warning: unknown interface '%s'" % feed_iface_uri)
 464                 return [iface_cache.get_interface(uri) for uri in feed_targets]
 465
 466         def get_icon_path(self, iface):
 467                 """Get an icon for this interface. If the icon is in the cache, use that.
 468                 If not, start a download. If we already started a download (successful or
 469                 not) do nothing.
 470                 @return: The cached icon's path, or None if no icon is currently available.
 471                 @rtype: str"""
 472                 path = iface_cache.get_icon_path(iface)
 473                 if path:
 474                         return path
 475
 476                 if self.network_use == network_offline:
 477                         info("No icon present for %s, but off-line so not downloading", iface)
 478                         return None
 479
 480                 self.begin_icon_download(iface)
 481                 return None
 482
 483         def get_best_source(self, impl):
 484                 """Return the best download source for this implementation.
 485                 @rtype: L{model.RetrievalMethod}"""
 486                 if impl.download_sources:
 487                         return impl.download_sources[0]
 488                 return None
 489
 490         def solve_with_downloads(self, force = False):
 491                 """Run the solver, then download any feeds that are missing or
 492                 that need to be updated. Each time a new feed is imported into
 493                 the cache, the solver is run again, possibly adding new downloads.
 494                 @param force: whether to download even if we're already ready to run
 495                 @return: a generator that can be used to create a L{support.tasks.Task}."""
 496
 497                 downloads_finished = set()              # Successful or otherwise
 498                 downloads_in_progress = {}              # URL -> Download
 499
 500                 host_arch = arch.get_host_architecture()
 501                 if self.src:
 502                         host_arch = arch.SourceArchitecture(host_arch)
 503
 504                 while True:
 505                         self.solver.solve(self.root, host_arch)
 506                         for w in self.watchers: w()
 507
 508                         if self.solver.ready and not force:
 509                                 break
 510                         else:
 511                                 # Once we've starting downloading some things,
 512                                 # we might as well get them all.
 513                                 force = True
 514
 515                         if not self.network_use == network_offline:
 516                                 for f in self.solver.feeds_used:
 517                                         if f in downloads_finished or f in downloads_in_progress:
 518                                                 continue
 519                                         if f.startswith('/'):
 520                                                 continue
 521                                         feed = iface_cache.get_interface(f)
 522                                         downloads_in_progress[f] = self.download_and_import_feed(f)
 523
 524                         if not downloads_in_progress:
 525                                 break
 526
 527                         blockers = downloads_in_progress.values()
 528                         yield blockers
 529                         tasks.check(blockers)
 530
 531                         for f in downloads_in_progress.keys():
 532                                 if downloads_in_progress[f].happened:
 533                                         del downloads_in_progress[f]
 534                                         downloads_finished.add(f)
 535
 536         def need_download(self):
 537                 """Decide whether we need to download anything (but don't do it!)
 538                 @return: true if we MUST download something (feeds or implementations)
 539                 @rtype: bool"""
 540                 host_arch = arch.get_host_architecture()
 541                 if self.src:
 542                         host_arch = arch.SourceArchitecture(host_arch)
 543                 self.solver.solve(self.root, host_arch)
 544                 for w in self.watchers: w()
 545
 546                 if not self.solver.ready:
 547                         return True             # Maybe a newer version will work?
 548
 549                 if self.get_uncached_implementations():
 550                         return True
 551
 552                 return False
 553
 554         def download_impls(self):
 555                 """Download all implementations that are missing from the cache."""
 556                 blockers = []
 557
 558                 for iface, impl in self.get_uncached_implementations():
 559                         debug("start_downloading_impls: for %s get %s", iface, impl)
 560                         source = self.get_best_source(impl)
 561                         if not source:
 562                                 raise model.SafeException("Implementation " + impl.id + " of "
 563                                         "interface " + iface.get_name() + " cannot be "
 564                                         "downloaded (no download locations given in "
 565                                         "interface!)")
 566                         blockers.append(tasks.Task(self.download_impl(impl, source), "fetch impl %s" % impl).finished)
 567
 568                 while blockers:
 569                         yield blockers
 570                         tasks.check(blockers)
 571
 572                         blockers = [b for b in blockers if not b.happened]