zeroinstall/injector/policy.py

   1 """
   2 This class brings together a L{solve.Solver} to choose a set of implmentations, a
   3 L{fetch.Fetcher} to download additional components, and the user's configuration
   4 settings.
   5 """
   6
   7 # Copyright (C) 2009, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 from zeroinstall import _
  11 import time
  12 import os
  13 from logging import info, debug, warn
  14 import ConfigParser
  15
  16 from zeroinstall import SafeException
  17 from zeroinstall.injector import arch
  18 from zeroinstall.injector.model import Interface, Implementation, network_levels, network_offline, DistributionImplementation, network_full
  19 from zeroinstall.injector.namespaces import config_site, config_prog
  20 from zeroinstall.support import tasks, basedir
  21 from zeroinstall.injector.iface_cache import iface_cache
  22
  23 # If we started a check within this period, don't start another one:
  24 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  25
  26 class Policy(object):
  27         """Chooses a set of implementations based on a policy.
  28         Typical use:
  29          1. Create a Policy object, giving it the URI of the program to be run and a handler.
  30          2. Call L{solve_with_downloads}. If more information is needed, a L{fetch.Fetcher} will be used to download it.
  31          3. When all downloads are complete, the L{solver} contains the chosen versions.
  32          4. Use L{get_uncached_implementations} to find where to get these versions and download them
  33             using L{download_uncached_implementations}.
  34
  35         @ivar target_arch: target architecture for binaries
  36         @type target_arch: L{arch.Architecture}
  37         @ivar root: URI of the root interface
  38         @ivar solver: solver used to choose a set of implementations
  39         @type solver: L{solve.Solver}
  40         @ivar watchers: callbacks to invoke after recalculating
  41         @ivar help_with_testing: default stability policy
  42         @type help_with_testing: bool
  43         @ivar network_use: one of the model.network_* values
  44         @ivar freshness: seconds allowed since last update
  45         @type freshness: int
  46         @ivar handler: handler for main-loop integration
  47         @type handler: L{handler.Handler}
  48         @ivar src: whether we are looking for source code
  49         @type src: bool
  50         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
  51         @type stale_feeds: set
  52         """
  53         __slots__ = ['root', 'watchers',
  54                      'freshness', 'handler', '_warned_offline',
  55                      'target_arch', 'src', 'stale_feeds', 'solver', '_fetcher']
  56
  57         help_with_testing = property(lambda self: self.solver.help_with_testing,
  58                                      lambda self, value: setattr(self.solver, 'help_with_testing', value))
  59
  60         network_use = property(lambda self: self.solver.network_use,
  61                                      lambda self, value: setattr(self.solver, 'network_use', value))
  62
  63         implementation = property(lambda self: self.solver.selections)
  64
  65         ready = property(lambda self: self.solver.ready)
  66
  67         def __init__(self, root, handler = None, src = False):
  68                 """
  69                 @param root: The URI of the root interface (the program we want to run).
  70                 @param handler: A handler for main-loop integration.
  71                 @type handler: L{zeroinstall.injector.handler.Handler}
  72                 @param src: Whether we are looking for source code.
  73                 @type src: bool
  74                 """
  75                 self.watchers = []
  76                 self.freshness = 60 * 60 * 24 * 30
  77                 self.src = src                          # Root impl must be a "src" machine type
  78                 self.stale_feeds = set()
  79
  80                 from zeroinstall.injector.solver import DefaultSolver
  81                 self.solver = DefaultSolver(network_full, iface_cache, iface_cache.stores)
  82
  83                 # If we need to download something but can't because we are offline,
  84                 # warn the user. But only the first time.
  85                 self._warned_offline = False
  86                 self._fetcher = None
  87
  88                 # (allow self for backwards compat)
  89                 self.handler = handler or self
  90
  91                 debug(_("Supported systems: '%s'"), arch.os_ranks)
  92                 debug(_("Supported processors: '%s'"), arch.machine_ranks)
  93
  94                 path = basedir.load_first_config(config_site, config_prog, 'global')
  95                 if path:
  96                         try:
  97                                 config = ConfigParser.ConfigParser()
  98                                 config.read(path)
  99                                 self.solver.help_with_testing = config.getboolean('global',
 100                                                                 'help_with_testing')
 101                                 self.solver.network_use = config.get('global', 'network_use')
 102                                 self.freshness = int(config.get('global', 'freshness'))
 103                                 assert self.solver.network_use in network_levels, self.solver.network_use
 104                         except Exception, ex:
 105                                 warn(_("Error loading config: %s"), str(ex) or repr(ex))
 106
 107                 self.set_root(root)
 108
 109                 self.target_arch = arch.get_host_architecture()
 110
 111         @property
 112         def fetcher(self):
 113                 if not self._fetcher:
 114                         import fetch
 115                         self._fetcher = fetch.Fetcher(self.handler)
 116                 return self._fetcher
 117
 118         def set_root(self, root):
 119                 """Change the root interface URI."""
 120                 assert isinstance(root, (str, unicode))
 121                 self.root = root
 122                 for w in self.watchers: w()
 123
 124         def save_config(self):
 125                 """Write global settings."""
 126                 config = ConfigParser.ConfigParser()
 127                 config.add_section('global')
 128
 129                 config.set('global', 'help_with_testing', self.help_with_testing)
 130                 config.set('global', 'network_use', self.network_use)
 131                 config.set('global', 'freshness', self.freshness)
 132
 133                 path = basedir.save_config_path(config_site, config_prog)
 134                 path = os.path.join(path, 'global')
 135                 config.write(file(path + '.new', 'w'))
 136                 os.rename(path + '.new', path)
 137
 138         def recalculate(self, fetch_stale_interfaces = True):
 139                 """@deprecated: see L{solve_with_downloads} """
 140                 self.stale_feeds = set()
 141
 142                 host_arch = self.target_arch
 143                 if self.src:
 144                         host_arch = arch.SourceArchitecture(host_arch)
 145                 self.solver.solve(self.root, host_arch)
 146
 147                 if self.network_use == network_offline:
 148                         fetch_stale_interfaces = False
 149
 150                 blockers = []
 151                 for f in self.solver.feeds_used:
 152                         if f.startswith('/'): continue
 153                         feed = iface_cache.get_feed(f)
 154                         if feed is None or feed.last_modified is None:
 155                                 self.download_and_import_feed_if_online(f)      # Will start a download
 156                         elif self.is_stale(feed):
 157                                 debug(_("Adding %s to stale set"), f)
 158                                 self.stale_feeds.add(iface_cache.get_interface(f))      # Legacy API
 159                                 if fetch_stale_interfaces:
 160                                         self.download_and_import_feed_if_online(f)      # Will start a download
 161
 162                 for w in self.watchers: w()
 163
 164                 return blockers
 165
 166         def usable_feeds(self, iface):
 167                 """Generator for C{iface.feeds} that are valid for our architecture.
 168                 @rtype: generator
 169                 @see: L{arch}"""
 170                 if self.src and iface.uri == self.root:
 171                         # Note: when feeds are recursive, we'll need a better test for root here
 172                         machine_ranks = {'src': 1}
 173                 else:
 174                         machine_ranks = arch.machine_ranks
 175
 176                 for f in iface.feeds:
 177                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 178                                 yield f
 179                         else:
 180                                 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
 181                                         {'feed': f, 'os': f.os, 'machine': f.machine})
 182
 183         def is_stale(self, feed):
 184                 """Check whether feed needs updating, based on the configured L{freshness}.
 185                 None is considered to be stale.
 186                 @return: true if feed is stale or missing."""
 187                 if feed is None:
 188                         return True
 189                 if feed.url.startswith('/'):
 190                         return False            # Local feeds are never stale
 191                 if feed.last_modified is None:
 192                         return True             # Don't even have it yet
 193                 now = time.time()
 194                 staleness = now - (feed.last_checked or 0)
 195                 debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0})
 196
 197                 if self.freshness == 0 or staleness < self.freshness:
 198                         return False            # Fresh enough for us
 199
 200                 last_check_attempt = iface_cache.get_last_check_attempt(feed.url)
 201                 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 202                         debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt))
 203                         return False
 204
 205                 return True
 206
 207         def download_and_import_feed_if_online(self, feed_url):
 208                 """If we're online, call L{fetch.Fetcher.download_and_import_feed}. Otherwise, log a suitable warning."""
 209                 if self.network_use != network_offline:
 210                         debug(_("Feed %s not cached and not off-line. Downloading..."), feed_url)
 211                         return self.fetcher.download_and_import_feed(feed_url, iface_cache)
 212                 else:
 213                         if self._warned_offline:
 214                                 debug(_("Not downloading feed '%s' because we are off-line."), feed_url)
 215                         else:
 216                                 warn(_("Not downloading feed '%s' because we are in off-line mode."), feed_url)
 217                                 self._warned_offline = True
 218
 219         def get_implementation_path(self, impl):
 220                 """Return the local path of impl.
 221                 @rtype: str
 222                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 223                 assert isinstance(impl, Implementation)
 224                 if impl.id.startswith('/'):
 225                         return impl.id
 226                 return iface_cache.stores.lookup(impl.id)
 227
 228         def get_implementation(self, interface):
 229                 """Get the chosen implementation.
 230                 @type interface: Interface
 231                 @rtype: L{model.Implementation}
 232                 @raise SafeException: if interface has not been fetched or no implementation could be
 233                 chosen."""
 234                 assert isinstance(interface, Interface)
 235
 236                 if not interface.name and not interface.feeds:
 237                         raise SafeException(_("We don't have enough information to "
 238                                             "run this program yet. "
 239                                             "Need to download:\n%s") % interface.uri)
 240                 try:
 241                         return self.implementation[interface]
 242                 except KeyError, ex:
 243                         if interface.implementations:
 244                                 offline = ""
 245                                 if self.network_use == network_offline:
 246                                         raise SafeException(_("No usable implementation found for '%s'.\n"
 247                                                         "This may be because 'Network Use' is set to Off-line.") %
 248                                                         interface.name)
 249                                 raise SafeException(_("No usable implementation found for '%s'.") %
 250                                                 interface.name)
 251                         raise ex
 252
 253         def get_cached(self, impl):
 254                 """Check whether an implementation is available locally.
 255                 @type impl: model.Implementation
 256                 @rtype: bool
 257                 """
 258                 if isinstance(impl, DistributionImplementation):
 259                         return impl.installed
 260                 if impl.id.startswith('/'):
 261                         return os.path.exists(impl.id)
 262                 else:
 263                         try:
 264                                 path = self.get_implementation_path(impl)
 265                                 assert path
 266                                 return True
 267                         except:
 268                                 pass # OK
 269                 return False
 270
 271         def get_uncached_implementations(self):
 272                 """List all chosen implementations which aren't yet available locally.
 273                 @rtype: [(L{model.Interface}, L{model.Implementation})]"""
 274                 uncached = []
 275                 for iface in self.solver.selections:
 276                         impl = self.solver.selections[iface]
 277                         assert impl, self.solver.selections
 278                         if not self.get_cached(impl):
 279                                 uncached.append((iface, impl))
 280                 return uncached
 281
 282         def refresh_all(self, force = True):
 283                 """Start downloading all feeds for all selected interfaces.
 284                 @param force: Whether to restart existing downloads."""
 285                 return self.solve_with_downloads(force = True)
 286
 287         def get_feed_targets(self, feed_iface_uri):
 288                 """Return a list of Interfaces for which feed_iface can be a feed.
 289                 This is used by B{0launch --feed}.
 290                 @rtype: [model.Interface]
 291                 @raise SafeException: If there are no known feeds."""
 292                 # TODO: what if it isn't cached yet?
 293                 feed_iface = iface_cache.get_interface(feed_iface_uri)
 294                 if not feed_iface.feed_for:
 295                         if not feed_iface.name:
 296                                 raise SafeException(_("Can't get feed targets for '%s'; failed to load interface.") %
 297                                                 feed_iface_uri)
 298                         raise SafeException(_("Missing <feed-for> element in '%s'; "
 299                                         "this interface can't be used as a feed.") % feed_iface_uri)
 300                 feed_targets = feed_iface.feed_for
 301                 debug(_("Feed targets: %s"), feed_targets)
 302                 if not feed_iface.name:
 303                         warn(_("Warning: unknown interface '%s'") % feed_iface_uri)
 304                 return [iface_cache.get_interface(uri) for uri in feed_targets]
 305
 306         @tasks.async
 307         def solve_with_downloads(self, force = False):
 308                 """Run the solver, then download any feeds that are missing or
 309                 that need to be updated. Each time a new feed is imported into
 310                 the cache, the solver is run again, possibly adding new downloads.
 311                 @param force: whether to download even if we're already ready to run."""
 312
 313                 downloads_finished = set()              # Successful or otherwise
 314                 downloads_in_progress = {}              # URL -> Download
 315
 316                 host_arch = self.target_arch
 317                 if self.src:
 318                         host_arch = arch.SourceArchitecture(host_arch)
 319
 320                 while True:
 321                         self.solver.solve(self.root, host_arch)
 322                         for w in self.watchers: w()
 323
 324                         if self.solver.ready and not force:
 325                                 break
 326                         else:
 327                                 if self.network_use == network_offline and not force:
 328                                         info(_("Can't choose versions and in off-line mode, so aborting"))
 329                                         break
 330                                 # Once we've starting downloading some things,
 331                                 # we might as well get them all.
 332                                 force = True
 333
 334                         for f in self.solver.feeds_used:
 335                                 if f in downloads_finished or f in downloads_in_progress:
 336                                         continue
 337                                 if f.startswith('/'):
 338                                         continue
 339                                 feed = iface_cache.get_interface(f)
 340                                 downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, iface_cache)
 341
 342                         if not downloads_in_progress:
 343                                 break
 344
 345                         blockers = downloads_in_progress.values()
 346                         yield blockers
 347                         tasks.check(blockers, self.handler.report_error)
 348
 349                         for f in downloads_in_progress.keys():
 350                                 if downloads_in_progress[f].happened:
 351                                         del downloads_in_progress[f]
 352                                         downloads_finished.add(f)
 353
 354         @tasks.async
 355         def solve_and_download_impls(self, refresh = False):
 356                 """Run L{solve_with_downloads} and then get the selected implementations too.
 357                 @raise SafeException: if we couldn't select a set of implementations
 358                 @since: 0.40"""
 359                 refreshed = self.solve_with_downloads(refresh)
 360                 if refreshed:
 361                         yield refreshed
 362                         tasks.check(refreshed)
 363
 364                 if not self.solver.ready:
 365                         raise SafeException(_("Can't find all required implementations:") + '\n' +
 366                                 '\n'.join(["- %s -> %s" % (iface, self.solver.selections[iface])
 367                                            for iface  in self.solver.selections]))
 368                 downloaded = self.download_uncached_implementations()
 369                 if downloaded:
 370                         yield downloaded
 371                         tasks.check(downloaded)
 372
 373         def need_download(self):
 374                 """Decide whether we need to download anything (but don't do it!)
 375                 @return: true if we MUST download something (feeds or implementations)
 376                 @rtype: bool"""
 377                 host_arch = self.target_arch
 378                 if self.src:
 379                         host_arch = arch.SourceArchitecture(host_arch)
 380                 self.solver.solve(self.root, host_arch)
 381                 for w in self.watchers: w()
 382
 383                 if not self.solver.ready:
 384                         return True             # Maybe a newer version will work?
 385
 386                 if self.get_uncached_implementations():
 387                         return True
 388
 389                 return False
 390
 391         def download_uncached_implementations(self):
 392                 """Download all implementations chosen by the solver that are missing from the cache."""
 393                 assert self.solver.ready, "Solver is not ready!\n%s" % self.solver.selections
 394                 return self.fetcher.download_impls([impl for impl in self.solver.selections.values() if not self.get_cached(impl)],
 395                                                    iface_cache.stores)
 396
 397         def download_icon(self, interface, force = False):
 398                 """Download an icon for this interface and add it to the
 399                 icon cache. If the interface has no icon or we are offline, do nothing.
 400                 @return: the task doing the import, or None
 401                 @rtype: L{tasks.Task}"""
 402                 if self.network_use == network_offline:
 403                         info("Not downloading icon for %s as we are off-line", interface)
 404                         return
 405
 406                 modification_time = None
 407
 408                 existing_icon = iface_cache.get_icon_path(interface)
 409                 if existing_icon:
 410                         file_mtime = os.stat(existing_icon).st_mtime
 411                         from email.utils import formatdate
 412                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 413
 414                 return self.fetcher.download_icon(interface, force, modification_time)
 415
 416         def get_interface(self, uri):
 417                 """@deprecated: use L{iface_cache.IfaceCache.get_interface} instead"""
 418                 import warnings
 419                 warnings.warn(_("Policy.get_interface is deprecated!"), DeprecationWarning, stacklevel = 2)
 420                 return iface_cache.get_interface(uri)