zeroinstall/injector/policy.py

   1 """
   2 This class brings together a L{solve.Solver} to choose a set of implmentations, a
   3 L{fetch.Fetcher} to download additional components, and the user's configuration
   4 settings.
   5 """
   6
   7 # Copyright (C) 2009, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 from zeroinstall import _
  11 import time
  12 import os
  13 from logging import info, debug, warn
  14 import ConfigParser
  15
  16 from zeroinstall import SafeException
  17 from zeroinstall.injector import arch
  18 from zeroinstall.injector.model import Interface, Implementation, network_levels, network_offline, DistributionImplementation, network_full
  19 from zeroinstall.injector.namespaces import config_site, config_prog
  20 from zeroinstall.support import tasks, basedir
  21 from zeroinstall.injector.iface_cache import iface_cache
  22
  23 # If we started a check within this period, don't start another one:
  24 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  25
  26 class Policy(object):
  27         """Chooses a set of implementations based on a policy.
  28         Typical use:
  29          1. Create a Policy object, giving it the URI of the program to be run and a handler.
  30          2. Call L{solve_with_downloads}. If more information is needed, a L{fetch.Fetcher} will be used to download it.
  31          3. When all downloads are complete, the L{solver} contains the chosen versions.
  32          4. Use L{get_uncached_implementations} to find where to get these versions and download them
  33             using L{download_uncached_implementations}.
  34
  35         @ivar target_arch: target architecture for binaries
  36         @type target_arch: L{arch.Architecture}
  37         @ivar root: URI of the root interface
  38         @ivar solver: solver used to choose a set of implementations
  39         @type solver: L{solve.Solver}
  40         @ivar watchers: callbacks to invoke after recalculating
  41         @ivar help_with_testing: default stability policy
  42         @type help_with_testing: bool
  43         @ivar network_use: one of the model.network_* values
  44         @ivar freshness: seconds allowed since last update
  45         @type freshness: int
  46         @ivar handler: handler for main-loop integration
  47         @type handler: L{handler.Handler}
  48         @ivar src: whether we are looking for source code
  49         @type src: bool
  50         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
  51         @type stale_feeds: set
  52         """
  53         __slots__ = ['root', 'watchers',
  54                      'freshness', 'handler', '_warned_offline',
  55                      'target_arch', 'src', 'stale_feeds', 'solver', '_fetcher']
  56
  57         help_with_testing = property(lambda self: self.solver.help_with_testing,
  58                                      lambda self, value: setattr(self.solver, 'help_with_testing', value))
  59
  60         network_use = property(lambda self: self.solver.network_use,
  61                                      lambda self, value: setattr(self.solver, 'network_use', value))
  62
  63         implementation = property(lambda self: self.solver.selections)
  64
  65         ready = property(lambda self: self.solver.ready)
  66
  67         def __init__(self, root, handler = None, src = False):
  68                 """
  69                 @param root: The URI of the root interface (the program we want to run).
  70                 @param handler: A handler for main-loop integration.
  71                 @type handler: L{zeroinstall.injector.handler.Handler}
  72                 @param src: Whether we are looking for source code.
  73                 @type src: bool
  74                 """
  75                 self.watchers = []
  76                 self.src = src                          # Root impl must be a "src" machine type
  77                 self.stale_feeds = set()
  78
  79                 from zeroinstall.injector.solver import DefaultSolver
  80                 self.solver = DefaultSolver(network_full, iface_cache, iface_cache.stores)
  81
  82                 # If we need to download something but can't because we are offline,
  83                 # warn the user. But only the first time.
  84                 self._warned_offline = False
  85                 self._fetcher = None
  86
  87                 # (allow self for backwards compat)
  88                 self.handler = handler or self
  89
  90                 debug(_("Supported systems: '%s'"), arch.os_ranks)
  91                 debug(_("Supported processors: '%s'"), arch.machine_ranks)
  92
  93                 config = ConfigParser.ConfigParser()
  94                 config.add_section('global')
  95                 config.set('global', 'help_with_testing', 'False')
  96                 config.set('global', 'freshness', str(60 * 60 * 24 * 30))       # One month
  97                 config.set('global', 'network_use', 'full')
  98
  99                 path = basedir.load_first_config(config_site, config_prog, 'global')
 100                 if path:
 101                         info("Loading configuration from %s", path)
 102                         try:
 103                                 config.read(path)
 104                         except Exception, ex:
 105                                 warn(_("Error loading config: %s"), str(ex) or repr(ex))
 106
 107                 self.solver.help_with_testing = config.getboolean('global', 'help_with_testing')
 108                 self.solver.network_use = config.get('global', 'network_use')
 109                 self.freshness = int(config.get('global', 'freshness'))
 110                 assert self.solver.network_use in network_levels, self.solver.network_use
 111
 112                 self.set_root(root)
 113
 114                 self.target_arch = arch.get_host_architecture()
 115
 116         @property
 117         def fetcher(self):
 118                 if not self._fetcher:
 119                         import fetch
 120                         self._fetcher = fetch.Fetcher(self.handler)
 121                 return self._fetcher
 122
 123         def set_root(self, root):
 124                 """Change the root interface URI."""
 125                 assert isinstance(root, (str, unicode))
 126                 self.root = root
 127                 for w in self.watchers: w()
 128
 129         def save_config(self):
 130                 """Write global settings."""
 131                 config = ConfigParser.ConfigParser()
 132                 config.add_section('global')
 133
 134                 config.set('global', 'help_with_testing', self.help_with_testing)
 135                 config.set('global', 'network_use', self.network_use)
 136                 config.set('global', 'freshness', self.freshness)
 137
 138                 path = basedir.save_config_path(config_site, config_prog)
 139                 path = os.path.join(path, 'global')
 140                 config.write(file(path + '.new', 'w'))
 141                 os.rename(path + '.new', path)
 142
 143         def recalculate(self, fetch_stale_interfaces = True):
 144                 """@deprecated: see L{solve_with_downloads} """
 145                 self.stale_feeds = set()
 146
 147                 host_arch = self.target_arch
 148                 if self.src:
 149                         host_arch = arch.SourceArchitecture(host_arch)
 150                 self.solver.solve(self.root, host_arch)
 151
 152                 if self.network_use == network_offline:
 153                         fetch_stale_interfaces = False
 154
 155                 blockers = []
 156                 for f in self.solver.feeds_used:
 157                         if os.path.isabs(f): continue
 158                         feed = iface_cache.get_feed(f)
 159                         if feed is None or feed.last_modified is None:
 160                                 self.download_and_import_feed_if_online(f)      # Will start a download
 161                         elif self.is_stale(feed):
 162                                 debug(_("Adding %s to stale set"), f)
 163                                 self.stale_feeds.add(iface_cache.get_interface(f))      # Legacy API
 164                                 if fetch_stale_interfaces:
 165                                         self.download_and_import_feed_if_online(f)      # Will start a download
 166
 167                 for w in self.watchers: w()
 168
 169                 return blockers
 170
 171         def usable_feeds(self, iface):
 172                 """Generator for C{iface.feeds} that are valid for our architecture.
 173                 @rtype: generator
 174                 @see: L{arch}"""
 175                 if self.src and iface.uri == self.root:
 176                         # Note: when feeds are recursive, we'll need a better test for root here
 177                         machine_ranks = {'src': 1}
 178                 else:
 179                         machine_ranks = arch.machine_ranks
 180
 181                 for f in iface_cache.get_feed_imports(iface):
 182                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 183                                 yield f
 184                         else:
 185                                 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
 186                                         {'feed': f, 'os': f.os, 'machine': f.machine})
 187
 188         def is_stale(self, feed):
 189                 """Check whether feed needs updating, based on the configured L{freshness}.
 190                 None is considered to be stale.
 191                 @return: true if feed is stale or missing."""
 192                 if feed is None:
 193                         return True
 194                 if os.path.isabs(feed.url):
 195                         return False            # Local feeds are never stale
 196                 if feed.last_modified is None:
 197                         return True             # Don't even have it yet
 198                 now = time.time()
 199                 staleness = now - (feed.last_checked or 0)
 200                 debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0})
 201
 202                 if self.freshness <= 0 or staleness < self.freshness:
 203                         return False            # Fresh enough for us
 204
 205                 last_check_attempt = iface_cache.get_last_check_attempt(feed.url)
 206                 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 207                         debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt))
 208                         return False
 209
 210                 return True
 211
 212         def download_and_import_feed_if_online(self, feed_url):
 213                 """If we're online, call L{fetch.Fetcher.download_and_import_feed}. Otherwise, log a suitable warning."""
 214                 if self.network_use != network_offline:
 215                         debug(_("Feed %s not cached and not off-line. Downloading..."), feed_url)
 216                         return self.fetcher.download_and_import_feed(feed_url, iface_cache)
 217                 else:
 218                         if self._warned_offline:
 219                                 debug(_("Not downloading feed '%s' because we are off-line."), feed_url)
 220                         else:
 221                                 warn(_("Not downloading feed '%s' because we are in off-line mode."), feed_url)
 222                                 self._warned_offline = True
 223
 224         def get_implementation_path(self, impl):
 225                 """Return the local path of impl.
 226                 @rtype: str
 227                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 228                 assert isinstance(impl, Implementation)
 229                 return impl.local_path or iface_cache.stores.lookup_any(impl.digests)
 230
 231         def get_implementation(self, interface):
 232                 """Get the chosen implementation.
 233                 @type interface: Interface
 234                 @rtype: L{model.Implementation}
 235                 @raise SafeException: if interface has not been fetched or no implementation could be
 236                 chosen."""
 237                 assert isinstance(interface, Interface)
 238
 239                 try:
 240                         return self.implementation[interface]
 241                 except KeyError, ex:
 242                         raise SafeException(_("No usable implementation found for '%s'.") % interface.uri)
 243
 244         def get_cached(self, impl):
 245                 """Check whether an implementation is available locally.
 246                 @type impl: model.Implementation
 247                 @rtype: bool
 248                 """
 249                 if isinstance(impl, DistributionImplementation):
 250                         return impl.installed
 251                 if impl.local_path:
 252                         return os.path.exists(impl.local_path)
 253                 else:
 254                         try:
 255                                 path = self.get_implementation_path(impl)
 256                                 assert path
 257                                 return True
 258                         except:
 259                                 pass # OK
 260                 return False
 261
 262         def get_uncached_implementations(self):
 263                 """List all chosen implementations which aren't yet available locally.
 264                 @rtype: [(L{model.Interface}, L{model.Implementation})]"""
 265                 uncached = []
 266                 for iface in self.solver.selections:
 267                         impl = self.solver.selections[iface]
 268                         assert impl, self.solver.selections
 269                         if not self.get_cached(impl):
 270                                 uncached.append((iface, impl))
 271                 return uncached
 272
 273         def refresh_all(self, force = True):
 274                 """Start downloading all feeds for all selected interfaces.
 275                 @param force: Whether to restart existing downloads."""
 276                 return self.solve_with_downloads(force = True)
 277
 278         def get_feed_targets(self, feed_iface_uri):
 279                 """Return a list of Interfaces for which feed_iface can be a feed.
 280                 This is used by B{0launch --feed}.
 281                 @rtype: [model.Interface]
 282                 @raise SafeException: If there are no known feeds."""
 283                 # TODO: what if it isn't cached yet?
 284                 feed_iface = iface_cache.get_interface(feed_iface_uri)
 285                 if not feed_iface.feed_for:
 286                         if not feed_iface.name:
 287                                 raise SafeException(_("Can't get feed targets for '%s'; failed to load interface.") %
 288                                                 feed_iface_uri)
 289                         raise SafeException(_("Missing <feed-for> element in '%s'; "
 290                                         "this interface can't be used as a feed.") % feed_iface_uri)
 291                 feed_targets = feed_iface.feed_for
 292                 debug(_("Feed targets: %s"), feed_targets)
 293                 if not feed_iface.name:
 294                         warn(_("Warning: unknown interface '%s'") % feed_iface_uri)
 295                 return [iface_cache.get_interface(uri) for uri in feed_targets]
 296
 297         @tasks.async
 298         def solve_with_downloads(self, force = False):
 299                 """Run the solver, then download any feeds that are missing or
 300                 that need to be updated. Each time a new feed is imported into
 301                 the cache, the solver is run again, possibly adding new downloads.
 302                 @param force: whether to download even if we're already ready to run."""
 303
 304                 downloads_finished = set()              # Successful or otherwise
 305                 downloads_in_progress = {}              # URL -> Download
 306
 307                 host_arch = self.target_arch
 308                 if self.src:
 309                         host_arch = arch.SourceArchitecture(host_arch)
 310
 311                 while True:
 312                         self.solver.solve(self.root, host_arch)
 313                         for w in self.watchers: w()
 314
 315                         if self.solver.ready and not force:
 316                                 break
 317                         else:
 318                                 if self.network_use == network_offline and not force:
 319                                         info(_("Can't choose versions and in off-line mode, so aborting"))
 320                                         break
 321                                 # Once we've starting downloading some things,
 322                                 # we might as well get them all.
 323                                 force = True
 324
 325                         for f in self.solver.feeds_used:
 326                                 if f in downloads_finished or f in downloads_in_progress:
 327                                         continue
 328                                 if os.path.isabs(f):
 329                                         continue
 330                                 downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, iface_cache)
 331
 332                         if not downloads_in_progress:
 333                                 break
 334
 335                         blockers = downloads_in_progress.values()
 336                         yield blockers
 337                         tasks.check(blockers, self.handler.report_error)
 338
 339                         for f in downloads_in_progress.keys():
 340                                 if downloads_in_progress[f].happened:
 341                                         del downloads_in_progress[f]
 342                                         downloads_finished.add(f)
 343
 344         @tasks.async
 345         def solve_and_download_impls(self, refresh = False, select_only = False):
 346                 """Run L{solve_with_downloads} and then get the selected implementations too.
 347                 @raise SafeException: if we couldn't select a set of implementations
 348                 @since: 0.40"""
 349                 refreshed = self.solve_with_downloads(refresh)
 350                 if refreshed:
 351                         yield refreshed
 352                         tasks.check(refreshed)
 353
 354                 if not self.solver.ready:
 355                         raise SafeException(_("Can't find all required implementations:") + '\n' +
 356                                 '\n'.join(["- %s -> %s" % (iface, self.solver.selections[iface])
 357                                            for iface  in self.solver.selections]))
 358
 359                 if not select_only:
 360                         downloaded = self.download_uncached_implementations()
 361                         if downloaded:
 362                                 yield downloaded
 363                                 tasks.check(downloaded)
 364
 365         def need_download(self):
 366                 """Decide whether we need to download anything (but don't do it!)
 367                 @return: true if we MUST download something (feeds or implementations)
 368                 @rtype: bool"""
 369                 host_arch = self.target_arch
 370                 if self.src:
 371                         host_arch = arch.SourceArchitecture(host_arch)
 372                 self.solver.solve(self.root, host_arch)
 373                 for w in self.watchers: w()
 374
 375                 if not self.solver.ready:
 376                         return True             # Maybe a newer version will work?
 377
 378                 if self.get_uncached_implementations():
 379                         return True
 380
 381                 return False
 382
 383         def download_uncached_implementations(self):
 384                 """Download all implementations chosen by the solver that are missing from the cache."""
 385                 assert self.solver.ready, "Solver is not ready!\n%s" % self.solver.selections
 386                 return self.fetcher.download_impls([impl for impl in self.solver.selections.values() if not self.get_cached(impl)],
 387                                                    iface_cache.stores)
 388
 389         def download_icon(self, interface, force = False):
 390                 """Download an icon for this interface and add it to the
 391                 icon cache. If the interface has no icon or we are offline, do nothing.
 392                 @return: the task doing the import, or None
 393                 @rtype: L{tasks.Task}"""
 394                 if self.network_use == network_offline:
 395                         info("Not downloading icon for %s as we are off-line", interface)
 396                         return
 397
 398                 modification_time = None
 399
 400                 existing_icon = iface_cache.get_icon_path(interface)
 401                 if existing_icon:
 402                         file_mtime = os.stat(existing_icon).st_mtime
 403                         from email.utils import formatdate
 404                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 405
 406                 return self.fetcher.download_icon(interface, force, modification_time)
 407
 408         def get_interface(self, uri):
 409                 """@deprecated: use L{iface_cache.IfaceCache.get_interface} instead"""
 410                 import warnings
 411                 warnings.warn("Policy.get_interface is deprecated!", DeprecationWarning, stacklevel = 2)
 412                 return iface_cache.get_interface(uri)