zeroinstall/injector/policy.py

   1 """
   2 This class brings together a L{solve.Solver} to choose a set of implmentations, a
   3 L{fetch.Fetcher} to download additional components, and the user's configuration
   4 settings.
   5 """
   6
   7 # Copyright (C) 2009, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 from zeroinstall import _
  11 import time
  12 import os
  13 from logging import info, debug, warn
  14 import ConfigParser
  15
  16 from zeroinstall import SafeException
  17 from zeroinstall.injector import arch
  18 from zeroinstall.injector.model import Interface, Implementation, network_levels, network_offline, DistributionImplementation, network_full
  19 from zeroinstall.injector.namespaces import config_site, config_prog
  20 from zeroinstall.support import tasks, basedir
  21 from zeroinstall.injector.iface_cache import iface_cache
  22
  23 # If we started a check within this period, don't start another one:
  24 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  25
  26 class Policy(object):
  27         """Chooses a set of implementations based on a policy.
  28         Typical use:
  29          1. Create a Policy object, giving it the URI of the program to be run and a handler.
  30          2. Call L{solve_with_downloads}. If more information is needed, a L{fetch.Fetcher} will be used to download it.
  31          3. When all downloads are complete, the L{solver} contains the chosen versions.
  32          4. Use L{get_uncached_implementations} to find where to get these versions and download them
  33             using L{download_uncached_implementations}.
  34
  35         @ivar target_arch: target architecture for binaries
  36         @type target_arch: L{arch.Architecture}
  37         @ivar root: URI of the root interface
  38         @ivar solver: solver used to choose a set of implementations
  39         @type solver: L{solve.Solver}
  40         @ivar watchers: callbacks to invoke after recalculating
  41         @ivar help_with_testing: default stability policy
  42         @type help_with_testing: bool
  43         @ivar network_use: one of the model.network_* values
  44         @ivar freshness: seconds allowed since last update
  45         @type freshness: int
  46         @ivar handler: handler for main-loop integration
  47         @type handler: L{handler.Handler}
  48         @ivar src: whether we are looking for source code
  49         @type src: bool
  50         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
  51         @type stale_feeds: set
  52         """
  53         __slots__ = ['root', 'watchers',
  54                      'freshness', 'handler', '_warned_offline',
  55                      'target_arch', 'src', 'stale_feeds', 'solver', '_fetcher']
  56
  57         help_with_testing = property(lambda self: self.solver.help_with_testing,
  58                                      lambda self, value: setattr(self.solver, 'help_with_testing', value))
  59
  60         network_use = property(lambda self: self.solver.network_use,
  61                                      lambda self, value: setattr(self.solver, 'network_use', value))
  62
  63         implementation = property(lambda self: self.solver.selections)
  64
  65         ready = property(lambda self: self.solver.ready)
  66
  67         def __init__(self, root, handler = None, src = False):
  68                 """
  69                 @param root: The URI of the root interface (the program we want to run).
  70                 @param handler: A handler for main-loop integration.
  71                 @type handler: L{zeroinstall.injector.handler.Handler}
  72                 @param src: Whether we are looking for source code.
  73                 @type src: bool
  74                 """
  75                 self.watchers = []
  76                 self.src = src                          # Root impl must be a "src" machine type
  77                 self.stale_feeds = set()
  78
  79                 from zeroinstall.injector.solver import DefaultSolver
  80                 self.solver = DefaultSolver(network_full, iface_cache, iface_cache.stores)
  81
  82                 # If we need to download something but can't because we are offline,
  83                 # warn the user. But only the first time.
  84                 self._warned_offline = False
  85                 self._fetcher = None
  86
  87                 # (allow self for backwards compat)
  88                 self.handler = handler or self
  89
  90                 debug(_("Supported systems: '%s'"), arch.os_ranks)
  91                 debug(_("Supported processors: '%s'"), arch.machine_ranks)
  92
  93                 config = ConfigParser.ConfigParser()
  94                 config.add_section('global')
  95                 config.set('global', 'help_with_testing', 'False')
  96                 config.set('global', 'freshness', str(60 * 60 * 24 * 30))       # One month
  97                 config.set('global', 'network_use', 'full')
  98
  99                 path = basedir.load_first_config(config_site, config_prog, 'global')
 100                 if path:
 101                         info("Loading configuration from %s", path)
 102                         try:
 103                                 config.read(path)
 104                         except Exception, ex:
 105                                 warn(_("Error loading config: %s"), str(ex) or repr(ex))
 106
 107                 self.solver.help_with_testing = config.getboolean('global', 'help_with_testing')
 108                 self.solver.network_use = config.get('global', 'network_use')
 109                 self.freshness = int(config.get('global', 'freshness'))
 110                 assert self.solver.network_use in network_levels, self.solver.network_use
 111
 112                 self.set_root(root)
 113
 114                 self.target_arch = arch.get_host_architecture()
 115
 116         @property
 117         def fetcher(self):
 118                 if not self._fetcher:
 119                         import fetch
 120                         self._fetcher = fetch.Fetcher(self.handler)
 121                 return self._fetcher
 122
 123         def set_root(self, root):
 124                 """Change the root interface URI."""
 125                 assert isinstance(root, (str, unicode))
 126                 self.root = root
 127                 for w in self.watchers: w()
 128
 129         def save_config(self):
 130                 """Write global settings."""
 131                 config = ConfigParser.ConfigParser()
 132                 config.add_section('global')
 133
 134                 config.set('global', 'help_with_testing', self.help_with_testing)
 135                 config.set('global', 'network_use', self.network_use)
 136                 config.set('global', 'freshness', self.freshness)
 137
 138                 path = basedir.save_config_path(config_site, config_prog)
 139                 path = os.path.join(path, 'global')
 140                 config.write(file(path + '.new', 'w'))
 141                 os.rename(path + '.new', path)
 142
 143         def recalculate(self, fetch_stale_interfaces = True):
 144                 """@deprecated: see L{solve_with_downloads} """
 145                 import warnings
 146                 warnings.warn("Policy.recalculate is deprecated!", DeprecationWarning, stacklevel = 2)
 147
 148                 self.stale_feeds = set()
 149
 150                 host_arch = self.target_arch
 151                 if self.src:
 152                         host_arch = arch.SourceArchitecture(host_arch)
 153                 self.solver.solve(self.root, host_arch)
 154
 155                 if self.network_use == network_offline:
 156                         fetch_stale_interfaces = False
 157
 158                 blockers = []
 159                 for f in self.solver.feeds_used:
 160                         if os.path.isabs(f): continue
 161                         feed = iface_cache.get_feed(f)
 162                         if feed is None or feed.last_modified is None:
 163                                 self.download_and_import_feed_if_online(f)      # Will start a download
 164                         elif self.is_stale(feed):
 165                                 debug(_("Adding %s to stale set"), f)
 166                                 self.stale_feeds.add(iface_cache.get_interface(f))      # Legacy API
 167                                 if fetch_stale_interfaces:
 168                                         self.download_and_import_feed_if_online(f)      # Will start a download
 169
 170                 for w in self.watchers: w()
 171
 172                 return blockers
 173
 174         def usable_feeds(self, iface):
 175                 """Generator for C{iface.feeds} that are valid for our architecture.
 176                 @rtype: generator
 177                 @see: L{arch}"""
 178                 if self.src and iface.uri == self.root:
 179                         # Note: when feeds are recursive, we'll need a better test for root here
 180                         machine_ranks = {'src': 1}
 181                 else:
 182                         machine_ranks = arch.machine_ranks
 183
 184                 for f in iface_cache.get_feed_imports(iface):
 185                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 186                                 yield f
 187                         else:
 188                                 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
 189                                         {'feed': f, 'os': f.os, 'machine': f.machine})
 190
 191         def is_stale(self, feed):
 192                 """Check whether feed needs updating, based on the configured L{freshness}.
 193                 None is considered to be stale.
 194                 @return: true if feed is stale or missing."""
 195                 if feed is None:
 196                         return True
 197                 if os.path.isabs(feed.url):
 198                         return False            # Local feeds are never stale
 199                 if feed.last_modified is None:
 200                         return True             # Don't even have it yet
 201                 now = time.time()
 202                 staleness = now - (feed.last_checked or 0)
 203                 debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0})
 204
 205                 if self.freshness <= 0 or staleness < self.freshness:
 206                         return False            # Fresh enough for us
 207
 208                 last_check_attempt = iface_cache.get_last_check_attempt(feed.url)
 209                 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 210                         debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt))
 211                         return False
 212
 213                 return True
 214
 215         def download_and_import_feed_if_online(self, feed_url):
 216                 """If we're online, call L{fetch.Fetcher.download_and_import_feed}. Otherwise, log a suitable warning."""
 217                 if self.network_use != network_offline:
 218                         debug(_("Feed %s not cached and not off-line. Downloading..."), feed_url)
 219                         return self.fetcher.download_and_import_feed(feed_url, iface_cache)
 220                 else:
 221                         if self._warned_offline:
 222                                 debug(_("Not downloading feed '%s' because we are off-line."), feed_url)
 223                         else:
 224                                 warn(_("Not downloading feed '%s' because we are in off-line mode."), feed_url)
 225                                 self._warned_offline = True
 226
 227         def get_implementation_path(self, impl):
 228                 """Return the local path of impl.
 229                 @rtype: str
 230                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 231                 assert isinstance(impl, Implementation)
 232                 return impl.local_path or iface_cache.stores.lookup_any(impl.digests)
 233
 234         def get_implementation(self, interface):
 235                 """Get the chosen implementation.
 236                 @type interface: Interface
 237                 @rtype: L{model.Implementation}
 238                 @raise SafeException: if interface has not been fetched or no implementation could be
 239                 chosen."""
 240                 assert isinstance(interface, Interface)
 241
 242                 try:
 243                         return self.implementation[interface]
 244                 except KeyError, ex:
 245                         raise SafeException(_("No usable implementation found for '%s'.") % interface.uri)
 246
 247         def get_cached(self, impl):
 248                 """Check whether an implementation is available locally.
 249                 @type impl: model.Implementation
 250                 @rtype: bool
 251                 """
 252                 if isinstance(impl, DistributionImplementation):
 253                         return impl.installed
 254                 if impl.local_path:
 255                         return os.path.exists(impl.local_path)
 256                 else:
 257                         try:
 258                                 path = self.get_implementation_path(impl)
 259                                 assert path
 260                                 return True
 261                         except:
 262                                 pass # OK
 263                 return False
 264
 265         def get_uncached_implementations(self):
 266                 """List all chosen implementations which aren't yet available locally.
 267                 @rtype: [(L{model.Interface}, L{model.Implementation})]"""
 268                 uncached = []
 269                 for iface in self.solver.selections:
 270                         impl = self.solver.selections[iface]
 271                         assert impl, self.solver.selections
 272                         if not self.get_cached(impl):
 273                                 uncached.append((iface, impl))
 274                 return uncached
 275
 276         def refresh_all(self, force = True):
 277                 """Start downloading all feeds for all selected interfaces.
 278                 @param force: Whether to restart existing downloads."""
 279                 return self.solve_with_downloads(force = True)
 280
 281         def get_feed_targets(self, feed_iface_uri):
 282                 """Return a list of Interfaces for which feed_iface can be a feed.
 283                 This is used by B{0launch --feed}.
 284                 @rtype: [model.Interface]
 285                 @raise SafeException: If there are no known feeds."""
 286                 # TODO: what if it isn't cached yet?
 287                 feed_iface = iface_cache.get_interface(feed_iface_uri)
 288                 if not feed_iface.feed_for:
 289                         if not feed_iface.name:
 290                                 raise SafeException(_("Can't get feed targets for '%s'; failed to load interface.") %
 291                                                 feed_iface_uri)
 292                         raise SafeException(_("Missing <feed-for> element in '%s'; "
 293                                         "this interface can't be used as a feed.") % feed_iface_uri)
 294                 feed_targets = feed_iface.feed_for
 295                 debug(_("Feed targets: %s"), feed_targets)
 296                 if not feed_iface.name:
 297                         warn(_("Warning: unknown interface '%s'") % feed_iface_uri)
 298                 return [iface_cache.get_interface(uri) for uri in feed_targets]
 299
 300         @tasks.async
 301         def solve_with_downloads(self, force = False, update_local = False):
 302                 """Run the solver, then download any feeds that are missing or
 303                 that need to be updated. Each time a new feed is imported into
 304                 the cache, the solver is run again, possibly adding new downloads.
 305                 @param force: whether to download even if we're already ready to run.
 306                 @param update_local: fetch PackageKit feeds even if we're ready to run."""
 307
 308                 downloads_finished = set()              # Successful or otherwise
 309                 downloads_in_progress = {}              # URL -> Download
 310
 311                 host_arch = self.target_arch
 312                 if self.src:
 313                         host_arch = arch.SourceArchitecture(host_arch)
 314
 315                 # There are three cases:
 316                 # 1. We want to run immediately if possible. If not, download all the information we can.
 317                 #    (force = False, update_local = False)
 318                 # 2. We're in no hurry, but don't want to use the network unnecessarily.
 319                 #    We should still update local information (from PackageKit).
 320                 #    (force = False, update_local = True)
 321                 # 3. The user explicitly asked us to refresh everything.
 322                 #    (force = True)
 323
 324                 try_quick_exit = not (force or update_local)
 325
 326                 while True:
 327                         self.solver.solve(self.root, host_arch)
 328                         for w in self.watchers: w()
 329
 330                         if try_quick_exit and self.solver.ready:
 331                                 break
 332                         try_quick_exit = False
 333
 334                         if not self.solver.ready:
 335                                 # Once we've starting downloading some things,
 336                                 # we might as well get them all.
 337                                 force = True
 338
 339                         for f in self.solver.feeds_used:
 340                                 if f in downloads_finished or f in downloads_in_progress:
 341                                         continue
 342                                 if os.path.isabs(f):
 343                                         continue
 344                                 elif f.startswith('distribution:'):
 345                                         if force or update_local:
 346                                                 downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, iface_cache)
 347                                 elif force and self.network_use != network_offline:
 348                                         downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, iface_cache)
 349
 350                         if not downloads_in_progress:
 351                                 if self.network_use == network_offline:
 352                                         info(_("Can't choose versions and in off-line mode, so aborting"))
 353                                 break
 354
 355                         # Wait for at least one download to finish
 356                         blockers = downloads_in_progress.values()
 357                         yield blockers
 358                         tasks.check(blockers, self.handler.report_error)
 359
 360                         for f in downloads_in_progress.keys():
 361                                 if downloads_in_progress[f].happened:
 362                                         del downloads_in_progress[f]
 363                                         downloads_finished.add(f)
 364
 365         @tasks.async
 366         def solve_and_download_impls(self, refresh = False, select_only = False):
 367                 """Run L{solve_with_downloads} and then get the selected implementations too.
 368                 @raise SafeException: if we couldn't select a set of implementations
 369                 @since: 0.40"""
 370                 refreshed = self.solve_with_downloads(refresh)
 371                 if refreshed:
 372                         yield refreshed
 373                         tasks.check(refreshed)
 374
 375                 if not self.solver.ready:
 376                         raise SafeException(_("Can't find all required implementations:") + '\n' +
 377                                 '\n'.join(["- %s -> %s" % (iface, self.solver.selections[iface])
 378                                            for iface  in self.solver.selections]))
 379
 380                 if not select_only:
 381                         downloaded = self.download_uncached_implementations()
 382                         if downloaded:
 383                                 yield downloaded
 384                                 tasks.check(downloaded)
 385
 386         def need_download(self):
 387                 """Decide whether we need to download anything (but don't do it!)
 388                 @return: true if we MUST download something (feeds or implementations)
 389                 @rtype: bool"""
 390                 host_arch = self.target_arch
 391                 if self.src:
 392                         host_arch = arch.SourceArchitecture(host_arch)
 393                 self.solver.solve(self.root, host_arch)
 394                 for w in self.watchers: w()
 395
 396                 if not self.solver.ready:
 397                         return True             # Maybe a newer version will work?
 398
 399                 if self.get_uncached_implementations():
 400                         return True
 401
 402                 return False
 403
 404         def download_uncached_implementations(self):
 405                 """Download all implementations chosen by the solver that are missing from the cache."""
 406                 assert self.solver.ready, "Solver is not ready!\n%s" % self.solver.selections
 407                 return self.fetcher.download_impls([impl for impl in self.solver.selections.values() if not self.get_cached(impl)],
 408                                                    iface_cache.stores)
 409
 410         def download_icon(self, interface, force = False):
 411                 """Download an icon for this interface and add it to the
 412                 icon cache. If the interface has no icon or we are offline, do nothing.
 413                 @return: the task doing the import, or None
 414                 @rtype: L{tasks.Task}"""
 415                 if self.network_use == network_offline:
 416                         info("Not downloading icon for %s as we are off-line", interface)
 417                         return
 418
 419                 modification_time = None
 420
 421                 existing_icon = iface_cache.get_icon_path(interface)
 422                 if existing_icon:
 423                         file_mtime = os.stat(existing_icon).st_mtime
 424                         from email.utils import formatdate
 425                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 426
 427                 return self.fetcher.download_icon(interface, force, modification_time)
 428
 429         def get_interface(self, uri):
 430                 """@deprecated: use L{iface_cache.IfaceCache.get_interface} instead"""
 431                 import warnings
 432                 warnings.warn("Policy.get_interface is deprecated!", DeprecationWarning, stacklevel = 2)
 433                 return iface_cache.get_interface(uri)