zeroinstall/injector/policy.py

   1 """
   2 This class brings together a L{solve.Solver} to choose a set of implmentations, a
   3 L{fetch.Fetcher} to download additional components, and the user's configuration
   4 settings.
   5 """
   6
   7 # Copyright (C) 2009, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 from zeroinstall import _
  11 import time
  12 import os
  13 from logging import info, debug, warn
  14 import ConfigParser
  15
  16 from zeroinstall import zerostore, SafeException
  17 from zeroinstall.injector import arch, model
  18 from zeroinstall.injector.model import Interface, Implementation, network_levels, network_offline, DistributionImplementation, network_full
  19 from zeroinstall.injector.handler import Handler
  20 from zeroinstall.injector.namespaces import config_site, config_prog
  21 from zeroinstall.support import tasks, basedir
  22
  23 # If we started a check within this period, don't start another one:
  24 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  25
  26 class Config(object):
  27         """
  28         @ivar handler: handler for main-loop integration
  29         @type handler: L{handler.Handler}
  30         """
  31
  32         __slots__ = ['help_with_testing', 'freshness', 'network_use', '_fetcher', '_stores', '_iface_cache', 'handler']
  33         def __init__(self, handler):
  34                 assert handler is not None
  35                 self.help_with_testing = False
  36                 self.freshness = 60 * 60 * 24 * 30
  37                 self.network_use = network_full
  38                 self.handler = handler
  39                 self._fetcher = self._stores = self._iface_cache = None
  40
  41         @property
  42         def stores(self):
  43                 if not self._stores:
  44                         self._stores = zerostore.Stores()
  45                 return self._stores
  46
  47         @property
  48         def iface_cache(self):
  49                 if not self._iface_cache:
  50                         from zeroinstall.injector import iface_cache
  51                         self._iface_cache = iface_cache.iface_cache
  52                         #self._iface_cache = iface_cache.IfaceCache()
  53                 return self._iface_cache
  54
  55         @property
  56         def fetcher(self):
  57                 if not self._fetcher:
  58                         from zeroinstall.injector import fetch
  59                         self._fetcher = fetch.Fetcher(self.handler)
  60                 return self._fetcher
  61
  62         def save_globals(self):
  63                """Write global settings."""
  64                parser = ConfigParser.ConfigParser()
  65                parser.add_section('global')
  66
  67                parser.set('global', 'help_with_testing', self.help_with_testing)
  68                parser.set('global', 'network_use', self.network_use)
  69                parser.set('global', 'freshness', self.freshness)
  70
  71                path = basedir.save_config_path(config_site, config_prog)
  72                path = os.path.join(path, 'global')
  73                parser.write(file(path + '.new', 'w'))
  74                os.rename(path + '.new', path)
  75
  76 def load_config(handler):
  77         config = Config(handler)
  78         parser = ConfigParser.RawConfigParser()
  79         parser.add_section('global')
  80         parser.set('global', 'help_with_testing', 'False')
  81         parser.set('global', 'freshness', str(60 * 60 * 24 * 30))       # One month
  82         parser.set('global', 'network_use', 'full')
  83
  84         path = basedir.load_first_config(config_site, config_prog, 'global')
  85         if path:
  86                 info("Loading configuration from %s", path)
  87                 try:
  88                         parser.read(path)
  89                 except Exception, ex:
  90                         warn(_("Error loading config: %s"), str(ex) or repr(ex))
  91
  92         config.help_with_testing = parser.getboolean('global', 'help_with_testing')
  93         config.network_use = parser.get('global', 'network_use')
  94         config.freshness = int(parser.get('global', 'freshness'))
  95
  96         assert config.network_use in network_levels, config.network_use
  97
  98         return config
  99
 100 class Policy(object):
 101         """Chooses a set of implementations based on a policy.
 102         Typical use:
 103          1. Create a Policy object, giving it the URI of the program to be run and a handler.
 104          2. Call L{solve_with_downloads}. If more information is needed, a L{fetch.Fetcher} will be used to download it.
 105          3. When all downloads are complete, the L{solver} contains the chosen versions.
 106          4. Use L{get_uncached_implementations} to find where to get these versions and download them
 107             using L{download_uncached_implementations}.
 108
 109         @ivar target_arch: target architecture for binaries
 110         @type target_arch: L{arch.Architecture}
 111         @ivar root: URI of the root interface
 112         @ivar solver: solver used to choose a set of implementations
 113         @type solver: L{solve.Solver}
 114         @ivar watchers: callbacks to invoke after recalculating
 115         @ivar help_with_testing: default stability policy
 116         @type help_with_testing: bool
 117         @ivar network_use: one of the model.network_* values
 118         @ivar freshness: seconds allowed since last update
 119         @type freshness: int
 120         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
 121         @type stale_feeds: set
 122         """
 123         __slots__ = ['root', 'watchers', 'requirements', 'config', '_warned_offline',
 124                      'command', 'target_arch',
 125                      'stale_feeds', 'solver']
 126
 127         help_with_testing = property(lambda self: self.config.help_with_testing,
 128                                      lambda self, value: setattr(self.config, 'help_with_testing', bool(value)))
 129
 130         network_use = property(lambda self: self.config.network_use,
 131                                lambda self, value: setattr(self.config, 'network_use', value))
 132
 133         freshness = property(lambda self: self.config.freshness,
 134                              lambda self, value: setattr(self.config, 'freshness', str(value)))
 135
 136         implementation = property(lambda self: self.solver.selections)
 137
 138         ready = property(lambda self: self.solver.ready)
 139
 140         # (was used by 0test)
 141         handler = property(lambda self: self.config.handler,
 142                            lambda self, value: setattr(self.config, 'handler', value))
 143
 144
 145         def __init__(self, root = None, handler = None, src = None, command = -1, config = None, requirements = None):
 146                 """
 147                 @param requirements: Details about the program we want to run
 148                 @type requirements: L{requirements.Requirements}
 149                 @param config: The configuration settings to use, or None to load from disk.
 150                 @type config: L{ConfigParser.ConfigParser}
 151                 Note: all other arguments are deprecated (since 0launch 0.52)
 152                 """
 153                 self.watchers = []
 154                 if requirements is None:
 155                         from zeroinstall.injector.requirements import Requirements
 156                         requirements = Requirements(root)
 157                         requirements.source = bool(src)                         # Root impl must be a "src" machine type
 158                         if command == -1:
 159                                 if src:
 160                                         command = 'compile'
 161                                 else:
 162                                         command = 'run'
 163                         requirements.command = command
 164                         self.target_arch = arch.get_host_architecture()
 165                 else:
 166                         assert root == src == None
 167                         assert command == -1
 168                         self.target_arch = arch.get_architecture(requirements.os, requirements.cpu)
 169                 self.requirements = requirements
 170
 171                 self.stale_feeds = set()
 172
 173                 if config is None:
 174                         self.config = load_config(handler or Handler())
 175                 else:
 176                         assert handler is None, "can't pass a handler and a config"
 177                         self.config = config
 178
 179                 from zeroinstall.injector.solver import DefaultSolver
 180                 self.solver = DefaultSolver(self.config)
 181
 182                 # If we need to download something but can't because we are offline,
 183                 # warn the user. But only the first time.
 184                 self._warned_offline = False
 185
 186                 debug(_("Supported systems: '%s'"), arch.os_ranks)
 187                 debug(_("Supported processors: '%s'"), arch.machine_ranks)
 188
 189                 if requirements.before or requirements.not_before:
 190                         self.solver.extra_restrictions[config.iface_cache.get_interface(requirements.interface_uri)] = [
 191                                         model.VersionRangeRestriction(model.parse_version(requirements.before),
 192                                                                       model.parse_version(requirements.not_before))]
 193
 194         @property
 195         def fetcher(self):
 196                 return self.config.fetcher
 197
 198         def save_config(self):
 199                 self.config.save_globals()
 200
 201         def recalculate(self, fetch_stale_interfaces = True):
 202                 """@deprecated: see L{solve_with_downloads} """
 203                 import warnings
 204                 warnings.warn("Policy.recalculate is deprecated!", DeprecationWarning, stacklevel = 2)
 205
 206                 self.stale_feeds = set()
 207
 208                 host_arch = self.target_arch
 209                 if self.requirements.source:
 210                         host_arch = arch.SourceArchitecture(host_arch)
 211                 self.solver.solve(self.root, host_arch, command_name = self.command)
 212
 213                 if self.network_use == network_offline:
 214                         fetch_stale_interfaces = False
 215
 216                 blockers = []
 217                 for f in self.solver.feeds_used:
 218                         if os.path.isabs(f): continue
 219                         feed = self.config.iface_cache.get_feed(f)
 220                         if feed is None or feed.last_modified is None:
 221                                 self.download_and_import_feed_if_online(f)      # Will start a download
 222                         elif self.is_stale(feed):
 223                                 debug(_("Adding %s to stale set"), f)
 224                                 self.stale_feeds.add(self.config.iface_cache.get_interface(f))  # Legacy API
 225                                 if fetch_stale_interfaces:
 226                                         self.download_and_import_feed_if_online(f)      # Will start a download
 227
 228                 for w in self.watchers: w()
 229
 230                 return blockers
 231
 232         def usable_feeds(self, iface):
 233                 """Generator for C{iface.feeds} that are valid for our architecture.
 234                 @rtype: generator
 235                 @see: L{arch}"""
 236                 if self.requirements.source and iface.uri == self.root:
 237                         # Note: when feeds are recursive, we'll need a better test for root here
 238                         machine_ranks = {'src': 1}
 239                 else:
 240                         machine_ranks = arch.machine_ranks
 241
 242                 for f in self.config.iface_cache.get_feed_imports(iface):
 243                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 244                                 yield f
 245                         else:
 246                                 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
 247                                         {'feed': f, 'os': f.os, 'machine': f.machine})
 248
 249         def is_stale(self, feed):
 250                 """Check whether feed needs updating, based on the configured L{freshness}.
 251                 None is considered to be stale.
 252                 @return: true if feed is stale or missing."""
 253                 if feed is None:
 254                         return True
 255                 if os.path.isabs(feed.url):
 256                         return False            # Local feeds are never stale
 257                 if feed.last_modified is None:
 258                         return True             # Don't even have it yet
 259                 now = time.time()
 260                 staleness = now - (feed.last_checked or 0)
 261                 debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0})
 262
 263                 if self.freshness <= 0 or staleness < self.freshness:
 264                         return False            # Fresh enough for us
 265
 266                 last_check_attempt = self.config.iface_cache.get_last_check_attempt(feed.url)
 267                 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 268                         debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt))
 269                         return False
 270
 271                 return True
 272
 273         def download_and_import_feed_if_online(self, feed_url):
 274                 """If we're online, call L{fetch.Fetcher.download_and_import_feed}. Otherwise, log a suitable warning."""
 275                 if self.network_use != network_offline:
 276                         debug(_("Feed %s not cached and not off-line. Downloading..."), feed_url)
 277                         return self.fetcher.download_and_import_feed(feed_url, self.config.iface_cache)
 278                 else:
 279                         if self._warned_offline:
 280                                 debug(_("Not downloading feed '%s' because we are off-line."), feed_url)
 281                         else:
 282                                 warn(_("Not downloading feed '%s' because we are in off-line mode."), feed_url)
 283                                 self._warned_offline = True
 284
 285         def get_implementation_path(self, impl):
 286                 """Return the local path of impl.
 287                 @rtype: str
 288                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 289                 assert isinstance(impl, Implementation)
 290                 return impl.local_path or self.config.stores.lookup_any(impl.digests)
 291
 292         def get_implementation(self, interface):
 293                 """Get the chosen implementation.
 294                 @type interface: Interface
 295                 @rtype: L{model.Implementation}
 296                 @raise SafeException: if interface has not been fetched or no implementation could be
 297                 chosen."""
 298                 assert isinstance(interface, Interface)
 299
 300                 try:
 301                         return self.implementation[interface]
 302                 except KeyError:
 303                         raise SafeException(_("No usable implementation found for '%s'.") % interface.uri)
 304
 305         def get_cached(self, impl):
 306                 """Check whether an implementation is available locally.
 307                 @type impl: model.Implementation
 308                 @rtype: bool
 309                 """
 310                 return impl.is_available(self.config.stores)
 311
 312         def get_uncached_implementations(self):
 313                 """List all chosen implementations which aren't yet available locally.
 314                 @rtype: [(L{model.Interface}, L{model.Implementation})]"""
 315                 iface_cache = self.config.iface_cache
 316                 uncached = []
 317                 for uri, selection in self.solver.selections.selections.iteritems():
 318                         impl = selection.impl
 319                         assert impl, self.solver.selections
 320                         if not self.get_cached(impl):
 321                                 uncached.append((iface_cache.get_interface(uri), impl))
 322                 return uncached
 323
 324         def refresh_all(self, force = True):
 325                 """Start downloading all feeds for all selected interfaces.
 326                 @param force: Whether to restart existing downloads."""
 327                 return self.solve_with_downloads(force = True)
 328
 329         def get_feed_targets(self, feed):
 330                 """Return a list of Interfaces for which feed can be a feed.
 331                 This is used by B{0install add-feed}.
 332                 @param feed: the feed
 333                 @type feed: L{model.ZeroInstallFeed} (or, deprecated, a URL)
 334                 @rtype: [model.Interface]
 335                 @raise SafeException: If there are no known feeds."""
 336
 337                 if not isinstance(feed, model.ZeroInstallFeed):
 338                         # (deprecated)
 339                         feed = self.config.iface_cache.get_feed(feed)
 340                         if feed is None:
 341                                 raise SafeException("Feed is not cached and using deprecated API")
 342
 343                 if not feed.feed_for:
 344                         raise SafeException(_("Missing <feed-for> element in '%s'; "
 345                                         "it can't be used as a feed for any other interface.") % feed.url)
 346                 feed_targets = feed.feed_for
 347                 debug(_("Feed targets: %s"), feed_targets)
 348                 return [self.config.iface_cache.get_interface(uri) for uri in feed_targets]
 349
 350         @tasks.async
 351         def solve_with_downloads(self, force = False, update_local = False):
 352                 """Run the solver, then download any feeds that are missing or
 353                 that need to be updated. Each time a new feed is imported into
 354                 the cache, the solver is run again, possibly adding new downloads.
 355                 @param force: whether to download even if we're already ready to run.
 356                 @param update_local: fetch PackageKit feeds even if we're ready to run."""
 357
 358                 downloads_finished = set()              # Successful or otherwise
 359                 downloads_in_progress = {}              # URL -> Download
 360
 361                 host_arch = self.target_arch
 362                 if self.requirements.source:
 363                         host_arch = arch.SourceArchitecture(host_arch)
 364
 365                 # There are three cases:
 366                 # 1. We want to run immediately if possible. If not, download all the information we can.
 367                 #    (force = False, update_local = False)
 368                 # 2. We're in no hurry, but don't want to use the network unnecessarily.
 369                 #    We should still update local information (from PackageKit).
 370                 #    (force = False, update_local = True)
 371                 # 3. The user explicitly asked us to refresh everything.
 372                 #    (force = True)
 373
 374                 try_quick_exit = not (force or update_local)
 375
 376                 while True:
 377                         self.solver.solve(self.root, host_arch, command_name = self.command)
 378                         for w in self.watchers: w()
 379
 380                         if try_quick_exit and self.solver.ready:
 381                                 break
 382                         try_quick_exit = False
 383
 384                         if not self.solver.ready:
 385                                 force = True
 386
 387                         for f in self.solver.feeds_used:
 388                                 if f in downloads_finished or f in downloads_in_progress:
 389                                         continue
 390                                 if os.path.isabs(f):
 391                                         if force:
 392                                                 self.config.iface_cache.get_feed(f, force = True)
 393                                                 downloads_in_progress[f] = tasks.IdleBlocker('Refresh local feed')
 394                                         continue
 395                                 elif f.startswith('distribution:'):
 396                                         if force or update_local:
 397                                                 downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, self.config.iface_cache)
 398                                 elif force and self.network_use != network_offline:
 399                                         downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, self.config.iface_cache)
 400                                         # Once we've starting downloading some things,
 401                                         # we might as well get them all.
 402                                         force = True
 403
 404                         if not downloads_in_progress:
 405                                 if self.network_use == network_offline:
 406                                         info(_("Can't choose versions and in off-line mode, so aborting"))
 407                                 break
 408
 409                         # Wait for at least one download to finish
 410                         blockers = downloads_in_progress.values()
 411                         yield blockers
 412                         tasks.check(blockers, self.handler.report_error)
 413
 414                         for f in downloads_in_progress.keys():
 415                                 if f in downloads_in_progress and downloads_in_progress[f].happened:
 416                                         del downloads_in_progress[f]
 417                                         downloads_finished.add(f)
 418
 419                                         # Need to refetch any "distribution" feed that
 420                                         # depends on this one
 421                                         distro_feed_url = 'distribution:' + f
 422                                         if distro_feed_url in downloads_finished:
 423                                                 downloads_finished.remove(distro_feed_url)
 424                                         if distro_feed_url in downloads_in_progress:
 425                                                 del downloads_in_progress[distro_feed_url]
 426
 427         @tasks.async
 428         def solve_and_download_impls(self, refresh = False, select_only = False):
 429                 """Run L{solve_with_downloads} and then get the selected implementations too.
 430                 @raise SafeException: if we couldn't select a set of implementations
 431                 @since: 0.40"""
 432                 refreshed = self.solve_with_downloads(refresh)
 433                 if refreshed:
 434                         yield refreshed
 435                         tasks.check(refreshed)
 436
 437                 if not self.solver.ready:
 438                         raise self.solver.get_failure_reason()
 439
 440                 if not select_only:
 441                         downloaded = self.download_uncached_implementations()
 442                         if downloaded:
 443                                 yield downloaded
 444                                 tasks.check(downloaded)
 445
 446         def need_download(self):
 447                 """Decide whether we need to download anything (but don't do it!)
 448                 @return: true if we MUST download something (feeds or implementations)
 449                 @rtype: bool"""
 450                 host_arch = self.target_arch
 451                 if self.requirements.source:
 452                         host_arch = arch.SourceArchitecture(host_arch)
 453                 self.solver.solve(self.root, host_arch, command_name = self.command)
 454                 for w in self.watchers: w()
 455
 456                 if not self.solver.ready:
 457                         return True             # Maybe a newer version will work?
 458
 459                 if self.get_uncached_implementations():
 460                         return True
 461
 462                 return False
 463
 464         def download_uncached_implementations(self):
 465                 """Download all implementations chosen by the solver that are missing from the cache."""
 466                 assert self.solver.ready, "Solver is not ready!\n%s" % self.solver.selections
 467                 return self.fetcher.download_impls([impl for impl in self.solver.selections.values() if not self.get_cached(impl)],
 468                                                    self.config.stores)
 469
 470         def download_icon(self, interface, force = False):
 471                 """Download an icon for this interface and add it to the
 472                 icon cache. If the interface has no icon or we are offline, do nothing.
 473                 @return: the task doing the import, or None
 474                 @rtype: L{tasks.Task}"""
 475                 if self.network_use == network_offline:
 476                         info("Not downloading icon for %s as we are off-line", interface)
 477                         return
 478
 479                 modification_time = None
 480
 481                 existing_icon = self.config.iface_cache.get_icon_path(interface)
 482                 if existing_icon:
 483                         file_mtime = os.stat(existing_icon).st_mtime
 484                         from email.utils import formatdate
 485                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 486
 487                 return self.fetcher.download_icon(interface, force, modification_time)
 488
 489         def get_interface(self, uri):
 490                 """@deprecated: use L{iface_cache.IfaceCache.get_interface} instead"""
 491                 import warnings
 492                 warnings.warn("Policy.get_interface is deprecated!", DeprecationWarning, stacklevel = 2)
 493                 return self.config.iface_cache.get_interface(uri)
 494
 495         @property
 496         def command(self):
 497                 return self.requirements.command
 498
 499         @property
 500         def root(self):
 501                 return self.requirements.interface_uri
 502
 503 _config = None
 504 def get_deprecated_singleton_config():
 505         global _config
 506         if _config is None:
 507                 _config = load_config(Handler())
 508         return _config