zeroinstall/injector/policy.py

   1 """
   2 This class brings together a L{solve.Solver} to choose a set of implmentations, a
   3 L{fetch.Fetcher} to download additional components, and the user's configuration
   4 settings.
   5 """
   6
   7 # Copyright (C) 2009, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 from zeroinstall import _
  11 import time
  12 import os
  13 from logging import info, debug, warn
  14 import ConfigParser
  15
  16 from zeroinstall import zerostore, SafeException
  17 from zeroinstall.injector import arch, model
  18 from zeroinstall.injector.model import Interface, Implementation, network_levels, network_offline, DistributionImplementation, network_full
  19 from zeroinstall.injector.handler import Handler
  20 from zeroinstall.injector.namespaces import config_site, config_prog
  21 from zeroinstall.support import tasks, basedir
  22
  23 # If we started a check within this period, don't start another one:
  24 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  25
  26 class Config(object):
  27         """
  28         @ivar handler: handler for main-loop integration
  29         @type handler: L{handler.Handler}
  30         """
  31
  32         __slots__ = ['help_with_testing', 'freshness', 'network_use', '_fetcher', '_stores', '_iface_cache', 'handler']
  33         def __init__(self, handler):
  34                 assert handler is not None
  35                 self.help_with_testing = False
  36                 self.freshness = 60 * 60 * 24 * 30
  37                 self.network_use = network_full
  38                 self.handler = handler
  39                 self._fetcher = self._stores = self._iface_cache = None
  40
  41         @property
  42         def stores(self):
  43                 if not self._stores:
  44                         self._stores = zerostore.Stores()
  45                 return self._stores
  46
  47         @property
  48         def iface_cache(self):
  49                 if not self._iface_cache:
  50                         from zeroinstall.injector import iface_cache
  51                         self._iface_cache = iface_cache.iface_cache
  52                         #self._iface_cache = iface_cache.IfaceCache()
  53                 return self._iface_cache
  54
  55         @property
  56         def fetcher(self):
  57                 if not self._fetcher:
  58                         from zeroinstall.injector import fetch
  59                         self._fetcher = fetch.Fetcher(self.handler)
  60                 return self._fetcher
  61
  62         def save_globals(self):
  63                """Write global settings."""
  64                parser = ConfigParser.ConfigParser()
  65                parser.add_section('global')
  66
  67                parser.set('global', 'help_with_testing', self.help_with_testing)
  68                parser.set('global', 'network_use', self.network_use)
  69                parser.set('global', 'freshness', self.freshness)
  70
  71                path = basedir.save_config_path(config_site, config_prog)
  72                path = os.path.join(path, 'global')
  73                parser.write(file(path + '.new', 'w'))
  74                os.rename(path + '.new', path)
  75
  76 def load_config(handler):
  77         config = Config(handler)
  78         parser = ConfigParser.RawConfigParser()
  79         parser.add_section('global')
  80         parser.set('global', 'help_with_testing', 'False')
  81         parser.set('global', 'freshness', str(60 * 60 * 24 * 30))       # One month
  82         parser.set('global', 'network_use', 'full')
  83
  84         path = basedir.load_first_config(config_site, config_prog, 'global')
  85         if path:
  86                 info("Loading configuration from %s", path)
  87                 try:
  88                         parser.read(path)
  89                 except Exception, ex:
  90                         warn(_("Error loading config: %s"), str(ex) or repr(ex))
  91
  92         config.help_with_testing = parser.getboolean('global', 'help_with_testing')
  93         config.network_use = parser.get('global', 'network_use')
  94         config.freshness = int(parser.get('global', 'freshness'))
  95
  96         assert config.network_use in network_levels, config.network_use
  97
  98         return config
  99
 100 class Policy(object):
 101         """Chooses a set of implementations based on a policy.
 102         Typical use:
 103          1. Create a Policy object, giving it the URI of the program to be run and a handler.
 104          2. Call L{solve_with_downloads}. If more information is needed, a L{fetch.Fetcher} will be used to download it.
 105          3. When all downloads are complete, the L{solver} contains the chosen versions.
 106          4. Use L{get_uncached_implementations} to find where to get these versions and download them
 107             using L{download_uncached_implementations}.
 108
 109         @ivar target_arch: target architecture for binaries
 110         @type target_arch: L{arch.Architecture}
 111         @ivar root: URI of the root interface
 112         @ivar solver: solver used to choose a set of implementations
 113         @type solver: L{solve.Solver}
 114         @ivar watchers: callbacks to invoke after recalculating
 115         @ivar help_with_testing: default stability policy
 116         @type help_with_testing: bool
 117         @ivar network_use: one of the model.network_* values
 118         @ivar freshness: seconds allowed since last update
 119         @type freshness: int
 120         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
 121         @type stale_feeds: set
 122         """
 123         __slots__ = ['root', 'watchers', 'requirements', 'config', '_warned_offline',
 124                      'command', 'target_arch',
 125                      'stale_feeds', 'solver']
 126
 127         help_with_testing = property(lambda self: self.config.help_with_testing,
 128                                      lambda self, value: setattr(self.config, 'help_with_testing', bool(value)))
 129
 130         network_use = property(lambda self: self.config.network_use,
 131                                lambda self, value: setattr(self.config, 'network_use', value))
 132
 133         freshness = property(lambda self: self.config.freshness,
 134                              lambda self, value: setattr(self.config, 'freshness', str(value)))
 135
 136         implementation = property(lambda self: self.solver.selections)
 137
 138         ready = property(lambda self: self.solver.ready)
 139
 140         # (used by 0test)
 141         handler = property(lambda self: self.config.handler,
 142                            lambda self, value: setattr(self.config, 'handler', value))
 143
 144
 145         def __init__(self, root = None, handler = None, src = None, command = -1, config = None, requirements = None):
 146                 """
 147                 @param requirements: Details about the program we want to run
 148                 @type requirements: L{requirements.Requirements}
 149                 @param config: The configuration settings to use, or None to load from disk.
 150                 @type config: L{ConfigParser.ConfigParser}
 151                 Note: all other arguments are deprecated (since 0launch 0.52)
 152                 """
 153                 self.watchers = []
 154                 if requirements is None:
 155                         from zeroinstall.injector.requirements import Requirements
 156                         requirements = Requirements(root)
 157                         requirements.source = bool(src)                         # Root impl must be a "src" machine type
 158                         if command == -1:
 159                                 if src:
 160                                         command = 'compile'
 161                                 else:
 162                                         command = 'run'
 163                         requirements.command = command
 164                         self.target_arch = arch.get_host_architecture()
 165                 else:
 166                         assert root == src == None
 167                         assert command == -1
 168                         self.target_arch = arch.get_architecture(requirements.os, requirements.cpu)
 169                 self.requirements = requirements
 170
 171                 self.stale_feeds = set()
 172
 173                 if config is None:
 174                         self.config = load_config(handler or Handler())
 175                 else:
 176                         assert handler is None, "can't pass a handler and a config"
 177                         self.config = config
 178
 179                 from zeroinstall.injector.solver import DefaultSolver
 180                 self.solver = DefaultSolver(self.config)
 181
 182                 # If we need to download something but can't because we are offline,
 183                 # warn the user. But only the first time.
 184                 self._warned_offline = False
 185
 186                 debug(_("Supported systems: '%s'"), arch.os_ranks)
 187                 debug(_("Supported processors: '%s'"), arch.machine_ranks)
 188
 189                 if requirements.before or requirements.not_before:
 190                         self.solver.extra_restrictions[config.iface_cache.get_interface(requirements.interface_uri)] = [
 191                                         model.VersionRangeRestriction(model.parse_version(requirements.before),
 192                                                                       model.parse_version(requirements.not_before))]
 193
 194         @property
 195         def fetcher(self):
 196                 return self.config.fetcher
 197
 198         def save_config(self):
 199                 self.config.save_globals()
 200
 201         def recalculate(self, fetch_stale_interfaces = True):
 202                 """@deprecated: see L{solve_with_downloads} """
 203                 import warnings
 204                 warnings.warn("Policy.recalculate is deprecated!", DeprecationWarning, stacklevel = 2)
 205
 206                 self.stale_feeds = set()
 207
 208                 host_arch = self.target_arch
 209                 if self.requirements.source:
 210                         host_arch = arch.SourceArchitecture(host_arch)
 211                 self.solver.solve(self.root, host_arch, command_name = self.command)
 212
 213                 if self.network_use == network_offline:
 214                         fetch_stale_interfaces = False
 215
 216                 blockers = []
 217                 for f in self.solver.feeds_used:
 218                         if os.path.isabs(f): continue
 219                         feed = self.config.iface_cache.get_feed(f)
 220                         if feed is None or feed.last_modified is None:
 221                                 self.download_and_import_feed_if_online(f)      # Will start a download
 222                         elif self.is_stale(feed):
 223                                 debug(_("Adding %s to stale set"), f)
 224                                 self.stale_feeds.add(self.config.iface_cache.get_interface(f))  # Legacy API
 225                                 if fetch_stale_interfaces:
 226                                         self.download_and_import_feed_if_online(f)      # Will start a download
 227
 228                 for w in self.watchers: w()
 229
 230                 return blockers
 231
 232         def usable_feeds(self, iface):
 233                 """Generator for C{iface.feeds} that are valid for our architecture.
 234                 @rtype: generator
 235                 @see: L{arch}"""
 236                 if self.requirements.source and iface.uri == self.root:
 237                         # Note: when feeds are recursive, we'll need a better test for root here
 238                         machine_ranks = {'src': 1}
 239                 else:
 240                         machine_ranks = arch.machine_ranks
 241
 242                 for f in self.config.iface_cache.get_feed_imports(iface):
 243                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 244                                 yield f
 245                         else:
 246                                 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
 247                                         {'feed': f, 'os': f.os, 'machine': f.machine})
 248
 249         def is_stale(self, feed):
 250                 """Check whether feed needs updating, based on the configured L{freshness}.
 251                 None is considered to be stale.
 252                 @return: true if feed is stale or missing."""
 253                 if feed is None:
 254                         return True
 255                 if os.path.isabs(feed.url):
 256                         return False            # Local feeds are never stale
 257                 if feed.last_modified is None:
 258                         return True             # Don't even have it yet
 259                 now = time.time()
 260                 staleness = now - (feed.last_checked or 0)
 261                 debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0})
 262
 263                 if self.freshness <= 0 or staleness < self.freshness:
 264                         return False            # Fresh enough for us
 265
 266                 last_check_attempt = self.config.iface_cache.get_last_check_attempt(feed.url)
 267                 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 268                         debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt))
 269                         return False
 270
 271                 return True
 272
 273         def download_and_import_feed_if_online(self, feed_url):
 274                 """If we're online, call L{fetch.Fetcher.download_and_import_feed}. Otherwise, log a suitable warning."""
 275                 if self.network_use != network_offline:
 276                         debug(_("Feed %s not cached and not off-line. Downloading..."), feed_url)
 277                         return self.fetcher.download_and_import_feed(feed_url, self.config.iface_cache)
 278                 else:
 279                         if self._warned_offline:
 280                                 debug(_("Not downloading feed '%s' because we are off-line."), feed_url)
 281                         else:
 282                                 warn(_("Not downloading feed '%s' because we are in off-line mode."), feed_url)
 283                                 self._warned_offline = True
 284
 285         def get_implementation_path(self, impl):
 286                 """Return the local path of impl.
 287                 @rtype: str
 288                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 289                 assert isinstance(impl, Implementation)
 290                 return impl.local_path or self.config.stores.lookup_any(impl.digests)
 291
 292         def get_implementation(self, interface):
 293                 """Get the chosen implementation.
 294                 @type interface: Interface
 295                 @rtype: L{model.Implementation}
 296                 @raise SafeException: if interface has not been fetched or no implementation could be
 297                 chosen."""
 298                 assert isinstance(interface, Interface)
 299
 300                 try:
 301                         return self.implementation[interface]
 302                 except KeyError:
 303                         raise SafeException(_("No usable implementation found for '%s'.") % interface.uri)
 304
 305         def get_cached(self, impl):
 306                 """Check whether an implementation is available locally.
 307                 @type impl: model.Implementation
 308                 @rtype: bool
 309                 """
 310                 if isinstance(impl, DistributionImplementation):
 311                         return impl.installed
 312                 if impl.local_path:
 313                         return os.path.exists(impl.local_path)
 314                 else:
 315                         try:
 316                                 path = self.get_implementation_path(impl)
 317                                 assert path
 318                                 return True
 319                         except:
 320                                 pass # OK
 321                 return False
 322
 323         def get_uncached_implementations(self):
 324                 """List all chosen implementations which aren't yet available locally.
 325                 @rtype: [(L{model.Interface}, L{model.Implementation})]"""
 326                 iface_cache = self.config.iface_cache
 327                 uncached = []
 328                 for uri, selection in self.solver.selections.selections.iteritems():
 329                         impl = selection.impl
 330                         assert impl, self.solver.selections
 331                         if not self.get_cached(impl):
 332                                 uncached.append((iface_cache.get_interface(uri), impl))
 333                 return uncached
 334
 335         def refresh_all(self, force = True):
 336                 """Start downloading all feeds for all selected interfaces.
 337                 @param force: Whether to restart existing downloads."""
 338                 return self.solve_with_downloads(force = True)
 339
 340         def get_feed_targets(self, feed):
 341                 """Return a list of Interfaces for which feed can be a feed.
 342                 This is used by B{0install add-feed}.
 343                 @param feed: the feed
 344                 @type feed: L{model.ZeroInstallFeed} (or, deprecated, a URL)
 345                 @rtype: [model.Interface]
 346                 @raise SafeException: If there are no known feeds."""
 347
 348                 if not isinstance(feed, model.ZeroInstallFeed):
 349                         # (deprecated)
 350                         feed = self.config.iface_cache.get_feed(feed)
 351                         if feed is None:
 352                                 raise SafeException("Feed is not cached and using deprecated API")
 353
 354                 if not feed.feed_for:
 355                         raise SafeException(_("Missing <feed-for> element in '%s'; "
 356                                         "it can't be used as a feed for any other interface.") % feed.url)
 357                 feed_targets = feed.feed_for
 358                 debug(_("Feed targets: %s"), feed_targets)
 359                 return [self.config.iface_cache.get_interface(uri) for uri in feed_targets]
 360
 361         @tasks.async
 362         def solve_with_downloads(self, force = False, update_local = False):
 363                 """Run the solver, then download any feeds that are missing or
 364                 that need to be updated. Each time a new feed is imported into
 365                 the cache, the solver is run again, possibly adding new downloads.
 366                 @param force: whether to download even if we're already ready to run.
 367                 @param update_local: fetch PackageKit feeds even if we're ready to run."""
 368
 369                 downloads_finished = set()              # Successful or otherwise
 370                 downloads_in_progress = {}              # URL -> Download
 371
 372                 host_arch = self.target_arch
 373                 if self.requirements.source:
 374                         host_arch = arch.SourceArchitecture(host_arch)
 375
 376                 # There are three cases:
 377                 # 1. We want to run immediately if possible. If not, download all the information we can.
 378                 #    (force = False, update_local = False)
 379                 # 2. We're in no hurry, but don't want to use the network unnecessarily.
 380                 #    We should still update local information (from PackageKit).
 381                 #    (force = False, update_local = True)
 382                 # 3. The user explicitly asked us to refresh everything.
 383                 #    (force = True)
 384
 385                 try_quick_exit = not (force or update_local)
 386
 387                 while True:
 388                         self.solver.solve(self.root, host_arch, command_name = self.command)
 389                         for w in self.watchers: w()
 390
 391                         if try_quick_exit and self.solver.ready:
 392                                 break
 393                         try_quick_exit = False
 394
 395                         if not self.solver.ready:
 396                                 force = True
 397
 398                         for f in self.solver.feeds_used:
 399                                 if f in downloads_finished or f in downloads_in_progress:
 400                                         continue
 401                                 if os.path.isabs(f):
 402                                         if force:
 403                                                 self.config.iface_cache.get_feed(f, force = True)
 404                                                 downloads_in_progress[f] = tasks.IdleBlocker('Refresh local feed')
 405                                         continue
 406                                 elif f.startswith('distribution:'):
 407                                         if force or update_local:
 408                                                 downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, self.config.iface_cache)
 409                                 elif force and self.network_use != network_offline:
 410                                         downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, self.config.iface_cache)
 411                                         # Once we've starting downloading some things,
 412                                         # we might as well get them all.
 413                                         force = True
 414
 415                         if not downloads_in_progress:
 416                                 if self.network_use == network_offline:
 417                                         info(_("Can't choose versions and in off-line mode, so aborting"))
 418                                 break
 419
 420                         # Wait for at least one download to finish
 421                         blockers = downloads_in_progress.values()
 422                         yield blockers
 423                         tasks.check(blockers, self.handler.report_error)
 424
 425                         for f in downloads_in_progress.keys():
 426                                 if f in downloads_in_progress and downloads_in_progress[f].happened:
 427                                         del downloads_in_progress[f]
 428                                         downloads_finished.add(f)
 429
 430                                         # Need to refetch any "distribution" feed that
 431                                         # depends on this one
 432                                         distro_feed_url = 'distribution:' + f
 433                                         if distro_feed_url in downloads_finished:
 434                                                 downloads_finished.remove(distro_feed_url)
 435                                         if distro_feed_url in downloads_in_progress:
 436                                                 del downloads_in_progress[distro_feed_url]
 437
 438         @tasks.async
 439         def solve_and_download_impls(self, refresh = False, select_only = False):
 440                 """Run L{solve_with_downloads} and then get the selected implementations too.
 441                 @raise SafeException: if we couldn't select a set of implementations
 442                 @since: 0.40"""
 443                 refreshed = self.solve_with_downloads(refresh)
 444                 if refreshed:
 445                         yield refreshed
 446                         tasks.check(refreshed)
 447
 448                 if not self.solver.ready:
 449                         raise self.solver.get_failure_reason()
 450
 451                 if not select_only:
 452                         downloaded = self.download_uncached_implementations()
 453                         if downloaded:
 454                                 yield downloaded
 455                                 tasks.check(downloaded)
 456
 457         def need_download(self):
 458                 """Decide whether we need to download anything (but don't do it!)
 459                 @return: true if we MUST download something (feeds or implementations)
 460                 @rtype: bool"""
 461                 host_arch = self.target_arch
 462                 if self.requirements.source:
 463                         host_arch = arch.SourceArchitecture(host_arch)
 464                 self.solver.solve(self.root, host_arch, command_name = self.command)
 465                 for w in self.watchers: w()
 466
 467                 if not self.solver.ready:
 468                         return True             # Maybe a newer version will work?
 469
 470                 if self.get_uncached_implementations():
 471                         return True
 472
 473                 return False
 474
 475         def download_uncached_implementations(self):
 476                 """Download all implementations chosen by the solver that are missing from the cache."""
 477                 assert self.solver.ready, "Solver is not ready!\n%s" % self.solver.selections
 478                 return self.fetcher.download_impls([impl for impl in self.solver.selections.values() if not self.get_cached(impl)],
 479                                                    self.config.stores)
 480
 481         def download_icon(self, interface, force = False):
 482                 """Download an icon for this interface and add it to the
 483                 icon cache. If the interface has no icon or we are offline, do nothing.
 484                 @return: the task doing the import, or None
 485                 @rtype: L{tasks.Task}"""
 486                 if self.network_use == network_offline:
 487                         info("Not downloading icon for %s as we are off-line", interface)
 488                         return
 489
 490                 modification_time = None
 491
 492                 existing_icon = self.config.iface_cache.get_icon_path(interface)
 493                 if existing_icon:
 494                         file_mtime = os.stat(existing_icon).st_mtime
 495                         from email.utils import formatdate
 496                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 497
 498                 return self.fetcher.download_icon(interface, force, modification_time)
 499
 500         def get_interface(self, uri):
 501                 """@deprecated: use L{iface_cache.IfaceCache.get_interface} instead"""
 502                 import warnings
 503                 warnings.warn("Policy.get_interface is deprecated!", DeprecationWarning, stacklevel = 2)
 504                 return self.config.iface_cache.get_interface(uri)
 505
 506         @property
 507         def command(self):
 508                 return self.requirements.command
 509
 510         @property
 511         def root(self):
 512                 return self.requirements.interface_uri
 513
 514 _config = None
 515 def get_deprecated_singleton_config():
 516         global _config
 517         if _config is None:
 518                 _config = load_config(Handler())
 519         return _config