zeroinstall/injector/policy.py

   1 """
   2 This class brings together a L{solve.Solver} to choose a set of implmentations, a
   3 L{fetch.Fetcher} to download additional components, and the user's configuration
   4 settings.
   5 """
   6
   7 # Copyright (C) 2009, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 from zeroinstall import _
  11 import time
  12 import os
  13 from logging import info, debug, warn
  14 import ConfigParser
  15
  16 from zeroinstall import zerostore, SafeException
  17 from zeroinstall.injector import arch, model, driver
  18 from zeroinstall.injector.model import Interface, Implementation, network_levels, network_offline, DistributionImplementation, network_full
  19 from zeroinstall.injector.handler import Handler
  20 from zeroinstall.injector.namespaces import config_site, config_prog
  21 from zeroinstall.support import tasks, basedir
  22
  23 # If we started a check within this period, don't start another one:
  24 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  25
  26 class Config(object):
  27         """
  28         @ivar handler: handler for main-loop integration
  29         @type handler: L{handler.Handler}
  30         """
  31
  32         __slots__ = ['help_with_testing', 'freshness', 'network_use', '_fetcher', '_stores', '_iface_cache', 'handler', 'driver_factory']
  33         def __init__(self, handler):
  34                 assert handler is not None
  35                 self.help_with_testing = False
  36                 self.freshness = 60 * 60 * 24 * 30
  37                 self.network_use = network_full
  38                 self.handler = handler
  39                 self._fetcher = self._stores = self._iface_cache = None
  40                 self.driver_factory = driver.DriverFactory(settings = self, iface_cache = self.iface_cache, stores = self.stores, user_interface = handler)
  41
  42         @property
  43         def stores(self):
  44                 if not self._stores:
  45                         self._stores = zerostore.Stores()
  46                 return self._stores
  47
  48         @property
  49         def iface_cache(self):
  50                 if not self._iface_cache:
  51                         from zeroinstall.injector import iface_cache
  52                         self._iface_cache = iface_cache.iface_cache
  53                         #self._iface_cache = iface_cache.IfaceCache()
  54                 return self._iface_cache
  55
  56         @property
  57         def fetcher(self):
  58                 if not self._fetcher:
  59                         from zeroinstall.injector import fetch
  60                         self._fetcher = fetch.Fetcher(self.handler)
  61                 return self._fetcher
  62
  63         def save_globals(self):
  64                """Write global settings."""
  65                parser = ConfigParser.ConfigParser()
  66                parser.add_section('global')
  67
  68                parser.set('global', 'help_with_testing', self.help_with_testing)
  69                parser.set('global', 'network_use', self.network_use)
  70                parser.set('global', 'freshness', self.freshness)
  71
  72                path = basedir.save_config_path(config_site, config_prog)
  73                path = os.path.join(path, 'global')
  74                parser.write(file(path + '.new', 'w'))
  75                os.rename(path + '.new', path)
  76
  77 def load_config(handler):
  78         config = Config(handler)
  79         parser = ConfigParser.RawConfigParser()
  80         parser.add_section('global')
  81         parser.set('global', 'help_with_testing', 'False')
  82         parser.set('global', 'freshness', str(60 * 60 * 24 * 30))       # One month
  83         parser.set('global', 'network_use', 'full')
  84
  85         path = basedir.load_first_config(config_site, config_prog, 'global')
  86         if path:
  87                 info("Loading configuration from %s", path)
  88                 try:
  89                         parser.read(path)
  90                 except Exception, ex:
  91                         warn(_("Error loading config: %s"), str(ex) or repr(ex))
  92
  93         config.help_with_testing = parser.getboolean('global', 'help_with_testing')
  94         config.network_use = parser.get('global', 'network_use')
  95         config.freshness = int(parser.get('global', 'freshness'))
  96
  97         assert config.network_use in network_levels, config.network_use
  98
  99         return config
 100
 101 class Policy(object):
 102         """Chooses a set of implementations based on a policy.
 103         Typical use:
 104          1. Create a Policy object, giving it the URI of the program to be run and a handler.
 105          2. Call L{solve_with_downloads}. If more information is needed, a L{fetch.Fetcher} will be used to download it.
 106          3. When all downloads are complete, the L{solver} contains the chosen versions.
 107          4. Use L{get_uncached_implementations} to find where to get these versions and download them
 108             using L{download_uncached_implementations}.
 109
 110         @ivar target_arch: target architecture for binaries
 111         @type target_arch: L{arch.Architecture}
 112         @ivar root: URI of the root interface
 113         @ivar solver: solver used to choose a set of implementations
 114         @type solver: L{solve.Solver}
 115         @ivar watchers: callbacks to invoke after recalculating
 116         @ivar help_with_testing: default stability policy
 117         @type help_with_testing: bool
 118         @ivar network_use: one of the model.network_* values
 119         @ivar freshness: seconds allowed since last update
 120         @type freshness: int
 121         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
 122         @type stale_feeds: set
 123         """
 124         __slots__ = ['root', 'watchers', 'config', '_warned_offline',
 125                      'target_arch', 'driver',
 126                      'stale_feeds', 'solver']
 127
 128         help_with_testing = property(lambda self: self.config.help_with_testing,
 129                                      lambda self, value: setattr(self.config, 'help_with_testing', bool(value)))
 130
 131         network_use = property(lambda self: self.config.network_use,
 132                                lambda self, value: setattr(self.config, 'network_use', value))
 133
 134         freshness = property(lambda self: self.config.freshness,
 135                              lambda self, value: setattr(self.config, 'freshness', str(value)))
 136
 137         implementation = property(lambda self: self.solver.selections)
 138
 139         requirements = property(lambda self: self.driver.requirements)
 140
 141         solver = property(lambda self: self.driver.solver)
 142
 143         ready = property(lambda self: self.solver.ready)
 144
 145         # (was used by 0test)
 146         handler = property(lambda self: self.config.handler,
 147                            lambda self, value: setattr(self.config, 'handler', value))
 148
 149
 150         def __init__(self, root = None, handler = None, src = None, command = -1, config = None, requirements = None):
 151                 """
 152                 @param requirements: Details about the program we want to run
 153                 @type requirements: L{requirements.Requirements}
 154                 @param config: The configuration settings to use, or None to load from disk.
 155                 @type config: L{ConfigParser.ConfigParser}
 156                 Note: all other arguments are deprecated (since 0launch 0.52)
 157                 """
 158                 self.watchers = []
 159                 if requirements is None:
 160                         from zeroinstall.injector.requirements import Requirements
 161                         requirements = Requirements(root)
 162                         requirements.source = bool(src)                         # Root impl must be a "src" machine type
 163                         if command == -1:
 164                                 if src:
 165                                         command = 'compile'
 166                                 else:
 167                                         command = 'run'
 168                         requirements.command = command
 169                         self.target_arch = arch.get_host_architecture()
 170                 else:
 171                         assert root == src == None
 172                         assert command == -1
 173                         self.target_arch = arch.get_architecture(requirements.os, requirements.cpu)
 174
 175                 self.stale_feeds = set()
 176
 177                 if config is None:
 178                         self.config = load_config(handler or Handler())
 179                 else:
 180                         assert handler is None, "can't pass a handler and a config"
 181                         self.config = config
 182
 183                 self.driver = config.driver_factory.make_driver(requirements)
 184
 185         @property
 186         def fetcher(self):
 187                 return self.config.fetcher
 188
 189         def save_config(self):
 190                 self.config.save_globals()
 191
 192         def recalculate(self, fetch_stale_interfaces = True):
 193                 """@deprecated: see L{solve_with_downloads} """
 194                 import warnings
 195                 warnings.warn("Policy.recalculate is deprecated!", DeprecationWarning, stacklevel = 2)
 196
 197                 self.stale_feeds = set()
 198
 199                 host_arch = self.target_arch
 200                 if self.requirements.source:
 201                         host_arch = arch.SourceArchitecture(host_arch)
 202                 self.solver.solve(self.root, host_arch, command_name = self.command)
 203
 204                 if self.network_use == network_offline:
 205                         fetch_stale_interfaces = False
 206
 207                 blockers = []
 208                 for f in self.solver.feeds_used:
 209                         if os.path.isabs(f): continue
 210                         feed = self.config.iface_cache.get_feed(f)
 211                         if feed is None or feed.last_modified is None:
 212                                 self.download_and_import_feed_if_online(f)      # Will start a download
 213                         elif self.is_stale(feed):
 214                                 debug(_("Adding %s to stale set"), f)
 215                                 self.stale_feeds.add(self.config.iface_cache.get_interface(f))  # Legacy API
 216                                 if fetch_stale_interfaces:
 217                                         self.download_and_import_feed_if_online(f)      # Will start a download
 218
 219                 for w in self.watchers: w()
 220
 221                 return blockers
 222
 223         def usable_feeds(self, iface):
 224                 """Generator for C{iface.feeds} that are valid for our architecture.
 225                 @rtype: generator
 226                 @see: L{arch}"""
 227                 if self.requirements.source and iface.uri == self.root:
 228                         # Note: when feeds are recursive, we'll need a better test for root here
 229                         machine_ranks = {'src': 1}
 230                 else:
 231                         machine_ranks = arch.machine_ranks
 232
 233                 for f in self.config.iface_cache.get_feed_imports(iface):
 234                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 235                                 yield f
 236                         else:
 237                                 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
 238                                         {'feed': f, 'os': f.os, 'machine': f.machine})
 239
 240         def is_stale(self, feed):
 241                 """Check whether feed needs updating, based on the configured L{freshness}.
 242                 None is considered to be stale.
 243                 @return: true if feed is stale or missing."""
 244                 if feed is None:
 245                         return True
 246                 if os.path.isabs(feed.url):
 247                         return False            # Local feeds are never stale
 248                 if feed.last_modified is None:
 249                         return True             # Don't even have it yet
 250                 now = time.time()
 251                 staleness = now - (feed.last_checked or 0)
 252                 debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0})
 253
 254                 if self.freshness <= 0 or staleness < self.freshness:
 255                         return False            # Fresh enough for us
 256
 257                 last_check_attempt = self.config.iface_cache.get_last_check_attempt(feed.url)
 258                 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 259                         debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt))
 260                         return False
 261
 262                 return True
 263
 264         def download_and_import_feed_if_online(self, feed_url):
 265                 """If we're online, call L{fetch.Fetcher.download_and_import_feed}. Otherwise, log a suitable warning."""
 266                 if self.network_use != network_offline:
 267                         debug(_("Feed %s not cached and not off-line. Downloading..."), feed_url)
 268                         return self.fetcher.download_and_import_feed(feed_url, self.config.iface_cache)
 269                 else:
 270                         if self._warned_offline:
 271                                 debug(_("Not downloading feed '%s' because we are off-line."), feed_url)
 272                         else:
 273                                 warn(_("Not downloading feed '%s' because we are in off-line mode."), feed_url)
 274                                 self._warned_offline = True
 275
 276         def get_implementation_path(self, impl):
 277                 """Return the local path of impl.
 278                 @rtype: str
 279                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 280                 assert isinstance(impl, Implementation)
 281                 return impl.local_path or self.config.stores.lookup_any(impl.digests)
 282
 283         def get_implementation(self, interface):
 284                 """Get the chosen implementation.
 285                 @type interface: Interface
 286                 @rtype: L{model.Implementation}
 287                 @raise SafeException: if interface has not been fetched or no implementation could be
 288                 chosen."""
 289                 assert isinstance(interface, Interface)
 290
 291                 try:
 292                         return self.implementation[interface]
 293                 except KeyError:
 294                         raise SafeException(_("No usable implementation found for '%s'.") % interface.uri)
 295
 296         def get_cached(self, impl):
 297                 """Check whether an implementation is available locally.
 298                 @type impl: model.Implementation
 299                 @rtype: bool
 300                 """
 301                 if isinstance(impl, DistributionImplementation):
 302                         return impl.installed
 303                 if impl.local_path:
 304                         return os.path.exists(impl.local_path)
 305                 else:
 306                         try:
 307                                 path = self.get_implementation_path(impl)
 308                                 assert path
 309                                 return True
 310                         except:
 311                                 pass # OK
 312                 return False
 313
 314         def get_uncached_implementations(self):
 315                 """List all chosen implementations which aren't yet available locally.
 316                 @rtype: [(L{model.Interface}, L{model.Implementation})]"""
 317                 iface_cache = self.config.iface_cache
 318                 uncached = []
 319                 for uri, selection in self.solver.selections.selections.iteritems():
 320                         impl = selection.impl
 321                         assert impl, self.solver.selections
 322                         if not self.get_cached(impl):
 323                                 uncached.append((iface_cache.get_interface(uri), impl))
 324                 return uncached
 325
 326         def refresh_all(self, force = True):
 327                 """Start downloading all feeds for all selected interfaces.
 328                 @param force: Whether to restart existing downloads."""
 329                 return self.solve_with_downloads(force = True)
 330
 331         def get_feed_targets(self, feed):
 332                 """Return a list of Interfaces for which feed can be a feed.
 333                 This is used by B{0install add-feed}.
 334                 @param feed: the feed
 335                 @type feed: L{model.ZeroInstallFeed} (or, deprecated, a URL)
 336                 @rtype: [model.Interface]
 337                 @raise SafeException: If there are no known feeds."""
 338
 339                 if not isinstance(feed, model.ZeroInstallFeed):
 340                         # (deprecated)
 341                         feed = self.config.iface_cache.get_feed(feed)
 342                         if feed is None:
 343                                 raise SafeException("Feed is not cached and using deprecated API")
 344
 345                 if not feed.feed_for:
 346                         raise SafeException(_("Missing <feed-for> element in '%s'; "
 347                                         "it can't be used as a feed for any other interface.") % feed.url)
 348                 feed_targets = feed.feed_for
 349                 debug(_("Feed targets: %s"), feed_targets)
 350                 return [self.config.iface_cache.get_interface(uri) for uri in feed_targets]
 351
 352         @tasks.async
 353         def solve_with_downloads(self, force = False, update_local = False):
 354                 """Run the solver, then download any feeds that are missing or
 355                 that need to be updated. Each time a new feed is imported into
 356                 the cache, the solver is run again, possibly adding new downloads.
 357                 @param force: whether to download even if we're already ready to run.
 358                 @param update_local: fetch PackageKit feeds even if we're ready to run."""
 359
 360                 downloads_finished = set()              # Successful or otherwise
 361                 downloads_in_progress = {}              # URL -> Download
 362
 363                 host_arch = self.target_arch
 364                 if self.requirements.source:
 365                         host_arch = arch.SourceArchitecture(host_arch)
 366
 367                 # There are three cases:
 368                 # 1. We want to run immediately if possible. If not, download all the information we can.
 369                 #    (force = False, update_local = False)
 370                 # 2. We're in no hurry, but don't want to use the network unnecessarily.
 371                 #    We should still update local information (from PackageKit).
 372                 #    (force = False, update_local = True)
 373                 # 3. The user explicitly asked us to refresh everything.
 374                 #    (force = True)
 375
 376                 try_quick_exit = not (force or update_local)
 377
 378                 while True:
 379                         self.solver.solve(self.root, host_arch, command_name = self.command)
 380                         for w in self.watchers: w()
 381
 382                         if try_quick_exit and self.solver.ready:
 383                                 break
 384                         try_quick_exit = False
 385
 386                         if not self.solver.ready:
 387                                 force = True
 388
 389                         for f in self.solver.feeds_used:
 390                                 if f in downloads_finished or f in downloads_in_progress:
 391                                         continue
 392                                 if os.path.isabs(f):
 393                                         if force:
 394                                                 self.config.iface_cache.get_feed(f, force = True)
 395                                                 downloads_in_progress[f] = tasks.IdleBlocker('Refresh local feed')
 396                                         continue
 397                                 elif f.startswith('distribution:'):
 398                                         if force or update_local:
 399                                                 downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, self.config.iface_cache)
 400                                 elif force and self.network_use != network_offline:
 401                                         downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, self.config.iface_cache)
 402                                         # Once we've starting downloading some things,
 403                                         # we might as well get them all.
 404                                         force = True
 405
 406                         if not downloads_in_progress:
 407                                 if self.network_use == network_offline:
 408                                         info(_("Can't choose versions and in off-line mode, so aborting"))
 409                                 break
 410
 411                         # Wait for at least one download to finish
 412                         blockers = downloads_in_progress.values()
 413                         yield blockers
 414                         tasks.check(blockers, self.handler.report_error)
 415
 416                         for f in downloads_in_progress.keys():
 417                                 if f in downloads_in_progress and downloads_in_progress[f].happened:
 418                                         del downloads_in_progress[f]
 419                                         downloads_finished.add(f)
 420
 421                                         # Need to refetch any "distribution" feed that
 422                                         # depends on this one
 423                                         distro_feed_url = 'distribution:' + f
 424                                         if distro_feed_url in downloads_finished:
 425                                                 downloads_finished.remove(distro_feed_url)
 426                                         if distro_feed_url in downloads_in_progress:
 427                                                 del downloads_in_progress[distro_feed_url]
 428
 429         @tasks.async
 430         def solve_and_download_impls(self, refresh = False, select_only = False):
 431                 """Run L{solve_with_downloads} and then get the selected implementations too.
 432                 @raise SafeException: if we couldn't select a set of implementations
 433                 @since: 0.40"""
 434                 refreshed = self.solve_with_downloads(refresh)
 435                 if refreshed:
 436                         yield refreshed
 437                         tasks.check(refreshed)
 438
 439                 if not self.solver.ready:
 440                         raise self.solver.get_failure_reason()
 441
 442                 if not select_only:
 443                         downloaded = self.download_uncached_implementations()
 444                         if downloaded:
 445                                 yield downloaded
 446                                 tasks.check(downloaded)
 447
 448         def need_download(self):
 449                 """Decide whether we need to download anything (but don't do it!)
 450                 @return: true if we MUST download something (feeds or implementations)
 451                 @rtype: bool"""
 452                 host_arch = self.target_arch
 453                 if self.requirements.source:
 454                         host_arch = arch.SourceArchitecture(host_arch)
 455                 self.solver.solve(self.root, host_arch, command_name = self.command)
 456                 for w in self.watchers: w()
 457
 458                 if not self.solver.ready:
 459                         return True             # Maybe a newer version will work?
 460
 461                 if self.get_uncached_implementations():
 462                         return True
 463
 464                 return False
 465
 466         def download_uncached_implementations(self):
 467                 """Download all implementations chosen by the solver that are missing from the cache."""
 468                 assert self.solver.ready, "Solver is not ready!\n%s" % self.solver.selections
 469                 return self.fetcher.download_impls([impl for impl in self.solver.selections.values() if not self.get_cached(impl)],
 470                                                    self.config.stores)
 471
 472         def download_icon(self, interface, force = False):
 473                 """Download an icon for this interface and add it to the
 474                 icon cache. If the interface has no icon or we are offline, do nothing.
 475                 @return: the task doing the import, or None
 476                 @rtype: L{tasks.Task}"""
 477                 if self.network_use == network_offline:
 478                         info("Not downloading icon for %s as we are off-line", interface)
 479                         return
 480
 481                 modification_time = None
 482
 483                 existing_icon = self.config.iface_cache.get_icon_path(interface)
 484                 if existing_icon:
 485                         file_mtime = os.stat(existing_icon).st_mtime
 486                         from email.utils import formatdate
 487                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 488
 489                 return self.fetcher.download_icon(interface, force, modification_time)
 490
 491         def get_interface(self, uri):
 492                 """@deprecated: use L{iface_cache.IfaceCache.get_interface} instead"""
 493                 import warnings
 494                 warnings.warn("Policy.get_interface is deprecated!", DeprecationWarning, stacklevel = 2)
 495                 return self.config.iface_cache.get_interface(uri)
 496
 497         @property
 498         def command(self):
 499                 return self.requirements.command
 500
 501         @property
 502         def root(self):
 503                 return self.requirements.interface_uri
 504
 505 _config = None
 506 def get_deprecated_singleton_config():
 507         global _config
 508         if _config is None:
 509                 _config = load_config(Handler())
 510         return _config