zeroinstall/injector/policy.py

   1 """
   2 This class brings together a L{solve.Solver} to choose a set of implmentations, a
   3 L{fetch.Fetcher} to download additional components, and the user's configuration
   4 settings.
   5 """
   6
   7 # Copyright (C) 2009, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 from zeroinstall import _
  11 import time
  12 import os
  13 from logging import info, debug, warn
  14 import ConfigParser
  15
  16 from zeroinstall import zerostore, SafeException
  17 from zeroinstall.injector import arch, model
  18 from zeroinstall.injector.model import Interface, Implementation, network_levels, network_offline, DistributionImplementation, network_full
  19 from zeroinstall.injector.handler import Handler
  20 from zeroinstall.injector.namespaces import config_site, config_prog
  21 from zeroinstall.support import tasks, basedir
  22
  23 # If we started a check within this period, don't start another one:
  24 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  25
  26 class Config(object):
  27         """
  28         @ivar handler: handler for main-loop integration
  29         @type handler: L{handler.Handler}
  30         """
  31
  32         __slots__ = ['help_with_testing', 'freshness', 'network_use', '_fetcher', '_stores', '_iface_cache', 'handler']
  33         def __init__(self, handler):
  34                 assert handler is not None
  35                 self.help_with_testing = False
  36                 self.freshness = 60 * 60 * 24 * 30
  37                 self.network_use = network_full
  38                 self.handler = handler
  39                 self._fetcher = self._stores = self._iface_cache = None
  40
  41         @property
  42         def stores(self):
  43                 if not self._stores:
  44                         self._stores = zerostore.Stores()
  45                 return self._stores
  46
  47         @property
  48         def iface_cache(self):
  49                 if not self._iface_cache:
  50                         from zeroinstall.injector import iface_cache
  51                         self._iface_cache = iface_cache.IfaceCache()
  52                 return self._iface_cache
  53
  54         @property
  55         def fetcher(self):
  56                 if not self._fetcher:
  57                         from zeroinstall.injector import fetch
  58                         self._fetcher = fetch.Fetcher(self.handler)
  59                 return self._fetcher
  60
  61         def save_globals(self):
  62                """Write global settings."""
  63                parser = ConfigParser.ConfigParser()
  64                parser.add_section('global')
  65
  66                parser.set('global', 'help_with_testing', self.help_with_testing)
  67                parser.set('global', 'network_use', self.network_use)
  68                parser.set('global', 'freshness', self.freshness)
  69
  70                path = basedir.save_config_path(config_site, config_prog)
  71                path = os.path.join(path, 'global')
  72                parser.write(file(path + '.new', 'w'))
  73                os.rename(path + '.new', path)
  74
  75 def load_config(handler):
  76         config = Config(handler)
  77         parser = ConfigParser.RawConfigParser()
  78         parser.add_section('global')
  79         parser.set('global', 'help_with_testing', 'False')
  80         parser.set('global', 'freshness', str(60 * 60 * 24 * 30))       # One month
  81         parser.set('global', 'network_use', 'full')
  82
  83         path = basedir.load_first_config(config_site, config_prog, 'global')
  84         if path:
  85                 info("Loading configuration from %s", path)
  86                 try:
  87                         parser.read(path)
  88                 except Exception, ex:
  89                         warn(_("Error loading config: %s"), str(ex) or repr(ex))
  90
  91         config.help_with_testing = parser.getboolean('global', 'help_with_testing')
  92         config.network_use = parser.get('global', 'network_use')
  93         config.freshness = int(parser.get('global', 'freshness'))
  94
  95         assert config.network_use in network_levels, config.network_use
  96
  97         return config
  98
  99 class Policy(object):
 100         """Chooses a set of implementations based on a policy.
 101         Typical use:
 102          1. Create a Policy object, giving it the URI of the program to be run and a handler.
 103          2. Call L{solve_with_downloads}. If more information is needed, a L{fetch.Fetcher} will be used to download it.
 104          3. When all downloads are complete, the L{solver} contains the chosen versions.
 105          4. Use L{get_uncached_implementations} to find where to get these versions and download them
 106             using L{download_uncached_implementations}.
 107
 108         @ivar target_arch: target architecture for binaries
 109         @type target_arch: L{arch.Architecture}
 110         @ivar root: URI of the root interface
 111         @ivar solver: solver used to choose a set of implementations
 112         @type solver: L{solve.Solver}
 113         @ivar watchers: callbacks to invoke after recalculating
 114         @ivar help_with_testing: default stability policy
 115         @type help_with_testing: bool
 116         @ivar network_use: one of the model.network_* values
 117         @ivar freshness: seconds allowed since last update
 118         @type freshness: int
 119         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
 120         @type stale_feeds: set
 121         """
 122         __slots__ = ['root', 'watchers', 'requirements', 'config', '_warned_offline',
 123                      'command', 'target_arch',
 124                      'stale_feeds', 'solver']
 125
 126         help_with_testing = property(lambda self: self.config.help_with_testing,
 127                                      lambda self, value: setattr(self.config, 'help_with_testing', bool(value)))
 128
 129         network_use = property(lambda self: self.config.network_use,
 130                                lambda self, value: setattr(self.config, 'network_use', value))
 131
 132         freshness = property(lambda self: self.config.freshness,
 133                              lambda self, value: setattr(self.config, 'freshness', str(value)))
 134
 135         implementation = property(lambda self: self.solver.selections)
 136
 137         ready = property(lambda self: self.solver.ready)
 138
 139         def __init__(self, root = None, handler = None, src = None, command = -1, config = None, requirements = None):
 140                 """
 141                 @param requirements: Details about the program we want to run
 142                 @type requirements: L{requirements.Requirements}
 143                 @param config: The configuration settings to use, or None to load from disk.
 144                 @type config: L{ConfigParser.ConfigParser}
 145                 Note: all other arguments are deprecated (since 0launch 0.52)
 146                 """
 147                 self.watchers = []
 148                 if requirements is None:
 149                         from zeroinstall.injector.requirements import Requirements
 150                         requirements = Requirements(root)
 151                         requirements.source = bool(src)                         # Root impl must be a "src" machine type
 152                         if command == -1:
 153                                 if src:
 154                                         command = 'compile'
 155                                 else:
 156                                         command = 'run'
 157                         requirements.command = command
 158                         self.target_arch = arch.get_host_architecture()
 159                 else:
 160                         assert root == src == None
 161                         assert command == -1
 162                         self.target_arch = arch.get_architecture(requirements.os, requirements.cpu)
 163                 self.requirements = requirements
 164
 165                 self.stale_feeds = set()
 166
 167                 if config is None:
 168                         assert False #XXX
 169                         self.config = load_config(handler or Handler())
 170                 else:
 171                         assert handler is None, "can't pass a handler and a config"
 172                         self.config = config
 173
 174                 from zeroinstall.injector.solver import DefaultSolver
 175                 self.solver = DefaultSolver(self.config)
 176
 177                 # If we need to download something but can't because we are offline,
 178                 # warn the user. But only the first time.
 179                 self._warned_offline = False
 180
 181                 debug(_("Supported systems: '%s'"), arch.os_ranks)
 182                 debug(_("Supported processors: '%s'"), arch.machine_ranks)
 183
 184                 if requirements.before or requirements.not_before:
 185                         self.solver.extra_restrictions[config.iface_cache.get_interface(requirements.interface_uri)] = [
 186                                         model.VersionRangeRestriction(model.parse_version(requirements.before),
 187                                                                       model.parse_version(requirements.not_before))]
 188
 189         @property
 190         def fetcher(self):
 191                 return self.config.fetcher
 192
 193         @property
 194         def handler(self):
 195                 return self.config.handler
 196
 197         def save_config(self):
 198                 self.config.save_globals()
 199
 200         def recalculate(self, fetch_stale_interfaces = True):
 201                 """@deprecated: see L{solve_with_downloads} """
 202                 import warnings
 203                 warnings.warn("Policy.recalculate is deprecated!", DeprecationWarning, stacklevel = 2)
 204
 205                 self.stale_feeds = set()
 206
 207                 host_arch = self.target_arch
 208                 if self.requirements.source:
 209                         host_arch = arch.SourceArchitecture(host_arch)
 210                 self.solver.solve(self.root, host_arch, command_name = self.command)
 211
 212                 if self.network_use == network_offline:
 213                         fetch_stale_interfaces = False
 214
 215                 blockers = []
 216                 for f in self.solver.feeds_used:
 217                         if os.path.isabs(f): continue
 218                         feed = self.config.iface_cache.get_feed(f)
 219                         if feed is None or feed.last_modified is None:
 220                                 self.download_and_import_feed_if_online(f)      # Will start a download
 221                         elif self.is_stale(feed):
 222                                 debug(_("Adding %s to stale set"), f)
 223                                 self.stale_feeds.add(self.config.iface_cache.get_interface(f))  # Legacy API
 224                                 if fetch_stale_interfaces:
 225                                         self.download_and_import_feed_if_online(f)      # Will start a download
 226
 227                 for w in self.watchers: w()
 228
 229                 return blockers
 230
 231         def usable_feeds(self, iface):
 232                 """Generator for C{iface.feeds} that are valid for our architecture.
 233                 @rtype: generator
 234                 @see: L{arch}"""
 235                 if self.requirements.source and iface.uri == self.root:
 236                         # Note: when feeds are recursive, we'll need a better test for root here
 237                         machine_ranks = {'src': 1}
 238                 else:
 239                         machine_ranks = arch.machine_ranks
 240
 241                 for f in self.config.iface_cache.get_feed_imports(iface):
 242                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 243                                 yield f
 244                         else:
 245                                 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
 246                                         {'feed': f, 'os': f.os, 'machine': f.machine})
 247
 248         def is_stale(self, feed):
 249                 """Check whether feed needs updating, based on the configured L{freshness}.
 250                 None is considered to be stale.
 251                 @return: true if feed is stale or missing."""
 252                 if feed is None:
 253                         return True
 254                 if os.path.isabs(feed.url):
 255                         return False            # Local feeds are never stale
 256                 if feed.last_modified is None:
 257                         return True             # Don't even have it yet
 258                 now = time.time()
 259                 staleness = now - (feed.last_checked or 0)
 260                 debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0})
 261
 262                 if self.freshness <= 0 or staleness < self.freshness:
 263                         return False            # Fresh enough for us
 264
 265                 last_check_attempt = self.config.iface_cache.get_last_check_attempt(feed.url)
 266                 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 267                         debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt))
 268                         return False
 269
 270                 return True
 271
 272         def download_and_import_feed_if_online(self, feed_url):
 273                 """If we're online, call L{fetch.Fetcher.download_and_import_feed}. Otherwise, log a suitable warning."""
 274                 if self.network_use != network_offline:
 275                         debug(_("Feed %s not cached and not off-line. Downloading..."), feed_url)
 276                         return self.fetcher.download_and_import_feed(feed_url, self.config.iface_cache)
 277                 else:
 278                         if self._warned_offline:
 279                                 debug(_("Not downloading feed '%s' because we are off-line."), feed_url)
 280                         else:
 281                                 warn(_("Not downloading feed '%s' because we are in off-line mode."), feed_url)
 282                                 self._warned_offline = True
 283
 284         def get_implementation_path(self, impl):
 285                 """Return the local path of impl.
 286                 @rtype: str
 287                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 288                 assert isinstance(impl, Implementation)
 289                 return impl.local_path or self.config.stores.lookup_any(impl.digests)
 290
 291         def get_implementation(self, interface):
 292                 """Get the chosen implementation.
 293                 @type interface: Interface
 294                 @rtype: L{model.Implementation}
 295                 @raise SafeException: if interface has not been fetched or no implementation could be
 296                 chosen."""
 297                 assert isinstance(interface, Interface)
 298
 299                 try:
 300                         return self.implementation[interface]
 301                 except KeyError:
 302                         raise SafeException(_("No usable implementation found for '%s'.") % interface.uri)
 303
 304         def get_cached(self, impl):
 305                 """Check whether an implementation is available locally.
 306                 @type impl: model.Implementation
 307                 @rtype: bool
 308                 """
 309                 if isinstance(impl, DistributionImplementation):
 310                         return impl.installed
 311                 if impl.local_path:
 312                         return os.path.exists(impl.local_path)
 313                 else:
 314                         try:
 315                                 path = self.get_implementation_path(impl)
 316                                 assert path
 317                                 return True
 318                         except:
 319                                 pass # OK
 320                 return False
 321
 322         def get_uncached_implementations(self):
 323                 """List all chosen implementations which aren't yet available locally.
 324                 @rtype: [(L{model.Interface}, L{model.Implementation})]"""
 325                 iface_cache = self.config.iface_cache
 326                 uncached = []
 327                 for uri, selection in self.solver.selections.selections.iteritems():
 328                         impl = selection.impl
 329                         assert impl, self.solver.selections
 330                         if not self.get_cached(impl):
 331                                 uncached.append((iface_cache.get_interface(uri), impl))
 332                 return uncached
 333
 334         def refresh_all(self, force = True):
 335                 """Start downloading all feeds for all selected interfaces.
 336                 @param force: Whether to restart existing downloads."""
 337                 return self.solve_with_downloads(force = True)
 338
 339         def get_feed_targets(self, feed_iface_uri):
 340                 """Return a list of Interfaces for which feed_iface can be a feed.
 341                 This is used by B{0launch --feed}.
 342                 @rtype: [model.Interface]
 343                 @raise SafeException: If there are no known feeds."""
 344                 # TODO: what if it isn't cached yet?
 345                 feed_iface = self.config.iface_cache.get_interface(feed_iface_uri)
 346                 if not feed_iface.feed_for:
 347                         if not feed_iface.name:
 348                                 raise SafeException(_("Can't get feed targets for '%s'; failed to load it.") %
 349                                                 feed_iface_uri)
 350                         raise SafeException(_("Missing <feed-for> element in '%s'; "
 351                                         "it can't be used as a feed for any other interface.") % feed_iface_uri)
 352                 feed_targets = feed_iface.feed_for
 353                 debug(_("Feed targets: %s"), feed_targets)
 354                 if not feed_iface.name:
 355                         warn(_("Warning: unknown interface '%s'") % feed_iface_uri)
 356                 return [self.config.iface_cache.get_interface(uri) for uri in feed_targets]
 357
 358         @tasks.async
 359         def solve_with_downloads(self, force = False, update_local = False):
 360                 """Run the solver, then download any feeds that are missing or
 361                 that need to be updated. Each time a new feed is imported into
 362                 the cache, the solver is run again, possibly adding new downloads.
 363                 @param force: whether to download even if we're already ready to run.
 364                 @param update_local: fetch PackageKit feeds even if we're ready to run."""
 365
 366                 downloads_finished = set()              # Successful or otherwise
 367                 downloads_in_progress = {}              # URL -> Download
 368
 369                 host_arch = self.target_arch
 370                 if self.requirements.source:
 371                         host_arch = arch.SourceArchitecture(host_arch)
 372
 373                 # There are three cases:
 374                 # 1. We want to run immediately if possible. If not, download all the information we can.
 375                 #    (force = False, update_local = False)
 376                 # 2. We're in no hurry, but don't want to use the network unnecessarily.
 377                 #    We should still update local information (from PackageKit).
 378                 #    (force = False, update_local = True)
 379                 # 3. The user explicitly asked us to refresh everything.
 380                 #    (force = True)
 381
 382                 try_quick_exit = not (force or update_local)
 383
 384                 while True:
 385                         self.solver.solve(self.root, host_arch, command_name = self.command)
 386                         for w in self.watchers: w()
 387
 388                         if try_quick_exit and self.solver.ready:
 389                                 break
 390                         try_quick_exit = False
 391
 392                         if not self.solver.ready:
 393                                 force = True
 394
 395                         for f in self.solver.feeds_used:
 396                                 if f in downloads_finished or f in downloads_in_progress:
 397                                         continue
 398                                 if os.path.isabs(f):
 399                                         if force:
 400                                                 self.config.iface_cache.get_feed(f, force = True)
 401                                                 downloads_in_progress[f] = tasks.IdleBlocker('Refresh local feed')
 402                                         continue
 403                                 elif f.startswith('distribution:'):
 404                                         if force or update_local:
 405                                                 downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, self.config.iface_cache)
 406                                 elif force and self.network_use != network_offline:
 407                                         downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, self.config.iface_cache)
 408                                         # Once we've starting downloading some things,
 409                                         # we might as well get them all.
 410                                         force = True
 411
 412                         if not downloads_in_progress:
 413                                 if self.network_use == network_offline:
 414                                         info(_("Can't choose versions and in off-line mode, so aborting"))
 415                                 break
 416
 417                         # Wait for at least one download to finish
 418                         blockers = downloads_in_progress.values()
 419                         yield blockers
 420                         tasks.check(blockers, self.handler.report_error)
 421
 422                         for f in downloads_in_progress.keys():
 423                                 if f in downloads_in_progress and downloads_in_progress[f].happened:
 424                                         del downloads_in_progress[f]
 425                                         downloads_finished.add(f)
 426
 427                                         # Need to refetch any "distribution" feed that
 428                                         # depends on this one
 429                                         distro_feed_url = 'distribution:' + f
 430                                         if distro_feed_url in downloads_finished:
 431                                                 downloads_finished.remove(distro_feed_url)
 432                                         if distro_feed_url in downloads_in_progress:
 433                                                 del downloads_in_progress[distro_feed_url]
 434
 435         @tasks.async
 436         def solve_and_download_impls(self, refresh = False, select_only = False):
 437                 """Run L{solve_with_downloads} and then get the selected implementations too.
 438                 @raise SafeException: if we couldn't select a set of implementations
 439                 @since: 0.40"""
 440                 refreshed = self.solve_with_downloads(refresh)
 441                 if refreshed:
 442                         yield refreshed
 443                         tasks.check(refreshed)
 444
 445                 if not self.solver.ready:
 446                         raise self.solver.get_failure_reason()
 447
 448                 if not select_only:
 449                         downloaded = self.download_uncached_implementations()
 450                         if downloaded:
 451                                 yield downloaded
 452                                 tasks.check(downloaded)
 453
 454         def need_download(self):
 455                 """Decide whether we need to download anything (but don't do it!)
 456                 @return: true if we MUST download something (feeds or implementations)
 457                 @rtype: bool"""
 458                 host_arch = self.target_arch
 459                 if self.requirements.source:
 460                         host_arch = arch.SourceArchitecture(host_arch)
 461                 self.solver.solve(self.root, host_arch, command_name = self.command)
 462                 for w in self.watchers: w()
 463
 464                 if not self.solver.ready:
 465                         return True             # Maybe a newer version will work?
 466
 467                 if self.get_uncached_implementations():
 468                         return True
 469
 470                 return False
 471
 472         def download_uncached_implementations(self):
 473                 """Download all implementations chosen by the solver that are missing from the cache."""
 474                 assert self.solver.ready, "Solver is not ready!\n%s" % self.solver.selections
 475                 return self.fetcher.download_impls([impl for impl in self.solver.selections.values() if not self.get_cached(impl)],
 476                                                    self.config.stores)
 477
 478         def download_icon(self, interface, force = False):
 479                 """Download an icon for this interface and add it to the
 480                 icon cache. If the interface has no icon or we are offline, do nothing.
 481                 @return: the task doing the import, or None
 482                 @rtype: L{tasks.Task}"""
 483                 if self.network_use == network_offline:
 484                         info("Not downloading icon for %s as we are off-line", interface)
 485                         return
 486
 487                 modification_time = None
 488
 489                 existing_icon = self.config.iface_cache.get_icon_path(interface)
 490                 if existing_icon:
 491                         file_mtime = os.stat(existing_icon).st_mtime
 492                         from email.utils import formatdate
 493                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 494
 495                 return self.fetcher.download_icon(interface, force, modification_time)
 496
 497         def get_interface(self, uri):
 498                 """@deprecated: use L{iface_cache.IfaceCache.get_interface} instead"""
 499                 import warnings
 500                 warnings.warn("Policy.get_interface is deprecated!", DeprecationWarning, stacklevel = 2)
 501                 return self.config.iface_cache.get_interface(uri)
 502
 503         @property
 504         def command(self):
 505                 return self.requirements.command
 506
 507         @property
 508         def root(self):
 509                 return self.requirements.interface_uri
 510
 511 _config = None
 512 def get_deprecated_singleton_config():
 513         global _config
 514         if _config is None:
 515                 _config = load_config(Handler())
 516         return _config