zeroinstall/injector/policy.py

   1 """
   2 This class brings together a L{solve.Solver} to choose a set of implmentations, a
   3 L{fetch.Fetcher} to download additional components, and the user's configuration
   4 settings.
   5 """
   6
   7 # Copyright (C) 2009, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 from zeroinstall import _
  11 import time
  12 import os
  13 from logging import info, debug, warn
  14 import ConfigParser
  15
  16 from zeroinstall import zerostore, SafeException
  17 from zeroinstall.injector import arch
  18 from zeroinstall.injector.model import Interface, Implementation, network_levels, network_offline, DistributionImplementation, network_full
  19 from zeroinstall.injector.handler import Handler
  20 from zeroinstall.injector.namespaces import config_site, config_prog
  21 from zeroinstall.support import tasks, basedir
  22
  23 # If we started a check within this period, don't start another one:
  24 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  25
  26 class Config(object):
  27         """
  28         @ivar handler: handler for main-loop integration
  29         @type handler: L{handler.Handler}
  30         """
  31
  32         __slots__ = ['help_with_testing', 'freshness', 'network_use', '_fetcher', '_stores', '_iface_cache', 'handler']
  33         def __init__(self, handler):
  34                 assert handler is not None
  35                 self.help_with_testing = False
  36                 self.freshness = 60 * 60 * 24 * 30
  37                 self.network_use = network_full
  38                 self.handler = handler
  39                 self._fetcher = self._stores = self._iface_cache = None
  40
  41         @property
  42         def stores(self):
  43                 if not self._stores:
  44                         self._stores = zerostore.Stores()
  45                 return self._stores
  46
  47         @property
  48         def iface_cache(self):
  49                 if not self._iface_cache:
  50                         from zeroinstall.injector import iface_cache
  51                         self._iface_cache = iface_cache.IfaceCache()
  52                 return self._iface_cache
  53
  54         @property
  55         def fetcher(self):
  56                 if not self._fetcher:
  57                         from zeroinstall.injector import fetch
  58                         self._fetcher = fetch.Fetcher(self.handler)
  59                 return self._fetcher
  60
  61         def save_globals(self):
  62                """Write global settings."""
  63                parser = ConfigParser.ConfigParser()
  64                parser.add_section('global')
  65
  66                parser.set('global', 'help_with_testing', self.help_with_testing)
  67                parser.set('global', 'network_use', self.network_use)
  68                parser.set('global', 'freshness', self.freshness)
  69
  70                path = basedir.save_config_path(config_site, config_prog)
  71                path = os.path.join(path, 'global')
  72                parser.write(file(path + '.new', 'w'))
  73                os.rename(path + '.new', path)
  74
  75 def load_config(handler):
  76         config = Config(handler)
  77         parser = ConfigParser.RawConfigParser()
  78         parser.add_section('global')
  79         parser.set('global', 'help_with_testing', 'False')
  80         parser.set('global', 'freshness', str(60 * 60 * 24 * 30))       # One month
  81         parser.set('global', 'network_use', 'full')
  82
  83         path = basedir.load_first_config(config_site, config_prog, 'global')
  84         if path:
  85                 info("Loading configuration from %s", path)
  86                 try:
  87                         parser.read(path)
  88                 except Exception, ex:
  89                         warn(_("Error loading config: %s"), str(ex) or repr(ex))
  90
  91         config.help_with_testing = parser.getboolean('global', 'help_with_testing')
  92         config.network_use = parser.get('global', 'network_use')
  93         config.freshness = int(parser.get('global', 'freshness'))
  94
  95         assert config.network_use in network_levels, self.network_use
  96
  97         return config
  98
  99 class Policy(object):
 100         """Chooses a set of implementations based on a policy.
 101         Typical use:
 102          1. Create a Policy object, giving it the URI of the program to be run and a handler.
 103          2. Call L{solve_with_downloads}. If more information is needed, a L{fetch.Fetcher} will be used to download it.
 104          3. When all downloads are complete, the L{solver} contains the chosen versions.
 105          4. Use L{get_uncached_implementations} to find where to get these versions and download them
 106             using L{download_uncached_implementations}.
 107
 108         @ivar target_arch: target architecture for binaries
 109         @type target_arch: L{arch.Architecture}
 110         @ivar root: URI of the root interface
 111         @ivar solver: solver used to choose a set of implementations
 112         @type solver: L{solve.Solver}
 113         @ivar watchers: callbacks to invoke after recalculating
 114         @ivar help_with_testing: default stability policy
 115         @type help_with_testing: bool
 116         @ivar network_use: one of the model.network_* values
 117         @ivar freshness: seconds allowed since last update
 118         @type freshness: int
 119         @ivar src: whether we are looking for source code
 120         @type src: bool
 121         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
 122         @type stale_feeds: set
 123         """
 124         __slots__ = ['root', 'watchers', 'requirements', 'config', '_warned_offline',
 125                      'command', 'target_arch', 'src',
 126                      'stale_feeds', 'solver']
 127
 128         help_with_testing = property(lambda self: self.config.help_with_testing,
 129                                      lambda self, value: setattr(self.config, 'help_with_testing', bool(value)))
 130
 131         network_use = property(lambda self: self.config.network_use,
 132                                lambda self, value: setattr(self.config, 'network_use', value))
 133
 134         freshness = property(lambda self: self.config.freshness,
 135                              lambda self, value: setattr(self.config, 'freshness', str(value)))
 136
 137         implementation = property(lambda self: self.solver.selections)
 138
 139         ready = property(lambda self: self.solver.ready)
 140
 141         def __init__(self, root = None, handler = None, src = None, command = -1, config = None, requirements = None):
 142                 """
 143                 @param root: The URI of the root interface (the program we want to run).
 144                 @param handler: A handler for main-loop integration.
 145                 @type handler: L{zeroinstall.injector.handler.Handler}
 146                 @param src: Whether we are looking for source code.
 147                 @type src: bool
 148                 @param command: The name of the command to run (e.g. 'run', 'test', 'compile', etc)
 149                 @type command: str
 150                 @param config: The configuration settings to use, or None to load from disk.
 151                 @type config: L{ConfigParser.ConfigParser}
 152                 """
 153                 self.watchers = []
 154                 if requirements is None:
 155                         from zeroinstall.injector.requirements import Requirements
 156                         requirements = Requirements(root)
 157                         requirements.source = bool(src)                         # Root impl must be a "src" machine type
 158                         if command == -1:
 159                                 if src:
 160                                         command = 'compile'
 161                                 else:
 162                                         command = 'run'
 163                         requirements.command = command
 164                         self.target_arch = arch.get_host_architecture()
 165                 else:
 166                         assert root == src == None
 167                         assert command == -1
 168                         self.target_arch = arch.get_architecture(requirements.os, requirements.cpu)
 169                 self.requirements = requirements
 170
 171                 self.stale_feeds = set()
 172
 173                 if config is None:
 174                         assert False #XXX
 175                         self.config = load_config(handler or Handler())
 176                 else:
 177                         assert handler is None, "can't pass a handler and a config"
 178                         self.config = config
 179
 180                 from zeroinstall.injector.solver import DefaultSolver
 181                 self.solver = DefaultSolver(self.config)
 182
 183                 # If we need to download something but can't because we are offline,
 184                 # warn the user. But only the first time.
 185                 self._warned_offline = False
 186
 187                 debug(_("Supported systems: '%s'"), arch.os_ranks)
 188                 debug(_("Supported processors: '%s'"), arch.machine_ranks)
 189
 190         @property
 191         def fetcher(self):
 192                 return self.config.fetcher
 193
 194         @property
 195         def handler(self):
 196                 return self.config.handler
 197
 198         def save_config(self):
 199                 self.config.save_globals()
 200
 201         def recalculate(self, fetch_stale_interfaces = True):
 202                 """@deprecated: see L{solve_with_downloads} """
 203                 import warnings
 204                 warnings.warn("Policy.recalculate is deprecated!", DeprecationWarning, stacklevel = 2)
 205
 206                 self.stale_feeds = set()
 207
 208                 host_arch = self.target_arch
 209                 if self.requirements.source:
 210                         host_arch = arch.SourceArchitecture(host_arch)
 211                 self.solver.solve(self.root, host_arch, command_name = self.command)
 212
 213                 if self.network_use == network_offline:
 214                         fetch_stale_interfaces = False
 215
 216                 blockers = []
 217                 for f in self.solver.feeds_used:
 218                         if os.path.isabs(f): continue
 219                         feed = self.config.iface_cache.get_feed(f)
 220                         if feed is None or feed.last_modified is None:
 221                                 self.download_and_import_feed_if_online(f)      # Will start a download
 222                         elif self.is_stale(feed):
 223                                 debug(_("Adding %s to stale set"), f)
 224                                 self.stale_feeds.add(self.config.iface_cache.get_interface(f))  # Legacy API
 225                                 if fetch_stale_interfaces:
 226                                         self.download_and_import_feed_if_online(f)      # Will start a download
 227
 228                 for w in self.watchers: w()
 229
 230                 return blockers
 231
 232         def usable_feeds(self, iface):
 233                 """Generator for C{iface.feeds} that are valid for our architecture.
 234                 @rtype: generator
 235                 @see: L{arch}"""
 236                 if self.requirements.source and iface.uri == self.root:
 237                         # Note: when feeds are recursive, we'll need a better test for root here
 238                         machine_ranks = {'src': 1}
 239                 else:
 240                         machine_ranks = arch.machine_ranks
 241
 242                 for f in self.config.iface_cache.get_feed_imports(iface):
 243                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 244                                 yield f
 245                         else:
 246                                 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
 247                                         {'feed': f, 'os': f.os, 'machine': f.machine})
 248
 249         def is_stale(self, feed):
 250                 """Check whether feed needs updating, based on the configured L{freshness}.
 251                 None is considered to be stale.
 252                 @return: true if feed is stale or missing."""
 253                 if feed is None:
 254                         return True
 255                 if os.path.isabs(feed.url):
 256                         return False            # Local feeds are never stale
 257                 if feed.last_modified is None:
 258                         return True             # Don't even have it yet
 259                 now = time.time()
 260                 staleness = now - (feed.last_checked or 0)
 261                 debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0})
 262
 263                 if self.freshness <= 0 or staleness < self.freshness:
 264                         return False            # Fresh enough for us
 265
 266                 last_check_attempt = self.config.iface_cache.get_last_check_attempt(feed.url)
 267                 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 268                         debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt))
 269                         return False
 270
 271                 return True
 272
 273         def download_and_import_feed_if_online(self, feed_url):
 274                 """If we're online, call L{fetch.Fetcher.download_and_import_feed}. Otherwise, log a suitable warning."""
 275                 if self.network_use != network_offline:
 276                         debug(_("Feed %s not cached and not off-line. Downloading..."), feed_url)
 277                         return self.fetcher.download_and_import_feed(feed_url, self.config.iface_cache)
 278                 else:
 279                         if self._warned_offline:
 280                                 debug(_("Not downloading feed '%s' because we are off-line."), feed_url)
 281                         else:
 282                                 warn(_("Not downloading feed '%s' because we are in off-line mode."), feed_url)
 283                                 self._warned_offline = True
 284
 285         def get_implementation_path(self, impl):
 286                 """Return the local path of impl.
 287                 @rtype: str
 288                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 289                 assert isinstance(impl, Implementation)
 290                 return impl.local_path or self.config.stores.lookup_any(impl.digests)
 291
 292         def get_implementation(self, interface):
 293                 """Get the chosen implementation.
 294                 @type interface: Interface
 295                 @rtype: L{model.Implementation}
 296                 @raise SafeException: if interface has not been fetched or no implementation could be
 297                 chosen."""
 298                 assert isinstance(interface, Interface)
 299
 300                 try:
 301                         return self.implementation[interface]
 302                 except KeyError, ex:
 303                         raise SafeException(_("No usable implementation found for '%s'.") % interface.uri)
 304
 305         def get_cached(self, impl):
 306                 """Check whether an implementation is available locally.
 307                 @type impl: model.Implementation
 308                 @rtype: bool
 309                 """
 310                 if isinstance(impl, DistributionImplementation):
 311                         return impl.installed
 312                 if impl.local_path:
 313                         return os.path.exists(impl.local_path)
 314                 else:
 315                         try:
 316                                 path = self.get_implementation_path(impl)
 317                                 assert path
 318                                 return True
 319                         except:
 320                                 pass # OK
 321                 return False
 322
 323         def get_uncached_implementations(self):
 324                 """List all chosen implementations which aren't yet available locally.
 325                 @rtype: [(L{model.Interface}, L{model.Implementation})]"""
 326                 uncached = []
 327                 for iface in self.solver.selections:
 328                         impl = self.solver.selections[iface]
 329                         assert impl, self.solver.selections
 330                         if not self.get_cached(impl):
 331                                 uncached.append((iface, impl))
 332                 return uncached
 333
 334         def refresh_all(self, force = True):
 335                 """Start downloading all feeds for all selected interfaces.
 336                 @param force: Whether to restart existing downloads."""
 337                 return self.solve_with_downloads(force = True)
 338
 339         def get_feed_targets(self, feed_iface_uri):
 340                 """Return a list of Interfaces for which feed_iface can be a feed.
 341                 This is used by B{0launch --feed}.
 342                 @rtype: [model.Interface]
 343                 @raise SafeException: If there are no known feeds."""
 344                 # TODO: what if it isn't cached yet?
 345                 feed_iface = self.config.iface_cache.get_interface(feed_iface_uri)
 346                 if not feed_iface.feed_for:
 347                         if not feed_iface.name:
 348                                 raise SafeException(_("Can't get feed targets for '%s'; failed to load it.") %
 349                                                 feed_iface_uri)
 350                         raise SafeException(_("Missing <feed-for> element in '%s'; "
 351                                         "it can't be used as a feed for any other interface.") % feed_iface_uri)
 352                 feed_targets = feed_iface.feed_for
 353                 debug(_("Feed targets: %s"), feed_targets)
 354                 if not feed_iface.name:
 355                         warn(_("Warning: unknown interface '%s'") % feed_iface_uri)
 356                 return [self.config.iface_cache.get_interface(uri) for uri in feed_targets]
 357
 358         @tasks.async
 359         def solve_with_downloads(self, force = False, update_local = False):
 360                 """Run the solver, then download any feeds that are missing or
 361                 that need to be updated. Each time a new feed is imported into
 362                 the cache, the solver is run again, possibly adding new downloads.
 363                 @param force: whether to download even if we're already ready to run.
 364                 @param update_local: fetch PackageKit feeds even if we're ready to run."""
 365
 366                 downloads_finished = set()              # Successful or otherwise
 367                 downloads_in_progress = {}              # URL -> Download
 368
 369                 host_arch = self.target_arch
 370                 if self.requirements.source:
 371                         host_arch = arch.SourceArchitecture(host_arch)
 372
 373                 # There are three cases:
 374                 # 1. We want to run immediately if possible. If not, download all the information we can.
 375                 #    (force = False, update_local = False)
 376                 # 2. We're in no hurry, but don't want to use the network unnecessarily.
 377                 #    We should still update local information (from PackageKit).
 378                 #    (force = False, update_local = True)
 379                 # 3. The user explicitly asked us to refresh everything.
 380                 #    (force = True)
 381
 382                 try_quick_exit = not (force or update_local)
 383
 384                 while True:
 385                         self.solver.solve(self.root, host_arch, command_name = self.command)
 386                         for w in self.watchers: w()
 387
 388                         if try_quick_exit and self.solver.ready:
 389                                 break
 390                         try_quick_exit = False
 391
 392                         if not self.solver.ready:
 393                                 force = True
 394
 395                         for f in self.solver.feeds_used:
 396                                 if f in downloads_finished or f in downloads_in_progress:
 397                                         continue
 398                                 if os.path.isabs(f):
 399                                         if force:
 400                                                 self.config.iface_cache.get_feed(f, force = True)
 401                                                 downloads_in_progress[f] = tasks.IdleBlocker('Refresh local feed')
 402                                         continue
 403                                 elif f.startswith('distribution:'):
 404                                         if force or update_local:
 405                                                 downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, self.config.iface_cache)
 406                                 elif force and self.network_use != network_offline:
 407                                         downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, self.config.iface_cache)
 408                                         # Once we've starting downloading some things,
 409                                         # we might as well get them all.
 410                                         force = True
 411
 412                         if not downloads_in_progress:
 413                                 if self.network_use == network_offline:
 414                                         info(_("Can't choose versions and in off-line mode, so aborting"))
 415                                 break
 416
 417                         # Wait for at least one download to finish
 418                         blockers = downloads_in_progress.values()
 419                         yield blockers
 420                         tasks.check(blockers, self.handler.report_error)
 421
 422                         for f in downloads_in_progress.keys():
 423                                 if f in downloads_in_progress and downloads_in_progress[f].happened:
 424                                         del downloads_in_progress[f]
 425                                         downloads_finished.add(f)
 426
 427                                         # Need to refetch any "distribution" feed that
 428                                         # depends on this one
 429                                         distro_feed_url = 'distribution:' + f
 430                                         if distro_feed_url in downloads_finished:
 431                                                 downloads_finished.remove(distro_feed_url)
 432                                         if distro_feed_url in downloads_in_progress:
 433                                                 del downloads_in_progress[distro_feed_url]
 434
 435         @tasks.async
 436         def solve_and_download_impls(self, refresh = False, select_only = False):
 437                 """Run L{solve_with_downloads} and then get the selected implementations too.
 438                 @raise SafeException: if we couldn't select a set of implementations
 439                 @since: 0.40"""
 440                 refreshed = self.solve_with_downloads(refresh)
 441                 if refreshed:
 442                         yield refreshed
 443                         tasks.check(refreshed)
 444
 445                 if not self.solver.ready:
 446                         raise self.solver.get_failure_reason()
 447
 448                 if not select_only:
 449                         downloaded = self.download_uncached_implementations()
 450                         if downloaded:
 451                                 yield downloaded
 452                                 tasks.check(downloaded)
 453
 454         def need_download(self):
 455                 """Decide whether we need to download anything (but don't do it!)
 456                 @return: true if we MUST download something (feeds or implementations)
 457                 @rtype: bool"""
 458                 host_arch = self.target_arch
 459                 if self.requirements.source:
 460                         host_arch = arch.SourceArchitecture(host_arch)
 461                 self.solver.solve(self.root, host_arch, command_name = self.command)
 462                 for w in self.watchers: w()
 463
 464                 if not self.solver.ready:
 465                         return True             # Maybe a newer version will work?
 466
 467                 if self.get_uncached_implementations():
 468                         return True
 469
 470                 return False
 471
 472         def download_uncached_implementations(self):
 473                 """Download all implementations chosen by the solver that are missing from the cache."""
 474                 assert self.solver.ready, "Solver is not ready!\n%s" % self.solver.selections
 475                 return self.fetcher.download_impls([impl for impl in self.solver.selections.values() if not self.get_cached(impl)],
 476                                                    self.config.stores)
 477
 478         def download_icon(self, interface, force = False):
 479                 """Download an icon for this interface and add it to the
 480                 icon cache. If the interface has no icon or we are offline, do nothing.
 481                 @return: the task doing the import, or None
 482                 @rtype: L{tasks.Task}"""
 483                 if self.network_use == network_offline:
 484                         info("Not downloading icon for %s as we are off-line", interface)
 485                         return
 486
 487                 modification_time = None
 488
 489                 existing_icon = self.config.iface_cache.get_icon_path(interface)
 490                 if existing_icon:
 491                         file_mtime = os.stat(existing_icon).st_mtime
 492                         from email.utils import formatdate
 493                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 494
 495                 return self.fetcher.download_icon(interface, force, modification_time)
 496
 497         def get_interface(self, uri):
 498                 """@deprecated: use L{iface_cache.IfaceCache.get_interface} instead"""
 499                 import warnings
 500                 warnings.warn("Policy.get_interface is deprecated!", DeprecationWarning, stacklevel = 2)
 501                 return self.config.iface_cache.get_interface(uri)
 502
 503         @property
 504         def command(self):
 505                 return self.requirements.command
 506
 507         @property
 508         def root(self):
 509                 return self.requirements.interface_uri
 510
 511 _config = None
 512 def get_deprecated_singleton_config():
 513         global _config
 514         if _config is None:
 515                 _config = load_config(Handler())
 516         return _config