zeroinstall/injector/policy.py

   1 """
   2 This class brings together a L{solve.Solver} to choose a set of implmentations, a
   3 L{fetch.Fetcher} to download additional components, and the user's configuration
   4 settings.
   5 """
   6
   7 # Copyright (C) 2009, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 from zeroinstall import _
  11 import time
  12 import os
  13 from logging import info, debug, warn
  14 import ConfigParser
  15
  16 from zeroinstall import zerostore, SafeException
  17 from zeroinstall.injector import arch
  18 from zeroinstall.injector.model import Interface, Implementation, network_levels, network_offline, DistributionImplementation, network_full
  19 from zeroinstall.injector.namespaces import config_site, config_prog
  20 from zeroinstall.support import tasks, basedir
  21
  22 # If we started a check within this period, don't start another one:
  23 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  24
  25 class Config(object):
  26         """
  27         @ivar handler: handler for main-loop integration
  28         @type handler: L{handler.Handler}
  29         """
  30
  31         __slots__ = ['help_with_testing', 'freshness', 'network_use', '_fetcher', '_stores', '_iface_cache', 'handler']
  32         def __init__(self, handler):
  33                 self.help_with_testing = False
  34                 self.freshness = 60 * 60 * 24 * 30
  35                 self.network_use = network_full
  36                 self.handler = handler
  37                 self._fetcher = self._stores = self._iface_cache = None
  38
  39         @property
  40         def stores(self):
  41                 if not self._stores:
  42                         self._stores = zerostore.Stores()
  43                 return self._stores
  44
  45         @property
  46         def iface_cache(self):
  47                 if not self._iface_cache:
  48                         from zeroinstall.injector import iface_cache
  49                         self._iface_cache = iface_cache.IfaceCache()
  50                 return self._iface_cache
  51
  52         @property
  53         def fetcher(self):
  54                 if not self._fetcher:
  55                         from zeroinstall.injector import fetch
  56                         self._fetcher = fetch.Fetcher(self.handler)
  57                 return self._fetcher
  58
  59         def save_globals(self):
  60                """Write global settings."""
  61                parser = ConfigParser.ConfigParser()
  62                parser.add_section('global')
  63
  64                parser.set('global', 'help_with_testing', self.help_with_testing)
  65                parser.set('global', 'network_use', self.network_use)
  66                parser.set('global', 'freshness', self.freshness)
  67
  68                path = basedir.save_config_path(config_site, config_prog)
  69                path = os.path.join(path, 'global')
  70                parser.write(file(path + '.new', 'w'))
  71                os.rename(path + '.new', path)
  72
  73 def load_config(handler):
  74         config = Config(handler)
  75         parser = ConfigParser.RawConfigParser()
  76         parser.add_section('global')
  77         parser.set('global', 'help_with_testing', 'False')
  78         parser.set('global', 'freshness', str(60 * 60 * 24 * 30))       # One month
  79         parser.set('global', 'network_use', 'full')
  80
  81         path = basedir.load_first_config(config_site, config_prog, 'global')
  82         if path:
  83                 info("Loading configuration from %s", path)
  84                 try:
  85                         parser.read(path)
  86                 except Exception, ex:
  87                         warn(_("Error loading config: %s"), str(ex) or repr(ex))
  88
  89         config.help_with_testing = parser.getboolean('global', 'help_with_testing')
  90         config.network_use = parser.get('global', 'network_use')
  91         config.freshness = int(parser.get('global', 'freshness'))
  92
  93         return config
  94
  95 class Policy(object):
  96         """Chooses a set of implementations based on a policy.
  97         Typical use:
  98          1. Create a Policy object, giving it the URI of the program to be run and a handler.
  99          2. Call L{solve_with_downloads}. If more information is needed, a L{fetch.Fetcher} will be used to download it.
 100          3. When all downloads are complete, the L{solver} contains the chosen versions.
 101          4. Use L{get_uncached_implementations} to find where to get these versions and download them
 102             using L{download_uncached_implementations}.
 103
 104         @ivar target_arch: target architecture for binaries
 105         @type target_arch: L{arch.Architecture}
 106         @ivar root: URI of the root interface
 107         @ivar solver: solver used to choose a set of implementations
 108         @type solver: L{solve.Solver}
 109         @ivar watchers: callbacks to invoke after recalculating
 110         @ivar help_with_testing: default stability policy
 111         @type help_with_testing: bool
 112         @ivar network_use: one of the model.network_* values
 113         @ivar freshness: seconds allowed since last update
 114         @type freshness: int
 115         @ivar src: whether we are looking for source code
 116         @type src: bool
 117         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
 118         @type stale_feeds: set
 119         """
 120         __slots__ = ['root', 'watchers', 'command', 'config',
 121                      '_warned_offline',
 122                      'target_arch', 'src', 'stale_feeds', 'solver', '_fetcher']
 123
 124         help_with_testing = property(lambda self: self.config.help_with_testing,
 125                                      lambda self, value: setattr(self.config, 'help_with_testing', bool(value)))
 126
 127         network_use = property(lambda self: self.config.network_use,
 128                                lambda self, value: setattr(self.config, 'network_use', value))
 129
 130         freshness = property(lambda self: self.config.freshness,
 131                              lambda self, value: setattr(self.config, 'freshness', str(value)))
 132
 133         implementation = property(lambda self: self.solver.selections)
 134
 135         ready = property(lambda self: self.solver.ready)
 136
 137         def __init__(self, root, handler = None, src = False, command = -1, config = None):
 138                 """
 139                 @param root: The URI of the root interface (the program we want to run).
 140                 @param handler: A handler for main-loop integration.
 141                 @type handler: L{zeroinstall.injector.handler.Handler}
 142                 @param src: Whether we are looking for source code.
 143                 @type src: bool
 144                 @param command: The name of the command to run (e.g. 'run', 'test', 'compile', etc)
 145                 @type command: str
 146                 @param config: The configuration settings to use, or None to load from disk.
 147                 @type config: L{ConfigParser.ConfigParser}
 148                 """
 149                 self.watchers = []
 150                 self.src = src                          # Root impl must be a "src" machine type
 151                 self.stale_feeds = set()
 152                 if command == -1:
 153                         if src:
 154                                 command = 'compile'
 155                         else:
 156                                 command = 'run'
 157                 self.command = command
 158
 159                 if config is None:
 160                         self.config = load_config(handler)
 161                 else:
 162                         assert handler is None, "can't pass a handler and a config"
 163                         self.config = config
 164
 165                 from zeroinstall.injector.solver import DefaultSolver
 166                 self.solver = DefaultSolver(self.config)
 167
 168                 # If we need to download something but can't because we are offline,
 169                 # warn the user. But only the first time.
 170                 self._warned_offline = False
 171                 self._fetcher = None
 172
 173                 debug(_("Supported systems: '%s'"), arch.os_ranks)
 174                 debug(_("Supported processors: '%s'"), arch.machine_ranks)
 175
 176                 assert self.network_use in network_levels, self.network_use
 177                 self.set_root(root)
 178
 179                 self.target_arch = arch.get_host_architecture()
 180
 181         @property
 182         def fetcher(self):
 183                 return self.config.fetcher
 184
 185         @property
 186         def handler(self):
 187                 return self.config.handler
 188
 189         def set_root(self, root):
 190                 """Change the root interface URI."""
 191                 assert isinstance(root, (str, unicode))
 192                 self.root = root
 193                 for w in self.watchers: w()
 194
 195         def save_config(self):
 196                 self.config.save_globals()
 197
 198         def recalculate(self, fetch_stale_interfaces = True):
 199                 """@deprecated: see L{solve_with_downloads} """
 200                 import warnings
 201                 warnings.warn("Policy.recalculate is deprecated!", DeprecationWarning, stacklevel = 2)
 202
 203                 self.stale_feeds = set()
 204
 205                 host_arch = self.target_arch
 206                 if self.src:
 207                         host_arch = arch.SourceArchitecture(host_arch)
 208                 self.solver.solve(self.root, host_arch, command_name = self.command)
 209
 210                 if self.network_use == network_offline:
 211                         fetch_stale_interfaces = False
 212
 213                 blockers = []
 214                 for f in self.solver.feeds_used:
 215                         if os.path.isabs(f): continue
 216                         feed = self.config.iface_cache.get_feed(f)
 217                         if feed is None or feed.last_modified is None:
 218                                 self.download_and_import_feed_if_online(f)      # Will start a download
 219                         elif self.is_stale(feed):
 220                                 debug(_("Adding %s to stale set"), f)
 221                                 self.stale_feeds.add(self.config.iface_cache.get_interface(f))  # Legacy API
 222                                 if fetch_stale_interfaces:
 223                                         self.download_and_import_feed_if_online(f)      # Will start a download
 224
 225                 for w in self.watchers: w()
 226
 227                 return blockers
 228
 229         def usable_feeds(self, iface):
 230                 """Generator for C{iface.feeds} that are valid for our architecture.
 231                 @rtype: generator
 232                 @see: L{arch}"""
 233                 if self.src and iface.uri == self.root:
 234                         # Note: when feeds are recursive, we'll need a better test for root here
 235                         machine_ranks = {'src': 1}
 236                 else:
 237                         machine_ranks = arch.machine_ranks
 238
 239                 for f in self.config.iface_cache.get_feed_imports(iface):
 240                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 241                                 yield f
 242                         else:
 243                                 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
 244                                         {'feed': f, 'os': f.os, 'machine': f.machine})
 245
 246         def is_stale(self, feed):
 247                 """Check whether feed needs updating, based on the configured L{freshness}.
 248                 None is considered to be stale.
 249                 @return: true if feed is stale or missing."""
 250                 if feed is None:
 251                         return True
 252                 if os.path.isabs(feed.url):
 253                         return False            # Local feeds are never stale
 254                 if feed.last_modified is None:
 255                         return True             # Don't even have it yet
 256                 now = time.time()
 257                 staleness = now - (feed.last_checked or 0)
 258                 debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0})
 259
 260                 if self.freshness <= 0 or staleness < self.freshness:
 261                         return False            # Fresh enough for us
 262
 263                 last_check_attempt = self.config.iface_cache.get_last_check_attempt(feed.url)
 264                 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 265                         debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt))
 266                         return False
 267
 268                 return True
 269
 270         def download_and_import_feed_if_online(self, feed_url):
 271                 """If we're online, call L{fetch.Fetcher.download_and_import_feed}. Otherwise, log a suitable warning."""
 272                 if self.network_use != network_offline:
 273                         debug(_("Feed %s not cached and not off-line. Downloading..."), feed_url)
 274                         return self.fetcher.download_and_import_feed(feed_url, self.config.iface_cache)
 275                 else:
 276                         if self._warned_offline:
 277                                 debug(_("Not downloading feed '%s' because we are off-line."), feed_url)
 278                         else:
 279                                 warn(_("Not downloading feed '%s' because we are in off-line mode."), feed_url)
 280                                 self._warned_offline = True
 281
 282         def get_implementation_path(self, impl):
 283                 """Return the local path of impl.
 284                 @rtype: str
 285                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 286                 assert isinstance(impl, Implementation)
 287                 return impl.local_path or self.config.stores.lookup_any(impl.digests)
 288
 289         def get_implementation(self, interface):
 290                 """Get the chosen implementation.
 291                 @type interface: Interface
 292                 @rtype: L{model.Implementation}
 293                 @raise SafeException: if interface has not been fetched or no implementation could be
 294                 chosen."""
 295                 assert isinstance(interface, Interface)
 296
 297                 try:
 298                         return self.implementation[interface]
 299                 except KeyError, ex:
 300                         raise SafeException(_("No usable implementation found for '%s'.") % interface.uri)
 301
 302         def get_cached(self, impl):
 303                 """Check whether an implementation is available locally.
 304                 @type impl: model.Implementation
 305                 @rtype: bool
 306                 """
 307                 if isinstance(impl, DistributionImplementation):
 308                         return impl.installed
 309                 if impl.local_path:
 310                         return os.path.exists(impl.local_path)
 311                 else:
 312                         try:
 313                                 path = self.get_implementation_path(impl)
 314                                 assert path
 315                                 return True
 316                         except:
 317                                 pass # OK
 318                 return False
 319
 320         def get_uncached_implementations(self):
 321                 """List all chosen implementations which aren't yet available locally.
 322                 @rtype: [(L{model.Interface}, L{model.Implementation})]"""
 323                 uncached = []
 324                 for iface in self.solver.selections:
 325                         impl = self.solver.selections[iface]
 326                         assert impl, self.solver.selections
 327                         if not self.get_cached(impl):
 328                                 uncached.append((iface, impl))
 329                 return uncached
 330
 331         def refresh_all(self, force = True):
 332                 """Start downloading all feeds for all selected interfaces.
 333                 @param force: Whether to restart existing downloads."""
 334                 return self.solve_with_downloads(force = True)
 335
 336         def get_feed_targets(self, feed_iface_uri):
 337                 """Return a list of Interfaces for which feed_iface can be a feed.
 338                 This is used by B{0launch --feed}.
 339                 @rtype: [model.Interface]
 340                 @raise SafeException: If there are no known feeds."""
 341                 # TODO: what if it isn't cached yet?
 342                 feed_iface = self.config.iface_cache.get_interface(feed_iface_uri)
 343                 if not feed_iface.feed_for:
 344                         if not feed_iface.name:
 345                                 raise SafeException(_("Can't get feed targets for '%s'; failed to load it.") %
 346                                                 feed_iface_uri)
 347                         raise SafeException(_("Missing <feed-for> element in '%s'; "
 348                                         "it can't be used as a feed for any other interface.") % feed_iface_uri)
 349                 feed_targets = feed_iface.feed_for
 350                 debug(_("Feed targets: %s"), feed_targets)
 351                 if not feed_iface.name:
 352                         warn(_("Warning: unknown interface '%s'") % feed_iface_uri)
 353                 return [self.config.iface_cache.get_interface(uri) for uri in feed_targets]
 354
 355         @tasks.async
 356         def solve_with_downloads(self, force = False, update_local = False):
 357                 """Run the solver, then download any feeds that are missing or
 358                 that need to be updated. Each time a new feed is imported into
 359                 the cache, the solver is run again, possibly adding new downloads.
 360                 @param force: whether to download even if we're already ready to run.
 361                 @param update_local: fetch PackageKit feeds even if we're ready to run."""
 362
 363                 downloads_finished = set()              # Successful or otherwise
 364                 downloads_in_progress = {}              # URL -> Download
 365
 366                 host_arch = self.target_arch
 367                 if self.src:
 368                         host_arch = arch.SourceArchitecture(host_arch)
 369
 370                 # There are three cases:
 371                 # 1. We want to run immediately if possible. If not, download all the information we can.
 372                 #    (force = False, update_local = False)
 373                 # 2. We're in no hurry, but don't want to use the network unnecessarily.
 374                 #    We should still update local information (from PackageKit).
 375                 #    (force = False, update_local = True)
 376                 # 3. The user explicitly asked us to refresh everything.
 377                 #    (force = True)
 378
 379                 try_quick_exit = not (force or update_local)
 380
 381                 while True:
 382                         self.solver.solve(self.root, host_arch, command_name = self.command)
 383                         for w in self.watchers: w()
 384
 385                         if try_quick_exit and self.solver.ready:
 386                                 break
 387                         try_quick_exit = False
 388
 389                         if not self.solver.ready:
 390                                 force = True
 391
 392                         for f in self.solver.feeds_used:
 393                                 if f in downloads_finished or f in downloads_in_progress:
 394                                         continue
 395                                 if os.path.isabs(f):
 396                                         if force:
 397                                                 self.config.iface_cache.get_feed(f, force = True)
 398                                                 downloads_in_progress[f] = tasks.IdleBlocker('Refresh local feed')
 399                                         continue
 400                                 elif f.startswith('distribution:'):
 401                                         if force or update_local:
 402                                                 downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, self.config.iface_cache)
 403                                 elif force and self.network_use != network_offline:
 404                                         downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, self.config.iface_cache)
 405                                         # Once we've starting downloading some things,
 406                                         # we might as well get them all.
 407                                         force = True
 408
 409                         if not downloads_in_progress:
 410                                 if self.network_use == network_offline:
 411                                         info(_("Can't choose versions and in off-line mode, so aborting"))
 412                                 break
 413
 414                         # Wait for at least one download to finish
 415                         blockers = downloads_in_progress.values()
 416                         yield blockers
 417                         tasks.check(blockers, self.handler.report_error)
 418
 419                         for f in downloads_in_progress.keys():
 420                                 if f in downloads_in_progress and downloads_in_progress[f].happened:
 421                                         del downloads_in_progress[f]
 422                                         downloads_finished.add(f)
 423
 424                                         # Need to refetch any "distribution" feed that
 425                                         # depends on this one
 426                                         distro_feed_url = 'distribution:' + f
 427                                         if distro_feed_url in downloads_finished:
 428                                                 downloads_finished.remove(distro_feed_url)
 429                                         if distro_feed_url in downloads_in_progress:
 430                                                 del downloads_in_progress[distro_feed_url]
 431
 432         @tasks.async
 433         def solve_and_download_impls(self, refresh = False, select_only = False):
 434                 """Run L{solve_with_downloads} and then get the selected implementations too.
 435                 @raise SafeException: if we couldn't select a set of implementations
 436                 @since: 0.40"""
 437                 refreshed = self.solve_with_downloads(refresh)
 438                 if refreshed:
 439                         yield refreshed
 440                         tasks.check(refreshed)
 441
 442                 if not self.solver.ready:
 443                         raise self.solver.get_failure_reason()
 444
 445                 if not select_only:
 446                         downloaded = self.download_uncached_implementations()
 447                         if downloaded:
 448                                 yield downloaded
 449                                 tasks.check(downloaded)
 450
 451         def need_download(self):
 452                 """Decide whether we need to download anything (but don't do it!)
 453                 @return: true if we MUST download something (feeds or implementations)
 454                 @rtype: bool"""
 455                 host_arch = self.target_arch
 456                 if self.src:
 457                         host_arch = arch.SourceArchitecture(host_arch)
 458                 self.solver.solve(self.root, host_arch, command_name = self.command)
 459                 for w in self.watchers: w()
 460
 461                 if not self.solver.ready:
 462                         return True             # Maybe a newer version will work?
 463
 464                 if self.get_uncached_implementations():
 465                         return True
 466
 467                 return False
 468
 469         def download_uncached_implementations(self):
 470                 """Download all implementations chosen by the solver that are missing from the cache."""
 471                 assert self.solver.ready, "Solver is not ready!\n%s" % self.solver.selections
 472                 return self.fetcher.download_impls([impl for impl in self.solver.selections.values() if not self.get_cached(impl)],
 473                                                    self.config.stores)
 474
 475         def download_icon(self, interface, force = False):
 476                 """Download an icon for this interface and add it to the
 477                 icon cache. If the interface has no icon or we are offline, do nothing.
 478                 @return: the task doing the import, or None
 479                 @rtype: L{tasks.Task}"""
 480                 if self.network_use == network_offline:
 481                         info("Not downloading icon for %s as we are off-line", interface)
 482                         return
 483
 484                 modification_time = None
 485
 486                 existing_icon = self.config.iface_cache.get_icon_path(interface)
 487                 if existing_icon:
 488                         file_mtime = os.stat(existing_icon).st_mtime
 489                         from email.utils import formatdate
 490                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 491
 492                 return self.fetcher.download_icon(interface, force, modification_time)
 493
 494         def get_interface(self, uri):
 495                 """@deprecated: use L{iface_cache.IfaceCache.get_interface} instead"""
 496                 import warnings
 497                 warnings.warn("Policy.get_interface is deprecated!", DeprecationWarning, stacklevel = 2)
 498                 return self.config.iface_cache.get_interface(uri)
 499
 500 _config = None
 501 def get_deprecated_singleton_config():
 502         global _config
 503         if _config is None:
 504                 from zeroinstall.injector import handler
 505                 _config = load_config(handler.Handler())
 506         return _config