zeroinstall/injector/policy.py

   1 """
   2 This class brings together a L{solve.Solver} to choose a set of implmentations, a
   3 L{fetch.Fetcher} to download additional components, and the user's configuration
   4 settings.
   5 """
   6
   7 # Copyright (C) 2009, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 from zeroinstall import _
  11 import time
  12 import os
  13 from logging import info, debug, warn
  14 import ConfigParser
  15
  16 from zeroinstall import SafeException
  17 from zeroinstall.injector import arch
  18 from zeroinstall.injector.model import Interface, Implementation, network_levels, network_offline, DistributionImplementation, network_full
  19 from zeroinstall.injector.namespaces import config_site, config_prog
  20 from zeroinstall.support import tasks, basedir
  21 from zeroinstall.injector.iface_cache import iface_cache
  22
  23 # If we started a check within this period, don't start another one:
  24 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  25
  26 class Policy(object):
  27         """Chooses a set of implementations based on a policy.
  28         Typical use:
  29          1. Create a Policy object, giving it the URI of the program to be run and a handler.
  30          2. Call L{solve_with_downloads}. If more information is needed, a L{fetch.Fetcher} will be used to download it.
  31          3. When all downloads are complete, the L{solver} contains the chosen versions.
  32          4. Use L{get_uncached_implementations} to find where to get these versions and download them
  33             using L{download_uncached_implementations}.
  34
  35         @ivar target_arch: target architecture for binaries
  36         @type target_arch: L{arch.Architecture}
  37         @ivar root: URI of the root interface
  38         @ivar solver: solver used to choose a set of implementations
  39         @type solver: L{solve.Solver}
  40         @ivar watchers: callbacks to invoke after recalculating
  41         @ivar help_with_testing: default stability policy
  42         @type help_with_testing: bool
  43         @ivar network_use: one of the model.network_* values
  44         @ivar freshness: seconds allowed since last update
  45         @type freshness: int
  46         @ivar handler: handler for main-loop integration
  47         @type handler: L{handler.Handler}
  48         @ivar src: whether we are looking for source code
  49         @type src: bool
  50         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
  51         @type stale_feeds: set
  52         """
  53         __slots__ = ['root', 'watchers', 'command',
  54                      'freshness', 'handler', '_warned_offline',
  55                      'target_arch', 'src', 'stale_feeds', 'solver', '_fetcher']
  56
  57         help_with_testing = property(lambda self: self.solver.help_with_testing,
  58                                      lambda self, value: setattr(self.solver, 'help_with_testing', value))
  59
  60         network_use = property(lambda self: self.solver.network_use,
  61                                      lambda self, value: setattr(self.solver, 'network_use', value))
  62
  63         implementation = property(lambda self: self.solver.selections)
  64
  65         ready = property(lambda self: self.solver.ready)
  66
  67         def __init__(self, root, handler = None, src = False, command = None):
  68                 """
  69                 @param root: The URI of the root interface (the program we want to run).
  70                 @param handler: A handler for main-loop integration.
  71                 @type handler: L{zeroinstall.injector.handler.Handler}
  72                 @param src: Whether we are looking for source code.
  73                 @type src: bool
  74                 @param command: The name of the command to run (e.g. 'run', 'test', 'compile', etc)
  75                 @type command: str
  76                 """
  77                 self.watchers = []
  78                 self.src = src                          # Root impl must be a "src" machine type
  79                 self.stale_feeds = set()
  80                 if command is None:
  81                         if src:
  82                                 command = 'compile'
  83                         else:
  84                                 command = 'run'
  85                 self.command = command
  86
  87                 from zeroinstall.injector.solver import DefaultSolver
  88                 self.solver = DefaultSolver(network_full, iface_cache, iface_cache.stores)
  89
  90                 # If we need to download something but can't because we are offline,
  91                 # warn the user. But only the first time.
  92                 self._warned_offline = False
  93                 self._fetcher = None
  94
  95                 # (allow self for backwards compat)
  96                 self.handler = handler or self
  97
  98                 debug(_("Supported systems: '%s'"), arch.os_ranks)
  99                 debug(_("Supported processors: '%s'"), arch.machine_ranks)
 100
 101                 config = ConfigParser.ConfigParser()
 102                 config.add_section('global')
 103                 config.set('global', 'help_with_testing', 'False')
 104                 config.set('global', 'freshness', str(60 * 60 * 24 * 30))       # One month
 105                 config.set('global', 'network_use', 'full')
 106
 107                 path = basedir.load_first_config(config_site, config_prog, 'global')
 108                 if path:
 109                         info("Loading configuration from %s", path)
 110                         try:
 111                                 config.read(path)
 112                         except Exception, ex:
 113                                 warn(_("Error loading config: %s"), str(ex) or repr(ex))
 114
 115                 self.solver.help_with_testing = config.getboolean('global', 'help_with_testing')
 116                 self.solver.network_use = config.get('global', 'network_use')
 117                 self.freshness = int(config.get('global', 'freshness'))
 118                 assert self.solver.network_use in network_levels, self.solver.network_use
 119
 120                 self.set_root(root)
 121
 122                 self.target_arch = arch.get_host_architecture()
 123
 124         @property
 125         def fetcher(self):
 126                 if not self._fetcher:
 127                         import fetch
 128                         self._fetcher = fetch.Fetcher(self.handler)
 129                 return self._fetcher
 130
 131         def set_root(self, root):
 132                 """Change the root interface URI."""
 133                 assert isinstance(root, (str, unicode))
 134                 self.root = root
 135                 for w in self.watchers: w()
 136
 137         def save_config(self):
 138                 """Write global settings."""
 139                 config = ConfigParser.ConfigParser()
 140                 config.add_section('global')
 141
 142                 config.set('global', 'help_with_testing', self.help_with_testing)
 143                 config.set('global', 'network_use', self.network_use)
 144                 config.set('global', 'freshness', self.freshness)
 145
 146                 path = basedir.save_config_path(config_site, config_prog)
 147                 path = os.path.join(path, 'global')
 148                 config.write(file(path + '.new', 'w'))
 149                 os.rename(path + '.new', path)
 150
 151         def recalculate(self, fetch_stale_interfaces = True):
 152                 """@deprecated: see L{solve_with_downloads} """
 153                 import warnings
 154                 warnings.warn("Policy.recalculate is deprecated!", DeprecationWarning, stacklevel = 2)
 155
 156                 self.stale_feeds = set()
 157
 158                 host_arch = self.target_arch
 159                 if self.src:
 160                         host_arch = arch.SourceArchitecture(host_arch)
 161                 self.solver.solve(self.root, host_arch, command_name = self.command)
 162
 163                 if self.network_use == network_offline:
 164                         fetch_stale_interfaces = False
 165
 166                 blockers = []
 167                 for f in self.solver.feeds_used:
 168                         if os.path.isabs(f): continue
 169                         feed = iface_cache.get_feed(f)
 170                         if feed is None or feed.last_modified is None:
 171                                 self.download_and_import_feed_if_online(f)      # Will start a download
 172                         elif self.is_stale(feed):
 173                                 debug(_("Adding %s to stale set"), f)
 174                                 self.stale_feeds.add(iface_cache.get_interface(f))      # Legacy API
 175                                 if fetch_stale_interfaces:
 176                                         self.download_and_import_feed_if_online(f)      # Will start a download
 177
 178                 for w in self.watchers: w()
 179
 180                 return blockers
 181
 182         def usable_feeds(self, iface):
 183                 """Generator for C{iface.feeds} that are valid for our architecture.
 184                 @rtype: generator
 185                 @see: L{arch}"""
 186                 if self.src and iface.uri == self.root:
 187                         # Note: when feeds are recursive, we'll need a better test for root here
 188                         machine_ranks = {'src': 1}
 189                 else:
 190                         machine_ranks = arch.machine_ranks
 191
 192                 for f in iface_cache.get_feed_imports(iface):
 193                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 194                                 yield f
 195                         else:
 196                                 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
 197                                         {'feed': f, 'os': f.os, 'machine': f.machine})
 198
 199         def is_stale(self, feed):
 200                 """Check whether feed needs updating, based on the configured L{freshness}.
 201                 None is considered to be stale.
 202                 @return: true if feed is stale or missing."""
 203                 if feed is None:
 204                         return True
 205                 if os.path.isabs(feed.url):
 206                         return False            # Local feeds are never stale
 207                 if feed.last_modified is None:
 208                         return True             # Don't even have it yet
 209                 now = time.time()
 210                 staleness = now - (feed.last_checked or 0)
 211                 debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0})
 212
 213                 if self.freshness <= 0 or staleness < self.freshness:
 214                         return False            # Fresh enough for us
 215
 216                 last_check_attempt = iface_cache.get_last_check_attempt(feed.url)
 217                 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 218                         debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt))
 219                         return False
 220
 221                 return True
 222
 223         def download_and_import_feed_if_online(self, feed_url):
 224                 """If we're online, call L{fetch.Fetcher.download_and_import_feed}. Otherwise, log a suitable warning."""
 225                 if self.network_use != network_offline:
 226                         debug(_("Feed %s not cached and not off-line. Downloading..."), feed_url)
 227                         return self.fetcher.download_and_import_feed(feed_url, iface_cache)
 228                 else:
 229                         if self._warned_offline:
 230                                 debug(_("Not downloading feed '%s' because we are off-line."), feed_url)
 231                         else:
 232                                 warn(_("Not downloading feed '%s' because we are in off-line mode."), feed_url)
 233                                 self._warned_offline = True
 234
 235         def get_implementation_path(self, impl):
 236                 """Return the local path of impl.
 237                 @rtype: str
 238                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 239                 assert isinstance(impl, Implementation)
 240                 return impl.local_path or iface_cache.stores.lookup_any(impl.digests)
 241
 242         def get_implementation(self, interface):
 243                 """Get the chosen implementation.
 244                 @type interface: Interface
 245                 @rtype: L{model.Implementation}
 246                 @raise SafeException: if interface has not been fetched or no implementation could be
 247                 chosen."""
 248                 assert isinstance(interface, Interface)
 249
 250                 try:
 251                         return self.implementation[interface]
 252                 except KeyError, ex:
 253                         raise SafeException(_("No usable implementation found for '%s'.") % interface.uri)
 254
 255         def get_cached(self, impl):
 256                 """Check whether an implementation is available locally.
 257                 @type impl: model.Implementation
 258                 @rtype: bool
 259                 """
 260                 if isinstance(impl, DistributionImplementation):
 261                         return impl.installed
 262                 if impl.local_path:
 263                         return os.path.exists(impl.local_path)
 264                 else:
 265                         try:
 266                                 path = self.get_implementation_path(impl)
 267                                 assert path
 268                                 return True
 269                         except:
 270                                 pass # OK
 271                 return False
 272
 273         def get_uncached_implementations(self):
 274                 """List all chosen implementations which aren't yet available locally.
 275                 @rtype: [(L{model.Interface}, L{model.Implementation})]"""
 276                 uncached = []
 277                 for iface in self.solver.selections:
 278                         impl = self.solver.selections[iface]
 279                         assert impl, self.solver.selections
 280                         if not self.get_cached(impl):
 281                                 uncached.append((iface, impl))
 282                 return uncached
 283
 284         def refresh_all(self, force = True):
 285                 """Start downloading all feeds for all selected interfaces.
 286                 @param force: Whether to restart existing downloads."""
 287                 return self.solve_with_downloads(force = True)
 288
 289         def get_feed_targets(self, feed_iface_uri):
 290                 """Return a list of Interfaces for which feed_iface can be a feed.
 291                 This is used by B{0launch --feed}.
 292                 @rtype: [model.Interface]
 293                 @raise SafeException: If there are no known feeds."""
 294                 # TODO: what if it isn't cached yet?
 295                 feed_iface = iface_cache.get_interface(feed_iface_uri)
 296                 if not feed_iface.feed_for:
 297                         if not feed_iface.name:
 298                                 raise SafeException(_("Can't get feed targets for '%s'; failed to load interface.") %
 299                                                 feed_iface_uri)
 300                         raise SafeException(_("Missing <feed-for> element in '%s'; "
 301                                         "this interface can't be used as a feed.") % feed_iface_uri)
 302                 feed_targets = feed_iface.feed_for
 303                 debug(_("Feed targets: %s"), feed_targets)
 304                 if not feed_iface.name:
 305                         warn(_("Warning: unknown interface '%s'") % feed_iface_uri)
 306                 return [iface_cache.get_interface(uri) for uri in feed_targets]
 307
 308         @tasks.async
 309         def solve_with_downloads(self, force = False, update_local = False):
 310                 """Run the solver, then download any feeds that are missing or
 311                 that need to be updated. Each time a new feed is imported into
 312                 the cache, the solver is run again, possibly adding new downloads.
 313                 @param force: whether to download even if we're already ready to run.
 314                 @param update_local: fetch PackageKit feeds even if we're ready to run."""
 315
 316                 downloads_finished = set()              # Successful or otherwise
 317                 downloads_in_progress = {}              # URL -> Download
 318
 319                 host_arch = self.target_arch
 320                 if self.src:
 321                         host_arch = arch.SourceArchitecture(host_arch)
 322
 323                 # There are three cases:
 324                 # 1. We want to run immediately if possible. If not, download all the information we can.
 325                 #    (force = False, update_local = False)
 326                 # 2. We're in no hurry, but don't want to use the network unnecessarily.
 327                 #    We should still update local information (from PackageKit).
 328                 #    (force = False, update_local = True)
 329                 # 3. The user explicitly asked us to refresh everything.
 330                 #    (force = True)
 331
 332                 try_quick_exit = not (force or update_local)
 333
 334                 while True:
 335                         self.solver.solve(self.root, host_arch, command_name = self.command)
 336                         for w in self.watchers: w()
 337
 338                         if try_quick_exit and self.solver.ready:
 339                                 break
 340                         try_quick_exit = False
 341
 342                         if not self.solver.ready:
 343                                 force = True
 344
 345                         for f in self.solver.feeds_used:
 346                                 if f in downloads_finished or f in downloads_in_progress:
 347                                         continue
 348                                 if os.path.isabs(f):
 349                                         continue
 350                                 elif f.startswith('distribution:'):
 351                                         if force or update_local:
 352                                                 downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, iface_cache)
 353                                 elif force and self.network_use != network_offline:
 354                                         downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, iface_cache)
 355                                         # Once we've starting downloading some things,
 356                                         # we might as well get them all.
 357                                         force = True
 358
 359                         if not downloads_in_progress:
 360                                 if self.network_use == network_offline:
 361                                         info(_("Can't choose versions and in off-line mode, so aborting"))
 362                                 break
 363
 364                         # Wait for at least one download to finish
 365                         blockers = downloads_in_progress.values()
 366                         yield blockers
 367                         tasks.check(blockers, self.handler.report_error)
 368
 369                         for f in downloads_in_progress.keys():
 370                                 if f in downloads_in_progress and downloads_in_progress[f].happened:
 371                                         del downloads_in_progress[f]
 372                                         downloads_finished.add(f)
 373
 374                                         # Need to refetch any "distribution" feed that
 375                                         # depends on this one
 376                                         distro_feed_url = 'distribution:' + f
 377                                         if distro_feed_url in downloads_finished:
 378                                                 downloads_finished.remove(distro_feed_url)
 379                                         if distro_feed_url in downloads_in_progress:
 380                                                 del downloads_in_progress[distro_feed_url]
 381
 382         @tasks.async
 383         def solve_and_download_impls(self, refresh = False, select_only = False):
 384                 """Run L{solve_with_downloads} and then get the selected implementations too.
 385                 @raise SafeException: if we couldn't select a set of implementations
 386                 @since: 0.40"""
 387                 refreshed = self.solve_with_downloads(refresh)
 388                 if refreshed:
 389                         yield refreshed
 390                         tasks.check(refreshed)
 391
 392                 if not self.solver.ready:
 393                         raise self.solver.get_failure_reason()
 394
 395                 if not select_only:
 396                         downloaded = self.download_uncached_implementations()
 397                         if downloaded:
 398                                 yield downloaded
 399                                 tasks.check(downloaded)
 400
 401         def need_download(self):
 402                 """Decide whether we need to download anything (but don't do it!)
 403                 @return: true if we MUST download something (feeds or implementations)
 404                 @rtype: bool"""
 405                 host_arch = self.target_arch
 406                 if self.src:
 407                         host_arch = arch.SourceArchitecture(host_arch)
 408                 self.solver.solve(self.root, host_arch, command_name = self.command)
 409                 for w in self.watchers: w()
 410
 411                 if not self.solver.ready:
 412                         return True             # Maybe a newer version will work?
 413
 414                 if self.get_uncached_implementations():
 415                         return True
 416
 417                 return False
 418
 419         def download_uncached_implementations(self):
 420                 """Download all implementations chosen by the solver that are missing from the cache."""
 421                 assert self.solver.ready, "Solver is not ready!\n%s" % self.solver.selections
 422                 return self.fetcher.download_impls([impl for impl in self.solver.selections.values() if not self.get_cached(impl)],
 423                                                    iface_cache.stores)
 424
 425         def download_icon(self, interface, force = False):
 426                 """Download an icon for this interface and add it to the
 427                 icon cache. If the interface has no icon or we are offline, do nothing.
 428                 @return: the task doing the import, or None
 429                 @rtype: L{tasks.Task}"""
 430                 if self.network_use == network_offline:
 431                         info("Not downloading icon for %s as we are off-line", interface)
 432                         return
 433
 434                 modification_time = None
 435
 436                 existing_icon = iface_cache.get_icon_path(interface)
 437                 if existing_icon:
 438                         file_mtime = os.stat(existing_icon).st_mtime
 439                         from email.utils import formatdate
 440                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 441
 442                 return self.fetcher.download_icon(interface, force, modification_time)
 443
 444         def get_interface(self, uri):
 445                 """@deprecated: use L{iface_cache.IfaceCache.get_interface} instead"""
 446                 import warnings
 447                 warnings.warn("Policy.get_interface is deprecated!", DeprecationWarning, stacklevel = 2)
 448                 return iface_cache.get_interface(uri)