zeroinstall/injector/policy.py

   1 """
   2 This class brings together a L{solve.Solver} to choose a set of implmentations, a
   3 L{fetch.Fetcher} to download additional components, and the user's configuration
   4 settings.
   5 """
   6
   7 # Copyright (C) 2009, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 from zeroinstall import _
  11 import time
  12 import os
  13 from logging import info, debug, warn
  14 import ConfigParser
  15
  16 from zeroinstall import SafeException
  17 from zeroinstall.injector import arch
  18 from zeroinstall.injector.model import Interface, Implementation, network_levels, network_offline, DistributionImplementation, network_full
  19 from zeroinstall.injector.namespaces import config_site, config_prog
  20 from zeroinstall.support import tasks, basedir
  21 from zeroinstall.injector.iface_cache import iface_cache
  22
  23 # If we started a check within this period, don't start another one:
  24 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  25
  26 class Policy(object):
  27         """Chooses a set of implementations based on a policy.
  28         Typical use:
  29          1. Create a Policy object, giving it the URI of the program to be run and a handler.
  30          2. Call L{solve_with_downloads}. If more information is needed, a L{fetch.Fetcher} will be used to download it.
  31          3. When all downloads are complete, the L{solver} contains the chosen versions.
  32          4. Use L{get_uncached_implementations} to find where to get these versions and download them
  33             using L{download_uncached_implementations}.
  34
  35         @ivar target_arch: target architecture for binaries
  36         @type target_arch: L{arch.Architecture}
  37         @ivar root: URI of the root interface
  38         @ivar solver: solver used to choose a set of implementations
  39         @type solver: L{solve.Solver}
  40         @ivar watchers: callbacks to invoke after recalculating
  41         @ivar help_with_testing: default stability policy
  42         @type help_with_testing: bool
  43         @ivar network_use: one of the model.network_* values
  44         @ivar freshness: seconds allowed since last update
  45         @type freshness: int
  46         @ivar handler: handler for main-loop integration
  47         @type handler: L{handler.Handler}
  48         @ivar src: whether we are looking for source code
  49         @type src: bool
  50         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
  51         @type stale_feeds: set
  52         """
  53         __slots__ = ['root', 'watchers',
  54                      'freshness', 'handler', '_warned_offline',
  55                      'target_arch', 'src', 'stale_feeds', 'solver', '_fetcher']
  56
  57         help_with_testing = property(lambda self: self.solver.help_with_testing,
  58                                      lambda self, value: setattr(self.solver, 'help_with_testing', value))
  59
  60         network_use = property(lambda self: self.solver.network_use,
  61                                      lambda self, value: setattr(self.solver, 'network_use', value))
  62
  63         implementation = property(lambda self: self.solver.selections)
  64
  65         ready = property(lambda self: self.solver.ready)
  66
  67         def __init__(self, root, handler = None, src = False):
  68                 """
  69                 @param root: The URI of the root interface (the program we want to run).
  70                 @param handler: A handler for main-loop integration.
  71                 @type handler: L{zeroinstall.injector.handler.Handler}
  72                 @param src: Whether we are looking for source code.
  73                 @type src: bool
  74                 """
  75                 self.watchers = []
  76                 self.freshness = 60 * 60 * 24 * 30
  77                 self.src = src                          # Root impl must be a "src" machine type
  78                 self.stale_feeds = set()
  79
  80                 from zeroinstall.injector.solver import DefaultSolver
  81                 self.solver = DefaultSolver(network_full, iface_cache, iface_cache.stores)
  82
  83                 # If we need to download something but can't because we are offline,
  84                 # warn the user. But only the first time.
  85                 self._warned_offline = False
  86                 self._fetcher = None
  87
  88                 # (allow self for backwards compat)
  89                 self.handler = handler or self
  90
  91                 debug(_("Supported systems: '%s'"), arch.os_ranks)
  92                 debug(_("Supported processors: '%s'"), arch.machine_ranks)
  93
  94                 path = basedir.load_first_config(config_site, config_prog, 'global')
  95                 if path:
  96                         try:
  97                                 config = ConfigParser.ConfigParser()
  98                                 config.read(path)
  99                                 self.solver.help_with_testing = config.getboolean('global',
 100                                                                 'help_with_testing')
 101                                 self.solver.network_use = config.get('global', 'network_use')
 102                                 self.freshness = int(config.get('global', 'freshness'))
 103                                 assert self.solver.network_use in network_levels, self.solver.network_use
 104                         except Exception, ex:
 105                                 warn(_("Error loading config: %s"), str(ex) or repr(ex))
 106
 107                 self.set_root(root)
 108
 109                 self.target_arch = arch.get_host_architecture()
 110
 111         @property
 112         def fetcher(self):
 113                 if not self._fetcher:
 114                         import fetch
 115                         self._fetcher = fetch.Fetcher(self.handler)
 116                 return self._fetcher
 117
 118         def set_root(self, root):
 119                 """Change the root interface URI."""
 120                 assert isinstance(root, (str, unicode))
 121                 self.root = root
 122                 for w in self.watchers: w()
 123
 124         def save_config(self):
 125                 """Write global settings."""
 126                 config = ConfigParser.ConfigParser()
 127                 config.add_section('global')
 128
 129                 config.set('global', 'help_with_testing', self.help_with_testing)
 130                 config.set('global', 'network_use', self.network_use)
 131                 config.set('global', 'freshness', self.freshness)
 132
 133                 path = basedir.save_config_path(config_site, config_prog)
 134                 path = os.path.join(path, 'global')
 135                 config.write(file(path + '.new', 'w'))
 136                 os.rename(path + '.new', path)
 137
 138         def recalculate(self, fetch_stale_interfaces = True):
 139                 """@deprecated: see L{solve_with_downloads} """
 140                 self.stale_feeds = set()
 141
 142                 host_arch = self.target_arch
 143                 if self.src:
 144                         host_arch = arch.SourceArchitecture(host_arch)
 145                 self.solver.solve(self.root, host_arch)
 146
 147                 if self.network_use == network_offline:
 148                         fetch_stale_interfaces = False
 149
 150                 blockers = []
 151                 for f in self.solver.feeds_used:
 152                         if f.startswith('/'): continue
 153                         feed = iface_cache.get_feed(f)
 154                         if feed is None or feed.last_modified is None:
 155                                 self.download_and_import_feed_if_online(f)      # Will start a download
 156                         elif self.is_stale(feed):
 157                                 debug(_("Adding %s to stale set"), f)
 158                                 self.stale_feeds.add(iface_cache.get_interface(f))      # Legacy API
 159                                 if fetch_stale_interfaces:
 160                                         self.download_and_import_feed_if_online(f)      # Will start a download
 161
 162                 for w in self.watchers: w()
 163
 164                 return blockers
 165
 166         def usable_feeds(self, iface):
 167                 """Generator for C{iface.feeds} that are valid for our architecture.
 168                 @rtype: generator
 169                 @see: L{arch}"""
 170                 if self.src and iface.uri == self.root:
 171                         # Note: when feeds are recursive, we'll need a better test for root here
 172                         machine_ranks = {'src': 1}
 173                 else:
 174                         machine_ranks = arch.machine_ranks
 175
 176                 for f in iface.feeds:
 177                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 178                                 yield f
 179                         else:
 180                                 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
 181                                         {'feed': f, 'os': f.os, 'machine': f.machine})
 182
 183         def is_stale(self, feed):
 184                 """Check whether feed needs updating, based on the configured L{freshness}.
 185                 None is considered to be stale.
 186                 @return: true if feed is stale or missing."""
 187                 if feed is None:
 188                         return True
 189                 if feed.url.startswith('/'):
 190                         return False            # Local feeds are never stale
 191                 if feed.last_modified is None:
 192                         return True             # Don't even have it yet
 193                 now = time.time()
 194                 staleness = now - (feed.last_checked or 0)
 195                 debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0})
 196
 197                 if self.freshness <= 0 or staleness < self.freshness:
 198                         return False            # Fresh enough for us
 199
 200                 last_check_attempt = iface_cache.get_last_check_attempt(feed.url)
 201                 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 202                         debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt))
 203                         return False
 204
 205                 return True
 206
 207         def download_and_import_feed_if_online(self, feed_url):
 208                 """If we're online, call L{fetch.Fetcher.download_and_import_feed}. Otherwise, log a suitable warning."""
 209                 if self.network_use != network_offline:
 210                         debug(_("Feed %s not cached and not off-line. Downloading..."), feed_url)
 211                         return self.fetcher.download_and_import_feed(feed_url, iface_cache)
 212                 else:
 213                         if self._warned_offline:
 214                                 debug(_("Not downloading feed '%s' because we are off-line."), feed_url)
 215                         else:
 216                                 warn(_("Not downloading feed '%s' because we are in off-line mode."), feed_url)
 217                                 self._warned_offline = True
 218
 219         def get_implementation_path(self, impl):
 220                 """Return the local path of impl.
 221                 @rtype: str
 222                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 223                 assert isinstance(impl, Implementation)
 224                 return impl.local_path or iface_cache.stores.lookup_any(impl.digests)
 225
 226         def get_implementation(self, interface):
 227                 """Get the chosen implementation.
 228                 @type interface: Interface
 229                 @rtype: L{model.Implementation}
 230                 @raise SafeException: if interface has not been fetched or no implementation could be
 231                 chosen."""
 232                 assert isinstance(interface, Interface)
 233
 234                 if not interface.name and not interface.feeds:
 235                         raise SafeException(_("We don't have enough information to "
 236                                             "run this program yet. "
 237                                             "Need to download:\n%s") % interface.uri)
 238                 try:
 239                         return self.implementation[interface]
 240                 except KeyError, ex:
 241                         if interface.implementations:
 242                                 offline = ""
 243                                 if self.network_use == network_offline:
 244                                         raise SafeException(_("No usable implementation found for '%s'.\n"
 245                                                         "This may be because 'Network Use' is set to Off-line.") %
 246                                                         interface.name)
 247                                 raise SafeException(_("No usable implementation found for '%s'.") %
 248                                                 interface.name)
 249                         raise ex
 250
 251         def get_cached(self, impl):
 252                 """Check whether an implementation is available locally.
 253                 @type impl: model.Implementation
 254                 @rtype: bool
 255                 """
 256                 if isinstance(impl, DistributionImplementation):
 257                         return impl.installed
 258                 if impl.local_path:
 259                         return os.path.exists(impl.local_path)
 260                 else:
 261                         try:
 262                                 path = self.get_implementation_path(impl)
 263                                 assert path
 264                                 return True
 265                         except:
 266                                 pass # OK
 267                 return False
 268
 269         def get_uncached_implementations(self):
 270                 """List all chosen implementations which aren't yet available locally.
 271                 @rtype: [(L{model.Interface}, L{model.Implementation})]"""
 272                 uncached = []
 273                 for iface in self.solver.selections:
 274                         impl = self.solver.selections[iface]
 275                         assert impl, self.solver.selections
 276                         if not self.get_cached(impl):
 277                                 uncached.append((iface, impl))
 278                 return uncached
 279
 280         def refresh_all(self, force = True):
 281                 """Start downloading all feeds for all selected interfaces.
 282                 @param force: Whether to restart existing downloads."""
 283                 return self.solve_with_downloads(force = True)
 284
 285         def get_feed_targets(self, feed_iface_uri):
 286                 """Return a list of Interfaces for which feed_iface can be a feed.
 287                 This is used by B{0launch --feed}.
 288                 @rtype: [model.Interface]
 289                 @raise SafeException: If there are no known feeds."""
 290                 # TODO: what if it isn't cached yet?
 291                 feed_iface = iface_cache.get_interface(feed_iface_uri)
 292                 if not feed_iface.feed_for:
 293                         if not feed_iface.name:
 294                                 raise SafeException(_("Can't get feed targets for '%s'; failed to load interface.") %
 295                                                 feed_iface_uri)
 296                         raise SafeException(_("Missing <feed-for> element in '%s'; "
 297                                         "this interface can't be used as a feed.") % feed_iface_uri)
 298                 feed_targets = feed_iface.feed_for
 299                 debug(_("Feed targets: %s"), feed_targets)
 300                 if not feed_iface.name:
 301                         warn(_("Warning: unknown interface '%s'") % feed_iface_uri)
 302                 return [iface_cache.get_interface(uri) for uri in feed_targets]
 303
 304         @tasks.async
 305         def solve_with_downloads(self, force = False):
 306                 """Run the solver, then download any feeds that are missing or
 307                 that need to be updated. Each time a new feed is imported into
 308                 the cache, the solver is run again, possibly adding new downloads.
 309                 @param force: whether to download even if we're already ready to run."""
 310
 311                 downloads_finished = set()              # Successful or otherwise
 312                 downloads_in_progress = {}              # URL -> Download
 313
 314                 host_arch = self.target_arch
 315                 if self.src:
 316                         host_arch = arch.SourceArchitecture(host_arch)
 317
 318                 while True:
 319                         self.solver.solve(self.root, host_arch)
 320                         for w in self.watchers: w()
 321
 322                         if self.solver.ready and not force:
 323                                 break
 324                         else:
 325                                 if self.network_use == network_offline and not force:
 326                                         info(_("Can't choose versions and in off-line mode, so aborting"))
 327                                         break
 328                                 # Once we've starting downloading some things,
 329                                 # we might as well get them all.
 330                                 force = True
 331
 332                         for f in self.solver.feeds_used:
 333                                 if f in downloads_finished or f in downloads_in_progress:
 334                                         continue
 335                                 if f.startswith('/'):
 336                                         continue
 337                                 feed = iface_cache.get_interface(f)
 338                                 downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, iface_cache)
 339
 340                         if not downloads_in_progress:
 341                                 break
 342
 343                         blockers = downloads_in_progress.values()
 344                         yield blockers
 345                         tasks.check(blockers, self.handler.report_error)
 346
 347                         for f in downloads_in_progress.keys():
 348                                 if downloads_in_progress[f].happened:
 349                                         del downloads_in_progress[f]
 350                                         downloads_finished.add(f)
 351
 352         @tasks.async
 353         def solve_and_download_impls(self, refresh = False):
 354                 """Run L{solve_with_downloads} and then get the selected implementations too.
 355                 @raise SafeException: if we couldn't select a set of implementations
 356                 @since: 0.40"""
 357                 refreshed = self.solve_with_downloads(refresh)
 358                 if refreshed:
 359                         yield refreshed
 360                         tasks.check(refreshed)
 361
 362                 if not self.solver.ready:
 363                         raise SafeException(_("Can't find all required implementations:") + '\n' +
 364                                 '\n'.join(["- %s -> %s" % (iface, self.solver.selections[iface])
 365                                            for iface  in self.solver.selections]))
 366                 downloaded = self.download_uncached_implementations()
 367                 if downloaded:
 368                         yield downloaded
 369                         tasks.check(downloaded)
 370
 371         def need_download(self):
 372                 """Decide whether we need to download anything (but don't do it!)
 373                 @return: true if we MUST download something (feeds or implementations)
 374                 @rtype: bool"""
 375                 host_arch = self.target_arch
 376                 if self.src:
 377                         host_arch = arch.SourceArchitecture(host_arch)
 378                 self.solver.solve(self.root, host_arch)
 379                 for w in self.watchers: w()
 380
 381                 if not self.solver.ready:
 382                         return True             # Maybe a newer version will work?
 383
 384                 if self.get_uncached_implementations():
 385                         return True
 386
 387                 return False
 388
 389         def download_uncached_implementations(self):
 390                 """Download all implementations chosen by the solver that are missing from the cache."""
 391                 assert self.solver.ready, "Solver is not ready!\n%s" % self.solver.selections
 392                 return self.fetcher.download_impls([impl for impl in self.solver.selections.values() if not self.get_cached(impl)],
 393                                                    iface_cache.stores)
 394
 395         def download_icon(self, interface, force = False):
 396                 """Download an icon for this interface and add it to the
 397                 icon cache. If the interface has no icon or we are offline, do nothing.
 398                 @return: the task doing the import, or None
 399                 @rtype: L{tasks.Task}"""
 400                 if self.network_use == network_offline:
 401                         info("Not downloading icon for %s as we are off-line", interface)
 402                         return
 403
 404                 modification_time = None
 405
 406                 existing_icon = iface_cache.get_icon_path(interface)
 407                 if existing_icon:
 408                         file_mtime = os.stat(existing_icon).st_mtime
 409                         from email.utils import formatdate
 410                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 411
 412                 return self.fetcher.download_icon(interface, force, modification_time)
 413
 414         def get_interface(self, uri):
 415                 """@deprecated: use L{iface_cache.IfaceCache.get_interface} instead"""
 416                 import warnings
 417                 warnings.warn(_("Policy.get_interface is deprecated!"), DeprecationWarning, stacklevel = 2)
 418                 return iface_cache.get_interface(uri)