zeroinstall/injector/policy.py

   1 """
   2 This class brings together a L{solve.Solver} to choose a set of implmentations, a
   3 L{fetch.Fetcher} to download additional components, and the user's configuration
   4 settings.
   5 """
   6
   7 # Copyright (C) 2009, Thomas Leonard
   8 # See the README file for details, or visit http://0install.net.
   9
  10 import time
  11 import os
  12 from logging import info, debug, warn
  13 import ConfigParser
  14
  15 from zeroinstall import SafeException
  16 from zeroinstall.injector import arch
  17 from zeroinstall.injector.model import Interface, Implementation, network_levels, network_offline, DistributionImplementation, network_full
  18 from zeroinstall.injector.namespaces import config_site, config_prog, injector_gui_uri
  19 from zeroinstall.support import tasks, basedir
  20 from zeroinstall.injector.iface_cache import iface_cache
  21
  22 # If we started a check within this period, don't start another one:
  23 FAILED_CHECK_DELAY = 60 * 60    # 1 Hour
  24
  25 class Policy(object):
  26         """Chooses a set of implementations based on a policy.
  27         Typical use:
  28          1. Create a Policy object, giving it the URI of the program to be run and a handler.
  29          2. Call L{solve_with_downloads}. If more information is needed, a L{fetch.Fetcher} will be used to download it.
  30          3. When all downloads are complete, the L{solver} contains the chosen versions.
  31          4. Use L{get_uncached_implementations} to find where to get these versions and download them
  32             using L{download_uncached_implementations}.
  33
  34         @ivar target_arch: target architecture for binaries
  35         @type target_arch: L{arch.Architecture}
  36         @ivar root: URI of the root interface
  37         @ivar solver: solver used to choose a set of implementations
  38         @type solver: L{solve.Solver}
  39         @ivar watchers: callbacks to invoke after recalculating
  40         @ivar help_with_testing: default stability policy
  41         @type help_with_testing: bool
  42         @ivar network_use: one of the model.network_* values
  43         @ivar freshness: seconds allowed since last update
  44         @type freshness: int
  45         @ivar handler: handler for main-loop integration
  46         @type handler: L{handler.Handler}
  47         @ivar src: whether we are looking for source code
  48         @type src: bool
  49         @ivar stale_feeds: set of feeds which are present but haven't been checked for a long time
  50         @type stale_feeds: set
  51         """
  52         __slots__ = ['root', 'watchers',
  53                      'freshness', 'handler', '_warned_offline',
  54                      'target_arch', 'src', 'stale_feeds', 'solver', '_fetcher']
  55
  56         help_with_testing = property(lambda self: self.solver.help_with_testing,
  57                                      lambda self, value: setattr(self.solver, 'help_with_testing', value))
  58
  59         network_use = property(lambda self: self.solver.network_use,
  60                                      lambda self, value: setattr(self.solver, 'network_use', value))
  61
  62         implementation = property(lambda self: self.solver.selections)
  63
  64         ready = property(lambda self: self.solver.ready)
  65
  66         def __init__(self, root, handler = None, src = False):
  67                 """
  68                 @param root: The URI of the root interface (the program we want to run).
  69                 @param handler: A handler for main-loop integration.
  70                 @type handler: L{zeroinstall.injector.handler.Handler}
  71                 @param src: Whether we are looking for source code.
  72                 @type src: bool
  73                 """
  74                 self.watchers = []
  75                 self.freshness = 60 * 60 * 24 * 30
  76                 self.src = src                          # Root impl must be a "src" machine type
  77                 self.stale_feeds = set()
  78
  79                 from zeroinstall.injector.solver import DefaultSolver
  80                 self.solver = DefaultSolver(network_full, iface_cache, iface_cache.stores)
  81
  82                 # If we need to download something but can't because we are offline,
  83                 # warn the user. But only the first time.
  84                 self._warned_offline = False
  85                 self._fetcher = None
  86
  87                 # (allow self for backwards compat)
  88                 self.handler = handler or self
  89
  90                 debug(_("Supported systems: '%s'"), arch.os_ranks)
  91                 debug(_("Supported processors: '%s'"), arch.machine_ranks)
  92
  93                 path = basedir.load_first_config(config_site, config_prog, 'global')
  94                 if path:
  95                         try:
  96                                 config = ConfigParser.ConfigParser()
  97                                 config.read(path)
  98                                 self.solver.help_with_testing = config.getboolean('global',
  99                                                                 'help_with_testing')
 100                                 self.solver.network_use = config.get('global', 'network_use')
 101                                 self.freshness = int(config.get('global', 'freshness'))
 102                                 assert self.solver.network_use in network_levels, self.solver.network_use
 103                         except Exception, ex:
 104                                 warn(_("Error loading config: %s"), str(ex) or repr(ex))
 105
 106                 self.set_root(root)
 107
 108                 self.target_arch = arch.get_host_architecture()
 109
 110         @property
 111         def fetcher(self):
 112                 if not self._fetcher:
 113                         import fetch
 114                         self._fetcher = fetch.Fetcher(self.handler)
 115                 return self._fetcher
 116
 117         def set_root(self, root):
 118                 """Change the root interface URI."""
 119                 assert isinstance(root, (str, unicode))
 120                 self.root = root
 121                 for w in self.watchers: w()
 122
 123         def save_config(self):
 124                 """Write global settings."""
 125                 config = ConfigParser.ConfigParser()
 126                 config.add_section('global')
 127
 128                 config.set('global', 'help_with_testing', self.help_with_testing)
 129                 config.set('global', 'network_use', self.network_use)
 130                 config.set('global', 'freshness', self.freshness)
 131
 132                 path = basedir.save_config_path(config_site, config_prog)
 133                 path = os.path.join(path, 'global')
 134                 config.write(file(path + '.new', 'w'))
 135                 os.rename(path + '.new', path)
 136
 137         def recalculate(self, fetch_stale_interfaces = True):
 138                 """@deprecated: see L{solve_with_downloads} """
 139                 self.stale_feeds = set()
 140
 141                 host_arch = self.target_arch
 142                 if self.src:
 143                         host_arch = arch.SourceArchitecture(host_arch)
 144                 self.solver.solve(self.root, host_arch)
 145
 146                 if self.network_use == network_offline:
 147                         fetch_stale_interfaces = False
 148
 149                 blockers = []
 150                 for f in self.solver.feeds_used:
 151                         if f.startswith('/'): continue
 152                         feed = iface_cache.get_feed(f)
 153                         if feed is None or feed.last_modified is None:
 154                                 self.download_and_import_feed_if_online(f)      # Will start a download
 155                         elif self.is_stale(feed):
 156                                 debug(_("Adding %s to stale set"), f)
 157                                 self.stale_feeds.add(iface_cache.get_interface(f))      # Legacy API
 158                                 if fetch_stale_interfaces:
 159                                         self.download_and_import_feed_if_online(f)      # Will start a download
 160
 161                 for w in self.watchers: w()
 162
 163                 return blockers
 164
 165         def usable_feeds(self, iface):
 166                 """Generator for C{iface.feeds} that are valid for our architecture.
 167                 @rtype: generator
 168                 @see: L{arch}"""
 169                 if self.src and iface.uri == self.root:
 170                         # Note: when feeds are recursive, we'll need a better test for root here
 171                         machine_ranks = {'src': 1}
 172                 else:
 173                         machine_ranks = arch.machine_ranks
 174
 175                 for f in iface.feeds:
 176                         if f.os in arch.os_ranks and f.machine in machine_ranks:
 177                                 yield f
 178                         else:
 179                                 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
 180                                         {'feed': f, 'os': f.os, 'machine': f.machine})
 181
 182         def is_stale(self, feed):
 183                 """Check whether feed needs updating, based on the configured L{freshness}.
 184                 None is considered to be stale.
 185                 @return: true if feed is stale or missing."""
 186                 if feed is None:
 187                         return True
 188                 if feed.url.startswith('/'):
 189                         return False            # Local feeds are never stale
 190                 if feed.last_modified is None:
 191                         return True             # Don't even have it yet
 192                 now = time.time()
 193                 staleness = now - (feed.last_checked or 0)
 194                 debug(_("Staleness for %(feed)s is %(staleness).2f hours"), {'feed': feed, 'staleness': staleness / 3600.0})
 195
 196                 if self.freshness == 0 or staleness < self.freshness:
 197                         return False            # Fresh enough for us
 198
 199                 last_check_attempt = iface_cache.get_last_check_attempt(feed.url)
 200                 if last_check_attempt and last_check_attempt > now - FAILED_CHECK_DELAY:
 201                         debug(_("Stale, but tried to check recently (%s) so not rechecking now."), time.ctime(last_check_attempt))
 202                         return False
 203
 204                 return True
 205
 206         def download_and_import_feed_if_online(self, feed_url):
 207                 """If we're online, call L{fetch.Fetcher.download_and_import_feed}. Otherwise, log a suitable warning."""
 208                 if self.network_use != network_offline:
 209                         debug(_("Feed %s not cached and not off-line. Downloading..."), feed_url)
 210                         return self.fetcher.download_and_import_feed(feed_url, iface_cache)
 211                 else:
 212                         if self._warned_offline:
 213                                 debug(_("Not downloading feed '%s' because we are off-line."), feed_url)
 214                         elif feed_url == injector_gui_uri:
 215                                 # Don't print a warning, because we always switch to off-line mode to
 216                                 # run the GUI the first time.
 217                                 info(_("Not downloading GUI feed '%s' because we are in off-line mode."), feed_url)
 218                         else:
 219                                 warn(_("Not downloading feed '%s' because we are in off-line mode."), feed_url)
 220                                 self._warned_offline = True
 221
 222         def get_implementation_path(self, impl):
 223                 """Return the local path of impl.
 224                 @rtype: str
 225                 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
 226                 assert isinstance(impl, Implementation)
 227                 if impl.id.startswith('/'):
 228                         return impl.id
 229                 return iface_cache.stores.lookup(impl.id)
 230
 231         def get_implementation(self, interface):
 232                 """Get the chosen implementation.
 233                 @type interface: Interface
 234                 @rtype: L{model.Implementation}
 235                 @raise SafeException: if interface has not been fetched or no implementation could be
 236                 chosen."""
 237                 assert isinstance(interface, Interface)
 238
 239                 if not interface.name and not interface.feeds:
 240                         raise SafeException(_("We don't have enough information to "
 241                                             "run this program yet. "
 242                                             "Need to download:\n%s") % interface.uri)
 243                 try:
 244                         return self.implementation[interface]
 245                 except KeyError, ex:
 246                         if interface.implementations:
 247                                 offline = ""
 248                                 if self.network_use == network_offline:
 249                                         raise SafeException(_("No usable implementation found for '%s'.\n"
 250                                                         "This may be because 'Network Use' is set to Off-line.") %
 251                                                         interface.name)
 252                                 raise SafeException(_("No usable implementation found for '%s'.") %
 253                                                 interface.name)
 254                         raise ex
 255
 256         def get_cached(self, impl):
 257                 """Check whether an implementation is available locally.
 258                 @type impl: model.Implementation
 259                 @rtype: bool
 260                 """
 261                 if isinstance(impl, DistributionImplementation):
 262                         return impl.installed
 263                 if impl.id.startswith('/'):
 264                         return os.path.exists(impl.id)
 265                 else:
 266                         try:
 267                                 path = self.get_implementation_path(impl)
 268                                 assert path
 269                                 return True
 270                         except:
 271                                 pass # OK
 272                 return False
 273
 274         def get_uncached_implementations(self):
 275                 """List all chosen implementations which aren't yet available locally.
 276                 @rtype: [(L{model.Interface}, L{model.Implementation})]"""
 277                 uncached = []
 278                 for iface in self.solver.selections:
 279                         impl = self.solver.selections[iface]
 280                         assert impl, self.solver.selections
 281                         if not self.get_cached(impl):
 282                                 uncached.append((iface, impl))
 283                 return uncached
 284
 285         def refresh_all(self, force = True):
 286                 """Start downloading all feeds for all selected interfaces.
 287                 @param force: Whether to restart existing downloads."""
 288                 return self.solve_with_downloads(force = True)
 289
 290         def get_feed_targets(self, feed_iface_uri):
 291                 """Return a list of Interfaces for which feed_iface can be a feed.
 292                 This is used by B{0launch --feed}.
 293                 @rtype: [model.Interface]
 294                 @raise SafeException: If there are no known feeds."""
 295                 # TODO: what if it isn't cached yet?
 296                 feed_iface = iface_cache.get_interface(feed_iface_uri)
 297                 if not feed_iface.feed_for:
 298                         if not feed_iface.name:
 299                                 raise SafeException(_("Can't get feed targets for '%s'; failed to load interface.") %
 300                                                 feed_iface_uri)
 301                         raise SafeException(_("Missing <feed-for> element in '%s'; "
 302                                         "this interface can't be used as a feed.") % feed_iface_uri)
 303                 feed_targets = feed_iface.feed_for
 304                 debug(_("Feed targets: %s"), feed_targets)
 305                 if not feed_iface.name:
 306                         warn(_("Warning: unknown interface '%s'") % feed_iface_uri)
 307                 return [iface_cache.get_interface(uri) for uri in feed_targets]
 308
 309         @tasks.async
 310         def solve_with_downloads(self, force = False):
 311                 """Run the solver, then download any feeds that are missing or
 312                 that need to be updated. Each time a new feed is imported into
 313                 the cache, the solver is run again, possibly adding new downloads.
 314                 @param force: whether to download even if we're already ready to run."""
 315
 316                 downloads_finished = set()              # Successful or otherwise
 317                 downloads_in_progress = {}              # URL -> Download
 318
 319                 host_arch = self.target_arch
 320                 if self.src:
 321                         host_arch = arch.SourceArchitecture(host_arch)
 322
 323                 while True:
 324                         self.solver.solve(self.root, host_arch)
 325                         for w in self.watchers: w()
 326
 327                         if self.solver.ready and not force:
 328                                 break
 329                         else:
 330                                 if self.network_use == network_offline and not force:
 331                                         info(_("Can't choose versions and in off-line mode, so aborting"))
 332                                         break
 333                                 # Once we've starting downloading some things,
 334                                 # we might as well get them all.
 335                                 force = True
 336
 337                         for f in self.solver.feeds_used:
 338                                 if f in downloads_finished or f in downloads_in_progress:
 339                                         continue
 340                                 if f.startswith('/'):
 341                                         continue
 342                                 feed = iface_cache.get_interface(f)
 343                                 downloads_in_progress[f] = self.fetcher.download_and_import_feed(f, iface_cache)
 344
 345                         if not downloads_in_progress:
 346                                 break
 347
 348                         blockers = downloads_in_progress.values()
 349                         yield blockers
 350                         tasks.check(blockers, self.handler.report_error)
 351
 352                         for f in downloads_in_progress.keys():
 353                                 if downloads_in_progress[f].happened:
 354                                         del downloads_in_progress[f]
 355                                         downloads_finished.add(f)
 356
 357         @tasks.async
 358         def solve_and_download_impls(self, refresh = False):
 359                 """Run L{solve_with_downloads} and then get the selected implementations too.
 360                 @raise SafeException: if we couldn't select a set of implementations
 361                 @since: 0.40"""
 362                 refreshed = self.solve_with_downloads(refresh)
 363                 if refreshed:
 364                         yield refreshed
 365                         tasks.check(refreshed)
 366
 367                 if not self.solver.ready:
 368                         raise SafeException(_("Can't find all required implementations:") + '\n' +
 369                                 '\n'.join(["- %s -> %s" % (iface, self.solver.selections[iface])
 370                                            for iface  in self.solver.selections]))
 371                 downloaded = self.download_uncached_implementations()
 372                 if downloaded:
 373                         yield downloaded
 374                         tasks.check(downloaded)
 375
 376         def need_download(self):
 377                 """Decide whether we need to download anything (but don't do it!)
 378                 @return: true if we MUST download something (feeds or implementations)
 379                 @rtype: bool"""
 380                 host_arch = self.target_arch
 381                 if self.src:
 382                         host_arch = arch.SourceArchitecture(host_arch)
 383                 self.solver.solve(self.root, host_arch)
 384                 for w in self.watchers: w()
 385
 386                 if not self.solver.ready:
 387                         return True             # Maybe a newer version will work?
 388
 389                 if self.get_uncached_implementations():
 390                         return True
 391
 392                 return False
 393
 394         def download_uncached_implementations(self):
 395                 """Download all implementations chosen by the solver that are missing from the cache."""
 396                 assert self.solver.ready, "Solver is not ready!\n%s" % self.solver.selections
 397                 return self.fetcher.download_impls([impl for impl in self.solver.selections.values() if not self.get_cached(impl)],
 398                                                    iface_cache.stores)
 399
 400         def download_icon(self, interface, force = False):
 401                 """Download an icon for this interface and add it to the
 402                 icon cache. If the interface has no icon or we are offline, do nothing.
 403                 @return: the task doing the import, or None
 404                 @rtype: L{tasks.Task}"""
 405                 if self.network_use == network_offline:
 406                         info("Not downloading icon for %s as we are off-line", interface)
 407                         return
 408
 409                 modification_time = None
 410
 411                 existing_icon = iface_cache.get_icon_path(interface)
 412                 if existing_icon:
 413                         file_mtime = os.stat(existing_icon).st_mtime
 414                         from email.utils import formatdate
 415                         modification_time = formatdate(timeval = file_mtime, localtime = False, usegmt = True)
 416
 417                 return self.fetcher.download_icon(interface, force, modification_time)
 418
 419         def get_interface(self, uri):
 420                 """@deprecated: use L{iface_cache.IfaceCache.get_interface} instead"""
 421                 import warnings
 422                 warnings.warn(_("Policy.get_interface is deprecated!"), DeprecationWarning, stacklevel = 2)
 423                 return iface_cache.get_interface(uri)