2 Chooses a set of implementations based on a policy.
5 # Copyright (C) 2006, Thomas Leonard
6 # See the README file for details, or visit http://0install.net.
10 from logging
import info
, debug
, warn
15 from namespaces
import *
18 from zeroinstall
import NeedDownload
19 from zeroinstall
.injector
.iface_cache
import iface_cache
, PendingFeed
20 from zeroinstall
.injector
.trust
import trust_db
23 """A Cook follows a Recipe."""
24 # Maybe we're taking this metaphor too far?
26 def __init__(self
, policy
, required_digest
, recipe
, force
= False):
27 """Start downloading all the ingredients."""
29 self
.required_digest
= required_digest
30 self
.downloads
= {} # Downloads that are not yet successful
31 self
.streams
= {} # Streams collected from successful downloads
33 # Start a download for each ingredient
34 for step
in recipe
.steps
:
35 dl
= policy
.begin_archive_download(step
, success_callback
=
36 lambda stream
, step
=step
: self
.ingredient_ready(step
, stream
),
38 self
.downloads
[step
] = dl
39 self
.test_done() # Needed for empty recipes
41 # Note: the only references to us are held by the on_success callback
42 # in each Download. On error this is removed, which will cause us
43 # to be destoryed, which will release all the temporary files we hold.
45 def ingredient_ready(self
, step
, stream
):
46 # Called when one archive has been fetched. Store it until the other
48 assert step
not in self
.streams
49 self
.streams
[step
] = stream
50 del self
.downloads
[step
]
54 # On success, a download is removed from here. If empty, it means that
55 # all archives have successfully been downloaded.
56 if self
.downloads
: return
58 from zeroinstall
.zerostore
import unpack
60 # Create an empty directory for the new implementation
61 store
= iface_cache
.stores
.stores
[0]
62 tmpdir
= store
.get_tmp_dir_for(self
.required_digest
)
64 # Unpack each of the downloaded archives into it in turn
65 for step
in self
.recipe
.steps
:
66 unpack
.unpack_archive(step
.url
, self
.streams
[step
], tmpdir
, step
.extract
)
67 # Check that the result is correct and store it in the cache
68 store
.check_manifest_and_rename(self
.required_digest
, tmpdir
)
71 # If unpacking fails, remove the temporary directory
72 if tmpdir
is not None:
77 """Chooses a set of implementations based on a policy.
79 1. Create a Policy object, giving it the URI of the program to be run and a handler.
80 2. Call L{recalculate}. If more information is needed, the handler will be used to download it.
81 3. When all downloads are complete, the L{implementation} map contains the chosen versions.
82 4. Use L{get_uncached_implementations} to find where to get these versions and download them
83 using L{begin_impl_download}.
85 @ivar root: URI of the root interface
86 @ivar implementation: chosen implementations
87 @type implementation: {model.Interface: model.Implementation or None}
88 @ivar watchers: callbacks to invoke after recalculating
89 @ivar help_with_testing: default stability policy
90 @type help_with_testing: bool
91 @ivar network_use: one of the model.network_* values
92 @ivar freshness: seconds allowed since last update
94 @ivar ready: whether L{implementation} is complete enough to run the program
96 @ivar handler: handler for main-loop integration
97 @type handler: L{handler.Handler}
98 @ivar restrictions: Currently known restrictions for each interface.
99 @type restrictions: {model.Interface -> [model.Restriction]}
100 @ivar src: whether we are looking for source code
103 __slots__
= ['root', 'implementation', 'watchers',
104 'help_with_testing', 'network_use',
105 'freshness', 'ready', 'handler', '_warned_offline',
106 'restrictions', 'src', 'root_restrictions']
108 def __init__(self
, root
, handler
= None, src
= False):
110 @param root: The URI of the root interface (the program we want to run).
111 @param handler: A handler for main-loop integration.
112 @type handler: L{zeroinstall.injector.handler.Handler}
113 @param src: Whether we are looking for source code.
117 self
.help_with_testing
= False
118 self
.network_use
= network_full
119 self
.freshness
= 60 * 60 * 24 * 30
121 self
.src
= src
# Root impl must be a "src" machine type
122 self
.restrictions
= {}
124 # This is used in is_unusable() to check whether the impl is
125 # for the root interface when looking for source. It is also
126 # used to add restrictions to the root (e.g. --before and --not-before)
127 self
.root_restrictions
= []
129 # If we need to download something but can't because we are offline,
130 # warn the user. But only the first time.
131 self
._warned
_offline
= False
133 # (allow self for backwards compat)
134 self
.handler
= handler
or self
136 debug("Supported systems: '%s'", arch
.os_ranks
)
137 debug("Supported processors: '%s'", arch
.machine_ranks
)
139 path
= basedir
.load_first_config(config_site
, config_prog
, 'global')
142 config
= ConfigParser
.ConfigParser()
144 self
.help_with_testing
= config
.getboolean('global',
146 self
.network_use
= config
.get('global', 'network_use')
147 self
.freshness
= int(config
.get('global', 'freshness'))
148 assert self
.network_use
in network_levels
149 except Exception, ex
:
150 warn("Error loading config: %s", ex
)
154 # Probably need weakrefs here...
155 iface_cache
.add_watcher(self
)
156 trust_db
.watchers
.append(self
.process_pending
)
158 def set_root(self
, root
):
159 """Change the root interface URI."""
160 assert isinstance(root
, (str, unicode))
162 self
.implementation
= {} # Interface -> [Implementation | None]
164 def save_config(self
):
165 """Write global settings."""
166 config
= ConfigParser
.ConfigParser()
167 config
.add_section('global')
169 config
.set('global', 'help_with_testing', self
.help_with_testing
)
170 config
.set('global', 'network_use', self
.network_use
)
171 config
.set('global', 'freshness', self
.freshness
)
173 path
= basedir
.save_config_path(config_site
, config_prog
)
174 path
= os
.path
.join(path
, 'global')
175 config
.write(file(path
+ '.new', 'w'))
176 os
.rename(path
+ '.new', path
)
178 def process_pending(self
):
179 """For each pending feed, either import it fully (if we now
180 trust one of the signatures) or start performing whatever action
181 is needed next (either downloading a key or confirming a
185 # process_pending must never be called from recalculate
187 for pending
in iface_cache
.pending
.values():
188 pending
.begin_key_downloads(self
.handler
, lambda pending
= pending
: self
._keys
_ready
(pending
))
190 def _keys_ready(self
, pending
):
192 iface
= iface_cache
.get_interface(pending
.url
)
193 # Note: this may call recalculate, but it shouldn't do any harm
195 updated
= iface_cache
.update_interface_if_trusted(iface
, pending
.sigs
, pending
.new_xml
)
196 except SafeException
, ex
:
197 self
.handler
.report_error(ex
)
198 # Ignore the problematic new version and continue...
201 self
.handler
.confirm_trust_keys(iface
, pending
.sigs
, pending
.new_xml
)
203 def recalculate(self
):
204 """Try to choose a set of implementations.
205 This may start downloading more interfaces, but will return immediately.
206 @postcondition: L{ready} indicates whether a possible set of implementations was chosen
207 @note: A policy may be ready before all feeds have been downloaded. As new feeds
208 arrive, the chosen versions may change.
211 self
.restrictions
= {}
212 self
.implementation
= {}
214 debug("Recalculate! root = %s", self
.root
)
216 iface
= self
.get_interface(dep
.interface
)
217 if iface
in self
.implementation
:
218 debug("Interface requested twice; skipping second %s", iface
)
220 warn("Interface requested twice; I've already chosen an implementation "
221 "of '%s' but there are more restrictions! Ignoring the second set.", iface
)
223 self
.implementation
[iface
] = None # Avoid cycles
225 assert iface
not in self
.restrictions
226 self
.restrictions
[iface
] = dep
.restrictions
228 impl
= self
._get
_best
_implementation
(iface
)
230 debug("Will use implementation %s (version %s)", impl
, impl
.get_version())
231 self
.implementation
[iface
] = impl
232 for d
in impl
.dependencies
.values():
233 debug("Considering dependency %s", d
)
236 debug("No implementation chould be chosen yet");
238 process(Dependency(self
.root
, restrictions
= self
.root_restrictions
))
239 for w
in self
.watchers
: w()
241 # Only to be called from recalculate, as it is quite slow.
242 # Use the results stored in self.implementation instead.
243 def _get_best_implementation(self
, iface
):
244 impls
= iface
.implementations
.values()
245 for f
in self
.usable_feeds(iface
):
246 debug("Processing feed %s", f
)
248 feed_iface
= self
.get_interface(f
.uri
)
249 if feed_iface
.name
and iface
.uri
not in feed_iface
.feed_for
:
250 warn("Missing <feed-for> for '%s' in '%s'",
252 if feed_iface
.implementations
:
253 impls
.extend(feed_iface
.implementations
.values())
254 except NeedDownload
, ex
:
256 except Exception, ex
:
257 warn("Failed to load feed %s for %s: %s",
260 debug("get_best_implementation(%s), with feeds: %s", iface
, iface
.feeds
)
263 info("Interface %s has no implementations!", iface
)
267 if self
.compare(iface
, x
, best
) < 0:
269 unusable
= self
.get_unusable_reason(best
, self
.restrictions
.get(iface
, []))
271 info("Best implementation of %s is %s, but unusable (%s)", iface
, best
, unusable
)
275 def compare(self
, interface
, b
, a
):
276 """Compare a and b to see which would be chosen first.
277 @param interface: The interface we are trying to resolve, which may
278 not be the interface of a or b if they are from feeds.
280 restrictions
= self
.restrictions
.get(interface
, [])
282 a_stab
= a
.get_stability()
283 b_stab
= b
.get_stability()
285 # Usable ones come first
286 r
= cmp(self
.is_unusable(b
, restrictions
), self
.is_unusable(a
, restrictions
))
289 # Preferred versions come first
290 r
= cmp(a_stab
== preferred
, b_stab
== preferred
)
293 if self
.network_use
!= network_full
:
294 r
= cmp(self
.get_cached(a
), self
.get_cached(b
))
298 stab_policy
= interface
.stability_policy
300 if self
.help_with_testing
: stab_policy
= testing
301 else: stab_policy
= stable
303 if a_stab
>= stab_policy
: a_stab
= preferred
304 if b_stab
>= stab_policy
: b_stab
= preferred
306 r
= cmp(a_stab
, b_stab
)
309 # Newer versions come before older ones
310 r
= cmp(a
.version
, b
.version
)
314 r
= cmp(arch
.os_ranks
.get(a
.os
, None),
315 arch
.os_ranks
.get(b
.os
, None))
319 r
= cmp(arch
.machine_ranks
.get(a
.machine
, None),
320 arch
.machine_ranks
.get(b
.machine
, None))
323 # Slightly prefer cached versions
324 if self
.network_use
== network_full
:
325 r
= cmp(self
.get_cached(a
), self
.get_cached(b
))
328 return cmp(a
.id, b
.id)
330 def usable_feeds(self
, iface
):
331 """Generator for C{iface.feeds} that are valid for our architecture.
334 if self
.src
and iface
.uri
== self
.root
:
335 # Note: when feeds are recursive, we'll need a better test for root here
336 machine_ranks
= {'src': 1}
338 machine_ranks
= arch
.machine_ranks
340 for f
in iface
.feeds
:
341 if f
.os
in arch
.os_ranks
and f
.machine
in machine_ranks
:
344 debug("Skipping '%s'; unsupported architecture %s-%s",
347 def get_ranked_implementations(self
, iface
):
348 """Get all implementations from all feeds, in order.
349 @type iface: Interface
350 @return: a sorted list of implementations.
351 @rtype: [model.Implementation]"""
352 impls
= iface
.implementations
.values()
353 for f
in self
.usable_feeds(iface
):
354 feed_iface
= self
.get_interface(f
.uri
)
355 if feed_iface
.implementations
:
356 impls
.extend(feed_iface
.implementations
.values())
357 impls
.sort(lambda a
, b
: self
.compare(iface
, a
, b
))
360 def is_unusable(self
, impl
, restrictions
= []):
361 """@return: whether this implementation is unusable.
363 return self
.get_unusable_reason(impl
, restrictions
) != None
365 def get_unusable_reason(self
, impl
, restrictions
= []):
367 @param impl: Implementation to test.
368 @type restrictions: [L{model.Restriction}]
369 @return: The reason why this impl is unusable, or None if it's OK.
371 @note: The restrictions are for the interface being requested, not the interface
372 of the implementation; they may be different when feeds are being used."""
373 for r
in restrictions
:
374 if not r
.meets_restriction(impl
):
375 return "Incompatible with another selected implementation"
376 stability
= impl
.get_stability()
377 if stability
<= buggy
:
378 return stability
.name
379 if self
.network_use
== network_offline
and not self
.get_cached(impl
):
380 return "Not cached and we are off-line"
381 if impl
.os
not in arch
.os_ranks
:
382 return "Unsupported OS"
383 # When looking for source code, we need to known if we're
384 # looking at an implementation of the root interface, even if
385 # it's from a feed, hence the sneaky restrictions identity check.
386 if self
.src
and restrictions
is self
.root_restrictions
:
387 if impl
.machine
!= 'src':
388 return "Not source code"
390 if impl
.machine
not in arch
.machine_ranks
:
391 if impl
.machine
== 'src':
393 return "Unsupported machine type"
396 def get_interface(self
, uri
):
397 """Get an interface from the L{iface_cache}. If it is missing or needs updating,
398 start a new download.
399 @rtype: L{model.Interface}"""
400 iface
= iface_cache
.get_interface(uri
)
402 if uri
in iface_cache
.pending
:
403 # Don't start another download while one is pending
404 # TODO: unless the pending version is very old
407 if iface
.last_modified
is None:
408 if self
.network_use
!= network_offline
:
409 debug("Interface not cached and not off-line. Downloading...")
410 self
.begin_iface_download(iface
)
412 if self
._warned
_offline
:
413 debug("Nothing known about interface, but we are off-line.")
416 info("Nothing known about interface '%s' and off-line. Trying feeds only.", uri
)
418 warn("Nothing known about interface '%s', but we are in off-line mode "
419 "(so not fetching).", uri
)
420 self
._warned
_offline
= True
421 elif not uri
.startswith('/'):
422 staleness
= time
.time() - (iface
.last_checked
or 0)
423 debug("Staleness for %s is %.2f hours", iface
, staleness
/ 3600.0)
425 if self
.network_use
!= network_offline
and self
.freshness
> 0 and staleness
> self
.freshness
:
426 debug("Updating %s", iface
)
427 self
.begin_iface_download(iface
, False)
428 #else: debug("Local interface, so not checking staleness.")
432 def begin_iface_download(self
, interface
, force
= False):
433 """Start downloading the interface, and add a callback to process it when
434 done. If it is already being downloaded, do nothing."""
436 debug("begin_iface_download %s (force = %d)", interface
, force
)
437 if interface
.uri
.startswith('/'):
439 debug("Need to download")
440 dl
= self
.handler
.get_download(interface
.uri
, force
= force
)
442 # Possibly we should handle this better, but it's unlikely anyone will need
443 # to use an interface as an icon or implementation as well, and some of the code
444 # assumes it's OK keep asking for the same interface to be downloaded.
445 info("Already have a handler for %s; not adding another", interface
)
448 def feed_downloaded(stream
):
449 pending
= PendingFeed(interface
.uri
, stream
)
450 iface_cache
.add_pending(pending
)
451 # This will trigger any required confirmations
452 self
.process_pending()
454 dl
.on_success
.append(feed_downloaded
)
456 def begin_impl_download(self
, impl
, retrieval_method
, force
= False):
457 """Start fetching impl, using retrieval_method. Each download started
458 will call monitor_download."""
460 assert retrieval_method
462 from zeroinstall
.zerostore
import manifest
463 alg
= impl
.id.split('=', 1)[0]
464 if alg
not in manifest
.algorithms
:
465 raise SafeException("Unknown digest algorithm '%s' for '%s' version %s" %
466 (alg
, impl
.interface
.get_name(), impl
.get_version()))
468 if isinstance(retrieval_method
, DownloadSource
):
469 def archive_ready(stream
):
470 iface_cache
.add_to_cache(retrieval_method
, stream
)
471 self
.begin_archive_download(retrieval_method
, success_callback
= archive_ready
, force
= force
)
472 elif isinstance(retrieval_method
, Recipe
):
473 _Cook(self
, impl
.id, retrieval_method
)
475 raise Exception("Unknown download type for '%s'" % retrieval_method
)
477 def begin_archive_download(self
, download_source
, success_callback
, force
= False):
478 """Start fetching an archive. You should normally call L{begin_impl_download}
479 instead, since it handles other kinds of retrieval method too."""
480 from zeroinstall
.zerostore
import unpack
481 mime_type
= download_source
.type
483 mime_type
= unpack
.type_from_url(download_source
.url
)
485 raise SafeException("No 'type' attribute on archive, and I can't guess from the name (%s)" % download_source
.url
)
486 unpack
.check_type_ok(mime_type
)
487 dl
= self
.handler
.get_download(download_source
.url
, force
= force
)
488 dl
.expected_size
= download_source
.size
+ (download_source
.start_offset
or 0)
489 dl
.on_success
.append(success_callback
)
492 def begin_icon_download(self
, interface
, force
= False):
493 """Start downloading an icon for this interface. On success, add it to the
494 icon cache. If the interface has no icon, do nothing."""
495 debug("begin_icon_download %s (force = %d)", interface
, force
)
497 # Find a suitable icon to download
498 for icon
in interface
.get_metadata(XMLNS_IFACE
, 'icon'):
499 type = icon
.getAttribute('type')
500 if type != 'image/png':
501 debug('Skipping non-PNG icon')
503 source
= icon
.getAttribute('href')
506 warn('Missing "href" attribute on <icon> in %s', interface
)
508 info('No PNG icons found in %s', interface
)
511 dl
= self
.handler
.get_download(source
, force
= force
)
513 # Possibly we should handle this better, but it's unlikely anyone will need
514 # to use an icon as an interface or implementation as well, and some of the code
515 # may assume it's OK keep asking for the same icon to be downloaded.
516 info("Already have a handler for %s; not adding another", source
)
518 dl
.on_success
.append(lambda stream
: self
.store_icon(interface
, stream
))
520 def store_icon(self
, interface
, stream
):
521 """Called when an icon has been successfully downloaded.
522 Subclasses may wish to wrap this to repaint the display."""
523 from zeroinstall
.injector
import basedir
525 icons_cache
= basedir
.save_cache_path(config_site
, 'interface_icons')
526 icon_file
= file(os
.path
.join(icons_cache
, escape(interface
.uri
)), 'w')
527 shutil
.copyfileobj(stream
, icon_file
)
529 def get_implementation_path(self
, impl
):
530 """Return the local path of impl.
532 @raise zeroinstall.zerostore.NotStored: if it needs to be added to the cache first."""
533 assert isinstance(impl
, Implementation
)
534 if impl
.id.startswith('/'):
536 return iface_cache
.stores
.lookup(impl
.id)
538 def get_implementation(self
, interface
):
539 """Get the chosen implementation.
540 @type interface: Interface
541 @rtype: L{model.Implementation}
542 @raise SafeException: if interface has not been fetched or no implementation could be
544 assert isinstance(interface
, Interface
)
546 if not interface
.name
and not interface
.feeds
:
547 raise SafeException("We don't have enough information to "
548 "run this program yet. "
549 "Need to download:\n%s" % interface
.uri
)
551 return self
.implementation
[interface
]
553 if interface
.implementations
:
555 if self
.network_use
== network_offline
:
556 offline
= "\nThis may be because 'Network Use' is set to Off-line."
557 raise SafeException("No usable implementation found for '%s'.%s" %
558 (interface
.name
, offline
))
561 def walk_interfaces(self
):
562 """@deprecated: use L{implementation} instead"""
563 return iter(self
.implementation
)
565 def get_cached(self
, impl
):
566 """Check whether an implementation is available locally.
567 @type impl: model.Implementation
570 if impl
.id.startswith('/'):
571 return os
.path
.exists(impl
.id)
574 path
= self
.get_implementation_path(impl
)
581 def add_to_cache(self
, source
, data
):
582 """Wrapper for L{iface_cache.IfaceCache.add_to_cache}."""
583 iface_cache
.add_to_cache(source
, data
)
585 def get_uncached_implementations(self
):
586 """List all chosen implementations which aren't yet available locally.
587 @rtype: [model.Implementation]"""
589 for iface
in self
.implementation
:
590 impl
= self
.implementation
[iface
]
592 if not self
.get_cached(impl
):
593 uncached
.append((iface
, impl
))
596 def refresh_all(self
, force
= True):
597 """Start downloading all feeds for all selected interfaces.
598 @param force: Whether to restart existing downloads."""
599 for x
in self
.implementation
:
600 self
.begin_iface_download(x
, force
)
601 for f
in self
.usable_feeds(x
):
602 feed_iface
= self
.get_interface(f
.uri
)
603 self
.begin_iface_download(feed_iface
, force
)
605 def interface_changed(self
, interface
):
606 """Callback used by L{iface_cache.IfaceCache.update_interface_from_network}."""
607 debug("interface_changed(%s): recalculating", interface
)
610 def get_feed_targets(self
, feed_iface_uri
):
611 """Return a list of Interfaces for which feed_iface can be a feed.
612 This is used by B{0launch --feed}.
613 @rtype: [model.Interface]
614 @raise SafeException: If there are no known feeds."""
615 feed_iface
= self
.get_interface(feed_iface_uri
)
616 if not feed_iface
.feed_for
:
617 if not feed_iface
.name
:
618 raise SafeException("Can't get feed targets for '%s'; failed to load interface." %
620 raise SafeException("Missing <feed-for> element in '%s'; "
621 "this interface can't be used as a feed." % feed_iface_uri
)
622 feed_targets
= feed_iface
.feed_for
623 if not feed_iface
.name
:
624 warn("Warning: unknown interface '%s'" % feed_iface_uri
)
625 return [self
.get_interface(uri
) for uri
in feed_targets
]
627 def get_icon_path(self
, iface
):
628 """Get an icon for this interface. If the icon is in the cache, use that.
629 If not, start a download. If we already started a download (successful or
631 @return: The cached icon's path, or None if no icon is currently available.
633 path
= iface_cache
.get_icon_path(iface
)
637 if self
.network_use
== network_offline
:
638 info("No icon present for %s, but off-line so not downloading", iface
)
641 self
.begin_icon_download(iface
)
644 def get_best_source(self
, impl
):
645 """Return the best download source for this implementation.
646 @rtype: L{model.RetrievalMethod}"""
647 if impl
.download_sources
:
648 return impl
.download_sources
[0]