2 Chooses a set of components to make a running program.
5 # Copyright (C) 2009, Thomas Leonard
6 # See the README file for details, or visit http://0install.net.
8 from zeroinstall
import _
9 import os
, tempfile
, subprocess
, sys
, locale
10 from logging
import debug
, warn
, info
12 from zeroinstall
.zerostore
import BadDigest
, NotStored
14 from zeroinstall
.injector
.arch
import machine_groups
15 from zeroinstall
.injector
import model
, sat
, selections
17 def _get_cached(stores
, impl
):
18 """Check whether an implementation is available locally.
19 @type impl: model.Implementation
22 if isinstance(impl
, model
.DistributionImplementation
):
25 return os
.path
.exists(impl
.local_path
)
29 warn("No digests given for %s!", impl
)
31 path
= stores
.lookup_any(impl
.digests
)
40 def __init__(self
, name
, command
, impl
, arch
):
42 self
.command
= command
47 name
= "%s_%s_%s_%s" % (self
.impl
.feed
.get_name(), self
.impl
.get_version(), self
.impl
.arch
, self
.name
)
48 return name
.replace('-', '_').replace('.', '_')
53 def __init__(self
, iface
, impl
, arch
, dummy
= False):
61 name
= "%s_%s_%s" % (self
.impl
.feed
.get_name(), self
.impl
.get_version(), self
.impl
.arch
)
62 return name
.replace('-', '_').replace('.', '_')
64 def _get_command_name(runner
):
65 """Returns the 'command' attribute of a <runner>, or 'run' if there isn't one."""
66 return runner
.qdom
.attrs
.get('command', 'run')
68 class _DummyImpl(object):
77 feed
= property(lambda self
: self
)
79 def get_version(self
):
86 """Chooses a set of implementations to satisfy the requirements of a program and its user.
88 1. Create a Solver object and configure it
90 3. If any of the returned feeds_used are stale or missing, you may like to start downloading them
91 4. If it is 'ready' then you can download and run the chosen versions.
92 @ivar selections: the chosen implementation of each interface
93 @type selections: L{selections.Selections}
94 @ivar requires: the selected dependencies for each chosen version
95 @type requires: {L{model.Interface}: [L{model.Dependency}]}
96 @ivar feeds_used: the feeds which contributed to the choice in L{selections}
97 @type feeds_used: set(str)
98 @ivar record_details: whether to record information about unselected implementations
99 @type record_details: {L{Interface}: [(L{Implementation}, str)]}
100 @ivar details: extra information, if record_details mode was used
101 @type details: {str: [(Implementation, comment)]}
103 __slots__
= ['selections', 'requires', 'feeds_used', 'details', 'record_details', 'ready']
106 self
.selections
= self
.requires
= self
.feeds_used
= self
.details
= None
107 self
.record_details
= False
110 def solve(self
, root_interface
, root_arch
, command_name
= 'run'):
111 """Get the best implementation of root_interface and all of its dependencies.
112 @param root_interface: the URI of the program to be solved
113 @type root_interface: str
114 @param root_arch: the desired target architecture
115 @type root_arch: L{arch.Architecture}
116 @param command_name: which <command> element to select
117 @type command_name: str | None
118 @postcondition: self.ready, self.selections and self.feeds_used are updated"""
119 raise NotImplementedError("Abstract")
121 class SATSolver(Solver
):
122 __slots__
= ['_failure_reason', 'config', 'iface_cache', 'stores', 'extra_restrictions', 'langs']
124 """Converts the problem to a set of pseudo-boolean constraints and uses a PB solver to solve them.
125 @ivar langs: the preferred languages (e.g. ["es_ES", "en"]). Initialised to the current locale.
127 def __init__(self
, config
, iface_cache
, stores
, extra_restrictions
= None):
129 @param network_use: how much use to make of the network
130 @type network_use: L{model.network_levels}
131 @param iface_cache: a cache of feeds containing information about available versions
132 @type iface_cache: L{iface_cache.IfaceCache}
133 @param stores: a cached of implementations (affects choice when offline or when minimising network use)
134 @type stores: L{zerostore.Stores}
135 @param extra_restrictions: extra restrictions on the chosen implementations
136 @type extra_restrictions: {L{model.Interface}: [L{model.Restriction}]}
138 Solver
.__init
__(self
)
139 assert not isinstance(config
, str), "API change!"
141 self
.iface_cache
= iface_cache
143 self
.extra_restrictions
= extra_restrictions
or {}
145 self
.langs
= [locale
.getlocale()[0] or 'en']
148 def network_use(self
):
149 return self
.config
.get('global', 'network_use')
152 def help_with_testing(self
):
153 return self
.config
.getboolean('global', 'help_with_testing')
155 def compare(self
, interface
, b
, a
, arch
):
156 """Compare a and b to see which would be chosen first.
157 Does not consider whether the implementations are usable (check for that yourself first).
158 @param interface: The interface we are trying to resolve, which may
159 not be the interface of a or b if they are from feeds.
162 # Languages we understand come first
163 a_langs
= (a
.langs
or 'en').split()
164 b_langs
= (b
.langs
or 'en').split()
165 main_langs
= set(l
.split('_')[0] for l
in self
.langs
)
166 r
= cmp(any(l
.split('_')[0] in main_langs
for l
in a_langs
),
167 any(l
.split('_')[0] in main_langs
for l
in b_langs
))
170 a_stab
= a
.get_stability()
171 b_stab
= b
.get_stability()
173 # Preferred versions come first
174 r
= cmp(a_stab
== model
.preferred
, b_stab
== model
.preferred
)
178 if self
.network_use
!= model
.network_full
:
179 r
= cmp(_get_cached(stores
, a
), _get_cached(stores
, b
))
182 # Packages that require admin access to install come last
183 r
= cmp(b
.requires_root_install
, a
.requires_root_install
)
187 stab_policy
= interface
.stability_policy
189 if self
.help_with_testing
: stab_policy
= model
.testing
190 else: stab_policy
= model
.stable
192 if a_stab
>= stab_policy
: a_stab
= model
.preferred
193 if b_stab
>= stab_policy
: b_stab
= model
.preferred
195 r
= cmp(a_stab
, b_stab
)
198 # Newer versions come before older ones
199 if a
.id.startswith('package:') != b
.id.startswith('package:'):
200 # If one of packages is native, do not compare full versions since
201 # it is useless to compare native and 0install version revisions
202 r
= cmp(a
.version
[0], b
.version
[0])
204 # Othewise, prefer native package
205 return cmp(a
.id.startswith('package:'), b
.id.startswith('package:'))
207 r
= cmp(a
.version
, b
.version
)
211 r
= cmp(arch
.os_ranks
.get(b
.os
, None),
212 arch
.os_ranks
.get(a
.os
, None))
216 r
= cmp(arch
.machine_ranks
.get(b
.machine
, None),
217 arch
.machine_ranks
.get(a
.machine
, None))
220 # Slightly prefer languages specialised to our country
221 r
= cmp(any(l
in self
.langs
for l
in a_langs
),
222 any(l
in self
.langs
for l
in b_langs
))
225 # Slightly prefer cached versions
226 if self
.network_use
== model
.network_full
:
227 r
= cmp(_get_cached(stores
, a
), _get_cached(stores
, b
))
230 return cmp(a
.id, b
.id)
232 def solve(self
, root_interface
, root_arch
, command_name
= 'run', closest_match
= False):
233 # closest_match is used internally. It adds a lowest-ranked
234 # by valid implementation to every interface, so we can always
235 # select something. Useful for diagnostics.
237 # TODO: We need some way to figure out which feeds to include.
238 # Currently, we include any feed referenced from anywhere but
239 # this is probably too much. We could insert a dummy optimial
240 # implementation in stale/uncached feeds and see whether it
244 problem
= sat
.SATProblem()
246 impl_to_var
= {} # Impl -> sat var
247 self
.feeds_used
= set()
250 self
.details
= self
.record_details
and {}
252 self
.selections
= None
253 self
._failure
_reason
= None
255 ifaces_processed
= set()
257 impls_for_machine_group
= {0 : []} # Machine group (e.g. "64") to [impl] in that group
258 for machine_group
in machine_groups
.values():
259 impls_for_machine_group
[machine_group
] = []
261 impls_for_iface
= {} # Iface -> [impl]
263 group_clause_for
= {} # Iface URI -> AtMostOneClause | bool
264 group_clause_for_command
= {} # (Iface URI, command name) -> AtMostOneClause | bool
266 # Return the dependencies of impl that we should consider.
267 # Skips dependencies if the use flag isn't what we need.
268 # (note: impl may also be a model.Command)
269 def deps_in_use(impl
, arch
):
270 for dep
in impl
.requires
:
271 use
= dep
.metadata
.get("use", None)
272 if use
not in arch
.use
:
276 # Add a clause so that if requiring_impl_var is True then an implementation
277 # matching 'dependency' must also be selected.
278 # Must have already done add_iface on dependency.interface.
279 def find_dependency_candidates(requiring_impl_var
, dependency
):
280 dep_iface
= self
.iface_cache
.get_interface(dependency
.interface
)
281 dep_union
= [sat
.neg(requiring_impl_var
)] # Either requiring_impl_var is False, or ...
282 for candidate
in impls_for_iface
[dep_iface
]:
283 for r
in dependency
.restrictions
:
284 if candidate
.__class
__ is not _DummyImpl
and not r
.meets_restriction(candidate
):
285 #warn("%s rejected due to %s", candidate.get_version(), r)
286 if candidate
.version
is not None:
289 c_var
= impl_to_var
.get(candidate
, None)
290 if c_var
is not None:
291 dep_union
.append(c_var
)
292 # else we filtered that version out, so ignore it
295 problem
.add_clause(dep_union
)
297 def is_unusable(impl
, restrictions
, arch
):
298 """@return: whether this implementation is unusable.
300 return get_unusable_reason(impl
, restrictions
, arch
) != None
302 def get_unusable_reason(impl
, restrictions
, arch
):
304 @param impl: Implementation to test.
305 @type restrictions: [L{model.Restriction}]
306 @return: The reason why this impl is unusable, or None if it's OK.
308 @note: The restrictions are for the interface being requested, not the feed
309 of the implementation; they may be different when feeds are being used."""
310 for r
in restrictions
:
311 if not r
.meets_restriction(impl
):
312 return _("Incompatible with another selected implementation")
313 stability
= impl
.get_stability()
314 if stability
<= model
.buggy
:
315 return stability
.name
316 if (self
.network_use
== model
.network_offline
or not impl
.download_sources
) and not _get_cached(self
.stores
, impl
):
317 if not impl
.download_sources
:
318 return _("No retrieval methods")
319 return _("Not cached and we are off-line")
320 if impl
.os
not in arch
.os_ranks
:
321 return _("Unsupported OS")
322 if impl
.machine
not in arch
.machine_ranks
:
323 if impl
.machine
== 'src':
324 return _("Source code")
325 return _("Unsupported machine type")
328 def usable_feeds(iface
, arch
):
329 """Return all feeds for iface that support arch.
330 @rtype: generator(ZeroInstallFeed)"""
333 for f
in self
.iface_cache
.get_feed_imports(iface
):
334 # Note: when searching for src, None is not in machine_ranks
335 if f
.os
in arch
.os_ranks
and \
336 (f
.machine
is None or f
.machine
in arch
.machine_ranks
):
339 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
340 {'feed': f
, 'os': f
.os
, 'machine': f
.machine
})
342 def add_iface(uri
, arch
):
343 """Name implementations from feed and assert that only one can be selected."""
344 if uri
in ifaces_processed
: return
345 ifaces_processed
.add(uri
)
346 iface_name
= 'i%d' % len(ifaces_processed
)
348 iface
= self
.iface_cache
.get_interface(uri
)
351 for f
in usable_feeds(iface
, arch
):
352 self
.feeds_used
.add(f
)
353 debug(_("Processing feed %s"), f
)
356 feed
= self
.iface_cache
.get_feed(f
)
357 if feed
is None: continue
358 #if feed.name and iface.uri != feed.url and iface.uri not in feed.feed_for:
359 # info(_("Missing <feed-for> for '%(uri)s' in '%(feed)s'"), {'uri': iface.uri, 'feed': f})
361 if feed
.implementations
:
362 impls
.extend(feed
.implementations
.values())
364 distro_feed_url
= feed
.get_distro_feed()
366 self
.feeds_used
.add(distro_feed_url
)
367 distro_feed
= self
.iface_cache
.get_feed(distro_feed_url
)
368 if distro_feed
.implementations
:
369 impls
.extend(distro_feed
.implementations
.values())
370 except Exception, ex
:
371 warn(_("Failed to load feed %(feed)s for %(interface)s: %(exception)s"), {'feed': f
, 'interface': iface
, 'exception': ex
})
373 impls
.sort(lambda a
, b
: self
.compare(iface
, a
, b
, arch
))
375 impls_for_iface
[iface
] = filtered_impls
= []
377 my_extra_restrictions
= self
.extra_restrictions
.get(iface
, [])
379 if self
.record_details
:
380 self
.details
[iface
] = [(impl
, get_unusable_reason(impl
, my_extra_restrictions
, arch
)) for impl
in impls
]
385 if is_unusable(impl
, my_extra_restrictions
, arch
):
388 filtered_impls
.append(impl
)
390 assert impl
not in impl_to_var
391 v
= problem
.add_variable(ImplInfo(iface
, impl
, arch
))
392 impl_to_var
[impl
] = v
396 if impl
.machine
and impl
.machine
!= 'src':
397 impls_for_machine_group
[machine_groups
.get(impl
.machine
, 0)].append(v
)
399 for d
in deps_in_use(impl
, arch
):
400 debug(_("Considering dependency %s"), d
)
402 add_iface(d
.interface
, arch
.child_arch
)
404 # Must choose one version of d if impl is selected
405 find_dependency_candidates(v
, d
)
408 dummy_impl
= _DummyImpl()
409 dummy_var
= problem
.add_variable(ImplInfo(iface
, dummy_impl
, arch
, dummy
= True))
410 var_names
.append(dummy_var
)
411 impl_to_var
[dummy_impl
] = dummy_var
412 filtered_impls
.append(dummy_impl
)
414 # Only one implementation of this interface can be selected
415 if uri
== root_interface
:
417 clause
= problem
.at_most_one(var_names
)
418 problem
.add_clause(var_names
) # at least one
423 clause
= problem
.at_most_one(var_names
)
425 # Don't need to add to group_clause_for because we should
426 # never get a possible selection involving this.
429 assert clause
is not True
430 assert clause
is not None
431 if clause
is not False:
432 group_clause_for
[uri
] = clause
434 def add_command_iface(uri
, arch
, command_name
):
435 """Add every <command> in interface 'uri' with this name.
436 Each one depends on the corresponding implementation and only
437 one can be selected."""
438 # First ensure that the interface itself has been processed
439 # We'll reuse the ordering of the implementations to order
443 iface
= self
.iface_cache
.get_interface(uri
)
444 filtered_impls
= impls_for_iface
[iface
]
447 for impl
in filtered_impls
:
448 command
= impl
.commands
.get(command_name
, None)
450 if not isinstance(impl
, _DummyImpl
):
451 # Mark implementation as unselectable
452 problem
.add_clause([sat
.neg(impl_to_var
[impl
])])
455 # We have a candidate <command>. Require that if it's selected
456 # then we select the corresponding <implementation> too.
457 command_var
= problem
.add_variable(CommandInfo(command_name
, command
, impl
, arch
))
458 problem
.add_clause([impl_to_var
[impl
], sat
.neg(command_var
)])
460 var_names
.append(command_var
)
462 runner
= command
.get_runner()
463 for d
in deps_in_use(command
, arch
):
465 # With a <runner>, we depend on a command rather than on an
466 # implementation. This allows us to support recursive <runner>s, etc.
467 debug(_("Considering command runner %s"), d
)
468 runner_command_name
= _get_command_name(d
)
469 runner_vars
= add_command_iface(d
.interface
, arch
.child_arch
, runner_command_name
)
472 dummy_command
= problem
.add_variable(None)
473 runner_vars
.append(dummy_command
)
474 # If the parent command is chosen, one of the candidate runner commands
475 # must be too. If there aren't any, then this command is unselectable.
476 problem
.add_clause([sat
.neg(command_var
)] + runner_vars
)
478 # Can't select more than one of them.
479 group_clause_for_command
[(d
.interface
, runner_command_name
)] = problem
.at_most_one(runner_vars
)
481 debug(_("Considering command dependency %s"), d
)
482 add_iface(d
.interface
, arch
.child_arch
)
484 # Must choose one version of d if impl is selected
485 find_dependency_candidates(command_var
, d
)
487 # Tell the user why we couldn't use this version
488 if self
.record_details
:
489 def new_reason(impl
, old_reason
):
490 if command_name
in impl
.commands
:
492 return old_reason
or (_('No %s command') % command_name
)
493 self
.details
[iface
] = [(impl
, new_reason(impl
, reason
)) for (impl
, reason
) in self
.details
[iface
]]
497 if command_name
is None:
498 add_iface(root_interface
, root_arch
)
500 commands
= add_command_iface(root_interface
, root_arch
, command_name
)
502 problem
.add_clause(commands
) # At least one
503 group_clause_for_command
[(root_interface
, command_name
)] = problem
.at_most_one(commands
)
505 # (note: might be because we haven't cached it yet)
506 info("No %s <command> in %s", command_name
, root_interface
)
508 impls
= impls_for_iface
[self
.iface_cache
.get_interface(root_interface
)]
509 if impls
== [] or (len(impls
) == 1 and isinstance(impls
[0], _DummyImpl
)):
510 # There were no candidates at all.
511 self
._failure
_reason
= _("Interface '%s' has no usable implementations") % root_interface
513 # We had some candidates implementations, but none for the command we need
514 self
._failure
_reason
= _("Interface '%s' cannot be executed directly; it is just a library "
515 "to be used by other programs (or missing '%s' command)") % (root_interface
, command_name
)
519 # Require m<group> to be true if we select an implementation in that group
521 for machine_group
, impls
in impls_for_machine_group
.iteritems():
522 m_group
= 'm%d' % machine_group
523 group_var
= problem
.add_variable(m_group
)
526 problem
.add_clause([group_var
, sat
.neg(impl
)])
527 m_groups
.append(group_var
)
529 m_groups_clause
= problem
.at_most_one(m_groups
)
531 m_groups_clause
= None
534 """Recurse through the current selections until we get to an interface with
535 no chosen version, then tell the solver to try the best version from that."""
537 def find_undecided_dep(impl_or_command
, arch
):
538 # Check for undecided dependencies of impl_or_command
539 for dep
in deps_in_use(impl_or_command
, arch
):
540 if dep
.qdom
.name
== 'runner':
541 dep_lit
= find_undecided_command(dep
.interface
, _get_command_name(dep
))
543 dep_lit
= find_undecided(dep
.interface
)
544 if dep_lit
is not None:
549 def find_undecided(uri
):
551 return # Break cycles
554 group
= group_clause_for
[uri
]
555 #print "Group for %s = %s" % (uri, group)
558 return group
.best_undecided()
559 # else there was only one choice anyway
561 # Check for undecided dependencies
562 lit_info
= problem
.get_varinfo_for_lit(lit
).obj
563 return find_undecided_dep(lit_info
.impl
, lit_info
.arch
)
565 def find_undecided_command(uri
, name
):
566 if name
is None: return find_undecided(uri
)
568 group
= group_clause_for_command
[(uri
, name
)]
571 return group
.best_undecided()
572 # else we've already chosen which <command> to use
574 # Check for undecided command-specific dependencies, and then for
575 # implementation dependencies.
576 lit_info
= problem
.get_varinfo_for_lit(lit
).obj
579 return None # (a dummy command added for better diagnostics; has no dependencies)
580 return find_undecided_dep(lit_info
.command
, lit_info
.arch
) or \
581 find_undecided_dep(lit_info
.impl
, lit_info
.arch
)
583 best
= find_undecided_command(root_interface
, command_name
)
587 # If we're chosen everything we need, we can probably
588 # set everything else to False.
589 for group
in group_clause_for
.values() + group_clause_for_command
.values() + [m_groups_clause
]:
590 if group
.current
is None:
591 best
= group
.best_undecided()
595 return None # Failed to find any valid combination
597 ready
= problem
.run_solver(decide
) is True
599 if not ready
and not closest_match
:
600 # We failed while trying to do a real solve.
601 # Try a closest match solve to get a better
602 # error report for the user.
603 self
.solve(root_interface
, root_arch
, command_name
= command_name
, closest_match
= True)
605 self
.ready
= ready
and not closest_match
606 self
.selections
= selections
.Selections(None)
607 self
.selections
.interface
= root_interface
609 sels
= self
.selections
.selections
611 for uri
, group
in group_clause_for
.iteritems():
612 if group
.current
is not None:
613 lit_info
= problem
.get_varinfo_for_lit(group
.current
).obj
614 if lit_info
.is_dummy
:
615 sels
[lit_info
.iface
.uri
] = None
619 deps
= self
.requires
[lit_info
.iface
] = []
620 for dep
in deps_in_use(lit_info
.impl
, lit_info
.arch
):
623 sels
[lit_info
.iface
.uri
] = selections
.ImplSelection(lit_info
.iface
.uri
, impl
, deps
)
625 def add_command(iface
, name
):
626 sel
= sels
.get(iface
, None)
628 command
= sel
.impl
.commands
[name
]
629 self
.selections
.commands
.append(command
)
630 runner
= command
.get_runner()
632 add_command(runner
.metadata
['interface'], _get_command_name(runner
))
634 if command_name
is not None:
635 add_command(root_interface
, command_name
)
637 def get_failure_reason(self
):
638 """Return an exception explaining why the solve failed."""
639 assert not self
.ready
641 if self
._failure
_reason
:
642 return model
.SafeException(self
._failure
_reason
)
644 return model
.SafeException(_("Can't find all required implementations:") + '\n' +
645 '\n'.join(["- %s -> %s" % (iface
, self
.selections
[iface
])
646 for iface
in self
.selections
]))
648 DefaultSolver
= SATSolver