Explain how to remove unneeded local feeds
[zeroinstall.git] / zeroinstall / injector / solver.py
blob6eea042f9e8627357d4b91a31c4d26a19c7ebdcc
1 """
2 Chooses a set of components to make a running program.
3 """
5 # Copyright (C) 2009, Thomas Leonard
6 # See the README file for details, or visit http://0install.net.
8 from zeroinstall import _
9 import locale
10 from logging import debug, warn, info
12 from zeroinstall.injector.reader import MissingLocalFeed
13 from zeroinstall.injector.arch import machine_groups
14 from zeroinstall.injector import model, sat, selections
16 class CommandInfo:
17 def __init__(self, name, command, impl, arch):
18 self.name = name
19 self.command = command
20 self.impl = impl
21 self.arch = arch
23 def __repr__(self):
24 name = "%s_%s_%s_%s" % (self.impl.feed.get_name(), self.impl.get_version(), self.impl.arch, self.name)
25 return name.replace('-', '_').replace('.', '_')
27 class ImplInfo:
28 is_dummy = False
30 def __init__(self, iface, impl, arch, dummy = False):
31 self.iface = iface
32 self.impl = impl
33 self.arch = arch
34 if dummy:
35 self.is_dummy = True
37 def __repr__(self):
38 name = "%s_%s_%s" % (self.impl.feed.get_name(), self.impl.get_version(), self.impl.arch)
39 return name.replace('-', '_').replace('.', '_')
41 def _get_command_name(runner):
42 """Returns the 'command' attribute of a <runner>, or 'run' if there isn't one."""
43 return runner.qdom.attrs.get('command', 'run')
45 class _DummyImpl(object):
46 requires = []
47 version = None
48 arch = None
49 commands = {}
51 def __repr__(self):
52 return "dummy"
54 feed = property(lambda self: self)
56 def get_version(self):
57 return "dummy"
59 def get_name(self):
60 return "dummy"
62 class Solver(object):
63 """Chooses a set of implementations to satisfy the requirements of a program and its user.
64 Typical use:
65 1. Create a Solver object and configure it
66 2. Call L{solve}.
67 3. If any of the returned feeds_used are stale or missing, you may like to start downloading them
68 4. If it is 'ready' then you can download and run the chosen versions.
69 @ivar selections: the chosen implementation of each interface
70 @type selections: L{selections.Selections}
71 @ivar requires: the selected dependencies for each chosen version
72 @type requires: {L{model.Interface}: [L{model.Dependency}]}
73 @ivar feeds_used: the feeds which contributed to the choice in L{selections}
74 @type feeds_used: set(str)
75 @ivar record_details: whether to record information about unselected implementations
76 @type record_details: {L{Interface}: [(L{Implementation}, str)]}
77 @ivar details: extra information, if record_details mode was used
78 @type details: {str: [(Implementation, comment)]}
79 """
80 __slots__ = ['selections', 'requires', 'feeds_used', 'details', 'record_details', 'ready']
82 def __init__(self):
83 self.selections = self.requires = self.feeds_used = self.details = None
84 self.record_details = False
85 self.ready = False
87 def solve(self, root_interface, root_arch, command_name = 'run'):
88 """Get the best implementation of root_interface and all of its dependencies.
89 @param root_interface: the URI of the program to be solved
90 @type root_interface: str
91 @param root_arch: the desired target architecture
92 @type root_arch: L{arch.Architecture}
93 @param command_name: which <command> element to select
94 @type command_name: str | None
95 @postcondition: self.ready, self.selections and self.feeds_used are updated"""
96 raise NotImplementedError("Abstract")
98 class SATSolver(Solver):
99 """Converts the problem to a set of pseudo-boolean constraints and uses a PB solver to solve them.
100 @ivar langs: the preferred languages (e.g. ["es_ES", "en"]). Initialised to the current locale.
101 @type langs: str"""
103 __slots__ = ['_failure_reason', 'config', 'extra_restrictions', '_lang_ranks', '_langs']
105 @property
106 def iface_cache(self):
107 return self.config.iface_cache # (deprecated; used by 0compile)
109 def __init__(self, config, extra_restrictions = None):
111 @param config: policy preferences (e.g. stability), the iface_cache and the stores to use
112 @type config: L{policy.Config}
113 @param extra_restrictions: extra restrictions on the chosen implementations
114 @type extra_restrictions: {L{model.Interface}: [L{model.Restriction}]}
116 Solver.__init__(self)
117 assert not isinstance(config, str), "API change!"
118 self.config = config
119 self.extra_restrictions = extra_restrictions or {}
121 # By default, prefer the current locale's language first and English second
122 self.langs = [locale.getlocale()[0] or 'en', 'en']
124 def set_langs(self, langs):
125 """Set the preferred languages.
126 @param lang: languages (and regions), first choice first
127 @type lang: [str]
129 # _lang_ranks is a map from locale string to score (higher is better)
130 _lang_ranks = {}
131 score = 0
132 i = len(langs)
133 # (is there are duplicates, the first occurance takes precedence)
134 while i > 0:
135 i -= 1
136 lang = langs[i].replace('_', '-')
137 _lang_ranks[lang.split('-')[0]] = score
138 _lang_ranks[lang] = score + 1
139 score += 2
140 self._langs = langs
141 self._lang_ranks = _lang_ranks
143 langs = property(lambda self: self._langs, set_langs)
145 def compare(self, interface, b, a, arch):
146 """Compare a and b to see which would be chosen first.
147 Does not consider whether the implementations are usable (check for that yourself first).
148 @param interface: The interface we are trying to resolve, which may
149 not be the interface of a or b if they are from feeds.
150 @rtype: int"""
152 # Languages we understand come first
153 a_langs = (a.langs or 'en').split()
154 b_langs = (b.langs or 'en').split()
155 my_langs = self._lang_ranks
157 r = cmp(max(my_langs.get(l.split('-')[0], -1) for l in a_langs),
158 max(my_langs.get(l.split('-')[0], -1) for l in b_langs))
159 if r: return r
161 a_stab = a.get_stability()
162 b_stab = b.get_stability()
164 # Preferred versions come first
165 r = cmp(a_stab == model.preferred, b_stab == model.preferred)
166 if r: return r
168 stores = self.config.stores
169 if self.config.network_use != model.network_full:
170 r = cmp(a.is_available(stores), b.is_available(stores))
171 if r: return r
173 # Packages that require admin access to install come last
174 r = cmp(b.requires_root_install, a.requires_root_install)
175 if r: return r
177 # Stability
178 stab_policy = interface.stability_policy
179 if not stab_policy:
180 if self.config.help_with_testing: stab_policy = model.testing
181 else: stab_policy = model.stable
183 if a_stab >= stab_policy: a_stab = model.preferred
184 if b_stab >= stab_policy: b_stab = model.preferred
186 r = cmp(a_stab, b_stab)
187 if r: return r
189 # Newer versions come before older ones
190 if a.id.startswith('package:') != b.id.startswith('package:'):
191 # If one of packages is native, do not compare full versions since
192 # it is useless to compare native and 0install version revisions
193 r = cmp(a.version[0], b.version[0])
194 if r: return r
195 # Othewise, prefer native package
196 return cmp(a.id.startswith('package:'), b.id.startswith('package:'))
197 else:
198 r = cmp(a.version, b.version)
199 if r: return r
201 # Get best OS
202 r = cmp(arch.os_ranks.get(b.os, None),
203 arch.os_ranks.get(a.os, None))
204 if r: return r
206 # Get best machine
207 r = cmp(arch.machine_ranks.get(b.machine, None),
208 arch.machine_ranks.get(a.machine, None))
209 if r: return r
211 # Slightly prefer languages specialised to our country
212 # (we know a and b have the same base language at this point)
213 r = cmp(max(my_langs.get(l, -1) for l in a_langs),
214 max(my_langs.get(l, -1) for l in b_langs))
215 if r: return r
217 # Slightly prefer cached versions
218 if self.config.network_use == model.network_full:
219 r = cmp(a.is_available(stores), b.is_available(stores))
220 if r: return r
222 return cmp(a.id, b.id)
224 def solve(self, root_interface, root_arch, command_name = 'run', closest_match = False):
225 # closest_match is used internally. It adds a lowest-ranked
226 # by valid implementation to every interface, so we can always
227 # select something. Useful for diagnostics.
229 # TODO: We need some way to figure out which feeds to include.
230 # Currently, we include any feed referenced from anywhere but
231 # this is probably too much. We could insert a dummy optimial
232 # implementation in stale/uncached feeds and see whether it
233 # selects that.
234 iface_cache = self.config.iface_cache
236 problem = sat.SATProblem()
238 impl_to_var = {} # Impl -> sat var
239 self.feeds_used = set()
240 self.requires = {}
241 self.ready = False
242 self.details = self.record_details and {}
244 self.selections = None
245 self._failure_reason = None
247 ifaces_processed = set()
249 impls_for_machine_group = {0 : []} # Machine group (e.g. "64") to [impl] in that group
250 for machine_group in machine_groups.values():
251 impls_for_machine_group[machine_group] = []
253 impls_for_iface = {} # Iface -> [impl]
255 group_clause_for = {} # Iface URI -> AtMostOneClause | bool
256 group_clause_for_command = {} # (Iface URI, command name) -> AtMostOneClause | bool
258 # Return the dependencies of impl that we should consider.
259 # Skips dependencies if the use flag isn't what we need.
260 # (note: impl may also be a model.Command)
261 def deps_in_use(impl, arch):
262 for dep in impl.requires:
263 use = dep.metadata.get("use", None)
264 if use not in arch.use:
265 continue
266 yield dep
268 # Add a clause so that if requiring_impl_var is True then an implementation
269 # matching 'dependency' must also be selected.
270 # Must have already done add_iface on dependency.interface.
271 def find_dependency_candidates(requiring_impl_var, dependency):
272 dep_iface = iface_cache.get_interface(dependency.interface)
273 dep_union = [sat.neg(requiring_impl_var)] # Either requiring_impl_var is False, or ...
274 for candidate in impls_for_iface[dep_iface]:
275 for r in dependency.restrictions:
276 if candidate.__class__ is not _DummyImpl and not r.meets_restriction(candidate):
277 #warn("%s rejected due to %s", candidate.get_version(), r)
278 if candidate.version is not None:
279 break
280 else:
281 c_var = impl_to_var.get(candidate, None)
282 if c_var is not None:
283 dep_union.append(c_var)
284 # else we filtered that version out, so ignore it
286 assert dep_union
287 problem.add_clause(dep_union)
289 def is_unusable(impl, restrictions, arch):
290 """@return: whether this implementation is unusable.
291 @rtype: bool"""
292 return get_unusable_reason(impl, restrictions, arch) != None
294 def get_unusable_reason(impl, restrictions, arch):
296 @param impl: Implementation to test.
297 @type restrictions: [L{model.Restriction}]
298 @return: The reason why this impl is unusable, or None if it's OK.
299 @rtype: str
300 @note: The restrictions are for the interface being requested, not the feed
301 of the implementation; they may be different when feeds are being used."""
302 for r in restrictions:
303 if not r.meets_restriction(impl):
304 return _("Incompatible with another selected implementation")
305 stability = impl.get_stability()
306 if stability <= model.buggy:
307 return stability.name
308 if (self.config.network_use == model.network_offline or not impl.download_sources) and not impl.is_available(self.config.stores):
309 if not impl.download_sources:
310 return _("No retrieval methods")
311 return _("Not cached and we are off-line")
312 if impl.os not in arch.os_ranks:
313 return _("Unsupported OS")
314 if impl.machine not in arch.machine_ranks:
315 if impl.machine == 'src':
316 return _("Source code")
317 return _("Unsupported machine type")
318 return None
320 def usable_feeds(iface, arch):
321 """Return all feeds for iface that support arch.
322 @rtype: generator(ZeroInstallFeed)"""
323 yield iface.uri
325 for f in iface_cache.get_feed_imports(iface):
326 # Note: when searching for src, None is not in machine_ranks
327 if f.os in arch.os_ranks and \
328 (f.machine is None or f.machine in arch.machine_ranks):
329 yield f.uri
330 else:
331 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
332 {'feed': f, 'os': f.os, 'machine': f.machine})
334 def add_iface(uri, arch):
335 """Name implementations from feed and assert that only one can be selected."""
336 if uri in ifaces_processed: return
337 ifaces_processed.add(uri)
339 iface = iface_cache.get_interface(uri)
341 impls = []
342 for f in usable_feeds(iface, arch):
343 self.feeds_used.add(f)
344 debug(_("Processing feed %s"), f)
346 try:
347 feed = iface_cache.get_feed(f)
348 if feed is None: continue
349 #if feed.name and iface.uri != feed.url and iface.uri not in feed.feed_for:
350 # info(_("Missing <feed-for> for '%(uri)s' in '%(feed)s'"), {'uri': iface.uri, 'feed': f})
352 if feed.implementations:
353 impls.extend(feed.implementations.values())
355 distro_feed_url = feed.get_distro_feed()
356 if distro_feed_url:
357 self.feeds_used.add(distro_feed_url)
358 distro_feed = iface_cache.get_feed(distro_feed_url)
359 if distro_feed.implementations:
360 impls.extend(distro_feed.implementations.values())
361 except MissingLocalFeed, ex:
362 warn(_("Missing local feed; if it's no longer required, remove it with:") +
363 '\n0install remove-feed ' + iface.uri + ' ' + f,
364 {'feed': f, 'interface': iface, 'exception': ex})
365 except Exception, ex:
366 warn(_("Failed to load feed %(feed)s for %(interface)s: %(exception)s"), {'feed': f, 'interface': iface, 'exception': ex})
367 #raise
369 impls.sort(lambda a, b: self.compare(iface, a, b, arch))
371 impls_for_iface[iface] = filtered_impls = []
373 my_extra_restrictions = self.extra_restrictions.get(iface, [])
375 if self.record_details:
376 self.details[iface] = [(impl, get_unusable_reason(impl, my_extra_restrictions, arch)) for impl in impls]
378 rank = 1
379 var_names = []
380 for impl in impls:
381 if is_unusable(impl, my_extra_restrictions, arch):
382 continue
384 filtered_impls.append(impl)
386 assert impl not in impl_to_var
387 v = problem.add_variable(ImplInfo(iface, impl, arch))
388 impl_to_var[impl] = v
389 rank += 1
390 var_names.append(v)
392 if impl.machine and impl.machine != 'src':
393 impls_for_machine_group[machine_groups.get(impl.machine, 0)].append(v)
395 for d in deps_in_use(impl, arch):
396 debug(_("Considering dependency %s"), d)
398 add_iface(d.interface, arch.child_arch)
400 # Must choose one version of d if impl is selected
401 find_dependency_candidates(v, d)
403 if closest_match:
404 dummy_impl = _DummyImpl()
405 dummy_var = problem.add_variable(ImplInfo(iface, dummy_impl, arch, dummy = True))
406 var_names.append(dummy_var)
407 impl_to_var[dummy_impl] = dummy_var
408 filtered_impls.append(dummy_impl)
410 # Only one implementation of this interface can be selected
411 if uri == root_interface:
412 if var_names:
413 clause = problem.at_most_one(var_names)
414 problem.add_clause(var_names) # at least one
415 else:
416 problem.impossible()
417 clause = False
418 elif var_names:
419 clause = problem.at_most_one(var_names)
420 else:
421 # Don't need to add to group_clause_for because we should
422 # never get a possible selection involving this.
423 return
425 assert clause is not True
426 assert clause is not None
427 if clause is not False:
428 group_clause_for[uri] = clause
430 def add_command_iface(uri, arch, command_name):
431 """Add every <command> in interface 'uri' with this name.
432 Each one depends on the corresponding implementation and only
433 one can be selected. If closest_match is on, include a dummy
434 command that can always be selected."""
436 # Check whether we've already processed this (interface,command) pair
437 existing = group_clause_for_command.get((uri, command_name), None)
438 if existing is not None:
439 return existing.lits
441 # First ensure that the interface itself has been processed
442 # We'll reuse the ordering of the implementations to order
443 # the commands too.
444 add_iface(uri, arch)
446 iface = iface_cache.get_interface(uri)
447 filtered_impls = impls_for_iface[iface]
449 var_names = []
450 for impl in filtered_impls:
451 command = impl.commands.get(command_name, None)
452 if not command:
453 if not isinstance(impl, _DummyImpl):
454 # Mark implementation as unselectable
455 problem.add_clause([sat.neg(impl_to_var[impl])])
456 continue
458 # We have a candidate <command>. Require that if it's selected
459 # then we select the corresponding <implementation> too.
460 command_var = problem.add_variable(CommandInfo(command_name, command, impl, arch))
461 problem.add_clause([impl_to_var[impl], sat.neg(command_var)])
463 var_names.append(command_var)
465 runner = command.get_runner()
466 for d in deps_in_use(command, arch):
467 if d is runner:
468 # With a <runner>, we depend on a command rather than on an
469 # implementation. This allows us to support recursive <runner>s, etc.
470 debug(_("Considering command runner %s"), d)
471 runner_command_name = _get_command_name(d)
472 runner_vars = add_command_iface(d.interface, arch.child_arch, runner_command_name)
474 # If the parent command is chosen, one of the candidate runner commands
475 # must be too. If there aren't any, then this command is unselectable.
476 problem.add_clause([sat.neg(command_var)] + runner_vars)
477 else:
478 debug(_("Considering command dependency %s"), d)
479 add_iface(d.interface, arch.child_arch)
481 # Must choose one version of d if impl is selected
482 find_dependency_candidates(command_var, d)
484 # Tell the user why we couldn't use this version
485 if self.record_details:
486 def new_reason(impl, old_reason):
487 if command_name in impl.commands:
488 return old_reason
489 return old_reason or (_('No %s command') % command_name)
490 self.details[iface] = [(impl, new_reason(impl, reason)) for (impl, reason) in self.details[iface]]
492 if closest_match:
493 dummy_command = problem.add_variable(None)
494 var_names.append(dummy_command)
496 if var_names:
497 # Can't select more than one of them.
498 assert (uri, command_name) not in group_clause_for_command
499 group_clause_for_command[(uri, command_name)] = problem.at_most_one(var_names)
501 return var_names
503 if command_name is None:
504 add_iface(root_interface, root_arch)
505 else:
506 commands = add_command_iface(root_interface, root_arch, command_name)
507 if len(commands) > int(closest_match):
508 # (we have at least one non-dummy command)
509 problem.add_clause(commands) # At least one
510 else:
511 # (note: might be because we haven't cached it yet)
512 info("No %s <command> in %s", command_name, root_interface)
514 impls = impls_for_iface[iface_cache.get_interface(root_interface)]
515 if impls == [] or (len(impls) == 1 and isinstance(impls[0], _DummyImpl)):
516 # There were no candidates at all.
517 self._failure_reason = _("Interface '%s' has no usable implementations") % root_interface
518 else:
519 # We had some candidates implementations, but none for the command we need
520 self._failure_reason = _("Interface '%s' cannot be executed directly; it is just a library "
521 "to be used by other programs (or missing '%s' command)") % (root_interface, command_name)
523 problem.impossible()
525 # Require m<group> to be true if we select an implementation in that group
526 m_groups = []
527 for machine_group, impls in impls_for_machine_group.iteritems():
528 m_group = 'm%d' % machine_group
529 group_var = problem.add_variable(m_group)
530 if impls:
531 for impl in impls:
532 problem.add_clause([group_var, sat.neg(impl)])
533 m_groups.append(group_var)
534 if m_groups:
535 m_groups_clause = problem.at_most_one(m_groups)
536 else:
537 m_groups_clause = None
539 def decide():
540 """Recurse through the current selections until we get to an interface with
541 no chosen version, then tell the solver to try the best version from that."""
543 def find_undecided_dep(impl_or_command, arch):
544 # Check for undecided dependencies of impl_or_command
545 for dep in deps_in_use(impl_or_command, arch):
546 if dep.qdom.name == 'runner':
547 dep_lit = find_undecided_command(dep.interface, _get_command_name(dep))
548 else:
549 dep_lit = find_undecided(dep.interface)
550 if dep_lit is not None:
551 return dep_lit
552 return None
554 seen = set()
555 def find_undecided(uri):
556 if uri in seen:
557 return # Break cycles
558 seen.add(uri)
560 group = group_clause_for[uri]
561 #print "Group for %s = %s" % (uri, group)
562 lit = group.current
563 if lit is None:
564 return group.best_undecided()
565 # else there was only one choice anyway
567 # Check for undecided dependencies
568 lit_info = problem.get_varinfo_for_lit(lit).obj
569 return find_undecided_dep(lit_info.impl, lit_info.arch)
571 def find_undecided_command(uri, name):
572 if name is None: return find_undecided(uri)
574 group = group_clause_for_command[(uri, name)]
575 lit = group.current
576 if lit is None:
577 return group.best_undecided()
578 # else we've already chosen which <command> to use
580 # Check for undecided command-specific dependencies, and then for
581 # implementation dependencies.
582 lit_info = problem.get_varinfo_for_lit(lit).obj
583 if lit_info is None:
584 assert closest_match
585 return None # (a dummy command added for better diagnostics; has no dependencies)
586 return find_undecided_dep(lit_info.command, lit_info.arch) or \
587 find_undecided_dep(lit_info.impl, lit_info.arch)
589 best = find_undecided_command(root_interface, command_name)
590 if best is not None:
591 return best
593 # If we're chosen everything we need, we can probably
594 # set everything else to False.
595 for group in group_clause_for.values() + group_clause_for_command.values() + [m_groups_clause]:
596 if group.current is None:
597 best = group.best_undecided()
598 if best is not None:
599 return sat.neg(best)
601 return None # Failed to find any valid combination
603 ready = problem.run_solver(decide) is True
605 if not ready and not closest_match:
606 # We failed while trying to do a real solve.
607 # Try a closest match solve to get a better
608 # error report for the user.
609 self.solve(root_interface, root_arch, command_name = command_name, closest_match = True)
610 else:
611 self.ready = ready and not closest_match
612 self.selections = selections.Selections(None)
613 self.selections.interface = root_interface
615 sels = self.selections.selections
617 for uri, group in group_clause_for.iteritems():
618 if group.current is not None:
619 lit_info = problem.get_varinfo_for_lit(group.current).obj
620 if lit_info.is_dummy:
621 sels[lit_info.iface.uri] = None
622 else:
623 impl = lit_info.impl
625 deps = self.requires[lit_info.iface] = []
626 for dep in deps_in_use(lit_info.impl, lit_info.arch):
627 deps.append(dep)
629 sels[lit_info.iface.uri] = selections.ImplSelection(lit_info.iface.uri, impl, deps)
631 def add_command(iface, name):
632 sel = sels.get(iface, None)
633 if sel:
634 command = sel.impl.commands[name]
635 self.selections.commands.append(command)
636 runner = command.get_runner()
637 if runner:
638 add_command(runner.metadata['interface'], _get_command_name(runner))
640 if command_name is not None:
641 add_command(root_interface, command_name)
643 def get_failure_reason(self):
644 """Return an exception explaining why the solve failed."""
645 assert not self.ready
647 if self._failure_reason:
648 return model.SafeException(self._failure_reason)
650 return model.SafeException(_("Can't find all required implementations:") + '\n' +
651 '\n'.join(["- %s -> %s" % (iface, self.selections[iface])
652 for iface in self.selections]))
654 DefaultSolver = SATSolver