2 Chooses a set of components to make a running program.
5 # Copyright (C) 2009, Thomas Leonard
6 # See the README file for details, or visit http://0install.net.
8 from zeroinstall
import _
10 from logging
import debug
, warn
, info
12 from zeroinstall
.injector
.reader
import MissingLocalFeed
13 from zeroinstall
.injector
.arch
import machine_groups
14 from zeroinstall
.injector
import model
, sat
, selections
17 def __init__(self
, name
, command
, impl
, arch
):
19 self
.command
= command
24 name
= "%s_%s_%s_%s" % (self
.impl
.feed
.get_name(), self
.impl
.get_version(), self
.impl
.arch
, self
.name
)
25 return name
.replace('-', '_').replace('.', '_')
30 def __init__(self
, iface
, impl
, arch
, dummy
= False):
38 name
= "%s_%s_%s" % (self
.impl
.feed
.get_name(), self
.impl
.get_version(), self
.impl
.arch
)
39 return name
.replace('-', '_').replace('.', '_')
41 def _get_command_name(runner
):
42 """Returns the 'command' attribute of a <runner>, or 'run' if there isn't one."""
43 return runner
.qdom
.attrs
.get('command', 'run')
45 class _DummyImpl(object):
54 feed
= property(lambda self
: self
)
56 def get_version(self
):
63 """Chooses a set of implementations to satisfy the requirements of a program and its user.
65 1. Create a Solver object and configure it
67 3. If any of the returned feeds_used are stale or missing, you may like to start downloading them
68 4. If it is 'ready' then you can download and run the chosen versions.
69 @ivar selections: the chosen implementation of each interface
70 @type selections: L{selections.Selections}
71 @ivar requires: the selected dependencies for each chosen version
72 @type requires: {L{model.Interface}: [L{model.Dependency}]}
73 @ivar feeds_used: the feeds which contributed to the choice in L{selections}
74 @type feeds_used: set(str)
75 @ivar record_details: whether to record information about unselected implementations
76 @type record_details: {L{Interface}: [(L{Implementation}, str)]}
77 @ivar details: extra information, if record_details mode was used
78 @type details: {str: [(Implementation, comment)]}
80 __slots__
= ['selections', 'requires', 'feeds_used', 'details', 'record_details', 'ready']
83 self
.selections
= self
.requires
= self
.feeds_used
= self
.details
= None
84 self
.record_details
= False
87 def solve(self
, root_interface
, root_arch
, command_name
= 'run'):
88 """Get the best implementation of root_interface and all of its dependencies.
89 @param root_interface: the URI of the program to be solved
90 @type root_interface: str
91 @param root_arch: the desired target architecture
92 @type root_arch: L{arch.Architecture}
93 @param command_name: which <command> element to select
94 @type command_name: str | None
95 @postcondition: self.ready, self.selections and self.feeds_used are updated"""
96 raise NotImplementedError("Abstract")
98 class SATSolver(Solver
):
99 """Converts the problem to a set of pseudo-boolean constraints and uses a PB solver to solve them.
100 @ivar langs: the preferred languages (e.g. ["es_ES", "en"]). Initialised to the current locale.
103 __slots__
= ['_failure_reason', 'config', 'extra_restrictions', '_lang_ranks', '_langs']
106 def iface_cache(self
):
107 return self
.config
.iface_cache
# (deprecated; used by 0compile)
109 def __init__(self
, config
, extra_restrictions
= None):
111 @param config: policy preferences (e.g. stability), the iface_cache and the stores to use
112 @type config: L{policy.Config}
113 @param extra_restrictions: extra restrictions on the chosen implementations
114 @type extra_restrictions: {L{model.Interface}: [L{model.Restriction}]}
116 Solver
.__init
__(self
)
117 assert not isinstance(config
, str), "API change!"
119 self
.extra_restrictions
= extra_restrictions
or {}
121 # By default, prefer the current locale's language first and English second
122 self
.langs
= [locale
.getlocale()[0] or 'en', 'en']
124 def set_langs(self
, langs
):
125 """Set the preferred languages.
126 @param lang: languages (and regions), first choice first
129 # _lang_ranks is a map from locale string to score (higher is better)
133 # (is there are duplicates, the first occurance takes precedence)
136 lang
= langs
[i
].replace('_', '-')
137 _lang_ranks
[lang
.split('-')[0]] = score
138 _lang_ranks
[lang
] = score
+ 1
141 self
._lang
_ranks
= _lang_ranks
143 langs
= property(lambda self
: self
._langs
, set_langs
)
145 def compare(self
, interface
, b
, a
, arch
):
146 """Compare a and b to see which would be chosen first.
147 Does not consider whether the implementations are usable (check for that yourself first).
148 @param interface: The interface we are trying to resolve, which may
149 not be the interface of a or b if they are from feeds.
152 # Languages we understand come first
153 a_langs
= (a
.langs
or 'en').split()
154 b_langs
= (b
.langs
or 'en').split()
155 my_langs
= self
._lang
_ranks
157 r
= cmp(max(my_langs
.get(l
.split('-')[0], -1) for l
in a_langs
),
158 max(my_langs
.get(l
.split('-')[0], -1) for l
in b_langs
))
161 a_stab
= a
.get_stability()
162 b_stab
= b
.get_stability()
164 # Preferred versions come first
165 r
= cmp(a_stab
== model
.preferred
, b_stab
== model
.preferred
)
168 stores
= self
.config
.stores
169 if self
.config
.network_use
!= model
.network_full
:
170 r
= cmp(a
.is_available(stores
), b
.is_available(stores
))
173 # Packages that require admin access to install come last
174 r
= cmp(b
.requires_root_install
, a
.requires_root_install
)
178 stab_policy
= interface
.stability_policy
180 if self
.config
.help_with_testing
: stab_policy
= model
.testing
181 else: stab_policy
= model
.stable
183 if a_stab
>= stab_policy
: a_stab
= model
.preferred
184 if b_stab
>= stab_policy
: b_stab
= model
.preferred
186 r
= cmp(a_stab
, b_stab
)
189 # Newer versions come before older ones
190 if a
.id.startswith('package:') != b
.id.startswith('package:'):
191 # If one of packages is native, do not compare full versions since
192 # it is useless to compare native and 0install version revisions
193 r
= cmp(a
.version
[0], b
.version
[0])
195 # Othewise, prefer native package
196 return cmp(a
.id.startswith('package:'), b
.id.startswith('package:'))
198 r
= cmp(a
.version
, b
.version
)
202 r
= cmp(arch
.os_ranks
.get(b
.os
, None),
203 arch
.os_ranks
.get(a
.os
, None))
207 r
= cmp(arch
.machine_ranks
.get(b
.machine
, None),
208 arch
.machine_ranks
.get(a
.machine
, None))
211 # Slightly prefer languages specialised to our country
212 # (we know a and b have the same base language at this point)
213 r
= cmp(max(my_langs
.get(l
, -1) for l
in a_langs
),
214 max(my_langs
.get(l
, -1) for l
in b_langs
))
217 # Slightly prefer cached versions
218 if self
.config
.network_use
== model
.network_full
:
219 r
= cmp(a
.is_available(stores
), b
.is_available(stores
))
222 return cmp(a
.id, b
.id)
224 def solve(self
, root_interface
, root_arch
, command_name
= 'run', closest_match
= False):
225 # closest_match is used internally. It adds a lowest-ranked
226 # by valid implementation to every interface, so we can always
227 # select something. Useful for diagnostics.
229 # TODO: We need some way to figure out which feeds to include.
230 # Currently, we include any feed referenced from anywhere but
231 # this is probably too much. We could insert a dummy optimial
232 # implementation in stale/uncached feeds and see whether it
234 iface_cache
= self
.config
.iface_cache
236 problem
= sat
.SATProblem()
238 impl_to_var
= {} # Impl -> sat var
239 self
.feeds_used
= set()
242 self
.details
= self
.record_details
and {}
244 self
.selections
= None
245 self
._failure
_reason
= None
247 ifaces_processed
= set()
249 impls_for_machine_group
= {0 : []} # Machine group (e.g. "64") to [impl] in that group
250 for machine_group
in machine_groups
.values():
251 impls_for_machine_group
[machine_group
] = []
253 impls_for_iface
= {} # Iface -> [impl]
255 group_clause_for
= {} # Iface URI -> AtMostOneClause | bool
256 group_clause_for_command
= {} # (Iface URI, command name) -> AtMostOneClause | bool
258 # Return the dependencies of impl that we should consider.
259 # Skips dependencies if the use flag isn't what we need.
260 # (note: impl may also be a model.Command)
261 def deps_in_use(impl
, arch
):
262 for dep
in impl
.requires
:
263 use
= dep
.metadata
.get("use", None)
264 if use
not in arch
.use
:
268 # Must have already done add_iface on dependency.interface.
269 # If dependency is essential:
270 # Add a clause so that if requiring_impl_var is True then an implementation
271 # matching 'dependency' must also be selected.
272 # If dependency is optional:
273 # Require that no incompatible version is selected.
274 def find_dependency_candidates(requiring_impl_var
, dependency
):
275 def meets_restrictions(candidate
):
276 for r
in dependency
.restrictions
:
277 if not r
.meets_restriction(candidate
):
278 #warn("%s rejected due to %s", candidate.get_version(), r)
282 essential
= dependency
.importance
== model
.Dependency
.Essential
284 dep_iface
= iface_cache
.get_interface(dependency
.interface
)
285 dep_union
= [sat
.neg(requiring_impl_var
)] # Either requiring_impl_var is False, or ...
286 for candidate
in impls_for_iface
[dep_iface
]:
287 if (candidate
.__class
__ is _DummyImpl
) or meets_restrictions(candidate
):
289 c_var
= impl_to_var
.get(candidate
, None)
290 if c_var
is not None:
291 dep_union
.append(c_var
)
292 # else we filtered that version out, so ignore it
294 # Candidate doesn't meet our requirements
295 # If the dependency is optional add a rule to make sure we don't
296 # select this candidate.
297 # (for essential dependencies this isn't necessary because we must
298 # select a good version and we can't select two)
300 c_var
= impl_to_var
.get(candidate
, None)
301 if c_var
is not None:
302 problem
.add_clause(dep_union
+ [sat
.neg(c_var
)])
303 # else we filtered that version out, so ignore it
306 problem
.add_clause(dep_union
)
308 def is_unusable(impl
, restrictions
, arch
):
309 """@return: whether this implementation is unusable.
311 return get_unusable_reason(impl
, restrictions
, arch
) != None
313 def get_unusable_reason(impl
, restrictions
, arch
):
315 @param impl: Implementation to test.
316 @type restrictions: [L{model.Restriction}]
317 @return: The reason why this impl is unusable, or None if it's OK.
319 @note: The restrictions are for the interface being requested, not the feed
320 of the implementation; they may be different when feeds are being used."""
321 for r
in restrictions
:
322 if not r
.meets_restriction(impl
):
323 return _("Incompatible with another selected implementation")
324 stability
= impl
.get_stability()
325 if stability
<= model
.buggy
:
326 return stability
.name
327 if (self
.config
.network_use
== model
.network_offline
or not impl
.download_sources
) and not impl
.is_available(self
.config
.stores
):
328 if not impl
.download_sources
:
329 return _("No retrieval methods")
330 return _("Not cached and we are off-line")
331 if impl
.os
not in arch
.os_ranks
:
332 return _("Unsupported OS")
333 if impl
.machine
not in arch
.machine_ranks
:
334 if impl
.machine
== 'src':
335 return _("Source code")
336 return _("Unsupported machine type")
339 def usable_feeds(iface
, arch
):
340 """Return all feeds for iface that support arch.
341 @rtype: generator(ZeroInstallFeed)"""
344 for f
in iface_cache
.get_feed_imports(iface
):
345 # Note: when searching for src, None is not in machine_ranks
346 if f
.os
in arch
.os_ranks
and \
347 (f
.machine
is None or f
.machine
in arch
.machine_ranks
):
350 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
351 {'feed': f
, 'os': f
.os
, 'machine': f
.machine
})
353 def add_iface(uri
, arch
):
354 """Name implementations from feed and assert that only one can be selected."""
355 if uri
in ifaces_processed
: return
356 ifaces_processed
.add(uri
)
358 iface
= iface_cache
.get_interface(uri
)
361 for f
in usable_feeds(iface
, arch
):
362 self
.feeds_used
.add(f
)
363 debug(_("Processing feed %s"), f
)
366 feed
= iface_cache
.get_feed(f
)
367 if feed
is None: continue
368 #if feed.name and iface.uri != feed.url and iface.uri not in feed.feed_for:
369 # info(_("Missing <feed-for> for '%(uri)s' in '%(feed)s'"), {'uri': iface.uri, 'feed': f})
371 if feed
.implementations
:
372 impls
.extend(feed
.implementations
.values())
374 distro_feed_url
= feed
.get_distro_feed()
376 self
.feeds_used
.add(distro_feed_url
)
377 distro_feed
= iface_cache
.get_feed(distro_feed_url
)
378 if distro_feed
.implementations
:
379 impls
.extend(distro_feed
.implementations
.values())
380 except MissingLocalFeed
as ex
:
381 warn(_("Missing local feed; if it's no longer required, remove it with:") +
382 '\n0install remove-feed ' + iface
.uri
+ ' ' + f
,
383 {'feed': f
, 'interface': iface
, 'exception': ex
})
384 except Exception as ex
:
385 warn(_("Failed to load feed %(feed)s for %(interface)s: %(exception)s"), {'feed': f
, 'interface': iface
, 'exception': ex
})
388 impls
.sort(lambda a
, b
: self
.compare(iface
, a
, b
, arch
))
390 impls_for_iface
[iface
] = filtered_impls
= []
392 my_extra_restrictions
= self
.extra_restrictions
.get(iface
, [])
394 if self
.record_details
:
395 self
.details
[iface
] = [(impl
, get_unusable_reason(impl
, my_extra_restrictions
, arch
)) for impl
in impls
]
400 if is_unusable(impl
, my_extra_restrictions
, arch
):
403 filtered_impls
.append(impl
)
405 assert impl
not in impl_to_var
406 v
= problem
.add_variable(ImplInfo(iface
, impl
, arch
))
407 impl_to_var
[impl
] = v
411 if impl
.machine
and impl
.machine
!= 'src':
412 impls_for_machine_group
[machine_groups
.get(impl
.machine
, 0)].append(v
)
414 for d
in deps_in_use(impl
, arch
):
415 debug(_("Considering dependency %s"), d
)
417 add_iface(d
.interface
, arch
.child_arch
)
419 # Must choose one version of d if impl is selected
420 find_dependency_candidates(v
, d
)
423 dummy_impl
= _DummyImpl()
424 dummy_var
= problem
.add_variable(ImplInfo(iface
, dummy_impl
, arch
, dummy
= True))
425 var_names
.append(dummy_var
)
426 impl_to_var
[dummy_impl
] = dummy_var
427 filtered_impls
.append(dummy_impl
)
429 # Only one implementation of this interface can be selected
430 if uri
== root_interface
:
432 clause
= problem
.at_most_one(var_names
)
433 problem
.add_clause(var_names
) # at least one
438 clause
= problem
.at_most_one(var_names
)
440 # Don't need to add to group_clause_for because we should
441 # never get a possible selection involving this.
444 assert clause
is not True
445 assert clause
is not None
446 if clause
is not False:
447 group_clause_for
[uri
] = clause
449 def add_command_iface(uri
, arch
, command_name
):
450 """Add every <command> in interface 'uri' with this name.
451 Each one depends on the corresponding implementation and only
452 one can be selected. If closest_match is on, include a dummy
453 command that can always be selected."""
455 # Check whether we've already processed this (interface,command) pair
456 existing
= group_clause_for_command
.get((uri
, command_name
), None)
457 if existing
is not None:
460 # First ensure that the interface itself has been processed
461 # We'll reuse the ordering of the implementations to order
465 iface
= iface_cache
.get_interface(uri
)
466 filtered_impls
= impls_for_iface
[iface
]
469 for impl
in filtered_impls
:
470 command
= impl
.commands
.get(command_name
, None)
472 if not isinstance(impl
, _DummyImpl
):
473 # Mark implementation as unselectable
474 problem
.add_clause([sat
.neg(impl_to_var
[impl
])])
477 # We have a candidate <command>. Require that if it's selected
478 # then we select the corresponding <implementation> too.
479 command_var
= problem
.add_variable(CommandInfo(command_name
, command
, impl
, arch
))
480 problem
.add_clause([impl_to_var
[impl
], sat
.neg(command_var
)])
482 var_names
.append(command_var
)
484 runner
= command
.get_runner()
485 for d
in deps_in_use(command
, arch
):
487 # With a <runner>, we depend on a command rather than on an
488 # implementation. This allows us to support recursive <runner>s, etc.
489 debug(_("Considering command runner %s"), d
)
490 runner_command_name
= _get_command_name(d
)
491 runner_vars
= add_command_iface(d
.interface
, arch
.child_arch
, runner_command_name
)
493 # If the parent command is chosen, one of the candidate runner commands
494 # must be too. If there aren't any, then this command is unselectable.
495 problem
.add_clause([sat
.neg(command_var
)] + runner_vars
)
497 debug(_("Considering command dependency %s"), d
)
498 add_iface(d
.interface
, arch
.child_arch
)
500 # Must choose one version of d if impl is selected
501 find_dependency_candidates(command_var
, d
)
503 # Tell the user why we couldn't use this version
504 if self
.record_details
:
505 def new_reason(impl
, old_reason
):
506 if command_name
in impl
.commands
:
508 return old_reason
or (_('No %s command') % command_name
)
509 self
.details
[iface
] = [(impl
, new_reason(impl
, reason
)) for (impl
, reason
) in self
.details
[iface
]]
512 dummy_command
= problem
.add_variable(None)
513 var_names
.append(dummy_command
)
516 # Can't select more than one of them.
517 assert (uri
, command_name
) not in group_clause_for_command
518 group_clause_for_command
[(uri
, command_name
)] = problem
.at_most_one(var_names
)
522 if command_name
is None:
523 add_iface(root_interface
, root_arch
)
525 commands
= add_command_iface(root_interface
, root_arch
, command_name
)
526 if len(commands
) > int(closest_match
):
527 # (we have at least one non-dummy command)
528 problem
.add_clause(commands
) # At least one
530 # (note: might be because we haven't cached it yet)
531 info("No %s <command> in %s", command_name
, root_interface
)
533 impls
= impls_for_iface
[iface_cache
.get_interface(root_interface
)]
534 if impls
== [] or (len(impls
) == 1 and isinstance(impls
[0], _DummyImpl
)):
535 # There were no candidates at all.
536 if self
.config
.network_use
== model
.network_offline
:
537 self
._failure
_reason
= _("Interface '%s' has no usable implementations in the cache (and 0install is in off-line mode)") % root_interface
539 self
._failure
_reason
= _("Interface '%s' has no usable implementations") % root_interface
541 # We had some candidates implementations, but none for the command we need
542 self
._failure
_reason
= _("Interface '%s' cannot be executed directly; it is just a library "
543 "to be used by other programs (or missing '%s' command)") % (root_interface
, command_name
)
547 # Require m<group> to be true if we select an implementation in that group
549 for machine_group
, impls
in impls_for_machine_group
.iteritems():
550 m_group
= 'm%d' % machine_group
551 group_var
= problem
.add_variable(m_group
)
554 problem
.add_clause([group_var
, sat
.neg(impl
)])
555 m_groups
.append(group_var
)
557 m_groups_clause
= problem
.at_most_one(m_groups
)
559 m_groups_clause
= None
562 """Recurse through the current selections until we get to an interface with
563 no chosen version, then tell the solver to try the best version from that."""
565 def find_undecided_dep(impl_or_command
, arch
):
566 # Check for undecided dependencies of impl_or_command
567 for dep
in deps_in_use(impl_or_command
, arch
):
568 if dep
.qdom
.name
== 'runner':
569 dep_lit
= find_undecided_command(dep
.interface
, _get_command_name(dep
))
571 dep_lit
= find_undecided(dep
.interface
)
572 if dep_lit
is not None:
577 def find_undecided(uri
):
579 return # Break cycles
582 group
= group_clause_for
[uri
]
583 #print "Group for %s = %s" % (uri, group)
586 return group
.best_undecided()
587 # else there was only one choice anyway
589 # Check for undecided dependencies
590 lit_info
= problem
.get_varinfo_for_lit(lit
).obj
591 return find_undecided_dep(lit_info
.impl
, lit_info
.arch
)
593 def find_undecided_command(uri
, name
):
594 if name
is None: return find_undecided(uri
)
596 group
= group_clause_for_command
[(uri
, name
)]
599 return group
.best_undecided()
600 # else we've already chosen which <command> to use
602 # Check for undecided command-specific dependencies, and then for
603 # implementation dependencies.
604 lit_info
= problem
.get_varinfo_for_lit(lit
).obj
607 return None # (a dummy command added for better diagnostics; has no dependencies)
608 return find_undecided_dep(lit_info
.command
, lit_info
.arch
) or \
609 find_undecided_dep(lit_info
.impl
, lit_info
.arch
)
611 best
= find_undecided_command(root_interface
, command_name
)
615 # If we're chosen everything we need, we can probably
616 # set everything else to False.
617 for group
in group_clause_for
.values() + group_clause_for_command
.values() + [m_groups_clause
]:
618 if group
.current
is None:
619 best
= group
.best_undecided()
623 return None # Failed to find any valid combination
625 ready
= problem
.run_solver(decide
) is True
627 if not ready
and not closest_match
:
628 # We failed while trying to do a real solve.
629 # Try a closest match solve to get a better
630 # error report for the user.
631 self
.solve(root_interface
, root_arch
, command_name
= command_name
, closest_match
= True)
633 self
.ready
= ready
and not closest_match
634 self
.selections
= selections
.Selections(None)
635 self
.selections
.interface
= root_interface
637 sels
= self
.selections
.selections
639 for uri
, group
in group_clause_for
.iteritems():
640 if group
.current
is not None:
641 lit_info
= problem
.get_varinfo_for_lit(group
.current
).obj
642 if lit_info
.is_dummy
:
643 sels
[lit_info
.iface
.uri
] = None
647 deps
= self
.requires
[lit_info
.iface
] = []
648 for dep
in deps_in_use(lit_info
.impl
, lit_info
.arch
):
651 sels
[lit_info
.iface
.uri
] = selections
.ImplSelection(lit_info
.iface
.uri
, impl
, deps
)
653 def add_command(iface
, name
):
654 sel
= sels
.get(iface
, None)
656 command
= sel
.impl
.commands
[name
]
657 sel
._used
_commands
.add(name
)
658 runner
= command
.get_runner()
660 add_command(runner
.metadata
['interface'], _get_command_name(runner
))
662 if command_name
is not None:
663 self
.selections
.command
= command_name
664 add_command(root_interface
, command_name
)
666 def get_failure_reason(self
):
667 """Return an exception explaining why the solve failed."""
668 assert not self
.ready
670 if self
._failure
_reason
:
671 return model
.SafeException(self
._failure
_reason
)
673 return model
.SafeException(_("Can't find all required implementations:") + '\n' +
674 '\n'.join(["- %s -> %s" % (iface
, self
.selections
[iface
])
675 for iface
in self
.selections
]))
677 DefaultSolver
= SATSolver