2 Chooses a set of components to make a running program.
5 # Copyright (C) 2009, Thomas Leonard
6 # See the README file for details, or visit http://0install.net.
8 from zeroinstall
import _
10 from logging
import debug
, warn
, info
12 from zeroinstall
.zerostore
import BadDigest
, NotStored
14 from zeroinstall
.injector
.arch
import machine_groups
15 from zeroinstall
.injector
import model
, sat
, selections
17 def _get_cached(stores
, impl
):
18 """Check whether an implementation is available locally.
19 @type impl: model.Implementation
22 if isinstance(impl
, model
.DistributionImplementation
):
25 return os
.path
.exists(impl
.local_path
)
29 warn("No digests given for %s!", impl
)
31 path
= stores
.lookup_any(impl
.digests
)
40 def __init__(self
, name
, command
, impl
, arch
):
42 self
.command
= command
47 name
= "%s_%s_%s_%s" % (self
.impl
.feed
.get_name(), self
.impl
.get_version(), self
.impl
.arch
, self
.name
)
48 return name
.replace('-', '_').replace('.', '_')
53 def __init__(self
, iface
, impl
, arch
, dummy
= False):
61 name
= "%s_%s_%s" % (self
.impl
.feed
.get_name(), self
.impl
.get_version(), self
.impl
.arch
)
62 return name
.replace('-', '_').replace('.', '_')
64 def _get_command_name(runner
):
65 """Returns the 'command' attribute of a <runner>, or 'run' if there isn't one."""
66 return runner
.qdom
.attrs
.get('command', 'run')
68 class _DummyImpl(object):
77 feed
= property(lambda self
: self
)
79 def get_version(self
):
86 """Chooses a set of implementations to satisfy the requirements of a program and its user.
88 1. Create a Solver object and configure it
90 3. If any of the returned feeds_used are stale or missing, you may like to start downloading them
91 4. If it is 'ready' then you can download and run the chosen versions.
92 @ivar selections: the chosen implementation of each interface
93 @type selections: L{selections.Selections}
94 @ivar requires: the selected dependencies for each chosen version
95 @type requires: {L{model.Interface}: [L{model.Dependency}]}
96 @ivar feeds_used: the feeds which contributed to the choice in L{selections}
97 @type feeds_used: set(str)
98 @ivar record_details: whether to record information about unselected implementations
99 @type record_details: {L{Interface}: [(L{Implementation}, str)]}
100 @ivar details: extra information, if record_details mode was used
101 @type details: {str: [(Implementation, comment)]}
103 __slots__
= ['selections', 'requires', 'feeds_used', 'details', 'record_details', 'ready']
106 self
.selections
= self
.requires
= self
.feeds_used
= self
.details
= None
107 self
.record_details
= False
110 def solve(self
, root_interface
, root_arch
, command_name
= 'run'):
111 """Get the best implementation of root_interface and all of its dependencies.
112 @param root_interface: the URI of the program to be solved
113 @type root_interface: str
114 @param root_arch: the desired target architecture
115 @type root_arch: L{arch.Architecture}
116 @param command_name: which <command> element to select
117 @type command_name: str | None
118 @postcondition: self.ready, self.selections and self.feeds_used are updated"""
119 raise NotImplementedError("Abstract")
121 class SATSolver(Solver
):
122 __slots__
= ['_failure_reason', 'config', 'extra_restrictions', 'langs']
124 """Converts the problem to a set of pseudo-boolean constraints and uses a PB solver to solve them.
125 @ivar langs: the preferred languages (e.g. ["es_ES", "en"]). Initialised to the current locale.
127 def __init__(self
, config
, extra_restrictions
= None):
129 @param network_use: how much use to make of the network
130 @type network_use: L{model.network_levels}
131 @param config: policy preferences (e.g. stability), the iface_cache and the stores to use
132 @type config: L{policy.Config}
133 @param extra_restrictions: extra restrictions on the chosen implementations
134 @type extra_restrictions: {L{model.Interface}: [L{model.Restriction}]}
136 Solver
.__init
__(self
)
137 assert not isinstance(config
, str), "API change!"
139 self
.extra_restrictions
= extra_restrictions
or {}
141 self
.langs
= [locale
.getlocale()[0] or 'en']
143 def compare(self
, interface
, b
, a
, arch
):
144 """Compare a and b to see which would be chosen first.
145 Does not consider whether the implementations are usable (check for that yourself first).
146 @param interface: The interface we are trying to resolve, which may
147 not be the interface of a or b if they are from feeds.
150 # Languages we understand come first
151 a_langs
= (a
.langs
or 'en').split()
152 b_langs
= (b
.langs
or 'en').split()
153 main_langs
= set(l
.split('_')[0] for l
in self
.langs
)
154 r
= cmp(any(l
.split('_')[0] in main_langs
for l
in a_langs
),
155 any(l
.split('_')[0] in main_langs
for l
in b_langs
))
158 a_stab
= a
.get_stability()
159 b_stab
= b
.get_stability()
161 # Preferred versions come first
162 r
= cmp(a_stab
== model
.preferred
, b_stab
== model
.preferred
)
165 stores
= self
.config
.stores
166 if self
.config
.network_use
!= model
.network_full
:
167 r
= cmp(_get_cached(stores
, a
), _get_cached(stores
, b
))
170 # Packages that require admin access to install come last
171 r
= cmp(b
.requires_root_install
, a
.requires_root_install
)
175 stab_policy
= interface
.stability_policy
177 if self
.config
.help_with_testing
: stab_policy
= model
.testing
178 else: stab_policy
= model
.stable
180 if a_stab
>= stab_policy
: a_stab
= model
.preferred
181 if b_stab
>= stab_policy
: b_stab
= model
.preferred
183 r
= cmp(a_stab
, b_stab
)
186 # Newer versions come before older ones
187 if a
.id.startswith('package:') != b
.id.startswith('package:'):
188 # If one of packages is native, do not compare full versions since
189 # it is useless to compare native and 0install version revisions
190 r
= cmp(a
.version
[0], b
.version
[0])
192 # Othewise, prefer native package
193 return cmp(a
.id.startswith('package:'), b
.id.startswith('package:'))
195 r
= cmp(a
.version
, b
.version
)
199 r
= cmp(arch
.os_ranks
.get(b
.os
, None),
200 arch
.os_ranks
.get(a
.os
, None))
204 r
= cmp(arch
.machine_ranks
.get(b
.machine
, None),
205 arch
.machine_ranks
.get(a
.machine
, None))
208 # Slightly prefer languages specialised to our country
209 r
= cmp(any(l
in self
.langs
for l
in a_langs
),
210 any(l
in self
.langs
for l
in b_langs
))
213 # Slightly prefer cached versions
214 if self
.config
.network_use
== model
.network_full
:
215 r
= cmp(_get_cached(stores
, a
), _get_cached(stores
, b
))
218 return cmp(a
.id, b
.id)
220 def solve(self
, root_interface
, root_arch
, command_name
= 'run', closest_match
= False):
221 # closest_match is used internally. It adds a lowest-ranked
222 # by valid implementation to every interface, so we can always
223 # select something. Useful for diagnostics.
225 # TODO: We need some way to figure out which feeds to include.
226 # Currently, we include any feed referenced from anywhere but
227 # this is probably too much. We could insert a dummy optimial
228 # implementation in stale/uncached feeds and see whether it
230 iface_cache
= self
.config
.iface_cache
232 problem
= sat
.SATProblem()
234 impl_to_var
= {} # Impl -> sat var
235 self
.feeds_used
= set()
238 self
.details
= self
.record_details
and {}
240 self
.selections
= None
241 self
._failure
_reason
= None
243 ifaces_processed
= set()
245 impls_for_machine_group
= {0 : []} # Machine group (e.g. "64") to [impl] in that group
246 for machine_group
in machine_groups
.values():
247 impls_for_machine_group
[machine_group
] = []
249 impls_for_iface
= {} # Iface -> [impl]
251 group_clause_for
= {} # Iface URI -> AtMostOneClause | bool
252 group_clause_for_command
= {} # (Iface URI, command name) -> AtMostOneClause | bool
254 # Return the dependencies of impl that we should consider.
255 # Skips dependencies if the use flag isn't what we need.
256 # (note: impl may also be a model.Command)
257 def deps_in_use(impl
, arch
):
258 for dep
in impl
.requires
:
259 use
= dep
.metadata
.get("use", None)
260 if use
not in arch
.use
:
264 # Add a clause so that if requiring_impl_var is True then an implementation
265 # matching 'dependency' must also be selected.
266 # Must have already done add_iface on dependency.interface.
267 def find_dependency_candidates(requiring_impl_var
, dependency
):
268 dep_iface
= iface_cache
.get_interface(dependency
.interface
)
269 dep_union
= [sat
.neg(requiring_impl_var
)] # Either requiring_impl_var is False, or ...
270 for candidate
in impls_for_iface
[dep_iface
]:
271 for r
in dependency
.restrictions
:
272 if candidate
.__class
__ is not _DummyImpl
and not r
.meets_restriction(candidate
):
273 #warn("%s rejected due to %s", candidate.get_version(), r)
274 if candidate
.version
is not None:
277 c_var
= impl_to_var
.get(candidate
, None)
278 if c_var
is not None:
279 dep_union
.append(c_var
)
280 # else we filtered that version out, so ignore it
283 problem
.add_clause(dep_union
)
285 def is_unusable(impl
, restrictions
, arch
):
286 """@return: whether this implementation is unusable.
288 return get_unusable_reason(impl
, restrictions
, arch
) != None
290 def get_unusable_reason(impl
, restrictions
, arch
):
292 @param impl: Implementation to test.
293 @type restrictions: [L{model.Restriction}]
294 @return: The reason why this impl is unusable, or None if it's OK.
296 @note: The restrictions are for the interface being requested, not the feed
297 of the implementation; they may be different when feeds are being used."""
298 for r
in restrictions
:
299 if not r
.meets_restriction(impl
):
300 return _("Incompatible with another selected implementation")
301 stability
= impl
.get_stability()
302 if stability
<= model
.buggy
:
303 return stability
.name
304 if (self
.config
.network_use
== model
.network_offline
or not impl
.download_sources
) and not _get_cached(self
.config
.stores
, impl
):
305 if not impl
.download_sources
:
306 return _("No retrieval methods")
307 return _("Not cached and we are off-line")
308 if impl
.os
not in arch
.os_ranks
:
309 return _("Unsupported OS")
310 if impl
.machine
not in arch
.machine_ranks
:
311 if impl
.machine
== 'src':
312 return _("Source code")
313 return _("Unsupported machine type")
316 def usable_feeds(iface
, arch
):
317 """Return all feeds for iface that support arch.
318 @rtype: generator(ZeroInstallFeed)"""
321 for f
in iface_cache
.get_feed_imports(iface
):
322 # Note: when searching for src, None is not in machine_ranks
323 if f
.os
in arch
.os_ranks
and \
324 (f
.machine
is None or f
.machine
in arch
.machine_ranks
):
327 debug(_("Skipping '%(feed)s'; unsupported architecture %(os)s-%(machine)s"),
328 {'feed': f
, 'os': f
.os
, 'machine': f
.machine
})
330 def add_iface(uri
, arch
):
331 """Name implementations from feed and assert that only one can be selected."""
332 if uri
in ifaces_processed
: return
333 ifaces_processed
.add(uri
)
335 iface
= iface_cache
.get_interface(uri
)
338 for f
in usable_feeds(iface
, arch
):
339 self
.feeds_used
.add(f
)
340 debug(_("Processing feed %s"), f
)
343 feed
= iface_cache
.get_feed(f
)
344 if feed
is None: continue
345 #if feed.name and iface.uri != feed.url and iface.uri not in feed.feed_for:
346 # info(_("Missing <feed-for> for '%(uri)s' in '%(feed)s'"), {'uri': iface.uri, 'feed': f})
348 if feed
.implementations
:
349 impls
.extend(feed
.implementations
.values())
351 distro_feed_url
= feed
.get_distro_feed()
353 self
.feeds_used
.add(distro_feed_url
)
354 distro_feed
= iface_cache
.get_feed(distro_feed_url
)
355 if distro_feed
.implementations
:
356 impls
.extend(distro_feed
.implementations
.values())
357 except Exception, ex
:
358 warn(_("Failed to load feed %(feed)s for %(interface)s: %(exception)s"), {'feed': f
, 'interface': iface
, 'exception': ex
})
361 impls
.sort(lambda a
, b
: self
.compare(iface
, a
, b
, arch
))
363 impls_for_iface
[iface
] = filtered_impls
= []
365 my_extra_restrictions
= self
.extra_restrictions
.get(iface
, [])
367 if self
.record_details
:
368 self
.details
[iface
] = [(impl
, get_unusable_reason(impl
, my_extra_restrictions
, arch
)) for impl
in impls
]
373 if is_unusable(impl
, my_extra_restrictions
, arch
):
376 filtered_impls
.append(impl
)
378 assert impl
not in impl_to_var
379 v
= problem
.add_variable(ImplInfo(iface
, impl
, arch
))
380 impl_to_var
[impl
] = v
384 if impl
.machine
and impl
.machine
!= 'src':
385 impls_for_machine_group
[machine_groups
.get(impl
.machine
, 0)].append(v
)
387 for d
in deps_in_use(impl
, arch
):
388 debug(_("Considering dependency %s"), d
)
390 add_iface(d
.interface
, arch
.child_arch
)
392 # Must choose one version of d if impl is selected
393 find_dependency_candidates(v
, d
)
396 dummy_impl
= _DummyImpl()
397 dummy_var
= problem
.add_variable(ImplInfo(iface
, dummy_impl
, arch
, dummy
= True))
398 var_names
.append(dummy_var
)
399 impl_to_var
[dummy_impl
] = dummy_var
400 filtered_impls
.append(dummy_impl
)
402 # Only one implementation of this interface can be selected
403 if uri
== root_interface
:
405 clause
= problem
.at_most_one(var_names
)
406 problem
.add_clause(var_names
) # at least one
411 clause
= problem
.at_most_one(var_names
)
413 # Don't need to add to group_clause_for because we should
414 # never get a possible selection involving this.
417 assert clause
is not True
418 assert clause
is not None
419 if clause
is not False:
420 group_clause_for
[uri
] = clause
422 def add_command_iface(uri
, arch
, command_name
):
423 """Add every <command> in interface 'uri' with this name.
424 Each one depends on the corresponding implementation and only
425 one can be selected."""
426 # First ensure that the interface itself has been processed
427 # We'll reuse the ordering of the implementations to order
431 iface
= iface_cache
.get_interface(uri
)
432 filtered_impls
= impls_for_iface
[iface
]
435 for impl
in filtered_impls
:
436 command
= impl
.commands
.get(command_name
, None)
438 if not isinstance(impl
, _DummyImpl
):
439 # Mark implementation as unselectable
440 problem
.add_clause([sat
.neg(impl_to_var
[impl
])])
443 # We have a candidate <command>. Require that if it's selected
444 # then we select the corresponding <implementation> too.
445 command_var
= problem
.add_variable(CommandInfo(command_name
, command
, impl
, arch
))
446 problem
.add_clause([impl_to_var
[impl
], sat
.neg(command_var
)])
448 var_names
.append(command_var
)
450 runner
= command
.get_runner()
451 for d
in deps_in_use(command
, arch
):
453 # With a <runner>, we depend on a command rather than on an
454 # implementation. This allows us to support recursive <runner>s, etc.
455 debug(_("Considering command runner %s"), d
)
456 runner_command_name
= _get_command_name(d
)
457 runner_vars
= add_command_iface(d
.interface
, arch
.child_arch
, runner_command_name
)
460 dummy_command
= problem
.add_variable(None)
461 runner_vars
.append(dummy_command
)
462 # If the parent command is chosen, one of the candidate runner commands
463 # must be too. If there aren't any, then this command is unselectable.
464 problem
.add_clause([sat
.neg(command_var
)] + runner_vars
)
466 # Can't select more than one of them.
467 group_clause_for_command
[(d
.interface
, runner_command_name
)] = problem
.at_most_one(runner_vars
)
469 debug(_("Considering command dependency %s"), d
)
470 add_iface(d
.interface
, arch
.child_arch
)
472 # Must choose one version of d if impl is selected
473 find_dependency_candidates(command_var
, d
)
475 # Tell the user why we couldn't use this version
476 if self
.record_details
:
477 def new_reason(impl
, old_reason
):
478 if command_name
in impl
.commands
:
480 return old_reason
or (_('No %s command') % command_name
)
481 self
.details
[iface
] = [(impl
, new_reason(impl
, reason
)) for (impl
, reason
) in self
.details
[iface
]]
485 if command_name
is None:
486 add_iface(root_interface
, root_arch
)
488 commands
= add_command_iface(root_interface
, root_arch
, command_name
)
490 problem
.add_clause(commands
) # At least one
491 group_clause_for_command
[(root_interface
, command_name
)] = problem
.at_most_one(commands
)
493 # (note: might be because we haven't cached it yet)
494 info("No %s <command> in %s", command_name
, root_interface
)
496 impls
= impls_for_iface
[iface_cache
.get_interface(root_interface
)]
497 if impls
== [] or (len(impls
) == 1 and isinstance(impls
[0], _DummyImpl
)):
498 # There were no candidates at all.
499 self
._failure
_reason
= _("Interface '%s' has no usable implementations") % root_interface
501 # We had some candidates implementations, but none for the command we need
502 self
._failure
_reason
= _("Interface '%s' cannot be executed directly; it is just a library "
503 "to be used by other programs (or missing '%s' command)") % (root_interface
, command_name
)
507 # Require m<group> to be true if we select an implementation in that group
509 for machine_group
, impls
in impls_for_machine_group
.iteritems():
510 m_group
= 'm%d' % machine_group
511 group_var
= problem
.add_variable(m_group
)
514 problem
.add_clause([group_var
, sat
.neg(impl
)])
515 m_groups
.append(group_var
)
517 m_groups_clause
= problem
.at_most_one(m_groups
)
519 m_groups_clause
= None
522 """Recurse through the current selections until we get to an interface with
523 no chosen version, then tell the solver to try the best version from that."""
525 def find_undecided_dep(impl_or_command
, arch
):
526 # Check for undecided dependencies of impl_or_command
527 for dep
in deps_in_use(impl_or_command
, arch
):
528 if dep
.qdom
.name
== 'runner':
529 dep_lit
= find_undecided_command(dep
.interface
, _get_command_name(dep
))
531 dep_lit
= find_undecided(dep
.interface
)
532 if dep_lit
is not None:
537 def find_undecided(uri
):
539 return # Break cycles
542 group
= group_clause_for
[uri
]
543 #print "Group for %s = %s" % (uri, group)
546 return group
.best_undecided()
547 # else there was only one choice anyway
549 # Check for undecided dependencies
550 lit_info
= problem
.get_varinfo_for_lit(lit
).obj
551 return find_undecided_dep(lit_info
.impl
, lit_info
.arch
)
553 def find_undecided_command(uri
, name
):
554 if name
is None: return find_undecided(uri
)
556 group
= group_clause_for_command
[(uri
, name
)]
559 return group
.best_undecided()
560 # else we've already chosen which <command> to use
562 # Check for undecided command-specific dependencies, and then for
563 # implementation dependencies.
564 lit_info
= problem
.get_varinfo_for_lit(lit
).obj
567 return None # (a dummy command added for better diagnostics; has no dependencies)
568 return find_undecided_dep(lit_info
.command
, lit_info
.arch
) or \
569 find_undecided_dep(lit_info
.impl
, lit_info
.arch
)
571 best
= find_undecided_command(root_interface
, command_name
)
575 # If we're chosen everything we need, we can probably
576 # set everything else to False.
577 for group
in group_clause_for
.values() + group_clause_for_command
.values() + [m_groups_clause
]:
578 if group
.current
is None:
579 best
= group
.best_undecided()
583 return None # Failed to find any valid combination
585 ready
= problem
.run_solver(decide
) is True
587 if not ready
and not closest_match
:
588 # We failed while trying to do a real solve.
589 # Try a closest match solve to get a better
590 # error report for the user.
591 self
.solve(root_interface
, root_arch
, command_name
= command_name
, closest_match
= True)
593 self
.ready
= ready
and not closest_match
594 self
.selections
= selections
.Selections(None)
595 self
.selections
.interface
= root_interface
597 sels
= self
.selections
.selections
599 for uri
, group
in group_clause_for
.iteritems():
600 if group
.current
is not None:
601 lit_info
= problem
.get_varinfo_for_lit(group
.current
).obj
602 if lit_info
.is_dummy
:
603 sels
[lit_info
.iface
.uri
] = None
607 deps
= self
.requires
[lit_info
.iface
] = []
608 for dep
in deps_in_use(lit_info
.impl
, lit_info
.arch
):
611 sels
[lit_info
.iface
.uri
] = selections
.ImplSelection(lit_info
.iface
.uri
, impl
, deps
)
613 def add_command(iface
, name
):
614 sel
= sels
.get(iface
, None)
616 command
= sel
.impl
.commands
[name
]
617 self
.selections
.commands
.append(command
)
618 runner
= command
.get_runner()
620 add_command(runner
.metadata
['interface'], _get_command_name(runner
))
622 if command_name
is not None:
623 add_command(root_interface
, command_name
)
625 def get_failure_reason(self
):
626 """Return an exception explaining why the solve failed."""
627 assert not self
.ready
629 if self
._failure
_reason
:
630 return model
.SafeException(self
._failure
_reason
)
632 return model
.SafeException(_("Can't find all required implementations:") + '\n' +
633 '\n'.join(["- %s -> %s" % (iface
, self
.selections
[iface
])
634 for iface
in self
.selections
]))
636 DefaultSolver
= SATSolver