| blob | 947aab6697951fe63b7cb2f793315434665756c4 |
1 --- libs.py.2.2_rc8 2008-08-14 15:45:30.000000000 -0500
2 +++ pym/portage/sets/libs.py 2008-08-17 20:59:09.000000000 -0500
3 @@ -2,10 +2,18 @@
4 # Distributed under the terms of the GNU General Public License v2
5 # $Id: libs.py 10759 2008-06-22 04:04:50Z zmedico $
7 +import os
8 +import re
9 +import time
10 +from portage.dbapi.vartree import dblink
11 +from portage.versions import catsplit
12 from portage.sets.base import PackageSet
13 from portage.sets import get_boolean
14 from portage.versions import catpkgsplit
16 +__all__ = ["LibraryConsumerSet", "PreservedLibraryConsumerSet",
17 + "MissingLibraryConsumerSet"]
18 +
19 class LibraryConsumerSet(PackageSet):
20 _operations = ["merge", "unmerge"]
22 @@ -45,3 +53,338 @@
23 debug = get_boolean(options, "debug", False)
24 return PreservedLibraryConsumerSet(trees["vartree"].dbapi, debug)
25 singleBuilder = classmethod(singleBuilder)
26 +
27 +
28 +class MissingLibraryConsumerSet(LibraryConsumerSet):
29 +
30 + """
31 + This class is the set of packages to emerge due to missing libraries.
32 +
33 + This class scans binaries for missing and broken shared library dependencies
34 + and fixes them by emerging the packages containing the broken binaries.
35 +
36 + The user may also emerge packages containing consumers of specified
37 + libraries by passing the name or a python regular expression through the
38 + environment variable, LIBRARY. Due to a limitation in passing flags to
39 + package sets through the portage cli, the user must set environment
40 + variables to modify the behaviour of this package set. So if the
41 + environment variable LIBRARY is set, the behaviour of this set changes.
42 +
43 + """
44 +
45 + description = "The set of packages to emerge due to missing libraries."
46 + _operations = ["merge"]
47 +
48 + def __init__(self, vardbapi, debug=False):
49 + super(MissingLibraryConsumerSet, self).__init__(vardbapi, debug)
50 + # FIXME Since we can't get command line arguments from the user, the
51 + # soname can be passed through an environment variable for now.
52 + self.libraryRegexp = os.getenv("LIBRARY")
53 + self.root = self.dbapi.root
54 + self.linkmap = self.dbapi.linkmap
55 +
56 + def load(self):
57 + # brokenDependencies: object -> set-of-unsatisfied-dependencies, where
58 + # object is an installed binary/library and
59 + # set-of-unsatisfied-dependencies are sonames or libraries required by
60 + # the object but have no corresponding libraries to fulfill the
61 + # dependency.
62 + brokenDependencies = {}
63 + atoms = set()
64 +
65 + # If the LIBRARY environment variable is set, the resulting package set
66 + # will be packages containing consumers of the libraries matched by the
67 + # variable.
68 + if self.libraryRegexp:
69 + atoms = self.findAtomsOfLibraryConsumers(self.libraryRegexp)
70 + self._setAtoms(atoms)
71 + if self.debug:
72 + print
73 + print "atoms to be emerged:"
74 + for x in sorted(atoms):
75 + print x
76 + return
77 +
78 + if self.debug:
79 + timeStart = time.time()
80 + self.linkmap.rebuild()
81 + if self.debug:
82 + timeRebuild = time.time() - timeStart
83 +
84 + if self.debug:
85 + timeStart = time.time()
86 + self.linkmap.listConsumers()
87 + if self.debug:
88 + timeConsumers = time.time() - timeStart
89 +
90 + if self.debug:
91 + timeStart = time.time()
92 + self.linkmap.listProviders()
93 + if self.debug:
94 + timeProviders = time.time() - timeStart
95 +
96 + # Get the list of broken dependencies from LinkageMap.
97 + if self.debug:
98 + timeStart = time.time()
99 + brokenDependencies = self.linkmap.listBrokenBinaries()
100 + if self.debug:
101 + timeListBrokenBinaries = time.time() - timeStart
102 +
103 + # Add broken libtool libraries into the brokenDependencies dict.
104 + if self.debug:
105 + timeStart = time.time()
106 + brokenDependencies.update(self.listBrokenLibtoolLibraries())
107 + if self.debug:
108 + timeLibtool = time.time() - timeStart
109 +
110 + # FIXME Too many atoms may be emerged because libraries in binary
111 + # packages are not being handled properly eg openoffice, nvidia-drivers,
112 + # sun-jdk. Certain binaries are run in an environment where additional
113 + # library paths are added via LD_LIBRARY_PATH. Since these paths aren't
114 + # registered in _obj_properties, they appear broken (and are if not run
115 + # in the correct environment). I have to determine if libraries and lib
116 + # paths should be masked using /etc/revdep-rebuild/* as done in
117 + # revdep-rebuild or if there is a better way to identify and deal with
118 + # these problematic packages (or if something entirely different should
119 + # be done). For now directory and library masks are used.
120 +
121 + # Remove masked directories and libraries.
122 + if self.debug:
123 + timeStart = time.time()
124 + if brokenDependencies:
125 + brokenDependencies = self.removeMaskedDependencies(brokenDependencies)
126 + if self.debug:
127 + timeMask = time.time() - timeStart
128 +
129 + # Determine atoms to emerge based on broken objects in
130 + # brokenDependencies.
131 + if self.debug:
132 + timeStart = time.time()
133 + if brokenDependencies:
134 + atoms = self.mapPathsToAtoms(set(brokenDependencies.keys()))
135 + if self.debug:
136 + timeAtoms = time.time() - timeStart
137 +
138 + # Debug output
139 + if self.debug:
140 + print
141 + print len(brokenDependencies), "brokenDependencies:"
142 + for x in sorted(brokenDependencies.keys()):
143 + print
144 + print x, "->"
145 + print '\t', brokenDependencies[x]
146 + print
147 + print "atoms to be emerged:"
148 + for x in sorted(atoms):
149 + print x
150 + print
151 + print "Rebuild time:", timeRebuild
152 + print "Providers time:", timeProviders
153 + print "Consumers time:", timeConsumers
154 + print "Broken binaries time:", timeListBrokenBinaries
155 + print "Broken libtool time:", timeLibtool
156 + print "Remove mask time:", timeMask
157 + print "mapPathsToAtoms time:", timeAtoms
158 + print
159 +
160 + self._setAtoms(atoms)
161 +
162 + def removeMaskedDependencies(self, dependencies):
163 + """
164 + Remove all masked dependencies and return the updated mapping.
165 +
166 + @param dependencies: dependencies from which to removed masked
167 + dependencies
168 + @type dependencies: dict (example: {'/usr/bin/foo': set(['libfoo.so'])})
169 + @rtype: dict
170 + @return: shallow copy of dependencies with masked items removed
171 +
172 + """
173 + rValue = dependencies.copy()
174 + dirMask, libMask = self.getDependencyMasks()
175 +
176 + # Remove entries that are masked.
177 + if dirMask or libMask:
178 + if self.debug:
179 + print "The following are masked:"
180 + for binary, libSet in rValue.items():
181 + for directory in dirMask:
182 + # Check if the broken binary lies within the masked directory or
183 + # its subdirectories.
184 + # XXX Perhaps we should allow regexps as masks.
185 + if binary.startswith(directory):
186 + del rValue[binary]
187 + if self.debug:
188 + print "dirMask:",binary
189 + break
190 + # Check if all the required libraries are masked.
191 + if binary in rValue and libSet.issubset(libMask):
192 + del rValue[binary]
193 + if self.debug:
194 + print "libMask:", binary, libSet & libMask
195 +
196 + if self.debug:
197 + print
198 + print "Directory mask:", dirMask
199 + print
200 + print "Library mask:", libMask
201 +
202 + return rValue
203 +
204 + def getDependencyMasks(self):
205 + """
206 + Return all dependency masks as a tuple.
207 +
208 + @rtype: 2-tuple of sets of strings
209 + @return: 2-tuple in which the first component is a set of directory
210 + masks and the second component is a set of library masks
211 +
212 + """
213 + dirMask = set()
214 + libMask = set()
215 + _dirMask_re = re.compile(r'SEARCH_DIRS_MASK\s*=\s*"([^"]*)"')
216 + _libMask_re = re.compile(r'LD_LIBRARY_MASK\s*=\s*"([^"]*)"')
217 + lines = []
218 +
219 + # Reads the contents of /etc/revdep-rebuild/*
220 + libMaskDir = os.path.join(self.root, "etc", "revdep-rebuild")
221 + if os.path.exists(libMaskDir):
222 + for file in os.listdir(libMaskDir):
223 + try:
224 + f = open(os.path.join(libMaskDir, file), "r")
225 + try:
226 + lines.extend(f.readlines())
227 + finally:
228 + f.close()
229 + except IOError: # OSError?
230 + continue
231 + # The following parses SEARCH_DIRS_MASK and LD_LIBRARY_MASK variables
232 + # from /etc/revdep-rebuild/*
233 + for line in lines:
234 + matchDir = _dirMask_re.match(line)
235 + matchLib = _libMask_re.match(line)
236 + if matchDir:
237 + dirMask.update(set(matchDir.group(1).split()))
238 + if matchLib:
239 + libMask.update(set(matchLib.group(1).split()))
240 +
241 + # These directories contain specially evaluated libraries.
242 + # app-emulation/vmware-workstation-6.0.1.55017
243 + dirMask.add('/opt/vmware/workstation/lib')
244 + # app-emulation/vmware-server-console-1.0.6.91891
245 + dirMask.add('/opt/vmware/server/console/lib')
246 + # www-client/mozilla-firefox-2.0.0.15
247 + dirMask.add('/usr/lib/mozilla-firefox/plugins')
248 + dirMask.add('/usr/lib64/mozilla-firefox/plugins')
249 + # app-office/openoffice-2.4.1
250 + dirMask.add('/opt/OpenOffice')
251 + dirMask.add('/usr/lib/openoffice')
252 + # dev-libs/libmix-2.05 libmix.so is missing soname entry
253 + libMask.add('libmix.so')
254 + # app-accessibility/speech-tools-1.2.96_beta missing sonames
255 + libMask.add('libestools.so')
256 + libMask.add('libestbase.so')
257 + libMask.add('libeststring.so')
258 + # app-emulation/emul-linux-x86-soundlibs-20080418
259 + dirMask.add('/usr/kde/3.5/lib32')
260 +
261 + return (dirMask, libMask)
262 +
263 + def findAtomsOfLibraryConsumers(self, searchString):
264 + """
265 + Return atoms containing consumers of libraries matching the argument.
266 +
267 + @param searchString: a string used to search for libraries
268 + @type searchString: string to be compiled as a regular expression
269 + (example: 'libfoo.*')
270 + @rtype: set of strings
271 + @return: the returned set of atoms are valid to be used by package sets
272 +
273 + """
274 + atoms = set()
275 + consumers = set()
276 + matchedLibraries = set()
277 + libraryObjects = []
278 + _librarySearch_re = re.compile(searchString)
279 +
280 + # Find libraries matching searchString.
281 + libraryObjects = self.linkmap.listLibraryObjects()
282 + for library in libraryObjects:
283 + m = _librarySearch_re.search(library)
284 + if m:
285 + matchedLibraries.add(library)
286 + consumers.update(self.linkmap.findConsumers(library))
287 +
288 + if self.debug:
289 + print
290 + print "Consumers of the following libraries will be emerged:"
291 + for x in matchedLibraries:
292 + print x
293 +
294 + if consumers:
295 + # The following prevents emerging the packages that own the matched
296 + # libraries. Note that this will prevent updating the packages owning
297 + # the libraries if there are newer versions available in the installed
298 + # slot. See bug #30095
299 + atoms = self.mapPathsToAtoms(consumers)
300 + libraryOwners = self.mapPathsToAtoms(matchedLibraries)
301 + atoms.difference_update(libraryOwners)
302 +
303 + return atoms
304 +
305 + def listBrokenLibtoolLibraries(self):
306 + """
307 + Find broken libtool libraries and their missing dependencies.
308 +
309 + @rtype: dict (example: {'/lib/libfoo.la': set(['/lib/libbar.la'])})
310 + @return: The return value is a library -> set-of-libraries mapping, where
311 + library is a broken library and the set consists of dependencies
312 + needed by library that do not exist on the filesystem.
313 +
314 + """
315 + rValue = {}
316 + lines = []
317 + dependencies = []
318 + _la_re = re.compile(r".*\.la$")
319 + _dependency_libs_re = re.compile(r"^dependency_libs\s*=\s*'(.*)'")
320 +
321 + # Loop over the contents of all packages.
322 + for cpv in self.dbapi.cpv_all():
323 + mysplit = catsplit(cpv)
324 + link = dblink(mysplit[0], mysplit[1], myroot=self.dbapi.root, \
325 + mysettings=self.dbapi.settings, treetype='vartree', \
326 + vartree=self.dbapi.vartree)
327 + for file in link.getcontents():
328 + # Check if the file ends with '.la'.
329 + matchLib = _la_re.match(file)
330 + if matchLib:
331 + # Read the lines from the library.
332 + lines = []
333 + try:
334 + f = open(file, "r")
335 + try:
336 + lines.extend(f.readlines())
337 + finally:
338 + f.close()
339 + except IOError:
340 + continue
341 + # Find the line listing the dependencies.
342 + for line in lines:
343 + matchLine = _dependency_libs_re.match(line)
344 + if matchLine:
345 + dependencies = matchLine.group(1).split()
346 + # For each dependency that is a pathname (begins with
347 + # os.sep), check that it exists on the filesystem. If it
348 + # does not exist, then add the library and the missing
349 + # dependency to rValue.
350 + for dependency in dependencies:
351 + if dependency[0] == os.sep and \
352 + not os.path.isfile(dependency):
353 + rValue.setdefault(file, set()).add(dependency)
354 +
355 + return rValue
356 +
357 + def singleBuilder(self, options, settings, trees):
358 + debug = get_boolean(options, "debug", False)
359 + return MissingLibraryConsumerSet(trees["vartree"].dbapi, debug)
360 + singleBuilder = classmethod(singleBuilder)