| blob | 05fa61a358dfec0bc65fbc4f301af1f4184b0f3f |
1 --- vartree.py.2.2_rc8 2008-08-17 21:11:41.000000000 -0500
2 +++ pym/portage/dbapi/vartree.py 2008-08-18 03:09:26.000000000 -0500
3 @@ -143,10 +143,12 @@
4 self._dbapi = vardbapi
5 self._libs = {}
6 self._obj_properties = {}
7 - self._defpath = getlibpaths()
8 -
9 + self._defpath = set(getlibpaths())
10 + self._obj_key_cache = {}
11 +
12 def rebuild(self, include_file=None):
13 libs = {}
14 + obj_key_cache = {}
15 obj_properties = {}
16 lines = []
17 for cpv in self._dbapi.cpv_all():
18 @@ -176,29 +178,60 @@
19 # insufficient field length
20 continue
21 arch = fields[0]
22 - obj = os.path.realpath(fields[1])
23 + obj = fields[1]
24 + obj_key = self._generateObjKey(obj)
25 soname = fields[2]
26 - path = filter(None, fields[3].replace(
27 + path = set([normalize_path(x)
28 + for x in filter(None, fields[3].replace(
29 "${ORIGIN}", os.path.dirname(obj)).replace(
30 - "$ORIGIN", os.path.dirname(obj)).split(":"))
31 + "$ORIGIN", os.path.dirname(obj)).split(":"))])
32 needed = filter(None, fields[4].split(","))
33 if soname:
34 - libs.setdefault(soname, {arch: {"providers": [], "consumers": []}})
35 - libs[soname].setdefault(arch, {"providers": [], "consumers": []})
36 - libs[soname][arch]["providers"].append(obj)
37 + libs.setdefault(soname, \
38 + {arch: {"providers": set(), "consumers": set()}})
39 + libs[soname].setdefault(arch, \
40 + {"providers": set(), "consumers": set()})
41 + libs[soname][arch]["providers"].add(obj_key)
42 for x in needed:
43 - libs.setdefault(x, {arch: {"providers": [], "consumers": []}})
44 - libs[x].setdefault(arch, {"providers": [], "consumers": []})
45 - libs[x][arch]["consumers"].append(obj)
46 - obj_properties[obj] = (arch, needed, path, soname)
47 -
48 + libs.setdefault(x, \
49 + {arch: {"providers": set(), "consumers": set()}})
50 + libs[x].setdefault(arch, {"providers": set(), "consumers": set()})
51 + libs[x][arch]["consumers"].add(obj_key)
52 + obj_key_cache.setdefault(obj, obj_key)
53 + # All object paths are added into the obj_properties tuple
54 + obj_properties.setdefault(obj_key, \
55 + (arch, needed, path, soname, set()))[4].add(obj)
56 +
57 self._libs = libs
58 self._obj_properties = obj_properties
59 + self._obj_key_cache = obj_key_cache
61 - def listBrokenBinaries(self):
62 + def _generateObjKey(self, obj):
63 + """
64 + Generate obj key for a given object.
65 +
66 + @param obj: path to an existing file
67 + @type obj: string (example: '/usr/bin/bar')
68 + @rtype: 2-tuple of longs if obj exists. string if obj does not exist.
69 + @return:
70 + 1. 2-tuple of obj's inode and device from a stat call, if obj exists.
71 + 2. realpath of object if obj does not exist.
72 +
73 + """
74 + try:
75 + obj_st = os.stat(obj)
76 + except OSError:
77 + # Use the realpath as the key if the file does not exists on the
78 + # filesystem.
79 + return os.path.realpath(obj)
80 + return (obj_st.st_dev, obj_st.st_ino)
81 +
82 + def listBrokenBinaries(self, debug=False):
83 """
84 Find binaries and their needed sonames, which have no providers.
86 + @param debug: Boolean to enable debug output
87 + @type debug: Boolean
88 @rtype: dict (example: {'/usr/bin/foo': set(['libbar.so'])})
89 @return: The return value is an object -> set-of-sonames mapping, where
90 object is a broken binary and the set consists of sonames needed by
91 @@ -208,7 +241,7 @@
92 class LibraryCache(object):
94 """
95 - Caches sonames and realpaths associated with paths.
96 + Caches properties associated with paths.
98 The purpose of this class is to prevent multiple calls of
99 os.path.realpath and os.path.isfile on the same paths.
100 @@ -218,55 +251,56 @@
101 def __init__(cache_self):
102 cache_self.cache = {}
104 - def get(cache_self, path):
105 + def get(cache_self, obj):
106 """
107 - Caches and returns the soname and realpath for a path.
108 + Caches and returns properties associated with an object.
110 - @param path: absolute path (can be symlink)
111 - @type path: string (example: '/usr/lib/libfoo.so')
112 - @rtype: 3-tuple with types (string or None, string, boolean)
113 - @return: 3-tuple with the following components:
114 - 1. soname as a string or None if it does not exist,
115 - 2. realpath as a string,
116 - 3. the result of os.path.isfile(realpath)
117 - (example: ('libfoo.so.1', '/usr/lib/libfoo.so.1.5.1', True))
118 + @param obj: absolute path (can be symlink)
119 + @type obj: string (example: '/usr/lib/libfoo.so')
120 + @rtype: 4-tuple with types
121 + (string or None, string or None, 2-tuple, Boolean)
122 + @return: 4-tuple with the following components:
123 + 1. arch as a string or None if it does not exist,
124 + 2. soname as a string or None if it does not exist,
125 + 3. obj_key as 2-tuple,
126 + 4. Boolean representing whether the object exists.
127 + (example: ('libfoo.so.1', (123L, 456L), True))
129 """
130 - if path in cache_self.cache:
131 - return cache_self.cache[path]
132 + if obj in cache_self.cache:
133 + return cache_self.cache[obj]
134 else:
135 - realpath = os.path.realpath(path)
136 + if obj in self._obj_key_cache:
137 + obj_key = self._obj_key_cache.get(obj)
138 + else:
139 + obj_key = self._generateObjKey(obj)
140 # Check that the library exists on the filesystem.
141 - if os.path.isfile(realpath):
142 - # Get the soname from LinkageMap._obj_properties if it
143 - # exists. Otherwise, None.
144 - soname = self._obj_properties.get(realpath, (None,)*3)[3]
145 - # Both path and realpath are cached and the result is
146 - # returned.
147 - cache_self.cache.setdefault(realpath, \
148 - (soname, realpath, True))
149 - return cache_self.cache.setdefault(path, \
150 - (soname, realpath, True))
151 + if isinstance(obj_key, tuple):
152 + # Get the arch and soname from LinkageMap._obj_properties if
153 + # it exists. Otherwise, None.
154 + arch, _, _, soname, _ = \
155 + self._obj_properties.get(obj_key, (None,)*5)
156 + return cache_self.cache.setdefault(obj, \
157 + (arch, soname, obj_key, True))
158 else:
159 - # realpath is not cached here, because the majority of cases
160 - # where realpath is not a file, path is the same as realpath.
161 - # Thus storing twice slows down the cache performance.
162 - return cache_self.cache.setdefault(path, \
163 - (None, realpath, False))
164 + return cache_self.cache.setdefault(obj, \
165 + (None, None, obj_key, False))
167 - debug = False
168 rValue = {}
169 cache = LibraryCache()
170 providers = self.listProviders()
172 - # Iterate over all binaries and their providers.
173 - for obj, sonames in providers.items():
174 + # Iterate over all obj_keys and their providers.
175 + for obj_key, sonames in providers.items():
176 + arch, _, path, _, objs = self._obj_properties[obj_key]
177 + path = path.union(self._defpath)
178 # Iterate over each needed soname and the set of library paths that
179 # fulfill the soname to determine if the dependency is broken.
180 for soname, libraries in sonames.items():
181 # validLibraries is used to store libraries, which satisfy soname,
182 # so if no valid libraries are found, the soname is not satisfied
183 - # for obj. Thus obj must be emerged.
184 + # for obj_key. If unsatisfied, objects associated with obj_key
185 + # must be emerged.
186 validLibraries = set()
187 # It could be the case that the library to satisfy the soname is
188 # not in the obj's runpath, but a symlink to the library is (eg
189 @@ -274,67 +308,60 @@
190 # does not catalog symlinks, broken or missing symlinks may go
191 # unnoticed. As a result of these cases, check that a file with
192 # the same name as the soname exists in obj's runpath.
193 - path = self._obj_properties[obj][2] + self._defpath
194 - for d in path:
195 - cachedSoname, cachedRealpath, cachedExists = \
196 - cache.get(os.path.join(d, soname))
197 - # Check that the this library provides the needed soname. Doing
198 + # XXX If we catalog symlinks in LinkageMap, this could be improved.
199 + for directory in path:
200 + cachedArch, cachedSoname, cachedKey, cachedExists = \
201 + cache.get(os.path.join(directory, soname))
202 + # Check that this library provides the needed soname. Doing
203 # this, however, will cause consumers of libraries missing
204 # sonames to be unnecessarily emerged. (eg libmix.so)
205 - if cachedSoname == soname:
206 - validLibraries.add(cachedRealpath)
207 - if debug and cachedRealpath not in libraries:
208 + if cachedSoname == soname and cachedArch == arch:
209 + validLibraries.add(cachedKey)
210 + if debug and cachedKey not in \
211 + set(map(self._obj_key_cache.get, libraries)):
212 + # XXX This is most often due to soname symlinks not in
213 + # a library's directory. We could catalog symlinks in
214 + # LinkageMap to avoid checking for this edge case here.
215 print "Found provider outside of findProviders:", \
216 - os.path.join(d, soname), "->", cachedRealpath
217 + os.path.join(directory, soname), "->", \
218 + self._obj_properties[cachedKey][4], libraries
219 # A valid library has been found, so there is no need to
220 # continue.
221 break
222 - if debug and cachedRealpath in self._obj_properties:
223 + if debug and cachedArch == arch and \
224 + cachedKey in self._obj_properties:
225 print "Broken symlink or missing/bad soname:", \
226 - os.path.join(d, soname), '->', cachedRealpath, \
227 - "with soname", cachedSoname, "but expecting", soname
228 + os.path.join(directory, soname), '->', \
229 + self._obj_properties[cachedKey], "with soname", \
230 + cachedSoname, "but expecting", soname
231 # This conditional checks if there are no libraries to satisfy the
232 # soname (empty set).
233 if not validLibraries:
234 - rValue.setdefault(obj, set()).add(soname)
235 + for obj in objs:
236 + rValue.setdefault(obj, set()).add(soname)
237 # If no valid libraries have been found by this point, then
238 # there are no files named with the soname within obj's runpath,
239 # but if there are libraries (from the providers mapping), it is
240 - # likely that symlinks or the actual libraries are missing.
241 - # Thus possible symlinks and missing libraries are added to
242 - # rValue in order to emerge corrupt library packages.
243 + # likely that soname symlinks or the actual libraries are
244 + # missing or broken. Thus those libraries are added to rValue
245 + # in order to emerge corrupt library packages.
246 for lib in libraries:
247 - cachedSoname, cachedRealpath, cachedExists = cache.get(lib)
248 - if not cachedExists:
249 - # The library's package needs to be emerged to repair the
250 - # missing library.
251 - rValue.setdefault(lib, set()).add(soname)
252 - else:
253 - # A library providing the soname exists in the obj's
254 - # runpath, but no file named as the soname exists, so add
255 - # the path constructed from the lib's directory and the
256 - # soname to rValue to fix cases of vanishing (or modified)
257 - # symlinks. This path is not guaranteed to exist, but it
258 - # follows the symlink convention found in the majority of
259 - # packages.
260 - rValue.setdefault(os.path.join(os.path.dirname(lib), \
261 - soname), set()).add(soname)
262 + rValue.setdefault(lib, set()).add(soname)
263 if debug:
264 - if not cachedExists:
265 + if not os.path.isfile(lib):
266 print "Missing library:", lib
267 else:
268 print "Possibly missing symlink:", \
269 os.path.join(os.path.dirname(lib), soname)
270 -
271 return rValue
273 def listProviders(self):
274 """
275 - Find the providers for all binaries.
276 + Find the providers for all object keys in LinkageMap.
278 @rtype: dict (example:
279 - {'/usr/bin/foo': {'libbar.so': set(['/lib/libbar.so.1.5'])}})
280 - @return: The return value is an object -> providers mapping, where
281 + {(123L, 456L): {'libbar.so': set(['/lib/libbar.so.1.5'])}})
282 + @return: The return value is an object key -> providers mapping, where
283 providers is a mapping of soname -> set-of-library-paths returned
284 from the findProviders method.
286 @@ -342,118 +369,187 @@
287 rValue = {}
288 if not self._libs:
289 self.rebuild()
290 - # Iterate over all binaries within LinkageMap.
291 - for obj in self._obj_properties:
292 - rValue.setdefault(obj, self.findProviders(obj))
293 + # Iterate over all objects within LinkageMap.
294 + for obj_key in self._obj_properties:
295 + rValue.setdefault(obj_key, self.findProviders(obj_key=obj_key))
296 return rValue
298 def isMasterLink(self, obj):
299 + """
300 + Determine whether an object is a master link.
301 +
302 + @param obj: absolute path to an object
303 + @type obj: string (example: '/usr/bin/foo')
304 + @rtype: Boolean
305 + @return:
306 + 1. True if obj is a master link
307 + 2. False if obj is not a master link
308 +
309 + """
310 basename = os.path.basename(obj)
311 - if obj not in self._obj_properties:
312 - obj = os.path.realpath(obj)
313 - if obj not in self._obj_properties:
314 - raise KeyError("%s not in object list" % obj)
315 - soname = self._obj_properties[obj][3]
316 + obj_key = self._generateObjKey(obj)
317 + if obj_key not in self._obj_properties:
318 + raise KeyError("%s (%s) not in object list" % (obj_key, obj))
319 + soname = self._obj_properties[obj_key][3]
320 return (len(basename) < len(soname))
321 -
322 +
323 def listLibraryObjects(self):
324 + """
325 + Return a list of library objects.
326 +
327 + Known limitation: library objects lacking an soname are not included.
328 +
329 + @rtype: list of strings
330 + @return: list of paths to all providers
331 +
332 + """
333 rValue = []
334 if not self._libs:
335 self.rebuild()
336 for soname in self._libs:
337 for arch in self._libs[soname]:
338 - rValue.extend(self._libs[soname][arch]["providers"])
339 + for obj_key in self._libs[soname][arch]["providers"]:
340 + rValue.extend(self._obj_properties[obj_key][4])
341 return rValue
343 def getSoname(self, obj):
344 + """
345 + Return the soname associated with an object.
346 +
347 + @param obj: absolute path to an object
348 + @type obj: string (example: '/usr/bin/bar')
349 + @rtype: string
350 + @return: soname
351 +
352 + """
353 if not self._libs:
354 self.rebuild()
355 - if obj not in self._obj_properties:
356 - obj = os.path.realpath(obj)
357 - if obj not in self._obj_properties:
358 - raise KeyError("%s not in object list" % obj)
359 - arch, needed, path, soname = self._obj_properties[obj]
360 - return soname
361 + if obj not in self._obj_key_cache:
362 + raise KeyError("%s not in object list" % obj)
363 + return self._obj_properties[self._obj_key_cache[obj]][3]
364 +
365 + def findProviders(self, obj=None, obj_key=None):
366 + """
367 + Find providers for an object or object key.
368 +
369 + This method should be called with either an obj or obj_key. If called
370 + with both, the obj_key is ignored. If called with neither, KeyError is
371 + raised as if an invalid obj was passed.
372 +
373 + In some cases, not all valid libraries are returned. This may occur when
374 + an soname symlink referencing a library is in and object's runpath while
375 + the actual library is not.
376 +
377 + @param obj: absolute path to an object
378 + @type obj: string (example: '/usr/bin/bar')
379 + @param obj_key: key from LinkageMap._generateObjKey
380 + @type obj_key: 2-tuple of longs or string
381 + @rtype: dict (example: {'libbar.so': set(['/lib/libbar.so.1.5'])})
382 + @return: The return value is a soname -> set-of-library-paths, where
383 + set-of-library-paths satisfy soname.
384 +
385 + """
386 + rValue = {}
388 - def findProviders(self, obj):
389 if not self._libs:
390 self.rebuild()
392 - realpath_cache = {}
393 - def realpath(p):
394 - real_path = realpath_cache.get(p)
395 - if real_path is None:
396 - real_path = os.path.realpath(p)
397 - realpath_cache[p] = real_path
398 - return real_path
399 + if obj is not None:
400 + obj_key = self._obj_key_cache.get(obj)
401 + if obj_key not in self._obj_properties:
402 + obj_key = self._generateObjKey(obj)
403 + if obj_key not in self._obj_properties:
404 + raise KeyError("%s (%s) not in object list" % (obj_key, obj))
405 + elif obj_key not in self._obj_properties:
406 + raise KeyError("%s not in object list" % obj_key)
408 - rValue = {}
409 - if obj not in self._obj_properties:
410 - obj = realpath(obj)
411 - if obj not in self._obj_properties:
412 - raise KeyError("%s not in object list" % obj)
413 - arch, needed, path, soname = self._obj_properties[obj]
414 - path = path[:]
415 - path.extend(self._defpath)
416 - path = set(realpath(x) for x in path)
417 + arch, needed, path, soname, objs = self._obj_properties[obj_key]
418 + path = path.union(self._defpath)
419 for x in needed:
420 rValue[x] = set()
421 if x not in self._libs or arch not in self._libs[x]:
422 continue
423 for y in self._libs[x][arch]["providers"]:
424 - if x[0] == os.sep and realpath(x) == realpath(y):
425 - rValue[x].add(y)
426 - elif realpath(os.path.dirname(y)) in path:
427 - rValue[x].add(y)
428 + objs = self._obj_properties[y][4]
429 + for o in objs:
430 + if os.path.dirname(o) in path:
431 + rValue[x].add(o)
432 +
433 return rValue
434 -
435 - def findConsumers(self, obj):
436 +
437 + def findConsumers(self, obj=None, obj_key=None):
438 + """
439 + Find consumers of an object or object key.
440 +
441 + This method should be called with either an obj or obj_key. If called
442 + with both, the obj_key is ignored. If called with neither, KeyError is
443 + raised as if an invalid obj was passed.
444 +
445 + In some cases, not all consumers are returned. This may occur when
446 + an soname symlink referencing a library is in and object's runpath while
447 + the actual library is not.
448 +
449 + @param obj: absolute path to an object
450 + @type obj: string (example: '/usr/bin/bar')
451 + @param obj_key: key from LinkageMap._generateObjKey
452 + @type obj_key: 2-tuple of longs or string
453 + @rtype: set of strings (example: )
454 + @return: The return value is a soname -> set-of-library-paths, where
455 + set-of-library-paths satisfy soname.
456 +
457 + """
458 + rValue = set()
459 +
460 if not self._libs:
461 self.rebuild()
463 - realpath_cache = {}
464 - def realpath(p):
465 - real_path = realpath_cache.get(p)
466 - if real_path is None:
467 - real_path = os.path.realpath(p)
468 - realpath_cache[p] = real_path
469 - return real_path
470 -
471 - if obj not in self._obj_properties:
472 - obj = realpath(obj)
473 - if obj not in self._obj_properties:
474 - raise KeyError("%s not in object list" % obj)
475 + if obj is not None:
476 + objs = set([obj])
477 + obj_key = self._obj_key_cache.get(obj)
478 + if obj_key not in self._obj_properties:
479 + obj_key = self._generateObjKey(obj)
480 + if obj_key not in self._obj_properties:
481 + raise KeyError("%s (%s) not in object list" % (obj_key, obj))
482 + else:
483 + if obj_key not in self._obj_properties:
484 + raise KeyError("%s not in object list" % obj_key)
485 + objs = self._obj_properties[obj_key][4]
486 +
487 + objs_dirs = [os.path.dirname(x) for x in objs]
489 # If there is another version of this lib with the
490 # same soname and the master link points to that
491 # other version, this lib will be shadowed and won't
492 # have any consumers.
493 - arch, needed, path, soname = self._obj_properties[obj]
494 - obj_dir = os.path.dirname(obj)
495 - master_link = os.path.join(obj_dir, soname)
496 - try:
497 - master_st = os.stat(master_link)
498 - obj_st = os.stat(obj)
499 - except OSError:
500 - pass
501 - else:
502 - if (obj_st.st_dev, obj_st.st_ino) != \
503 - (master_st.st_dev, master_st.st_ino):
504 - return set()
505 + if obj is not None:
506 + soname = self._obj_properties[obj_key][3]
507 + obj_dir = os.path.dirname(obj)
508 + master_link = os.path.join(obj_dir, soname)
509 + try:
510 + master_st = os.stat(master_link)
511 + obj_st = os.stat(obj)
512 + except OSError:
513 + pass
514 + else:
515 + if (obj_st.st_dev, obj_st.st_ino) != \
516 + (master_st.st_dev, master_st.st_ino):
517 + return set()
518 +
519 + arch, needed, path, soname, objs = self._obj_properties[obj_key]
520 + path = path.union(self._defpath)
521 +
522 + if soname in self._libs and arch in self._libs[soname]:
523 + for consumer_key in self._libs[soname][arch]["consumers"]:
524 + _, _, path, _, consumer_objs = \
525 + self._obj_properties[consumer_key]
526 + path = path.union(self._defpath)
527 + for consumer_obj in consumer_objs:
528 + for directory in objs_dirs:
529 + if directory in path:
530 + rValue.add(consumer_obj)
532 - rValue = set()
533 - for soname in self._libs:
534 - for arch in self._libs[soname]:
535 - if obj in self._libs[soname][arch]["providers"]:
536 - for x in self._libs[soname][arch]["consumers"]:
537 - path = self._obj_properties[x][2]
538 - path = [realpath(y) for y in path+self._defpath]
539 - if soname[0] == os.sep and realpath(soname) == realpath(obj):
540 - rValue.add(x)
541 - elif realpath(obj_dir) in path:
542 - rValue.add(x)
543 return rValue
544 -
545 +
546 class vardbapi(dbapi):
548 _excluded_dirs = ["CVS", "lost+found"]