| blob | 742a1e1b078800d863f1f02ae0250deb27d6cbe5 |
1 --- vartree.py.2.2_rc8 2008-08-14 15:45:44.000000000 -0500
2 +++ pym/portage/dbapi/vartree.py 2008-08-17 20:59:09.000000000 -0500
3 @@ -143,10 +143,12 @@
4 self._dbapi = vardbapi
5 self._libs = {}
6 self._obj_properties = {}
7 - self._defpath = getlibpaths()
8 -
9 + self._defpath = set(getlibpaths())
10 + self._obj_key_cache = {}
11 +
12 def rebuild(self, include_file=None):
13 libs = {}
14 + obj_key_cache = {}
15 obj_properties = {}
16 lines = []
17 for cpv in self._dbapi.cpv_all():
18 @@ -176,26 +178,56 @@
19 # insufficient field length
20 continue
21 arch = fields[0]
22 - obj = os.path.realpath(fields[1])
23 + obj = fields[1]
24 + obj_key = self._generateObjKey(obj)
25 soname = fields[2]
26 - path = filter(None, fields[3].replace(
27 + path = set([normalize_path(x)
28 + for x in filter(None, fields[3].replace(
29 "${ORIGIN}", os.path.dirname(obj)).replace(
30 - "$ORIGIN", os.path.dirname(obj)).split(":"))
31 + "$ORIGIN", os.path.dirname(obj)).split(":"))])
32 needed = filter(None, fields[4].split(","))
33 if soname:
34 - libs.setdefault(soname, {arch: {"providers": [], "consumers": []}})
35 - libs[soname].setdefault(arch, {"providers": [], "consumers": []})
36 - libs[soname][arch]["providers"].append(obj)
37 + libs.setdefault(soname, \
38 + {arch: {"providers": set(), "consumers": set()}})
39 + libs[soname].setdefault(arch, \
40 + {"providers": set(), "consumers": set()})
41 + libs[soname][arch]["providers"].add(obj_key)
42 for x in needed:
43 - libs.setdefault(x, {arch: {"providers": [], "consumers": []}})
44 - libs[x].setdefault(arch, {"providers": [], "consumers": []})
45 - libs[x][arch]["consumers"].append(obj)
46 - obj_properties[obj] = (arch, needed, path, soname)
47 -
48 + libs.setdefault(x, \
49 + {arch: {"providers": set(), "consumers": set()}})
50 + libs[x].setdefault(arch, {"providers": set(), "consumers": set()})
51 + libs[x][arch]["consumers"].add(obj_key)
52 + obj_key_cache.setdefault(obj, obj_key)
53 + # All object paths are added into the obj_properties tuple
54 + obj_properties.setdefault(obj_key, \
55 + (arch, needed, path, soname, set()))[4].add(obj)
56 +
57 self._libs = libs
58 self._obj_properties = obj_properties
59 + self._obj_key_cache = obj_key_cache
60 +
61 + def _generateObjKey(self, obj):
62 + """
63 + Generate obj key for a given object.
64 +
65 + @param obj: path to an existing file
66 + @type obj: string (example: '/usr/bin/bar')
67 + @rtype: 2-tuple of longs or string
68 + @return: If obj exists, a 2-tuple of obj's inode and device from a stat
69 + call is returned. Otherwise, the obj is returned.
70 +
71 + """
72 + try:
73 + return os.stat(obj)[1:3]
74 + except OSError:
75 + # XXX pull this out
76 + print '_generateObjKey(%s)' % obj
77 +# from portage.output import teal
78 +# writemsg(bold(red("Error in ")) + \
79 +# bold(teal("_generateObjKey. Stat failed on %s" % obj)) + '\n')
80 + return obj
82 - def listBrokenBinaries(self):
83 + def listBrokenBinaries(self, debug=False):
84 """
85 Find binaries and their needed sonames, which have no providers.
87 @@ -218,55 +250,56 @@
88 def __init__(cache_self):
89 cache_self.cache = {}
91 - def get(cache_self, path):
92 + def get(cache_self, obj):
93 """
94 - Caches and returns the soname and realpath for a path.
95 + Caches and returns properties associated with an object.
97 - @param path: absolute path (can be symlink)
98 - @type path: string (example: '/usr/lib/libfoo.so')
99 - @rtype: 3-tuple with types (string or None, string, boolean)
100 - @return: 3-tuple with the following components:
101 - 1. soname as a string or None if it does not exist,
102 - 2. realpath as a string,
103 - 3. the result of os.path.isfile(realpath)
104 - (example: ('libfoo.so.1', '/usr/lib/libfoo.so.1.5.1', True))
105 + @param obj: absolute path (can be symlink)
106 + @type obj: string (example: '/usr/lib/libfoo.so')
107 + @rtype: 4-tuple with types
108 + (string or None, string or None, 2-tuple, Boolean)
109 + @return: 4-tuple with the following components:
110 + 1. arch as a string or None if it does not exist,
111 + 2. soname as a string or None if it does not exist,
112 + 3. obj_key as 2-tuple,
113 + 4. Boolean representing whether the object exists.
114 + (example: ('libfoo.so.1', (123L, 456L), True))
116 """
117 - if path in cache_self.cache:
118 - return cache_self.cache[path]
119 + if obj in cache_self.cache:
120 + return cache_self.cache[obj]
121 else:
122 - realpath = os.path.realpath(path)
123 + if obj in self._obj_key_cache:
124 + obj_key = self._obj_key_cache.get(obj)
125 + else:
126 + obj_key = self._generateObjKey(obj)
127 # Check that the library exists on the filesystem.
128 - if os.path.isfile(realpath):
129 - # Get the soname from LinkageMap._obj_properties if it
130 - # exists. Otherwise, None.
131 - soname = self._obj_properties.get(realpath, (None,)*3)[3]
132 - # Both path and realpath are cached and the result is
133 - # returned.
134 - cache_self.cache.setdefault(realpath, \
135 - (soname, realpath, True))
136 - return cache_self.cache.setdefault(path, \
137 - (soname, realpath, True))
138 + if isinstance(obj_key, tuple):
139 + # Get the arch and soname from LinkageMap._obj_properties if
140 + # it exists. Otherwise, None.
141 + arch, _, _, soname, _ = \
142 + self._obj_properties.get(obj_key, (None,)*5)
143 + return cache_self.cache.setdefault(obj, \
144 + (arch, soname, obj_key, True))
145 else:
146 - # realpath is not cached here, because the majority of cases
147 - # where realpath is not a file, path is the same as realpath.
148 - # Thus storing twice slows down the cache performance.
149 - return cache_self.cache.setdefault(path, \
150 - (None, realpath, False))
151 + return cache_self.cache.setdefault(obj, \
152 + (None, None, obj_key, False))
154 - debug = False
155 rValue = {}
156 cache = LibraryCache()
157 providers = self.listProviders()
159 - # Iterate over all binaries and their providers.
160 - for obj, sonames in providers.items():
161 + # Iterate over all obj_keys and their providers.
162 + for obj_key, sonames in providers.items():
163 + arch, _, path, _, objs = self._obj_properties[obj_key]
164 + path = path.union(self._defpath)
165 # Iterate over each needed soname and the set of library paths that
166 # fulfill the soname to determine if the dependency is broken.
167 for soname, libraries in sonames.items():
168 # validLibraries is used to store libraries, which satisfy soname,
169 # so if no valid libraries are found, the soname is not satisfied
170 - # for obj. Thus obj must be emerged.
171 + # for obj_key. If unsatisfied, objects associated with obj_key
172 + # must be emerged.
173 validLibraries = set()
174 # It could be the case that the library to satisfy the soname is
175 # not in the obj's runpath, but a symlink to the library is (eg
176 @@ -274,29 +307,34 @@
177 # does not catalog symlinks, broken or missing symlinks may go
178 # unnoticed. As a result of these cases, check that a file with
179 # the same name as the soname exists in obj's runpath.
180 - path = self._obj_properties[obj][2] + self._defpath
181 - for d in path:
182 - cachedSoname, cachedRealpath, cachedExists = \
183 - cache.get(os.path.join(d, soname))
184 - # Check that the this library provides the needed soname. Doing
185 + # XXX If we catalog symlinks in LinkageMap, this could be improved.
186 + for directory in path:
187 + cachedArch, cachedSoname, cachedKey, cachedExists = \
188 + cache.get(os.path.join(directory, soname))
189 + # Check that this library provides the needed soname. Doing
190 # this, however, will cause consumers of libraries missing
191 # sonames to be unnecessarily emerged. (eg libmix.so)
192 - if cachedSoname == soname:
193 - validLibraries.add(cachedRealpath)
194 - if debug and cachedRealpath not in libraries:
195 + if cachedSoname == soname and cachedArch == arch:
196 + validLibraries.add(cachedKey)
197 + if debug and cachedKey not in \
198 + set(map(self._obj_key_cache.get, libraries)):
199 print "Found provider outside of findProviders:", \
200 - os.path.join(d, soname), "->", cachedRealpath
201 + os.path.join(directory, soname), "->", \
202 + self._obj_properties[cachedKey][4]
203 # A valid library has been found, so there is no need to
204 # continue.
205 break
206 - if debug and cachedRealpath in self._obj_properties:
207 + if debug and cachedArch == arch and \
208 + cachedKey in self._obj_properties:
209 print "Broken symlink or missing/bad soname:", \
210 - os.path.join(d, soname), '->', cachedRealpath, \
211 - "with soname", cachedSoname, "but expecting", soname
212 + os.path.join(directory, soname), '->', \
213 + self._obj_properties[cachedKey], "with soname", \
214 + cachedSoname, "but expecting", soname
215 # This conditional checks if there are no libraries to satisfy the
216 # soname (empty set).
217 if not validLibraries:
218 - rValue.setdefault(obj, set()).add(soname)
219 + for obj in objs:
220 + rValue.setdefault(obj, set()).add(soname)
221 # If no valid libraries have been found by this point, then
222 # there are no files named with the soname within obj's runpath,
223 # but if there are libraries (from the providers mapping), it is
224 @@ -304,7 +342,8 @@
225 # Thus possible symlinks and missing libraries are added to
226 # rValue in order to emerge corrupt library packages.
227 for lib in libraries:
228 - cachedSoname, cachedRealpath, cachedExists = cache.get(lib)
229 + cachedArch, cachedSoname, cachedKey, cachedExists = \
230 + cache.get(lib)
231 if not cachedExists:
232 # The library's package needs to be emerged to repair the
233 # missing library.
234 @@ -317,10 +356,11 @@
235 # symlinks. This path is not guaranteed to exist, but it
236 # follows the symlink convention found in the majority of
237 # packages.
238 + # XXX merge this into block above
239 rValue.setdefault(os.path.join(os.path.dirname(lib), \
240 soname), set()).add(soname)
241 if debug:
242 - if not cachedExists:
243 + if not os.path.isfile(lib):
244 print "Missing library:", lib
245 else:
246 print "Possibly missing symlink:", \
247 @@ -328,13 +368,28 @@
249 return rValue
251 + def listConsumers(self):
252 + i = 0
253 + rValue = {}
254 + if not self._libs:
255 + self.rebuild()
256 + # Iterate over all objects within LinkageMap.
257 + for obj_key in self._obj_properties:
258 + try:
259 + i += len(self.findConsumersNew(obj_key=obj_key))
260 + except:
261 + print 'oshi', self._obj_properties[obj_key][4]
262 + rValue.setdefault(obj_key, self.findConsumers(obj_key=obj_key))
263 + print i
264 + return rValue
265 +
266 def listProviders(self):
267 """
268 - Find the providers for all binaries.
269 + Find the providers for all objects in LinkageMap.
271 @rtype: dict (example:
272 - {'/usr/bin/foo': {'libbar.so': set(['/lib/libbar.so.1.5'])}})
273 - @return: The return value is an object -> providers mapping, where
274 + {(123L, 456L): {'libbar.so': set(['/lib/libbar.so.1.5'])}})
275 + @return: The return value is an object key -> providers mapping, where
276 providers is a mapping of soname -> set-of-library-paths returned
277 from the findProviders method.
279 @@ -342,118 +397,258 @@
280 rValue = {}
281 if not self._libs:
282 self.rebuild()
283 - # Iterate over all binaries within LinkageMap.
284 - for obj in self._obj_properties:
285 - rValue.setdefault(obj, self.findProviders(obj))
286 + # Iterate over all objects within LinkageMap.
287 + for obj_key in self._obj_properties:
288 + # XXX remove this
289 + if len(self._obj_properties[obj_key][4]) != 1:
290 + writemsg(bold(red(self._obj_properties[obj_key])))
291 + rValue.setdefault(obj_key, self.findProviders(obj_key=obj_key))
292 return rValue
294 def isMasterLink(self, obj):
295 basename = os.path.basename(obj)
296 - if obj not in self._obj_properties:
297 - obj = os.path.realpath(obj)
298 - if obj not in self._obj_properties:
299 - raise KeyError("%s not in object list" % obj)
300 - soname = self._obj_properties[obj][3]
301 + obj_key = self._generateObjKey(obj)
302 + if obj_key not in self._obj_properties:
303 + raise KeyError("%s (%s) not in object list" % (obj_key, obj))
304 + soname = self._obj_properties[obj_key][3]
305 return (len(basename) < len(soname))
306 -
307 +
308 def listLibraryObjects(self):
309 rValue = []
310 if not self._libs:
311 self.rebuild()
312 for soname in self._libs:
313 for arch in self._libs[soname]:
314 - rValue.extend(self._libs[soname][arch]["providers"])
315 + for obj_key in self._libs[soname][arch]["providers"]:
316 + rValue.extend(self._obj_properties[obj_key][4])
317 return rValue
319 def getSoname(self, obj):
320 if not self._libs:
321 self.rebuild()
322 - if obj not in self._obj_properties:
323 - obj = os.path.realpath(obj)
324 - if obj not in self._obj_properties:
325 - raise KeyError("%s not in object list" % obj)
326 - arch, needed, path, soname = self._obj_properties[obj]
327 - return soname
328 + if obj not in self._obj_key_cache:
329 + raise KeyError("%s not in object list" % obj)
330 + return self._obj_properties[self._obj_key_cache[obj]][3]
331 +
332 + def findProviders(self, obj=None, obj_key=None):
333 + """
334 + Find providers for an object or object key.
335 +
336 + This method should be called with either an obj or obj_key. If called
337 + with both, the obj_key is ignored. If called with neither, KeyError is
338 + raised as if an invalid obj was passed.
339 +
340 + @param obj:
341 + @type obj:
342 + @param obj_key:
343 + @type obj_key:
344 + @rtype:
345 + @return:
346 +
347 + """
348 + rValue = {}
350 - def findProviders(self, obj):
351 if not self._libs:
352 self.rebuild()
354 - realpath_cache = {}
355 - def realpath(p):
356 - real_path = realpath_cache.get(p)
357 - if real_path is None:
358 - real_path = os.path.realpath(p)
359 - realpath_cache[p] = real_path
360 - return real_path
361 -
362 - rValue = {}
363 - if obj not in self._obj_properties:
364 - obj = realpath(obj)
365 - if obj not in self._obj_properties:
366 - raise KeyError("%s not in object list" % obj)
367 - arch, needed, path, soname = self._obj_properties[obj]
368 - path = path[:]
369 - path.extend(self._defpath)
370 - path = set(realpath(x) for x in path)
371 + if obj is not None:
372 + obj_key = self._obj_key_cache.get(obj)
373 + if obj_key not in self._obj_properties:
374 + obj_key = self._generateObjKey(obj)
375 + if obj_key not in self._obj_properties:
376 + raise KeyError("%s (%s) not in object list" % (obj_key, obj))
377 + elif obj_key not in self._obj_properties:
378 + raise KeyError("%s not in object list" % obj_key)
379 +
380 + arch, needed, path, soname, objs = self._obj_properties[obj_key]
381 + path = path.union(self._defpath)
382 + # XXX test this
383 + # path = set(realpath(x) for x in path)
384 for x in needed:
385 rValue[x] = set()
386 if x not in self._libs or arch not in self._libs[x]:
387 continue
388 for y in self._libs[x][arch]["providers"]:
389 - if x[0] == os.sep and realpath(x) == realpath(y):
390 - rValue[x].add(y)
391 - elif realpath(os.path.dirname(y)) in path:
392 - rValue[x].add(y)
393 + objs = self._obj_properties[y][4]
394 + # XXX x is an soname, so it should never start with os.sep, right?
395 + #if x[0] == os.sep and realpath(x) == realpath(y):
396 + # rValue[x].add(y)
397 + for o in objs:
398 + if os.path.dirname(o) in path:
399 + rValue[x].add(o)
400 +
401 return rValue
402 -
403 - def findConsumers(self, obj):
404 +
405 + def findConsumersNew(self, obj=None, obj_key=None):
406 + """
407 + Find consumers of an object or object key.
408 +
409 + This method should be called with either an obj or obj_key. If called
410 + with both, the obj_key is ignored. If called with neither, KeyError is
411 + raised as if an invalid obj was passed.
412 +
413 + @param obj:
414 + @type obj:
415 + @param obj_key:
416 + @type obj_key:
417 + @rtype:
418 + @return:
419 +
420 + """
421 + rValue = set()
422 +
423 if not self._libs:
424 self.rebuild()
426 - realpath_cache = {}
427 - def realpath(p):
428 - real_path = realpath_cache.get(p)
429 - if real_path is None:
430 - real_path = os.path.realpath(p)
431 - realpath_cache[p] = real_path
432 - return real_path
433 -
434 - if obj not in self._obj_properties:
435 - obj = realpath(obj)
436 - if obj not in self._obj_properties:
437 - raise KeyError("%s not in object list" % obj)
438 + if obj is not None:
439 + objs = set([obj])
440 + obj_key = self._obj_key_cache.get(obj)
441 + if obj_key not in self._obj_properties:
442 + obj_key = self._generateObjKey(obj)
443 + if obj_key not in self._obj_properties:
444 + raise KeyError("%s (%s) not in object list" % (obj_key, obj))
445 + else:
446 + if obj_key not in self._obj_properties:
447 + raise KeyError("%s not in object list" % obj_key)
448 + objs = self._obj_properties[obj_key][4]
449 +
450 + objs_dirs = [os.path.dirname(x) for x in objs]
452 # If there is another version of this lib with the
453 # same soname and the master link points to that
454 # other version, this lib will be shadowed and won't
455 # have any consumers.
456 - arch, needed, path, soname = self._obj_properties[obj]
457 - obj_dir = os.path.dirname(obj)
458 - master_link = os.path.join(obj_dir, soname)
459 - try:
460 - master_st = os.stat(master_link)
461 - obj_st = os.stat(obj)
462 - except OSError:
463 - pass
464 - else:
465 - if (obj_st.st_dev, obj_st.st_ino) != \
466 - (master_st.st_dev, master_st.st_ino):
467 - return set()
468 + if obj is not None:
469 + soname = self._obj_properties[obj_key][3]
470 + obj_dir = os.path.dirname(obj)
471 + master_link = os.path.join(obj_dir, soname)
472 + try:
473 + master_st = os.stat(master_link)
474 + obj_st = os.stat(obj)
475 + except OSError:
476 + pass
477 + else:
478 + if (obj_st.st_dev, obj_st.st_ino) != \
479 + (master_st.st_dev, master_st.st_ino):
480 + return set()
481 +
482 + arch, needed, path, soname, objs = self._obj_properties[obj_key]
483 + path = path.union(self._defpath)
484 +
485 + if soname in self._libs and arch in self._libs[soname]:
486 + for consumer_key in self._libs[soname][arch]["consumers"]:
487 + _, _, path, _, consumer_objs = \
488 + self._obj_properties[consumer_key]
489 + # XXX test this
490 + #path = [realpath(y) for y in path+self._defpath]
491 + path = path.union(self._defpath)
492 + # XXX x is an soname, so it should never start with os.sep,
493 + # right?
494 + #if soname[0] == os.sep and realpath(soname) == realpath(obj):
495 + # rValue.add(x)
496 + #if realpath(obj_dir) in path:
497 + # rValue.add(x)
498 + for consumer_obj in consumer_objs:
499 + for directory in objs_dirs:
500 + if directory in path:
501 + rValue.add(consumer_obj)
502 +
503 +
504 +# for soname in self._libs:
505 +# for arch in self._libs[soname]:
506 +# if obj_key in self._libs[soname][arch]["providers"]:
507 +# for consumer_key in self._libs[soname][arch]["consumers"]:
508 +# _, _, path, _, consumer_objs = \
509 +# self._obj_properties[consumer_key]
510 +# # XXX test this
511 +# #path = [realpath(y) for y in path+self._defpath]
512 +# path = path.union(self._defpath)
513 +# # XXX x is an soname, so it should never start with os.sep,
514 +# # right?
515 +# #if soname[0] == os.sep and realpath(soname) == realpath(obj):
516 +# # rValue.add(x)
517 +# #if realpath(obj_dir) in path:
518 +# # rValue.add(x)
519 +# for consumer_obj in consumer_objs:
520 +# for directory in objs_dirs:
521 +# if directory in path:
522 +# rValue.add(consumer_obj)
523 + return rValue
525 + def findConsumers(self, obj=None, obj_key=None):
526 + """
527 + Find consumers of an object or object key.
528 +
529 + This method should be called with either an obj or obj_key. If called
530 + with both, the obj_key is ignored. If called with neither, KeyError is
531 + raised as if an invalid obj was passed.
532 +
533 + @param obj:
534 + @type obj:
535 + @param obj_key:
536 + @type obj_key:
537 + @rtype:
538 + @return:
539 +
540 + """
541 rValue = set()
542 +
543 + if not self._libs:
544 + self.rebuild()
545 +
546 + if obj is not None:
547 + objs = set([obj])
548 + obj_key = self._obj_key_cache.get(obj)
549 + if obj_key not in self._obj_properties:
550 + obj_key = self._generateObjKey(obj)
551 + if obj_key not in self._obj_properties:
552 + raise KeyError("%s (%s) not in object list" % (obj_key, obj))
553 + else:
554 + if obj_key not in self._obj_properties:
555 + raise KeyError("%s not in object list" % obj_key)
556 + objs = self._obj_properties[obj_key][4]
557 +
558 + objs_dirs = [os.path.dirname(x) for x in objs]
559 +
560 + # If there is another version of this lib with the
561 + # same soname and the master link points to that
562 + # other version, this lib will be shadowed and won't
563 + # have any consumers.
564 + if obj is not None:
565 + soname = self._obj_properties[obj_key][3]
566 + obj_dir = os.path.dirname(obj)
567 + master_link = os.path.join(obj_dir, soname)
568 + try:
569 + master_st = os.stat(master_link)
570 + obj_st = os.stat(obj)
571 + except OSError:
572 + pass
573 + else:
574 + if (obj_st.st_dev, obj_st.st_ino) != \
575 + (master_st.st_dev, master_st.st_ino):
576 + return set()
577 +
578 for soname in self._libs:
579 for arch in self._libs[soname]:
580 - if obj in self._libs[soname][arch]["providers"]:
581 - for x in self._libs[soname][arch]["consumers"]:
582 - path = self._obj_properties[x][2]
583 - path = [realpath(y) for y in path+self._defpath]
584 - if soname[0] == os.sep and realpath(soname) == realpath(obj):
585 - rValue.add(x)
586 - elif realpath(obj_dir) in path:
587 - rValue.add(x)
588 + if obj_key in self._libs[soname][arch]["providers"]:
589 + for consumer_key in self._libs[soname][arch]["consumers"]:
590 + _, _, path, _, consumer_objs = \
591 + self._obj_properties[consumer_key]
592 + # XXX test this
593 + #path = [realpath(y) for y in path+self._defpath]
594 + path = path.union(self._defpath)
595 + # XXX x is an soname, so it should never start with os.sep,
596 + # right?
597 + #if soname[0] == os.sep and realpath(soname) == realpath(obj):
598 + # rValue.add(x)
599 + #if realpath(obj_dir) in path:
600 + # rValue.add(x)
601 + for consumer_obj in consumer_objs:
602 + for directory in objs_dirs:
603 + if directory in path:
604 + rValue.add(consumer_obj)
605 return rValue
606 -
607 +
608 class vardbapi(dbapi):
610 _excluded_dirs = ["CVS", "lost+found"]