From eac5528887656abec65fc3a825506187397482e4 Mon Sep 17 00:00:00 2001 From: worch Date: Fri, 22 Aug 2008 23:52:19 -0500 Subject: [PATCH] Added _ObjectKey helper class to LinkageMap. --- pym/portage/dbapi/vartree.py | 148 +++++++++++++++++++------------ vartree.py.2.2_rc8.patch | 205 +++++++++++++++++++++++++------------------ 2 files changed, 212 insertions(+), 141 deletions(-) diff --git a/pym/portage/dbapi/vartree.py b/pym/portage/dbapi/vartree.py index a10c133..48cc283 100644 --- a/pym/portage/dbapi/vartree.py +++ b/pym/portage/dbapi/vartree.py @@ -139,6 +139,9 @@ class PreservedLibsRegistry(object): return rValue class LinkageMap(object): + + """Models dynamic linker dependencies.""" + def __init__(self, vardbapi): self._dbapi = vardbapi self._libs = {} @@ -146,6 +149,61 @@ class LinkageMap(object): self._defpath = set(getlibpaths()) self._obj_key_cache = {} + class _ObjectKey(object): + + """Helper class used as _obj_properties keys for objects.""" + + def __init__(self, object): + """ + This takes a path to an object. + + @param object: path to a file + @type object: string (example: '/usr/bin/bar') + + """ + self._key = self._generate_object_key(object) + + def __hash__(self): + return hash(self._key) + + def __eq__(self, other): + return self._key == other._key + + def _generate_object_key(self, object): + """ + Generate object key for a given object. + + @param object: path to a file + @type object: string (example: '/usr/bin/bar') + @rtype: 2-tuple of types (long, int) if object exists. string if + object does not exist. + @return: + 1. 2-tuple of object's inode and device from a stat call, if object + exists. + 2. realpath of object if object does not exist. + + """ + try: + object_stat = os.stat(object) + except OSError: + # Use the realpath as the key if the file does not exists on the + # filesystem. + return os.path.realpath(object) + # Return a tuple of the device and inode. + return (object_stat.st_dev, object_stat.st_ino) + + def file_exists(self): + """ + Determine if the file for this key exists on the filesystem. + + @rtype: Boolean + @return: + 1. True if the file exists. + 2. False if the file does not exist or is a broken symlink. + + """ + return isinstance(self._key, tuple) + def rebuild(self, include_file=None): libs = {} obj_key_cache = {} @@ -179,7 +237,7 @@ class LinkageMap(object): continue arch = fields[0] obj = fields[1] - obj_key = self._generateObjKey(obj) + obj_key = self._ObjectKey(obj) soname = fields[2] path = set([normalize_path(x) for x in filter(None, fields[3].replace( @@ -206,27 +264,6 @@ class LinkageMap(object): self._obj_properties = obj_properties self._obj_key_cache = obj_key_cache - def _generateObjKey(self, obj): - """ - Generate obj key for a given object. - - @param obj: path to an existing file - @type obj: string (example: '/usr/bin/bar') - @rtype: 2-tuple of longs if obj exists. string if obj does not exist. - @return: - 1. 2-tuple of obj's inode and device from a stat call, if obj exists. - 2. realpath of object if obj does not exist. - - """ - try: - obj_st = os.stat(obj) - except OSError: - # Use the realpath as the key if the file does not exists on the - # filesystem. - return os.path.realpath(obj) - # Return a tuple of the device and inode. - return (obj_st.st_dev, obj_st.st_ino) - def listBrokenBinaries(self, debug=False): """ Find binaries and their needed sonames, which have no providers. @@ -239,13 +276,13 @@ class LinkageMap(object): object that have no corresponding libraries to fulfill the dependency. """ - class LibraryCache(object): + class _LibraryCache(object): """ Caches properties associated with paths. - The purpose of this class is to prevent multiple calls of - _generateObjKey on the same paths. + The purpose of this class is to prevent multiple instances of + _ObjectKey for the same paths. """ @@ -274,9 +311,9 @@ class LinkageMap(object): if obj in self._obj_key_cache: obj_key = self._obj_key_cache.get(obj) else: - obj_key = self._generateObjKey(obj) + obj_key = self._ObjectKey(obj) # Check that the library exists on the filesystem. - if isinstance(obj_key, tuple): + if obj_key.file_exists(): # Get the arch and soname from LinkageMap._obj_properties if # it exists. Otherwise, None. arch, _, _, soname, _ = \ @@ -288,7 +325,7 @@ class LinkageMap(object): (None, None, obj_key, False)) rValue = {} - cache = LibraryCache() + cache = _LibraryCache() providers = self.listProviders() # Iterate over all obj_keys and their providers. @@ -372,7 +409,7 @@ class LinkageMap(object): self.rebuild() # Iterate over all object keys within LinkageMap. for obj_key in self._obj_properties: - rValue.setdefault(obj_key, self.findProviders(obj_key=obj_key)) + rValue.setdefault(obj_key, self.findProviders(obj_key)) return rValue def isMasterLink(self, obj): @@ -388,7 +425,7 @@ class LinkageMap(object): """ basename = os.path.basename(obj) - obj_key = self._generateObjKey(obj) + obj_key = self._ObjectKey(obj) if obj_key not in self._obj_properties: raise KeyError("%s (%s) not in object list" % (obj_key, obj)) soname = self._obj_properties[obj_key][3] @@ -429,13 +466,11 @@ class LinkageMap(object): raise KeyError("%s not in object list" % obj) return self._obj_properties[self._obj_key_cache[obj]][3] - def findProviders(self, obj=None, obj_key=None): + def findProviders(self, obj): """ Find providers for an object or object key. - This method should be called with either an obj or obj_key. If called - with both, the obj_key is ignored. If called with neither, KeyError is - raised as if an invalid obj was passed. + This method may be called with a key from _obj_properties. In some cases, not all valid libraries are returned. This may occur when an soname symlink referencing a library is in an object's runpath while @@ -444,10 +479,8 @@ class LinkageMap(object): library dependencies (since the dynamic linker actually searches for files named with the soname in the runpaths). - @param obj: absolute path to an object - @type obj: string (example: '/usr/bin/bar') - @param obj_key: key from LinkageMap._generateObjKey - @type obj_key: 2-tuple of longs or string + @param obj: absolute path to an object or a key from _obj_properties + @type obj: string (example: '/usr/bin/bar') or _ObjectKey @rtype: dict (example: {'libbar.so': set(['/lib/libbar.so.1.5'])}) @return: The return value is a soname -> set-of-library-paths, where set-of-library-paths satisfy soname. @@ -459,14 +492,16 @@ class LinkageMap(object): self.rebuild() # Determine the obj_key from the arguments. - if obj is not None: + if isinstance(obj, self._ObjectKey): + obj_key = obj + if obj_key not in self._obj_properties: + raise KeyError("%s not in object list" % obj_key) + else: obj_key = self._obj_key_cache.get(obj) if obj_key not in self._obj_properties: - obj_key = self._generateObjKey(obj) + obj_key = self._ObjectKey(obj) if obj_key not in self._obj_properties: raise KeyError("%s (%s) not in object list" % (obj_key, obj)) - elif obj_key not in self._obj_properties: - raise KeyError("%s not in object list" % obj_key) arch, needed, path, _, _ = self._obj_properties[obj_key] path = path.union(self._defpath) @@ -483,23 +518,19 @@ class LinkageMap(object): rValue[soname].add(provider) return rValue - def findConsumers(self, obj=None, obj_key=None): + def findConsumers(self, obj): """ Find consumers of an object or object key. - This method should be called with either an obj or obj_key. If called - with both, the obj_key is ignored. If called with neither, KeyError is - raised as if an invalid obj was passed. + This method may be called with a key from _obj_properties. If this method is going to be called with object keys, to avoid not catching shadowed libraryies, do not pass new _ObjectKey instances to this method. Instead pass the library as a string. In some cases, not all consumers are returned. This may occur when an soname symlink referencing a library is in an object's runpath while the actual library is not. - @param obj: absolute path to an object - @type obj: string (example: '/usr/bin/bar') - @param obj_key: key from LinkageMap._generateObjKey - @type obj_key: 2-tuple of longs or string - @rtype: set of strings (example: ) + @param obj: absolute path to an object or a key from _obj_properties + @type obj: string (example: '/usr/bin/bar') or _ObjectKey + @rtype: set of strings (example: set(['/bin/foo', '/usr/bin/bar'])) @return: The return value is a soname -> set-of-library-paths, where set-of-library-paths satisfy soname. @@ -510,17 +541,18 @@ class LinkageMap(object): self.rebuild() # Determine the obj_key and the set of objects matching the arguments. - if obj is not None: + if isinstance(obj, self._ObjectKey): + obj_key = obj + if obj_key not in self._obj_properties: + raise KeyError("%s not in object list" % obj_key) + objs = self._obj_properties[obj_key][4] + else: objs = set([obj]) obj_key = self._obj_key_cache.get(obj) if obj_key not in self._obj_properties: - obj_key = self._generateObjKey(obj) + obj_key = self._ObjectKey(obj) if obj_key not in self._obj_properties: raise KeyError("%s (%s) not in object list" % (obj_key, obj)) - else: - if obj_key not in self._obj_properties: - raise KeyError("%s not in object list" % obj_key) - objs = self._obj_properties[obj_key][4] # Determine the directory(ies) from the set of objects. objs_dirs = set([os.path.dirname(x) for x in objs]) @@ -529,7 +561,7 @@ class LinkageMap(object): # same soname and the master link points to that # other version, this lib will be shadowed and won't # have any consumers. - if obj is not None: + if not isinstance(obj, self._ObjectKey): soname = self._obj_properties[obj_key][3] obj_dir = os.path.dirname(obj) master_link = os.path.join(obj_dir, soname) diff --git a/vartree.py.2.2_rc8.patch b/vartree.py.2.2_rc8.patch index 2340005..ea996a6 100644 --- a/vartree.py.2.2_rc8.patch +++ b/vartree.py.2.2_rc8.patch @@ -1,6 +1,13 @@ --- vartree.py.2.2_rc8 2008-08-20 20:49:18.000000000 -0500 -+++ pym/portage/dbapi/vartree.py 2008-08-20 22:53:19.000000000 -0500 -@@ -143,10 +143,12 @@ ++++ pym/portage/dbapi/vartree.py 2008-08-22 23:52:19.000000000 -0500 +@@ -139,14 +139,74 @@ + return rValue + + class LinkageMap(object): ++ ++ """Models dynamic linker dependencies.""" ++ + def __init__(self, vardbapi): self._dbapi = vardbapi self._libs = {} self._obj_properties = {} @@ -9,19 +16,74 @@ + self._defpath = set(getlibpaths()) + self._obj_key_cache = {} + ++ class _ObjectKey(object): ++ ++ """Helper class used as _obj_properties keys for objects.""" ++ ++ def __init__(self, object): ++ """ ++ This takes a path to an object. ++ ++ @param object: path to a file ++ @type object: string (example: '/usr/bin/bar') ++ ++ """ ++ self._key = self._generate_object_key(object) ++ ++ def __hash__(self): ++ return hash(self._key) ++ ++ def __eq__(self, other): ++ return self._key == other._key ++ ++ def _generate_object_key(self, object): ++ """ ++ Generate object key for a given object. ++ ++ @param object: path to a file ++ @type object: string (example: '/usr/bin/bar') ++ @rtype: 2-tuple of types (long, int) if object exists. string if ++ object does not exist. ++ @return: ++ 1. 2-tuple of object's inode and device from a stat call, if object ++ exists. ++ 2. realpath of object if object does not exist. ++ ++ """ ++ try: ++ object_stat = os.stat(object) ++ except OSError: ++ # Use the realpath as the key if the file does not exists on the ++ # filesystem. ++ return os.path.realpath(object) ++ # Return a tuple of the device and inode. ++ return (object_stat.st_dev, object_stat.st_ino) ++ ++ def file_exists(self): ++ """ ++ Determine if the file for this key exists on the filesystem. ++ ++ @rtype: Boolean ++ @return: ++ 1. True if the file exists. ++ 2. False if the file does not exist or is a broken symlink. ++ ++ """ ++ return isinstance(self._key, tuple) ++ def rebuild(self, include_file=None): libs = {} + obj_key_cache = {} obj_properties = {} lines = [] for cpv in self._dbapi.cpv_all(): -@@ -176,29 +178,61 @@ +@@ -176,97 +236,109 @@ # insufficient field length continue arch = fields[0] - obj = os.path.realpath(fields[1]) + obj = fields[1] -+ obj_key = self._generateObjKey(obj) ++ obj_key = self._ObjectKey(obj) soname = fields[2] - path = filter(None, fields[3].replace( + path = set([normalize_path(x) @@ -59,27 +121,6 @@ + self._obj_key_cache = obj_key_cache - def listBrokenBinaries(self): -+ def _generateObjKey(self, obj): -+ """ -+ Generate obj key for a given object. -+ -+ @param obj: path to an existing file -+ @type obj: string (example: '/usr/bin/bar') -+ @rtype: 2-tuple of longs if obj exists. string if obj does not exist. -+ @return: -+ 1. 2-tuple of obj's inode and device from a stat call, if obj exists. -+ 2. realpath of object if obj does not exist. -+ -+ """ -+ try: -+ obj_st = os.stat(obj) -+ except OSError: -+ # Use the realpath as the key if the file does not exists on the -+ # filesystem. -+ return os.path.realpath(obj) -+ # Return a tuple of the device and inode. -+ return (obj_st.st_dev, obj_st.st_ino) -+ + def listBrokenBinaries(self, debug=False): """ Find binaries and their needed sonames, which have no providers. @@ -89,16 +130,20 @@ @rtype: dict (example: {'/usr/bin/foo': set(['libbar.so'])}) @return: The return value is an object -> set-of-sonames mapping, where object is a broken binary and the set consists of sonames needed by -@@ -208,65 +242,66 @@ - class LibraryCache(object): + object that have no corresponding libraries to fulfill the dependency. + + """ +- class LibraryCache(object): ++ class _LibraryCache(object): """ - Caches sonames and realpaths associated with paths. + Caches properties associated with paths. - The purpose of this class is to prevent multiple calls of +- The purpose of this class is to prevent multiple calls of - os.path.realpath and os.path.isfile on the same paths. -+ _generateObjKey on the same paths. ++ The purpose of this class is to prevent multiple instances of ++ _ObjectKey for the same paths. """ @@ -140,7 +185,7 @@ + if obj in self._obj_key_cache: + obj_key = self._obj_key_cache.get(obj) + else: -+ obj_key = self._generateObjKey(obj) ++ obj_key = self._ObjectKey(obj) # Check that the library exists on the filesystem. - if os.path.isfile(realpath): - # Get the soname from LinkageMap._obj_properties if it @@ -152,7 +197,7 @@ - (soname, realpath, True)) - return cache_self.cache.setdefault(path, \ - (soname, realpath, True)) -+ if isinstance(obj_key, tuple): ++ if obj_key.file_exists(): + # Get the arch and soname from LinkageMap._obj_properties if + # it exists. Otherwise, None. + arch, _, _, soname, _ = \ @@ -170,7 +215,8 @@ - debug = False rValue = {} - cache = LibraryCache() +- cache = LibraryCache() ++ cache = _LibraryCache() providers = self.listProviders() - # Iterate over all binaries and their providers. @@ -190,7 +236,7 @@ validLibraries = set() # It could be the case that the library to satisfy the soname is # not in the obj's runpath, but a symlink to the library is (eg -@@ -274,67 +309,60 @@ +@@ -274,67 +346,60 @@ # does not catalog symlinks, broken or missing symlinks may go # unnoticed. As a result of these cases, check that a file with # the same name as the soname exists in obj's runpath. @@ -287,7 +333,7 @@ providers is a mapping of soname -> set-of-library-paths returned from the findProviders method. -@@ -342,118 +370,191 @@ +@@ -342,118 +407,186 @@ rValue = {} if not self._libs: self.rebuild() @@ -296,7 +342,7 @@ - rValue.setdefault(obj, self.findProviders(obj)) + # Iterate over all object keys within LinkageMap. + for obj_key in self._obj_properties: -+ rValue.setdefault(obj_key, self.findProviders(obj_key=obj_key)) ++ rValue.setdefault(obj_key, self.findProviders(obj_key)) return rValue def isMasterLink(self, obj): @@ -317,7 +363,7 @@ - if obj not in self._obj_properties: - raise KeyError("%s not in object list" % obj) - soname = self._obj_properties[obj][3] -+ obj_key = self._generateObjKey(obj) ++ obj_key = self._ObjectKey(obj) + if obj_key not in self._obj_properties: + raise KeyError("%s (%s) not in object list" % (obj_key, obj)) + soname = self._obj_properties[obj_key][3] @@ -365,15 +411,22 @@ + if obj not in self._obj_key_cache: + raise KeyError("%s not in object list" % obj) + return self._obj_properties[self._obj_key_cache[obj]][3] -+ -+ def findProviders(self, obj=None, obj_key=None): + + def findProviders(self, obj): +- if not self._libs: +- self.rebuild() + """ + Find providers for an object or object key. + -+ This method should be called with either an obj or obj_key. If called -+ with both, the obj_key is ignored. If called with neither, KeyError is -+ raised as if an invalid obj was passed. -+ ++ This method may be called with a key from _obj_properties. + +- realpath_cache = {} +- def realpath(p): +- real_path = realpath_cache.get(p) +- if real_path is None: +- real_path = os.path.realpath(p) +- realpath_cache[p] = real_path +- return real_path + In some cases, not all valid libraries are returned. This may occur when + an soname symlink referencing a library is in an object's runpath while + the actual library is not. We should consider cataloging symlinks within @@ -381,30 +434,14 @@ + library dependencies (since the dynamic linker actually searches for + files named with the soname in the runpaths). + -+ @param obj: absolute path to an object -+ @type obj: string (example: '/usr/bin/bar') -+ @param obj_key: key from LinkageMap._generateObjKey -+ @type obj_key: 2-tuple of longs or string ++ @param obj: absolute path to an object or a key from _obj_properties ++ @type obj: string (example: '/usr/bin/bar') or _ObjectKey + @rtype: dict (example: {'libbar.so': set(['/lib/libbar.so.1.5'])}) + @return: The return value is a soname -> set-of-library-paths, where + set-of-library-paths satisfy soname. -+ -+ """ -+ rValue = {} - -- def findProviders(self, obj): - if not self._libs: - self.rebuild() -- realpath_cache = {} -- def realpath(p): -- real_path = realpath_cache.get(p) -- if real_path is None: -- real_path = os.path.realpath(p) -- realpath_cache[p] = real_path -- return real_path -- -- rValue = {} ++ """ + rValue = {} - if obj not in self._obj_properties: - obj = realpath(obj) - if obj not in self._obj_properties: @@ -422,15 +459,21 @@ - rValue[x].add(y) - elif realpath(os.path.dirname(y)) in path: - rValue[x].add(y) ++ ++ if not self._libs: ++ self.rebuild() ++ + # Determine the obj_key from the arguments. -+ if obj is not None: ++ if isinstance(obj, self._ObjectKey): ++ obj_key = obj ++ if obj_key not in self._obj_properties: ++ raise KeyError("%s not in object list" % obj_key) ++ else: + obj_key = self._obj_key_cache.get(obj) + if obj_key not in self._obj_properties: -+ obj_key = self._generateObjKey(obj) ++ obj_key = self._ObjectKey(obj) + if obj_key not in self._obj_properties: + raise KeyError("%s (%s) not in object list" % (obj_key, obj)) -+ elif obj_key not in self._obj_properties: -+ raise KeyError("%s not in object list" % obj_key) + + arch, needed, path, _, _ = self._obj_properties[obj_key] + path = path.union(self._defpath) @@ -447,25 +490,20 @@ + rValue[soname].add(provider) return rValue - -- def findConsumers(self, obj): + -+ def findConsumers(self, obj=None, obj_key=None): + def findConsumers(self, obj): + """ + Find consumers of an object or object key. + -+ This method should be called with either an obj or obj_key. If called -+ with both, the obj_key is ignored. If called with neither, KeyError is -+ raised as if an invalid obj was passed. ++ This method may be called with a key from _obj_properties. If this method is going to be called with object keys, to avoid not catching shadowed libraryies, do not pass new _ObjectKey instances to this method. Instead pass the library as a string. + + In some cases, not all consumers are returned. This may occur when + an soname symlink referencing a library is in an object's runpath while + the actual library is not. + -+ @param obj: absolute path to an object -+ @type obj: string (example: '/usr/bin/bar') -+ @param obj_key: key from LinkageMap._generateObjKey -+ @type obj_key: 2-tuple of longs or string -+ @rtype: set of strings (example: ) ++ @param obj: absolute path to an object or a key from _obj_properties ++ @type obj: string (example: '/usr/bin/bar') or _ObjectKey ++ @rtype: set of strings (example: set(['/bin/foo', '/usr/bin/bar'])) + @return: The return value is a soname -> set-of-library-paths, where + set-of-library-paths satisfy soname. + @@ -488,17 +526,18 @@ - if obj not in self._obj_properties: - raise KeyError("%s not in object list" % obj) + # Determine the obj_key and the set of objects matching the arguments. -+ if obj is not None: ++ if isinstance(obj, self._ObjectKey): ++ obj_key = obj ++ if obj_key not in self._obj_properties: ++ raise KeyError("%s not in object list" % obj_key) ++ objs = self._obj_properties[obj_key][4] ++ else: + objs = set([obj]) + obj_key = self._obj_key_cache.get(obj) + if obj_key not in self._obj_properties: -+ obj_key = self._generateObjKey(obj) ++ obj_key = self._ObjectKey(obj) + if obj_key not in self._obj_properties: + raise KeyError("%s (%s) not in object list" % (obj_key, obj)) -+ else: -+ if obj_key not in self._obj_properties: -+ raise KeyError("%s not in object list" % obj_key) -+ objs = self._obj_properties[obj_key][4] + + # Determine the directory(ies) from the set of objects. + objs_dirs = set([os.path.dirname(x) for x in objs]) @@ -531,7 +570,7 @@ - rValue.add(x) - elif realpath(obj_dir) in path: - rValue.add(x) -+ if obj is not None: ++ if not isinstance(obj, self._ObjectKey): + soname = self._obj_properties[obj_key][3] + obj_dir = os.path.dirname(obj) + master_link = os.path.join(obj_dir, soname) -- 2.11.4.GIT