pym/portage/sets/libs.py

   1 # Copyright 2007 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3 # $Id: libs.py 10759 2008-06-22 04:04:50Z zmedico $
   4
   5 import os
   6 import re
   7 import time
   8 from portage.dbapi.vartree import dblink
   9 from portage.versions import catsplit
  10 from portage.sets.base import PackageSet
  11 from portage.sets import get_boolean
  12 from portage.versions import catpkgsplit
  13
  14 __all__ = ["LibraryConsumerSet", "PreservedLibraryConsumerSet",
  15                 "MissingLibraryConsumerSet"]
  16
  17 class LibraryConsumerSet(PackageSet):
  18         _operations = ["merge", "unmerge"]
  19
  20         def __init__(self, vardbapi, debug=False):
  21                 super(LibraryConsumerSet, self).__init__()
  22                 self.dbapi = vardbapi
  23                 self.debug = debug
  24
  25         def mapPathsToAtoms(self, paths):
  26                 rValue = set()
  27                 for link, p in self.dbapi._owners.iter_owners(paths):
  28                         cat, pn = catpkgsplit(link.mycpv)[:2]
  29                         slot = self.dbapi.aux_get(link.mycpv, ["SLOT"])[0]
  30                         rValue.add("%s/%s:%s" % (cat, pn, slot))
  31                 return rValue
  32
  33 class PreservedLibraryConsumerSet(LibraryConsumerSet):
  34         def load(self):
  35                 reg = self.dbapi.plib_registry
  36                 consumers = set()
  37                 if reg:
  38                         for libs in reg.getPreservedLibs().values():
  39                                 for lib in libs:
  40                                         if self.debug:
  41                                                 print lib
  42                                                 for x in sorted(self.dbapi.linkmap.findConsumers(lib)):
  43                                                         print "    ", x
  44                                                 print "-"*40
  45                                         consumers.update(self.dbapi.linkmap.findConsumers(lib))
  46                 else:
  47                         return
  48                 if not consumers:
  49                         return
  50                 self._setAtoms(self.mapPathsToAtoms(consumers))
  51
  52         def singleBuilder(cls, options, settings, trees):
  53                 debug = get_boolean(options, "debug", False)
  54                 return PreservedLibraryConsumerSet(trees["vartree"].dbapi, debug)
  55         singleBuilder = classmethod(singleBuilder)
  56
  57
  58 class MissingLibraryConsumerSet(LibraryConsumerSet):
  59
  60         """
  61         This class is the set of packages to emerge due to missing libraries.
  62
  63         This class scans binaries for missing and broken shared library dependencies
  64         and fixes them by emerging the packages containing the broken binaries.
  65
  66         The user may also emerge packages containing consumers of specified
  67         libraries by passing the name or a python regular expression through the
  68         environment variable, LIBRARY.  Due to a limitation in passing flags to
  69         package sets through the portage cli, the user must set environment
  70         variables to modify the behaviour of this package set.  So if the
  71         environment variable LIBRARY is set, the behaviour of this set changes.
  72
  73         """
  74
  75         description = "The set of packages to emerge due to missing libraries."
  76         _operations = ["merge"]
  77
  78         def __init__(self, vardbapi, debug=False):
  79                 super(MissingLibraryConsumerSet, self).__init__(vardbapi, debug)
  80                 # FIXME Since we can't get command line arguments from the user, the
  81                 # soname can be passed through an environment variable for now.
  82                 self.libraryRegexp = os.getenv("LIBRARY")
  83                 self.root = self.dbapi.root
  84                 self.linkmap = self.dbapi.linkmap
  85
  86         def load(self):
  87                 # brokenDependencies: object -> set-of-unsatisfied-dependencies, where
  88                 # object is an installed binary/library and
  89                 # set-of-unsatisfied-dependencies are sonames or libraries required by
  90                 # the object but have no corresponding libraries to fulfill the
  91                 # dependency.
  92                 brokenDependencies = {}
  93                 atoms = set()
  94
  95                 # If the LIBRARY environment variable is set, the resulting package set
  96                 # will be packages containing consumers of the libraries matched by the
  97                 # variable.
  98                 if self.libraryRegexp:
  99                         atoms = self.findAtomsOfLibraryConsumers(self.libraryRegexp)
 100                         self._setAtoms(atoms)
 101                         if self.debug:
 102                                 print
 103                                 print "atoms to be emerged:"
 104                                 for x in sorted(atoms):
 105                                         print x
 106                         return
 107
 108                 # Rebuild LinkageMap.
 109                 if self.debug:
 110                         timeStart = time.time()
 111                 self.linkmap.rebuild()
 112                 if self.debug:
 113                         timeRebuild = time.time() - timeStart
 114
 115                 # Get the list of broken dependencies from LinkageMap.
 116                 if self.debug:
 117                         timeStart = time.time()
 118                 brokenDependencies = self.linkmap.listBrokenBinaries(self.debug)
 119                 if self.debug:
 120                         timeListBrokenBinaries = time.time() - timeStart
 121
 122                 # Add broken libtool libraries into the brokenDependencies dict.
 123                 if self.debug:
 124                         timeStart = time.time()
 125                 brokenDependencies.update(self.listBrokenLibtoolLibraries())
 126                 if self.debug:
 127                         timeLibtool = time.time() - timeStart
 128
 129                 # FIXME Too many atoms may be emerged because libraries in binary
 130                 # packages are not being handled properly eg openoffice, nvidia-drivers,
 131                 # sun-jdk.  Certain binaries are run in an environment where additional
 132                 # library paths are added via LD_LIBRARY_PATH.  Since these paths aren't
 133                 # registered in _obj_properties, they appear broken (and are if not run
 134                 # in the correct environment).  I have to determine if libraries and lib
 135                 # paths should be masked using /etc/revdep-rebuild/* as done in
 136                 # revdep-rebuild or if there is a better way to identify and deal with
 137                 # these problematic packages (or if something entirely different should
 138                 # be done).  For now directory and library masks are used.
 139
 140                 # Remove masked directories and libraries.
 141                 if self.debug:
 142                         timeStart = time.time()
 143                 if brokenDependencies:
 144                         brokenDependencies = self.removeMaskedDependencies(brokenDependencies)
 145                 if self.debug:
 146                         timeMask = time.time() - timeStart
 147
 148                 # Determine atoms to emerge based on broken objects in
 149                 # brokenDependencies.
 150                 if self.debug:
 151                         timeStart = time.time()
 152                 if brokenDependencies:
 153                         atoms = self.mapPathsToAtoms(set(brokenDependencies.keys()))
 154                 if self.debug:
 155                         timeAtoms = time.time() - timeStart
 156
 157                 # Debug output
 158                 if self.debug:
 159                         print
 160                         print len(brokenDependencies), "brokenDependencies:"
 161                         for x in sorted(brokenDependencies.keys()):
 162                                 print
 163                                 print x, "->"
 164                                 print '\t', brokenDependencies[x]
 165                         print
 166                         print "atoms to be emerged:"
 167                         for x in sorted(atoms):
 168                                 print x
 169                         print
 170                         print "Rebuild time:", timeRebuild
 171                         print "Broken binaries time:", timeListBrokenBinaries
 172                         print "Broken libtool time:", timeLibtool
 173                         print "Remove mask time:", timeMask
 174                         print "mapPathsToAtoms time:", timeAtoms
 175                         print
 176
 177                 self._setAtoms(atoms)
 178
 179         def removeMaskedDependencies(self, dependencies):
 180                 """
 181                 Remove all masked dependencies and return the updated mapping.
 182
 183                 @param dependencies: dependencies from which to removed masked
 184                         dependencies
 185                 @type dependencies: dict (example: {'/usr/bin/foo': set(['libfoo.so'])})
 186                 @rtype: dict
 187                 @return: shallow copy of dependencies with masked items removed
 188
 189                 """
 190                 rValue = dependencies.copy()
 191                 dirMask, libMask = self.getDependencyMasks()
 192
 193                 # Remove entries that are masked.
 194                 if dirMask or libMask:
 195                         if self.debug:
 196                                 print "The following are masked:"
 197                         for binary, libSet in rValue.items():
 198                                 for directory in dirMask:
 199                                         # Check if the broken binary lies within the masked directory or
 200                                         # its subdirectories.
 201                                         # XXX Perhaps we should allow regexps as masks.
 202                                         if binary.startswith(directory):
 203                                                 del rValue[binary]
 204                                                 if self.debug:
 205                                                         print "dirMask:",binary
 206                                                 break
 207                                 # Check if all the required libraries are masked.
 208                                 if binary in rValue and libSet.issubset(libMask):
 209                                         del rValue[binary]
 210                                         if self.debug:
 211                                                 print "libMask:", binary, libSet & libMask
 212
 213                 if self.debug:
 214                         print
 215                         print "Directory mask:", dirMask
 216                         print
 217                         print "Library mask:", libMask
 218
 219                 return rValue
 220
 221         def getDependencyMasks(self):
 222                 """
 223                 Return all dependency masks as a tuple.
 224
 225                 @rtype: 2-tuple of sets of strings
 226                 @return: 2-tuple in which the first component is a set of directory
 227                         masks and the second component is a set of library masks
 228
 229                 """
 230                 dirMask = set()
 231                 libMask = set()
 232                 _dirMask_re = re.compile(r'SEARCH_DIRS_MASK\s*=\s*"([^"]*)"')
 233                 _libMask_re = re.compile(r'LD_LIBRARY_MASK\s*=\s*"([^"]*)"')
 234                 lines = []
 235
 236                 # Reads the contents of /etc/revdep-rebuild/*
 237                 libMaskDir = os.path.join(self.root, "etc", "revdep-rebuild")
 238                 if os.path.exists(libMaskDir):
 239                         for file in os.listdir(libMaskDir):
 240                                 try:
 241                                         f = open(os.path.join(libMaskDir, file), "r")
 242                                         try:
 243                                                 lines.extend(f.readlines())
 244                                         finally:
 245                                                 f.close()
 246                                 except IOError: # OSError?
 247                                         continue
 248                         # The following parses SEARCH_DIRS_MASK and LD_LIBRARY_MASK variables
 249                         # from /etc/revdep-rebuild/*
 250                         for line in lines:
 251                                 matchDir = _dirMask_re.match(line)
 252                                 matchLib = _libMask_re.match(line)
 253                                 if matchDir:
 254                                         dirMask.update(set(matchDir.group(1).split()))
 255                                 if matchLib:
 256                                         libMask.update(set(matchLib.group(1).split()))
 257
 258                 # These directories contain specially evaluated libraries.
 259                 # app-emulation/vmware-workstation-6.0.1.55017
 260                 dirMask.add('/opt/vmware/workstation/lib')
 261                 # app-emulation/vmware-server-console-1.0.6.91891
 262                 dirMask.add('/opt/vmware/server/console/lib')
 263                 # www-client/mozilla-firefox-2.0.0.15
 264                 dirMask.add('/usr/lib/mozilla-firefox/plugins')
 265                 dirMask.add('/usr/lib64/mozilla-firefox/plugins')
 266                 # app-office/openoffice-2.4.1
 267                 dirMask.add('/opt/OpenOffice')
 268                 dirMask.add('/usr/lib/openoffice')
 269                 # dev-libs/libmix-2.05  libmix.so is missing soname entry
 270                 libMask.add('libmix.so')
 271                 # app-accessibility/speech-tools-1.2.96_beta  missing sonames
 272                 libMask.add('libestools.so')
 273                 libMask.add('libestbase.so')
 274                 libMask.add('libeststring.so')
 275                 # app-emulation/emul-linux-x86-soundlibs-20080418
 276                 dirMask.add('/usr/kde/3.5/lib32')
 277
 278                 return (dirMask, libMask)
 279
 280         def listBrokenLibtoolLibraries(self):
 281                 """
 282                 Find broken libtool libraries and their missing dependencies.
 283
 284                 Consider adding new entry into /var/db/pkg to catalog libtool libraries
 285                 (similar in nature to NEEDED.ELF.2).  Currently, the contents of all
 286                 packages are searched in order to find libtool libraries, so having them
 287                 available in the vdb_path will speed things up.
 288
 289                 @rtype: dict (example: {'/lib/libfoo.la': set(['/lib/libbar.la'])})
 290                 @return: The return value is a library -> set-of-libraries mapping, where
 291                         library is a broken library and the set consists of dependencies
 292                         needed by library that do not exist on the filesystem.
 293
 294                 """
 295                 rValue = {}
 296                 lines = []
 297                 dependencies = []
 298                 _la_re = re.compile(r".*\.la$")
 299                 _dependency_libs_re = re.compile(r"^dependency_libs\s*=\s*'(.*)'")
 300
 301                 # Loop over the contents of all packages.
 302                 for cpv in self.dbapi.cpv_all():
 303                         mysplit = catsplit(cpv)
 304                         link = dblink(mysplit[0], mysplit[1], myroot=self.dbapi.root, \
 305                                         mysettings=self.dbapi.settings, treetype='vartree', \
 306                                         vartree=self.dbapi.vartree)
 307                         for file in link.getcontents():
 308                                 # Check if the file ends with '.la'.
 309                                 matchLib = _la_re.match(file)
 310                                 if matchLib:
 311                                         # Read the lines from the library.
 312                                         lines = []
 313                                         try:
 314                                                 f = open(file, "r")
 315                                                 try:
 316                                                         lines.extend(f.readlines())
 317                                                 finally:
 318                                                         f.close()
 319                                         except IOError:
 320                                                 continue
 321                                         # Find the line listing the dependencies.
 322                                         for line in lines:
 323                                                 matchLine = _dependency_libs_re.match(line)
 324                                                 if matchLine:
 325                                                         dependencies = matchLine.group(1).split()
 326                                                         # For each dependency that is a pathname (begins with
 327                                                         # os.sep), check that it exists on the filesystem.  If it
 328                                                         # does not exist, then add the library and the missing
 329                                                         # dependency to rValue.
 330                                                         for dependency in dependencies:
 331                                                                 if dependency[0] == os.sep and \
 332                                                                                 not os.path.isfile(dependency):
 333                                                                         rValue.setdefault(file, set()).add(dependency)
 334
 335                 return rValue
 336
 337         def findAtomsOfLibraryConsumers(self, searchString):
 338                 """
 339                 Return atoms containing consumers of libraries matching the argument.
 340
 341                 The libraries returned by LinkageMap.listLibraryObjects() are matched
 342                 against the searchString regular expression.  Consequently, only files
 343                 cataloged in LinkageMap will be considered.  Since symlinks aren't
 344                 entered into NEEDED.ELF.2 files, LinkageMap doesn't catalog them.  So if
 345                 /usr/lib were a symlink to /usr/lib64 and the regular expression were
 346                 /usr/lib/libfoo*, nothing would be matched (libfoo would work however).
 347                 This can be fixed by adding symlink entries into LinkageMap, searching
 348                 CONTENTS files for symlinks, or utilizing the find utility.
 349
 350                 @param searchString: a string used to search for libraries
 351                 @type searchString: string to be compiled as a regular expression
 352                         (example: 'libfoo.*')
 353                 @rtype: set of strings
 354                 @return: the returned set of atoms are valid to be used by package sets
 355
 356                 """
 357                 atoms = set()
 358                 consumers = set()
 359                 matchedLibraries = set()
 360                 libraryObjects = self.linkmap.listLibraryObjects()
 361                 _librarySearch_re = re.compile(searchString)
 362
 363                 # Find libraries matching searchString.
 364                 for library in libraryObjects:
 365                         m = _librarySearch_re.search(library)
 366                         if m:
 367                                 matchedLibraries.add(library)
 368                                 consumers.update(self.linkmap.findConsumers(library))
 369
 370                 if self.debug:
 371                         print
 372                         print "Consumers of the following libraries will be emerged:"
 373                         for x in matchedLibraries:
 374                                 print x
 375
 376                 if consumers:
 377                         # The following prevents emerging the packages that own the matched
 378                         # libraries.  Note that this will prevent updating the packages owning
 379                         # the libraries if there are newer versions available in the installed
 380                         # slot.  See bug #30095
 381                         atoms = self.mapPathsToAtoms(consumers)
 382                         libraryOwners = self.mapPathsToAtoms(matchedLibraries)
 383                         atoms.difference_update(libraryOwners)
 384
 385                 return atoms
 386
 387         def singleBuilder(self, options, settings, trees):
 388                 debug = get_boolean(options, "debug", False)
 389                 return MissingLibraryConsumerSet(trees["vartree"].dbapi, debug)
 390         singleBuilder = classmethod(singleBuilder)