obj_properties now uses sets for paths.

[revdep-rebuild-reimplementation.git] / pym / portage / dbapi / vartree.py

# Copyright 1998-2007 Gentoo Foundation
# Distributed under the terms of the GNU General Public License v2
# $Id: vartree.py 11392 2008-08-10 10:30:20Z zmedico $

__all__ = ["PreservedLibsRegistry", "LinkageMap",
        "vardbapi", "vartree", "dblink"] + \
        ["write_contents", "tar_contents"]

from portage.checksum import perform_md5
from portage.const import CACHE_PATH, CONFIG_MEMORY_FILE, PORTAGE_BIN_PATH, \
        PRIVATE_PATH, VDB_PATH
from portage.data import portage_gid, portage_uid, secpass
from portage.dbapi import dbapi
from portage.dep import dep_getslot, use_reduce, paren_reduce, isvalidatom, \
        isjustname, dep_getkey, match_from_list
from portage.exception import InvalidAtom, InvalidData, InvalidPackageName, \
        FileNotFound, PermissionDenied, UnsupportedAPIException
from portage.locks import lockdir, unlockdir
from portage.output import bold, red, green
from portage.update import fixdbentries
from portage.util import apply_secpass_permissions, ConfigProtect, ensure_dirs, \
        writemsg, writemsg_stdout, writemsg_level, \
        write_atomic, atomic_ofstream, writedict, \
        grabfile, grabdict, normalize_path, new_protect_filename, getlibpaths
from portage.versions import pkgsplit, catpkgsplit, catsplit, best, pkgcmp

from portage import listdir, dep_expand, flatten, key_expand, \
        doebuild_environment, doebuild, env_update, prepare_build_dirs, \
        abssymlink, movefile, _movefile, bsd_chflags, cpv_getkey

from portage.elog import elog_process
from portage.elog.filtering import filter_mergephases, filter_unmergephases

import os, re, sys, stat, errno, commands, copy, time, subprocess
import logging
import shlex
from itertools import izip

try:
        import cPickle
except ImportError:
        import pickle as cPickle

class PreservedLibsRegistry(object):
        """ This class handles the tracking of preserved library objects """
        def __init__(self, filename, autocommit=True):
                """ @param filename: absolute path for saving the preserved libs records
                    @type filename: String
                        @param autocommit: determines if the file is written after every update
                        @type autocommit: Boolean
                """
                self._filename = filename
                self._autocommit = autocommit
                self.load()
                self.pruneNonExisting()

        def load(self):
                """ Reload the registry data from file """
                try:
                        self._data = cPickle.load(open(self._filename, "r"))
                except (EOFError, IOError), e:
                        if isinstance(e, EOFError) or e.errno == errno.ENOENT:
                                self._data = {}
                        elif e.errno == PermissionDenied.errno:
                                raise PermissionDenied(self._filename)
                        else:
                                raise e
                
        def store(self):
                """ Store the registry data to file. No need to call this if autocommit
                    was enabled.
                """
                if os.environ.get("SANDBOX_ON") == "1":
                        return
                try:
                        f = atomic_ofstream(self._filename)
                        cPickle.dump(self._data, f)
                        f.close()
                except EnvironmentError, e:
                        if e.errno != PermissionDenied.errno:
                                writemsg("!!! %s %s\n" % (e, self._filename), noiselevel=-1)

        def register(self, cpv, slot, counter, paths):
                """ Register new objects in the registry. If there is a record with the
                        same packagename (internally derived from cpv) and slot it is 
                        overwritten with the new data.
                        @param cpv: package instance that owns the objects
                        @type cpv: CPV (as String)
                        @param slot: the value of SLOT of the given package instance
                        @type slot: String
                        @param counter: vdb counter value for the package instace
                        @type counter: Integer
                        @param paths: absolute paths of objects that got preserved during an update
                        @type paths: List
                """
                cp = "/".join(catpkgsplit(cpv)[:2])
                cps = cp+":"+slot
                if len(paths) == 0 and cps in self._data \
                                and self._data[cps][0] == cpv and int(self._data[cps][1]) == int(counter):
                        del self._data[cps]
                elif len(paths) > 0:
                        self._data[cps] = (cpv, counter, paths)
                if self._autocommit:
                        self.store()
        
        def unregister(self, cpv, slot, counter):
                """ Remove a previous registration of preserved objects for the given package.
                        @param cpv: package instance whose records should be removed
                        @type cpv: CPV (as String)
                        @param slot: the value of SLOT of the given package instance
                        @type slot: String
                """
                self.register(cpv, slot, counter, [])
        
        def pruneNonExisting(self):
                """ Remove all records for objects that no longer exist on the filesystem. """
                for cps in self._data.keys():
                        cpv, counter, paths = self._data[cps]
                        paths = [f for f in paths if os.path.exists(f)]
                        if len(paths) > 0:
                                self._data[cps] = (cpv, counter, paths)
                        else:
                                del self._data[cps]
                if self._autocommit:
                        self.store()
        
        def hasEntries(self):
                """ Check if this registry contains any records. """
                return len(self._data) > 0
        
        def getPreservedLibs(self):
                """ Return a mapping of packages->preserved objects.
                        @returns mapping of package instances to preserved objects
                        @rtype Dict cpv->list-of-paths
                """
                rValue = {}
                for cps in self._data:
                        rValue[self._data[cps][0]] = self._data[cps][2]
                return rValue

class LinkageMap(object):
        def __init__(self, vardbapi):
                self._dbapi = vardbapi
                self._libs = {}
                self._obj_properties = {}
                self._defpath = set(getlibpaths())
                self._obj_key_cache = {}

        def rebuild(self, include_file=None):
                libs = {}
                obj_key_cache = {}
                obj_properties = {}
                lines = []
                for cpv in self._dbapi.cpv_all():
                        lines += self._dbapi.aux_get(cpv, ["NEEDED.ELF.2"])[0].split('\n')
                # Cache NEEDED.* files avoid doing excessive IO for every rebuild.
                self._dbapi.flush_cache()
                
                if include_file:
                        lines += grabfile(include_file)
                
                # have to call scanelf for preserved libs here as they aren't 
                # registered in NEEDED.ELF.2 files
                if self._dbapi.plib_registry and self._dbapi.plib_registry.getPreservedLibs():
                        args = ["/usr/bin/scanelf", "-qF", "%a;%F;%S;%r;%n"]
                        for items in self._dbapi.plib_registry.getPreservedLibs().values():
                                args += [x.lstrip(".") for x in items]
                        proc = subprocess.Popen(args, stdout=subprocess.PIPE)
                        output = [l[3:] for l in proc.communicate()[0].split("\n")]
                        lines += output

                for l in lines:
                        if l.strip() == "":
                                continue
                        fields = l.strip("\n").split(";")
                        if len(fields) < 5:
                                print "Error", fields
                                # insufficient field length
                                continue
                        arch = fields[0]
                        obj = fields[1]
                        obj_key = self._generateObjKey(obj)
                        soname = fields[2]
                        path = set([normalize_path(x)
                                for x in filter(None, fields[3].replace(
                                "${ORIGIN}", os.path.dirname(obj)).replace(
                                "$ORIGIN", os.path.dirname(obj)).split(":"))])
                        needed = filter(None, fields[4].split(","))
                        if soname:
                                libs.setdefault(soname, \
                                                {arch: {"providers": set(), "consumers": set()}})
                                libs[soname].setdefault(arch, \
                                                {"providers": set(), "consumers": set()})
                                libs[soname][arch]["providers"].add(obj_key)
                        for x in needed:
                                libs.setdefault(x, \
                                                {arch: {"providers": set(), "consumers": set()}})
                                libs[x].setdefault(arch, {"providers": set(), "consumers": set()})
                                libs[x][arch]["consumers"].add(obj_key)
                        obj_key_cache.setdefault(obj, obj_key)
                        # All object paths are added into the obj_properties tuple
                        obj_properties.setdefault(obj_key, \
                                        (arch, needed, path, soname, set()))[4].add(obj)

                self._libs = libs
                self._obj_properties = obj_properties
                self._obj_key_cache = obj_key_cache

        def _generateObjKey(self, obj):
                """
                Generate obj key for a given object.

                @param obj: path to an existing file
                @type obj: string (example: '/usr/bin/bar')
                @rtype: 2-tuple of longs or string
                @return: If obj exists, a 2-tuple of obj's inode and device from a stat
                        call is returned.  Otherwise, the obj is returned.

                """
                try:
                        return os.stat(obj)[1:3]
                except OSError:
                        # XXX pull this out
#                       from portage.output import teal
#                       writemsg(bold(red("Error in ")) + \
#                                       bold(teal("_generateObjKey. Stat failed on %s" % obj)) + '\n')
                        return obj

        def listBrokenBinaries(self):
                """
                Find binaries and their needed sonames, which have no providers.

                @rtype: dict (example: {'/usr/bin/foo': set(['libbar.so'])})
                @return: The return value is an object -> set-of-sonames mapping, where
                        object is a broken binary and the set consists of sonames needed by
                        object that have no corresponding libraries to fulfill the dependency.

                """
                class LibraryCache(object):

                        """
                        Caches sonames and realpaths associated with paths.

                        The purpose of this class is to prevent multiple calls of
                        os.path.realpath and os.path.isfile on the same paths.

                        """

                        def __init__(cache_self):
                                cache_self.cache = {}

                        def get(cache_self, obj):
                                """
                                Caches and returns properties associated with an object.

                                @param obj: absolute path (can be symlink)
                                @type obj: string (example: '/usr/lib/libfoo.so')
                                @rtype: 4-tuple with types
                                        (string or None, string or None, 2-tuple, Boolean)
                                @return: 4-tuple with the following components:
                                        1. arch as a string or None if it does not exist,
                                        2. soname as a string or None if it does not exist,
                                        3. obj_key as 2-tuple,
                                        4. Boolean representing whether the object exists.
                                        (example: ('libfoo.so.1', (123L, 456L), True))

                                """
                                if obj in cache_self.cache:
                                        return cache_self.cache[obj]
                                else:
                                        if obj in self._obj_key_cache:
                                                obj_key = self._obj_key_cache.get(obj)
                                        else:
                                                obj_key = self._generateObjKey(obj)
                                        # Check that the library exists on the filesystem.
                                        if isinstance(obj_key, tuple):
                                                # Get the arch and soname from LinkageMap._obj_properties if
                                                # it exists. Otherwise, None.
                                                arch, _, _, soname, _ = \
                                                                self._obj_properties.get(obj_key, (None,)*5)
                                                return cache_self.cache.setdefault(obj, \
                                                                (arch, soname, obj_key, True))
                                        else:
                                                return cache_self.cache.setdefault(obj, \
                                                                (None, None, obj_key, False))

                debug = False
                rValue = {}
                cache = LibraryCache()
                providers = self.listProviders()

                # Iterate over all obj_keys and their providers.
                for obj_key, sonames in providers.items():
                        arch, _, path, _, objs = self._obj_properties[obj_key]
                        path = path.union(self._defpath)
                        # Iterate over each needed soname and the set of library paths that
                        # fulfill the soname to determine if the dependency is broken.
                        for soname, libraries in sonames.items():
                                # validLibraries is used to store libraries, which satisfy soname,
                                # so if no valid libraries are found, the soname is not satisfied
                                # for obj_key.  If unsatisfied, objects associated with obj_key
                                # must be emerged.
                                validLibraries = set()
                                # It could be the case that the library to satisfy the soname is
                                # not in the obj's runpath, but a symlink to the library is (eg
                                # libnvidia-tls.so.1 in nvidia-drivers).  Also, since LinkageMap
                                # does not catalog symlinks, broken or missing symlinks may go
                                # unnoticed.  As a result of these cases, check that a file with
                                # the same name as the soname exists in obj's runpath.
                                # XXX If we catalog symlinks in LinkageMap, this could be improved.
                                for directory in path:
                                        cachedArch, cachedSoname, cachedKey, cachedExists = \
                                                        cache.get(os.path.join(directory, soname))
                                        # Check that this library provides the needed soname.  Doing
                                        # this, however, will cause consumers of libraries missing
                                        # sonames to be unnecessarily emerged. (eg libmix.so)
                                        if cachedSoname == soname and cachedArch == arch:
                                                validLibraries.add(cachedKey)
                                                if debug and cachedKey not in \
                                                                map(self._obj_key_cache.get, libraries):
                                                        print "Found provider outside of findProviders:", \
                                                                        os.path.join(directory, soname), "->", \
                                                                        cachedKey
                                                # A valid library has been found, so there is no need to
                                                # continue.
                                                break
                                        if debug and cachedKey in self._obj_properties:
                                                print "Broken symlink or missing/bad soname:", \
                                                                os.path.join(directory, soname), '->', \
                                                                cachedKey, "with soname", cachedSoname, \
                                                                "but expecting", soname
                                # This conditional checks if there are no libraries to satisfy the
                                # soname (empty set).
                                if not validLibraries:
                                        for obj in objs:
                                                rValue.setdefault(obj, set()).add(soname)
                                        # If no valid libraries have been found by this point, then
                                        # there are no files named with the soname within obj's runpath,
                                        # but if there are libraries (from the providers mapping), it is
                                        # likely that symlinks or the actual libraries are missing.
                                        # Thus possible symlinks and missing libraries are added to
                                        # rValue in order to emerge corrupt library packages.
                                        for lib in libraries:
                                                cachedArch, cachedSoname, cachedKey, cachedExists = \
                                                                cache.get(lib)
                                                if not cachedExists:
                                                        # The library's package needs to be emerged to repair the
                                                        # missing library.
                                                        rValue.setdefault(lib, set()).add(soname)
                                                else:
                                                        # A library providing the soname exists in the obj's
                                                        # runpath, but no file named as the soname exists, so add
                                                        # the path constructed from the lib's directory and the
                                                        # soname to rValue to fix cases of vanishing (or modified)
                                                        # symlinks.  This path is not guaranteed to exist, but it
                                                        # follows the symlink convention found in the majority of
                                                        # packages.
                                                        # XXX merge this into block above
                                                        rValue.setdefault(os.path.join(os.path.dirname(lib), \
                                                                        soname), set()).add(soname)
                                                if debug:
                                                        if not os.path.isfile(lib):
                                                                print "Missing library:", lib
                                                        else:
                                                                print "Possibly missing symlink:", \
                                                                                os.path.join(os.path.dirname(lib), soname)

                return rValue

        def listConsumers(self):
                rValue = {}
                if not self._libs:
                        self.rebuild()
                # Iterate over all objects within LinkageMap.
                for obj_key in self._obj_properties:
                        rValue.setdefault(obj_key, self.findConsumers(obj_key=obj_key))
                return rValue

        def listProviders(self):
                """
                Find the providers for all objects in LinkageMap.

                @rtype: dict (example:
                        {(123L, 456L): {'libbar.so': set(['/lib/libbar.so.1.5'])}})
                @return: The return value is an object key -> providers mapping, where
                        providers is a mapping of soname -> set-of-library-paths returned
                        from the findProviders method.

                """
                rValue = {}
                if not self._libs:
                        self.rebuild()
                # Iterate over all objects within LinkageMap.
                for obj_key in self._obj_properties:
                        # XXX remove this
                        if len(self._obj_properties[obj_key][4]) != 1:
                                writemsg(bold(red(self._obj_properties[obj_key])))
                        rValue.setdefault(obj_key, self.findProviders(obj_key=obj_key))
                return rValue

        def isMasterLink(self, obj):
                basename = os.path.basename(obj)
                obj_key = self._generateObjKey(obj)
                if obj_key not in self._obj_properties:
                        raise KeyError("%s (%s) not in object list" % (obj_key, obj))
                soname = self._obj_properties[obj_key][3]
                return (len(basename) < len(soname))

        def listLibraryObjects(self):
                rValue = []
                if not self._libs:
                        self.rebuild()
                for soname in self._libs:
                        for arch in self._libs[soname]:
                                for obj_key in self._libs[soname][arch]["providers"]:
                                        rValue.extend(self._obj_properties[obj_key][4])
                return rValue

        def getSoname(self, obj):
                if not self._libs:
                        self.rebuild()
                if obj not in self._obj_key_cache:
                        raise KeyError("%s not in object list" % obj)
                return self._obj_properties[self._obj_key_cache[obj]][3]

        def findProviders(self, obj=None, obj_key=None):
                """
                Find providers for an object or object key.

                This method should be called with either an obj or obj_key.  If called
                with both, the obj_key is ignored.  If called with neither, KeyError is
                raised as if an invalid obj was passed.

                @param obj:
                @type obj:
                @param obj_key:
                @type obj_key:
                @rtype:
                @return:

                """
                rValue = {}

                if not self._libs:
                        self.rebuild()

                if obj is not None:
                        obj_key = self._obj_key_cache.get(obj)
                        if obj_key not in self._obj_properties:
                                obj_key = self._generateObjKey(obj)
                                if obj_key not in self._obj_properties:
                                        raise KeyError("%s (%s) not in object list" % (obj_key, obj))
                elif obj_key not in self._obj_properties:
                        raise KeyError("%s not in object list" % obj_key)

                arch, needed, path, soname, objs = self._obj_properties[obj_key]
                path = path.union(self._defpath)
                # XXX test this
                # path = set(realpath(x) for x in path)
                for x in needed:
                        rValue[x] = set()
                        if x not in self._libs or arch not in self._libs[x]:
                                continue
                        for y in self._libs[x][arch]["providers"]:
                                objs = self._obj_properties[y][4]
                                # XXX x is an soname, so it should never start with os.sep, right?
                                #if x[0] == os.sep and realpath(x) == realpath(y):
                                #       rValue[x].add(y)
                                for o in objs:
                                        if os.path.dirname(o) in path:
                                                rValue[x].add(o)

                return rValue

        def findConsumers(self, obj=None, obj_key=None):
                """
                Find consumers of an object or object key.

                This method should be called with either an obj or obj_key.  If called
                with both, the obj_key is ignored.  If called with neither, KeyError is
                raised as if an invalid obj was passed.

                @param obj:
                @type obj:
                @param obj_key:
                @type obj_key:
                @rtype:
                @return:

                """
                rValue = set()

                if not self._libs:
                        self.rebuild()

                if obj is not None:
                        objs = set([obj])
                        obj_key = self._obj_key_cache.get(obj)
                        if obj_key not in self._obj_properties:
                                obj_key = self._generateObjKey(obj)
                                if obj_key not in self._obj_properties:
                                        raise KeyError("%s (%s) not in object list" % (obj_key, obj))
                else:
                        if obj_key not in self._obj_properties:
                                raise KeyError("%s not in object list" % obj_key)
                        objs = self._obj_properties[obj_key][4]

                objs_dirs = [os.path.dirname(x) for x in objs]

                # If there is another version of this lib with the
                # same soname and the master link points to that
                # other version, this lib will be shadowed and won't
                # have any consumers.
                # Only necessary if given obj, which may be a symlink.
                if obj is not None:
                        soname = self._obj_properties[obj_key][3]
                        obj_dir = os.path.dirname(obj)
                        master_link = os.path.join(obj_dir, soname)
                        try:
                                master_st = os.stat(master_link)
                                obj_st = os.stat(obj)
                        except OSError:
                                pass
                        else:
                                if (obj_st.st_dev, obj_st.st_ino) != \
                                        (master_st.st_dev, master_st.st_ino):
                                        return set()

                for soname in self._libs:
                        for arch in self._libs[soname]:
                                if obj_key in self._libs[soname][arch]["providers"]:
                                        for consumer_key in self._libs[soname][arch]["consumers"]:
                                                _, _, path, _, consumer_objs = \
                                                                self._obj_properties[consumer_key]
                                                # XXX test this
                                                #path = [realpath(y) for y in path+self._defpath]
                                                path = path.union(self._defpath)
                                                # XXX x is an soname, so it should never start with os.sep,
                                                # right?
                                                #if soname[0] == os.sep and realpath(soname) == realpath(obj):
                                                #       rValue.add(x)
                                                #if realpath(obj_dir) in path:
                                                #       rValue.add(x)
                                                for consumer_obj in consumer_objs:
                                                        for directory in objs_dirs:
                                                                if directory in path:
                                                                        rValue.add(consumer_obj)
                return rValue

class vardbapi(dbapi):

        _excluded_dirs = ["CVS", "lost+found"]
        _excluded_dirs = [re.escape(x) for x in _excluded_dirs]
        _excluded_dirs = re.compile(r'^(\..*|-MERGING-.*|' + \
                "|".join(_excluded_dirs) + r')$')

        _aux_cache_version        = "1"
        _owners_cache_version     = "1"

        # Number of uncached packages to trigger cache update, since
        # it's wasteful to update it for every vdb change.
        _aux_cache_threshold = 5

        _aux_cache_keys_re = re.compile(r'^NEEDED\..*$')
        _aux_multi_line_re = re.compile(r'^(CONTENTS|NEEDED\..*)$')

        def __init__(self, root, categories=None, settings=None, vartree=None):
                """
                The categories parameter is unused since the dbapi class
                now has a categories property that is generated from the
                available packages.
                """
                self.root = root[:]

                #cache for category directory mtimes
                self.mtdircache = {}

                #cache for dependency checks
                self.matchcache = {}

                #cache for cp_list results
                self.cpcache = {}

                self.blockers = None
                if settings is None:
                        from portage import settings
                self.settings = settings
                if vartree is None:
                        from portage import db
                        vartree = db[root]["vartree"]
                self.vartree = vartree
                self._aux_cache_keys = set(
                        ["CHOST", "COUNTER", "DEPEND", "DESCRIPTION",
                        "EAPI", "HOMEPAGE", "IUSE", "KEYWORDS",
                        "LICENSE", "PDEPEND", "PROVIDE", "RDEPEND",
                        "repository", "RESTRICT" , "SLOT", "USE"])
                self._aux_cache_obj = None
                self._aux_cache_filename = os.path.join(self.root,
                        CACHE_PATH.lstrip(os.path.sep), "vdb_metadata.pickle")
                self._counter_path = os.path.join(root,
                        CACHE_PATH.lstrip(os.path.sep), "counter")

                try:
                        self.plib_registry = PreservedLibsRegistry(
                                os.path.join(self.root, PRIVATE_PATH, "preserved_libs_registry"))
                except PermissionDenied:
                        # apparently this user isn't allowed to access PRIVATE_PATH
                        self.plib_registry = None

                self.linkmap = LinkageMap(self)
                self._owners = self._owners_db(self)

        def getpath(self, mykey, filename=None):
                rValue = os.path.join(self.root, VDB_PATH, mykey)
                if filename != None:
                        rValue = os.path.join(rValue, filename)
                return rValue

        def cpv_exists(self, mykey):
                "Tells us whether an actual ebuild exists on disk (no masking)"
                return os.path.exists(self.getpath(mykey))

        def cpv_counter(self, mycpv):
                "This method will grab the COUNTER. Returns a counter value."
                try:
                        return long(self.aux_get(mycpv, ["COUNTER"])[0])
                except (KeyError, ValueError):
                        pass
                cdir = self.getpath(mycpv)
                cpath = self.getpath(mycpv, filename="COUNTER")

                # We write our new counter value to a new file that gets moved into
                # place to avoid filesystem corruption on XFS (unexpected reboot.)
                corrupted = 0
                if os.path.exists(cpath):
                        cfile = open(cpath, "r")
                        try:
                                counter = long(cfile.readline())
                        except ValueError:
                                print "portage: COUNTER for", mycpv, "was corrupted; resetting to value of 0"
                                counter = long(0)
                                corrupted = 1
                        cfile.close()
                elif os.path.exists(cdir):
                        mys = pkgsplit(mycpv)
                        myl = self.match(mys[0], use_cache=0)
                        print mys, myl
                        if len(myl) == 1:
                                try:
                                        # Only one package... Counter doesn't matter.
                                        write_atomic(cpath, "1")
                                        counter = 1
                                except SystemExit, e:
                                        raise
                                except Exception, e:
                                        writemsg("!!! COUNTER file is missing for "+str(mycpv)+" in /var/db.\n",
                                                noiselevel=-1)
                                        writemsg("!!! Please run %s/fix-db.py or\n" % PORTAGE_BIN_PATH,
                                                noiselevel=-1)
                                        writemsg("!!! unmerge this exact version.\n", noiselevel=-1)
                                        writemsg("!!! %s\n" % e, noiselevel=-1)
                                        sys.exit(1)
                        else:
                                writemsg("!!! COUNTER file is missing for "+str(mycpv)+" in /var/db.\n",
                                        noiselevel=-1)
                                writemsg("!!! Please run %s/fix-db.py or\n" % PORTAGE_BIN_PATH,
                                        noiselevel=-1)
                                writemsg("!!! remerge the package.\n", noiselevel=-1)
                                sys.exit(1)
                else:
                        counter = long(0)
                if corrupted:
                        # update new global counter file
                        write_atomic(cpath, str(counter))
                return counter

        def cpv_inject(self, mycpv):
                "injects a real package into our on-disk database; assumes mycpv is valid and doesn't already exist"
                os.makedirs(self.getpath(mycpv))
                counter = self.counter_tick(self.root, mycpv=mycpv)
                # write local package counter so that emerge clean does the right thing
                write_atomic(self.getpath(mycpv, filename="COUNTER"), str(counter))

        def isInjected(self, mycpv):
                if self.cpv_exists(mycpv):
                        if os.path.exists(self.getpath(mycpv, filename="INJECTED")):
                                return True
                        if not os.path.exists(self.getpath(mycpv, filename="CONTENTS")):
                                return True
                return False

        def move_ent(self, mylist):
                origcp = mylist[1]
                newcp = mylist[2]

                # sanity check
                for cp in [origcp, newcp]:
                        if not (isvalidatom(cp) and isjustname(cp)):
                                raise InvalidPackageName(cp)
                origmatches = self.match(origcp, use_cache=0)
                moves = 0
                if not origmatches:
                        return moves
                for mycpv in origmatches:
                        mycpsplit = catpkgsplit(mycpv)
                        mynewcpv = newcp + "-" + mycpsplit[2]
                        mynewcat = newcp.split("/")[0]
                        if mycpsplit[3] != "r0":
                                mynewcpv += "-" + mycpsplit[3]
                        mycpsplit_new = catpkgsplit(mynewcpv)
                        origpath = self.getpath(mycpv)
                        if not os.path.exists(origpath):
                                continue
                        moves += 1
                        if not os.path.exists(self.getpath(mynewcat)):
                                #create the directory
                                os.makedirs(self.getpath(mynewcat))
                        newpath = self.getpath(mynewcpv)
                        if os.path.exists(newpath):
                                #dest already exists; keep this puppy where it is.
                                continue
                        _movefile(origpath, newpath, mysettings=self.settings)

                        # We need to rename the ebuild now.
                        old_pf = catsplit(mycpv)[1]
                        new_pf = catsplit(mynewcpv)[1]
                        if new_pf != old_pf:
                                try:
                                        os.rename(os.path.join(newpath, old_pf + ".ebuild"),
                                                os.path.join(newpath, new_pf + ".ebuild"))
                                except EnvironmentError, e:
                                        if e.errno != errno.ENOENT:
                                                raise
                                        del e
                        write_atomic(os.path.join(newpath, "PF"), new_pf+"\n")
                        write_atomic(os.path.join(newpath, "CATEGORY"), mynewcat+"\n")
                        fixdbentries([mylist], newpath)
                return moves

        def cp_list(self, mycp, use_cache=1):
                mysplit=catsplit(mycp)
                if mysplit[0] == '*':
                        mysplit[0] = mysplit[0][1:]
                try:
                        mystat = os.stat(self.getpath(mysplit[0]))[stat.ST_MTIME]
                except OSError:
                        mystat = 0
                if use_cache and mycp in self.cpcache:
                        cpc = self.cpcache[mycp]
                        if cpc[0] == mystat:
                                return cpc[1][:]
                cat_dir = self.getpath(mysplit[0])
                try:
                        dir_list = os.listdir(cat_dir)
                except EnvironmentError, e:
                        if e.errno == PermissionDenied.errno:
                                raise PermissionDenied(cat_dir)
                        del e
                        dir_list = []

                returnme = []
                for x in dir_list:
                        if self._excluded_dirs.match(x) is not None:
                                continue
                        ps = pkgsplit(x)
                        if not ps:
                                self.invalidentry(os.path.join(self.getpath(mysplit[0]), x))
                                continue
                        if len(mysplit) > 1:
                                if ps[0] == mysplit[1]:
                                        returnme.append(mysplit[0]+"/"+x)
                self._cpv_sort_ascending(returnme)
                if use_cache:
                        self.cpcache[mycp] = [mystat, returnme[:]]
                elif mycp in self.cpcache:
                        del self.cpcache[mycp]
                return returnme

        def cpv_all(self, use_cache=1):
                """
                Set use_cache=0 to bypass the portage.cachedir() cache in cases
                when the accuracy of mtime staleness checks should not be trusted
                (generally this is only necessary in critical sections that
                involve merge or unmerge of packages).
                """
                returnme = []
                basepath = os.path.join(self.root, VDB_PATH) + os.path.sep

                if use_cache:
                        from portage import listdir
                else:
                        def listdir(p, **kwargs):
                                try:
                                        return [x for x in os.listdir(p) \
                                                if os.path.isdir(os.path.join(p, x))]
                                except EnvironmentError, e:
                                        if e.errno == PermissionDenied.errno:
                                                raise PermissionDenied(p)
                                        del e
                                        return []

                for x in listdir(basepath, EmptyOnError=1, ignorecvs=1, dirsonly=1):
                        if self._excluded_dirs.match(x) is not None:
                                continue
                        if not self._category_re.match(x):
                                continue
                        for y in listdir(basepath + x, EmptyOnError=1, dirsonly=1):
                                if self._excluded_dirs.match(y) is not None:
                                        continue
                                subpath = x + "/" + y
                                # -MERGING- should never be a cpv, nor should files.
                                try:
                                        if catpkgsplit(subpath) is None:
                                                self.invalidentry(os.path.join(self.root, subpath))
                                                continue
                                except InvalidData:
                                        self.invalidentry(os.path.join(self.root, subpath))
                                        continue
                                returnme.append(subpath)
                return returnme

        def cp_all(self, use_cache=1):
                mylist = self.cpv_all(use_cache=use_cache)
                d={}
                for y in mylist:
                        if y[0] == '*':
                                y = y[1:]
                        try:
                                mysplit = catpkgsplit(y)
                        except InvalidData:
                                self.invalidentry(self.getpath(y))
                                continue
                        if not mysplit:
                                self.invalidentry(self.getpath(y))
                                continue
                        d[mysplit[0]+"/"+mysplit[1]] = None
                return d.keys()

        def checkblockers(self, origdep):
                pass

        def _clear_cache(self):
                self.mtdircache.clear()
                self.matchcache.clear()
                self.cpcache.clear()
                self._aux_cache_obj = None

        def _add(self, pkg_dblink):
                self._clear_pkg_cache(pkg_dblink)

        def _remove(self, pkg_dblink):
                self._clear_pkg_cache(pkg_dblink)

        def _clear_pkg_cache(self, pkg_dblink):
                # Due to 1 second mtime granularity in <python-2.5, mtime checks
                # are not always sufficient to invalidate vardbapi caches. Therefore,
                # the caches need to be actively invalidated here.
                self.mtdircache.pop(pkg_dblink.cat, None)
                self.matchcache.pop(pkg_dblink.cat, None)
                self.cpcache.pop(pkg_dblink.mysplit[0], None)
                from portage import dircache
                dircache.pop(pkg_dblink.dbcatdir, None)

        def match(self, origdep, use_cache=1):
                "caching match function"
                mydep = dep_expand(
                        origdep, mydb=self, use_cache=use_cache, settings=self.settings)
                mykey = dep_getkey(mydep)
                mycat = catsplit(mykey)[0]
                if not use_cache:
                        if mycat in self.matchcache:
                                del self.mtdircache[mycat]
                                del self.matchcache[mycat]
                        return list(self._iter_match(mydep,
                                self.cp_list(mydep.cp, use_cache=use_cache)))
                try:
                        curmtime = os.stat(self.root+VDB_PATH+"/"+mycat).st_mtime
                except (IOError, OSError):
                        curmtime=0

                if mycat not in self.matchcache or \
                        self.mtdircache[mycat] != curmtime:
                        # clear cache entry
                        self.mtdircache[mycat] = curmtime
                        self.matchcache[mycat] = {}
                if mydep not in self.matchcache[mycat]:
                        mymatch = list(self._iter_match(mydep,
                                self.cp_list(mydep.cp, use_cache=use_cache)))
                        self.matchcache[mycat][mydep] = mymatch
                return self.matchcache[mycat][mydep][:]

        def findname(self, mycpv):
                return self.getpath(str(mycpv), filename=catsplit(mycpv)[1]+".ebuild")

        def flush_cache(self):
                """If the current user has permission and the internal aux_get cache has
                been updated, save it to disk and mark it unmodified.  This is called
                by emerge after it has loaded the full vdb for use in dependency
                calculations.  Currently, the cache is only written if the user has
                superuser privileges (since that's required to obtain a lock), but all
                users have read access and benefit from faster metadata lookups (as
                long as at least part of the cache is still valid)."""
                if self._aux_cache is not None and \
                        len(self._aux_cache["modified"]) >= self._aux_cache_threshold and \
                        secpass >= 2:
                        self._owners.populate() # index any unindexed contents
                        valid_nodes = set(self.cpv_all())
                        for cpv in self._aux_cache["packages"].keys():
                                if cpv not in valid_nodes:
                                        del self._aux_cache["packages"][cpv]
                        del self._aux_cache["modified"]
                        try:
                                f = atomic_ofstream(self._aux_cache_filename)
                                cPickle.dump(self._aux_cache, f, -1)
                                f.close()
                                apply_secpass_permissions(
                                        self._aux_cache_filename, gid=portage_gid, mode=0644)
                        except (IOError, OSError), e:
                                pass
                        self._aux_cache["modified"] = set()

        @property
        def _aux_cache(self):
                if self._aux_cache_obj is None:
                        self._aux_cache_init()
                return self._aux_cache_obj

        def _aux_cache_init(self):
                aux_cache = None
                try:
                        f = open(self._aux_cache_filename)
                        mypickle = cPickle.Unpickler(f)
                        mypickle.find_global = None
                        aux_cache = mypickle.load()
                        f.close()
                        del f
                except (IOError, OSError, EOFError, cPickle.UnpicklingError), e:
                        if isinstance(e, cPickle.UnpicklingError):
                                writemsg("!!! Error loading '%s': %s\n" % \
                                        (self._aux_cache_filename, str(e)), noiselevel=-1)
                        del e

                if not aux_cache or \
                        not isinstance(aux_cache, dict) or \
                        aux_cache.get("version") != self._aux_cache_version or \
                        not aux_cache.get("packages"):
                        aux_cache = {"version": self._aux_cache_version}
                        aux_cache["packages"] = {}

                owners = aux_cache.get("owners")
                if owners is not None:
                        if not isinstance(owners, dict):
                                owners = None
                        elif "version" not in owners:
                                owners = None
                        elif owners["version"] != self._owners_cache_version:
                                owners = None
                        elif "base_names" not in owners:
                                owners = None
                        elif not isinstance(owners["base_names"], dict):
                                owners = None

                if owners is None:
                        owners = {
                                "base_names" : {},
                                "version"    : self._owners_cache_version
                        }
                        aux_cache["owners"] = owners

                aux_cache["modified"] = set()
                self._aux_cache_obj = aux_cache

        def aux_get(self, mycpv, wants):
                """This automatically caches selected keys that are frequently needed
                by emerge for dependency calculations.  The cached metadata is
                considered valid if the mtime of the package directory has not changed
                since the data was cached.  The cache is stored in a pickled dict
                object with the following format:

                {version:"1", "packages":{cpv1:(mtime,{k1,v1, k2,v2, ...}), cpv2...}}

                If an error occurs while loading the cache pickle or the version is
                unrecognized, the cache will simple be recreated from scratch (it is
                completely disposable).
                """
                cache_these_wants = self._aux_cache_keys.intersection(wants)
                for x in wants:
                        if self._aux_cache_keys_re.match(x) is not None:
                                cache_these_wants.add(x)

                if not cache_these_wants:
                        return self._aux_get(mycpv, wants)

                cache_these = set(self._aux_cache_keys)
                cache_these.update(cache_these_wants)

                mydir = self.getpath(mycpv)
                mydir_stat = None
                try:
                        mydir_stat = os.stat(mydir)
                except OSError, e:
                        if e.errno != errno.ENOENT:
                                raise
                        raise KeyError(mycpv)
                mydir_mtime = long(mydir_stat.st_mtime)
                pkg_data = self._aux_cache["packages"].get(mycpv)
                pull_me = cache_these.union(wants)
                mydata = {"_mtime_" : mydir_mtime}
                cache_valid = False
                cache_incomplete = False
                cache_mtime = None
                metadata = None
                if pkg_data is not None:
                        if not isinstance(pkg_data, tuple) or len(pkg_data) != 2:
                                pkg_data = None
                        else:
                                cache_mtime, metadata = pkg_data
                                if not isinstance(cache_mtime, (long, int)) or \
                                        not isinstance(metadata, dict):
                                        pkg_data = None

                if pkg_data:
                        cache_mtime, metadata = pkg_data
                        cache_valid = cache_mtime == mydir_mtime
                if cache_valid:
                        mydata.update(metadata)
                        pull_me.difference_update(mydata)

                if pull_me:
                        # pull any needed data and cache it
                        aux_keys = list(pull_me)
                        for k, v in izip(aux_keys,
                                self._aux_get(mycpv, aux_keys, st=mydir_stat)):
                                mydata[k] = v
                        if not cache_valid or cache_these.difference(metadata):
                                cache_data = {}
                                if cache_valid and metadata:
                                        cache_data.update(metadata)
                                for aux_key in cache_these:
                                        cache_data[aux_key] = mydata[aux_key]
                                self._aux_cache["packages"][mycpv] = (mydir_mtime, cache_data)
                                self._aux_cache["modified"].add(mycpv)
                return [mydata[x] for x in wants]

        def _aux_get(self, mycpv, wants, st=None):
                mydir = self.getpath(mycpv)
                if st is None:
                        try:
                                st = os.stat(mydir)
                        except OSError, e:
                                if e.errno == errno.ENOENT:
                                        raise KeyError(mycpv)
                                elif e.errno == PermissionDenied.errno:
                                        raise PermissionDenied(mydir)
                                else:
                                        raise
                if not stat.S_ISDIR(st.st_mode):
                        raise KeyError(mycpv)
                results = []
                for x in wants:
                        if x == "_mtime_":
                                results.append(st.st_mtime)
                                continue
                        try:
                                myf = open(os.path.join(mydir, x), "r")
                                try:
                                        myd = myf.read()
                                finally:
                                        myf.close()
                                # Preserve \n for metadata that is known to
                                # contain multiple lines.
                                if self._aux_multi_line_re.match(x) is None:
                                        myd = " ".join(myd.split())
                        except IOError:
                                myd = ""
                        if x == "EAPI" and not myd:
                                results.append("0")
                        else:
                                results.append(myd)
                return results

        def aux_update(self, cpv, values):
                cat, pkg = catsplit(cpv)
                mylink = dblink(cat, pkg, self.root, self.settings,
                treetype="vartree", vartree=self.vartree)
                if not mylink.exists():
                        raise KeyError(cpv)
                for k, v in values.iteritems():
                        if v:
                                mylink.setfile(k, v)
                        else:
                                try:
                                        os.unlink(os.path.join(self.getpath(cpv), k))
                                except EnvironmentError:
                                        pass

        def counter_tick(self, myroot, mycpv=None):
                return self.counter_tick_core(myroot, incrementing=1, mycpv=mycpv)

        def get_counter_tick_core(self, myroot, mycpv=None):
                """
                Use this method to retrieve the counter instead
                of having to trust the value of a global counter
                file that can lead to invalid COUNTER
                generation. When cache is valid, the package COUNTER
                files are not read and we rely on the timestamp of
                the package directory to validate cache. The stat
                calls should only take a short time, so performance
                is sufficient without having to rely on a potentially
                corrupt global counter file.

                The global counter file located at
                $CACHE_PATH/counter serves to record the
                counter of the last installed package and
                it also corresponds to the total number of
                installation actions that have occurred in
                the history of this package database.
                """
                cp_list = self.cp_list
                max_counter = 0
                for cp in self.cp_all():
                        for cpv in cp_list(cp):
                                try:
                                        counter = int(self.aux_get(cpv, ["COUNTER"])[0])
                                except (KeyError, OverflowError, ValueError):
                                        continue
                                if counter > max_counter:
                                        max_counter = counter

                new_vdb = False
                counter = -1
                try:
                        cfile = open(self._counter_path, "r")
                except EnvironmentError, e:
                        new_vdb = not bool(self.cpv_all())
                        if not new_vdb:
                                writemsg("!!! Unable to read COUNTER file: '%s'\n" % \
                                        self._counter_path, noiselevel=-1)
                                writemsg("!!! %s\n" % str(e), noiselevel=-1)
                        del e
                else:
                        try:
                                try:
                                        counter = long(cfile.readline().strip())
                                finally:
                                        cfile.close()
                        except (OverflowError, ValueError), e:
                                writemsg("!!! COUNTER file is corrupt: '%s'\n" % \
                                        self._counter_path, noiselevel=-1)
                                writemsg("!!! %s\n" % str(e), noiselevel=-1)
                                del e

                # We must ensure that we return a counter
                # value that is at least as large as the
                # highest one from the installed packages,
                # since having a corrupt value that is too low
                # can trigger incorrect AUTOCLEAN behavior due
                # to newly installed packages having lower
                # COUNTERs than the previous version in the
                # same slot.
                if counter > max_counter:
                        max_counter = counter

                if counter < 0 and not new_vdb:
                        writemsg("!!! Initializing COUNTER to " + \
                                "value of %d\n" % max_counter, noiselevel=-1)

                return max_counter + 1

        def counter_tick_core(self, myroot, incrementing=1, mycpv=None):
                "This method will grab the next COUNTER value and record it back to the global file.  Returns new counter value."
                counter = self.get_counter_tick_core(myroot, mycpv=mycpv) - 1
                if incrementing:
                        #increment counter
                        counter += 1
                        # update new global counter file
                        write_atomic(self._counter_path, str(counter))
                return counter

        def _dblink(self, cpv):
                category, pf = catsplit(cpv)
                return dblink(category, pf, self.root,
                        self.settings, vartree=self.vartree, treetype="vartree")

        def removeFromContents(self, pkg, paths, relative_paths=True):
                """
                @param pkg: cpv for an installed package
                @type pkg: string
                @param paths: paths of files to remove from contents
                @type paths: iterable
                """
                if not hasattr(pkg, "getcontents"):
                        pkg = self._dblink(pkg)
                root = self.root
                root_len = len(root) - 1
                new_contents = pkg.getcontents().copy()
                removed = 0

                for filename in paths:
                        filename = normalize_path(filename)
                        if relative_paths:
                                relative_filename = filename
                        else:
                                relative_filename = filename[root_len:]
                        contents_key = pkg._match_contents(relative_filename, root)
                        if contents_key:
                                del new_contents[contents_key]
                                removed += 1

                if removed:
                        f = atomic_ofstream(os.path.join(pkg.dbdir, "CONTENTS"))
                        write_contents(new_contents, root, f)
                        f.close()

        class _owners_cache(object):
                """
                This class maintains an hash table that serves to index package
                contents by mapping the basename of file to a list of possible
                packages that own it. This is used to optimize owner lookups
                by narrowing the search down to a smaller number of packages.
                """
                try:
                        from hashlib import md5 as _new_hash
                except ImportError:
                        from md5 import new as _new_hash

                _hash_bits = 16
                _hex_chars = _hash_bits / 4

                def __init__(self, vardb):
                        self._vardb = vardb

                def add(self, cpv):
                        root_len = len(self._vardb.root)
                        contents = self._vardb._dblink(cpv).getcontents()
                        pkg_hash = self._hash_pkg(cpv)
                        if not contents:
                                # Empty path is a code used to represent empty contents.
                                self._add_path("", pkg_hash)
                        for x in contents:
                                self._add_path(x[root_len:], pkg_hash)
                        self._vardb._aux_cache["modified"].add(cpv)

                def _add_path(self, path, pkg_hash):
                        """
                        Empty path is a code that represents empty contents.
                        """
                        if path:
                                name = os.path.basename(path.rstrip(os.path.sep))
                                if not name:
                                        return
                        else:
                                name = path
                        name_hash = self._hash_str(name)
                        base_names = self._vardb._aux_cache["owners"]["base_names"]
                        pkgs = base_names.get(name_hash)
                        if pkgs is None:
                                pkgs = {}
                                base_names[name_hash] = pkgs
                        pkgs[pkg_hash] = None

                def _hash_str(self, s):
                        h = self._new_hash()
                        h.update(s)
                        h = h.hexdigest()
                        h = h[-self._hex_chars:]
                        h = int(h, 16)
                        return h

                def _hash_pkg(self, cpv):
                        counter, mtime = self._vardb.aux_get(
                                cpv, ["COUNTER", "_mtime_"])
                        try:
                                counter = int(counter)
                        except ValueError:
                                counter = 0
                        return (cpv, counter, mtime)

        class _owners_db(object):

                def __init__(self, vardb):
                        self._vardb = vardb

                def populate(self):
                        self._populate()

                def _populate(self):
                        owners_cache = vardbapi._owners_cache(self._vardb)
                        cached_hashes = set()
                        base_names = self._vardb._aux_cache["owners"]["base_names"]

                        # Take inventory of all cached package hashes.
                        for name, hash_values in base_names.items():
                                if not isinstance(hash_values, dict):
                                        del base_names[name]
                                        continue
                                cached_hashes.update(hash_values)

                        # Create sets of valid package hashes and uncached packages.
                        uncached_pkgs = set()
                        hash_pkg = owners_cache._hash_pkg
                        valid_pkg_hashes = set()
                        for cpv in self._vardb.cpv_all():
                                hash_value = hash_pkg(cpv)
                                valid_pkg_hashes.add(hash_value)
                                if hash_value not in cached_hashes:
                                        uncached_pkgs.add(cpv)

                        # Cache any missing packages.
                        for cpv in uncached_pkgs:
                                owners_cache.add(cpv)

                        # Delete any stale cache.
                        stale_hashes = cached_hashes.difference(valid_pkg_hashes)
                        if stale_hashes:
                                for base_name_hash, bucket in base_names.items():
                                        for hash_value in stale_hashes.intersection(bucket):
                                                del bucket[hash_value]
                                        if not bucket:
                                                del base_names[base_name_hash]

                        return owners_cache

                def get_owners(self, path_iter):
                        """
                        @return the owners as a dblink -> set(files) mapping.
                        """
                        owners = {}
                        for owner, f in self.iter_owners(path_iter):
                                owned_files = owners.get(owner)
                                if owned_files is None:
                                        owned_files = set()
                                        owners[owner] = owned_files
                                owned_files.add(f)
                        return owners

                def getFileOwnerMap(self, path_iter):
                        owners = self.get_owners(path_iter)
                        file_owners = {}
                        for pkg_dblink, files in owners.iteritems():
                                for f in files:
                                        owner_set = file_owners.get(f)
                                        if owner_set is None:
                                                owner_set = set()
                                                file_owners[f] = owner_set
                                        owner_set.add(pkg_dblink)
                        return file_owners

                def iter_owners(self, path_iter):
                        """
                        Iterate over tuples of (dblink, path). In order to avoid
                        consuming too many resources for too much time, resources
                        are only allocated for the duration of a given iter_owners()
                        call. Therefore, to maximize reuse of resources when searching
                        for multiple files, it's best to search for them all in a single
                        call.
                        """

                        owners_cache = self._populate()

                        vardb = self._vardb
                        root = vardb.root
                        hash_pkg = owners_cache._hash_pkg
                        hash_str = owners_cache._hash_str
                        base_names = self._vardb._aux_cache["owners"]["base_names"]

                        dblink_cache = {}

                        def dblink(cpv):
                                x = dblink_cache.get(cpv)
                                if x is None:
                                        x = self._vardb._dblink(cpv)
                                        dblink_cache[cpv] = x
                                return x

                        for path in path_iter:
                                name = os.path.basename(path.rstrip(os.path.sep))
                                if not name:
                                        continue

                                name_hash = hash_str(name)
                                pkgs = base_names.get(name_hash)
                                if pkgs is not None:
                                        for hash_value in pkgs:
                                                if not isinstance(hash_value, tuple) or \
                                                        len(hash_value) != 3:
                                                        continue
                                                cpv, counter, mtime = hash_value
                                                if not isinstance(cpv, basestring):
                                                        continue
                                                try:
                                                        current_hash = hash_pkg(cpv)
                                                except KeyError:
                                                        continue

                                                if current_hash != hash_value:
                                                        continue
                                                if dblink(cpv).isowner(path, root):
                                                        yield dblink(cpv), path

class vartree(object):
        "this tree will scan a var/db/pkg database located at root (passed to init)"
        def __init__(self, root="/", virtual=None, clone=None, categories=None,
                settings=None):
                if clone:
                        writemsg("vartree.__init__(): deprecated " + \
                                "use of clone parameter\n", noiselevel=-1)
                        self.root = clone.root[:]
                        self.dbapi = copy.deepcopy(clone.dbapi)
                        self.populated = 1
                        from portage import config
                        self.settings = config(clone=clone.settings)
                else:
                        self.root = root[:]
                        if settings is None:
                                from portage import settings
                        self.settings = settings # for key_expand calls
                        if categories is None:
                                categories = settings.categories
                        self.dbapi = vardbapi(self.root, categories=categories,
                                settings=settings, vartree=self)
                        self.populated = 1

        def getpath(self, mykey, filename=None):
                return self.dbapi.getpath(mykey, filename=filename)

        def zap(self, mycpv):
                return

        def inject(self, mycpv):
                return

        def get_provide(self, mycpv):
                myprovides = []
                mylines = None
                try:
                        mylines, myuse = self.dbapi.aux_get(mycpv, ["PROVIDE", "USE"])
                        if mylines:
                                myuse = myuse.split()
                                mylines = flatten(use_reduce(paren_reduce(mylines), uselist=myuse))
                                for myprovide in mylines:
                                        mys = catpkgsplit(myprovide)
                                        if not mys:
                                                mys = myprovide.split("/")
                                        myprovides += [mys[0] + "/" + mys[1]]
                        return myprovides
                except SystemExit, e:
                        raise
                except Exception, e:
                        mydir = os.path.join(self.root, VDB_PATH, mycpv)
                        writemsg("\nParse Error reading PROVIDE and USE in '%s'\n" % mydir,
                                noiselevel=-1)
                        if mylines:
                                writemsg("Possibly Invalid: '%s'\n" % str(mylines),
                                        noiselevel=-1)
                        writemsg("Exception: %s\n\n" % str(e), noiselevel=-1)
                        return []

        def get_all_provides(self):
                myprovides = {}
                for node in self.getallcpv():
                        for mykey in self.get_provide(node):
                                if mykey in myprovides:
                                        myprovides[mykey] += [node]
                                else:
                                        myprovides[mykey] = [node]
                return myprovides

        def dep_bestmatch(self, mydep, use_cache=1):
                "compatibility method -- all matches, not just visible ones"
                #mymatch=best(match(dep_expand(mydep,self.dbapi),self.dbapi))
                mymatch = best(self.dbapi.match(
                        dep_expand(mydep, mydb=self.dbapi, settings=self.settings),
                        use_cache=use_cache))
                if mymatch is None:
                        return ""
                else:
                        return mymatch

        def dep_match(self, mydep, use_cache=1):
                "compatibility method -- we want to see all matches, not just visible ones"
                #mymatch = match(mydep,self.dbapi)
                mymatch = self.dbapi.match(mydep, use_cache=use_cache)
                if mymatch is None:
                        return []
                else:
                        return mymatch

        def exists_specific(self, cpv):
                return self.dbapi.cpv_exists(cpv)

        def getallcpv(self):
                """temporary function, probably to be renamed --- Gets a list of all
                category/package-versions installed on the system."""
                return self.dbapi.cpv_all()

        def getallnodes(self):
                """new behavior: these are all *unmasked* nodes.  There may or may not be available
                masked package for nodes in this nodes list."""
                return self.dbapi.cp_all()

        def exists_specific_cat(self, cpv, use_cache=1):
                cpv = key_expand(cpv, mydb=self.dbapi, use_cache=use_cache,
                        settings=self.settings)
                a = catpkgsplit(cpv)
                if not a:
                        return 0
                mylist = listdir(self.getpath(a[0]), EmptyOnError=1)
                for x in mylist:
                        b = pkgsplit(x)
                        if not b:
                                self.dbapi.invalidentry(self.getpath(a[0], filename=x))
                                continue
                        if a[1] == b[0]:
                                return 1
                return 0

        def getebuildpath(self, fullpackage):
                cat, package = catsplit(fullpackage)
                return self.getpath(fullpackage, filename=package+".ebuild")

        def getnode(self, mykey, use_cache=1):
                mykey = key_expand(mykey, mydb=self.dbapi, use_cache=use_cache,
                        settings=self.settings)
                if not mykey:
                        return []
                mysplit = catsplit(mykey)
                mydirlist = listdir(self.getpath(mysplit[0]),EmptyOnError=1)
                returnme = []
                for x in mydirlist:
                        mypsplit = pkgsplit(x)
                        if not mypsplit:
                                self.dbapi.invalidentry(self.getpath(mysplit[0], filename=x))
                                continue
                        if mypsplit[0] == mysplit[1]:
                                appendme = [mysplit[0]+"/"+x, [mysplit[0], mypsplit[0], mypsplit[1], mypsplit[2]]]
                                returnme.append(appendme)
                return returnme


        def getslot(self, mycatpkg):
                "Get a slot for a catpkg; assume it exists."
                try:
                        return self.dbapi.aux_get(mycatpkg, ["SLOT"])[0]
                except KeyError:
                        return ""

        def hasnode(self, mykey, use_cache):
                """Does the particular node (cat/pkg key) exist?"""
                mykey = key_expand(mykey, mydb=self.dbapi, use_cache=use_cache,
                        settings=self.settings)
                mysplit = catsplit(mykey)
                mydirlist = listdir(self.getpath(mysplit[0]), EmptyOnError=1)
                for x in mydirlist:
                        mypsplit = pkgsplit(x)
                        if not mypsplit:
                                self.dbapi.invalidentry(self.getpath(mysplit[0], filename=x))
                                continue
                        if mypsplit[0] == mysplit[1]:
                                return 1
                return 0

        def populate(self):
                self.populated=1

class dblink(object):
        """
        This class provides an interface to the installed package database
        At present this is implemented as a text backend in /var/db/pkg.
        """

        import re
        _normalize_needed = re.compile(r'.*//.*|^[^/]|.+/$|(^|.*/)\.\.?(/.*|$)')
        _contents_split_counts = {
                "dev": 2,
                "dir": 2,
                "fif": 2,
                "obj": 4,
                "sym": 5
        }

        # When looping over files for merge/unmerge, temporarily yield to the
        # scheduler each time this many files are processed.
        _file_merge_yield_interval = 20

        def __init__(self, cat, pkg, myroot, mysettings, treetype=None,
                vartree=None, blockers=None, scheduler=None):
                """
                Creates a DBlink object for a given CPV.
                The given CPV may not be present in the database already.
                
                @param cat: Category
                @type cat: String
                @param pkg: Package (PV)
                @type pkg: String
                @param myroot: Typically ${ROOT}
                @type myroot: String (Path)
                @param mysettings: Typically portage.config
                @type mysettings: An instance of portage.config
                @param treetype: one of ['porttree','bintree','vartree']
                @type treetype: String
                @param vartree: an instance of vartree corresponding to myroot.
                @type vartree: vartree
                """
                
                self.cat = cat
                self.pkg = pkg
                self.mycpv = self.cat + "/" + self.pkg
                self.mysplit = list(catpkgsplit(self.mycpv)[1:])
                self.mysplit[0] = "%s/%s" % (self.cat, self.mysplit[0])
                self.treetype = treetype
                if vartree is None:
                        from portage import db
                        vartree = db[myroot]["vartree"]
                self.vartree = vartree
                self._blockers = blockers
                self._scheduler = scheduler

                self.dbroot = normalize_path(os.path.join(myroot, VDB_PATH))
                self.dbcatdir = self.dbroot+"/"+cat
                self.dbpkgdir = self.dbcatdir+"/"+pkg
                self.dbtmpdir = self.dbcatdir+"/-MERGING-"+pkg
                self.dbdir = self.dbpkgdir

                self._lock_vdb = None

                self.settings = mysettings
                if self.settings == 1:
                        raise ValueError

                self.myroot=myroot
                protect_obj = ConfigProtect(myroot,
                        mysettings.get("CONFIG_PROTECT","").split(),
                        mysettings.get("CONFIG_PROTECT_MASK","").split())
                self.updateprotect = protect_obj.updateprotect
                self.isprotected = protect_obj.isprotected
                self._installed_instance = None
                self.contentscache = None
                self._contents_inodes = None
                self._contents_basenames = None

        def lockdb(self):
                if self._lock_vdb:
                        raise AssertionError("Lock already held.")
                # At least the parent needs to exist for the lock file.
                ensure_dirs(self.dbroot)
                self._lock_vdb = lockdir(self.dbroot)

        def unlockdb(self):
                if self._lock_vdb:
                        unlockdir(self._lock_vdb)
                        self._lock_vdb = None

        def getpath(self):
                "return path to location of db information (for >>> informational display)"
                return self.dbdir

        def exists(self):
                "does the db entry exist?  boolean."
                return os.path.exists(self.dbdir)

        def delete(self):
                """
                Remove this entry from the database
                """
                if not os.path.exists(self.dbdir):
                        return

                # Check validity of self.dbdir before attempting to remove it.
                if not self.dbdir.startswith(self.dbroot):
                        writemsg("portage.dblink.delete(): invalid dbdir: %s\n" % \
                                self.dbdir, noiselevel=-1)
                        return
                import shutil
                shutil.rmtree(self.dbdir)
                self.vartree.dbapi._remove(self)

        def clearcontents(self):
                """
                For a given db entry (self), erase the CONTENTS values.
                """
                if os.path.exists(self.dbdir+"/CONTENTS"):
                        os.unlink(self.dbdir+"/CONTENTS")

        def _clear_contents_cache(self):
                self.contentscache = None
                self._contents_inodes = None
                self._contents_basenames = None

        def getcontents(self):
                """
                Get the installed files of a given package (aka what that package installed)
                """
                contents_file = os.path.join(self.dbdir, "CONTENTS")
                if self.contentscache is not None:
                        return self.contentscache
                pkgfiles = {}
                try:
                        myc = open(contents_file,"r")
                except EnvironmentError, e:
                        if e.errno != errno.ENOENT:
                                raise
                        del e
                        self.contentscache = pkgfiles
                        return pkgfiles
                mylines = myc.readlines()
                myc.close()
                null_byte = "\0"
                normalize_needed = self._normalize_needed
                contents_split_counts = self._contents_split_counts
                myroot = self.myroot
                if myroot == os.path.sep:
                        myroot = None
                pos = 0
                errors = []
                for pos, line in enumerate(mylines):
                        if null_byte in line:
                                # Null bytes are a common indication of corruption.
                                errors.append((pos + 1, "Null byte found in CONTENTS entry"))
                                continue
                        line = line.rstrip("\n")
                        # Split on " " so that even file paths that
                        # end with spaces can be handled.
                        mydat = line.split(" ")
                        entry_type = mydat[0] # empty string if line is empty
                        correct_split_count = contents_split_counts.get(entry_type)
                        if correct_split_count and len(mydat) > correct_split_count:
                                # Apparently file paths contain spaces, so reassemble
                                # the split have the correct_split_count.
                                newsplit = [entry_type]
                                spaces_total = len(mydat) - correct_split_count
                                if entry_type == "sym":
                                        try:
                                                splitter = mydat.index("->", 2, len(mydat) - 2)
                                        except ValueError:
                                                errors.append((pos + 1, "Unrecognized CONTENTS entry"))
                                                continue
                                        spaces_in_path = splitter - 2
                                        spaces_in_target = spaces_total - spaces_in_path
                                        newsplit.append(" ".join(mydat[1:splitter]))
                                        newsplit.append("->")
                                        target_end = splitter + spaces_in_target + 2
                                        newsplit.append(" ".join(mydat[splitter + 1:target_end]))
                                        newsplit.extend(mydat[target_end:])
                                else:
                                        path_end = spaces_total + 2
                                        newsplit.append(" ".join(mydat[1:path_end]))
                                        newsplit.extend(mydat[path_end:])
                                mydat = newsplit

                        # we do this so we can remove from non-root filesystems
                        # (use the ROOT var to allow maintenance on other partitions)
                        try:
                                if normalize_needed.match(mydat[1]):
                                        mydat[1] = normalize_path(mydat[1])
                                        if not mydat[1].startswith(os.path.sep):
                                                mydat[1] = os.path.sep + mydat[1]
                                if myroot:
                                        mydat[1] = os.path.join(myroot, mydat[1].lstrip(os.path.sep))
                                if mydat[0] == "obj":
                                        #format: type, mtime, md5sum
                                        pkgfiles[mydat[1]] = [mydat[0], mydat[3], mydat[2]]
                                elif mydat[0] == "dir":
                                        #format: type
                                        pkgfiles[mydat[1]] = [mydat[0]]
                                elif mydat[0] == "sym":
                                        #format: type, mtime, dest
                                        pkgfiles[mydat[1]] = [mydat[0], mydat[4], mydat[3]]
                                elif mydat[0] == "dev":
                                        #format: type
                                        pkgfiles[mydat[1]] = [mydat[0]]
                                elif mydat[0]=="fif":
                                        #format: type
                                        pkgfiles[mydat[1]] = [mydat[0]]
                                else:
                                        errors.append((pos + 1, "Unrecognized CONTENTS entry"))
                        except (KeyError, IndexError):
                                errors.append((pos + 1, "Unrecognized CONTENTS entry"))
                if errors:
                        writemsg("!!! Parse error in '%s'\n" % contents_file, noiselevel=-1)
                        for pos, e in errors:
                                writemsg("!!!   line %d: %s\n" % (pos, e), noiselevel=-1)
                self.contentscache = pkgfiles
                return pkgfiles

        def unmerge(self, pkgfiles=None, trimworld=1, cleanup=1,
                ldpath_mtimes=None, others_in_slot=None):
                """
                Calls prerm
                Unmerges a given package (CPV)
                calls postrm
                calls cleanrm
                calls env_update
                
                @param pkgfiles: files to unmerge (generally self.getcontents() )
                @type pkgfiles: Dictionary
                @param trimworld: Remove CPV from world file if True, not if False
                @type trimworld: Boolean
                @param cleanup: cleanup to pass to doebuild (see doebuild)
                @type cleanup: Boolean
                @param ldpath_mtimes: mtimes to pass to env_update (see env_update)
                @type ldpath_mtimes: Dictionary
                @param others_in_slot: all dblink instances in this slot, excluding self
                @type others_in_slot: list
                @rtype: Integer
                @returns:
                1. os.EX_OK if everything went well.
                2. return code of the failed phase (for prerm, postrm, cleanrm)
                
                Notes:
                The caller must ensure that lockdb() and unlockdb() are called
                before and after this method.
                """
                showMessage = self._display_merge
                if self.vartree.dbapi._categories is not None:
                        self.vartree.dbapi._categories = None
                # When others_in_slot is supplied, the security check has already been
                # done for this slot, so it shouldn't be repeated until the next
                # replacement or unmerge operation.
                if others_in_slot is None:
                        slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0]
                        slot_matches = self.vartree.dbapi.match(
                                "%s:%s" % (dep_getkey(self.mycpv), slot))
                        others_in_slot = []
                        for cur_cpv in slot_matches:
                                if cur_cpv == self.mycpv:
                                        continue
                                others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1],
                                        self.vartree.root, self.settings, vartree=self.vartree,
                                        treetype="vartree"))

                        retval = self._security_check([self] + others_in_slot)
                        if retval:
                                return retval

                contents = self.getcontents()
                # Now, don't assume that the name of the ebuild is the same as the
                # name of the dir; the package may have been moved.
                myebuildpath = None
                ebuild_phase = "prerm"
                log_path = None
                mystuff = os.listdir(self.dbdir)
                for x in mystuff:
                        if x.endswith(".ebuild"):
                                myebuildpath = os.path.join(self.dbdir, self.pkg + ".ebuild")
                                if x[:-7] != self.pkg:
                                        # Clean up after vardbapi.move_ent() breakage in
                                        # portage versions before 2.1.2
                                        os.rename(os.path.join(self.dbdir, x), myebuildpath)
                                        write_atomic(os.path.join(self.dbdir, "PF"), self.pkg+"\n")
                                break

                self.settings.setcpv(self.mycpv, mydb=self.vartree.dbapi)
                if myebuildpath:
                        try:
                                doebuild_environment(myebuildpath, "prerm", self.myroot,
                                        self.settings, 0, 0, self.vartree.dbapi)
                        except UnsupportedAPIException, e:
                                # Sometimes this happens due to corruption of the EAPI file.
                                writemsg("!!! FAILED prerm: %s\n" % \
                                        os.path.join(self.dbdir, "EAPI"), noiselevel=-1)
                                writemsg("%s\n" % str(e), noiselevel=-1)
                                myebuildpath = None
                        else:
                                catdir = os.path.dirname(self.settings["PORTAGE_BUILDDIR"])
                                ensure_dirs(os.path.dirname(catdir), uid=portage_uid,
                                        gid=portage_gid, mode=070, mask=0)

                builddir_lock = None
                catdir_lock = None
                scheduler = self._scheduler
                retval = -1
                try:
                        if myebuildpath:
                                catdir_lock = lockdir(catdir)
                                ensure_dirs(catdir,
                                        uid=portage_uid, gid=portage_gid,
                                        mode=070, mask=0)
                                builddir_lock = lockdir(
                                        self.settings["PORTAGE_BUILDDIR"])
                                try:
                                        unlockdir(catdir_lock)
                                finally:
                                        catdir_lock = None

                                prepare_build_dirs(self.myroot, self.settings, 1)
                                log_path = self.settings.get("PORTAGE_LOG_FILE")

                                if scheduler is None:
                                        retval = doebuild(myebuildpath, ebuild_phase, self.myroot,
                                                self.settings, cleanup=cleanup, use_cache=0,
                                                mydbapi=self.vartree.dbapi, tree=self.treetype,
                                                vartree=self.vartree)
                                else:
                                        retval = scheduler.dblinkEbuildPhase(
                                                self, self.vartree.dbapi, myebuildpath, ebuild_phase)

                                # XXX: Decide how to handle failures here.
                                if retval != os.EX_OK:
                                        writemsg("!!! FAILED prerm: %s\n" % retval, noiselevel=-1)

                        self._unmerge_pkgfiles(pkgfiles, others_in_slot)
                        
                        # Remove the registration of preserved libs for this pkg instance
                        plib_registry = self.vartree.dbapi.plib_registry
                        plib_registry.unregister(self.mycpv, self.settings["SLOT"],
                                self.vartree.dbapi.cpv_counter(self.mycpv))

                        if myebuildpath:
                                ebuild_phase = "postrm"
                                if scheduler is None:
                                        retval = doebuild(myebuildpath, ebuild_phase, self.myroot,
                                                self.settings, use_cache=0, tree=self.treetype,
                                                mydbapi=self.vartree.dbapi, vartree=self.vartree)
                                else:
                                        retval = scheduler.dblinkEbuildPhase(
                                                self, self.vartree.dbapi, myebuildpath, ebuild_phase)

                                # XXX: Decide how to handle failures here.
                                if retval != os.EX_OK:
                                        writemsg("!!! FAILED postrm: %s\n" % retval, noiselevel=-1)

                        # regenerate reverse NEEDED map
                        self.vartree.dbapi.linkmap.rebuild()

                        # remove preserved libraries that don't have any consumers left
                        # FIXME: this code is quite ugly and can likely be optimized in several ways
                        plib_dict = plib_registry.getPreservedLibs()
                        for cpv in plib_dict:
                                plib_dict[cpv].sort()
                                # for the loop below to work correctly, we need all
                                # symlinks to come before the actual files, such that
                                # the recorded symlinks (sonames) will be resolved into
                                # their real target before the object is found not to be
                                # in the reverse NEEDED map
                                def symlink_compare(x, y):
                                        if os.path.islink(x):
                                                if os.path.islink(y):
                                                        return 0
                                                else:
                                                        return -1
                                        elif os.path.islink(y):
                                                return 1
                                        else:
                                                return 0

                                plib_dict[cpv].sort(symlink_compare)
                                for f in plib_dict[cpv]:
                                        if not os.path.exists(f):
                                                continue
                                        unlink_list = []
                                        consumers = self.vartree.dbapi.linkmap.findConsumers(f)
                                        if not consumers:
                                                unlink_list.append(f)
                                        else:
                                                keep=False
                                                for c in consumers:
                                                        if c not in self.getcontents():
                                                                keep=True
                                                                break
                                                if not keep:
                                                        unlink_list.append(f)
                                        for obj in unlink_list:
                                                try:
                                                        if os.path.islink(obj):
                                                                obj_type = "sym"
                                                        else:
                                                                obj_type = "obj"
                                                        os.unlink(obj)
                                                        showMessage("<<< !needed   %s %s\n" % (obj_type, obj))
                                                except OSError, e:
                                                        if e.errno == errno.ENOENT:
                                                                pass
                                                        else:
                                                                raise e
                        plib_registry.pruneNonExisting()
                                                
                finally:
                        if builddir_lock:
                                try:
                                        if myebuildpath:
                                                if retval != os.EX_OK:
                                                        msg_lines = []
                                                        msg = ("The '%s' " % ebuild_phase) + \
                                                        ("phase of the '%s' package " % self.mycpv) + \
                                                        ("has failed with exit value %s." % retval)
                                                        from textwrap import wrap
                                                        msg_lines.extend(wrap(msg, 72))
                                                        msg_lines.append("")

                                                        ebuild_name = os.path.basename(myebuildpath)
                                                        ebuild_dir = os.path.dirname(myebuildpath)
                                                        msg = "The problem occurred while executing " + \
                                                        ("the ebuild file named '%s' " % ebuild_name) + \
                                                        ("located in the '%s' directory. " \
                                                        % ebuild_dir) + \
                                                        "If necessary, manually remove " + \
                                                        "the environment.bz2 file and/or the " + \
                                                        "ebuild file located in that directory."
                                                        msg_lines.extend(wrap(msg, 72))
                                                        msg_lines.append("")

                                                        msg = "Removal " + \
                                                        "of the environment.bz2 file is " + \
                                                        "preferred since it may allow the " + \
                                                        "removal phases to execute successfully. " + \
                                                        "The ebuild will be " + \
                                                        "sourced and the eclasses " + \
                                                        "from the current portage tree will be used " + \
                                                        "when necessary. Removal of " + \
                                                        "the ebuild file will cause the " + \
                                                        "pkg_prerm() and pkg_postrm() removal " + \
                                                        "phases to be skipped entirely."
                                                        msg_lines.extend(wrap(msg, 72))

                                                        self._eerror(ebuild_phase, msg_lines)

                                                # process logs created during pre/postrm
                                                elog_process(self.mycpv, self.settings, phasefilter=filter_unmergephases)
                                                if retval == os.EX_OK:
                                                        doebuild(myebuildpath, "cleanrm", self.myroot,
                                                                self.settings, tree="vartree",
                                                                mydbapi=self.vartree.dbapi,
                                                                vartree=self.vartree)
                                finally:
                                        unlockdir(builddir_lock)
                        try:
                                if myebuildpath and not catdir_lock:
                                        # Lock catdir for removal if empty.
                                        catdir_lock = lockdir(catdir)
                        finally:
                                if catdir_lock:
                                        try:
                                                os.rmdir(catdir)
                                        except OSError, e:
                                                if e.errno not in (errno.ENOENT,
                                                        errno.ENOTEMPTY, errno.EEXIST):
                                                        raise
                                                del e
                                        unlockdir(catdir_lock)

                if log_path is not None and os.path.exists(log_path):
                        # Restore this since it gets lost somewhere above and it
                        # needs to be set for _display_merge() to be able to log.
                        # Note that the log isn't necessarily supposed to exist
                        # since if PORT_LOGDIR is unset then it's a temp file
                        # so it gets cleaned above.
                        self.settings["PORTAGE_LOG_FILE"] = log_path
                else:
                        self.settings.pop("PORTAGE_LOG_FILE", None)

                env_update(target_root=self.myroot, prev_mtimes=ldpath_mtimes,
                        contents=contents, env=self.settings.environ(),
                        writemsg_level=self._display_merge)
                return os.EX_OK

        def _display_merge(self, msg, level=0, noiselevel=0):
                if self._scheduler is not None:
                        self._scheduler.dblinkDisplayMerge(self, msg,
                                level=level, noiselevel=noiselevel)
                        return
                writemsg_level(msg, level=level, noiselevel=noiselevel)

        def _unmerge_pkgfiles(self, pkgfiles, others_in_slot):
                """
                
                Unmerges the contents of a package from the liveFS
                Removes the VDB entry for self
                
                @param pkgfiles: typically self.getcontents()
                @type pkgfiles: Dictionary { filename: [ 'type', '?', 'md5sum' ] }
                @param others_in_slot: all dblink instances in this slot, excluding self
                @type others_in_slot: list
                @rtype: None
                """

                showMessage = self._display_merge
                scheduler = self._scheduler

                if not pkgfiles:
                        showMessage("No package files given... Grabbing a set.\n")
                        pkgfiles = self.getcontents()

                if others_in_slot is None:
                        others_in_slot = []
                        slot = self.vartree.dbapi.aux_get(self.mycpv, ["SLOT"])[0]
                        slot_matches = self.vartree.dbapi.match(
                                "%s:%s" % (dep_getkey(self.mycpv), slot))
                        for cur_cpv in slot_matches:
                                if cur_cpv == self.mycpv:
                                        continue
                                others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1],
                                        self.vartree.root, self.settings,
                                        vartree=self.vartree, treetype="vartree"))

                dest_root = normalize_path(self.vartree.root).rstrip(os.path.sep) + \
                        os.path.sep
                dest_root_len = len(dest_root) - 1

                conf_mem_file = os.path.join(dest_root, CONFIG_MEMORY_FILE)
                cfgfiledict = grabdict(conf_mem_file)
                stale_confmem = []

                unmerge_orphans = "unmerge-orphans" in self.settings.features

                if pkgfiles:
                        self.updateprotect()
                        mykeys = pkgfiles.keys()
                        mykeys.sort()
                        mykeys.reverse()

                        #process symlinks second-to-last, directories last.
                        mydirs = []
                        ignored_unlink_errnos = (
                                errno.EBUSY, errno.ENOENT,
                                errno.ENOTDIR, errno.EISDIR)
                        ignored_rmdir_errnos = (
                                errno.EEXIST, errno.ENOTEMPTY,
                                errno.EBUSY, errno.ENOENT,
                                errno.ENOTDIR, errno.EISDIR)
                        modprotect = os.path.join(self.vartree.root, "lib/modules/")

                        def unlink(file_name, lstatobj):
                                if bsd_chflags:
                                        if lstatobj.st_flags != 0:
                                                bsd_chflags.lchflags(file_name, 0)
                                        parent_name = os.path.dirname(file_name)
                                        # Use normal stat/chflags for the parent since we want to
                                        # follow any symlinks to the real parent directory.
                                        pflags = os.stat(parent_name).st_flags
                                        if pflags != 0:
                                                bsd_chflags.chflags(parent_name, 0)
                                try:
                                        if not stat.S_ISLNK(lstatobj.st_mode):
                                                # Remove permissions to ensure that any hardlinks to
                                                # suid/sgid files are rendered harmless.
                                                os.chmod(file_name, 0)
                                        os.unlink(file_name)
                                finally:
                                        if bsd_chflags and pflags != 0:
                                                # Restore the parent flags we saved before unlinking
                                                bsd_chflags.chflags(parent_name, pflags)

                        def show_unmerge(zing, desc, file_type, file_name):
                                        showMessage("%s %s %s %s\n" % \
                                                (zing, desc.ljust(8), file_type, file_name))
                        for i, objkey in enumerate(mykeys):

                                if scheduler is not None and \
                                        0 == i % self._file_merge_yield_interval:
                                        scheduler.scheduleYield()

                                obj = normalize_path(objkey)
                                file_data = pkgfiles[objkey]
                                file_type = file_data[0]
                                statobj = None
                                try:
                                        statobj = os.stat(obj)
                                except OSError:
                                        pass
                                lstatobj = None
                                try:
                                        lstatobj = os.lstat(obj)
                                except (OSError, AttributeError):
                                        pass
                                islink = lstatobj is not None and stat.S_ISLNK(lstatobj.st_mode)
                                if lstatobj is None:
                                                show_unmerge("---", "!found", file_type, obj)
                                                continue
                                if obj.startswith(dest_root):
                                        relative_path = obj[dest_root_len:]
                                        is_owned = False
                                        for dblnk in others_in_slot:
                                                if dblnk.isowner(relative_path, dest_root):
                                                        is_owned = True
                                                        break
                                        if is_owned:
                                                # A new instance of this package claims the file, so
                                                # don't unmerge it.
                                                show_unmerge("---", "replaced", file_type, obj)
                                                continue
                                        elif relative_path in cfgfiledict:
                                                stale_confmem.append(relative_path)
                                # next line includes a tweak to protect modules from being unmerged,
                                # but we don't protect modules from being overwritten if they are
                                # upgraded. We effectively only want one half of the config protection
                                # functionality for /lib/modules. For portage-ng both capabilities
                                # should be able to be independently specified.
                                if obj.startswith(modprotect):
                                        show_unmerge("---", "cfgpro", file_type, obj)
                                        continue

                                # Don't unlink symlinks to directories here since that can
                                # remove /lib and /usr/lib symlinks.
                                if unmerge_orphans and \
                                        lstatobj and not stat.S_ISDIR(lstatobj.st_mode) and \
                                        not (islink and statobj and stat.S_ISDIR(statobj.st_mode)) and \
                                        not self.isprotected(obj):
                                        try:
                                                unlink(obj, lstatobj)
                                        except EnvironmentError, e:
                                                if e.errno not in ignored_unlink_errnos:
                                                        raise
                                                del e
                                        show_unmerge("<<<", "", file_type, obj)
                                        continue

                                lmtime = str(lstatobj[stat.ST_MTIME])
                                if (pkgfiles[objkey][0] not in ("dir", "fif", "dev")) and (lmtime != pkgfiles[objkey][1]):
                                        show_unmerge("---", "!mtime", file_type, obj)
                                        continue

                                if pkgfiles[objkey][0] == "dir":
                                        if statobj is None or not stat.S_ISDIR(statobj.st_mode):
                                                show_unmerge("---", "!dir", file_type, obj)
                                                continue
                                        mydirs.append(obj)
                                elif pkgfiles[objkey][0] == "sym":
                                        if not islink:
                                                show_unmerge("---", "!sym", file_type, obj)
                                                continue
                                        # Go ahead and unlink symlinks to directories here when
                                        # they're actually recorded as symlinks in the contents.
                                        # Normally, symlinks such as /lib -> lib64 are not recorded
                                        # as symlinks in the contents of a package.  If a package
                                        # installs something into ${D}/lib/, it is recorded in the
                                        # contents as a directory even if it happens to correspond
                                        # to a symlink when it's merged to the live filesystem.
                                        try:
                                                unlink(obj, lstatobj)
                                                show_unmerge("<<<", "", file_type, obj)
                                        except (OSError, IOError),e:
                                                if e.errno not in ignored_unlink_errnos:
                                                        raise
                                                del e
                                                show_unmerge("!!!", "", file_type, obj)
                                elif pkgfiles[objkey][0] == "obj":
                                        if statobj is None or not stat.S_ISREG(statobj.st_mode):
                                                show_unmerge("---", "!obj", file_type, obj)
                                                continue
                                        mymd5 = None
                                        try:
                                                mymd5 = perform_md5(obj, calc_prelink=1)
                                        except FileNotFound, e:
                                                # the file has disappeared between now and our stat call
                                                show_unmerge("---", "!obj", file_type, obj)
                                                continue

                                        # string.lower is needed because db entries used to be in upper-case.  The
                                        # string.lower allows for backwards compatibility.
                                        if mymd5 != pkgfiles[objkey][2].lower():
                                                show_unmerge("---", "!md5", file_type, obj)
                                                continue
                                        try:
                                                unlink(obj, lstatobj)
                                        except (OSError, IOError), e:
                                                if e.errno not in ignored_unlink_errnos:
                                                        raise
                                                del e
                                        show_unmerge("<<<", "", file_type, obj)
                                elif pkgfiles[objkey][0] == "fif":
                                        if not stat.S_ISFIFO(lstatobj[stat.ST_MODE]):
                                                show_unmerge("---", "!fif", file_type, obj)
                                                continue
                                        show_unmerge("---", "", file_type, obj)
                                elif pkgfiles[objkey][0] == "dev":
                                        show_unmerge("---", "", file_type, obj)

                        mydirs.sort()
                        mydirs.reverse()

                        for obj in mydirs:
                                try:
                                        if bsd_chflags:
                                                lstatobj = os.lstat(obj)
                                                if lstatobj.st_flags != 0:
                                                        bsd_chflags.lchflags(obj, 0)
                                                parent_name = os.path.dirname(obj)
                                                # Use normal stat/chflags for the parent since we want to
                                                # follow any symlinks to the real parent directory.
                                                pflags = os.stat(parent_name).st_flags
                                                if pflags != 0:
                                                        bsd_chflags.chflags(parent_name, 0)
                                        try:
                                                os.rmdir(obj)
                                        finally:
                                                if bsd_chflags and pflags != 0:
                                                        # Restore the parent flags we saved before unlinking
                                                        bsd_chflags.chflags(parent_name, pflags)
                                        show_unmerge("<<<", "", "dir", obj)
                                except EnvironmentError, e:
                                        if e.errno not in ignored_rmdir_errnos:
                                                raise
                                        if e.errno != errno.ENOENT:
                                                show_unmerge("---", "!empty", "dir", obj)
                                        del e

                # Remove stale entries from config memory.
                if stale_confmem:
                        for filename in stale_confmem:
                                del cfgfiledict[filename]
                        writedict(cfgfiledict, conf_mem_file)

                #remove self from vartree database so that our own virtual gets zapped if we're the last node
                self.vartree.zap(self.mycpv)

        def isowner(self, filename, destroot):
                """ 
                Check if a file belongs to this package. This may
                result in a stat call for the parent directory of
                every installed file, since the inode numbers are
                used to work around the problem of ambiguous paths
                caused by symlinked directories. The results of
                stat calls are cached to optimize multiple calls
                to this method.

                @param filename:
                @type filename:
                @param destroot:
                @type destroot:
                @rtype: Boolean
                @returns:
                1. True if this package owns the file.
                2. False if this package does not own the file.
                """
                return bool(self._match_contents(filename, destroot))

        def _match_contents(self, filename, destroot):
                """
                The matching contents entry is returned, which is useful
                since the path may differ from the one given by the caller,
                due to symlinks.

                @rtype: String
                @return: the contents entry corresponding to the given path, or False
                        if the file is not owned by this package.
                """

                destfile = normalize_path(
                        os.path.join(destroot, filename.lstrip(os.path.sep)))

                pkgfiles = self.getcontents()
                if pkgfiles and destfile in pkgfiles:
                        return destfile
                if pkgfiles:
                        basename = os.path.basename(destfile)
                        if self._contents_basenames is None:
                                self._contents_basenames = set(
                                        os.path.basename(x) for x in pkgfiles)
                        if basename not in self._contents_basenames:
                                # This is a shortcut that, in most cases, allows us to
                                # eliminate this package as an owner without the need
                                # to examine inode numbers of parent directories.
                                return False

                        # Use stat rather than lstat since we want to follow
                        # any symlinks to the real parent directory.
                        parent_path = os.path.dirname(destfile)
                        try:
                                parent_stat = os.stat(parent_path)
                        except EnvironmentError, e:
                                if e.errno != errno.ENOENT:
                                        raise
                                del e
                                return False
                        if self._contents_inodes is None:
                                self._contents_inodes = {}
                                parent_paths = set()
                                for x in pkgfiles:
                                        p_path = os.path.dirname(x)
                                        if p_path in parent_paths:
                                                continue
                                        parent_paths.add(p_path)
                                        try:
                                                s = os.stat(p_path)
                                        except OSError:
                                                pass
                                        else:
                                                inode_key = (s.st_dev, s.st_ino)
                                                # Use lists of paths in case multiple
                                                # paths reference the same inode.
                                                p_path_list = self._contents_inodes.get(inode_key)
                                                if p_path_list is None:
                                                        p_path_list = []
                                                        self._contents_inodes[inode_key] = p_path_list
                                                if p_path not in p_path_list:
                                                        p_path_list.append(p_path)
                        p_path_list = self._contents_inodes.get(
                                (parent_stat.st_dev, parent_stat.st_ino))
                        if p_path_list:
                                for p_path in p_path_list:
                                        x = os.path.join(p_path, basename)
                                        if x in pkgfiles:
                                                return x

                return False

        def _preserve_libs(self, srcroot, destroot, mycontents, counter, inforoot):
                showMessage = self._display_merge
                # read global reverse NEEDED map
                linkmap = self.vartree.dbapi.linkmap
                linkmap.rebuild(include_file=os.path.join(inforoot, "NEEDED.ELF.2"))
                liblist = linkmap.listLibraryObjects()

                # get list of libraries from old package instance
                root_len = len(self.myroot) - 1
                old_contents = set(p[root_len:] \
                        for p in self._installed_instance.getcontents())
                old_libs = old_contents.intersection(liblist)

                # get list of libraries from new package instance
                mylibs = set([os.path.join(os.sep, x) for x in mycontents]).intersection(liblist)
                
                # check which libs are present in the old, but not the new package instance
                candidates = old_libs.difference(mylibs)
                
                for x in old_contents:
                        if os.path.islink(x) and os.path.realpath(x) in candidates and x not in mycontents:
                                candidates.add(x)

                provider_cache = {}
                consumer_cache = {}

                # ignore any libs that are only internally used by the package
                def has_external_consumers(lib, contents, otherlibs):
                        consumers = consumer_cache.get(lib)
                        if consumers is None:
                                consumers = linkmap.findConsumers(lib)
                                consumer_cache[lib] = consumers
                        contents_without_libs = [x for x in contents if x not in otherlibs]
                        
                        # just used by objects that will be autocleaned
                        if len(consumers.difference(contents_without_libs)) == 0:
                                return False
                        # used by objects that are referenced as well, need to check those 
                        # recursively to break any reference cycles
                        elif len(consumers.difference(contents)) == 0:
                                otherlibs = set(otherlibs)
                                for ol in otherlibs.intersection(consumers):
                                        if has_external_consumers(ol, contents, otherlibs.difference([lib])):
                                                return True
                                return False
                        # used by external objects directly
                        else:
                                return True

                for lib in list(candidates):
                        if not has_external_consumers(lib, old_contents, candidates):
                                candidates.remove(lib)
                                continue
                        if linkmap.isMasterLink(lib):
                                candidates.remove(lib)
                                continue
                        # only preserve the lib if there is no other copy to use for each consumer
                        keep = False

                        lib_consumers = consumer_cache.get(lib)
                        if lib_consumers is None:
                                lib_consumers = linkmap.findConsumers(lib)
                                consumer_cache[lib] = lib_consumers

                        for c in lib_consumers:
                                localkeep = True
                                providers = provider_cache.get(c)
                                if providers is None:
                                        providers = linkmap.findProviders(c)
                                        provider_cache[c] = providers

                                for soname in providers:
                                        if lib in providers[soname]:
                                                for p in providers[soname]:
                                                        if p not in candidates or os.path.exists(os.path.join(srcroot, p.lstrip(os.sep))):
                                                                localkeep = False
                                                                break
                                                break
                                if localkeep:
                                        keep = True
                        if not keep:
                                candidates.remove(lib)
                                continue
                
                del mylibs, mycontents, old_contents, liblist
                
                # inject files that should be preserved into our image dir
                import shutil
                preserve_paths = []
                candidates_stack = list(candidates)
                while candidates_stack:
                        x = candidates_stack.pop()
                        # skip existing files so the 'new' libs aren't overwritten
                        if os.path.exists(os.path.join(srcroot, x.lstrip(os.sep))):
                                continue
                        showMessage("injecting %s into %s\n" % (x, srcroot),
                                noiselevel=-1)
                        if not os.path.exists(os.path.join(destroot, x.lstrip(os.sep))):
                                showMessage("%s does not exist so can't be preserved\n" % x,
                                        noiselevel=-1)
                                continue
                        mydir = os.path.join(srcroot, os.path.dirname(x).lstrip(os.sep))
                        if not os.path.exists(mydir):
                                os.makedirs(mydir)

                        # resolve symlinks and extend preserve list
                        # NOTE: we're extending the list in the loop to emulate recursion to
                        #       also get indirect symlinks
                        if os.path.islink(x):
                                linktarget = os.readlink(x)
                                os.symlink(linktarget, os.path.join(srcroot, x.lstrip(os.sep)))
                                if linktarget[0] != os.sep:
                                        linktarget = os.path.join(os.path.dirname(x), linktarget)
                                if linktarget not in candidates:
                                        candidates.add(linktarget)
                                        candidates_stack.append(linktarget)
                        else:
                                shutil.copy2(os.path.join(destroot, x.lstrip(os.sep)),
                                        os.path.join(srcroot, x.lstrip(os.sep)))
                        preserve_paths.append(x)
                        
                del candidates

                # keep track of the libs we preserved
                self.vartree.dbapi.plib_registry.register(self.mycpv, self.settings["SLOT"], counter, preserve_paths)

                del preserve_paths
        
        def _collision_protect(self, srcroot, destroot, mypkglist, mycontents):
                        collision_ignore = set([normalize_path(myignore) for myignore in \
                                shlex.split(self.settings.get("COLLISION_IGNORE", ""))])

                        showMessage = self._display_merge
                        scheduler = self._scheduler
                        stopmerge = False
                        collisions = []
                        destroot = normalize_path(destroot).rstrip(os.path.sep) + \
                                os.path.sep
                        showMessage("%s checking %d files for package collisions\n" % \
                                (green("*"), len(mycontents)))
                        for i, f in enumerate(mycontents):
                                if i % 1000 == 0 and i != 0:
                                        showMessage("%d files checked ...\n" % i)

                                if scheduler is not None and \
                                        0 == i % self._file_merge_yield_interval:
                                        scheduler.scheduleYield()

                                dest_path = normalize_path(
                                        os.path.join(destroot, f.lstrip(os.path.sep)))
                                try:
                                        dest_lstat = os.lstat(dest_path)
                                except EnvironmentError, e:
                                        if e.errno == errno.ENOENT:
                                                del e
                                                continue
                                        elif e.errno == errno.ENOTDIR:
                                                del e
                                                # A non-directory is in a location where this package
                                                # expects to have a directory.
                                                dest_lstat = None
                                                parent_path = dest_path
                                                while len(parent_path) > len(destroot):
                                                        parent_path = os.path.dirname(parent_path)
                                                        try:
                                                                dest_lstat = os.lstat(parent_path)
                                                                break
                                                        except EnvironmentError, e:
                                                                if e.errno != errno.ENOTDIR:
                                                                        raise
                                                                del e
                                                if not dest_lstat:
                                                        raise AssertionError(
                                                                "unable to find non-directory " + \
                                                                "parent for '%s'" % dest_path)
                                                dest_path = parent_path
                                                f = os.path.sep + dest_path[len(destroot):]
                                                if f in collisions:
                                                        continue
                                        else:
                                                raise
                                if f[0] != "/":
                                        f="/"+f
                                isowned = False
                                for ver in [self] + mypkglist:
                                        if (ver.isowner(f, destroot) or ver.isprotected(f)):
                                                isowned = True
                                                break
                                if not isowned:
                                        stopmerge = True
                                        if collision_ignore:
                                                if f in collision_ignore:
                                                        stopmerge = False
                                                else:
                                                        for myignore in collision_ignore:
                                                                if f.startswith(myignore + os.path.sep):
                                                                        stopmerge = False
                                                                        break
                                        if stopmerge:
                                                collisions.append(f)
                        return collisions

        def _security_check(self, installed_instances):
                if not installed_instances:
                        return 0

                showMessage = self._display_merge
                scheduler = self._scheduler

                file_paths = set()
                for dblnk in installed_instances:
                        file_paths.update(dblnk.getcontents())
                inode_map = {}
                real_paths = set()
                for i, path in enumerate(file_paths):

                        if scheduler is not None and \
                                0 == i % self._file_merge_yield_interval:
                                scheduler.scheduleYield()

                        try:
                                s = os.lstat(path)
                        except OSError, e:
                                if e.errno not in (errno.ENOENT, errno.ENOTDIR):
                                        raise
                                del e
                                continue
                        if not stat.S_ISREG(s.st_mode):
                                continue
                        path = os.path.realpath(path)
                        if path in real_paths:
                                continue
                        real_paths.add(path)
                        if s.st_nlink > 1 and \
                                s.st_mode & (stat.S_ISUID | stat.S_ISGID):
                                k = (s.st_dev, s.st_ino)
                                inode_map.setdefault(k, []).append((path, s))
                suspicious_hardlinks = []
                for path_list in inode_map.itervalues():
                        path, s = path_list[0]
                        if len(path_list) == s.st_nlink:
                                # All hardlinks seem to be owned by this package.
                                continue
                        suspicious_hardlinks.append(path_list)
                if not suspicious_hardlinks:
                        return 0

                msg = []
                msg.append("suid/sgid file(s) " + \
                        "with suspicious hardlink(s):")
                msg.append("")
                for path_list in suspicious_hardlinks:
                        for path, s in path_list:
                                msg.append("\t%s" % path)
                msg.append("")
                msg.append("See the Gentoo Security Handbook " + \
                        "guide for advice on how to proceed.")

                self._eerror("preinst", msg)

                return 1

        def _eerror(self, phase, lines):
                from portage.elog.messages import eerror as _eerror
                if self._scheduler is None:
                        for l in lines:
                                _eerror(l, phase=phase, key=self.settings.mycpv)
                else:
                        self._scheduler.dblinkElog(self,
                                phase, _eerror, lines)

        def treewalk(self, srcroot, destroot, inforoot, myebuild, cleanup=0,
                mydbapi=None, prev_mtimes=None):
                """
                
                This function does the following:
                
                calls self._preserve_libs if FEATURES=preserve-libs
                calls self._collision_protect if FEATURES=collision-protect
                calls doebuild(mydo=pkg_preinst)
                Merges the package to the livefs
                unmerges old version (if required)
                calls doebuild(mydo=pkg_postinst)
                calls env_update
                calls elog_process
                
                @param srcroot: Typically this is ${D}
                @type srcroot: String (Path)
                @param destroot: Path to merge to (usually ${ROOT})
                @type destroot: String (Path)
                @param inforoot: root of the vardb entry ?
                @type inforoot: String (Path)
                @param myebuild: path to the ebuild that we are processing
                @type myebuild: String (Path)
                @param mydbapi: dbapi which is handed to doebuild.
                @type mydbapi: portdbapi instance
                @param prev_mtimes: { Filename:mtime } mapping for env_update
                @type prev_mtimes: Dictionary
                @rtype: Boolean
                @returns:
                1. 0 on success
                2. 1 on failure
                
                secondhand is a list of symlinks that have been skipped due to their target
                not existing; we will merge these symlinks at a later time.
                """

                showMessage = self._display_merge
                scheduler = self._scheduler

                srcroot = normalize_path(srcroot).rstrip(os.path.sep) + os.path.sep
                destroot = normalize_path(destroot).rstrip(os.path.sep) + os.path.sep

                if not os.path.isdir(srcroot):
                        showMessage("!!! Directory Not Found: D='%s'\n" % srcroot,
                                level=logging.ERROR, noiselevel=-1)
                        return 1

                inforoot_slot_file = os.path.join(inforoot, "SLOT")
                slot = None
                try:
                        f = open(inforoot_slot_file)
                        try:
                                slot = f.read().strip()
                        finally:
                                f.close()
                except EnvironmentError, e:
                        if e.errno != errno.ENOENT:
                                raise
                        del e

                if slot is None:
                        slot = ""

                def eerror(lines):
                        self._eerror("preinst", lines)

                if slot != self.settings["SLOT"]:
                        showMessage("!!! WARNING: Expected SLOT='%s', got '%s'\n" % \
                                (self.settings["SLOT"], slot), level=logging.WARN)

                if not os.path.exists(self.dbcatdir):
                        os.makedirs(self.dbcatdir)

                otherversions = []
                for v in self.vartree.dbapi.cp_list(self.mysplit[0]):
                        otherversions.append(v.split("/")[1])

                # filter any old-style virtual matches
                slot_matches = [cpv for cpv in self.vartree.dbapi.match(
                        "%s:%s" % (cpv_getkey(self.mycpv), slot)) \
                        if cpv_getkey(cpv) == cpv_getkey(self.mycpv)]

                if self.mycpv not in slot_matches and \
                        self.vartree.dbapi.cpv_exists(self.mycpv):
                        # handle multislot or unapplied slotmove
                        slot_matches.append(self.mycpv)

                others_in_slot = []
                from portage import config
                for cur_cpv in slot_matches:
                        # Clone the config in case one of these has to be unmerged since
                        # we need it to have private ${T} etc... for things like elog.
                        others_in_slot.append(dblink(self.cat, catsplit(cur_cpv)[1],
                                self.vartree.root, config(clone=self.settings),
                                vartree=self.vartree, treetype="vartree",
                                scheduler=self._scheduler))

                retval = self._security_check(others_in_slot)
                if retval:
                        return retval

                if slot_matches:
                        # Used by self.isprotected().
                        max_dblnk = None
                        max_counter = -1
                        for dblnk in others_in_slot:
                                cur_counter = self.vartree.dbapi.cpv_counter(dblnk.mycpv)
                                if cur_counter > max_counter:
                                        max_counter = cur_counter
                                        max_dblnk = dblnk
                        self._installed_instance = max_dblnk

                # get current counter value (counter_tick also takes care of incrementing it)
                # XXX Need to make this destroot, but it needs to be initialized first. XXX
                # XXX bis: leads to some invalidentry() call through cp_all().
                # Note: The counter is generated here but written later because preserve_libs
                #       needs the counter value but has to be before dbtmpdir is made (which
                #       has to be before the counter is written) - genone
                counter = self.vartree.dbapi.counter_tick(self.myroot, mycpv=self.mycpv)

                # Save this for unregistering preserved-libs if the merge fails.
                self.settings["COUNTER"] = str(counter)
                self.settings.backup_changes("COUNTER")

                myfilelist = []
                mylinklist = []
                def onerror(e):
                        raise
                for parent, dirs, files in os.walk(srcroot, onerror=onerror):
                        for f in files:
                                file_path = os.path.join(parent, f)
                                file_mode = os.lstat(file_path).st_mode
                                if stat.S_ISREG(file_mode):
                                        myfilelist.append(file_path[len(srcroot):])
                                elif stat.S_ISLNK(file_mode):
                                        # Note: os.walk puts symlinks to directories in the "dirs"
                                        # list and it does not traverse them since that could lead
                                        # to an infinite recursion loop.
                                        mylinklist.append(file_path[len(srcroot):])

                # If there are no files to merge, and an installed package in the same
                # slot has files, it probably means that something went wrong.
                if self.settings.get("PORTAGE_PACKAGE_EMPTY_ABORT") == "1" and \
                        not myfilelist and not mylinklist and others_in_slot:
                        installed_files = None
                        for other_dblink in others_in_slot:
                                installed_files = other_dblink.getcontents()
                                if not installed_files:
                                        continue
                                from textwrap import wrap
                                wrap_width = 72
                                msg = []
                                d = (
                                        self.mycpv,
                                        other_dblink.mycpv
                                )
                                msg.extend(wrap(("The '%s' package will not install " + \
                                        "any files, but the currently installed '%s'" + \
                                        " package has the following files: ") % d, wrap_width))
                                msg.append("")
                                msg.extend(sorted(installed_files))
                                msg.append("")
                                msg.append("package %s NOT merged" % self.mycpv)
                                msg.append("")
                                msg.extend(wrap(
                                        ("Manually run `emerge --unmerge =%s` " % \
                                        other_dblink.mycpv) + "if you really want to " + \
                                        "remove the above files. Set " + \
                                        "PORTAGE_PACKAGE_EMPTY_ABORT=\"0\" in " + \
                                        "/etc/make.conf if you do not want to " + \
                                        "abort in cases like this.",
                                        wrap_width))
                                eerror(msg)
                        if installed_files:
                                return 1

                # Preserve old libs if they are still in use
                if slot_matches and "preserve-libs" in self.settings.features:
                        self._preserve_libs(srcroot, destroot, myfilelist+mylinklist, counter, inforoot)

                # check for package collisions
                blockers = None
                if self._blockers is not None:
                        # This is only supposed to be called when
                        # the vdb is locked, like it is here.
                        blockers = self._blockers()
                if blockers is None:
                        blockers = []
                collisions = self._collision_protect(srcroot, destroot,
                        others_in_slot + blockers, myfilelist + mylinklist)

                # Make sure the ebuild environment is initialized and that ${T}/elog
                # exists for logging of collision-protect eerror messages.
                if myebuild is None:
                        myebuild = os.path.join(inforoot, self.pkg + ".ebuild")
                doebuild_environment(myebuild, "preinst", destroot,
                        self.settings, 0, 0, mydbapi)
                prepare_build_dirs(destroot, self.settings, cleanup)

                if collisions:
                        collision_protect = "collision-protect" in self.settings.features
                        msg = "This package will overwrite one or more files that" + \
                        " may belong to other packages (see list below)."
                        if not collision_protect:
                                msg += " Add \"collision-protect\" to FEATURES in" + \
                                " make.conf if you would like the merge to abort" + \
                                " in cases like this."
                        if self.settings.get("PORTAGE_QUIET") != "1":
                                msg += " You can use a command such as" + \
                                " `portageq owners / <filename>` to identify the" + \
                                " installed package that owns a file. If portageq" + \
                                " reports that only one package owns a file then do NOT" + \
                                " file a bug report. A bug report is only useful if it" + \
                                " identifies at least two or more packages that are known" + \
                                " to install the same file(s)." + \
                                " If a collision occurs and you" + \
                                " can not explain where the file came from then you" + \
                                " should simply ignore the collision since there is not" + \
                                " enough information to determine if a real problem" + \
                                " exists. Please do NOT file a bug report at" + \
                                " http://bugs.gentoo.org unless you report exactly which" + \
                                " two packages install the same file(s). Once again," + \
                                " please do NOT file a bug report unless you have" + \
                                " completely understood the above message."

                        self.settings["EBUILD_PHASE"] = "preinst"
                        from textwrap import wrap
                        msg = wrap(msg, 70)
                        if collision_protect:
                                msg.append("")
                                msg.append("package %s NOT merged" % self.settings.mycpv)
                        msg.append("")
                        msg.append("Detected file collision(s):")
                        msg.append("")

                        for f in collisions:
                                msg.append("\t%s" % \
                                        os.path.join(destroot, f.lstrip(os.path.sep)))

                        eerror(msg)

                        msg = []
                        msg.append("")
                        msg.append("Searching all installed" + \
                                " packages for file collisions...")
                        msg.append("")
                        msg.append("Press Ctrl-C to Stop")
                        msg.append("")
                        eerror(msg)

                        owners = self.vartree.dbapi._owners.get_owners(collisions)
                        self.vartree.dbapi.flush_cache()

                        for pkg, owned_files in owners.iteritems():
                                cpv = pkg.mycpv
                                msg = []
                                msg.append("%s" % cpv)
                                for f in sorted(owned_files):
                                        msg.append("\t%s" % os.path.join(destroot,
                                                f.lstrip(os.path.sep)))
                                msg.append("")
                                eerror(msg)

                        if not owners:
                                eerror(["None of the installed" + \
                                        " packages claim the file(s).", ""])

                        # The explanation about the collision and how to solve
                        # it may not be visible via a scrollback buffer, especially
                        # if the number of file collisions is large. Therefore,
                        # show a summary at the end.
                        if collision_protect:
                                msg = "Package '%s' NOT merged due to file collisions." % \
                                        self.settings.mycpv
                        else:
                                msg = "Package '%s' merged despite file collisions." % \
                                        self.settings.mycpv
                        msg += " If necessary, refer to your elog " + \
                                "messages for the whole content of the above message."
                        eerror(wrap(msg, 70))

                        if collision_protect:
                                return 1

                # The merge process may move files out of the image directory,
                # which causes invalidation of the .installed flag.
                try:
                        os.unlink(os.path.join(
                                os.path.dirname(normalize_path(srcroot)), ".installed"))
                except OSError, e:
                        if e.errno != errno.ENOENT:
                                raise
                        del e

                self.dbdir = self.dbtmpdir
                self.delete()
                ensure_dirs(self.dbtmpdir)

                # run preinst script
                if scheduler is None:
                        showMessage(">>> Merging %s to %s\n" % (self.mycpv, destroot))
                        a = doebuild(myebuild, "preinst", destroot, self.settings,
                                use_cache=0, tree=self.treetype, mydbapi=mydbapi,
                                vartree=self.vartree)
                else:
                        a = scheduler.dblinkEbuildPhase(
                                self, mydbapi, myebuild, "preinst")

                # XXX: Decide how to handle failures here.
                if a != os.EX_OK:
                        showMessage("!!! FAILED preinst: "+str(a)+"\n",
                                level=logging.ERROR, noiselevel=-1)
                        return a

                # copy "info" files (like SLOT, CFLAGS, etc.) into the database
                for x in os.listdir(inforoot):
                        self.copyfile(inforoot+"/"+x)

                # write local package counter for recording
                lcfile = open(os.path.join(self.dbtmpdir, "COUNTER"),"w")
                lcfile.write(str(counter))
                lcfile.close()

                # open CONTENTS file (possibly overwriting old one) for recording
                outfile = open(os.path.join(self.dbtmpdir, "CONTENTS"),"w")

                self.updateprotect()

                #if we have a file containing previously-merged config file md5sums, grab it.
                conf_mem_file = os.path.join(destroot, CONFIG_MEMORY_FILE)
                cfgfiledict = grabdict(conf_mem_file)
                if "NOCONFMEM" in self.settings:
                        cfgfiledict["IGNORE"]=1
                else:
                        cfgfiledict["IGNORE"]=0

                # Always behave like --noconfmem is enabled for downgrades
                # so that people who don't know about this option are less
                # likely to get confused when doing upgrade/downgrade cycles.
                pv_split = catpkgsplit(self.mycpv)[1:]
                for other in others_in_slot:
                        if pkgcmp(pv_split, catpkgsplit(other.mycpv)[1:]) < 0:
                                cfgfiledict["IGNORE"] = 1
                                break

                # Don't bump mtimes on merge since some application require
                # preservation of timestamps.  This means that the unmerge phase must
                # check to see if file belongs to an installed instance in the same
                # slot.
                mymtime = None

                # set umask to 0 for merging; back up umask, save old one in prevmask (since this is a global change)
                prevmask = os.umask(0)
                secondhand = []

                # we do a first merge; this will recurse through all files in our srcroot but also build up a
                # "second hand" of symlinks to merge later
                if self.mergeme(srcroot, destroot, outfile, secondhand, "", cfgfiledict, mymtime):
                        return 1

                # now, it's time for dealing our second hand; we'll loop until we can't merge anymore.  The rest are
                # broken symlinks.  We'll merge them too.
                lastlen = 0
                while len(secondhand) and len(secondhand)!=lastlen:
                        # clear the thirdhand.  Anything from our second hand that
                        # couldn't get merged will be added to thirdhand.

                        thirdhand = []
                        self.mergeme(srcroot, destroot, outfile, thirdhand, secondhand, cfgfiledict, mymtime)

                        #swap hands
                        lastlen = len(secondhand)

                        # our thirdhand now becomes our secondhand.  It's ok to throw
                        # away secondhand since thirdhand contains all the stuff that
                        # couldn't be merged.
                        secondhand = thirdhand

                if len(secondhand):
                        # force merge of remaining symlinks (broken or circular; oh well)
                        self.mergeme(srcroot, destroot, outfile, None, secondhand, cfgfiledict, mymtime)

                #restore umask
                os.umask(prevmask)

                #if we opened it, close it
                outfile.flush()
                outfile.close()

                # write out our collection of md5sums
                cfgfiledict.pop("IGNORE", None)
                ensure_dirs(os.path.dirname(conf_mem_file),
                        gid=portage_gid, mode=02750, mask=02)
                writedict(cfgfiledict, conf_mem_file)

                # These caches are populated during collision-protect and the data
                # they contain is now invalid. It's very important to invalidate
                # the contents_inodes cache so that FEATURES=unmerge-orphans
                # doesn't unmerge anything that belongs to this package that has
                # just been merged.
                others_in_slot.append(self)  # self has just been merged
                for dblnk in others_in_slot:
                        dblnk.contentscache = None
                        dblnk._contents_inodes = None
                        dblnk._contents_basenames = None

                # If portage is reinstalling itself, remove the old
                # version now since we want to use the temporary
                # PORTAGE_BIN_PATH that will be removed when we return.
                reinstall_self = False
                if self.myroot == "/" and \
                        "sys-apps" == self.cat and \
                        "portage" == pkgsplit(self.pkg)[0]:
                        reinstall_self = True

                autoclean = self.settings.get("AUTOCLEAN", "yes") == "yes"
                for dblnk in list(others_in_slot):
                        if dblnk is self:
                                continue
                        if not (autoclean or dblnk.mycpv == self.mycpv or reinstall_self):
                                continue
                        showMessage(">>> Safely unmerging already-installed instance...\n")
                        others_in_slot.remove(dblnk) # dblnk will unmerge itself now
                        dblnk.unmerge(trimworld=0, ldpath_mtimes=prev_mtimes,
                                others_in_slot=others_in_slot)
                        # TODO: Check status and abort if necessary.
                        dblnk.delete()
                        showMessage(">>> Original instance of package unmerged safely.\n")

                if len(others_in_slot) > 1:
                        from portage.output import colorize
                        showMessage(colorize("WARN", "WARNING:")
                                + " AUTOCLEAN is disabled.  This can cause serious"
                                + " problems due to overlapping packages.\n",
                                level=logging.WARN, noiselevel=-1)

                # We hold both directory locks.
                self.dbdir = self.dbpkgdir
                self.delete()
                _movefile(self.dbtmpdir, self.dbpkgdir, mysettings=self.settings)

                # Check for file collisions with blocking packages
                # and remove any colliding files from their CONTENTS
                # since they now belong to this package.
                self._clear_contents_cache()
                contents = self.getcontents()
                destroot_len = len(destroot) - 1
                for blocker in blockers:
                        self.vartree.dbapi.removeFromContents(blocker, iter(contents),
                                relative_paths=False)

                self.vartree.dbapi._add(self)
                contents = self.getcontents()

                # regenerate reverse NEEDED map
                self.vartree.dbapi.linkmap.rebuild()

                #do postinst script
                self.settings["PORTAGE_UPDATE_ENV"] = \
                        os.path.join(self.dbpkgdir, "environment.bz2")
                self.settings.backup_changes("PORTAGE_UPDATE_ENV")
                try:
                        if scheduler is None:
                                a = doebuild(myebuild, "postinst", destroot, self.settings,
                                        use_cache=0, tree=self.treetype, mydbapi=mydbapi,
                                        vartree=self.vartree)
                                if a == os.EX_OK:
                                        showMessage(">>> %s %s\n" % (self.mycpv, "merged."))
                        else:
                                a = scheduler.dblinkEbuildPhase(
                                        self, mydbapi, myebuild, "postinst")
                finally:
                        self.settings.pop("PORTAGE_UPDATE_ENV", None)

                # XXX: Decide how to handle failures here.
                if a != os.EX_OK:
                        showMessage("!!! FAILED postinst: "+str(a)+"\n",
                                level=logging.ERROR, noiselevel=-1)
                        return a

                downgrade = False
                for v in otherversions:
                        if pkgcmp(catpkgsplit(self.pkg)[1:], catpkgsplit(v)[1:]) < 0:
                                downgrade = True

                #update environment settings, library paths. DO NOT change symlinks.
                env_update(makelinks=(not downgrade),
                        target_root=self.settings["ROOT"], prev_mtimes=prev_mtimes,
                        contents=contents, env=self.settings.environ(),
                        writemsg_level=self._display_merge)

                return os.EX_OK

        def mergeme(self, srcroot, destroot, outfile, secondhand, stufftomerge, cfgfiledict, thismtime):
                """
                
                This function handles actual merging of the package contents to the livefs.
                It also handles config protection.
                
                @param srcroot: Where are we copying files from (usually ${D})
                @type srcroot: String (Path)
                @param destroot: Typically ${ROOT}
                @type destroot: String (Path)
                @param outfile: File to log operations to
                @type outfile: File Object
                @param secondhand: A set of items to merge in pass two (usually
                or symlinks that point to non-existing files that may get merged later)
                @type secondhand: List
                @param stufftomerge: Either a diretory to merge, or a list of items.
                @type stufftomerge: String or List
                @param cfgfiledict: { File:mtime } mapping for config_protected files
                @type cfgfiledict: Dictionary
                @param thismtime: The current time (typically long(time.time())
                @type thismtime: Long
                @rtype: None or Boolean
                @returns:
                1. True on failure
                2. None otherwise
                
                """

                showMessage = self._display_merge
                scheduler = self._scheduler

                from os.path import sep, join
                srcroot = normalize_path(srcroot).rstrip(sep) + sep
                destroot = normalize_path(destroot).rstrip(sep) + sep
                
                # this is supposed to merge a list of files.  There will be 2 forms of argument passing.
                if isinstance(stufftomerge, basestring):
                        #A directory is specified.  Figure out protection paths, listdir() it and process it.
                        mergelist = os.listdir(join(srcroot, stufftomerge))
                        offset = stufftomerge
                else:
                        mergelist = stufftomerge
                        offset = ""

                for i, x in enumerate(mergelist):

                        if scheduler is not None and \
                                0 == i % self._file_merge_yield_interval:
                                scheduler.scheduleYield()

                        mysrc = join(srcroot, offset, x)
                        mydest = join(destroot, offset, x)
                        # myrealdest is mydest without the $ROOT prefix (makes a difference if ROOT!="/")
                        myrealdest = join(sep, offset, x)
                        # stat file once, test using S_* macros many times (faster that way)
                        try:
                                mystat = os.lstat(mysrc)
                        except OSError, e:
                                writemsg("\n")
                                writemsg(red("!!! ERROR: There appears to be ")+bold("FILE SYSTEM CORRUPTION.")+red(" A file that is listed\n"))
                                writemsg(red("!!!        as existing is not capable of being stat'd. If you are using an\n"))
                                writemsg(red("!!!        experimental kernel, please boot into a stable one, force an fsck,\n"))
                                writemsg(red("!!!        and ensure your filesystem is in a sane state. ")+bold("'shutdown -Fr now'\n"))
                                writemsg(red("!!!        File:  ")+str(mysrc)+"\n", noiselevel=-1)
                                writemsg(red("!!!        Error: ")+str(e)+"\n", noiselevel=-1)
                                sys.exit(1)
                        except Exception, e:
                                writemsg("\n")
                                writemsg(red("!!! ERROR: An unknown error has occurred during the merge process.\n"))
                                writemsg(red("!!!        A stat call returned the following error for the following file:"))
                                writemsg(    "!!!        Please ensure that your filesystem is intact, otherwise report\n")
                                writemsg(    "!!!        this as a portage bug at bugs.gentoo.org. Append 'emerge info'.\n")
                                writemsg(    "!!!        File:  "+str(mysrc)+"\n", noiselevel=-1)
                                writemsg(    "!!!        Error: "+str(e)+"\n", noiselevel=-1)
                                sys.exit(1)


                        mymode = mystat[stat.ST_MODE]
                        # handy variables; mydest is the target object on the live filesystems;
                        # mysrc is the source object in the temporary install dir
                        try:
                                mydstat = os.lstat(mydest)
                                mydmode = mydstat.st_mode
                        except OSError, e:
                                if e.errno != errno.ENOENT:
                                        raise
                                del e
                                #dest file doesn't exist
                                mydstat = None
                                mydmode = None

                        if stat.S_ISLNK(mymode):
                                # we are merging a symbolic link
                                myabsto = abssymlink(mysrc)
                                if myabsto.startswith(srcroot):
                                        myabsto = myabsto[len(srcroot):]
                                myabsto = myabsto.lstrip(sep)
                                myto = os.readlink(mysrc)
                                if self.settings and self.settings["D"]:
                                        if myto.startswith(self.settings["D"]):
                                                myto = myto[len(self.settings["D"]):]
                                # myrealto contains the path of the real file to which this symlink points.
                                # we can simply test for existence of this file to see if the target has been merged yet
                                myrealto = normalize_path(os.path.join(destroot, myabsto))
                                if mydmode!=None:
                                        #destination exists
                                        if not stat.S_ISLNK(mydmode):
                                                if stat.S_ISDIR(mydmode):
                                                        # directory in the way: we can't merge a symlink over a directory
                                                        # we won't merge this, continue with next file...
                                                        continue

                                                if os.path.exists(mysrc) and stat.S_ISDIR(os.stat(mysrc)[stat.ST_MODE]):
                                                        # Kill file blocking installation of symlink to dir #71787
                                                        pass
                                                elif self.isprotected(mydest):
                                                        # Use md5 of the target in ${D} if it exists...
                                                        try:
                                                                newmd5 = perform_md5(join(srcroot, myabsto))
                                                        except FileNotFound:
                                                                # Maybe the target is merged already.
                                                                try:
                                                                        newmd5 = perform_md5(myrealto)
                                                                except FileNotFound:
                                                                        newmd5 = None
                                                        mydest = new_protect_filename(mydest, newmd5=newmd5)

                                # if secondhand is None it means we're operating in "force" mode and should not create a second hand.
                                if (secondhand != None) and (not os.path.exists(myrealto)):
                                        # either the target directory doesn't exist yet or the target file doesn't exist -- or
                                        # the target is a broken symlink.  We will add this file to our "second hand" and merge
                                        # it later.
                                        secondhand.append(mysrc[len(srcroot):])
                                        continue
                                # unlinking no longer necessary; "movefile" will overwrite symlinks atomically and correctly
                                mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings)
                                if mymtime != None:
                                        showMessage(">>> %s -> %s\n" % (mydest, myto))
                                        outfile.write("sym "+myrealdest+" -> "+myto+" "+str(mymtime)+"\n")
                                else:
                                        print "!!! Failed to move file."
                                        print "!!!", mydest, "->", myto
                                        sys.exit(1)
                        elif stat.S_ISDIR(mymode):
                                # we are merging a directory
                                if mydmode != None:
                                        # destination exists

                                        if bsd_chflags:
                                                # Save then clear flags on dest.
                                                dflags = mydstat.st_flags
                                                if dflags != 0:
                                                        bsd_chflags.lchflags(mydest, 0)

                                        if not os.access(mydest, os.W_OK):
                                                pkgstuff = pkgsplit(self.pkg)
                                                writemsg("\n!!! Cannot write to '"+mydest+"'.\n", noiselevel=-1)
                                                writemsg("!!! Please check permissions and directories for broken symlinks.\n")
                                                writemsg("!!! You may start the merge process again by using ebuild:\n")
                                                writemsg("!!! ebuild "+self.settings["PORTDIR"]+"/"+self.cat+"/"+pkgstuff[0]+"/"+self.pkg+".ebuild merge\n")
                                                writemsg("!!! And finish by running this: env-update\n\n")
                                                return 1

                                        if stat.S_ISLNK(mydmode) or stat.S_ISDIR(mydmode):
                                                # a symlink to an existing directory will work for us; keep it:
                                                showMessage("--- %s/\n" % mydest)
                                                if bsd_chflags:
                                                        bsd_chflags.lchflags(mydest, dflags)
                                        else:
                                                # a non-directory and non-symlink-to-directory.  Won't work for us.  Move out of the way.
                                                if movefile(mydest, mydest+".backup", mysettings=self.settings) is None:
                                                        sys.exit(1)
                                                print "bak", mydest, mydest+".backup"
                                                #now create our directory
                                                if self.settings.selinux_enabled():
                                                        import selinux
                                                        sid = selinux.get_sid(mysrc)
                                                        selinux.secure_mkdir(mydest,sid)
                                                else:
                                                        os.mkdir(mydest)
                                                if bsd_chflags:
                                                        bsd_chflags.lchflags(mydest, dflags)
                                                os.chmod(mydest, mystat[0])
                                                os.chown(mydest, mystat[4], mystat[5])
                                                showMessage(">>> %s/\n" % mydest)
                                else:
                                        #destination doesn't exist
                                        if self.settings.selinux_enabled():
                                                import selinux
                                                sid = selinux.get_sid(mysrc)
                                                selinux.secure_mkdir(mydest, sid)
                                        else:
                                                os.mkdir(mydest)
                                        os.chmod(mydest, mystat[0])
                                        os.chown(mydest, mystat[4], mystat[5])
                                        showMessage(">>> %s/\n" % mydest)
                                outfile.write("dir "+myrealdest+"\n")
                                # recurse and merge this directory
                                if self.mergeme(srcroot, destroot, outfile, secondhand,
                                        join(offset, x), cfgfiledict, thismtime):
                                        return 1
                        elif stat.S_ISREG(mymode):
                                # we are merging a regular file
                                mymd5 = perform_md5(mysrc, calc_prelink=1)
                                # calculate config file protection stuff
                                mydestdir = os.path.dirname(mydest)
                                moveme = 1
                                zing = "!!!"
                                mymtime = None
                                if mydmode != None:
                                        # destination file exists
                                        if stat.S_ISDIR(mydmode):
                                                # install of destination is blocked by an existing directory with the same name
                                                moveme = 0
                                                showMessage("!!! %s\n" % mydest,
                                                        level=logging.ERROR, noiselevel=-1)
                                        elif stat.S_ISREG(mydmode) or (stat.S_ISLNK(mydmode) and os.path.exists(mydest) and stat.S_ISREG(os.stat(mydest)[stat.ST_MODE])):
                                                cfgprot = 0
                                                # install of destination is blocked by an existing regular file,
                                                # or by a symlink to an existing regular file;
                                                # now, config file management may come into play.
                                                # we only need to tweak mydest if cfg file management is in play.
                                                if self.isprotected(mydest):
                                                        # we have a protection path; enable config file management.
                                                        destmd5 = perform_md5(mydest, calc_prelink=1)
                                                        if mymd5 == destmd5:
                                                                #file already in place; simply update mtimes of destination
                                                                moveme = 1
                                                        else:
                                                                if mymd5 == cfgfiledict.get(myrealdest, [None])[0]:
                                                                        """ An identical update has previously been
                                                                        merged.  Skip it unless the user has chosen
                                                                        --noconfmem."""
                                                                        moveme = cfgfiledict["IGNORE"]
                                                                        cfgprot = cfgfiledict["IGNORE"]
                                                                        if not moveme:
                                                                                zing = "---"
                                                                                mymtime = long(mystat.st_mtime)
                                                                else:
                                                                        moveme = 1
                                                                        cfgprot = 1
                                                        if moveme:
                                                                # Merging a new file, so update confmem.
                                                                cfgfiledict[myrealdest] = [mymd5]
                                                        elif destmd5 == cfgfiledict.get(myrealdest, [None])[0]:
                                                                """A previously remembered update has been
                                                                accepted, so it is removed from confmem."""
                                                                del cfgfiledict[myrealdest]
                                                if cfgprot:
                                                        mydest = new_protect_filename(mydest, newmd5=mymd5)

                                # whether config protection or not, we merge the new file the
                                # same way.  Unless moveme=0 (blocking directory)
                                if moveme:
                                        mymtime = movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings)
                                        if mymtime is None:
                                                sys.exit(1)
                                        zing = ">>>"

                                if mymtime != None:
                                        outfile.write("obj "+myrealdest+" "+mymd5+" "+str(mymtime)+"\n")
                                showMessage("%s %s\n" % (zing,mydest))
                        else:
                                # we are merging a fifo or device node
                                zing = "!!!"
                                if mydmode is None:
                                        # destination doesn't exist
                                        if movefile(mysrc, mydest, newmtime=thismtime, sstat=mystat, mysettings=self.settings) != None:
                                                zing = ">>>"
                                        else:
                                                sys.exit(1)
                                if stat.S_ISFIFO(mymode):
                                        outfile.write("fif %s\n" % myrealdest)
                                else:
                                        outfile.write("dev %s\n" % myrealdest)
                                showMessage(zing + " " + mydest + "\n")

        def merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0,
                mydbapi=None, prev_mtimes=None):
                """
                If portage is reinstalling itself, create temporary
                copies of PORTAGE_BIN_PATH and PORTAGE_PYM_PATH in order
                to avoid relying on the new versions which may be
                incompatible. Register an atexit hook to clean up the
                temporary directories. Pre-load elog modules here since
                we won't be able to later if they get unmerged (happens
                when namespace changes).
                """
                if self.vartree.dbapi._categories is not None:
                        self.vartree.dbapi._categories = None
                if self.myroot == "/" and \
                        "sys-apps" == self.cat and \
                        "portage" == pkgsplit(self.pkg)[0]:
                        settings = self.settings
                        base_path_orig = os.path.dirname(settings["PORTAGE_BIN_PATH"])
                        from tempfile import mkdtemp
                        import shutil
                        # Make the temp directory inside PORTAGE_TMPDIR since, unlike
                        # /tmp, it can't be mounted with the "noexec" option.
                        base_path_tmp = mkdtemp("", "._portage_reinstall_.",
                                settings["PORTAGE_TMPDIR"])
                        from portage.process import atexit_register
                        atexit_register(shutil.rmtree, base_path_tmp)
                        dir_perms = 0755
                        for subdir in "bin", "pym":
                                var_name = "PORTAGE_%s_PATH" % subdir.upper()
                                var_orig = settings[var_name]
                                var_new = os.path.join(base_path_tmp, subdir)
                                settings[var_name] = var_new
                                settings.backup_changes(var_name)
                                shutil.copytree(var_orig, var_new, symlinks=True)
                                os.chmod(var_new, dir_perms)
                        os.chmod(base_path_tmp, dir_perms)
                        # This serves so pre-load the modules.
                        elog_process(self.mycpv, self.settings,
                                phasefilter=filter_mergephases)

                return self._merge(mergeroot, inforoot,
                                myroot, myebuild=myebuild, cleanup=cleanup,
                                mydbapi=mydbapi, prev_mtimes=prev_mtimes)

        def _merge(self, mergeroot, inforoot, myroot, myebuild=None, cleanup=0,
                mydbapi=None, prev_mtimes=None):
                retval = -1
                self.lockdb()
                try:
                        retval = self.treewalk(mergeroot, myroot, inforoot, myebuild,
                                cleanup=cleanup, mydbapi=mydbapi, prev_mtimes=prev_mtimes)
                        # undo registrations of preserved libraries, bug #210501
                        if retval != os.EX_OK:
                                self.vartree.dbapi.plib_registry.unregister(self.mycpv, self.settings["SLOT"], self.settings["COUNTER"])
                        # Process ebuild logfiles
                        elog_process(self.mycpv, self.settings, phasefilter=filter_mergephases)
                        if retval == os.EX_OK and "noclean" not in self.settings.features:
                                if myebuild is None:
                                        myebuild = os.path.join(inforoot, self.pkg + ".ebuild")
                                doebuild(myebuild, "clean", myroot, self.settings,
                                        tree=self.treetype, mydbapi=mydbapi, vartree=self.vartree)
                finally:
                        self.unlockdb()
                return retval

        def getstring(self,name):
                "returns contents of a file with whitespace converted to spaces"
                if not os.path.exists(self.dbdir+"/"+name):
                        return ""
                myfile = open(self.dbdir+"/"+name,"r")
                mydata = myfile.read().split()
                myfile.close()
                return " ".join(mydata)

        def copyfile(self,fname):
                import shutil
                shutil.copyfile(fname,self.dbdir+"/"+os.path.basename(fname))

        def getfile(self,fname):
                if not os.path.exists(self.dbdir+"/"+fname):
                        return ""
                myfile = open(self.dbdir+"/"+fname,"r")
                mydata = myfile.read()
                myfile.close()
                return mydata

        def setfile(self,fname,data):
                write_atomic(os.path.join(self.dbdir, fname), data)

        def getelements(self,ename):
                if not os.path.exists(self.dbdir+"/"+ename):
                        return []
                myelement = open(self.dbdir+"/"+ename,"r")
                mylines = myelement.readlines()
                myreturn = []
                for x in mylines:
                        for y in x[:-1].split():
                                myreturn.append(y)
                myelement.close()
                return myreturn

        def setelements(self,mylist,ename):
                myelement = open(self.dbdir+"/"+ename,"w")
                for x in mylist:
                        myelement.write(x+"\n")
                myelement.close()

        def isregular(self):
                "Is this a regular package (does it have a CATEGORY file?  A dblink can be virtual *and* regular)"
                return os.path.exists(os.path.join(self.dbdir, "CATEGORY"))

def write_contents(contents, root, f):
        """
        Write contents to any file like object. The file will be left open.
        """
        root_len = len(root) - 1
        for filename in sorted(contents):
                entry_data = contents[filename]
                entry_type = entry_data[0]
                relative_filename = filename[root_len:]
                if entry_type == "obj":
                        entry_type, mtime, md5sum = entry_data
                        line = "%s %s %s %s\n" % \
                                (entry_type, relative_filename, md5sum, mtime)
                elif entry_type == "sym":
                        entry_type, mtime, link = entry_data
                        line = "%s %s -> %s %s\n" % \
                                (entry_type, relative_filename, link, mtime)
                else: # dir, dev, fif
                        line = "%s %s\n" % (entry_type, relative_filename)
                f.write(line)

def tar_contents(contents, root, tar, protect=None, onProgress=None):
        from portage.util import normalize_path
        import tarfile
        root = normalize_path(root).rstrip(os.path.sep) + os.path.sep
        id_strings = {}
        maxval = len(contents)
        curval = 0
        if onProgress:
                onProgress(maxval, 0)
        paths = contents.keys()
        paths.sort()
        for path in paths:
                curval += 1
                try:
                        lst = os.lstat(path)
                except OSError, e:
                        if e.errno != errno.ENOENT:
                                raise
                        del e
                        if onProgress:
                                onProgress(maxval, curval)
                        continue
                contents_type = contents[path][0]
                if path.startswith(root):
                        arcname = path[len(root):]
                else:
                        raise ValueError("invalid root argument: '%s'" % root)
                live_path = path
                if 'dir' == contents_type and \
                        not stat.S_ISDIR(lst.st_mode) and \
                        os.path.isdir(live_path):
                        # Even though this was a directory in the original ${D}, it exists
                        # as a symlink to a directory in the live filesystem.  It must be
                        # recorded as a real directory in the tar file to ensure that tar
                        # can properly extract it's children.
                        live_path = os.path.realpath(live_path)
                tarinfo = tar.gettarinfo(live_path, arcname)
                # store numbers instead of real names like tar's --numeric-owner
                tarinfo.uname = id_strings.setdefault(tarinfo.uid, str(tarinfo.uid))
                tarinfo.gname = id_strings.setdefault(tarinfo.gid, str(tarinfo.gid))

                if stat.S_ISREG(lst.st_mode):
                        # break hardlinks due to bug #185305
                        tarinfo.type = tarfile.REGTYPE
                        if protect and protect(path):
                                # Create an empty file as a place holder in order to avoid
                                # potential collision-protect issues.
                                tarinfo.size = 0
                                tar.addfile(tarinfo)
                        else:
                                f = open(path)
                                try:
                                        tar.addfile(tarinfo, f)
                                finally:
                                        f.close()
                else:
                        tar.addfile(tarinfo)
                if onProgress:
                        onProgress(maxval, curval)