upgradeprovision: improve info messages

[Samba/gebeck_regimport.git] / source4 / scripting / bin / upgradeprovision

#!/usr/bin/python
#
# Copyright (C) Matthieu Patou <mat@matws.net> 2009
#
# Based on provision a Samba4 server by
# Copyright (C) Jelmer Vernooij <jelmer@samba.org> 2007-2008
# Copyright (C) Andrew Bartlett <abartlet@samba.org> 2008
#
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


import getopt
import shutil
import optparse
import os
import sys
import random
import string
import re
import base64
import tempfile
# Find right directory when running from source tree
sys.path.insert(0, "bin/python")

from base64 import b64encode

import samba
from samba.credentials import DONT_USE_KERBEROS
from samba.auth import system_session, admin_session
from samba import Ldb, DS_DOMAIN_FUNCTION_2000, DS_DOMAIN_FUNCTION_2003, DS_DOMAIN_FUNCTION_2008, DS_DC_FUNCTION_2008_R2
from ldb import SCOPE_SUBTREE, SCOPE_ONELEVEL, SCOPE_BASE, LdbError
import ldb
import samba.getopt as options
from samba.samdb import SamDB
from samba import param
from samba import glue
from samba.misc import messageEltFlagToString
from samba.provision import  ProvisionNames,provision_paths_from_lp,find_setup_dir,FILL_FULL,provision, get_domain_descriptor, get_config_descriptor, secretsdb_self_join
from samba.provisionexceptions import ProvisioningError
from samba.schema import get_dnsyntax_attributes, get_linked_attributes, Schema, get_schema_descriptor
from samba.dcerpc import misc, security
from samba.ndr import ndr_pack, ndr_unpack
from samba.dcerpc.misc import SEC_CHAN_BDC

replace=2^ldb.FLAG_MOD_REPLACE
add=2^ldb.FLAG_MOD_ADD
delete=2^ldb.FLAG_MOD_DELETE

#Errors are always logged
ERROR =         -1
SIMPLE =        0x00
CHANGE =        0x01
CHANGESD =      0x02
GUESS =         0x04
PROVISION =     0x08
CHANGEALL =     0xff

# Attributes that are never copied from the reference provision (even if they
# do not exist in the destination object).
# This is most probably because they are populated automatcally when object is
# created
hashAttrNotCopied = {   "dn": 1,"whenCreated": 1,"whenChanged": 1,"objectGUID": 1,"replPropertyMetaData": 1,"uSNChanged": 1,\
                                                "uSNCreated": 1,"parentGUID": 1,"objectCategory": 1,"distinguishedName": 1,\
                                                "showInAdvancedViewOnly": 1,"instanceType": 1, "cn": 1, "msDS-Behavior-Version":1, "nextRid":1,\
                                                "nTMixedDomain": 1,"versionNumber":1, "lmPwdHistory":1, "pwdLastSet": 1, "ntPwdHistory":1, "unicodePwd":1,\
                                                "dBCSPwd":1,"supplementalCredentials":1,"gPCUserExtensionNames":1, "gPCMachineExtensionNames":1,\
                                                "maxPwdAge":1, "mail":1, "secret":1,"possibleInferiors":1, "sAMAccountType":1}

# Usually for an object that already exists we do not overwrite attributes as
# they might have been changed for good reasons. Anyway for a few of them it's
# mandatory to replace them otherwise the provision will be broken somehow.
hashOverwrittenAtt = {  "prefixMap": replace, "systemMayContain": replace,"systemOnly":replace, "searchFlags":replace,\
                        "mayContain":replace,  "systemFlags":replace,"description":replace,
                        "oEMInformation":replace, "operatingSystemVersion":replace, "adminPropertyPages":replace,
                        "defaultSecurityDescriptor": replace,"wellKnownObjects":replace,"privilege":delete}
backlinked = []

def define_what_to_log(opts):
        what = 0
        if opts.debugchange:
                what = what | CHANGE
        if opts.debugchangesd:
                what = what | CHANGESD
        if opts.debugguess:
                what = what | GUESS
        if opts.debugprovision:
                what = what | PROVISION
        if opts.debugall:
                what = what | CHANGEALL
        return what


parser = optparse.OptionParser("provision [options]")
sambaopts = options.SambaOptions(parser)
parser.add_option_group(sambaopts)
parser.add_option_group(options.VersionOptions(parser))
credopts = options.CredentialsOptions(parser)
parser.add_option_group(credopts)
parser.add_option("--setupdir", type="string", metavar="DIR",
                                        help="directory with setup files")
parser.add_option("--debugprovision", help="Debug provision", action="store_true")
parser.add_option("--debugguess", help="Print information on what is different but won't be changed", action="store_true")
parser.add_option("--debugchange", help="Print information on what is different but won't be changed", action="store_true")
parser.add_option("--debugchangesd", help="Print information security descriptors differences", action="store_true")
parser.add_option("--debugall", help="Print all available information (very verbose)", action="store_true")
parser.add_option("--full", help="Perform full upgrade of the samdb (schema, configuration, new objects, ...", action="store_true")

opts = parser.parse_args()[0]

whatToLog = define_what_to_log(opts)

def messageprovision(text):
        """print a message if quiet is not set."""
        if opts.debugprovision or opts.debugall:
                print text

def message(what,text):
        """print a message if quiet is not set."""
        if (whatToLog & what) or (what <= 0 ):
                print text

if len(sys.argv) == 1:
        opts.interactive = True
lp = sambaopts.get_loadparm()
smbconf = lp.configfile

creds = credopts.get_credentials(lp)
creds.set_kerberos_state(DONT_USE_KERBEROS)
setup_dir = opts.setupdir
if setup_dir is None:
    setup_dir = find_setup_dir()

session = system_session()

# simple helper to allow back and forth rename
def identic_rename(ldbobj,dn):
        (before,sep,after)=str(dn).partition('=')
        ldbobj.rename(dn,ldb.Dn(ldbobj,"%s=foo%s"%(before,after)))
        ldbobj.rename(ldb.Dn(ldbobj,"%s=foo%s"%(before,after)),dn)

# Create an array of backlinked attributes
def populate_backlink(newpaths,creds,session,schemadn):
        newsam_ldb = Ldb(newpaths.samdb, session_info=session, credentials=creds,lp=lp)
        backlinked.extend(get_linked_attributes(ldb.Dn(newsam_ldb,str(schemadn)),newsam_ldb).values())

# Get Paths for important objects (ldb, keytabs ...)
def get_paths(targetdir=None,smbconf=None):
        if targetdir is not None:
                if (not os.path.exists(os.path.join(targetdir, "etc"))):
                        os.makedirs(os.path.join(targetdir, "etc"))
                smbconf = os.path.join(targetdir, "etc", "smb.conf")
        if smbconf is None:
                        smbconf = param.default_path()

        if not os.path.exists(smbconf):
                message(ERROR,"Unable to find smb.conf ..")
                parser.print_usage()
                sys.exit(1)

        lp = param.LoadParm()
        lp.load(smbconf)
# Normally we need the domain name for this function but for our needs it's
# pointless
        paths = provision_paths_from_lp(lp,"foo")
        return paths

# This function guesses (fetches) informations needed to make a fresh provision
# from the current provision
# It includes: realm, workgroup, partitions, netbiosname, domain guid, ...
def guess_names_from_current_provision(credentials,session_info,paths):
        lp = param.LoadParm()
        lp.load(paths.smbconf)
        names = ProvisionNames()
        # NT domain, kerberos realm, root dn, domain dn, domain dns name
        names.domain = string.upper(lp.get("workgroup"))
        names.realm = lp.get("realm")
        basedn = "DC=" + names.realm.replace(".",",DC=")
        names.dnsdomain = names.realm
        names.realm = string.upper(names.realm)
        # netbiosname
        secrets_ldb = Ldb(paths.secrets, session_info=session_info, credentials=credentials,lp=lp, options=["modules:samba_secrets"])
        # Get the netbiosname first (could be obtained from smb.conf in theory)
        attrs = ["sAMAccountName"]
        res = secrets_ldb.search(expression="(flatname=%s)"%names.domain,base="CN=Primary Domains", scope=SCOPE_SUBTREE, attrs=attrs)
        names.netbiosname = str(res[0]["sAMAccountName"]).replace("$","")

        names.smbconf = smbconf
        # It's important here to let ldb load with the old module or it's quite
        # certain that the LDB won't load ...
        samdb = Ldb(paths.samdb, session_info=session_info,
                    credentials=credentials, lp=lp, options=["modules:samba_dsdb"])

        # That's a bit simplistic but it's ok as long as we have only 3
        # partitions
        attrs2 = ["defaultNamingContext", "schemaNamingContext","configurationNamingContext","rootDomainNamingContext"]
        current = samdb.search(expression="(objectClass=*)",base="", scope=SCOPE_BASE, attrs=attrs2)

        names.configdn = current[0]["configurationNamingContext"]
        configdn = str(names.configdn)
        names.schemadn = current[0]["schemaNamingContext"]
        if not (ldb.Dn(samdb, basedn) == (ldb.Dn(samdb, current[0]["defaultNamingContext"][0]))):
                raise ProvisioningError(("basedn in %s (%s) and from %s (%s) is not the same ..." % (paths.samdb, str(current[0]["defaultNamingContext"][0]), paths.smbconf, basedn)))

        names.domaindn=current[0]["defaultNamingContext"]
        names.rootdn=current[0]["rootDomainNamingContext"]
        # default site name
        attrs3 = ["cn"]
        res3= samdb.search(expression="(objectClass=*)",base="CN=Sites,"+configdn, scope=SCOPE_ONELEVEL, attrs=attrs3)
        names.sitename = str(res3[0]["cn"])

        # dns hostname and server dn
        attrs4 = ["dNSHostName"]
        res4= samdb.search(expression="(CN=%s)"%names.netbiosname,base="OU=Domain Controllers,"+basedn, \
                                                scope=SCOPE_ONELEVEL, attrs=attrs4)
        names.hostname = str(res4[0]["dNSHostName"]).replace("."+names.dnsdomain,"")

        server_res = samdb.search(expression="serverReference=%s"%res4[0].dn, attrs=[], base=configdn)
        names.serverdn = server_res[0].dn

        # invocation id/objectguid
        res5 = samdb.search(expression="(objectClass=*)",base="CN=NTDS Settings,%s" % str(names.serverdn), scope=SCOPE_BASE, attrs=["invocationID","objectGUID"])
        names.invocation = str(ndr_unpack( misc.GUID,res5[0]["invocationId"][0]))
        names.ntdsguid = str(ndr_unpack( misc.GUID,res5[0]["objectGUID"][0]))

        # domain guid/sid
        attrs6 = ["objectGUID", "objectSid","msDS-Behavior-Version" ]
        res6 = samdb.search(expression="(objectClass=*)",base=basedn, scope=SCOPE_BASE, attrs=attrs6)
        names.domainguid = str(ndr_unpack( misc.GUID,res6[0]["objectGUID"][0]))
        names.domainsid = ndr_unpack( security.dom_sid,res6[0]["objectSid"][0])
        if res6[0].get("msDS-Behavior-Version") == None or int(res6[0]["msDS-Behavior-Version"][0]) < DS_DOMAIN_FUNCTION_2000:
                names.domainlevel = DS_DOMAIN_FUNCTION_2000
        else:
                names.domainlevel = int(res6[0]["msDS-Behavior-Version"][0])

        # policy guid
        attrs7 = ["cn","displayName"]
        res7 = samdb.search(expression="(displayName=Default Domain Policy)",base="CN=Policies,CN=System,"+basedn, \
                                                        scope=SCOPE_ONELEVEL, attrs=attrs7)
        names.policyid = str(res7[0]["cn"]).replace("{","").replace("}","")
        # dc policy guid
        attrs8 = ["cn","displayName"]
        res8 = samdb.search(expression="(displayName=Default Domain Controllers Policy)",base="CN=Policies,CN=System,"+basedn, \
                                                        scope=SCOPE_ONELEVEL, attrs=attrs7)
        if len(res8) == 1:
                names.policyid_dc = str(res8[0]["cn"]).replace("{","").replace("}","")
        else:
                names.policyid_dc = None


        return names

# Debug a little bit
def print_names(names):
        message(GUESS, "rootdn      :"+str(names.rootdn))
        message(GUESS, "configdn    :"+str(names.configdn))
        message(GUESS, "schemadn    :"+str(names.schemadn))
        message(GUESS, "serverdn    :"+str(names.serverdn))
        message(GUESS, "netbiosname :"+names.netbiosname)
        message(GUESS, "defaultsite :"+names.sitename)
        message(GUESS, "dnsdomain   :"+names.dnsdomain)
        message(GUESS, "hostname    :"+names.hostname)
        message(GUESS, "domain      :"+names.domain)
        message(GUESS, "realm       :"+names.realm)
        message(GUESS, "invocationid:"+names.invocation)
        message(GUESS, "policyguid  :"+names.policyid)
        message(GUESS, "policyguiddc:"+str(names.policyid_dc))
        message(GUESS, "domainsid   :"+str(names.domainsid))
        message(GUESS, "domainguid  :"+names.domainguid)
        message(GUESS, "ntdsguid    :"+names.ntdsguid)
        message(GUESS, "domainlevel :"+str(names.domainlevel))

# Create a fresh new reference provision
# This provision will be the reference for knowing what has changed in the
# since the latest upgrade in the current provision
def newprovision(names,setup_dir,creds,session,smbconf):
        message(SIMPLE, "Creating a reference provision")
        provdir=tempfile.mkdtemp(dir=paths.private_dir, prefix="referenceprovision")
        if os.path.isdir(provdir):
                rmall(provdir)
        logstd=os.path.join(provdir,"log.std")
        os.chdir(os.path.join(setup_dir,".."))
        os.mkdir(provdir)
        os.close(2)
        sys.stderr = open("%s/provision.log"%provdir, "w")
        message(PROVISION, "Reference provision stored in %s"%provdir)
        message(PROVISION, "STDERR message of provision will be logged in %s/provision.log"%provdir)
        sys.stderr = open("/dev/stdout", "w")
        provision(setup_dir, messageprovision,
                session, creds, smbconf=smbconf, targetdir=provdir,
                samdb_fill=FILL_FULL, realm=names.realm, domain=names.domain,
                domainguid=names.domainguid, domainsid=str(names.domainsid),ntdsguid=names.ntdsguid,
                policyguid=names.policyid,policyguid_dc=names.policyid_dc,hostname=names.netbiosname,
                hostip=None, hostip6=None,
                invocationid=names.invocation, adminpass=None,
                krbtgtpass=None, machinepass=None,
                dnspass=None, root=None, nobody=None,
                wheel=None, users=None,
                serverrole="domain controller",
                ldap_backend_extra_port=None,
                backend_type=None,
                ldapadminpass=None,
                ol_mmr_urls=None,
                slapd_path=None,
                setup_ds_path=None,
                nosync=None,
                dom_for_fun_level=names.domainlevel,
                ldap_dryrun_mode=None,useeadb=True)
        return provdir

# This function sorts two DNs in the lexicographical order and put higher level
# DN before.
# So given the dns cn=bar,cn=foo and cn=foo the later will be return as smaller
# (-1) as it has less level
def dn_sort(x,y):
        p = re.compile(r'(?<!\\),')
        tab1 = p.split(str(x))
        tab2 = p.split(str(y))
        min = 0
        if (len(tab1) > len(tab2)):
                min = len(tab2)
        elif (len(tab1) < len(tab2)):
                min = len(tab1)
        else:
                min = len(tab1)
        len1=len(tab1)-1
        len2=len(tab2)-1
        space = " "
        # Note: python range go up to upper limit but do not include it
        for i in range(0,min):
                ret=cmp(tab1[len1-i],tab2[len2-i])
                if(ret != 0):
                        return ret
                else:
                        if(i==min-1):
                                if(len1==len2):
                                        message(ERROR,"PB PB PB"+space.join(tab1)+" / "+space.join(tab2))
                                if(len1>len2):
                                        return 1
                                else:
                                        return -1
        return ret

# Check for security descriptors modifications return 1 if it is and 0 otherwise
# it also populate hash structure for later use in the upgrade process
def handle_security_desc(ischema,att,msgElt,hashallSD,old,new):
        if ischema == 1 and att == "defaultSecurityDescriptor"  and msgElt.flags() == ldb.FLAG_MOD_REPLACE:
                hashSD = {}
                hashSD["oldSD"] = old[0][att]
                hashSD["newSD"] = new[0][att]
                hashallSD[str(old[0].dn)] = hashSD
                return 0
        if att == "nTSecurityDescriptor"  and msgElt.flags() == ldb.FLAG_MOD_REPLACE:
                if ischema == 0:
                        hashSD = {}
                        hashSD["oldSD"] =  ndr_unpack(security.descriptor,str(old[0][att]))
                        hashSD["newSD"] =  ndr_unpack(security.descriptor,str(new[0][att]))
                        hashallSD[str(old[0].dn)] = hashSD
                return 1
        return 0

# Handle special cases ... That's when we want to update a particular attribute
# only, e.g. if it has a certain value or if it's for a certain object or
# a class of object.
# It can be also if we want to do a merge of value instead of a simple replace
def handle_special_case(att,delta,new,old,ischema):
        flag = delta.get(att).flags()
        if (att == "gPLink" or att == "gPCFileSysPath") and flag ==  ldb.FLAG_MOD_REPLACE and str(new[0].dn).lower() == str(old[0].dn).lower():
                delta.remove(att)
                return 1
        if att == "forceLogoff":
                ref=0x8000000000000000
                oldval=int(old[0][att][0])
                newval=int(new[0][att][0])
                ref == old and ref == abs(new)
                return 1
        if (att == "adminDisplayName" or att == "adminDescription") and ischema:
                return 1
        if (str(old[0].dn) == "CN=Samba4-Local-Domain,%s"%(str(names.schemadn)) and att == "defaultObjectCategory" and flag  == ldb.FLAG_MOD_REPLACE):
                return 1
#       if (str(old[0].dn) == "CN=S-1-5-11,CN=ForeignSecurityPrincipals,%s"%(str(names.rootdn)) and att == "description" and flag  == ldb.FLAG_MOD_DELETE):
#               return 1
        if (str(old[0].dn) == "CN=Title,%s"%(str(names.schemadn)) and att == "rangeUpper" and flag  == ldb.FLAG_MOD_REPLACE):
                return 1
        if ( (att == "member" or att == "servicePrincipalName") and flag  == ldb.FLAG_MOD_REPLACE):

                hash = {}
                newval = []
                changeDelta=0
                for elem in old[0][att]:
                        hash[str(elem)]=1
                        newval.append(str(elem))

                for elem in new[0][att]:
                        if not hash.has_key(str(elem)):
                                changeDelta=1
                                newval.append(str(elem))
                if changeDelta == 1:
                        delta[att] = ldb.MessageElement(newval, ldb.FLAG_MOD_REPLACE, att)
                else:
                        delta.remove(att)
                return 1
        if (str(old[0].dn) == "%s"%(str(names.rootdn)) and att == "subRefs" and flag  == ldb.FLAG_MOD_REPLACE):
                return 1
        if str(delta.dn).endswith("CN=DisplaySpecifiers,%s"%names.configdn):
                return 1
        return 0

def update_secrets(newpaths,paths,creds,session):
        message(SIMPLE,"update secrets.ldb")
        newsecrets_ldb = Ldb(newpaths.secrets, session_info=session, credentials=creds,lp=lp)
        secrets_ldb = Ldb(paths.secrets, session_info=session, credentials=creds,lp=lp, options=["modules:samba_secrets"])
        reference = newsecrets_ldb.search(expression="dn=@MODULES",base="", scope=SCOPE_SUBTREE)
        current = secrets_ldb.search(expression="dn=@MODULES",base="", scope=SCOPE_SUBTREE)
        delta = secrets_ldb.msg_diff(current[0],reference[0])
        delta.dn = current[0].dn
        secrets_ldb.modify(delta)

        newsecrets_ldb = Ldb(newpaths.secrets, session_info=session, credentials=creds,lp=lp)
        secrets_ldb = Ldb(paths.secrets, session_info=session, credentials=creds,lp=lp)
        reference = newsecrets_ldb.search(expression="objectClass=top",base="", scope=SCOPE_SUBTREE,attrs=["dn"])
        current = secrets_ldb.search(expression="objectClass=top",base="", scope=SCOPE_SUBTREE,attrs=["dn"])
        hash_new = {}
        hash = {}
        listMissing = []
        listPresent = []

        empty = ldb.Message()
        for i in range(0,len(reference)):
                hash_new[str(reference[i]["dn"]).lower()] = reference[i]["dn"]

        # Create a hash for speeding the search of existing object in the
        # current provision
        for i in range(0,len(current)):
                hash[str(current[i]["dn"]).lower()] = current[i]["dn"]

        for k in hash_new.keys():
                if not hash.has_key(k):
                        listMissing.append(hash_new[k])
                else:
                        listPresent.append(hash_new[k])
        for entry in listMissing:
                reference = newsecrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
                current = secrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
                delta = secrets_ldb.msg_diff(empty,reference[0])
                for att in hashAttrNotCopied.keys():
                        delta.remove(att)
                message(CHANGE,"Entry %s is missing from secrets.ldb"%reference[0].dn)
                for att in delta:
                        message(CHANGE," Adding attribute %s"%att)
                delta.dn = reference[0].dn
                secrets_ldb.add(delta)

        for entry in listPresent:
                reference = newsecrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
                current = secrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
                delta = secrets_ldb.msg_diff(current[0],reference[0])
                i=0
                for att in hashAttrNotCopied.keys():
                        delta.remove(att)
                for att in delta:
                        i = i + 1

                        if att == "name":
                                message(CHANGE,"Found attribute name on  %s, must rename the DN "%(current[0].dn))
                                identic_rename(secrets_ldb,reference[0].dn)
                        else:
                                delta.remove(att)


        for entry in listPresent:
                reference = newsecrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
                current = secrets_ldb.search(expression="dn=%s"%entry,base="", scope=SCOPE_SUBTREE)
                delta = secrets_ldb.msg_diff(current[0],reference[0])
                i=0
                for att in hashAttrNotCopied.keys():
                        delta.remove(att)
                for att in delta:
                        i = i + 1
                        if att != "dn":
                                message(CHANGE," Adding/Changing attribute %s to %s"%(att,current[0].dn))

                delta.dn = current[0].dn
                secrets_ldb.modify(delta)


# Check difference between the current provision and the reference provision.
# It looks for all objects which base DN is name. If ischema is "false" then
# the scan is done in cross partition mode.
# If "ischema" is true, then special handling is done for dealing with schema
def check_diff_name(newpaths,paths,creds,session,basedn,names,ischema):
        hash_new = {}
        hash = {}
        hashallSD = {}
        listMissing = []
        listPresent = []
        reference = []
        current = []
        # Connect to the reference provision and get all the attribute in the
        # partition referred by name
        newsam_ldb = Ldb(newpaths.samdb, session_info=session, credentials=creds,lp=lp)
        sam_ldb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp, options=["modules:samba_dsdb"])
        sam_ldb.transaction_start()
        if ischema:
                reference = newsam_ldb.search(expression="objectClass=*",base=basedn, scope=SCOPE_SUBTREE,attrs=["dn"])
                current = sam_ldb.search(expression="objectClass=*",base=basedn, scope=SCOPE_SUBTREE,attrs=["dn"])
        else:
                reference = newsam_ldb.search(expression="objectClass=*",base=basedn, scope=SCOPE_SUBTREE,attrs=["dn"],controls=["search_options:1:2"])
                current = sam_ldb.search(expression="objectClass=*",base=basedn, scope=SCOPE_SUBTREE,attrs=["dn"],controls=["search_options:1:2"])

        sam_ldb.transaction_commit()
        # Create a hash for speeding the search of new object
        for i in range(0,len(reference)):
                hash_new[str(reference[i]["dn"]).lower()] = reference[i]["dn"]

        # Create a hash for speeding the search of existing object in the
        # current provision
        for i in range(0,len(current)):
                hash[str(current[i]["dn"]).lower()] = current[i]["dn"]

        for k in hash_new.keys():
                if not hash.has_key(k):
                        listMissing.append(hash_new[k])
                else:
                        listPresent.append(hash_new[k])

        # Sort the missing object in order to have object of the lowest level
        # first (which can be containers for higher level objects)
        listMissing.sort(dn_sort)
        listPresent.sort(dn_sort)

        if ischema:
                # The following lines (up to the for loop) is to load the up to
                # date schema into our current LDB
                # a complete schema is needed as the insertion of attributes
                # and class is done against it
                # and the schema is self validated
                # The double ldb open and schema validation is taken from the
                # initial provision script
                # it's not certain that it is really needed ....
                sam_ldb = Ldb(session_info=session, credentials=creds, lp=lp)
                schema = Schema(setup_path, names.domainsid, schemadn=basedn, serverdn=str(names.serverdn))
                # Load the schema from the one we computed earlier
                sam_ldb.set_schema_from_ldb(schema.ldb)
                # And now we can connect to the DB - the schema won't be loaded
                # from the DB
                sam_ldb.connect(paths.samdb)
        else:
                sam_ldb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp, options=["modules:samba_dsdb"])

        sam_ldb.transaction_start()

        empty = ldb.Message()
        message(SIMPLE,"There are %d missing objects"%(len(listMissing)))
        for dn in listMissing:
                reference = newsam_ldb.search(expression="dn=%s"%(str(dn)),base=basedn, scope=SCOPE_SUBTREE,controls=["search_options:1:2"])
                delta = sam_ldb.msg_diff(empty,reference[0])
                for att in hashAttrNotCopied.keys():
                        delta.remove(att)
                for att in backlinked:
                        delta.remove(att)
                delta.dn = dn

                sam_ldb.add(delta,["relax:0"])

        changed = 0
        for dn in listPresent:
                reference = newsam_ldb.search(expression="dn=%s"%(str(dn)),base=basedn, scope=SCOPE_SUBTREE,controls=["search_options:1:2"])
                current = sam_ldb.search(expression="dn=%s"%(str(dn)),base=basedn, scope=SCOPE_SUBTREE,controls=["search_options:1:2"])
                if ((str(current[0].dn) != str(reference[0].dn)) and (str(current[0].dn).upper() == str(reference[0].dn).upper())):
                        message(CHANGE,"Name are the same but case change, let's rename %s to %s"%(str(current[0].dn),str(reference[0].dn)))
                        identic_rename(sam_ldb,reference[0].dn)
                        current = sam_ldb.search(expression="dn=%s"%(str(dn)),base=basedn, scope=SCOPE_SUBTREE,controls=["search_options:1:2"])

                delta = sam_ldb.msg_diff(current[0],reference[0])
                for att in hashAttrNotCopied.keys():
                        delta.remove(att)
                for att in backlinked:
                        delta.remove(att)
                delta.remove("parentGUID")
                nb = 0
                
                for att in delta:
                        msgElt = delta.get(att)
                        if att == "dn":
                                continue
                        if att == "name":
                                delta.remove(att)
                                continue
                        if handle_security_desc(ischema,att,msgElt,hashallSD,current,reference):
                                delta.remove(att)
                                continue
                        if (not hashOverwrittenAtt.has_key(att) or not (hashOverwrittenAtt.get(att)&2^msgElt.flags())):
                                if  handle_special_case(att,delta,reference,current,ischema)==0 and msgElt.flags()!=ldb.FLAG_MOD_ADD:
                                        i = 0
                                        if opts.debugchange:
                                                try:
                                                        dump_denied_change(dn,att,messageEltFlagToString(msgElt.flags()),current[0][att],reference[0][att])
                                                except:
                                                        dump_denied_change(dn,att,messageEltFlagToString(msgElt.flags()),current[0][att],None)
                                        delta.remove(att)
                delta.dn = dn
                if len(delta.items()) >1:
                        attributes=",".join(delta.keys())
                        message(CHANGE,"%s is different from the reference one, changed attributes: %s"%(dn,attributes))
                        changed = changed + 1
                        sam_ldb.modify(delta)

        sam_ldb.transaction_commit()
        message(SIMPLE,"There are %d changed objects"%(changed))
        return hashallSD

# Check that SD are correct
def check_updated_sd(newpaths,paths,creds,session,names):
        newsam_ldb = Ldb(newpaths.samdb, session_info=session, credentials=creds,lp=lp)
        sam_ldb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp)
        reference = newsam_ldb.search(expression="objectClass=*",base=str(names.rootdn), scope=SCOPE_SUBTREE,attrs=["dn","nTSecurityDescriptor"],controls=["search_options:1:2"])
        current = sam_ldb.search(expression="objectClass=*",base=str(names.rootdn), scope=SCOPE_SUBTREE,attrs=["dn","nTSecurityDescriptor"],controls=["search_options:1:2"])
        hash_new = {}
        for i in range(0,len(reference)):
                hash_new[str(reference[i]["dn"]).lower()] = ndr_unpack(security.descriptor,str(reference[i]["nTSecurityDescriptor"])).as_sddl(names.domainsid)

        for i in range(0,len(current)):
                key = str(current[i]["dn"]).lower()
                if hash_new.has_key(key):
                        sddl = ndr_unpack(security.descriptor,str(current[i]["nTSecurityDescriptor"])).as_sddl(names.domainsid)
                        if sddl != hash_new[key]:
                                print "%s new sddl/sddl in ref"%key
                                print "%s\n%s"%(sddl,hash_new[key])

# Simple update method for updating the SD that rely on the fact that nobody
# should have modified the SD
# This assumption is safe right now (alpha9) but should be removed asap
def update_sd(paths,creds,session,names):
        sam_ldb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp,options=["modules:samba_dsdb"])
        sam_ldb.transaction_start()
        # First update the SD for the rootdn
        sam_ldb.set_session_info(session)
        res = sam_ldb.search(expression="objectClass=*",base=str(names.rootdn), scope=SCOPE_BASE,attrs=["dn","whenCreated"],controls=["search_options:1:2"])
        delta = ldb.Message()
        delta.dn = ldb.Dn(sam_ldb,str(res[0]["dn"]))
        descr = get_domain_descriptor(names.domainsid)
        delta["nTSecurityDescriptor"] = ldb.MessageElement( descr,ldb.FLAG_MOD_REPLACE,"nTSecurityDescriptor" )
        sam_ldb.modify(delta,["recalculate_sd:0"])
        # Then the config dn
        res = sam_ldb.search(expression="objectClass=*",base=str(names.configdn), scope=SCOPE_BASE,attrs=["dn","whenCreated"],controls=["search_options:1:2"])
        delta = ldb.Message()
        delta.dn = ldb.Dn(sam_ldb,str(res[0]["dn"]))
        descr = get_config_descriptor(names.domainsid)
        delta["nTSecurityDescriptor"] = ldb.MessageElement( descr,ldb.FLAG_MOD_REPLACE,"nTSecurityDescriptor" )
        sam_ldb.modify(delta,["recalculate_sd:0"])
        # Then the schema dn
        res = sam_ldb.search(expression="objectClass=*",base=str(names.schemadn), scope=SCOPE_BASE,attrs=["dn","whenCreated"],controls=["search_options:1:2"])
        delta = ldb.Message()
        delta.dn = ldb.Dn(sam_ldb,str(res[0]["dn"]))
        descr = get_schema_descriptor(names.domainsid)
        delta["nTSecurityDescriptor"] = ldb.MessageElement( descr,ldb.FLAG_MOD_REPLACE,"nTSecurityDescriptor" )
        sam_ldb.modify(delta,["recalculate_sd:0"])

        # Then the rest
        hash = {}
        res = sam_ldb.search(expression="objectClass=*",base=str(names.rootdn), scope=SCOPE_SUBTREE,attrs=["dn","whenCreated"],controls=["search_options:1:2"])
        for obj in res:
                if not (str(obj["dn"]) == str(names.rootdn) or
                        str(obj["dn"]) == str(names.configdn) or \
                        str(obj["dn"]) == str(names.schemadn)):
                        hash[str(obj["dn"])] = obj["whenCreated"]

        listkeys = hash.keys()
        listkeys.sort(dn_sort)

        for key in listkeys:
                try:
                        delta = ldb.Message()
                        delta.dn = ldb.Dn(sam_ldb,key)
                        delta["whenCreated"] = ldb.MessageElement( hash[key],ldb.FLAG_MOD_REPLACE,"whenCreated" )
                        sam_ldb.modify(delta,["recalculate_sd:0"])
                except:
                        sam_ldb.transaction_cancel()
                        res = sam_ldb.search(expression="objectClass=*",base=str(names.rootdn), scope=SCOPE_SUBTREE,attrs=["dn","nTSecurityDescriptor"],controls=["search_options:1:2"])
                        print "bad stuff" +ndr_unpack(security.descriptor,str(res[0]["nTSecurityDescriptor"])).as_sddl(names.domainsid)
                        return
        sam_ldb.transaction_commit()

def rmall(topdir):
        for root, dirs, files in os.walk(topdir, topdown=False):
                for name in files:
                        os.remove(os.path.join(root, name))
                for name in dirs:
                        os.rmdir(os.path.join(root, name))
        os.rmdir(topdir)


def update_basesamdb(newpaths,paths,names):
        message(SIMPLE,"Copy samdb")
        shutil.copy(newpaths.samdb,paths.samdb)

        message(SIMPLE,"Update partitions filename if needed")
        schemaldb=os.path.join(paths.private_dir,"schema.ldb")
        configldb=os.path.join(paths.private_dir,"configuration.ldb")
        usersldb=os.path.join(paths.private_dir,"users.ldb")
        samldbdir=os.path.join(paths.private_dir,"sam.ldb.d")

        if not os.path.isdir(samldbdir):
                os.mkdir(samldbdir)
                os.chmod(samldbdir,0700)
        if os.path.isfile(schemaldb):
                shutil.copy(schemaldb,os.path.join(samldbdir,"%s.ldb"%str(names.schemadn).upper()))
                os.remove(schemaldb)
        if os.path.isfile(usersldb):
                shutil.copy(usersldb,os.path.join(samldbdir,"%s.ldb"%str(names.rootdn).upper()))
                os.remove(usersldb)
        if os.path.isfile(configldb):
                shutil.copy(configldb,os.path.join(samldbdir,"%s.ldb"%str(names.configdn).upper()))
                os.remove(configldb)

def update_privilege(newpaths,paths):
        message(SIMPLE,"Copy privilege")
        shutil.copy(os.path.join(newpaths.private_dir,"privilege.ldb"),os.path.join(paths.private_dir,"privilege.ldb"))

# For each partition check the differences
def update_samdb(newpaths,paths,creds,session,names):

        message(SIMPLE, "Doing schema update")
        hashdef = check_diff_name(newpaths,paths,creds,session,str(names.schemadn),names,1)
        message(SIMPLE,"Done with schema update")
        message(SIMPLE,"Scanning whole provision for updates and additions")
        hashSD = check_diff_name(newpaths,paths,creds,session,str(names.rootdn),names,0)
        message(SIMPLE,"Done with scanning")

def update_machine_account_password(paths,creds,session,names):

        secrets_ldb = Ldb(paths.secrets, session_info=session, credentials=creds,lp=lp)
        secrets_ldb.transaction_start()
        secrets_msg = secrets_ldb.search(expression=("samAccountName=%s$" % names.netbiosname), attrs=["secureChannelType"])
        sam_ldb = Ldb(paths.samdb, session_info=session, credentials=creds,lp=lp)
        sam_ldb.transaction_start()
        if int(secrets_msg[0]["secureChannelType"][0]) == SEC_CHAN_BDC:
                res = sam_ldb.search(expression=("samAccountName=%s$" % names.netbiosname), attrs=[])
                assert(len(res) == 1)

                msg = ldb.Message(res[0].dn)
                machinepass = glue.generate_random_str(12)
                msg["userPassword"] = ldb.MessageElement(machinepass, ldb.FLAG_MOD_REPLACE, "userPassword")
                sam_ldb.modify(msg)

                res = sam_ldb.search(expression=("samAccountName=%s$" % names.netbiosname),
                                     attrs=["msDs-keyVersionNumber"])
                assert(len(res) == 1)
                kvno = int(str(res[0]["msDs-keyVersionNumber"]))

                secretsdb_self_join(secrets_ldb, domain=names.domain,
                                    realm=names.realm,
                                    domainsid=names.domainsid,
                                    dnsdomain=names.dnsdomain,
                                    netbiosname=names.netbiosname,
                                    machinepass=machinepass,
                                    key_version_number=kvno,
                                    secure_channel_type=int(secrets_msg[0]["secureChannelType"][0]))
                sam_ldb.transaction_prepare_commit()
                secrets_ldb.transaction_prepare_commit()
                sam_ldb.transaction_commit()
                secrets_ldb.transaction_commit()
        else:
                secrets_ldb.transaction_cancel()

# From here start the big steps of the program
# First get files paths
paths=get_paths(smbconf=smbconf)
paths.setup = setup_dir
def setup_path(file):
        return os.path.join(setup_dir, file)
# Guess all the needed names (variables in fact) from the current
# provision.
names = guess_names_from_current_provision(creds,session,paths)
# Let's see them
print_names(names)
# With all this information let's create a fresh new provision used as reference
provisiondir = newprovision(names,setup_dir,creds,session,smbconf)
# Get file paths of this new provision
newpaths = get_paths(targetdir=provisiondir)
populate_backlink(newpaths,creds,session,names.schemadn)
# Check the difference
update_basesamdb(newpaths,paths,names)

if opts.full:
        update_samdb(newpaths,paths,creds,session,names)
update_secrets(newpaths,paths,creds,session)
update_privilege(newpaths,paths)
update_machine_account_password(paths,creds,session,names)
# SD should be created with admin but as some previous acl were so wrong that admin can't modify them we have first
# to recreate them with the good form but with system account and then give the ownership to admin ...
admin_session_info = admin_session(lp, str(names.domainsid))
message(SIMPLE,"Updating SD")
update_sd(paths,creds,session,names)
update_sd(paths,creds,admin_session_info,names)
check_updated_sd(newpaths,paths,creds,session,names)
message(SIMPLE,"Upgrade finished !")
# remove reference provision now that everything is done !
rmall(provisiondir)