python/samba/dbchecker.py

   1 # Samba4 AD database checker
   2 #
   3 # Copyright (C) Andrew Tridgell 2011
   4 # Copyright (C) Matthieu Patou <mat@matws.net> 2011
   5 #
   6 # This program is free software; you can redistribute it and/or modify
   7 # it under the terms of the GNU General Public License as published by
   8 # the Free Software Foundation; either version 3 of the License, or
   9 # (at your option) any later version.
  10 #
  11 # This program is distributed in the hope that it will be useful,
  12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14 # GNU General Public License for more details.
  15 #
  16 # You should have received a copy of the GNU General Public License
  17 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18 #
  19
  20 import ldb
  21 import samba
  22 import time
  23 from base64 import b64decode
  24 from samba import dsdb
  25 from samba import common
  26 from samba.dcerpc import misc
  27 from samba.dcerpc import drsuapi
  28 from samba.ndr import ndr_unpack, ndr_pack
  29 from samba.dcerpc import drsblobs
  30 from samba.common import dsdb_Dn
  31 from samba.dcerpc import security
  32 from samba.descriptor import get_wellknown_sds, get_diff_sds
  33 from samba.auth import system_session, admin_session
  34
  35
  36 class dbcheck(object):
  37     """check a SAM database for errors"""
  38
  39     def __init__(self, samdb, samdb_schema=None, verbose=False, fix=False,
  40                  yes=False, quiet=False, in_transaction=False,
  41                  reset_well_known_acls=False):
  42         self.samdb = samdb
  43         self.dict_oid_name = None
  44         self.samdb_schema = (samdb_schema or samdb)
  45         self.verbose = verbose
  46         self.fix = fix
  47         self.yes = yes
  48         self.quiet = quiet
  49         self.remove_all_unknown_attributes = False
  50         self.remove_all_empty_attributes = False
  51         self.fix_all_normalisation = False
  52         self.fix_all_duplicates = False
  53         self.fix_all_DN_GUIDs = False
  54         self.fix_all_binary_dn = False
  55         self.remove_all_deleted_DN_links = False
  56         self.fix_all_string_dn_component_mismatch = False
  57         self.fix_all_GUID_dn_component_mismatch = False
  58         self.fix_all_SID_dn_component_mismatch = False
  59         self.fix_all_metadata = False
  60         self.fix_time_metadata = False
  61         self.fix_undead_linked_attributes = False
  62         self.fix_all_missing_backlinks = False
  63         self.fix_all_orphaned_backlinks = False
  64         self.fix_rmd_flags = False
  65         self.fix_ntsecuritydescriptor = False
  66         self.fix_ntsecuritydescriptor_owner_group = False
  67         self.seize_fsmo_role = False
  68         self.move_to_lost_and_found = False
  69         self.fix_instancetype = False
  70         self.fix_replmetadata_zero_invocationid = False
  71         self.fix_replmetadata_duplicate_attid = False
  72         self.fix_replmetadata_wrong_attid = False
  73         self.fix_replmetadata_unsorted_attid = False
  74         self.fix_deleted_deleted_objects = False
  75         self.fix_incorrect_deleted_objects = False
  76         self.fix_dn = False
  77         self.fix_base64_userparameters = False
  78         self.fix_utf8_userparameters = False
  79         self.fix_doubled_userparameters = False
  80         self.reset_well_known_acls = reset_well_known_acls
  81         self.reset_all_well_known_acls = False
  82         self.in_transaction = in_transaction
  83         self.infrastructure_dn = ldb.Dn(samdb, "CN=Infrastructure," + samdb.domain_dn())
  84         self.naming_dn = ldb.Dn(samdb, "CN=Partitions,%s" % samdb.get_config_basedn())
  85         self.schema_dn = samdb.get_schema_basedn()
  86         self.rid_dn = ldb.Dn(samdb, "CN=RID Manager$,CN=System," + samdb.domain_dn())
  87         self.ntds_dsa = ldb.Dn(samdb, samdb.get_dsServiceName())
  88         self.class_schemaIDGUID = {}
  89         self.wellknown_sds = get_wellknown_sds(self.samdb)
  90         self.fix_all_missing_objectclass = False
  91         self.fix_missing_deleted_objects = False
  92
  93         self.dn_set = set()
  94         self.link_id_cache = {}
  95         self.name_map = {}
  96         try:
  97             res = samdb.search(base="CN=DnsAdmins,CN=Users,%s" % samdb.domain_dn(), scope=ldb.SCOPE_BASE,
  98                            attrs=["objectSid"])
  99             dnsadmins_sid = ndr_unpack(security.dom_sid, res[0]["objectSid"][0])
 100             self.name_map['DnsAdmins'] = str(dnsadmins_sid)
 101         except ldb.LdbError, (enum, estr):
 102             if enum != ldb.ERR_NO_SUCH_OBJECT:
 103                 raise
 104             pass
 105
 106         self.system_session_info = system_session()
 107         self.admin_session_info = admin_session(None, samdb.get_domain_sid())
 108
 109         res = self.samdb.search(base=self.ntds_dsa, scope=ldb.SCOPE_BASE, attrs=['msDS-hasMasterNCs', 'hasMasterNCs'])
 110         if "msDS-hasMasterNCs" in res[0]:
 111             self.write_ncs = res[0]["msDS-hasMasterNCs"]
 112         else:
 113             # If the Forest Level is less than 2003 then there is no
 114             # msDS-hasMasterNCs, so we fall back to hasMasterNCs
 115             # no need to merge as all the NCs that are in hasMasterNCs must
 116             # also be in msDS-hasMasterNCs (but not the opposite)
 117             if "hasMasterNCs" in res[0]:
 118                 self.write_ncs = res[0]["hasMasterNCs"]
 119             else:
 120                 self.write_ncs = None
 121
 122         res = self.samdb.search(base="", scope=ldb.SCOPE_BASE, attrs=['namingContexts'])
 123         self.deleted_objects_containers = []
 124         self.ncs_lacking_deleted_containers = []
 125         try:
 126             self.ncs = res[0]["namingContexts"]
 127         except KeyError:
 128             pass
 129         except IndexError:
 130             pass
 131
 132         for nc in self.ncs:
 133             try:
 134                 dn = self.samdb.get_wellknown_dn(ldb.Dn(self.samdb, nc),
 135                                                  dsdb.DS_GUID_DELETED_OBJECTS_CONTAINER)
 136                 self.deleted_objects_containers.append(dn)
 137             except KeyError:
 138                 self.ncs_lacking_deleted_containers.append(ldb.Dn(self.samdb, nc))
 139
 140     def check_database(self, DN=None, scope=ldb.SCOPE_SUBTREE, controls=[], attrs=['*']):
 141         '''perform a database check, returning the number of errors found'''
 142         res = self.samdb.search(base=DN, scope=scope, attrs=['dn'], controls=controls)
 143         self.report('Checking %u objects' % len(res))
 144         error_count = 0
 145
 146         error_count += self.check_deleted_objects_containers()
 147
 148         self.attribute_or_class_ids = set()
 149
 150         for object in res:
 151             self.dn_set.add(str(object.dn))
 152             error_count += self.check_object(object.dn, attrs=attrs)
 153
 154         if DN is None:
 155             error_count += self.check_rootdse()
 156
 157         if error_count != 0 and not self.fix:
 158             self.report("Please use --fix to fix these errors")
 159
 160         self.report('Checked %u objects (%u errors)' % (len(res), error_count))
 161         return error_count
 162
 163
 164     def check_deleted_objects_containers(self):
 165         """This function only fixes conflicts on the Deleted Objects
 166         containers, not the attributes"""
 167         error_count = 0
 168         for nc in self.ncs_lacking_deleted_containers:
 169             if nc == self.schema_dn:
 170                 continue
 171             error_count += 1
 172             self.report("ERROR: NC %s lacks a reference to a Deleted Objects container" % nc)
 173             if not self.confirm_all('Fix missing Deleted Objects container for %s?' % (nc), 'fix_missing_deleted_objects'):
 174                 continue
 175
 176             dn = ldb.Dn(self.samdb, "CN=Deleted Objects")
 177             dn.add_base(nc)
 178
 179             conflict_dn = None
 180             try:
 181                 # If something already exists here, add a conflict
 182                 res = self.samdb.search(base=dn, scope=ldb.SCOPE_BASE, attrs=[],
 183                                         controls=["show_deleted:1", "extended_dn:1:1",
 184                                                   "show_recycled:1", "reveal_internals:0"])
 185                 if len(res) != 0:
 186                     guid = res[0].dn.get_extended_component("GUID")
 187                     conflict_dn = ldb.Dn(self.samdb,
 188                                          "CN=Deleted Objects\\0ACNF:%s" % str(misc.GUID(guid)))
 189                     conflict_dn.add_base(nc)
 190
 191             except ldb.LdbError, (enum, estr):
 192                 if enum == ldb.ERR_NO_SUCH_OBJECT:
 193                     pass
 194                 else:
 195                     self.report("Couldn't check for conflicting Deleted Objects container: %s" % estr)
 196                     return 1
 197
 198             if conflict_dn is not None:
 199                 try:
 200                     self.samdb.rename(dn, conflict_dn, ["show_deleted:1", "relax:0", "show_recycled:1"])
 201                 except ldb.LdbError, (enum, estr):
 202                     self.report("Couldn't move old Deleted Objects placeholder: %s to %s: %s" % (dn, conflict_dn, estr))
 203                     return 1
 204
 205             # Refresh wellKnownObjects links
 206             res = self.samdb.search(base=nc, scope=ldb.SCOPE_BASE,
 207                                     attrs=['wellKnownObjects'],
 208                                     controls=["show_deleted:1", "extended_dn:0",
 209                                               "show_recycled:1", "reveal_internals:0"])
 210             if len(res) != 1:
 211                 self.report("wellKnownObjects was not found for NC %s" % nc)
 212                 return 1
 213
 214             # Prevent duplicate deleted objects containers just in case
 215             wko = res[0]["wellKnownObjects"]
 216             listwko = []
 217             proposed_objectguid = None
 218             for o in wko:
 219                 dsdb_dn = dsdb_Dn(self.samdb, o, dsdb.DSDB_SYNTAX_BINARY_DN)
 220                 if self.is_deleted_objects_dn(dsdb_dn):
 221                     self.report("wellKnownObjects had duplicate Deleted Objects value %s" % o)
 222                     # We really want to put this back in the same spot
 223                     # as the original one, so that on replication we
 224                     # merge, rather than conflict.
 225                     proposed_objectguid = dsdb_dn.dn.get_extended_component("GUID")
 226                 listwko.append(o)
 227
 228             if proposed_objectguid is not None:
 229                 guid_suffix = "\nobjectGUID: %s" % str(misc.GUID(proposed_objectguid))
 230             else:
 231                 wko_prefix = "B:32:%s" % dsdb.DS_GUID_DELETED_OBJECTS_CONTAINER
 232                 listwko.append('%s:%s' % (wko_prefix, dn))
 233                 guid_suffix = ""
 234
 235             # Insert a brand new Deleted Objects container
 236             self.samdb.add_ldif("""dn: %s
 237 objectClass: top
 238 objectClass: container
 239 description: Container for deleted objects
 240 isDeleted: TRUE
 241 isCriticalSystemObject: TRUE
 242 showInAdvancedViewOnly: TRUE
 243 systemFlags: -1946157056%s""" % (dn, guid_suffix),
 244                                 controls=["relax:0", "provision:0"])
 245
 246             delta = ldb.Message()
 247             delta.dn = ldb.Dn(self.samdb, str(res[0]["dn"]))
 248             delta["wellKnownObjects"] = ldb.MessageElement(listwko,
 249                                                            ldb.FLAG_MOD_REPLACE,
 250                                                            "wellKnownObjects")
 251
 252             # Insert the link to the brand new container
 253             if self.do_modify(delta, ["relax:0"],
 254                               "NC %s lacks Deleted Objects WKGUID" % nc,
 255                               validate=False):
 256                 self.report("Added %s well known guid link" % dn)
 257
 258             self.deleted_objects_containers.append(dn)
 259
 260         return error_count
 261
 262     def report(self, msg):
 263         '''print a message unless quiet is set'''
 264         if not self.quiet:
 265             print(msg)
 266
 267     def confirm(self, msg, allow_all=False, forced=False):
 268         '''confirm a change'''
 269         if not self.fix:
 270             return False
 271         if self.quiet:
 272             return self.yes
 273         if self.yes:
 274             forced = True
 275         return common.confirm(msg, forced=forced, allow_all=allow_all)
 276
 277     ################################################################
 278     # a local confirm function with support for 'all'
 279     def confirm_all(self, msg, all_attr):
 280         '''confirm a change with support for "all" '''
 281         if not self.fix:
 282             return False
 283         if self.quiet:
 284             return self.yes
 285         if getattr(self, all_attr) == 'NONE':
 286             return False
 287         if getattr(self, all_attr) == 'ALL':
 288             forced = True
 289         else:
 290             forced = self.yes
 291         c = common.confirm(msg, forced=forced, allow_all=True)
 292         if c == 'ALL':
 293             setattr(self, all_attr, 'ALL')
 294             return True
 295         if c == 'NONE':
 296             setattr(self, all_attr, 'NONE')
 297             return False
 298         return c
 299
 300     def do_delete(self, dn, controls, msg):
 301         '''delete dn with optional verbose output'''
 302         if self.verbose:
 303             self.report("delete DN %s" % dn)
 304         try:
 305             controls = controls + ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK]
 306             self.samdb.delete(dn, controls=controls)
 307         except Exception, err:
 308             self.report("%s : %s" % (msg, err))
 309             return False
 310         return True
 311
 312     def do_modify(self, m, controls, msg, validate=True):
 313         '''perform a modify with optional verbose output'''
 314         if self.verbose:
 315             self.report(self.samdb.write_ldif(m, ldb.CHANGETYPE_MODIFY))
 316         try:
 317             controls = controls + ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK]
 318             self.samdb.modify(m, controls=controls, validate=validate)
 319         except Exception, err:
 320             self.report("%s : %s" % (msg, err))
 321             return False
 322         return True
 323
 324     def do_rename(self, from_dn, to_rdn, to_base, controls, msg):
 325         '''perform a modify with optional verbose output'''
 326         if self.verbose:
 327             self.report("""dn: %s
 328 changeType: modrdn
 329 newrdn: %s
 330 deleteOldRdn: 1
 331 newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
 332         try:
 333             to_dn = to_rdn + to_base
 334             controls = controls + ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK]
 335             self.samdb.rename(from_dn, to_dn, controls=controls)
 336         except Exception, err:
 337             self.report("%s : %s" % (msg, err))
 338             return False
 339         return True
 340
 341     def get_attr_linkID_and_reverse_name(self, attrname):
 342         if attrname in self.link_id_cache:
 343             return self.link_id_cache[attrname]
 344         linkID = self.samdb_schema.get_linkId_from_lDAPDisplayName(attrname)
 345         if linkID:
 346             revname = self.samdb_schema.get_backlink_from_lDAPDisplayName(attrname)
 347         else:
 348             revname = None
 349         self.link_id_cache[attrname] = (linkID, revname)
 350         return linkID, revname
 351
 352     def err_empty_attribute(self, dn, attrname):
 353         '''fix empty attributes'''
 354         self.report("ERROR: Empty attribute %s in %s" % (attrname, dn))
 355         if not self.confirm_all('Remove empty attribute %s from %s?' % (attrname, dn), 'remove_all_empty_attributes'):
 356             self.report("Not fixing empty attribute %s" % attrname)
 357             return
 358
 359         m = ldb.Message()
 360         m.dn = dn
 361         m[attrname] = ldb.MessageElement('', ldb.FLAG_MOD_DELETE, attrname)
 362         if self.do_modify(m, ["relax:0", "show_recycled:1"],
 363                           "Failed to remove empty attribute %s" % attrname, validate=False):
 364             self.report("Removed empty attribute %s" % attrname)
 365
 366     def err_normalise_mismatch(self, dn, attrname, values):
 367         '''fix attribute normalisation errors'''
 368         self.report("ERROR: Normalisation error for attribute %s in %s" % (attrname, dn))
 369         mod_list = []
 370         for val in values:
 371             normalised = self.samdb.dsdb_normalise_attributes(
 372                 self.samdb_schema, attrname, [val])
 373             if len(normalised) != 1:
 374                 self.report("Unable to normalise value '%s'" % val)
 375                 mod_list.append((val, ''))
 376             elif (normalised[0] != val):
 377                 self.report("value '%s' should be '%s'" % (val, normalised[0]))
 378                 mod_list.append((val, normalised[0]))
 379         if not self.confirm_all('Fix normalisation for %s from %s?' % (attrname, dn), 'fix_all_normalisation'):
 380             self.report("Not fixing attribute %s" % attrname)
 381             return
 382
 383         m = ldb.Message()
 384         m.dn = dn
 385         for i in range(0, len(mod_list)):
 386             (val, nval) = mod_list[i]
 387             m['value_%u' % i] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
 388             if nval != '':
 389                 m['normv_%u' % i] = ldb.MessageElement(nval, ldb.FLAG_MOD_ADD,
 390                     attrname)
 391
 392         if self.do_modify(m, ["relax:0", "show_recycled:1"],
 393                           "Failed to normalise attribute %s" % attrname,
 394                           validate=False):
 395             self.report("Normalised attribute %s" % attrname)
 396
 397     def err_normalise_mismatch_replace(self, dn, attrname, values):
 398         '''fix attribute normalisation errors'''
 399         normalised = self.samdb.dsdb_normalise_attributes(self.samdb_schema, attrname, values)
 400         self.report("ERROR: Normalisation error for attribute '%s' in '%s'" % (attrname, dn))
 401         self.report("Values/Order of values do/does not match: %s/%s!" % (values, list(normalised)))
 402         if list(normalised) == values:
 403             return
 404         if not self.confirm_all("Fix normalisation for '%s' from '%s'?" % (attrname, dn), 'fix_all_normalisation'):
 405             self.report("Not fixing attribute '%s'" % attrname)
 406             return
 407
 408         m = ldb.Message()
 409         m.dn = dn
 410         m[attrname] = ldb.MessageElement(normalised, ldb.FLAG_MOD_REPLACE, attrname)
 411
 412         if self.do_modify(m, ["relax:0", "show_recycled:1"],
 413                           "Failed to normalise attribute %s" % attrname,
 414                           validate=False):
 415             self.report("Normalised attribute %s" % attrname)
 416
 417     def err_duplicate_values(self, dn, attrname, dup_values, values):
 418         '''fix attribute normalisation errors'''
 419         self.report("ERROR: Duplicate values for attribute '%s' in '%s'" % (attrname, dn))
 420         self.report("Values contain a duplicate: [%s]/[%s]!" % (','.join(dup_values), ','.join(values)))
 421         if not self.confirm_all("Fix duplicates for '%s' from '%s'?" % (attrname, dn), 'fix_all_duplicates'):
 422             self.report("Not fixing attribute '%s'" % attrname)
 423             return
 424
 425         m = ldb.Message()
 426         m.dn = dn
 427         m[attrname] = ldb.MessageElement(values, ldb.FLAG_MOD_REPLACE, attrname)
 428
 429         if self.do_modify(m, ["relax:0", "show_recycled:1"],
 430                           "Failed to remove duplicate value on attribute %s" % attrname,
 431                           validate=False):
 432             self.report("Removed duplicate value on attribute %s" % attrname)
 433
 434     def is_deleted_objects_dn(self, dsdb_dn):
 435         '''see if a dsdb_Dn is the special Deleted Objects DN'''
 436         return dsdb_dn.prefix == "B:32:%s:" % dsdb.DS_GUID_DELETED_OBJECTS_CONTAINER
 437
 438     def err_missing_objectclass(self, dn):
 439         """handle object without objectclass"""
 440         self.report("ERROR: missing objectclass in object %s.  If you have another working DC, please run 'samba-tool drs replicate --full-sync --local <destinationDC> <sourceDC> %s'" % (dn, self.samdb.get_nc_root(dn)))
 441         if not self.confirm_all("If you cannot re-sync from another DC, do you wish to delete object '%s'?" % dn, 'fix_all_missing_objectclass'):
 442             self.report("Not deleting object with missing objectclass '%s'" % dn)
 443             return
 444         if self.do_delete(dn, ["relax:0"],
 445                           "Failed to remove DN %s" % dn):
 446             self.report("Removed DN %s" % dn)
 447
 448     def err_deleted_dn(self, dn, attrname, val, dsdb_dn, correct_dn):
 449         """handle a DN pointing to a deleted object"""
 450         self.report("ERROR: target DN is deleted for %s in object %s - %s" % (attrname, dn, val))
 451         self.report("Target GUID points at deleted DN %r" % str(correct_dn))
 452         if not self.confirm_all('Remove DN link?', 'remove_all_deleted_DN_links'):
 453             self.report("Not removing")
 454             return
 455         m = ldb.Message()
 456         m.dn = dn
 457         m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
 458         if self.do_modify(m, ["show_recycled:1",
 459                               "local_oid:%s:0" % dsdb.DSDB_CONTROL_REPLMD_VANISH_LINKS],
 460                           "Failed to remove deleted DN attribute %s" % attrname):
 461             self.report("Removed deleted DN on attribute %s" % attrname)
 462
 463     def err_missing_dn_GUID(self, dn, attrname, val, dsdb_dn):
 464         """handle a missing target DN (both GUID and DN string form are missing)"""
 465         # check if its a backlink
 466         linkID, _ = self.get_attr_linkID_and_reverse_name(attrname)
 467         if (linkID & 1 == 0) and str(dsdb_dn).find('\\0ADEL') == -1:
 468             self.report("Not removing dangling forward link")
 469             return
 470         self.err_deleted_dn(dn, attrname, val, dsdb_dn, dsdb_dn)
 471
 472     def err_incorrect_dn_GUID(self, dn, attrname, val, dsdb_dn, errstr):
 473         """handle a missing GUID extended DN component"""
 474         self.report("ERROR: %s component for %s in object %s - %s" % (errstr, attrname, dn, val))
 475         controls=["extended_dn:1:1", "show_recycled:1"]
 476         try:
 477             res = self.samdb.search(base=str(dsdb_dn.dn), scope=ldb.SCOPE_BASE,
 478                                     attrs=[], controls=controls)
 479         except ldb.LdbError, (enum, estr):
 480             self.report("unable to find object for DN %s - (%s)" % (dsdb_dn.dn, estr))
 481             self.err_missing_dn_GUID(dn, attrname, val, dsdb_dn)
 482             return
 483         if len(res) == 0:
 484             self.report("unable to find object for DN %s" % dsdb_dn.dn)
 485             self.err_missing_dn_GUID(dn, attrname, val, dsdb_dn)
 486             return
 487         dsdb_dn.dn = res[0].dn
 488
 489         if not self.confirm_all('Change DN to %s?' % str(dsdb_dn), 'fix_all_DN_GUIDs'):
 490             self.report("Not fixing %s" % errstr)
 491             return
 492         m = ldb.Message()
 493         m.dn = dn
 494         m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
 495         m['new_value'] = ldb.MessageElement(str(dsdb_dn), ldb.FLAG_MOD_ADD, attrname)
 496
 497         if self.do_modify(m, ["show_recycled:1"],
 498                           "Failed to fix %s on attribute %s" % (errstr, attrname)):
 499             self.report("Fixed %s on attribute %s" % (errstr, attrname))
 500
 501     def err_incorrect_binary_dn(self, dn, attrname, val, dsdb_dn, errstr):
 502         """handle an incorrect binary DN component"""
 503         self.report("ERROR: %s binary component for %s in object %s - %s" % (errstr, attrname, dn, val))
 504         controls=["extended_dn:1:1", "show_recycled:1"]
 505
 506         if not self.confirm_all('Change DN to %s?' % str(dsdb_dn), 'fix_all_binary_dn'):
 507             self.report("Not fixing %s" % errstr)
 508             return
 509         m = ldb.Message()
 510         m.dn = dn
 511         m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
 512         m['new_value'] = ldb.MessageElement(str(dsdb_dn), ldb.FLAG_MOD_ADD, attrname)
 513
 514         if self.do_modify(m, ["show_recycled:1"],
 515                           "Failed to fix %s on attribute %s" % (errstr, attrname)):
 516             self.report("Fixed %s on attribute %s" % (errstr, attrname))
 517
 518     def err_dn_component_target_mismatch(self, dn, attrname, val, dsdb_dn, correct_dn, mismatch_type):
 519         """handle a DN string being incorrect"""
 520         self.report("ERROR: incorrect DN %s component for %s in object %s - %s" % (mismatch_type, attrname, dn, val))
 521         dsdb_dn.dn = correct_dn
 522
 523         if not self.confirm_all('Change DN to %s?' % str(dsdb_dn),
 524                                 'fix_all_%s_dn_component_mismatch' % mismatch_type):
 525             self.report("Not fixing %s component mismatch" % mismatch_type)
 526             return
 527         m = ldb.Message()
 528         m.dn = dn
 529         m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
 530         m['new_value'] = ldb.MessageElement(str(dsdb_dn), ldb.FLAG_MOD_ADD, attrname)
 531         if self.do_modify(m, ["show_recycled:1"],
 532                           "Failed to fix incorrect DN %s on attribute %s" % (mismatch_type, attrname)):
 533             self.report("Fixed incorrect DN %s on attribute %s" % (mismatch_type, attrname))
 534
 535     def err_unknown_attribute(self, obj, attrname):
 536         '''handle an unknown attribute error'''
 537         self.report("ERROR: unknown attribute '%s' in %s" % (attrname, obj.dn))
 538         if not self.confirm_all('Remove unknown attribute %s' % attrname, 'remove_all_unknown_attributes'):
 539             self.report("Not removing %s" % attrname)
 540             return
 541         m = ldb.Message()
 542         m.dn = obj.dn
 543         m['old_value'] = ldb.MessageElement([], ldb.FLAG_MOD_DELETE, attrname)
 544         if self.do_modify(m, ["relax:0", "show_recycled:1"],
 545                           "Failed to remove unknown attribute %s" % attrname):
 546             self.report("Removed unknown attribute %s" % (attrname))
 547
 548     def err_undead_linked_attribute(self, obj, attrname, val):
 549         '''handle a link that should not be there on a deleted object'''
 550         self.report("ERROR: linked attribute '%s' to '%s' is present on "
 551                     "deleted object %s" % (attrname, val, obj.dn))
 552         if not self.confirm_all('Remove linked attribute %s' % attrname, 'fix_undead_linked_attributes'):
 553             self.report("Not removing linked attribute %s" % attrname)
 554             return
 555         m = ldb.Message()
 556         m.dn = obj.dn
 557         m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
 558
 559         if self.do_modify(m, ["show_recycled:1", "show_deleted:1", "reveal_internals:0",
 560                               "local_oid:%s:0" % dsdb.DSDB_CONTROL_REPLMD_VANISH_LINKS],
 561                           "Failed to delete forward link %s" % attrname):
 562             self.report("Fixed undead forward link %s" % (attrname))
 563
 564     def err_missing_backlink(self, obj, attrname, val, backlink_name, target_dn):
 565         '''handle a missing backlink value'''
 566         self.report("ERROR: missing backlink attribute '%s' in %s for link %s in %s" % (backlink_name, target_dn, attrname, obj.dn))
 567         if not self.confirm_all('Fix missing backlink %s' % backlink_name, 'fix_all_missing_backlinks'):
 568             self.report("Not fixing missing backlink %s" % backlink_name)
 569             return
 570         m = ldb.Message()
 571         m.dn = obj.dn
 572         m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
 573         m['new_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_ADD, attrname)
 574         if self.do_modify(m, ["show_recycled:1"],
 575                           "Failed to fix missing backlink %s" % backlink_name):
 576             self.report("Fixed missing backlink %s" % (backlink_name))
 577
 578     def err_incorrect_rmd_flags(self, obj, attrname, revealed_dn):
 579         '''handle a incorrect RMD_FLAGS value'''
 580         rmd_flags = int(revealed_dn.dn.get_extended_component("RMD_FLAGS"))
 581         self.report("ERROR: incorrect RMD_FLAGS value %u for attribute '%s' in %s for link %s" % (rmd_flags, attrname, obj.dn, revealed_dn.dn.extended_str()))
 582         if not self.confirm_all('Fix incorrect RMD_FLAGS %u' % rmd_flags, 'fix_rmd_flags'):
 583             self.report("Not fixing incorrect RMD_FLAGS %u" % rmd_flags)
 584             return
 585         m = ldb.Message()
 586         m.dn = obj.dn
 587         m['old_value'] = ldb.MessageElement(str(revealed_dn), ldb.FLAG_MOD_DELETE, attrname)
 588         if self.do_modify(m, ["show_recycled:1", "reveal_internals:0", "show_deleted:0"],
 589                           "Failed to fix incorrect RMD_FLAGS %u" % rmd_flags):
 590             self.report("Fixed incorrect RMD_FLAGS %u" % (rmd_flags))
 591
 592     def err_orphaned_backlink(self, obj, attrname, val, link_name, target_dn):
 593         '''handle a orphaned backlink value'''
 594         self.report("ERROR: orphaned backlink attribute '%s' in %s for link %s in %s" % (attrname, obj.dn, link_name, target_dn))
 595         if not self.confirm_all('Remove orphaned backlink %s' % link_name, 'fix_all_orphaned_backlinks'):
 596             self.report("Not removing orphaned backlink %s" % link_name)
 597             return
 598         m = ldb.Message()
 599         m.dn = obj.dn
 600         m['value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
 601         if self.do_modify(m, ["show_recycled:1", "relax:0"],
 602                           "Failed to fix orphaned backlink %s" % link_name):
 603             self.report("Fixed orphaned backlink %s" % (link_name))
 604
 605     def err_no_fsmoRoleOwner(self, obj):
 606         '''handle a missing fSMORoleOwner'''
 607         self.report("ERROR: fSMORoleOwner not found for role %s" % (obj.dn))
 608         res = self.samdb.search("",
 609                                 scope=ldb.SCOPE_BASE, attrs=["dsServiceName"])
 610         assert len(res) == 1
 611         serviceName = res[0]["dsServiceName"][0]
 612         if not self.confirm_all('Sieze role %s onto current DC by adding fSMORoleOwner=%s' % (obj.dn, serviceName), 'seize_fsmo_role'):
 613             self.report("Not Siezing role %s onto current DC by adding fSMORoleOwner=%s" % (obj.dn, serviceName))
 614             return
 615         m = ldb.Message()
 616         m.dn = obj.dn
 617         m['value'] = ldb.MessageElement(serviceName, ldb.FLAG_MOD_ADD, 'fSMORoleOwner')
 618         if self.do_modify(m, [],
 619                           "Failed to sieze role %s onto current DC by adding fSMORoleOwner=%s" % (obj.dn, serviceName)):
 620             self.report("Siezed role %s onto current DC by adding fSMORoleOwner=%s" % (obj.dn, serviceName))
 621
 622     def err_missing_parent(self, obj):
 623         '''handle a missing parent'''
 624         self.report("ERROR: parent object not found for %s" % (obj.dn))
 625         if not self.confirm_all('Move object %s into LostAndFound?' % (obj.dn), 'move_to_lost_and_found'):
 626             self.report('Not moving object %s into LostAndFound' % (obj.dn))
 627             return
 628
 629         keep_transaction = True
 630         self.samdb.transaction_start()
 631         try:
 632             nc_root = self.samdb.get_nc_root(obj.dn);
 633             lost_and_found = self.samdb.get_wellknown_dn(nc_root, dsdb.DS_GUID_LOSTANDFOUND_CONTAINER)
 634             new_dn = ldb.Dn(self.samdb, str(obj.dn))
 635             new_dn.remove_base_components(len(new_dn) - 1)
 636             if self.do_rename(obj.dn, new_dn, lost_and_found, ["show_deleted:0", "relax:0"],
 637                               "Failed to rename object %s into lostAndFound at %s" % (obj.dn, new_dn + lost_and_found)):
 638                 self.report("Renamed object %s into lostAndFound at %s" % (obj.dn, new_dn + lost_and_found))
 639
 640                 m = ldb.Message()
 641                 m.dn = obj.dn
 642                 m['lastKnownParent'] = ldb.MessageElement(str(obj.dn.parent()), ldb.FLAG_MOD_REPLACE, 'lastKnownParent')
 643
 644                 if self.do_modify(m, [],
 645                                   "Failed to set lastKnownParent on lostAndFound object at %s" % (new_dn + lost_and_found)):
 646                     self.report("Set lastKnownParent on lostAndFound object at %s" % (new_dn + lost_and_found))
 647                     keep_transaction = True
 648         except:
 649             self.samdb.transaction_cancel()
 650             raise
 651
 652         if keep_transaction:
 653             self.samdb.transaction_commit()
 654         else:
 655             self.samdb.transaction_cancel()
 656
 657     def err_wrong_dn(self, obj, new_dn, rdn_attr, rdn_val, name_val):
 658         '''handle a wrong dn'''
 659
 660         new_rdn = ldb.Dn(self.samdb, str(new_dn))
 661         new_rdn.remove_base_components(len(new_rdn) - 1)
 662         new_parent = new_dn.parent()
 663
 664         attributes = ""
 665         if rdn_val != name_val:
 666             attributes += "%s=%r " % (rdn_attr, rdn_val)
 667         attributes += "name=%r" % (name_val)
 668
 669         self.report("ERROR: wrong dn[%s] %s new_dn[%s]" % (obj.dn, attributes, new_dn))
 670         if not self.confirm_all("Rename %s to %s?" % (obj.dn, new_dn), 'fix_dn'):
 671             self.report("Not renaming %s to %s" % (obj.dn, new_dn))
 672             return
 673
 674         if self.do_rename(obj.dn, new_rdn, new_parent, ["show_recycled:1", "relax:0"],
 675                           "Failed to rename object %s into %s" % (obj.dn, new_dn)):
 676             self.report("Renamed %s into %s" % (obj.dn, new_dn))
 677
 678     def err_wrong_instancetype(self, obj, calculated_instancetype):
 679         '''handle a wrong instanceType'''
 680         self.report("ERROR: wrong instanceType %s on %s, should be %d" % (obj["instanceType"], obj.dn, calculated_instancetype))
 681         if not self.confirm_all('Change instanceType from %s to %d on %s?' % (obj["instanceType"], calculated_instancetype, obj.dn), 'fix_instancetype'):
 682             self.report('Not changing instanceType from %s to %d on %s' % (obj["instanceType"], calculated_instancetype, obj.dn))
 683             return
 684
 685         m = ldb.Message()
 686         m.dn = obj.dn
 687         m['value'] = ldb.MessageElement(str(calculated_instancetype), ldb.FLAG_MOD_REPLACE, 'instanceType')
 688         if self.do_modify(m, ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK_MODIFY_RO_REPLICA],
 689                           "Failed to correct missing instanceType on %s by setting instanceType=%d" % (obj.dn, calculated_instancetype)):
 690             self.report("Corrected instancetype on %s by setting instanceType=%d" % (obj.dn, calculated_instancetype))
 691
 692     def err_short_userParameters(self, obj, attrname, value):
 693         # This is a truncated userParameters due to a pre 4.1 replication bug
 694         self.report("ERROR: incorrect userParameters value on object %s.  If you have another working DC that does not give this warning, please run 'samba-tool drs replicate --full-sync --local <destinationDC> <sourceDC> %s'" % (obj.dn, self.samdb.get_nc_root(obj.dn)))
 695
 696     def err_base64_userParameters(self, obj, attrname, value):
 697         '''handle a wrong userParameters'''
 698         self.report("ERROR: wrongly formatted userParameters %s on %s, should not be base64-encoded" % (value, obj.dn))
 699         if not self.confirm_all('Convert userParameters from base64 encoding on %s?' % (obj.dn), 'fix_base64_userparameters'):
 700             self.report('Not changing userParameters from base64 encoding on %s' % (obj.dn))
 701             return
 702
 703         m = ldb.Message()
 704         m.dn = obj.dn
 705         m['value'] = ldb.MessageElement(b64decode(obj[attrname][0]), ldb.FLAG_MOD_REPLACE, 'userParameters')
 706         if self.do_modify(m, [],
 707                           "Failed to correct base64-encoded userParameters on %s by converting from base64" % (obj.dn)):
 708             self.report("Corrected base64-encoded userParameters on %s by converting from base64" % (obj.dn))
 709
 710     def err_utf8_userParameters(self, obj, attrname, value):
 711         '''handle a wrong userParameters'''
 712         self.report("ERROR: wrongly formatted userParameters on %s, should not be psudo-UTF8 encoded" % (obj.dn))
 713         if not self.confirm_all('Convert userParameters from UTF8 encoding on %s?' % (obj.dn), 'fix_utf8_userparameters'):
 714             self.report('Not changing userParameters from UTF8 encoding on %s' % (obj.dn))
 715             return
 716
 717         m = ldb.Message()
 718         m.dn = obj.dn
 719         m['value'] = ldb.MessageElement(obj[attrname][0].decode('utf8').encode('utf-16-le'),
 720                                         ldb.FLAG_MOD_REPLACE, 'userParameters')
 721         if self.do_modify(m, [],
 722                           "Failed to correct psudo-UTF8 encoded userParameters on %s by converting from UTF8" % (obj.dn)):
 723             self.report("Corrected psudo-UTF8 encoded userParameters on %s by converting from UTF8" % (obj.dn))
 724
 725     def err_doubled_userParameters(self, obj, attrname, value):
 726         '''handle a wrong userParameters'''
 727         self.report("ERROR: wrongly formatted userParameters on %s, should not be double UTF16 encoded" % (obj.dn))
 728         if not self.confirm_all('Convert userParameters from doubled UTF-16 encoding on %s?' % (obj.dn), 'fix_doubled_userparameters'):
 729             self.report('Not changing userParameters from doubled UTF-16 encoding on %s' % (obj.dn))
 730             return
 731
 732         m = ldb.Message()
 733         m.dn = obj.dn
 734         m['value'] = ldb.MessageElement(obj[attrname][0].decode('utf-16-le').decode('utf-16-le').encode('utf-16-le'),
 735                                         ldb.FLAG_MOD_REPLACE, 'userParameters')
 736         if self.do_modify(m, [],
 737                           "Failed to correct doubled-UTF16 encoded userParameters on %s by converting" % (obj.dn)):
 738             self.report("Corrected doubled-UTF16 encoded userParameters on %s by converting" % (obj.dn))
 739
 740     def err_odd_userParameters(self, obj, attrname):
 741         # This is a truncated userParameters due to a pre 4.1 replication bug
 742         self.report("ERROR: incorrect userParameters value on object %s (odd length).  If you have another working DC that does not give this warning, please run 'samba-tool drs replicate --full-sync --local <destinationDC> <sourceDC> %s'" % (obj.dn, self.samdb.get_nc_root(obj.dn)))
 743
 744     def find_revealed_link(self, dn, attrname, guid):
 745         '''return a revealed link in an object'''
 746         res = self.samdb.search(base=dn, scope=ldb.SCOPE_BASE, attrs=[attrname],
 747                                 controls=["show_deleted:0", "extended_dn:0", "reveal_internals:0"])
 748         syntax_oid = self.samdb_schema.get_syntax_oid_from_lDAPDisplayName(attrname)
 749         for val in res[0][attrname]:
 750             dsdb_dn = dsdb_Dn(self.samdb, val, syntax_oid)
 751             guid2 = dsdb_dn.dn.get_extended_component("GUID")
 752             if guid == guid2:
 753                 return dsdb_dn
 754         return None
 755
 756     def check_dn(self, obj, attrname, syntax_oid):
 757         '''check a DN attribute for correctness'''
 758         error_count = 0
 759         obj_guid = obj['objectGUID'][0]
 760
 761         for val in obj[attrname]:
 762             dsdb_dn = dsdb_Dn(self.samdb, val, syntax_oid)
 763
 764             # all DNs should have a GUID component
 765             guid = dsdb_dn.dn.get_extended_component("GUID")
 766             if guid is None:
 767                 error_count += 1
 768                 self.err_incorrect_dn_GUID(obj.dn, attrname, val, dsdb_dn,
 769                     "missing GUID")
 770                 continue
 771
 772             guidstr = str(misc.GUID(guid))
 773             attrs = ['isDeleted']
 774
 775             if (str(attrname).lower() == 'msds-hasinstantiatedncs') and (obj.dn == self.ntds_dsa):
 776                 fixing_msDS_HasInstantiatedNCs = True
 777                 attrs.append("instanceType")
 778             else:
 779                 fixing_msDS_HasInstantiatedNCs = False
 780
 781             linkID, reverse_link_name = self.get_attr_linkID_and_reverse_name(attrname)
 782             if reverse_link_name is not None:
 783                 attrs.append(reverse_link_name)
 784
 785             # check its the right GUID
 786             try:
 787                 res = self.samdb.search(base="<GUID=%s>" % guidstr, scope=ldb.SCOPE_BASE,
 788                                         attrs=attrs, controls=["extended_dn:1:1", "show_recycled:1",
 789                                                                "reveal_internals:0"
 790                                         ])
 791             except ldb.LdbError, (enum, estr):
 792                 error_count += 1
 793                 self.err_incorrect_dn_GUID(obj.dn, attrname, val, dsdb_dn, "incorrect GUID")
 794                 continue
 795
 796             if fixing_msDS_HasInstantiatedNCs:
 797                 dsdb_dn.prefix = "B:8:%08X:" % int(res[0]['instanceType'][0])
 798                 dsdb_dn.binary = "%08X" % int(res[0]['instanceType'][0])
 799
 800                 if str(dsdb_dn) != val:
 801                     error_count +=1
 802                     self.err_incorrect_binary_dn(obj.dn, attrname, val, dsdb_dn, "incorrect instanceType part of Binary DN")
 803                     continue
 804
 805             # now we have two cases - the source object might or might not be deleted
 806             is_deleted = 'isDeleted' in obj and obj['isDeleted'][0].upper() == 'TRUE'
 807             target_is_deleted = 'isDeleted' in res[0] and res[0]['isDeleted'][0].upper() == 'TRUE'
 808
 809
 810             if is_deleted and not obj.dn in self.deleted_objects_containers and linkID:
 811                 # A fully deleted object should not have any linked
 812                 # attributes. (MS-ADTS 3.1.1.5.5.1.1 Tombstone
 813                 # Requirements and 3.1.1.5.5.1.3 Recycled-Object
 814                 # Requirements)
 815                 self.err_undead_linked_attribute(obj, attrname, val)
 816                 error_count += 1
 817                 continue
 818             elif target_is_deleted and not self.is_deleted_objects_dn(dsdb_dn) and linkID:
 819                 # the target DN is not allowed to be deleted, unless the target DN is the
 820                 # special Deleted Objects container
 821                 error_count += 1
 822                 self.err_deleted_dn(obj.dn, attrname, val, dsdb_dn, res[0].dn)
 823                 continue
 824
 825             # check the DN matches in string form
 826             if str(res[0].dn) != str(dsdb_dn.dn):
 827                 error_count += 1
 828                 self.err_dn_component_target_mismatch(obj.dn, attrname, val, dsdb_dn,
 829                                                       res[0].dn, "string")
 830                 continue
 831
 832             if res[0].dn.get_extended_component("GUID") != dsdb_dn.dn.get_extended_component("GUID"):
 833                 error_count += 1
 834                 self.err_dn_component_target_mismatch(obj.dn, attrname, val, dsdb_dn,
 835                                                       res[0].dn, "GUID")
 836                 continue
 837
 838             if res[0].dn.get_extended_component("SID") != dsdb_dn.dn.get_extended_component("SID"):
 839                 error_count += 1
 840                 self.err_dn_component_target_mismatch(obj.dn, attrname, val, dsdb_dn,
 841                                                       res[0].dn, "SID")
 842                 continue
 843
 844
 845             # check the reverse_link is correct if there should be one
 846             if reverse_link_name is not None:
 847                 match_count = 0
 848                 if reverse_link_name in res[0]:
 849                     for v in res[0][reverse_link_name]:
 850                         v_guid = dsdb_Dn(self.samdb, v).dn.get_extended_component("GUID")
 851                         if v_guid == obj_guid:
 852                             match_count += 1
 853                 if match_count != 1:
 854                     if target_is_deleted:
 855                         error_count += 1
 856                         if linkID & 1:
 857                             self.err_missing_backlink(obj, attrname, val, reverse_link_name, dsdb_dn.dn)
 858                         else:
 859                             self.err_orphaned_backlink(obj, attrname, val, reverse_link_name, dsdb_dn.dn)
 860                     continue
 861
 862
 863
 864
 865         return error_count
 866
 867
 868     def get_originating_time(self, val, attid):
 869         '''Read metadata properties and return the originating time for
 870            a given attributeId.
 871
 872            :return: the originating time or 0 if not found
 873         '''
 874
 875         repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, str(val))
 876         obj = repl.ctr
 877
 878         for o in repl.ctr.array:
 879             if o.attid == attid:
 880                 return o.originating_change_time
 881
 882         return 0
 883
 884     def process_metadata(self, dn, val):
 885         '''Read metadata properties and list attributes in it.
 886            raises KeyError if the attid is unknown.'''
 887
 888         set_att = set()
 889         wrong_attids = set()
 890         list_attid = []
 891         in_schema_nc = dn.is_child_of(self.schema_dn)
 892
 893         repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob, str(val))
 894         obj = repl.ctr
 895
 896         for o in repl.ctr.array:
 897             att = self.samdb_schema.get_lDAPDisplayName_by_attid(o.attid)
 898             set_att.add(att.lower())
 899             list_attid.append(o.attid)
 900             correct_attid = self.samdb_schema.get_attid_from_lDAPDisplayName(att,
 901                                                                              is_schema_nc=in_schema_nc)
 902             if correct_attid != o.attid:
 903                 wrong_attids.add(o.attid)
 904
 905         return (set_att, list_attid, wrong_attids)
 906
 907
 908     def fix_metadata(self, dn, attr):
 909         '''re-write replPropertyMetaData elements for a single attribute for a
 910         object. This is used to fix missing replPropertyMetaData elements'''
 911         res = self.samdb.search(base = dn, scope=ldb.SCOPE_BASE, attrs = [attr],
 912                                 controls = ["search_options:1:2", "show_recycled:1"])
 913         msg = res[0]
 914         nmsg = ldb.Message()
 915         nmsg.dn = dn
 916         nmsg[attr] = ldb.MessageElement(msg[attr], ldb.FLAG_MOD_REPLACE, attr)
 917         if self.do_modify(nmsg, ["relax:0", "provision:0", "show_recycled:1"],
 918                           "Failed to fix metadata for attribute %s" % attr):
 919             self.report("Fixed metadata for attribute %s" % attr)
 920
 921     def ace_get_effective_inherited_type(self, ace):
 922         if ace.flags & security.SEC_ACE_FLAG_INHERIT_ONLY:
 923             return None
 924
 925         check = False
 926         if ace.type == security.SEC_ACE_TYPE_ACCESS_ALLOWED_OBJECT:
 927             check = True
 928         elif ace.type == security.SEC_ACE_TYPE_ACCESS_DENIED_OBJECT:
 929             check = True
 930         elif ace.type == security.SEC_ACE_TYPE_SYSTEM_AUDIT_OBJECT:
 931             check = True
 932         elif ace.type == security.SEC_ACE_TYPE_SYSTEM_ALARM_OBJECT:
 933             check = True
 934
 935         if not check:
 936             return None
 937
 938         if not ace.object.flags & security.SEC_ACE_INHERITED_OBJECT_TYPE_PRESENT:
 939             return None
 940
 941         return str(ace.object.inherited_type)
 942
 943     def lookup_class_schemaIDGUID(self, cls):
 944         if cls in self.class_schemaIDGUID:
 945             return self.class_schemaIDGUID[cls]
 946
 947         flt = "(&(ldapDisplayName=%s)(objectClass=classSchema))" % cls
 948         res = self.samdb.search(base=self.schema_dn,
 949                                 expression=flt,
 950                                 attrs=["schemaIDGUID"])
 951         t = str(ndr_unpack(misc.GUID, res[0]["schemaIDGUID"][0]))
 952
 953         self.class_schemaIDGUID[cls] = t
 954         return t
 955
 956     def process_sd(self, dn, obj):
 957         sd_attr = "nTSecurityDescriptor"
 958         sd_val = obj[sd_attr]
 959
 960         sd = ndr_unpack(security.descriptor, str(sd_val))
 961
 962         is_deleted = 'isDeleted' in obj and obj['isDeleted'][0].upper() == 'TRUE'
 963         if is_deleted:
 964             # we don't fix deleted objects
 965             return (sd, None)
 966
 967         sd_clean = security.descriptor()
 968         sd_clean.owner_sid = sd.owner_sid
 969         sd_clean.group_sid = sd.group_sid
 970         sd_clean.type = sd.type
 971         sd_clean.revision = sd.revision
 972
 973         broken = False
 974         last_inherited_type = None
 975
 976         aces = []
 977         if sd.sacl is not None:
 978             aces = sd.sacl.aces
 979         for i in range(0, len(aces)):
 980             ace = aces[i]
 981
 982             if not ace.flags & security.SEC_ACE_FLAG_INHERITED_ACE:
 983                 sd_clean.sacl_add(ace)
 984                 continue
 985
 986             t = self.ace_get_effective_inherited_type(ace)
 987             if t is None:
 988                 continue
 989
 990             if last_inherited_type is not None:
 991                 if t != last_inherited_type:
 992                     # if it inherited from more than
 993                     # one type it's very likely to be broken
 994                     #
 995                     # If not the recalculation will calculate
 996                     # the same result.
 997                     broken = True
 998                 continue
 999
1000             last_inherited_type = t
1001
1002         aces = []
1003         if sd.dacl is not None:
1004             aces = sd.dacl.aces
1005         for i in range(0, len(aces)):
1006             ace = aces[i]
1007
1008             if not ace.flags & security.SEC_ACE_FLAG_INHERITED_ACE:
1009                 sd_clean.dacl_add(ace)
1010                 continue
1011
1012             t = self.ace_get_effective_inherited_type(ace)
1013             if t is None:
1014                 continue
1015
1016             if last_inherited_type is not None:
1017                 if t != last_inherited_type:
1018                     # if it inherited from more than
1019                     # one type it's very likely to be broken
1020                     #
1021                     # If not the recalculation will calculate
1022                     # the same result.
1023                     broken = True
1024                 continue
1025
1026             last_inherited_type = t
1027
1028         if broken:
1029             return (sd_clean, sd)
1030
1031         if last_inherited_type is None:
1032             # ok
1033             return (sd, None)
1034
1035         cls = None
1036         try:
1037             cls = obj["objectClass"][-1]
1038         except KeyError, e:
1039             pass
1040
1041         if cls is None:
1042             res = self.samdb.search(base=dn, scope=ldb.SCOPE_BASE,
1043                                     attrs=["isDeleted", "objectClass"],
1044                                     controls=["show_recycled:1"])
1045             o = res[0]
1046             is_deleted = 'isDeleted' in o and o['isDeleted'][0].upper() == 'TRUE'
1047             if is_deleted:
1048                 # we don't fix deleted objects
1049                 return (sd, None)
1050             cls = o["objectClass"][-1]
1051
1052         t = self.lookup_class_schemaIDGUID(cls)
1053
1054         if t != last_inherited_type:
1055             # broken
1056             return (sd_clean, sd)
1057
1058         # ok
1059         return (sd, None)
1060
1061     def err_wrong_sd(self, dn, sd, sd_broken):
1062         '''re-write the SD due to incorrect inherited ACEs'''
1063         sd_attr = "nTSecurityDescriptor"
1064         sd_val = ndr_pack(sd)
1065         sd_flags = security.SECINFO_DACL | security.SECINFO_SACL
1066
1067         if not self.confirm_all('Fix %s on %s?' % (sd_attr, dn), 'fix_ntsecuritydescriptor'):
1068             self.report('Not fixing %s on %s\n' % (sd_attr, dn))
1069             return
1070
1071         nmsg = ldb.Message()
1072         nmsg.dn = dn
1073         nmsg[sd_attr] = ldb.MessageElement(sd_val, ldb.FLAG_MOD_REPLACE, sd_attr)
1074         if self.do_modify(nmsg, ["sd_flags:1:%d" % sd_flags],
1075                           "Failed to fix attribute %s" % sd_attr):
1076             self.report("Fixed attribute '%s' of '%s'\n" % (sd_attr, dn))
1077
1078     def err_wrong_default_sd(self, dn, sd, sd_old, diff):
1079         '''re-write the SD due to not matching the default (optional mode for fixing an incorrect provision)'''
1080         sd_attr = "nTSecurityDescriptor"
1081         sd_val = ndr_pack(sd)
1082         sd_old_val = ndr_pack(sd_old)
1083         sd_flags = security.SECINFO_DACL | security.SECINFO_SACL
1084         if sd.owner_sid is not None:
1085             sd_flags |= security.SECINFO_OWNER
1086         if sd.group_sid is not None:
1087             sd_flags |= security.SECINFO_GROUP
1088
1089         if not self.confirm_all('Reset %s on %s back to provision default?\n%s' % (sd_attr, dn, diff), 'reset_all_well_known_acls'):
1090             self.report('Not resetting %s on %s\n' % (sd_attr, dn))
1091             return
1092
1093         m = ldb.Message()
1094         m.dn = dn
1095         m[sd_attr] = ldb.MessageElement(sd_val, ldb.FLAG_MOD_REPLACE, sd_attr)
1096         if self.do_modify(m, ["sd_flags:1:%d" % sd_flags],
1097                           "Failed to reset attribute %s" % sd_attr):
1098             self.report("Fixed attribute '%s' of '%s'\n" % (sd_attr, dn))
1099
1100     def err_missing_sd_owner(self, dn, sd):
1101         '''re-write the SD due to a missing owner or group'''
1102         sd_attr = "nTSecurityDescriptor"
1103         sd_val = ndr_pack(sd)
1104         sd_flags = security.SECINFO_OWNER | security.SECINFO_GROUP
1105
1106         if not self.confirm_all('Fix missing owner or group in %s on %s?' % (sd_attr, dn), 'fix_ntsecuritydescriptor_owner_group'):
1107             self.report('Not fixing missing owner or group %s on %s\n' % (sd_attr, dn))
1108             return
1109
1110         nmsg = ldb.Message()
1111         nmsg.dn = dn
1112         nmsg[sd_attr] = ldb.MessageElement(sd_val, ldb.FLAG_MOD_REPLACE, sd_attr)
1113
1114         # By setting the session_info to admin_session_info and
1115         # setting the security.SECINFO_OWNER | security.SECINFO_GROUP
1116         # flags we cause the descriptor module to set the correct
1117         # owner and group on the SD, replacing the None/NULL values
1118         # for owner_sid and group_sid currently present.
1119         #
1120         # The admin_session_info matches that used in provision, and
1121         # is the best guess we can make for an existing object that
1122         # hasn't had something specifically set.
1123         #
1124         # This is important for the dns related naming contexts.
1125         self.samdb.set_session_info(self.admin_session_info)
1126         if self.do_modify(nmsg, ["sd_flags:1:%d" % sd_flags],
1127                           "Failed to fix metadata for attribute %s" % sd_attr):
1128             self.report("Fixed attribute '%s' of '%s'\n" % (sd_attr, dn))
1129         self.samdb.set_session_info(self.system_session_info)
1130
1131
1132     def has_replmetadata_zero_invocationid(self, dn, repl_meta_data):
1133         repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob,
1134                           str(repl_meta_data))
1135         ctr = repl.ctr
1136         found = False
1137         for o in ctr.array:
1138             # Search for a zero invocationID
1139             if o.originating_invocation_id != misc.GUID("00000000-0000-0000-0000-000000000000"):
1140                 continue
1141
1142             found = True
1143             self.report('''ERROR: on replPropertyMetaData of %s, the instanceType on attribute 0x%08x,
1144                            version %d changed at %s is 00000000-0000-0000-0000-000000000000,
1145                            but should be non-zero.  Proposed fix is to set to our invocationID (%s).'''
1146                         % (dn, o.attid, o.version,
1147                            time.ctime(samba.nttime2unix(o.originating_change_time)),
1148                            self.samdb.get_invocation_id()))
1149
1150         return found
1151
1152
1153     def err_replmetadata_zero_invocationid(self, dn, attr, repl_meta_data):
1154         repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob,
1155                           str(repl_meta_data))
1156         ctr = repl.ctr
1157         now = samba.unix2nttime(int(time.time()))
1158         found = False
1159         for o in ctr.array:
1160             # Search for a zero invocationID
1161             if o.originating_invocation_id != misc.GUID("00000000-0000-0000-0000-000000000000"):
1162                 continue
1163
1164             found = True
1165             seq = self.samdb.sequence_number(ldb.SEQ_NEXT)
1166             o.version = o.version + 1
1167             o.originating_change_time = now
1168             o.originating_invocation_id = misc.GUID(self.samdb.get_invocation_id())
1169             o.originating_usn = seq
1170             o.local_usn = seq
1171
1172         if found:
1173             replBlob = ndr_pack(repl)
1174             msg = ldb.Message()
1175             msg.dn = dn
1176
1177             if not self.confirm_all('Fix %s on %s by setting originating_invocation_id on some elements to our invocationID %s?'
1178                                     % (attr, dn, self.samdb.get_invocation_id()), 'fix_replmetadata_zero_invocationid'):
1179                 self.report('Not fixing zero originating_invocation_id in %s on %s\n' % (attr, dn))
1180                 return
1181
1182             nmsg = ldb.Message()
1183             nmsg.dn = dn
1184             nmsg[attr] = ldb.MessageElement(replBlob, ldb.FLAG_MOD_REPLACE, attr)
1185             if self.do_modify(nmsg, ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK_MODIFY_RO_REPLICA,
1186                                      "local_oid:1.3.6.1.4.1.7165.4.3.14:0"],
1187                               "Failed to fix attribute %s" % attr):
1188                 self.report("Fixed attribute '%s' of '%s'\n" % (attr, dn))
1189
1190
1191     def err_replmetadata_unknown_attid(self, dn, attr, repl_meta_data):
1192         repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob,
1193                           str(repl_meta_data))
1194         ctr = repl.ctr
1195         for o in ctr.array:
1196             # Search for an invalid attid
1197             try:
1198                 att = self.samdb_schema.get_lDAPDisplayName_by_attid(o.attid)
1199             except KeyError:
1200                 self.report('ERROR: attributeID 0X%0X is not known in our schema, not fixing %s on %s\n' % (o.attid, attr, dn))
1201                 return
1202
1203
1204     def err_replmetadata_incorrect_attid(self, dn, attr, repl_meta_data, wrong_attids):
1205         repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob,
1206                           str(repl_meta_data))
1207         fix = False
1208
1209         set_att = set()
1210         remove_attid = set()
1211         hash_att = {}
1212
1213         in_schema_nc = dn.is_child_of(self.schema_dn)
1214
1215         ctr = repl.ctr
1216         # Sort the array, except for the last element.  This strange
1217         # construction, creating a new list, due to bugs in samba's
1218         # array handling in IDL generated objects.
1219         ctr.array = sorted(ctr.array[:], key=lambda o: o.attid)
1220         # Now walk it in reverse, so we see the low (and so incorrect,
1221         # the correct values are above 0x80000000) values first and
1222         # remove the 'second' value we see.
1223         for o in reversed(ctr.array):
1224             print "%s: 0x%08x" % (dn, o.attid)
1225             att = self.samdb_schema.get_lDAPDisplayName_by_attid(o.attid)
1226             if att.lower() in set_att:
1227                 self.report('ERROR: duplicate attributeID values for %s in %s on %s\n' % (att, attr, dn))
1228                 if not self.confirm_all('Fix %s on %s by removing the duplicate value 0x%08x for %s (keeping 0x%08x)?'
1229                                         % (attr, dn, o.attid, att, hash_att[att].attid),
1230                                         'fix_replmetadata_duplicate_attid'):
1231                     self.report('Not fixing duplicate value 0x%08x for %s in %s on %s\n'
1232                                 % (o.attid, att, attr, dn))
1233                     return
1234                 fix = True
1235                 remove_attid.add(o.attid)
1236                 # We want to set the metadata for the most recent
1237                 # update to have been applied locally, that is the metadata
1238                 # matching the (eg string) value in the attribute
1239                 if o.local_usn > hash_att[att].local_usn:
1240                     # This is always what we would have sent over DRS,
1241                     # because the DRS server will have sent the
1242                     # msDS-IntID, but with the values from both
1243                     # attribute entries.
1244                     hash_att[att].version = o.version
1245                     hash_att[att].originating_change_time = o.originating_change_time
1246                     hash_att[att].originating_invocation_id = o.originating_invocation_id
1247                     hash_att[att].originating_usn = o.originating_usn
1248                     hash_att[att].local_usn = o.local_usn
1249
1250                 # Do not re-add the value to the set or overwrite the hash value
1251                 continue
1252
1253             hash_att[att] = o
1254             set_att.add(att.lower())
1255
1256         # Generate a real list we can sort on properly
1257         new_list = [o for o in ctr.array if o.attid not in remove_attid]
1258
1259         if (len(wrong_attids) > 0):
1260             for o in new_list:
1261                 if o.attid in wrong_attids:
1262                     att = self.samdb_schema.get_lDAPDisplayName_by_attid(o.attid)
1263                     correct_attid = self.samdb_schema.get_attid_from_lDAPDisplayName(att, is_schema_nc=in_schema_nc)
1264                     self.report('ERROR: incorrect attributeID values in %s on %s\n' % (attr, dn))
1265                     if not self.confirm_all('Fix %s on %s by replacing incorrect value 0x%08x for %s (new 0x%08x)?'
1266                                             % (attr, dn, o.attid, att, hash_att[att].attid), 'fix_replmetadata_wrong_attid'):
1267                         self.report('Not fixing incorrect value 0x%08x with 0x%08x for %s in %s on %s\n'
1268                                     % (o.attid, correct_attid, att, attr, dn))
1269                         return
1270                     fix = True
1271                     o.attid = correct_attid
1272             if fix:
1273                 # Sort the array, (we changed the value so must re-sort)
1274                 new_list[:] = sorted(new_list[:], key=lambda o: o.attid)
1275
1276         # If we did not already need to fix it, then ask about sorting
1277         if not fix:
1278             self.report('ERROR: unsorted attributeID values in %s on %s\n' % (attr, dn))
1279             if not self.confirm_all('Fix %s on %s by sorting the attribute list?'
1280                                     % (attr, dn), 'fix_replmetadata_unsorted_attid'):
1281                 self.report('Not fixing %s on %s\n' % (attr, dn))
1282                 return
1283
1284             # The actual sort done is done at the top of the function
1285
1286         ctr.count = len(new_list)
1287         ctr.array = new_list
1288         replBlob = ndr_pack(repl)
1289
1290         nmsg = ldb.Message()
1291         nmsg.dn = dn
1292         nmsg[attr] = ldb.MessageElement(replBlob, ldb.FLAG_MOD_REPLACE, attr)
1293         if self.do_modify(nmsg, ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK_MODIFY_RO_REPLICA,
1294                              "local_oid:1.3.6.1.4.1.7165.4.3.14:0",
1295                              "local_oid:1.3.6.1.4.1.7165.4.3.25:0"],
1296                       "Failed to fix attribute %s" % attr):
1297             self.report("Fixed attribute '%s' of '%s'\n" % (attr, dn))
1298
1299
1300     def is_deleted_deleted_objects(self, obj):
1301         faulty = False
1302         if "description" not in obj:
1303             self.report("ERROR: description not present on Deleted Objects container %s" % obj.dn)
1304             faulty = True
1305         if "showInAdvancedViewOnly" not in obj or obj['showInAdvancedViewOnly'][0].upper() == 'FALSE':
1306             self.report("ERROR: showInAdvancedViewOnly not present on Deleted Objects container %s" % obj.dn)
1307             faulty = True
1308         if "objectCategory" not in obj:
1309             self.report("ERROR: objectCategory not present on Deleted Objects container %s" % obj.dn)
1310             faulty = True
1311         if "isCriticalSystemObject" not in obj or obj['isCriticalSystemObject'][0].upper() == 'FALSE':
1312             self.report("ERROR: isCriticalSystemObject not present on Deleted Objects container %s" % obj.dn)
1313             faulty = True
1314         if "isRecycled" in obj:
1315             self.report("ERROR: isRecycled present on Deleted Objects container %s" % obj.dn)
1316             faulty = True
1317         if "isDeleted" in obj and obj['isDeleted'][0].upper() == 'FALSE':
1318             self.report("ERROR: isDeleted not set on Deleted Objects container %s" % obj.dn)
1319             faulty = True
1320         if "objectClass" not in obj or (len(obj['objectClass']) != 2 or
1321                                         obj['objectClass'][0] != 'top' or
1322                                         obj['objectClass'][1] != 'container'):
1323             self.report("ERROR: objectClass incorrectly set on Deleted Objects container %s" % obj.dn)
1324             faulty = True
1325         if "systemFlags" not in obj or obj['systemFlags'][0] != '-1946157056':
1326             self.report("ERROR: systemFlags incorrectly set on Deleted Objects container %s" % obj.dn)
1327             faulty = True
1328         return faulty
1329
1330     def err_deleted_deleted_objects(self, obj):
1331         nmsg = ldb.Message()
1332         nmsg.dn = dn = obj.dn
1333
1334         if "description" not in obj:
1335             nmsg["description"] = ldb.MessageElement("Container for deleted objects", ldb.FLAG_MOD_REPLACE, "description")
1336         if "showInAdvancedViewOnly" not in obj:
1337             nmsg["showInAdvancedViewOnly"] = ldb.MessageElement("TRUE", ldb.FLAG_MOD_REPLACE, "showInAdvancedViewOnly")
1338         if "objectCategory" not in obj:
1339             nmsg["objectCategory"] = ldb.MessageElement("CN=Container,%s" % self.schema_dn, ldb.FLAG_MOD_REPLACE, "objectCategory")
1340         if "isCriticalSystemObject" not in obj:
1341             nmsg["isCriticalSystemObject"] = ldb.MessageElement("TRUE", ldb.FLAG_MOD_REPLACE, "isCriticalSystemObject")
1342         if "isRecycled" in obj:
1343             nmsg["isRecycled"] = ldb.MessageElement("TRUE", ldb.FLAG_MOD_DELETE, "isRecycled")
1344
1345         nmsg["isDeleted"] = ldb.MessageElement("TRUE", ldb.FLAG_MOD_REPLACE, "isDeleted")
1346         nmsg["systemFlags"] = ldb.MessageElement("-1946157056", ldb.FLAG_MOD_REPLACE, "systemFlags")
1347         nmsg["objectClass"] = ldb.MessageElement(["top", "container"], ldb.FLAG_MOD_REPLACE, "objectClass")
1348
1349         if not self.confirm_all('Fix Deleted Objects container %s by restoring default attributes?'
1350                                 % (dn), 'fix_deleted_deleted_objects'):
1351             self.report('Not fixing missing/incorrect attributes on %s\n' % (dn))
1352             return
1353
1354         if self.do_modify(nmsg, ["relax:0"],
1355                           "Failed to fix Deleted Objects container  %s" % dn):
1356             self.report("Fixed Deleted Objects container '%s'\n" % (dn))
1357
1358
1359     def is_fsmo_role(self, dn):
1360         if dn == self.samdb.domain_dn:
1361             return True
1362         if dn == self.infrastructure_dn:
1363             return True
1364         if dn == self.naming_dn:
1365             return True
1366         if dn == self.schema_dn:
1367             return True
1368         if dn == self.rid_dn:
1369             return True
1370
1371         return False
1372
1373     def calculate_instancetype(self, dn):
1374         instancetype = 0
1375         nc_root = self.samdb.get_nc_root(dn)
1376         if dn == nc_root:
1377             instancetype |= dsdb.INSTANCE_TYPE_IS_NC_HEAD
1378             try:
1379                 self.samdb.search(base=dn.parent(), scope=ldb.SCOPE_BASE, attrs=[], controls=["show_recycled:1"])
1380             except ldb.LdbError, (enum, estr):
1381                 if enum != ldb.ERR_NO_SUCH_OBJECT:
1382                     raise
1383             else:
1384                 instancetype |= dsdb.INSTANCE_TYPE_NC_ABOVE
1385
1386         if self.write_ncs is not None and str(nc_root) in self.write_ncs:
1387             instancetype |= dsdb.INSTANCE_TYPE_WRITE
1388
1389         return instancetype
1390
1391     def get_wellknown_sd(self, dn):
1392         for [sd_dn, descriptor_fn] in self.wellknown_sds:
1393             if dn == sd_dn:
1394                 domain_sid = security.dom_sid(self.samdb.get_domain_sid())
1395                 return ndr_unpack(security.descriptor,
1396                                   descriptor_fn(domain_sid,
1397                                                 name_map=self.name_map))
1398
1399         raise KeyError
1400
1401     def check_object(self, dn, attrs=['*']):
1402         '''check one object'''
1403         if self.verbose:
1404             self.report("Checking object %s" % dn)
1405
1406         # If we modify the pass-by-reference attrs variable, then we get a
1407         # replPropertyMetadata for every object that we check.
1408         attrs = list(attrs)
1409         if "dn" in map(str.lower, attrs):
1410             attrs.append("name")
1411         if "distinguishedname" in map(str.lower, attrs):
1412             attrs.append("name")
1413         if str(dn.get_rdn_name()).lower() in map(str.lower, attrs):
1414             attrs.append("name")
1415         if 'name' in map(str.lower, attrs):
1416             attrs.append(dn.get_rdn_name())
1417             attrs.append("isDeleted")
1418             attrs.append("systemFlags")
1419         if '*' in attrs:
1420             attrs.append("replPropertyMetaData")
1421         else:
1422             attrs.append("objectGUID")
1423
1424         try:
1425             sd_flags = 0
1426             sd_flags |= security.SECINFO_OWNER
1427             sd_flags |= security.SECINFO_GROUP
1428             sd_flags |= security.SECINFO_DACL
1429             sd_flags |= security.SECINFO_SACL
1430
1431             res = self.samdb.search(base=dn, scope=ldb.SCOPE_BASE,
1432                                     controls=[
1433                                         "extended_dn:1:1",
1434                                         "show_recycled:1",
1435                                         "show_deleted:1",
1436                                         "sd_flags:1:%d" % sd_flags,
1437                                         "reveal_internals:0",
1438                                     ],
1439                                     attrs=attrs)
1440         except ldb.LdbError, (enum, estr):
1441             if enum == ldb.ERR_NO_SUCH_OBJECT:
1442                 if self.in_transaction:
1443                     self.report("ERROR: Object %s disappeared during check" % dn)
1444                     return 1
1445                 return 0
1446             raise
1447         if len(res) != 1:
1448             self.report("ERROR: Object %s failed to load during check" % dn)
1449             return 1
1450         obj = res[0]
1451         error_count = 0
1452         set_attrs_from_md = set()
1453         set_attrs_seen = set()
1454         got_repl_property_meta_data = False
1455         got_objectclass = False
1456
1457         nc_dn = self.samdb.get_nc_root(obj.dn)
1458         try:
1459             deleted_objects_dn = self.samdb.get_wellknown_dn(nc_dn,
1460                                                              samba.dsdb.DS_GUID_DELETED_OBJECTS_CONTAINER)
1461         except KeyError:
1462             # We have no deleted objects DN for schema, and we check for this above for the other
1463             # NCs
1464             deleted_objects_dn = None
1465
1466
1467         object_rdn_attr = None
1468         object_rdn_val = None
1469         name_val = None
1470         isDeleted = False
1471         systemFlags = 0
1472
1473         for attrname in obj:
1474             if attrname == 'dn' or attrname == "distinguishedName":
1475                 continue
1476
1477             if str(attrname).lower() == 'objectclass':
1478                 got_objectclass = True
1479
1480             if str(attrname).lower() == "name":
1481                 if len(obj[attrname]) != 1:
1482                     error_count += 1
1483                     self.report("ERROR: Not fixing num_values(%d) for '%s' on '%s'" %
1484                                 (len(obj[attrname]), attrname, str(obj.dn)))
1485                 else:
1486                     name_val = obj[attrname][0]
1487
1488             if str(attrname).lower() == str(obj.dn.get_rdn_name()).lower():
1489                 object_rdn_attr = attrname
1490                 if len(obj[attrname]) != 1:
1491                     error_count += 1
1492                     self.report("ERROR: Not fixing num_values(%d) for '%s' on '%s'" %
1493                                 (len(obj[attrname]), attrname, str(obj.dn)))
1494                 else:
1495                     object_rdn_val = obj[attrname][0]
1496
1497             if str(attrname).lower() == 'isdeleted':
1498                 if obj[attrname][0] != "FALSE":
1499                     isDeleted = True
1500
1501             if str(attrname).lower() == 'systemflags':
1502                 systemFlags = int(obj[attrname][0])
1503
1504             if str(attrname).lower() == 'replpropertymetadata':
1505                 if self.has_replmetadata_zero_invocationid(dn, obj[attrname]):
1506                     error_count += 1
1507                     self.err_replmetadata_zero_invocationid(dn, attrname, obj[attrname])
1508                     # We don't continue, as we may also have other fixes for this attribute
1509                     # based on what other attributes we see.
1510
1511                 try:
1512                     (set_attrs_from_md, list_attid_from_md, wrong_attids) \
1513                         = self.process_metadata(dn, obj[attrname])
1514                 except KeyError:
1515                     error_count += 1
1516                     self.err_replmetadata_unknown_attid(dn, attrname, obj[attrname])
1517                     continue
1518
1519                 if len(set_attrs_from_md) < len(list_attid_from_md) \
1520                    or len(wrong_attids) > 0 \
1521                    or sorted(list_attid_from_md) != list_attid_from_md:
1522                     error_count +=1
1523                     self.err_replmetadata_incorrect_attid(dn, attrname, obj[attrname], wrong_attids)
1524
1525                 else:
1526                     # Here we check that the first attid is 0
1527                     # (objectClass).
1528                     if list_attid_from_md[0] != 0:
1529                         error_count += 1
1530                         self.report("ERROR: Not fixing incorrect inital attributeID in '%s' on '%s', it should be objectClass" %
1531                                     (attrname, str(dn)))
1532
1533                 got_repl_property_meta_data = True
1534                 continue
1535
1536             if str(attrname).lower() == 'ntsecuritydescriptor':
1537                 (sd, sd_broken) = self.process_sd(dn, obj)
1538                 if sd_broken is not None:
1539                     self.err_wrong_sd(dn, sd, sd_broken)
1540                     error_count += 1
1541                     continue
1542
1543                 if sd.owner_sid is None or sd.group_sid is None:
1544                     self.err_missing_sd_owner(dn, sd)
1545                     error_count += 1
1546                     continue
1547
1548                 if self.reset_well_known_acls:
1549                     try:
1550                         well_known_sd = self.get_wellknown_sd(dn)
1551                     except KeyError:
1552                         continue
1553
1554                     current_sd = ndr_unpack(security.descriptor,
1555                                             str(obj[attrname][0]))
1556
1557                     diff = get_diff_sds(well_known_sd, current_sd, security.dom_sid(self.samdb.get_domain_sid()))
1558                     if diff != "":
1559                         self.err_wrong_default_sd(dn, well_known_sd, current_sd, diff)
1560                         error_count += 1
1561                         continue
1562                 continue
1563
1564             if str(attrname).lower() == 'objectclass':
1565                 normalised = self.samdb.dsdb_normalise_attributes(self.samdb_schema, attrname, obj[attrname])
1566                 # Do not consider the attribute incorrect if:
1567                 #  - The sorted (alphabetically) list is the same, inclding case
1568                 #  - The first and last elements are the same
1569                 #
1570                 # This avoids triggering an error due to
1571                 # non-determinism in the sort routine in (at least)
1572                 # 4.3 and earlier, and the fact that any AUX classes
1573                 # in these attributes are also not sorted when
1574                 # imported from Windows (they are just in the reverse
1575                 # order of last set)
1576                 if sorted(normalised) != sorted(obj[attrname]) \
1577                    or normalised[0] != obj[attrname][0] \
1578                    or normalised[-1] != obj[attrname][-1]:
1579                     self.err_normalise_mismatch_replace(dn, attrname, list(obj[attrname]))
1580                     error_count += 1
1581                 continue
1582
1583             if str(attrname).lower() == 'userparameters':
1584                 if len(obj[attrname][0]) == 1 and obj[attrname][0][0] == '\x20':
1585                     error_count += 1
1586                     self.err_short_userParameters(obj, attrname, obj[attrname])
1587                     continue
1588
1589                 elif obj[attrname][0][:16] == '\x20\x00\x20\x00\x20\x00\x20\x00\x20\x00\x20\x00\x20\x00\x20\x00':
1590                     # This is the correct, normal prefix
1591                     continue
1592
1593                 elif obj[attrname][0][:20] == 'IAAgACAAIAAgACAAIAAg':
1594                     # this is the typical prefix from a windows migration
1595                     error_count += 1
1596                     self.err_base64_userParameters(obj, attrname, obj[attrname])
1597                     continue
1598
1599                 elif obj[attrname][0][1] != '\x00' and obj[attrname][0][3] != '\x00' and obj[attrname][0][5] != '\x00' and obj[attrname][0][7] != '\x00' and obj[attrname][0][9] != '\x00':
1600                     # This is a prefix that is not in UTF-16 format for the space or munged dialback prefix
1601                     error_count += 1
1602                     self.err_utf8_userParameters(obj, attrname, obj[attrname])
1603                     continue
1604
1605                 elif len(obj[attrname][0]) % 2 != 0:
1606                     # This is a value that isn't even in length
1607                     error_count += 1
1608                     self.err_odd_userParameters(obj, attrname, obj[attrname])
1609                     continue
1610
1611                 elif obj[attrname][0][1] == '\x00' and obj[attrname][0][2] == '\x00' and obj[attrname][0][3] == '\x00' and obj[attrname][0][4] != '\x00' and obj[attrname][0][5] == '\x00':
1612                     # This is a prefix that would happen if a SAMR-written value was replicated from a Samba 4.1 server to a working server
1613                     error_count += 1
1614                     self.err_doubled_userParameters(obj, attrname, obj[attrname])
1615                     continue
1616
1617             if attrname.lower() == 'attributeid' or attrname.lower() == 'governsid':
1618                 if obj[attrname][0] in self.attribute_or_class_ids:
1619                     error_count += 1
1620                     self.report('Error: %s %s on %s already exists as an attributeId or governsId'
1621                                 % (attrname, obj.dn, obj[attrname][0]))
1622                 else:
1623                     self.attribute_or_class_ids.add(obj[attrname][0])
1624
1625             # check for empty attributes
1626             for val in obj[attrname]:
1627                 if val == '':
1628                     self.err_empty_attribute(dn, attrname)
1629                     error_count += 1
1630                     continue
1631
1632             # get the syntax oid for the attribute, so we can can have
1633             # special handling for some specific attribute types
1634             try:
1635                 syntax_oid = self.samdb_schema.get_syntax_oid_from_lDAPDisplayName(attrname)
1636             except Exception, msg:
1637                 self.err_unknown_attribute(obj, attrname)
1638                 error_count += 1
1639                 continue
1640
1641             linkID, reverse_link_name = self.get_attr_linkID_and_reverse_name(attrname)
1642
1643             flag = self.samdb_schema.get_systemFlags_from_lDAPDisplayName(attrname)
1644             if (not flag & dsdb.DS_FLAG_ATTR_NOT_REPLICATED
1645                 and not flag & dsdb.DS_FLAG_ATTR_IS_CONSTRUCTED
1646                 and not linkID):
1647                 set_attrs_seen.add(str(attrname).lower())
1648
1649             if syntax_oid in [ dsdb.DSDB_SYNTAX_BINARY_DN, dsdb.DSDB_SYNTAX_OR_NAME,
1650                                dsdb.DSDB_SYNTAX_STRING_DN, ldb.SYNTAX_DN ]:
1651                 # it's some form of DN, do specialised checking on those
1652                 error_count += self.check_dn(obj, attrname, syntax_oid)
1653             else:
1654
1655                 values = set()
1656                 # check for incorrectly normalised attributes
1657                 for val in obj[attrname]:
1658                     values.add(str(val))
1659
1660                     normalised = self.samdb.dsdb_normalise_attributes(self.samdb_schema, attrname, [val])
1661                     if len(normalised) != 1 or normalised[0] != val:
1662                         self.err_normalise_mismatch(dn, attrname, obj[attrname])
1663                         error_count += 1
1664                         break
1665
1666                 if len(obj[attrname]) != len(values):
1667                     self.err_duplicate_values(dn, attrname, obj[attrname], list(values))
1668                     error_count += 1
1669                     break
1670
1671             if str(attrname).lower() == "instancetype":
1672                 calculated_instancetype = self.calculate_instancetype(dn)
1673                 if len(obj["instanceType"]) != 1 or obj["instanceType"][0] != str(calculated_instancetype):
1674                     error_count += 1
1675                     self.err_wrong_instancetype(obj, calculated_instancetype)
1676
1677         if not got_objectclass and ("*" in attrs or "objectclass" in map(str.lower, attrs)):
1678             error_count += 1
1679             self.err_missing_objectclass(dn)
1680
1681         if ("*" in attrs or "name" in map(str.lower, attrs)):
1682             if name_val is None:
1683                 error_count += 1
1684                 self.report("ERROR: Not fixing missing 'name' on '%s'" % (str(obj.dn)))
1685             if object_rdn_attr is None:
1686                 error_count += 1
1687                 self.report("ERROR: Not fixing missing '%s' on '%s'" % (obj.dn.get_rdn_name(), str(obj.dn)))
1688
1689         if name_val is not None:
1690             parent_dn = None
1691             if isDeleted:
1692                 if not (systemFlags & samba.dsdb.SYSTEM_FLAG_DISALLOW_MOVE_ON_DELETE):
1693                     parent_dn = deleted_objects_dn
1694             if parent_dn is None:
1695                 parent_dn = obj.dn.parent()
1696             expected_dn = ldb.Dn(self.samdb, "RDN=RDN,%s" % (parent_dn))
1697             expected_dn.set_component(0, obj.dn.get_rdn_name(), name_val)
1698
1699             if obj.dn == deleted_objects_dn:
1700                 expected_dn = obj.dn
1701
1702             if expected_dn != obj.dn:
1703                 error_count += 1
1704                 self.err_wrong_dn(obj, expected_dn, object_rdn_attr, object_rdn_val, name_val)
1705             elif obj.dn.get_rdn_value() != object_rdn_val:
1706                 error_count += 1
1707                 self.report("ERROR: Not fixing %s=%r on '%s'" % (object_rdn_attr, object_rdn_val, str(obj.dn)))
1708
1709         show_dn = True
1710         if got_repl_property_meta_data:
1711             if obj.dn == deleted_objects_dn:
1712                 isDeletedAttId = 131120
1713                 # It's 29/12/9999 at 23:59:59 UTC as specified in MS-ADTS 7.1.1.4.2 Deleted Objects Container
1714
1715                 expectedTimeDo = 2650466015990000000
1716                 originating = self.get_originating_time(obj["replPropertyMetaData"], isDeletedAttId)
1717                 if originating != expectedTimeDo:
1718                     if self.confirm_all("Fix isDeleted originating_change_time on '%s'" % str(dn), 'fix_time_metadata'):
1719                         nmsg = ldb.Message()
1720                         nmsg.dn = dn
1721                         nmsg["isDeleted"] = ldb.MessageElement("TRUE", ldb.FLAG_MOD_REPLACE, "isDeleted")
1722                         error_count += 1
1723                         self.samdb.modify(nmsg, controls=["provision:0"])
1724
1725                     else:
1726                         self.report("Not fixing isDeleted originating_change_time on '%s'" % str(dn))
1727
1728             for att in set_attrs_seen.difference(set_attrs_from_md):
1729                 if show_dn:
1730                     self.report("On object %s" % dn)
1731                     show_dn = False
1732                 error_count += 1
1733                 self.report("ERROR: Attribute %s not present in replication metadata" % att)
1734                 if not self.confirm_all("Fix missing replPropertyMetaData element '%s'" % att, 'fix_all_metadata'):
1735                     self.report("Not fixing missing replPropertyMetaData element '%s'" % att)
1736                     continue
1737                 self.fix_metadata(dn, att)
1738
1739         if self.is_fsmo_role(dn):
1740             if "fSMORoleOwner" not in obj and ("*" in attrs or "fsmoroleowner" in map(str.lower, attrs)):
1741                 self.err_no_fsmoRoleOwner(obj)
1742                 error_count += 1
1743
1744         try:
1745             if dn != self.samdb.get_root_basedn() and str(dn.parent()) not in self.dn_set:
1746                 res = self.samdb.search(base=dn.parent(), scope=ldb.SCOPE_BASE,
1747                                         controls=["show_recycled:1", "show_deleted:1"])
1748         except ldb.LdbError, (enum, estr):
1749             if enum == ldb.ERR_NO_SUCH_OBJECT:
1750                 self.err_missing_parent(obj)
1751                 error_count += 1
1752             else:
1753                 raise
1754
1755         if dn in self.deleted_objects_containers and '*' in attrs:
1756             if self.is_deleted_deleted_objects(obj):
1757                 self.err_deleted_deleted_objects(obj)
1758                 error_count += 1
1759
1760         return error_count
1761
1762     ################################################################
1763     # check special @ROOTDSE attributes
1764     def check_rootdse(self):
1765         '''check the @ROOTDSE special object'''
1766         dn = ldb.Dn(self.samdb, '@ROOTDSE')
1767         if self.verbose:
1768             self.report("Checking object %s" % dn)
1769         res = self.samdb.search(base=dn, scope=ldb.SCOPE_BASE)
1770         if len(res) != 1:
1771             self.report("Object %s disappeared during check" % dn)
1772             return 1
1773         obj = res[0]
1774         error_count = 0
1775
1776         # check that the dsServiceName is in GUID form
1777         if not 'dsServiceName' in obj:
1778             self.report('ERROR: dsServiceName missing in @ROOTDSE')
1779             return error_count+1
1780
1781         if not obj['dsServiceName'][0].startswith('<GUID='):
1782             self.report('ERROR: dsServiceName not in GUID form in @ROOTDSE')
1783             error_count += 1
1784             if not self.confirm('Change dsServiceName to GUID form?'):
1785                 return error_count
1786             res = self.samdb.search(base=ldb.Dn(self.samdb, obj['dsServiceName'][0]),
1787                                     scope=ldb.SCOPE_BASE, attrs=['objectGUID'])
1788             guid_str = str(ndr_unpack(misc.GUID, res[0]['objectGUID'][0]))
1789             m = ldb.Message()
1790             m.dn = dn
1791             m['dsServiceName'] = ldb.MessageElement("<GUID=%s>" % guid_str,
1792                                                     ldb.FLAG_MOD_REPLACE, 'dsServiceName')
1793             if self.do_modify(m, [], "Failed to change dsServiceName to GUID form", validate=False):
1794                 self.report("Changed dsServiceName to GUID form")
1795         return error_count
1796
1797
1798     ###############################################
1799     # re-index the database
1800     def reindex_database(self):
1801         '''re-index the whole database'''
1802         m = ldb.Message()
1803         m.dn = ldb.Dn(self.samdb, "@ATTRIBUTES")
1804         m['add']    = ldb.MessageElement('NONE', ldb.FLAG_MOD_ADD, 'force_reindex')
1805         m['delete'] = ldb.MessageElement('NONE', ldb.FLAG_MOD_DELETE, 'force_reindex')
1806         return self.do_modify(m, [], 're-indexed database', validate=False)
1807
1808     ###############################################
1809     # reset @MODULES
1810     def reset_modules(self):
1811         '''reset @MODULES to that needed for current sam.ldb (to read a very old database)'''
1812         m = ldb.Message()
1813         m.dn = ldb.Dn(self.samdb, "@MODULES")
1814         m['@LIST'] = ldb.MessageElement('samba_dsdb', ldb.FLAG_MOD_REPLACE, '@LIST')
1815         return self.do_modify(m, [], 'reset @MODULES on database', validate=False)