source4/torture/drs/python/link_conflicts.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Tests replication scenarios that involve conflicting linked attribute
   5 # information between the 2 DCs.
   6 #
   7 # Copyright (C) Catalyst.Net Ltd. 2017
   8 #
   9 # This program is free software; you can redistribute it and/or modify
  10 # it under the terms of the GNU General Public License as published by
  11 # the Free Software Foundation; either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21 #
  22
  23 #
  24 # Usage:
  25 #  export DC1=dc1_dns_name
  26 #  export DC2=dc2_dns_name
  27 #  export SUBUNITRUN=$samba4srcdir/scripting/bin/subunitrun
  28 #  PYTHONPATH="$PYTHONPATH:$samba4srcdir/torture/drs/python" $SUBUNITRUN link_conflicts -U"$DOMAIN/$DC_USERNAME"%"$DC_PASSWORD"
  29 #
  30
  31 import drs_base
  32 import samba.tests
  33 import ldb
  34 from ldb import SCOPE_BASE
  35 import random
  36 import time
  37
  38 from drs_base import AbstractLink
  39 from samba.dcerpc import drsuapi, misc
  40
  41 # specifies the order to sync DCs in
  42 DC1_TO_DC2 = 1
  43 DC2_TO_DC1 = 2
  44
  45 class DrsReplicaLinkConflictTestCase(drs_base.DrsBaseTestCase):
  46     def setUp(self):
  47         super(DrsReplicaLinkConflictTestCase, self).setUp()
  48
  49         # add some randomness to the test OU. (Deletion of the last test's
  50         # objects can be slow to replicate out. So the OU created by a previous
  51         # testenv may still exist at this point).
  52         rand = random.randint(1, 10000000)
  53         self.base_dn = self.ldb_dc1.get_default_basedn()
  54         self.ou = "OU=test_link_conflict%d,%s" %(rand, self.base_dn)
  55         self.ldb_dc1.add({
  56             "dn": self.ou,
  57             "objectclass": "organizationalUnit"})
  58
  59         (self.drs, self.drs_handle) = self._ds_bind(self.dnsname_dc1)
  60         (self.drs2, self.drs2_handle) = self._ds_bind(self.dnsname_dc2)
  61
  62         # disable replication for the tests so we can control at what point
  63         # the DCs try to replicate
  64         self._disable_inbound_repl(self.dnsname_dc1)
  65         self._disable_inbound_repl(self.dnsname_dc2)
  66
  67     def tearDown(self):
  68         # re-enable replication
  69         self._enable_inbound_repl(self.dnsname_dc1)
  70         self._enable_inbound_repl(self.dnsname_dc2)
  71         self.ldb_dc1.delete(self.ou, ["tree_delete:1"])
  72         super(DrsReplicaLinkConflictTestCase, self).tearDown()
  73
  74     def get_guid(self, samdb, dn):
  75         """Returns an object's GUID (in string format)"""
  76         res = samdb.search(base=dn, attrs=["objectGUID"], scope=ldb.SCOPE_BASE)
  77         return self._GUID_string(res[0]['objectGUID'][0])
  78
  79     def add_object(self, samdb, dn, objectclass="organizationalunit"):
  80         """Adds an object"""
  81         samdb.add({"dn": dn, "objectclass": objectclass})
  82         return self.get_guid(samdb, dn)
  83
  84     def modify_object(self, samdb, dn, attr, value):
  85         """Modifies an attribute for an object"""
  86         m = ldb.Message()
  87         m.dn = ldb.Dn(samdb, dn)
  88         m[attr] = ldb.MessageElement(value, ldb.FLAG_MOD_ADD, attr)
  89         samdb.modify(m)
  90
  91     def add_link_attr(self, samdb, source_dn, attr, target_dn):
  92         """Adds a linked attribute between 2 objects"""
  93         # add the specified attribute to the source object
  94         self.modify_object(samdb, source_dn, attr, target_dn)
  95
  96     def del_link_attr(self, samdb, src, attr, target):
  97         m = ldb.Message()
  98         m.dn = ldb.Dn(samdb, src)
  99         m[attr] = ldb.MessageElement(target, ldb.FLAG_MOD_DELETE, attr)
 100         samdb.modify(m)
 101
 102     def sync_DCs(self, sync_order=DC1_TO_DC2):
 103         """Manually syncs the 2 DCs to ensure they're in sync"""
 104         if sync_order == DC1_TO_DC2:
 105             # sync DC1-->DC2, then DC2-->DC1
 106             self._net_drs_replicate(DC=self.dnsname_dc2, fromDC=self.dnsname_dc1)
 107             self._net_drs_replicate(DC=self.dnsname_dc1, fromDC=self.dnsname_dc2)
 108         else:
 109             # sync DC2-->DC1, then DC1-->DC2
 110             self._net_drs_replicate(DC=self.dnsname_dc1, fromDC=self.dnsname_dc2)
 111             self._net_drs_replicate(DC=self.dnsname_dc2, fromDC=self.dnsname_dc1)
 112
 113     def ensure_unique_timestamp(self):
 114         """Waits a second to ensure a unique timestamp between 2 objects"""
 115         time.sleep(1)
 116
 117     def unique_dn(self, obj_name):
 118         """Returns a unique object DN"""
 119         # Because we run each test case twice, we need to create a unique DN so
 120         # that the 2nd run doesn't hit objects that already exist. Add some
 121         # randomness to the object DN to make it unique
 122         rand = random.randint(1, 10000000)
 123         return "%s-%d,%s" %(obj_name, rand, self.ou)
 124
 125     def assert_attrs_match(self, res1, res2, attr, expected_count):
 126         """
 127         Asserts that the search results contain the expected number of
 128         attributes and the results match on both DCs
 129         """
 130         actual_len = len(res1[0][attr])
 131         self.assertTrue(actual_len == expected_count,
 132                         "Expected %u %s attributes, but got %u" %(expected_count,
 133                                                                   attr, actual_len))
 134         actual_len = len(res2[0][attr])
 135         self.assertTrue(actual_len == expected_count,
 136                         "Expected %u %s attributes, but got %u" %(expected_count,
 137                                                                   attr, actual_len))
 138
 139         # check DCs both agree on the same linked attributes
 140         for val in res1[0][attr]:
 141             self.assertTrue(val in res2[0][attr],
 142                             "%s '%s' not found on DC2" %(attr, val))
 143
 144     def _check_replicated_links(self, src_obj_dn, expected_links):
 145         """Checks that replication sends back the expected linked attributes"""
 146
 147         hwm = drsuapi.DsReplicaHighWaterMark()
 148         hwm.tmp_highest_usn = 0
 149         hwm.reserved_usn = 0
 150         hwm.highest_usn = 0
 151
 152         self._check_replication([src_obj_dn],
 153                                 drsuapi.DRSUAPI_DRS_WRIT_REP,
 154                                 dest_dsa=None,
 155                                 drs_error=drsuapi.DRSUAPI_EXOP_ERR_SUCCESS,
 156                                 nc_dn_str=src_obj_dn,
 157                                 exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 158                                 expected_links=expected_links,
 159                                 highwatermark=hwm)
 160
 161         # Check DC2 as well
 162         self.set_test_ldb_dc(self.ldb_dc2)
 163
 164         self._check_replication([src_obj_dn],
 165                                 drsuapi.DRSUAPI_DRS_WRIT_REP,
 166                                 dest_dsa=None,
 167                                 drs_error=drsuapi.DRSUAPI_EXOP_ERR_SUCCESS,
 168                                 nc_dn_str=src_obj_dn,
 169                                 exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 170                                 expected_links=expected_links,
 171                                 highwatermark=hwm,
 172                                 drs=self.drs2, drs_handle=self.drs2_handle)
 173         self.set_test_ldb_dc(self.ldb_dc1)
 174
 175     def _test_conflict_single_valued_link(self, sync_order):
 176         """
 177         Tests a simple single-value link conflict, i.e. each DC adds a link to
 178         the same source object but linking to different targets.
 179         """
 180         src_ou = self.unique_dn("OU=src")
 181         src_guid = self.add_object(self.ldb_dc1, src_ou)
 182         self.sync_DCs()
 183
 184         # create a unique target on each DC
 185         target1_ou = self.unique_dn("OU=target1")
 186         target2_ou = self.unique_dn("OU=target2")
 187
 188         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 189         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 190
 191         # link the test OU to the respective targets created
 192         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 193         self.ensure_unique_timestamp()
 194         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 195
 196         # try to sync the 2 DCs (this currently fails)
 197         try:
 198             self.sync_DCs(sync_order=sync_order)
 199         except Exception, e:
 200             self.fail("Replication could not resolve link conflict: %s" % e)
 201
 202         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 203                                   scope=SCOPE_BASE, attrs=["managedBy"])
 204         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 205                                   scope=SCOPE_BASE, attrs=["managedBy"])
 206
 207         # check the object has only have one occurence of the single-valued
 208         # attribute and it matches on both DCs
 209         self.assert_attrs_match(res1, res2, "managedBy", 1)
 210
 211         self.assertTrue(res1[0]["managedBy"][0] == target2_ou,
 212                         "Expected most recent update to win conflict")
 213
 214         # we can't query the deleted links over LDAP, but we can check DRS
 215         # to make sure the DC kept a copy of the conflicting link
 216         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 217                              misc.GUID(src_guid), misc.GUID(target1_guid))
 218         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 219                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 220                              misc.GUID(src_guid), misc.GUID(target2_guid))
 221         self._check_replicated_links(src_ou, [link1, link2])
 222
 223
 224     def test_conflict_single_valued_link(self):
 225         # repeat the test twice, to give each DC a chance to resolve the conflict
 226         self._test_conflict_single_valued_link(sync_order=DC1_TO_DC2)
 227         self._test_conflict_single_valued_link(sync_order=DC2_TO_DC1)
 228
 229     def _test_duplicate_single_valued_link(self, sync_order):
 230         """
 231         Adds the same single-valued link on 2 DCs and checks we don't end up
 232         with 2 copies of the link.
 233         """
 234         # create unique objects for the link
 235         target_ou = self.unique_dn("OU=target")
 236         target_guid = self.add_object(self.ldb_dc1, target_ou)
 237         src_ou = self.unique_dn("OU=src")
 238         src_guid = self.add_object(self.ldb_dc1, src_ou)
 239         self.sync_DCs()
 240
 241         # link the same test OU to the same target on both DCs
 242         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target_ou)
 243         self.ensure_unique_timestamp()
 244         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target_ou)
 245
 246         # sync the 2 DCs
 247         self.sync_DCs(sync_order=sync_order)
 248
 249         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 250                                   scope=SCOPE_BASE, attrs=["managedBy"])
 251         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 252                                   scope=SCOPE_BASE, attrs=["managedBy"])
 253
 254         # check the object has only have one occurence of the single-valued
 255         # attribute and it matches on both DCs
 256         self.assert_attrs_match(res1, res2, "managedBy", 1)
 257
 258     def test_duplicate_single_valued_link(self):
 259         # repeat the test twice, to give each DC a chance to resolve the conflict
 260         self._test_duplicate_single_valued_link(sync_order=DC1_TO_DC2)
 261         self._test_duplicate_single_valued_link(sync_order=DC2_TO_DC1)
 262
 263     def _test_conflict_multi_valued_link(self, sync_order):
 264         """
 265         Tests a simple multi-valued link conflict. This adds 2 objects with the
 266         same username on 2 different DCs and checks their group membership is
 267         preserved after the conflict is resolved.
 268         """
 269
 270         # create a common link source
 271         src_dn = self.unique_dn("CN=src")
 272         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 273         self.sync_DCs()
 274
 275         # create the same user (link target) on each DC.
 276         # Note that the GUIDs will differ between the DCs
 277         target_dn = self.unique_dn("CN=target")
 278         target1_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 279         self.ensure_unique_timestamp()
 280         target2_guid = self.add_object(self.ldb_dc2, target_dn, objectclass="user")
 281
 282         # link the src group to the respective target created
 283         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 284         self.ensure_unique_timestamp()
 285         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 286
 287         # sync the 2 DCs. We expect the more recent target2 object to win
 288         self.sync_DCs(sync_order=sync_order)
 289
 290         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 291                                    scope=SCOPE_BASE, attrs=["member"])
 292         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 293                                    scope=SCOPE_BASE, attrs=["member"])
 294         target1_conflict = False
 295
 296         # we expect exactly 2 members in our test group (both DCs should agree)
 297         self.assert_attrs_match(res1, res2, "member", 2)
 298
 299         for val in res1[0]["member"]:
 300             # check the expected conflicting object was renamed
 301             self.assertFalse("CNF:%s" % target2_guid in val)
 302             if "CNF:%s" % target1_guid in val:
 303                 target1_conflict = True
 304
 305         self.assertTrue(target1_conflict,
 306                         "Expected link to conflicting target object not found")
 307
 308     def test_conflict_multi_valued_link(self):
 309         # repeat the test twice, to give each DC a chance to resolve the conflict
 310         self._test_conflict_multi_valued_link(sync_order=DC1_TO_DC2)
 311         self._test_conflict_multi_valued_link(sync_order=DC2_TO_DC1)
 312
 313     def _test_duplicate_multi_valued_link(self, sync_order):
 314         """
 315         Adds the same multivalued link on 2 DCs and checks we don't end up
 316         with 2 copies of the link.
 317         """
 318
 319         # create the link source/target objects
 320         src_dn = self.unique_dn("CN=src")
 321         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 322         target_dn = self.unique_dn("CN=target")
 323         target_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 324         self.sync_DCs()
 325
 326         # link the src group to the same target user separately on each DC
 327         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 328         self.ensure_unique_timestamp()
 329         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 330
 331         self.sync_DCs(sync_order=sync_order)
 332
 333         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 334                                    scope=SCOPE_BASE, attrs=["member"])
 335         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 336                                    scope=SCOPE_BASE, attrs=["member"])
 337
 338         # we expect to still have only 1 member in our test group
 339         self.assert_attrs_match(res1, res2, "member", 1)
 340
 341     def test_duplicate_multi_valued_link(self):
 342         # repeat the test twice, to give each DC a chance to resolve the conflict
 343         self._test_duplicate_multi_valued_link(sync_order=DC1_TO_DC2)
 344         self._test_duplicate_multi_valued_link(sync_order=DC2_TO_DC1)
 345
 346     def _test_conflict_backlinks(self, sync_order):
 347         """
 348         Tests that resolving a source object conflict fixes up any backlinks,
 349         e.g. the same user is added to a conflicting group.
 350         """
 351
 352         # create a common link target
 353         target_dn = self.unique_dn("CN=target")
 354         target_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 355         self.sync_DCs()
 356
 357         # create the same group (link source) on each DC.
 358         # Note that the GUIDs will differ between the DCs
 359         src_dn = self.unique_dn("CN=src")
 360         src1_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 361         self.ensure_unique_timestamp()
 362         src2_guid = self.add_object(self.ldb_dc2, src_dn, objectclass="group")
 363
 364         # link the src group to the respective target created
 365         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 366         self.ensure_unique_timestamp()
 367         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 368
 369         # sync the 2 DCs. We expect the more recent src2 object to win
 370         self.sync_DCs(sync_order=sync_order)
 371
 372         res1 = self.ldb_dc1.search(base="<GUID=%s>" % target_guid,
 373                                    scope=SCOPE_BASE, attrs=["memberOf"])
 374         res2 = self.ldb_dc2.search(base="<GUID=%s>" % target_guid,
 375                                    scope=SCOPE_BASE, attrs=["memberOf"])
 376         src1_backlink = False
 377
 378         # our test user should still be a member of 2 groups (check both DCs agree)
 379         self.assert_attrs_match(res1, res2, "memberOf", 2)
 380
 381         for val in res1[0]["memberOf"]:
 382             # check the conflicting object was renamed
 383             self.assertFalse("CNF:%s" % src2_guid in val)
 384             if "CNF:%s" % src1_guid in val:
 385                 src1_backlink = True
 386
 387         self.assertTrue(src1_backlink,
 388                         "Expected backlink to conflicting source object not found")
 389
 390     def test_conflict_backlinks(self):
 391         # repeat the test twice, to give each DC a chance to resolve the conflict
 392         self._test_conflict_backlinks(sync_order=DC1_TO_DC2)
 393         self._test_conflict_backlinks(sync_order=DC2_TO_DC1)
 394
 395     def _test_link_deletion_conflict(self, sync_order):
 396         """
 397         Checks that a deleted link conflicting with an active link is
 398         resolved correctly.
 399         """
 400
 401         # Add the link objects
 402         target_dn = self.unique_dn("CN=target")
 403         target_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 404         src_dn = self.unique_dn("CN=src")
 405         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 406         self.sync_DCs()
 407
 408         # add the same link on both DCs, and resolve any conflict
 409         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 410         self.ensure_unique_timestamp()
 411         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 412         self.sync_DCs(sync_order=sync_order)
 413
 414         # delete and re-add the link on one DC
 415         self.del_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 416         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 417
 418         # just delete it on the other DC
 419         self.ensure_unique_timestamp()
 420         self.del_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 421         # sanity-check the link is gone on this DC
 422         res1 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 423                                    scope=SCOPE_BASE, attrs=["member"])
 424         self.assertFalse("member" in res1[0], "Couldn't delete member attr")
 425
 426         # sync the 2 DCs. We expect the more older DC1 attribute to win
 427         # because it has a higher version number (even though it's older)
 428         self.sync_DCs(sync_order=sync_order)
 429
 430         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 431                                    scope=SCOPE_BASE, attrs=["member"])
 432         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 433                                    scope=SCOPE_BASE, attrs=["member"])
 434
 435         # our test user should still be a member of the group (check both DCs agree)
 436         self.assertTrue("member" in res1[0], "Expected member attribute missing")
 437         self.assert_attrs_match(res1, res2, "member", 1)
 438
 439     def test_link_deletion_conflict(self):
 440         # repeat the test twice, to give each DC a chance to resolve the conflict
 441         self._test_link_deletion_conflict(sync_order=DC1_TO_DC2)
 442         self._test_link_deletion_conflict(sync_order=DC2_TO_DC1)
 443
 444     def _test_obj_deletion_conflict(self, sync_order, del_target):
 445         """
 446         Checks that a receiving a new link for a deleted object gets
 447         resolved correctly.
 448         """
 449
 450         target_dn = self.unique_dn("CN=target")
 451         target_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 452         src_dn = self.unique_dn("CN=src")
 453         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 454
 455         self.sync_DCs()
 456
 457         # delete the object on one DC
 458         if del_target:
 459             search_guid = src_guid
 460             self.ldb_dc2.delete(target_dn)
 461         else:
 462             search_guid = target_guid
 463             self.ldb_dc2.delete(src_dn)
 464
 465         # add a link on the other DC
 466         self.ensure_unique_timestamp()
 467         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 468
 469         self.sync_DCs(sync_order=sync_order)
 470
 471         # the object deletion should trump the link addition.
 472         # Check the link no longer exists on the remaining object
 473         res1 = self.ldb_dc1.search(base="<GUID=%s>" % search_guid,
 474                                    scope=SCOPE_BASE, attrs=["member", "memberOf"])
 475         res2 = self.ldb_dc2.search(base="<GUID=%s>" % search_guid,
 476                                    scope=SCOPE_BASE, attrs=["member", "memberOf"])
 477
 478         self.assertFalse("member" in res1[0], "member attr shouldn't exist")
 479         self.assertFalse("member" in res2[0], "member attr shouldn't exist")
 480         self.assertFalse("memberOf" in res1[0], "member attr shouldn't exist")
 481         self.assertFalse("memberOf" in res2[0], "member attr shouldn't exist")
 482
 483     def test_obj_deletion_conflict(self):
 484         # repeat the test twice, to give each DC a chance to resolve the conflict
 485         self._test_obj_deletion_conflict(sync_order=DC1_TO_DC2, del_target=True)
 486         self._test_obj_deletion_conflict(sync_order=DC2_TO_DC1, del_target=True)
 487
 488         # and also try deleting the source object instead of the link target
 489         self._test_obj_deletion_conflict(sync_order=DC1_TO_DC2, del_target=False)
 490         self._test_obj_deletion_conflict(sync_order=DC2_TO_DC1, del_target=False)
 491
 492     def _test_full_sync_link_conflict(self, sync_order):
 493         """
 494         Checks that doing a full sync doesn't affect how conflicts get resolved
 495         """
 496
 497         # create the objects for the linked attribute
 498         src_dn = self.unique_dn("CN=src")
 499         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 500         target_dn = self.unique_dn("CN=target")
 501         target1_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 502         self.sync_DCs()
 503
 504         # add the same link on both DCs
 505         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 506         self.ensure_unique_timestamp()
 507         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 508
 509         # Do a couple of full syncs which should resolve the conflict
 510         # (but only for one DC)
 511         if sync_order == DC1_TO_DC2:
 512             self._net_drs_replicate(DC=self.dnsname_dc2, fromDC=self.dnsname_dc1, full_sync=True)
 513             self._net_drs_replicate(DC=self.dnsname_dc2, fromDC=self.dnsname_dc1, full_sync=True)
 514         else:
 515             self._net_drs_replicate(DC=self.dnsname_dc1, fromDC=self.dnsname_dc2, full_sync=True)
 516             self._net_drs_replicate(DC=self.dnsname_dc1, fromDC=self.dnsname_dc2, full_sync=True)
 517
 518         # delete and re-add the link on one DC
 519         self.del_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 520         self.ensure_unique_timestamp()
 521         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 522
 523         # just delete the link on the 2nd DC
 524         self.ensure_unique_timestamp()
 525         self.del_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 526
 527         # sync the 2 DCs. We expect DC1 to win based on version number
 528         self.sync_DCs(sync_order=sync_order)
 529
 530         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 531                                    scope=SCOPE_BASE, attrs=["member"])
 532         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 533                                    scope=SCOPE_BASE, attrs=["member"])
 534
 535         # check the membership still exits (and both DCs agree)
 536         self.assertTrue("member" in res1[0], "Expected member attribute missing")
 537         self.assert_attrs_match(res1, res2, "member", 1)
 538
 539     def test_full_sync_link_conflict(self):
 540         # repeat the test twice, to give each DC a chance to resolve the conflict
 541         self._test_full_sync_link_conflict(sync_order=DC1_TO_DC2)
 542         self._test_full_sync_link_conflict(sync_order=DC2_TO_DC1)
 543
 544     def _test_conflict_single_valued_link_deleted_winner(self, sync_order):
 545         """
 546         Tests a single-value link conflict where the more-up-to-date link value
 547         is deleted.
 548         """
 549         src_ou = self.unique_dn("OU=src")
 550         src_guid = self.add_object(self.ldb_dc1, src_ou)
 551         self.sync_DCs()
 552
 553         # create a unique target on each DC
 554         target1_ou = self.unique_dn("OU=target1")
 555         target2_ou = self.unique_dn("OU=target2")
 556
 557         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 558         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 559
 560         # add the links for the respective targets, and delete one of the links
 561         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 562         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 563         self.ensure_unique_timestamp()
 564         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 565
 566         # sync the 2 DCs
 567         self.sync_DCs(sync_order=sync_order)
 568
 569         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 570                                   scope=SCOPE_BASE, attrs=["managedBy"])
 571         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 572                                   scope=SCOPE_BASE, attrs=["managedBy"])
 573
 574         # Although the more up-to-date link value is deleted, this shouldn't
 575         # trump DC1's active link
 576         self.assert_attrs_match(res1, res2, "managedBy", 1)
 577
 578         self.assertTrue(res1[0]["managedBy"][0] == target2_ou,
 579                         "Expected active link win conflict")
 580
 581         # we can't query the deleted links over LDAP, but we can check that
 582         # the deleted links exist using DRS
 583         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 584                              misc.GUID(src_guid), misc.GUID(target1_guid))
 585         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 586                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 587                              misc.GUID(src_guid), misc.GUID(target2_guid))
 588         self._check_replicated_links(src_ou, [link1, link2])
 589
 590     def test_conflict_single_valued_link_deleted_winner(self):
 591         # repeat the test twice, to give each DC a chance to resolve the conflict
 592         self._test_conflict_single_valued_link_deleted_winner(sync_order=DC1_TO_DC2)
 593         self._test_conflict_single_valued_link_deleted_winner(sync_order=DC2_TO_DC1)
 594
 595     def _test_conflict_single_valued_link_deleted_loser(self, sync_order):
 596         """
 597         Tests a single-valued link conflict, where the losing link value is deleted.
 598         """
 599         src_ou = self.unique_dn("OU=src")
 600         src_guid = self.add_object(self.ldb_dc1, src_ou)
 601         self.sync_DCs()
 602
 603         # create a unique target on each DC
 604         target1_ou = self.unique_dn("OU=target1")
 605         target2_ou = self.unique_dn("OU=target2")
 606
 607         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 608         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 609
 610         # add the links - we want the link to end up deleted on DC2, but active on
 611         # DC1. DC1 has the better version and DC2 has the better timestamp - the
 612         # better version should win
 613         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 614         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 615         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 616         self.ensure_unique_timestamp()
 617         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 618         self.del_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 619
 620         self.sync_DCs(sync_order=sync_order)
 621
 622         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 623                                   scope=SCOPE_BASE, attrs=["managedBy"])
 624         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 625                                   scope=SCOPE_BASE, attrs=["managedBy"])
 626
 627         # check the object has only have one occurence of the single-valued
 628         # attribute and it matches on both DCs
 629         self.assert_attrs_match(res1, res2, "managedBy", 1)
 630
 631         self.assertTrue(res1[0]["managedBy"][0] == target1_ou,
 632                         "Expected most recent update to win conflict")
 633
 634         # we can't query the deleted links over LDAP, but we can check DRS
 635         # to make sure the DC kept a copy of the conflicting link
 636         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 637                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 638                              misc.GUID(src_guid), misc.GUID(target1_guid))
 639         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 640                              misc.GUID(src_guid), misc.GUID(target2_guid))
 641         self._check_replicated_links(src_ou, [link1, link2])
 642
 643     def test_conflict_single_valued_link_deleted_loser(self):
 644         # repeat the test twice, to give each DC a chance to resolve the conflict
 645         self._test_conflict_single_valued_link_deleted_loser(sync_order=DC1_TO_DC2)
 646         self._test_conflict_single_valued_link_deleted_loser(sync_order=DC2_TO_DC1)
 647