source4/torture/drs/python/link_conflicts.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Tests replication scenarios that involve conflicting linked attribute
   5 # information between the 2 DCs.
   6 #
   7 # Copyright (C) Catalyst.Net Ltd. 2017
   8 #
   9 # This program is free software; you can redistribute it and/or modify
  10 # it under the terms of the GNU General Public License as published by
  11 # the Free Software Foundation; either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21 #
  22
  23 #
  24 # Usage:
  25 #  export DC1=dc1_dns_name
  26 #  export DC2=dc2_dns_name
  27 #  export SUBUNITRUN=$samba4srcdir/scripting/bin/subunitrun
  28 #  PYTHONPATH="$PYTHONPATH:$samba4srcdir/torture/drs/python" $SUBUNITRUN link_conflicts -U"$DOMAIN/$DC_USERNAME"%"$DC_PASSWORD"
  29 #
  30
  31 import drs_base
  32 import samba.tests
  33 import ldb
  34 from ldb import SCOPE_BASE
  35 import random
  36 import time
  37
  38 from drs_base import AbstractLink
  39 from samba.dcerpc import drsuapi, misc
  40
  41 # specifies the order to sync DCs in
  42 DC1_TO_DC2 = 1
  43 DC2_TO_DC1 = 2
  44
  45 class DrsReplicaLinkConflictTestCase(drs_base.DrsBaseTestCase):
  46     def setUp(self):
  47         super(DrsReplicaLinkConflictTestCase, self).setUp()
  48
  49         self.ou = samba.tests.create_test_ou(self.ldb_dc1, "test_link_conflict")
  50         self.base_dn = self.ldb_dc1.get_default_basedn()
  51
  52         (self.drs, self.drs_handle) = self._ds_bind(self.dnsname_dc1)
  53         (self.drs2, self.drs2_handle) = self._ds_bind(self.dnsname_dc2)
  54
  55         # disable replication for the tests so we can control at what point
  56         # the DCs try to replicate
  57         self._disable_inbound_repl(self.dnsname_dc1)
  58         self._disable_inbound_repl(self.dnsname_dc2)
  59
  60     def tearDown(self):
  61         # re-enable replication
  62         self._enable_inbound_repl(self.dnsname_dc1)
  63         self._enable_inbound_repl(self.dnsname_dc2)
  64         self.ldb_dc1.delete(self.ou, ["tree_delete:1"])
  65         super(DrsReplicaLinkConflictTestCase, self).tearDown()
  66
  67     def get_guid(self, samdb, dn):
  68         """Returns an object's GUID (in string format)"""
  69         res = samdb.search(base=dn, attrs=["objectGUID"], scope=ldb.SCOPE_BASE)
  70         return self._GUID_string(res[0]['objectGUID'][0])
  71
  72     def add_object(self, samdb, dn, objectclass="organizationalunit"):
  73         """Adds an object"""
  74         samdb.add({"dn": dn, "objectclass": objectclass})
  75         return self.get_guid(samdb, dn)
  76
  77     def modify_object(self, samdb, dn, attr, value):
  78         """Modifies an attribute for an object"""
  79         m = ldb.Message()
  80         m.dn = ldb.Dn(samdb, dn)
  81         m[attr] = ldb.MessageElement(value, ldb.FLAG_MOD_ADD, attr)
  82         samdb.modify(m)
  83
  84     def add_link_attr(self, samdb, source_dn, attr, target_dn):
  85         """Adds a linked attribute between 2 objects"""
  86         # add the specified attribute to the source object
  87         self.modify_object(samdb, source_dn, attr, target_dn)
  88
  89     def del_link_attr(self, samdb, src, attr, target):
  90         m = ldb.Message()
  91         m.dn = ldb.Dn(samdb, src)
  92         m[attr] = ldb.MessageElement(target, ldb.FLAG_MOD_DELETE, attr)
  93         samdb.modify(m)
  94
  95     def sync_DCs(self, sync_order=DC1_TO_DC2):
  96         """Manually syncs the 2 DCs to ensure they're in sync"""
  97         if sync_order == DC1_TO_DC2:
  98             # sync DC1-->DC2, then DC2-->DC1
  99             self._net_drs_replicate(DC=self.dnsname_dc2, fromDC=self.dnsname_dc1)
 100             self._net_drs_replicate(DC=self.dnsname_dc1, fromDC=self.dnsname_dc2)
 101         else:
 102             # sync DC2-->DC1, then DC1-->DC2
 103             self._net_drs_replicate(DC=self.dnsname_dc1, fromDC=self.dnsname_dc2)
 104             self._net_drs_replicate(DC=self.dnsname_dc2, fromDC=self.dnsname_dc1)
 105
 106     def ensure_unique_timestamp(self):
 107         """Waits a second to ensure a unique timestamp between 2 objects"""
 108         time.sleep(1)
 109
 110     def unique_dn(self, obj_name):
 111         """Returns a unique object DN"""
 112         # Because we run each test case twice, we need to create a unique DN so
 113         # that the 2nd run doesn't hit objects that already exist. Add some
 114         # randomness to the object DN to make it unique
 115         rand = random.randint(1, 10000000)
 116         return "%s-%d,%s" %(obj_name, rand, self.ou)
 117
 118     def assert_attrs_match(self, res1, res2, attr, expected_count):
 119         """
 120         Asserts that the search results contain the expected number of
 121         attributes and the results match on both DCs
 122         """
 123         actual_len = len(res1[0][attr])
 124         self.assertTrue(actual_len == expected_count,
 125                         "Expected %u %s attributes, but got %u" %(expected_count,
 126                                                                   attr, actual_len))
 127         actual_len = len(res2[0][attr])
 128         self.assertTrue(actual_len == expected_count,
 129                         "Expected %u %s attributes, but got %u" %(expected_count,
 130                                                                   attr, actual_len))
 131
 132         # check DCs both agree on the same linked attributes
 133         for val in res1[0][attr]:
 134             self.assertTrue(val in res2[0][attr],
 135                             "%s '%s' not found on DC2" %(attr, val))
 136
 137     def zero_highwatermark(self):
 138         """Returns a zeroed highwatermark so that all DRS data gets returned"""
 139         hwm = drsuapi.DsReplicaHighWaterMark()
 140         hwm.tmp_highest_usn = 0
 141         hwm.reserved_usn = 0
 142         hwm.highest_usn = 0
 143         return hwm
 144
 145     def _check_replicated_links(self, src_obj_dn, expected_links):
 146         """Checks that replication sends back the expected linked attributes"""
 147         self._check_replication([src_obj_dn],
 148                                 drsuapi.DRSUAPI_DRS_WRIT_REP,
 149                                 dest_dsa=None,
 150                                 drs_error=drsuapi.DRSUAPI_EXOP_ERR_SUCCESS,
 151                                 nc_dn_str=src_obj_dn,
 152                                 exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 153                                 expected_links=expected_links,
 154                                 highwatermark=self.zero_highwatermark())
 155
 156         # Check DC2 as well
 157         self.set_test_ldb_dc(self.ldb_dc2)
 158
 159         self._check_replication([src_obj_dn],
 160                                 drsuapi.DRSUAPI_DRS_WRIT_REP,
 161                                 dest_dsa=None,
 162                                 drs_error=drsuapi.DRSUAPI_EXOP_ERR_SUCCESS,
 163                                 nc_dn_str=src_obj_dn,
 164                                 exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 165                                 expected_links=expected_links,
 166                                 highwatermark=self.zero_highwatermark(),
 167                                 drs=self.drs2, drs_handle=self.drs2_handle)
 168         self.set_test_ldb_dc(self.ldb_dc1)
 169
 170     def _test_conflict_single_valued_link(self, sync_order):
 171         """
 172         Tests a simple single-value link conflict, i.e. each DC adds a link to
 173         the same source object but linking to different targets.
 174         """
 175         src_ou = self.unique_dn("OU=src")
 176         src_guid = self.add_object(self.ldb_dc1, src_ou)
 177         self.sync_DCs()
 178
 179         # create a unique target on each DC
 180         target1_ou = self.unique_dn("OU=target1")
 181         target2_ou = self.unique_dn("OU=target2")
 182
 183         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 184         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 185
 186         # link the test OU to the respective targets created
 187         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 188         self.ensure_unique_timestamp()
 189         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 190
 191         # sync the 2 DCs
 192         self.sync_DCs(sync_order=sync_order)
 193
 194         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 195                                   scope=SCOPE_BASE, attrs=["managedBy"])
 196         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 197                                   scope=SCOPE_BASE, attrs=["managedBy"])
 198
 199         # check the object has only have one occurence of the single-valued
 200         # attribute and it matches on both DCs
 201         self.assert_attrs_match(res1, res2, "managedBy", 1)
 202
 203         self.assertTrue(res1[0]["managedBy"][0] == target2_ou,
 204                         "Expected most recent update to win conflict")
 205
 206         # we can't query the deleted links over LDAP, but we can check DRS
 207         # to make sure the DC kept a copy of the conflicting link
 208         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 209                              misc.GUID(src_guid), misc.GUID(target1_guid))
 210         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 211                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 212                              misc.GUID(src_guid), misc.GUID(target2_guid))
 213         self._check_replicated_links(src_ou, [link1, link2])
 214
 215
 216     def test_conflict_single_valued_link(self):
 217         # repeat the test twice, to give each DC a chance to resolve the conflict
 218         self._test_conflict_single_valued_link(sync_order=DC1_TO_DC2)
 219         self._test_conflict_single_valued_link(sync_order=DC2_TO_DC1)
 220
 221     def _test_duplicate_single_valued_link(self, sync_order):
 222         """
 223         Adds the same single-valued link on 2 DCs and checks we don't end up
 224         with 2 copies of the link.
 225         """
 226         # create unique objects for the link
 227         target_ou = self.unique_dn("OU=target")
 228         self.add_object(self.ldb_dc1, target_ou)
 229         src_ou = self.unique_dn("OU=src")
 230         src_guid = self.add_object(self.ldb_dc1, src_ou)
 231         self.sync_DCs()
 232
 233         # link the same test OU to the same target on both DCs
 234         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target_ou)
 235         self.ensure_unique_timestamp()
 236         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target_ou)
 237
 238         # sync the 2 DCs
 239         self.sync_DCs(sync_order=sync_order)
 240
 241         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 242                                   scope=SCOPE_BASE, attrs=["managedBy"])
 243         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 244                                   scope=SCOPE_BASE, attrs=["managedBy"])
 245
 246         # check the object has only have one occurence of the single-valued
 247         # attribute and it matches on both DCs
 248         self.assert_attrs_match(res1, res2, "managedBy", 1)
 249
 250     def test_duplicate_single_valued_link(self):
 251         # repeat the test twice, to give each DC a chance to resolve the conflict
 252         self._test_duplicate_single_valued_link(sync_order=DC1_TO_DC2)
 253         self._test_duplicate_single_valued_link(sync_order=DC2_TO_DC1)
 254
 255     def _test_conflict_multi_valued_link(self, sync_order):
 256         """
 257         Tests a simple multi-valued link conflict. This adds 2 objects with the
 258         same username on 2 different DCs and checks their group membership is
 259         preserved after the conflict is resolved.
 260         """
 261
 262         # create a common link source
 263         src_dn = self.unique_dn("CN=src")
 264         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 265         self.sync_DCs()
 266
 267         # create the same user (link target) on each DC.
 268         # Note that the GUIDs will differ between the DCs
 269         target_dn = self.unique_dn("CN=target")
 270         target1_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 271         self.ensure_unique_timestamp()
 272         target2_guid = self.add_object(self.ldb_dc2, target_dn, objectclass="user")
 273
 274         # link the src group to the respective target created
 275         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 276         self.ensure_unique_timestamp()
 277         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 278
 279         # sync the 2 DCs. We expect the more recent target2 object to win
 280         self.sync_DCs(sync_order=sync_order)
 281
 282         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 283                                    scope=SCOPE_BASE, attrs=["member"])
 284         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 285                                    scope=SCOPE_BASE, attrs=["member"])
 286         target1_conflict = False
 287
 288         # we expect exactly 2 members in our test group (both DCs should agree)
 289         self.assert_attrs_match(res1, res2, "member", 2)
 290
 291         for val in res1[0]["member"]:
 292             # check the expected conflicting object was renamed
 293             self.assertFalse("CNF:%s" % target2_guid in val)
 294             if "CNF:%s" % target1_guid in val:
 295                 target1_conflict = True
 296
 297         self.assertTrue(target1_conflict,
 298                         "Expected link to conflicting target object not found")
 299
 300     def test_conflict_multi_valued_link(self):
 301         # repeat the test twice, to give each DC a chance to resolve the conflict
 302         self._test_conflict_multi_valued_link(sync_order=DC1_TO_DC2)
 303         self._test_conflict_multi_valued_link(sync_order=DC2_TO_DC1)
 304
 305     def _test_duplicate_multi_valued_link(self, sync_order):
 306         """
 307         Adds the same multivalued link on 2 DCs and checks we don't end up
 308         with 2 copies of the link.
 309         """
 310
 311         # create the link source/target objects
 312         src_dn = self.unique_dn("CN=src")
 313         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 314         target_dn = self.unique_dn("CN=target")
 315         self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 316         self.sync_DCs()
 317
 318         # link the src group to the same target user separately on each DC
 319         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 320         self.ensure_unique_timestamp()
 321         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 322
 323         self.sync_DCs(sync_order=sync_order)
 324
 325         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 326                                    scope=SCOPE_BASE, attrs=["member"])
 327         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 328                                    scope=SCOPE_BASE, attrs=["member"])
 329
 330         # we expect to still have only 1 member in our test group
 331         self.assert_attrs_match(res1, res2, "member", 1)
 332
 333     def test_duplicate_multi_valued_link(self):
 334         # repeat the test twice, to give each DC a chance to resolve the conflict
 335         self._test_duplicate_multi_valued_link(sync_order=DC1_TO_DC2)
 336         self._test_duplicate_multi_valued_link(sync_order=DC2_TO_DC1)
 337
 338     def _test_conflict_backlinks(self, sync_order):
 339         """
 340         Tests that resolving a source object conflict fixes up any backlinks,
 341         e.g. the same user is added to a conflicting group.
 342         """
 343
 344         # create a common link target
 345         target_dn = self.unique_dn("CN=target")
 346         target_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 347         self.sync_DCs()
 348
 349         # create the same group (link source) on each DC.
 350         # Note that the GUIDs will differ between the DCs
 351         src_dn = self.unique_dn("CN=src")
 352         src1_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 353         self.ensure_unique_timestamp()
 354         src2_guid = self.add_object(self.ldb_dc2, src_dn, objectclass="group")
 355
 356         # link the src group to the respective target created
 357         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 358         self.ensure_unique_timestamp()
 359         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 360
 361         # sync the 2 DCs. We expect the more recent src2 object to win
 362         self.sync_DCs(sync_order=sync_order)
 363
 364         res1 = self.ldb_dc1.search(base="<GUID=%s>" % target_guid,
 365                                    scope=SCOPE_BASE, attrs=["memberOf"])
 366         res2 = self.ldb_dc2.search(base="<GUID=%s>" % target_guid,
 367                                    scope=SCOPE_BASE, attrs=["memberOf"])
 368         src1_backlink = False
 369
 370         # our test user should still be a member of 2 groups (check both DCs agree)
 371         self.assert_attrs_match(res1, res2, "memberOf", 2)
 372
 373         for val in res1[0]["memberOf"]:
 374             # check the conflicting object was renamed
 375             self.assertFalse("CNF:%s" % src2_guid in val)
 376             if "CNF:%s" % src1_guid in val:
 377                 src1_backlink = True
 378
 379         self.assertTrue(src1_backlink,
 380                         "Expected backlink to conflicting source object not found")
 381
 382     def test_conflict_backlinks(self):
 383         # repeat the test twice, to give each DC a chance to resolve the conflict
 384         self._test_conflict_backlinks(sync_order=DC1_TO_DC2)
 385         self._test_conflict_backlinks(sync_order=DC2_TO_DC1)
 386
 387     def _test_link_deletion_conflict(self, sync_order):
 388         """
 389         Checks that a deleted link conflicting with an active link is
 390         resolved correctly.
 391         """
 392
 393         # Add the link objects
 394         target_dn = self.unique_dn("CN=target")
 395         self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 396         src_dn = self.unique_dn("CN=src")
 397         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 398         self.sync_DCs()
 399
 400         # add the same link on both DCs, and resolve any conflict
 401         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 402         self.ensure_unique_timestamp()
 403         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 404         self.sync_DCs(sync_order=sync_order)
 405
 406         # delete and re-add the link on one DC
 407         self.del_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 408         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 409
 410         # just delete it on the other DC
 411         self.ensure_unique_timestamp()
 412         self.del_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 413         # sanity-check the link is gone on this DC
 414         res1 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 415                                    scope=SCOPE_BASE, attrs=["member"])
 416         self.assertFalse("member" in res1[0], "Couldn't delete member attr")
 417
 418         # sync the 2 DCs. We expect the more older DC1 attribute to win
 419         # because it has a higher version number (even though it's older)
 420         self.sync_DCs(sync_order=sync_order)
 421
 422         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 423                                    scope=SCOPE_BASE, attrs=["member"])
 424         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 425                                    scope=SCOPE_BASE, attrs=["member"])
 426
 427         # our test user should still be a member of the group (check both DCs agree)
 428         self.assertTrue("member" in res1[0], "Expected member attribute missing")
 429         self.assert_attrs_match(res1, res2, "member", 1)
 430
 431     def test_link_deletion_conflict(self):
 432         # repeat the test twice, to give each DC a chance to resolve the conflict
 433         self._test_link_deletion_conflict(sync_order=DC1_TO_DC2)
 434         self._test_link_deletion_conflict(sync_order=DC2_TO_DC1)
 435
 436     def _test_obj_deletion_conflict(self, sync_order, del_target):
 437         """
 438         Checks that a receiving a new link for a deleted object gets
 439         resolved correctly.
 440         """
 441
 442         target_dn = self.unique_dn("CN=target")
 443         target_guid = self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 444         src_dn = self.unique_dn("CN=src")
 445         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 446
 447         self.sync_DCs()
 448
 449         # delete the object on one DC
 450         if del_target:
 451             search_guid = src_guid
 452             self.ldb_dc2.delete(target_dn)
 453         else:
 454             search_guid = target_guid
 455             self.ldb_dc2.delete(src_dn)
 456
 457         # add a link on the other DC
 458         self.ensure_unique_timestamp()
 459         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 460
 461         self.sync_DCs(sync_order=sync_order)
 462
 463         # the object deletion should trump the link addition.
 464         # Check the link no longer exists on the remaining object
 465         res1 = self.ldb_dc1.search(base="<GUID=%s>" % search_guid,
 466                                    scope=SCOPE_BASE, attrs=["member", "memberOf"])
 467         res2 = self.ldb_dc2.search(base="<GUID=%s>" % search_guid,
 468                                    scope=SCOPE_BASE, attrs=["member", "memberOf"])
 469
 470         self.assertFalse("member" in res1[0], "member attr shouldn't exist")
 471         self.assertFalse("member" in res2[0], "member attr shouldn't exist")
 472         self.assertFalse("memberOf" in res1[0], "member attr shouldn't exist")
 473         self.assertFalse("memberOf" in res2[0], "member attr shouldn't exist")
 474
 475     def test_obj_deletion_conflict(self):
 476         # repeat the test twice, to give each DC a chance to resolve the conflict
 477         self._test_obj_deletion_conflict(sync_order=DC1_TO_DC2, del_target=True)
 478         self._test_obj_deletion_conflict(sync_order=DC2_TO_DC1, del_target=True)
 479
 480         # and also try deleting the source object instead of the link target
 481         self._test_obj_deletion_conflict(sync_order=DC1_TO_DC2, del_target=False)
 482         self._test_obj_deletion_conflict(sync_order=DC2_TO_DC1, del_target=False)
 483
 484     def _test_full_sync_link_conflict(self, sync_order):
 485         """
 486         Checks that doing a full sync doesn't affect how conflicts get resolved
 487         """
 488
 489         # create the objects for the linked attribute
 490         src_dn = self.unique_dn("CN=src")
 491         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 492         target_dn = self.unique_dn("CN=target")
 493         self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 494         self.sync_DCs()
 495
 496         # add the same link on both DCs
 497         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 498         self.ensure_unique_timestamp()
 499         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 500
 501         # Do a couple of full syncs which should resolve the conflict
 502         # (but only for one DC)
 503         if sync_order == DC1_TO_DC2:
 504             self._net_drs_replicate(DC=self.dnsname_dc2, fromDC=self.dnsname_dc1, full_sync=True)
 505             self._net_drs_replicate(DC=self.dnsname_dc2, fromDC=self.dnsname_dc1, full_sync=True)
 506         else:
 507             self._net_drs_replicate(DC=self.dnsname_dc1, fromDC=self.dnsname_dc2, full_sync=True)
 508             self._net_drs_replicate(DC=self.dnsname_dc1, fromDC=self.dnsname_dc2, full_sync=True)
 509
 510         # delete and re-add the link on one DC
 511         self.del_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 512         self.ensure_unique_timestamp()
 513         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 514
 515         # just delete the link on the 2nd DC
 516         self.ensure_unique_timestamp()
 517         self.del_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 518
 519         # sync the 2 DCs. We expect DC1 to win based on version number
 520         self.sync_DCs(sync_order=sync_order)
 521
 522         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 523                                    scope=SCOPE_BASE, attrs=["member"])
 524         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 525                                    scope=SCOPE_BASE, attrs=["member"])
 526
 527         # check the membership still exits (and both DCs agree)
 528         self.assertTrue("member" in res1[0], "Expected member attribute missing")
 529         self.assert_attrs_match(res1, res2, "member", 1)
 530
 531     def test_full_sync_link_conflict(self):
 532         # repeat the test twice, to give each DC a chance to resolve the conflict
 533         self._test_full_sync_link_conflict(sync_order=DC1_TO_DC2)
 534         self._test_full_sync_link_conflict(sync_order=DC2_TO_DC1)
 535
 536     def _test_conflict_single_valued_link_deleted_winner(self, sync_order):
 537         """
 538         Tests a single-value link conflict where the more-up-to-date link value
 539         is deleted.
 540         """
 541         src_ou = self.unique_dn("OU=src")
 542         src_guid = self.add_object(self.ldb_dc1, src_ou)
 543         self.sync_DCs()
 544
 545         # create a unique target on each DC
 546         target1_ou = self.unique_dn("OU=target1")
 547         target2_ou = self.unique_dn("OU=target2")
 548
 549         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 550         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 551
 552         # add the links for the respective targets, and delete one of the links
 553         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 554         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 555         self.ensure_unique_timestamp()
 556         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 557
 558         # sync the 2 DCs
 559         self.sync_DCs(sync_order=sync_order)
 560
 561         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 562                                   scope=SCOPE_BASE, attrs=["managedBy"])
 563         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 564                                   scope=SCOPE_BASE, attrs=["managedBy"])
 565
 566         # Although the more up-to-date link value is deleted, this shouldn't
 567         # trump DC1's active link
 568         self.assert_attrs_match(res1, res2, "managedBy", 1)
 569
 570         self.assertTrue(res1[0]["managedBy"][0] == target2_ou,
 571                         "Expected active link win conflict")
 572
 573         # we can't query the deleted links over LDAP, but we can check that
 574         # the deleted links exist using DRS
 575         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 576                              misc.GUID(src_guid), misc.GUID(target1_guid))
 577         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 578                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 579                              misc.GUID(src_guid), misc.GUID(target2_guid))
 580         self._check_replicated_links(src_ou, [link1, link2])
 581
 582     def test_conflict_single_valued_link_deleted_winner(self):
 583         # repeat the test twice, to give each DC a chance to resolve the conflict
 584         self._test_conflict_single_valued_link_deleted_winner(sync_order=DC1_TO_DC2)
 585         self._test_conflict_single_valued_link_deleted_winner(sync_order=DC2_TO_DC1)
 586
 587     def _test_conflict_single_valued_link_deleted_loser(self, sync_order):
 588         """
 589         Tests a single-valued link conflict, where the losing link value is deleted.
 590         """
 591         src_ou = self.unique_dn("OU=src")
 592         src_guid = self.add_object(self.ldb_dc1, src_ou)
 593         self.sync_DCs()
 594
 595         # create a unique target on each DC
 596         target1_ou = self.unique_dn("OU=target1")
 597         target2_ou = self.unique_dn("OU=target2")
 598
 599         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 600         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 601
 602         # add the links - we want the link to end up deleted on DC2, but active on
 603         # DC1. DC1 has the better version and DC2 has the better timestamp - the
 604         # better version should win
 605         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 606         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 607         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 608         self.ensure_unique_timestamp()
 609         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 610         self.del_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 611
 612         self.sync_DCs(sync_order=sync_order)
 613
 614         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 615                                   scope=SCOPE_BASE, attrs=["managedBy"])
 616         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 617                                   scope=SCOPE_BASE, attrs=["managedBy"])
 618
 619         # check the object has only have one occurence of the single-valued
 620         # attribute and it matches on both DCs
 621         self.assert_attrs_match(res1, res2, "managedBy", 1)
 622
 623         self.assertTrue(res1[0]["managedBy"][0] == target1_ou,
 624                         "Expected most recent update to win conflict")
 625
 626         # we can't query the deleted links over LDAP, but we can check DRS
 627         # to make sure the DC kept a copy of the conflicting link
 628         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 629                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 630                              misc.GUID(src_guid), misc.GUID(target1_guid))
 631         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 632                              misc.GUID(src_guid), misc.GUID(target2_guid))
 633         self._check_replicated_links(src_ou, [link1, link2])
 634
 635     def test_conflict_single_valued_link_deleted_loser(self):
 636         # repeat the test twice, to give each DC a chance to resolve the conflict
 637         self._test_conflict_single_valued_link_deleted_loser(sync_order=DC1_TO_DC2)
 638         self._test_conflict_single_valued_link_deleted_loser(sync_order=DC2_TO_DC1)
 639
 640     def _test_conflict_existing_single_valued_link(self, sync_order):
 641         """
 642         Tests a single-valued link conflict, where the conflicting link value
 643         already exists (as inactive) on both DCs.
 644         """
 645         # create the link objects
 646         src_ou = self.unique_dn("OU=src")
 647         src_guid = self.add_object(self.ldb_dc1, src_ou)
 648
 649         target1_ou = self.unique_dn("OU=target1")
 650         target2_ou = self.unique_dn("OU=target2")
 651         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 652         target2_guid = self.add_object(self.ldb_dc1, target2_ou)
 653
 654         # add the links, but then delete them
 655         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 656         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 657         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target2_ou)
 658         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target2_ou)
 659         self.sync_DCs()
 660
 661         # re-add the links independently on each DC
 662         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 663         self.ensure_unique_timestamp()
 664         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 665
 666         # try to sync the 2 DCs
 667         self.sync_DCs(sync_order=sync_order)
 668
 669         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 670                                   scope=SCOPE_BASE, attrs=["managedBy"])
 671         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 672                                   scope=SCOPE_BASE, attrs=["managedBy"])
 673
 674         # check the object has only have one occurence of the single-valued
 675         # attribute and it matches on both DCs
 676         self.assert_attrs_match(res1, res2, "managedBy", 1)
 677
 678         # here we expect DC2 to win because it has the more recent link
 679         self.assertTrue(res1[0]["managedBy"][0] == target2_ou,
 680                         "Expected most recent update to win conflict")
 681
 682         # we can't query the deleted links over LDAP, but we can check DRS
 683         # to make sure the DC kept a copy of the conflicting link
 684         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 685                              misc.GUID(src_guid), misc.GUID(target1_guid))
 686         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 687                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 688                              misc.GUID(src_guid), misc.GUID(target2_guid))
 689         self._check_replicated_links(src_ou, [link1, link2])
 690
 691     def test_conflict_existing_single_valued_link(self):
 692         # repeat the test twice, to give each DC a chance to resolve the conflict
 693         self._test_conflict_existing_single_valued_link(sync_order=DC1_TO_DC2)
 694         self._test_conflict_existing_single_valued_link(sync_order=DC2_TO_DC1)
 695
 696     def test_link_attr_version(self):
 697         """
 698         Checks the link attribute version starts from the correct value
 699         """
 700         # create some objects and add a link
 701         src_ou = self.unique_dn("OU=src")
 702         self.add_object(self.ldb_dc1, src_ou)
 703         target1_ou = self.unique_dn("OU=target1")
 704         self.add_object(self.ldb_dc1, target1_ou)
 705         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 706
 707         # get the link info via replication
 708         ctr6 = self._get_replication(drsuapi.DRSUAPI_DRS_WRIT_REP,
 709                                      dest_dsa=None,
 710                                      drs_error=drsuapi.DRSUAPI_EXOP_ERR_SUCCESS,
 711                                      exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 712                                      highwatermark=self.zero_highwatermark(),
 713                                      nc_dn_str=src_ou)
 714
 715         self.assertTrue(ctr6.linked_attributes_count == 1,
 716                         "DRS didn't return a link")
 717         link = ctr6.linked_attributes[0]
 718         self.assertTrue(link.meta_data.version == 1,
 719                         "Link version started from %u, not 1" % link.meta_data.version)
 720