source4/torture/drs/python/link_conflicts.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Tests replication scenarios that involve conflicting linked attribute
   5 # information between the 2 DCs.
   6 #
   7 # Copyright (C) Catalyst.Net Ltd. 2017
   8 #
   9 # This program is free software; you can redistribute it and/or modify
  10 # it under the terms of the GNU General Public License as published by
  11 # the Free Software Foundation; either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21 #
  22
  23 #
  24 # Usage:
  25 #  export DC1=dc1_dns_name
  26 #  export DC2=dc2_dns_name
  27 #  export SUBUNITRUN=$samba4srcdir/scripting/bin/subunitrun
  28 #  PYTHONPATH="$PYTHONPATH:$samba4srcdir/torture/drs/python" $SUBUNITRUN \
  29 #       link_conflicts -U"$DOMAIN/$DC_USERNAME"%"$DC_PASSWORD"
  30 #
  31
  32 import drs_base
  33 import samba.tests
  34 import ldb
  35 from ldb import SCOPE_BASE
  36 import random
  37 import time
  38
  39 from drs_base import AbstractLink
  40 from samba.dcerpc import drsuapi, misc
  41 from samba.dcerpc.drsuapi import DRSUAPI_EXOP_ERR_SUCCESS
  42
  43 # specifies the order to sync DCs in
  44 DC1_TO_DC2 = 1
  45 DC2_TO_DC1 = 2
  46
  47
  48 class DrsReplicaLinkConflictTestCase(drs_base.DrsBaseTestCase):
  49     def setUp(self):
  50         super(DrsReplicaLinkConflictTestCase, self).setUp()
  51
  52         self.ou = samba.tests.create_test_ou(self.ldb_dc1,
  53                                              "test_link_conflict")
  54         self.base_dn = self.ldb_dc1.get_default_basedn()
  55
  56         (self.drs, self.drs_handle) = self._ds_bind(self.dnsname_dc1)
  57         (self.drs2, self.drs2_handle) = self._ds_bind(self.dnsname_dc2)
  58
  59         # disable replication for the tests so we can control at what point
  60         # the DCs try to replicate
  61         self._disable_inbound_repl(self.dnsname_dc1)
  62         self._disable_inbound_repl(self.dnsname_dc2)
  63
  64     def tearDown(self):
  65         # re-enable replication
  66         self._enable_inbound_repl(self.dnsname_dc1)
  67         self._enable_inbound_repl(self.dnsname_dc2)
  68         self.ldb_dc1.delete(self.ou, ["tree_delete:1"])
  69         super(DrsReplicaLinkConflictTestCase, self).tearDown()
  70
  71     def get_guid(self, samdb, dn):
  72         """Returns an object's GUID (in string format)"""
  73         res = samdb.search(base=dn, attrs=["objectGUID"], scope=ldb.SCOPE_BASE)
  74         return self._GUID_string(res[0]['objectGUID'][0])
  75
  76     def add_object(self, samdb, dn, objectclass="organizationalunit"):
  77         """Adds an object"""
  78         samdb.add({"dn": dn, "objectclass": objectclass})
  79         return self.get_guid(samdb, dn)
  80
  81     def modify_object(self, samdb, dn, attr, value):
  82         """Modifies an attribute for an object"""
  83         m = ldb.Message()
  84         m.dn = ldb.Dn(samdb, dn)
  85         m[attr] = ldb.MessageElement(value, ldb.FLAG_MOD_ADD, attr)
  86         samdb.modify(m)
  87
  88     def add_link_attr(self, samdb, source_dn, attr, target_dn):
  89         """Adds a linked attribute between 2 objects"""
  90         # add the specified attribute to the source object
  91         self.modify_object(samdb, source_dn, attr, target_dn)
  92
  93     def del_link_attr(self, samdb, src, attr, target):
  94         m = ldb.Message()
  95         m.dn = ldb.Dn(samdb, src)
  96         m[attr] = ldb.MessageElement(target, ldb.FLAG_MOD_DELETE, attr)
  97         samdb.modify(m)
  98
  99     def sync_DCs(self, sync_order=DC1_TO_DC2):
 100         """Manually syncs the 2 DCs to ensure they're in sync"""
 101         if sync_order == DC1_TO_DC2:
 102             # sync DC1-->DC2, then DC2-->DC1
 103             self._net_drs_replicate(DC=self.dnsname_dc2,
 104                                     fromDC=self.dnsname_dc1)
 105             self._net_drs_replicate(DC=self.dnsname_dc1,
 106                                     fromDC=self.dnsname_dc2)
 107         else:
 108             # sync DC2-->DC1, then DC1-->DC2
 109             self._net_drs_replicate(DC=self.dnsname_dc1,
 110                                     fromDC=self.dnsname_dc2)
 111             self._net_drs_replicate(DC=self.dnsname_dc2,
 112                                     fromDC=self.dnsname_dc1)
 113
 114     def ensure_unique_timestamp(self):
 115         """Waits a second to ensure a unique timestamp between 2 objects"""
 116         time.sleep(1)
 117
 118     def unique_dn(self, obj_name):
 119         """Returns a unique object DN"""
 120         # Because we run each test case twice, we need to create a unique DN so
 121         # that the 2nd run doesn't hit objects that already exist. Add some
 122         # randomness to the object DN to make it unique
 123         rand = random.randint(1, 10000000)
 124         return "%s-%d,%s" % (obj_name, rand, self.ou)
 125
 126     def assert_attrs_match(self, res1, res2, attr, expected_count):
 127         """
 128         Asserts that the search results contain the expected number of
 129         attributes and the results match on both DCs
 130         """
 131         actual_len = len(res1[0][attr])
 132         self.assertTrue(actual_len == expected_count,
 133                         "Expected %u %s attributes, got %u" % (expected_count,
 134                                                                attr,
 135                                                                actual_len))
 136         actual_len = len(res2[0][attr])
 137         self.assertTrue(actual_len == expected_count,
 138                         "Expected %u %s attributes, got %u" % (expected_count,
 139                                                                attr,
 140                                                                actual_len))
 141
 142         # check DCs both agree on the same linked attributes
 143         for val in res1[0][attr]:
 144             self.assertTrue(val in res2[0][attr],
 145                             "%s '%s' not found on DC2" % (attr, val))
 146
 147     def zero_highwatermark(self):
 148         """Returns a zeroed highwatermark so that all DRS data gets returned"""
 149         hwm = drsuapi.DsReplicaHighWaterMark()
 150         hwm.tmp_highest_usn = 0
 151         hwm.reserved_usn = 0
 152         hwm.highest_usn = 0
 153         return hwm
 154
 155     def _check_replicated_links(self, src_obj_dn, expected_links):
 156         """Checks that replication sends back the expected linked attributes"""
 157         self._check_replication([src_obj_dn],
 158                                 drsuapi.DRSUAPI_DRS_WRIT_REP,
 159                                 dest_dsa=None,
 160                                 drs_error=drsuapi.DRSUAPI_EXOP_ERR_SUCCESS,
 161                                 nc_dn_str=src_obj_dn,
 162                                 exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 163                                 expected_links=expected_links,
 164                                 highwatermark=self.zero_highwatermark())
 165
 166         # Check DC2 as well
 167         self.set_test_ldb_dc(self.ldb_dc2)
 168
 169         self._check_replication([src_obj_dn],
 170                                 drsuapi.DRSUAPI_DRS_WRIT_REP,
 171                                 dest_dsa=None,
 172                                 drs_error=drsuapi.DRSUAPI_EXOP_ERR_SUCCESS,
 173                                 nc_dn_str=src_obj_dn,
 174                                 exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 175                                 expected_links=expected_links,
 176                                 highwatermark=self.zero_highwatermark(),
 177                                 drs=self.drs2, drs_handle=self.drs2_handle)
 178         self.set_test_ldb_dc(self.ldb_dc1)
 179
 180     def _test_conflict_single_valued_link(self, sync_order):
 181         """
 182         Tests a simple single-value link conflict, i.e. each DC adds a link to
 183         the same source object but linking to different targets.
 184         """
 185         src_ou = self.unique_dn("OU=src")
 186         src_guid = self.add_object(self.ldb_dc1, src_ou)
 187         self.sync_DCs()
 188
 189         # create a unique target on each DC
 190         target1_ou = self.unique_dn("OU=target1")
 191         target2_ou = self.unique_dn("OU=target2")
 192
 193         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 194         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 195
 196         # link the test OU to the respective targets created
 197         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 198         self.ensure_unique_timestamp()
 199         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 200
 201         # sync the 2 DCs
 202         self.sync_DCs(sync_order=sync_order)
 203
 204         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 205                                   scope=SCOPE_BASE, attrs=["managedBy"])
 206         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 207                                   scope=SCOPE_BASE, attrs=["managedBy"])
 208
 209         # check the object has only have one occurence of the single-valued
 210         # attribute and it matches on both DCs
 211         self.assert_attrs_match(res1, res2, "managedBy", 1)
 212
 213         self.assertTrue(res1[0]["managedBy"][0] == target2_ou,
 214                         "Expected most recent update to win conflict")
 215
 216         # we can't query the deleted links over LDAP, but we can check DRS
 217         # to make sure the DC kept a copy of the conflicting link
 218         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 219                              misc.GUID(src_guid), misc.GUID(target1_guid))
 220         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 221                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 222                              misc.GUID(src_guid), misc.GUID(target2_guid))
 223         self._check_replicated_links(src_ou, [link1, link2])
 224
 225     def test_conflict_single_valued_link(self):
 226         # repeat the test twice, to give each DC a chance to resolve
 227         # the conflict
 228         self._test_conflict_single_valued_link(sync_order=DC1_TO_DC2)
 229         self._test_conflict_single_valued_link(sync_order=DC2_TO_DC1)
 230
 231     def _test_duplicate_single_valued_link(self, sync_order):
 232         """
 233         Adds the same single-valued link on 2 DCs and checks we don't end up
 234         with 2 copies of the link.
 235         """
 236         # create unique objects for the link
 237         target_ou = self.unique_dn("OU=target")
 238         self.add_object(self.ldb_dc1, target_ou)
 239         src_ou = self.unique_dn("OU=src")
 240         src_guid = self.add_object(self.ldb_dc1, src_ou)
 241         self.sync_DCs()
 242
 243         # link the same test OU to the same target on both DCs
 244         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target_ou)
 245         self.ensure_unique_timestamp()
 246         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target_ou)
 247
 248         # sync the 2 DCs
 249         self.sync_DCs(sync_order=sync_order)
 250
 251         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 252                                   scope=SCOPE_BASE, attrs=["managedBy"])
 253         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 254                                   scope=SCOPE_BASE, attrs=["managedBy"])
 255
 256         # check the object has only have one occurence of the single-valued
 257         # attribute and it matches on both DCs
 258         self.assert_attrs_match(res1, res2, "managedBy", 1)
 259
 260     def test_duplicate_single_valued_link(self):
 261         # repeat the test twice, to give each DC a chance to resolve
 262         # the conflict
 263         self._test_duplicate_single_valued_link(sync_order=DC1_TO_DC2)
 264         self._test_duplicate_single_valued_link(sync_order=DC2_TO_DC1)
 265
 266     def _test_conflict_multi_valued_link(self, sync_order):
 267         """
 268         Tests a simple multi-valued link conflict. This adds 2 objects with the
 269         same username on 2 different DCs and checks their group membership is
 270         preserved after the conflict is resolved.
 271         """
 272
 273         # create a common link source
 274         src_dn = self.unique_dn("CN=src")
 275         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 276         self.sync_DCs()
 277
 278         # create the same user (link target) on each DC.
 279         # Note that the GUIDs will differ between the DCs
 280         target_dn = self.unique_dn("CN=target")
 281         target1_guid = self.add_object(self.ldb_dc1, target_dn,
 282                                        objectclass="user")
 283         self.ensure_unique_timestamp()
 284         target2_guid = self.add_object(self.ldb_dc2, target_dn,
 285                                        objectclass="user")
 286
 287         # link the src group to the respective target created
 288         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 289         self.ensure_unique_timestamp()
 290         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 291
 292         # sync the 2 DCs. We expect the more recent target2 object to win
 293         self.sync_DCs(sync_order=sync_order)
 294
 295         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 296                                    scope=SCOPE_BASE, attrs=["member"])
 297         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 298                                    scope=SCOPE_BASE, attrs=["member"])
 299         target1_conflict = False
 300
 301         # we expect exactly 2 members in our test group (both DCs should agree)
 302         self.assert_attrs_match(res1, res2, "member", 2)
 303
 304         for val in res1[0]["member"]:
 305             # check the expected conflicting object was renamed
 306             self.assertFalse("CNF:%s" % target2_guid in val)
 307             if "CNF:%s" % target1_guid in val:
 308                 target1_conflict = True
 309
 310         self.assertTrue(target1_conflict,
 311                         "Expected link to conflicting target object not found")
 312
 313     def test_conflict_multi_valued_link(self):
 314         # repeat the test twice, to give each DC a chance to resolve
 315         # the conflict
 316         self._test_conflict_multi_valued_link(sync_order=DC1_TO_DC2)
 317         self._test_conflict_multi_valued_link(sync_order=DC2_TO_DC1)
 318
 319     def _test_duplicate_multi_valued_link(self, sync_order):
 320         """
 321         Adds the same multivalued link on 2 DCs and checks we don't end up
 322         with 2 copies of the link.
 323         """
 324
 325         # create the link source/target objects
 326         src_dn = self.unique_dn("CN=src")
 327         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 328         target_dn = self.unique_dn("CN=target")
 329         self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 330         self.sync_DCs()
 331
 332         # link the src group to the same target user separately on each DC
 333         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 334         self.ensure_unique_timestamp()
 335         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 336
 337         self.sync_DCs(sync_order=sync_order)
 338
 339         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 340                                    scope=SCOPE_BASE, attrs=["member"])
 341         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 342                                    scope=SCOPE_BASE, attrs=["member"])
 343
 344         # we expect to still have only 1 member in our test group
 345         self.assert_attrs_match(res1, res2, "member", 1)
 346
 347     def test_duplicate_multi_valued_link(self):
 348         # repeat the test twice, to give each DC a chance to resolve
 349         # the conflict
 350         self._test_duplicate_multi_valued_link(sync_order=DC1_TO_DC2)
 351         self._test_duplicate_multi_valued_link(sync_order=DC2_TO_DC1)
 352
 353     def _test_conflict_backlinks(self, sync_order):
 354         """
 355         Tests that resolving a source object conflict fixes up any backlinks,
 356         e.g. the same user is added to a conflicting group.
 357         """
 358
 359         # create a common link target
 360         target_dn = self.unique_dn("CN=target")
 361         target_guid = self.add_object(self.ldb_dc1, target_dn,
 362                                       objectclass="user")
 363         self.sync_DCs()
 364
 365         # create the same group (link source) on each DC.
 366         # Note that the GUIDs will differ between the DCs
 367         src_dn = self.unique_dn("CN=src")
 368         src1_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 369         self.ensure_unique_timestamp()
 370         src2_guid = self.add_object(self.ldb_dc2, src_dn, objectclass="group")
 371
 372         # link the src group to the respective target created
 373         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 374         self.ensure_unique_timestamp()
 375         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 376
 377         # sync the 2 DCs. We expect the more recent src2 object to win
 378         self.sync_DCs(sync_order=sync_order)
 379
 380         res1 = self.ldb_dc1.search(base="<GUID=%s>" % target_guid,
 381                                    scope=SCOPE_BASE, attrs=["memberOf"])
 382         res2 = self.ldb_dc2.search(base="<GUID=%s>" % target_guid,
 383                                    scope=SCOPE_BASE, attrs=["memberOf"])
 384         src1_backlink = False
 385
 386         # our test user should still be a member of 2 groups (check both
 387         # DCs agree)
 388         self.assert_attrs_match(res1, res2, "memberOf", 2)
 389
 390         for val in res1[0]["memberOf"]:
 391             # check the conflicting object was renamed
 392             self.assertFalse("CNF:%s" % src2_guid in val)
 393             if "CNF:%s" % src1_guid in val:
 394                 src1_backlink = True
 395
 396         self.assertTrue(src1_backlink,
 397                         "Backlink to conflicting source object not found")
 398
 399     def test_conflict_backlinks(self):
 400         # repeat the test twice, to give each DC a chance to resolve
 401         # the conflict
 402         self._test_conflict_backlinks(sync_order=DC1_TO_DC2)
 403         self._test_conflict_backlinks(sync_order=DC2_TO_DC1)
 404
 405     def _test_link_deletion_conflict(self, sync_order):
 406         """
 407         Checks that a deleted link conflicting with an active link is
 408         resolved correctly.
 409         """
 410
 411         # Add the link objects
 412         target_dn = self.unique_dn("CN=target")
 413         self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 414         src_dn = self.unique_dn("CN=src")
 415         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 416         self.sync_DCs()
 417
 418         # add the same link on both DCs, and resolve any conflict
 419         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 420         self.ensure_unique_timestamp()
 421         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 422         self.sync_DCs(sync_order=sync_order)
 423
 424         # delete and re-add the link on one DC
 425         self.del_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 426         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 427
 428         # just delete it on the other DC
 429         self.ensure_unique_timestamp()
 430         self.del_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 431         # sanity-check the link is gone on this DC
 432         res1 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 433                                    scope=SCOPE_BASE, attrs=["member"])
 434         self.assertFalse("member" in res1[0], "Couldn't delete member attr")
 435
 436         # sync the 2 DCs. We expect the more older DC1 attribute to win
 437         # because it has a higher version number (even though it's older)
 438         self.sync_DCs(sync_order=sync_order)
 439
 440         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 441                                    scope=SCOPE_BASE, attrs=["member"])
 442         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 443                                    scope=SCOPE_BASE, attrs=["member"])
 444
 445         # our test user should still be a member of the group (check both
 446         # DCs agree)
 447         self.assertTrue("member" in res1[0],
 448                         "Expected member attribute missing")
 449         self.assert_attrs_match(res1, res2, "member", 1)
 450
 451     def test_link_deletion_conflict(self):
 452         # repeat the test twice, to give each DC a chance to resolve
 453         # the conflict
 454         self._test_link_deletion_conflict(sync_order=DC1_TO_DC2)
 455         self._test_link_deletion_conflict(sync_order=DC2_TO_DC1)
 456
 457     def _test_obj_deletion_conflict(self, sync_order, del_target):
 458         """
 459         Checks that a receiving a new link for a deleted object gets
 460         resolved correctly.
 461         """
 462
 463         target_dn = self.unique_dn("CN=target")
 464         target_guid = self.add_object(self.ldb_dc1, target_dn,
 465                                       objectclass="user")
 466         src_dn = self.unique_dn("CN=src")
 467         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 468
 469         self.sync_DCs()
 470
 471         # delete the object on one DC
 472         if del_target:
 473             search_guid = src_guid
 474             self.ldb_dc2.delete(target_dn)
 475         else:
 476             search_guid = target_guid
 477             self.ldb_dc2.delete(src_dn)
 478
 479         # add a link on the other DC
 480         self.ensure_unique_timestamp()
 481         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 482
 483         self.sync_DCs(sync_order=sync_order)
 484
 485         # the object deletion should trump the link addition.
 486         # Check the link no longer exists on the remaining object
 487         res1 = self.ldb_dc1.search(base="<GUID=%s>" % search_guid,
 488                                    scope=SCOPE_BASE,
 489                                    attrs=["member", "memberOf"])
 490         res2 = self.ldb_dc2.search(base="<GUID=%s>" % search_guid,
 491                                    scope=SCOPE_BASE,
 492                                    attrs=["member", "memberOf"])
 493
 494         self.assertFalse("member" in res1[0], "member attr shouldn't exist")
 495         self.assertFalse("member" in res2[0], "member attr shouldn't exist")
 496         self.assertFalse("memberOf" in res1[0], "member attr shouldn't exist")
 497         self.assertFalse("memberOf" in res2[0], "member attr shouldn't exist")
 498
 499     def test_obj_deletion_conflict(self):
 500         # repeat the test twice, to give each DC a chance to resolve
 501         # the conflict
 502         self._test_obj_deletion_conflict(sync_order=DC1_TO_DC2,
 503                                          del_target=True)
 504         self._test_obj_deletion_conflict(sync_order=DC2_TO_DC1,
 505                                          del_target=True)
 506
 507         # and also try deleting the source object instead of the link target
 508         self._test_obj_deletion_conflict(sync_order=DC1_TO_DC2,
 509                                          del_target=False)
 510         self._test_obj_deletion_conflict(sync_order=DC2_TO_DC1,
 511                                          del_target=False)
 512
 513     def _test_full_sync_link_conflict(self, sync_order):
 514         """
 515         Checks that doing a full sync doesn't affect how conflicts get resolved
 516         """
 517
 518         # create the objects for the linked attribute
 519         src_dn = self.unique_dn("CN=src")
 520         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 521         target_dn = self.unique_dn("CN=target")
 522         self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 523         self.sync_DCs()
 524
 525         # add the same link on both DCs
 526         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 527         self.ensure_unique_timestamp()
 528         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 529
 530         # Do a couple of full syncs which should resolve the conflict
 531         # (but only for one DC)
 532         if sync_order == DC1_TO_DC2:
 533             self._net_drs_replicate(DC=self.dnsname_dc2,
 534                                     fromDC=self.dnsname_dc1,
 535                                     full_sync=True)
 536             self._net_drs_replicate(DC=self.dnsname_dc2,
 537                                     fromDC=self.dnsname_dc1,
 538                                     full_sync=True)
 539         else:
 540             self._net_drs_replicate(DC=self.dnsname_dc1,
 541                                     fromDC=self.dnsname_dc2,
 542                                     full_sync=True)
 543             self._net_drs_replicate(DC=self.dnsname_dc1,
 544                                     fromDC=self.dnsname_dc2,
 545                                     full_sync=True)
 546
 547         # delete and re-add the link on one DC
 548         self.del_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 549         self.ensure_unique_timestamp()
 550         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 551
 552         # just delete the link on the 2nd DC
 553         self.ensure_unique_timestamp()
 554         self.del_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 555
 556         # sync the 2 DCs. We expect DC1 to win based on version number
 557         self.sync_DCs(sync_order=sync_order)
 558
 559         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 560                                    scope=SCOPE_BASE, attrs=["member"])
 561         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 562                                    scope=SCOPE_BASE, attrs=["member"])
 563
 564         # check the membership still exits (and both DCs agree)
 565         self.assertTrue("member" in res1[0],
 566                         "Expected member attribute missing")
 567         self.assert_attrs_match(res1, res2, "member", 1)
 568
 569     def test_full_sync_link_conflict(self):
 570         # repeat the test twice, to give each DC a chance to resolve
 571         # the conflict
 572         self._test_full_sync_link_conflict(sync_order=DC1_TO_DC2)
 573         self._test_full_sync_link_conflict(sync_order=DC2_TO_DC1)
 574
 575     def _test_conflict_single_valued_link_deleted_winner(self, sync_order):
 576         """
 577         Tests a single-value link conflict where the more-up-to-date link value
 578         is deleted.
 579         """
 580         src_ou = self.unique_dn("OU=src")
 581         src_guid = self.add_object(self.ldb_dc1, src_ou)
 582         self.sync_DCs()
 583
 584         # create a unique target on each DC
 585         target1_ou = self.unique_dn("OU=target1")
 586         target2_ou = self.unique_dn("OU=target2")
 587
 588         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 589         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 590
 591         # add the links for the respective targets, and delete one of the links
 592         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 593         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 594         self.ensure_unique_timestamp()
 595         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 596
 597         # sync the 2 DCs
 598         self.sync_DCs(sync_order=sync_order)
 599
 600         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 601                                   scope=SCOPE_BASE, attrs=["managedBy"])
 602         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 603                                   scope=SCOPE_BASE, attrs=["managedBy"])
 604
 605         # Although the more up-to-date link value is deleted, this shouldn't
 606         # trump DC1's active link
 607         self.assert_attrs_match(res1, res2, "managedBy", 1)
 608
 609         self.assertTrue(res1[0]["managedBy"][0] == target2_ou,
 610                         "Expected active link win conflict")
 611
 612         # we can't query the deleted links over LDAP, but we can check that
 613         # the deleted links exist using DRS
 614         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 615                              misc.GUID(src_guid), misc.GUID(target1_guid))
 616         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 617                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 618                              misc.GUID(src_guid), misc.GUID(target2_guid))
 619         self._check_replicated_links(src_ou, [link1, link2])
 620
 621     def test_conflict_single_valued_link_deleted_winner(self):
 622         # repeat the test twice, to give each DC a chance to resolve the conflict
 623         self._test_conflict_single_valued_link_deleted_winner(sync_order=DC1_TO_DC2)
 624         self._test_conflict_single_valued_link_deleted_winner(sync_order=DC2_TO_DC1)
 625
 626     def _test_conflict_single_valued_link_deleted_loser(self, sync_order):
 627         """
 628         Tests a single-valued link conflict, where the losing link value is
 629         deleted.
 630         """
 631         src_ou = self.unique_dn("OU=src")
 632         src_guid = self.add_object(self.ldb_dc1, src_ou)
 633         self.sync_DCs()
 634
 635         # create a unique target on each DC
 636         target1_ou = self.unique_dn("OU=target1")
 637         target2_ou = self.unique_dn("OU=target2")
 638
 639         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 640         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 641
 642         # add the links - we want the link to end up deleted on DC2, but active
 643         # on DC1. DC1 has the better version and DC2 has the better timestamp -
 644         # the better version should win
 645         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 646         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 647         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 648         self.ensure_unique_timestamp()
 649         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 650         self.del_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 651
 652         self.sync_DCs(sync_order=sync_order)
 653
 654         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 655                                   scope=SCOPE_BASE, attrs=["managedBy"])
 656         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 657                                   scope=SCOPE_BASE, attrs=["managedBy"])
 658
 659         # check the object has only have one occurence of the single-valued
 660         # attribute and it matches on both DCs
 661         self.assert_attrs_match(res1, res2, "managedBy", 1)
 662
 663         self.assertTrue(res1[0]["managedBy"][0] == target1_ou,
 664                         "Expected most recent update to win conflict")
 665
 666         # we can't query the deleted links over LDAP, but we can check DRS
 667         # to make sure the DC kept a copy of the conflicting link
 668         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 669                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 670                              misc.GUID(src_guid), misc.GUID(target1_guid))
 671         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 672                              misc.GUID(src_guid), misc.GUID(target2_guid))
 673         self._check_replicated_links(src_ou, [link1, link2])
 674
 675     def test_conflict_single_valued_link_deleted_loser(self):
 676         # repeat the test twice, to give each DC a chance to resolve the conflict
 677         self._test_conflict_single_valued_link_deleted_loser(sync_order=DC1_TO_DC2)
 678         self._test_conflict_single_valued_link_deleted_loser(sync_order=DC2_TO_DC1)
 679
 680     def _test_conflict_existing_single_valued_link(self, sync_order):
 681         """
 682         Tests a single-valued link conflict, where the conflicting link value
 683         already exists (as inactive) on both DCs.
 684         """
 685         # create the link objects
 686         src_ou = self.unique_dn("OU=src")
 687         src_guid = self.add_object(self.ldb_dc1, src_ou)
 688
 689         target1_ou = self.unique_dn("OU=target1")
 690         target2_ou = self.unique_dn("OU=target2")
 691         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 692         target2_guid = self.add_object(self.ldb_dc1, target2_ou)
 693
 694         # add the links, but then delete them
 695         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 696         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 697         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target2_ou)
 698         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target2_ou)
 699         self.sync_DCs()
 700
 701         # re-add the links independently on each DC
 702         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 703         self.ensure_unique_timestamp()
 704         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 705
 706         # try to sync the 2 DCs
 707         self.sync_DCs(sync_order=sync_order)
 708
 709         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 710                                   scope=SCOPE_BASE, attrs=["managedBy"])
 711         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 712                                   scope=SCOPE_BASE, attrs=["managedBy"])
 713
 714         # check the object has only have one occurence of the single-valued
 715         # attribute and it matches on both DCs
 716         self.assert_attrs_match(res1, res2, "managedBy", 1)
 717
 718         # here we expect DC2 to win because it has the more recent link
 719         self.assertTrue(res1[0]["managedBy"][0] == target2_ou,
 720                         "Expected most recent update to win conflict")
 721
 722         # we can't query the deleted links over LDAP, but we can check DRS
 723         # to make sure the DC kept a copy of the conflicting link
 724         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 725                              misc.GUID(src_guid), misc.GUID(target1_guid))
 726         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 727                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 728                              misc.GUID(src_guid), misc.GUID(target2_guid))
 729         self._check_replicated_links(src_ou, [link1, link2])
 730
 731     def test_conflict_existing_single_valued_link(self):
 732         # repeat the test twice, to give each DC a chance to resolve
 733         # the conflict
 734         self._test_conflict_existing_single_valued_link(sync_order=DC1_TO_DC2)
 735         self._test_conflict_existing_single_valued_link(sync_order=DC2_TO_DC1)
 736
 737     def test_link_attr_version(self):
 738         """
 739         Checks the link attribute version starts from the correct value
 740         """
 741         # create some objects and add a link
 742         src_ou = self.unique_dn("OU=src")
 743         self.add_object(self.ldb_dc1, src_ou)
 744         target1_ou = self.unique_dn("OU=target1")
 745         self.add_object(self.ldb_dc1, target1_ou)
 746         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 747
 748         # get the link info via replication
 749         ctr6 = self._get_replication(drsuapi.DRSUAPI_DRS_WRIT_REP,
 750                                      dest_dsa=None,
 751                                      drs_error=DRSUAPI_EXOP_ERR_SUCCESS,
 752                                      exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 753                                      highwatermark=self.zero_highwatermark(),
 754                                      nc_dn_str=src_ou)
 755
 756         self.assertTrue(ctr6.linked_attributes_count == 1,
 757                         "DRS didn't return a link")
 758         link = ctr6.linked_attributes[0]
 759         rcvd_version = link.meta_data.version
 760         self.assertTrue(rcvd_version == 1,
 761                         "Link version started from %u, not 1" % rcvd_version)