source4/torture/drs/python/link_conflicts.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Tests replication scenarios that involve conflicting linked attribute
   5 # information between the 2 DCs.
   6 #
   7 # Copyright (C) Catalyst.Net Ltd. 2017
   8 #
   9 # This program is free software; you can redistribute it and/or modify
  10 # it under the terms of the GNU General Public License as published by
  11 # the Free Software Foundation; either version 3 of the License, or
  12 # (at your option) any later version.
  13 #
  14 # This program is distributed in the hope that it will be useful,
  15 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 # GNU General Public License for more details.
  18 #
  19 # You should have received a copy of the GNU General Public License
  20 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  21 #
  22
  23 #
  24 # Usage:
  25 #  export DC1=dc1_dns_name
  26 #  export DC2=dc2_dns_name
  27 #  export SUBUNITRUN=$samba4srcdir/scripting/bin/subunitrun
  28 #  PYTHONPATH="$PYTHONPATH:$samba4srcdir/torture/drs/python" $SUBUNITRUN \
  29 #       link_conflicts -U"$DOMAIN/$DC_USERNAME"%"$DC_PASSWORD"
  30 #
  31
  32 import drs_base
  33 import samba.tests
  34 import ldb
  35 from ldb import SCOPE_BASE
  36 import random
  37 import time
  38
  39 from drs_base import AbstractLink
  40 from samba.dcerpc import drsuapi, misc
  41 from samba.dcerpc.drsuapi import DRSUAPI_EXOP_ERR_SUCCESS
  42
  43 # specifies the order to sync DCs in
  44 DC1_TO_DC2 = 1
  45 DC2_TO_DC1 = 2
  46
  47
  48 class DrsReplicaLinkConflictTestCase(drs_base.DrsBaseTestCase):
  49     def setUp(self):
  50         super(DrsReplicaLinkConflictTestCase, self).setUp()
  51
  52         self.ou = samba.tests.create_test_ou(self.ldb_dc1,
  53                                              "test_link_conflict")
  54         self.base_dn = self.ldb_dc1.get_default_basedn()
  55
  56         (self.drs, self.drs_handle) = self._ds_bind(self.dnsname_dc1)
  57         (self.drs2, self.drs2_handle) = self._ds_bind(self.dnsname_dc2)
  58
  59         # disable replication for the tests so we can control at what point
  60         # the DCs try to replicate
  61         self._disable_inbound_repl(self.dnsname_dc1)
  62         self._disable_inbound_repl(self.dnsname_dc2)
  63
  64     def tearDown(self):
  65         # re-enable replication
  66         self._enable_inbound_repl(self.dnsname_dc1)
  67         self._enable_inbound_repl(self.dnsname_dc2)
  68         self.ldb_dc1.delete(self.ou, ["tree_delete:1"])
  69         super(DrsReplicaLinkConflictTestCase, self).tearDown()
  70
  71     def get_guid(self, samdb, dn):
  72         """Returns an object's GUID (in string format)"""
  73         res = samdb.search(base=dn, attrs=["objectGUID"], scope=ldb.SCOPE_BASE)
  74         return self._GUID_string(res[0]['objectGUID'][0])
  75
  76     def add_object(self, samdb, dn, objectclass="organizationalunit"):
  77         """Adds an object"""
  78         samdb.add({"dn": dn, "objectclass": objectclass})
  79         return self.get_guid(samdb, dn)
  80
  81     def modify_object(self, samdb, dn, attr, value):
  82         """Modifies an attribute for an object"""
  83         m = ldb.Message()
  84         m.dn = ldb.Dn(samdb, dn)
  85         m[attr] = ldb.MessageElement(value, ldb.FLAG_MOD_ADD, attr)
  86         samdb.modify(m)
  87
  88     def add_link_attr(self, samdb, source_dn, attr, target_dn):
  89         """Adds a linked attribute between 2 objects"""
  90         # add the specified attribute to the source object
  91         self.modify_object(samdb, source_dn, attr, target_dn)
  92
  93     def del_link_attr(self, samdb, src, attr, target):
  94         m = ldb.Message()
  95         m.dn = ldb.Dn(samdb, src)
  96         m[attr] = ldb.MessageElement(target, ldb.FLAG_MOD_DELETE, attr)
  97         samdb.modify(m)
  98
  99     def sync_DCs(self, sync_order=DC1_TO_DC2):
 100         """Manually syncs the 2 DCs to ensure they're in sync"""
 101         if sync_order == DC1_TO_DC2:
 102             # sync DC1-->DC2, then DC2-->DC1
 103             self._net_drs_replicate(DC=self.dnsname_dc2,
 104                                     fromDC=self.dnsname_dc1)
 105             self._net_drs_replicate(DC=self.dnsname_dc1,
 106                                     fromDC=self.dnsname_dc2)
 107         else:
 108             # sync DC2-->DC1, then DC1-->DC2
 109             self._net_drs_replicate(DC=self.dnsname_dc1,
 110                                     fromDC=self.dnsname_dc2)
 111             self._net_drs_replicate(DC=self.dnsname_dc2,
 112                                     fromDC=self.dnsname_dc1)
 113
 114     def ensure_unique_timestamp(self):
 115         """Waits a second to ensure a unique timestamp between 2 objects"""
 116         time.sleep(1)
 117
 118     def unique_dn(self, obj_name):
 119         """Returns a unique object DN"""
 120         # Because we run each test case twice, we need to create a unique DN so
 121         # that the 2nd run doesn't hit objects that already exist. Add some
 122         # randomness to the object DN to make it unique
 123         rand = random.randint(1, 10000000)
 124         return "%s-%d,%s" % (obj_name, rand, self.ou)
 125
 126     def assert_attrs_match(self, res1, res2, attr, expected_count):
 127         """
 128         Asserts that the search results contain the expected number of
 129         attributes and the results match on both DCs
 130         """
 131         actual_len = len(res1[0][attr])
 132         self.assertTrue(actual_len == expected_count,
 133                         "Expected %u %s attributes, got %u" % (expected_count,
 134                                                                attr,
 135                                                                actual_len))
 136         actual_len = len(res2[0][attr])
 137         self.assertTrue(actual_len == expected_count,
 138                         "Expected %u %s attributes, got %u" % (expected_count,
 139                                                                attr,
 140                                                                actual_len))
 141
 142         # check DCs both agree on the same linked attributes
 143         for val in res1[0][attr]:
 144             self.assertTrue(val in res2[0][attr],
 145                             "%s '%s' not found on DC2" % (attr, val))
 146
 147     def zero_highwatermark(self):
 148         """Returns a zeroed highwatermark so that all DRS data gets returned"""
 149         hwm = drsuapi.DsReplicaHighWaterMark()
 150         hwm.tmp_highest_usn = 0
 151         hwm.reserved_usn = 0
 152         hwm.highest_usn = 0
 153         return hwm
 154
 155     def _check_replicated_links(self, src_obj_dn, expected_links):
 156         """Checks that replication sends back the expected linked attributes"""
 157         self._check_replication([src_obj_dn],
 158                                 drsuapi.DRSUAPI_DRS_WRIT_REP,
 159                                 dest_dsa=None,
 160                                 drs_error=drsuapi.DRSUAPI_EXOP_ERR_SUCCESS,
 161                                 nc_dn_str=src_obj_dn,
 162                                 exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 163                                 expected_links=expected_links,
 164                                 highwatermark=self.zero_highwatermark())
 165
 166         # Check DC2 as well
 167         self.set_test_ldb_dc(self.ldb_dc2)
 168
 169         self._check_replication([src_obj_dn],
 170                                 drsuapi.DRSUAPI_DRS_WRIT_REP,
 171                                 dest_dsa=None,
 172                                 drs_error=drsuapi.DRSUAPI_EXOP_ERR_SUCCESS,
 173                                 nc_dn_str=src_obj_dn,
 174                                 exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 175                                 expected_links=expected_links,
 176                                 highwatermark=self.zero_highwatermark(),
 177                                 drs=self.drs2, drs_handle=self.drs2_handle)
 178         self.set_test_ldb_dc(self.ldb_dc1)
 179
 180     def _test_conflict_single_valued_link(self, sync_order):
 181         """
 182         Tests a simple single-value link conflict, i.e. each DC adds a link to
 183         the same source object but linking to different targets.
 184         """
 185         src_ou = self.unique_dn("OU=src")
 186         src_guid = self.add_object(self.ldb_dc1, src_ou)
 187         self.sync_DCs()
 188
 189         # create a unique target on each DC
 190         target1_ou = self.unique_dn("OU=target1")
 191         target2_ou = self.unique_dn("OU=target2")
 192
 193         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 194         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 195
 196         # link the test OU to the respective targets created
 197         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 198         self.ensure_unique_timestamp()
 199         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 200
 201         # sync the 2 DCs
 202         self.sync_DCs(sync_order=sync_order)
 203
 204         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 205                                    scope=SCOPE_BASE, attrs=["managedBy"])
 206         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 207                                    scope=SCOPE_BASE, attrs=["managedBy"])
 208
 209         # check the object has only have one occurence of the single-valued
 210         # attribute and it matches on both DCs
 211         self.assert_attrs_match(res1, res2, "managedBy", 1)
 212
 213         self.assertTrue(str(res1[0]["managedBy"][0]) == target2_ou,
 214                         "Expected most recent update to win conflict")
 215
 216         # we can't query the deleted links over LDAP, but we can check DRS
 217         # to make sure the DC kept a copy of the conflicting link
 218         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 219                              misc.GUID(src_guid), misc.GUID(target1_guid))
 220         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 221                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 222                              misc.GUID(src_guid), misc.GUID(target2_guid))
 223         self._check_replicated_links(src_ou, [link1, link2])
 224
 225     def test_conflict_single_valued_link(self):
 226         # repeat the test twice, to give each DC a chance to resolve
 227         # the conflict
 228         self._test_conflict_single_valued_link(sync_order=DC1_TO_DC2)
 229         self._test_conflict_single_valued_link(sync_order=DC2_TO_DC1)
 230
 231     def _test_duplicate_single_valued_link(self, sync_order):
 232         """
 233         Adds the same single-valued link on 2 DCs and checks we don't end up
 234         with 2 copies of the link.
 235         """
 236         # create unique objects for the link
 237         target_ou = self.unique_dn("OU=target")
 238         self.add_object(self.ldb_dc1, target_ou)
 239         src_ou = self.unique_dn("OU=src")
 240         src_guid = self.add_object(self.ldb_dc1, src_ou)
 241         self.sync_DCs()
 242
 243         # link the same test OU to the same target on both DCs
 244         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target_ou)
 245         self.ensure_unique_timestamp()
 246         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target_ou)
 247
 248         # sync the 2 DCs
 249         self.sync_DCs(sync_order=sync_order)
 250
 251         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 252                                    scope=SCOPE_BASE, attrs=["managedBy"])
 253         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 254                                    scope=SCOPE_BASE, attrs=["managedBy"])
 255
 256         # check the object has only have one occurence of the single-valued
 257         # attribute and it matches on both DCs
 258         self.assert_attrs_match(res1, res2, "managedBy", 1)
 259
 260     def test_duplicate_single_valued_link(self):
 261         # repeat the test twice, to give each DC a chance to resolve
 262         # the conflict
 263         self._test_duplicate_single_valued_link(sync_order=DC1_TO_DC2)
 264         self._test_duplicate_single_valued_link(sync_order=DC2_TO_DC1)
 265
 266     def _test_conflict_multi_valued_link(self, sync_order):
 267         """
 268         Tests a simple multi-valued link conflict. This adds 2 objects with the
 269         same username on 2 different DCs and checks their group membership is
 270         preserved after the conflict is resolved.
 271         """
 272
 273         # create a common link source
 274         src_dn = self.unique_dn("CN=src")
 275         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 276         self.sync_DCs()
 277
 278         # create the same user (link target) on each DC.
 279         # Note that the GUIDs will differ between the DCs
 280         target_dn = self.unique_dn("CN=target")
 281         target1_guid = self.add_object(self.ldb_dc1, target_dn,
 282                                        objectclass="user")
 283         self.ensure_unique_timestamp()
 284         target2_guid = self.add_object(self.ldb_dc2, target_dn,
 285                                        objectclass="user")
 286
 287         # link the src group to the respective target created
 288         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 289         self.ensure_unique_timestamp()
 290         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 291
 292         # sync the 2 DCs. We expect the more recent target2 object to win
 293         self.sync_DCs(sync_order=sync_order)
 294
 295         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 296                                    scope=SCOPE_BASE, attrs=["member"])
 297         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 298                                    scope=SCOPE_BASE, attrs=["member"])
 299         target1_conflict = False
 300
 301         # we expect exactly 2 members in our test group (both DCs should agree)
 302         self.assert_attrs_match(res1, res2, "member", 2)
 303
 304         for val in [str(val) for val in res1[0]["member"]]:
 305             # check the expected conflicting object was renamed
 306             #val = str(val)
 307             self.assertFalse("CNF:%s" % target2_guid in val)
 308             if "CNF:%s" % target1_guid in val:
 309                 target1_conflict = True
 310
 311         self.assertTrue(target1_conflict,
 312                         "Expected link to conflicting target object not found")
 313
 314     def test_conflict_multi_valued_link(self):
 315         # repeat the test twice, to give each DC a chance to resolve
 316         # the conflict
 317         self._test_conflict_multi_valued_link(sync_order=DC1_TO_DC2)
 318         self._test_conflict_multi_valued_link(sync_order=DC2_TO_DC1)
 319
 320     def _test_duplicate_multi_valued_link(self, sync_order):
 321         """
 322         Adds the same multivalued link on 2 DCs and checks we don't end up
 323         with 2 copies of the link.
 324         """
 325
 326         # create the link source/target objects
 327         src_dn = self.unique_dn("CN=src")
 328         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 329         target_dn = self.unique_dn("CN=target")
 330         self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 331         self.sync_DCs()
 332
 333         # link the src group to the same target user separately on each DC
 334         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 335         self.ensure_unique_timestamp()
 336         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 337
 338         self.sync_DCs(sync_order=sync_order)
 339
 340         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 341                                    scope=SCOPE_BASE, attrs=["member"])
 342         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 343                                    scope=SCOPE_BASE, attrs=["member"])
 344
 345         # we expect to still have only 1 member in our test group
 346         self.assert_attrs_match(res1, res2, "member", 1)
 347
 348     def test_duplicate_multi_valued_link(self):
 349         # repeat the test twice, to give each DC a chance to resolve
 350         # the conflict
 351         self._test_duplicate_multi_valued_link(sync_order=DC1_TO_DC2)
 352         self._test_duplicate_multi_valued_link(sync_order=DC2_TO_DC1)
 353
 354     def _test_conflict_backlinks(self, sync_order):
 355         """
 356         Tests that resolving a source object conflict fixes up any backlinks,
 357         e.g. the same user is added to a conflicting group.
 358         """
 359
 360         # create a common link target
 361         target_dn = self.unique_dn("CN=target")
 362         target_guid = self.add_object(self.ldb_dc1, target_dn,
 363                                       objectclass="user")
 364         self.sync_DCs()
 365
 366         # create the same group (link source) on each DC.
 367         # Note that the GUIDs will differ between the DCs
 368         src_dn = self.unique_dn("CN=src")
 369         src1_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 370         self.ensure_unique_timestamp()
 371         src2_guid = self.add_object(self.ldb_dc2, src_dn, objectclass="group")
 372
 373         # link the src group to the respective target created
 374         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 375         self.ensure_unique_timestamp()
 376         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 377
 378         # sync the 2 DCs. We expect the more recent src2 object to win
 379         self.sync_DCs(sync_order=sync_order)
 380
 381         res1 = self.ldb_dc1.search(base="<GUID=%s>" % target_guid,
 382                                    scope=SCOPE_BASE, attrs=["memberOf"])
 383         res2 = self.ldb_dc2.search(base="<GUID=%s>" % target_guid,
 384                                    scope=SCOPE_BASE, attrs=["memberOf"])
 385         src1_backlink = False
 386
 387         # our test user should still be a member of 2 groups (check both
 388         # DCs agree)
 389         self.assert_attrs_match(res1, res2, "memberOf", 2)
 390
 391         for val in [str(val) for val in res1[0]["memberOf"]]:
 392             # check the conflicting object was renamed
 393             #val = str(val)
 394             self.assertFalse("CNF:%s" % src2_guid in val)
 395             if "CNF:%s" % src1_guid in val:
 396                 src1_backlink = True
 397
 398         self.assertTrue(src1_backlink,
 399                         "Backlink to conflicting source object not found")
 400
 401     def test_conflict_backlinks(self):
 402         # repeat the test twice, to give each DC a chance to resolve
 403         # the conflict
 404         self._test_conflict_backlinks(sync_order=DC1_TO_DC2)
 405         self._test_conflict_backlinks(sync_order=DC2_TO_DC1)
 406
 407     def _test_link_deletion_conflict(self, sync_order):
 408         """
 409         Checks that a deleted link conflicting with an active link is
 410         resolved correctly.
 411         """
 412
 413         # Add the link objects
 414         target_dn = self.unique_dn("CN=target")
 415         self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 416         src_dn = self.unique_dn("CN=src")
 417         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 418         self.sync_DCs()
 419
 420         # add the same link on both DCs, and resolve any conflict
 421         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 422         self.ensure_unique_timestamp()
 423         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 424         self.sync_DCs(sync_order=sync_order)
 425
 426         # delete and re-add the link on one DC
 427         self.del_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 428         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 429
 430         # just delete it on the other DC
 431         self.ensure_unique_timestamp()
 432         self.del_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 433         # sanity-check the link is gone on this DC
 434         res1 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 435                                    scope=SCOPE_BASE, attrs=["member"])
 436         self.assertFalse("member" in res1[0], "Couldn't delete member attr")
 437
 438         # sync the 2 DCs. We expect the more older DC1 attribute to win
 439         # because it has a higher version number (even though it's older)
 440         self.sync_DCs(sync_order=sync_order)
 441
 442         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 443                                    scope=SCOPE_BASE, attrs=["member"])
 444         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 445                                    scope=SCOPE_BASE, attrs=["member"])
 446
 447         # our test user should still be a member of the group (check both
 448         # DCs agree)
 449         self.assertTrue("member" in res1[0],
 450                         "Expected member attribute missing")
 451         self.assert_attrs_match(res1, res2, "member", 1)
 452
 453     def test_link_deletion_conflict(self):
 454         # repeat the test twice, to give each DC a chance to resolve
 455         # the conflict
 456         self._test_link_deletion_conflict(sync_order=DC1_TO_DC2)
 457         self._test_link_deletion_conflict(sync_order=DC2_TO_DC1)
 458
 459     def _test_obj_deletion_conflict(self, sync_order, del_target):
 460         """
 461         Checks that a receiving a new link for a deleted object gets
 462         resolved correctly.
 463         """
 464
 465         target_dn = self.unique_dn("CN=target")
 466         target_guid = self.add_object(self.ldb_dc1, target_dn,
 467                                       objectclass="user")
 468         src_dn = self.unique_dn("CN=src")
 469         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 470
 471         self.sync_DCs()
 472
 473         # delete the object on one DC
 474         if del_target:
 475             search_guid = src_guid
 476             self.ldb_dc2.delete(target_dn)
 477         else:
 478             search_guid = target_guid
 479             self.ldb_dc2.delete(src_dn)
 480
 481         # add a link on the other DC
 482         self.ensure_unique_timestamp()
 483         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 484
 485         self.sync_DCs(sync_order=sync_order)
 486
 487         # the object deletion should trump the link addition.
 488         # Check the link no longer exists on the remaining object
 489         res1 = self.ldb_dc1.search(base="<GUID=%s>" % search_guid,
 490                                    scope=SCOPE_BASE,
 491                                    attrs=["member", "memberOf"])
 492         res2 = self.ldb_dc2.search(base="<GUID=%s>" % search_guid,
 493                                    scope=SCOPE_BASE,
 494                                    attrs=["member", "memberOf"])
 495
 496         self.assertFalse("member" in res1[0], "member attr shouldn't exist")
 497         self.assertFalse("member" in res2[0], "member attr shouldn't exist")
 498         self.assertFalse("memberOf" in res1[0], "member attr shouldn't exist")
 499         self.assertFalse("memberOf" in res2[0], "member attr shouldn't exist")
 500
 501     def test_obj_deletion_conflict(self):
 502         # repeat the test twice, to give each DC a chance to resolve
 503         # the conflict
 504         self._test_obj_deletion_conflict(sync_order=DC1_TO_DC2,
 505                                          del_target=True)
 506         self._test_obj_deletion_conflict(sync_order=DC2_TO_DC1,
 507                                          del_target=True)
 508
 509         # and also try deleting the source object instead of the link target
 510         self._test_obj_deletion_conflict(sync_order=DC1_TO_DC2,
 511                                          del_target=False)
 512         self._test_obj_deletion_conflict(sync_order=DC2_TO_DC1,
 513                                          del_target=False)
 514
 515     def _test_full_sync_link_conflict(self, sync_order):
 516         """
 517         Checks that doing a full sync doesn't affect how conflicts get resolved
 518         """
 519
 520         # create the objects for the linked attribute
 521         src_dn = self.unique_dn("CN=src")
 522         src_guid = self.add_object(self.ldb_dc1, src_dn, objectclass="group")
 523         target_dn = self.unique_dn("CN=target")
 524         self.add_object(self.ldb_dc1, target_dn, objectclass="user")
 525         self.sync_DCs()
 526
 527         # add the same link on both DCs
 528         self.add_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 529         self.ensure_unique_timestamp()
 530         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 531
 532         # Do a couple of full syncs which should resolve the conflict
 533         # (but only for one DC)
 534         if sync_order == DC1_TO_DC2:
 535             self._net_drs_replicate(DC=self.dnsname_dc2,
 536                                     fromDC=self.dnsname_dc1,
 537                                     full_sync=True)
 538             self._net_drs_replicate(DC=self.dnsname_dc2,
 539                                     fromDC=self.dnsname_dc1,
 540                                     full_sync=True)
 541         else:
 542             self._net_drs_replicate(DC=self.dnsname_dc1,
 543                                     fromDC=self.dnsname_dc2,
 544                                     full_sync=True)
 545             self._net_drs_replicate(DC=self.dnsname_dc1,
 546                                     fromDC=self.dnsname_dc2,
 547                                     full_sync=True)
 548
 549         # delete and re-add the link on one DC
 550         self.del_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 551         self.ensure_unique_timestamp()
 552         self.add_link_attr(self.ldb_dc1, src_dn, "member", target_dn)
 553
 554         # just delete the link on the 2nd DC
 555         self.ensure_unique_timestamp()
 556         self.del_link_attr(self.ldb_dc2, src_dn, "member", target_dn)
 557
 558         # sync the 2 DCs. We expect DC1 to win based on version number
 559         self.sync_DCs(sync_order=sync_order)
 560
 561         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 562                                    scope=SCOPE_BASE, attrs=["member"])
 563         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 564                                    scope=SCOPE_BASE, attrs=["member"])
 565
 566         # check the membership still exits (and both DCs agree)
 567         self.assertTrue("member" in res1[0],
 568                         "Expected member attribute missing")
 569         self.assert_attrs_match(res1, res2, "member", 1)
 570
 571     def test_full_sync_link_conflict(self):
 572         # repeat the test twice, to give each DC a chance to resolve
 573         # the conflict
 574         self._test_full_sync_link_conflict(sync_order=DC1_TO_DC2)
 575         self._test_full_sync_link_conflict(sync_order=DC2_TO_DC1)
 576
 577     def _singleval_link_conflict_deleted_winner(self, sync_order):
 578         """
 579         Tests a single-value link conflict where the more-up-to-date link value
 580         is deleted.
 581         """
 582         src_ou = self.unique_dn("OU=src")
 583         src_guid = self.add_object(self.ldb_dc1, src_ou)
 584         self.sync_DCs()
 585
 586         # create a unique target on each DC
 587         target1_ou = self.unique_dn("OU=target1")
 588         target2_ou = self.unique_dn("OU=target2")
 589
 590         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 591         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 592
 593         # add the links for the respective targets, and delete one of the links
 594         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 595         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 596         self.ensure_unique_timestamp()
 597         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 598
 599         # sync the 2 DCs
 600         self.sync_DCs(sync_order=sync_order)
 601
 602         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 603                                    scope=SCOPE_BASE, attrs=["managedBy"])
 604         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 605                                    scope=SCOPE_BASE, attrs=["managedBy"])
 606
 607         # Although the more up-to-date link value is deleted, this shouldn't
 608         # trump DC1's active link
 609         self.assert_attrs_match(res1, res2, "managedBy", 1)
 610
 611         self.assertTrue(str(res1[0]["managedBy"][0]) == target2_ou,
 612                         "Expected active link win conflict")
 613
 614         # we can't query the deleted links over LDAP, but we can check that
 615         # the deleted links exist using DRS
 616         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 617                              misc.GUID(src_guid), misc.GUID(target1_guid))
 618         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 619                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 620                              misc.GUID(src_guid), misc.GUID(target2_guid))
 621         self._check_replicated_links(src_ou, [link1, link2])
 622
 623     def test_conflict_single_valued_link_deleted_winner(self):
 624         # repeat the test twice, to give each DC a chance to resolve
 625         # the conflict
 626         self._singleval_link_conflict_deleted_winner(sync_order=DC1_TO_DC2)
 627         self._singleval_link_conflict_deleted_winner(sync_order=DC2_TO_DC1)
 628
 629     def _singleval_link_conflict_deleted_loser(self, sync_order):
 630         """
 631         Tests a single-valued link conflict, where the losing link value is
 632         deleted.
 633         """
 634         src_ou = self.unique_dn("OU=src")
 635         src_guid = self.add_object(self.ldb_dc1, src_ou)
 636         self.sync_DCs()
 637
 638         # create a unique target on each DC
 639         target1_ou = self.unique_dn("OU=target1")
 640         target2_ou = self.unique_dn("OU=target2")
 641
 642         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 643         target2_guid = self.add_object(self.ldb_dc2, target2_ou)
 644
 645         # add the links - we want the link to end up deleted on DC2, but active
 646         # on DC1. DC1 has the better version and DC2 has the better timestamp -
 647         # the better version should win
 648         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 649         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 650         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 651         self.ensure_unique_timestamp()
 652         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 653         self.del_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 654
 655         self.sync_DCs(sync_order=sync_order)
 656
 657         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 658                                    scope=SCOPE_BASE, attrs=["managedBy"])
 659         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 660                                    scope=SCOPE_BASE, attrs=["managedBy"])
 661
 662         # check the object has only have one occurence of the single-valued
 663         # attribute and it matches on both DCs
 664         self.assert_attrs_match(res1, res2, "managedBy", 1)
 665
 666         self.assertTrue(str(res1[0]["managedBy"][0]) == target1_ou,
 667                         "Expected most recent update to win conflict")
 668
 669         # we can't query the deleted links over LDAP, but we can check DRS
 670         # to make sure the DC kept a copy of the conflicting link
 671         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 672                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 673                              misc.GUID(src_guid), misc.GUID(target1_guid))
 674         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 675                              misc.GUID(src_guid), misc.GUID(target2_guid))
 676         self._check_replicated_links(src_ou, [link1, link2])
 677
 678     def test_conflict_single_valued_link_deleted_loser(self):
 679         # repeat the test twice, to give each DC a chance to resolve
 680         # the conflict
 681         self._singleval_link_conflict_deleted_loser(sync_order=DC1_TO_DC2)
 682         self._singleval_link_conflict_deleted_loser(sync_order=DC2_TO_DC1)
 683
 684     def _test_conflict_existing_single_valued_link(self, sync_order):
 685         """
 686         Tests a single-valued link conflict, where the conflicting link value
 687         already exists (as inactive) on both DCs.
 688         """
 689         # create the link objects
 690         src_ou = self.unique_dn("OU=src")
 691         src_guid = self.add_object(self.ldb_dc1, src_ou)
 692
 693         target1_ou = self.unique_dn("OU=target1")
 694         target2_ou = self.unique_dn("OU=target2")
 695         target1_guid = self.add_object(self.ldb_dc1, target1_ou)
 696         target2_guid = self.add_object(self.ldb_dc1, target2_ou)
 697
 698         # add the links, but then delete them
 699         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 700         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 701         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target2_ou)
 702         self.del_link_attr(self.ldb_dc1, src_ou, "managedBy", target2_ou)
 703         self.sync_DCs()
 704
 705         # re-add the links independently on each DC
 706         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 707         self.ensure_unique_timestamp()
 708         self.add_link_attr(self.ldb_dc2, src_ou, "managedBy", target2_ou)
 709
 710         # try to sync the 2 DCs
 711         self.sync_DCs(sync_order=sync_order)
 712
 713         res1 = self.ldb_dc1.search(base="<GUID=%s>" % src_guid,
 714                                    scope=SCOPE_BASE, attrs=["managedBy"])
 715         res2 = self.ldb_dc2.search(base="<GUID=%s>" % src_guid,
 716                                    scope=SCOPE_BASE, attrs=["managedBy"])
 717
 718         # check the object has only have one occurence of the single-valued
 719         # attribute and it matches on both DCs
 720         self.assert_attrs_match(res1, res2, "managedBy", 1)
 721
 722         # here we expect DC2 to win because it has the more recent link
 723         self.assertTrue(str(res1[0]["managedBy"][0]) == target2_ou,
 724                         "Expected most recent update to win conflict")
 725
 726         # we can't query the deleted links over LDAP, but we can check DRS
 727         # to make sure the DC kept a copy of the conflicting link
 728         link1 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy, 0,
 729                              misc.GUID(src_guid), misc.GUID(target1_guid))
 730         link2 = AbstractLink(drsuapi.DRSUAPI_ATTID_managedBy,
 731                              drsuapi.DRSUAPI_DS_LINKED_ATTRIBUTE_FLAG_ACTIVE,
 732                              misc.GUID(src_guid), misc.GUID(target2_guid))
 733         self._check_replicated_links(src_ou, [link1, link2])
 734
 735     def test_conflict_existing_single_valued_link(self):
 736         # repeat the test twice, to give each DC a chance to resolve
 737         # the conflict
 738         self._test_conflict_existing_single_valued_link(sync_order=DC1_TO_DC2)
 739         self._test_conflict_existing_single_valued_link(sync_order=DC2_TO_DC1)
 740
 741     def test_link_attr_version(self):
 742         """
 743         Checks the link attribute version starts from the correct value
 744         """
 745         # create some objects and add a link
 746         src_ou = self.unique_dn("OU=src")
 747         self.add_object(self.ldb_dc1, src_ou)
 748         target1_ou = self.unique_dn("OU=target1")
 749         self.add_object(self.ldb_dc1, target1_ou)
 750         self.add_link_attr(self.ldb_dc1, src_ou, "managedBy", target1_ou)
 751
 752         # get the link info via replication
 753         ctr6 = self._get_replication(drsuapi.DRSUAPI_DRS_WRIT_REP,
 754                                      dest_dsa=None,
 755                                      drs_error=DRSUAPI_EXOP_ERR_SUCCESS,
 756                                      exop=drsuapi.DRSUAPI_EXOP_REPL_OBJ,
 757                                      highwatermark=self.zero_highwatermark(),
 758                                      nc_dn_str=src_ou)
 759
 760         self.assertTrue(ctr6.linked_attributes_count == 1,
 761                         "DRS didn't return a link")
 762         link = ctr6.linked_attributes[0]
 763         rcvd_version = link.meta_data.version
 764         self.assertTrue(rcvd_version == 1,
 765                         "Link version started from %u, not 1" % rcvd_version)