PEP8: fix E226: missing whitespace around arithmetic operator
[samba.git] / python / samba / dbchecker.py
index 193374041fe64d5bd43287434d1cfe83fd5f6b1e..6e84f1903c8b0ee32bcc1b3271f7ca370ec7db91 100644 (file)
@@ -17,6 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 
+from __future__ import print_function
 import ldb
 import samba
 import time
@@ -65,7 +66,9 @@ class dbcheck(object):
         self.fix_undead_linked_attributes = False
         self.fix_all_missing_backlinks = False
         self.fix_all_orphaned_backlinks = False
-        self.fix_all_duplicate_links = False
+        self.fix_all_missing_forward_links = False
+        self.duplicate_link_cache = dict()
+        self.recover_all_forward_links = False
         self.fix_rmd_flags = False
         self.fix_ntsecuritydescriptor = False
         self.fix_ntsecuritydescriptor_owner_group = False
@@ -103,10 +106,11 @@ class dbcheck(object):
         self.name_map = {}
         try:
             res = samdb.search(base="CN=DnsAdmins,CN=Users,%s" % samdb.domain_dn(), scope=ldb.SCOPE_BASE,
-                           attrs=["objectSid"])
+                               attrs=["objectSid"])
             dnsadmins_sid = ndr_unpack(security.dom_sid, res[0]["objectSid"][0])
             self.name_map['DnsAdmins'] = str(dnsadmins_sid)
-        except ldb.LdbError, (enum, estr):
+        except ldb.LdbError as e5:
+            (enum, estr) = e5.args
             if enum != ldb.ERR_NO_SUCH_OBJECT:
                 raise
             pass
@@ -140,11 +144,11 @@ class dbcheck(object):
 
         for nc in self.ncs:
             try:
-                dn = self.samdb.get_wellknown_dn(ldb.Dn(self.samdb, nc),
+                dn = self.samdb.get_wellknown_dn(ldb.Dn(self.samdb, nc.decode('utf8')),
                                                  dsdb.DS_GUID_DELETED_OBJECTS_CONTAINER)
                 self.deleted_objects_containers.append(dn)
             except KeyError:
-                self.ncs_lacking_deleted_containers.append(ldb.Dn(self.samdb, nc))
+                self.ncs_lacking_deleted_containers.append(ldb.Dn(self.samdb, nc.decode('utf8')))
 
         domaindns_zone = 'DC=DomainDnsZones,%s' % self.samdb.get_default_basedn()
         forestdns_zone = 'DC=ForestDnsZones,%s' % self.samdb.get_root_basedn()
@@ -174,16 +178,34 @@ class dbcheck(object):
         res = self.samdb.search(base=ldb.Dn(self.samdb, self.samdb.get_serverName()),
                                 scope=ldb.SCOPE_BASE, attrs=["serverReference"])
         # 2. Get server reference
-        self.server_ref_dn = ldb.Dn(self.samdb, res[0]['serverReference'][0])
+        self.server_ref_dn = ldb.Dn(self.samdb, res[0]['serverReference'][0].decode('utf8'))
 
         # 3. Get RID Set
         res = self.samdb.search(base=self.server_ref_dn,
                                 scope=ldb.SCOPE_BASE, attrs=['rIDSetReferences'])
         if "rIDSetReferences" in res[0]:
-            self.rid_set_dn = ldb.Dn(self.samdb, res[0]['rIDSetReferences'][0])
+            self.rid_set_dn = ldb.Dn(self.samdb, res[0]['rIDSetReferences'][0].decode('utf8'))
         else:
             self.rid_set_dn = None
 
+        self.compatibleFeatures = []
+        self.requiredFeatures = []
+
+        try:
+            res = self.samdb.search(scope=ldb.SCOPE_BASE,
+                                    base="@SAMBA_DSDB",
+                                    attrs=["compatibleFeatures",
+                                           "requiredFeatures"])
+            if "compatibleFeatures" in res[0]:
+                self.compatibleFeatures = res[0]["compatibleFeatures"]
+            if "requiredFeatures" in res[0]:
+                self.requiredFeatures = res[0]["requiredFeatures"]
+        except ldb.LdbError as e6:
+            (enum, estr) = e6.args
+            if enum != ldb.ERR_NO_SUCH_OBJECT:
+                raise
+            pass
+
     def check_database(self, DN=None, scope=ldb.SCOPE_SUBTREE, controls=[], attrs=['*']):
         '''perform a database check, returning the number of errors found'''
         res = self.samdb.search(base=DN, scope=scope, attrs=['dn'], controls=controls)
@@ -234,7 +256,8 @@ class dbcheck(object):
                                          "CN=Deleted Objects\\0ACNF:%s" % str(misc.GUID(guid)))
                     conflict_dn.add_base(nc)
 
-            except ldb.LdbError, (enum, estr):
+            except ldb.LdbError as e2:
+                (enum, estr) = e2.args
                 if enum == ldb.ERR_NO_SUCH_OBJECT:
                     pass
                 else:
@@ -244,7 +267,8 @@ class dbcheck(object):
             if conflict_dn is not None:
                 try:
                     self.samdb.rename(dn, conflict_dn, ["show_deleted:1", "relax:0", "show_recycled:1"])
-                except ldb.LdbError, (enum, estr):
+                except ldb.LdbError as e1:
+                    (enum, estr) = e1.args
                     self.report("Couldn't move old Deleted Objects placeholder: %s to %s: %s" % (dn, conflict_dn, estr))
                     return 1
 
@@ -262,7 +286,7 @@ class dbcheck(object):
             listwko = []
             proposed_objectguid = None
             for o in wko:
-                dsdb_dn = dsdb_Dn(self.samdb, o, dsdb.DSDB_SYNTAX_BINARY_DN)
+                dsdb_dn = dsdb_Dn(self.samdb, o.decode('utf8'), dsdb.DSDB_SYNTAX_BINARY_DN)
                 if self.is_deleted_objects_dn(dsdb_dn):
                     self.report("wellKnownObjects had duplicate Deleted Objects value %s" % o)
                     # We really want to put this back in the same spot
@@ -350,7 +374,7 @@ systemFlags: -1946157056%s""" % (dn, guid_suffix),
         try:
             controls = controls + ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK]
             self.samdb.delete(dn, controls=controls)
-        except Exception, err:
+        except Exception as err:
             if self.in_transaction:
                 raise CommandError("%s : %s" % (msg, err))
             self.report("%s : %s" % (msg, err))
@@ -364,7 +388,7 @@ systemFlags: -1946157056%s""" % (dn, guid_suffix),
         try:
             controls = controls + ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK]
             self.samdb.modify(m, controls=controls, validate=validate)
-        except Exception, err:
+        except Exception as err:
             if self.in_transaction:
                 raise CommandError("%s : %s" % (msg, err))
             self.report("%s : %s" % (msg, err))
@@ -383,7 +407,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             to_dn = to_rdn + to_base
             controls = controls + ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK]
             self.samdb.rename(from_dn, to_dn, controls=controls)
-        except Exception, err:
+        except Exception as err:
             if self.in_transaction:
                 raise CommandError("%s : %s" % (msg, err))
             self.report("%s : %s" % (msg, err))
@@ -439,7 +463,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             m['value_%u' % i] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
             if nval != '':
                 m['normv_%u' % i] = ldb.MessageElement(nval, ldb.FLAG_MOD_ADD,
-                    attrname)
+                                                       attrname)
 
         if self.do_modify(m, ["relax:0", "show_recycled:1"],
                           "Failed to normalise attribute %s" % attrname,
@@ -573,11 +597,12 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
     def err_missing_dn_GUID_component(self, dn, attrname, val, dsdb_dn, errstr):
         """handle a missing GUID extended DN component"""
         self.report("ERROR: %s component for %s in object %s - %s" % (errstr, attrname, dn, val))
-        controls=["extended_dn:1:1", "show_recycled:1"]
+        controls = ["extended_dn:1:1", "show_recycled:1"]
         try:
             res = self.samdb.search(base=str(dsdb_dn.dn), scope=ldb.SCOPE_BASE,
                                     attrs=[], controls=controls)
-        except ldb.LdbError, (enum, estr):
+        except ldb.LdbError as e7:
+            (enum, estr) = e7.args
             self.report("unable to find object for DN %s - (%s)" % (dsdb_dn.dn, estr))
             if enum != ldb.ERR_NO_SUCH_OBJECT:
                 raise
@@ -604,7 +629,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
     def err_incorrect_binary_dn(self, dn, attrname, val, dsdb_dn, errstr):
         """handle an incorrect binary DN component"""
         self.report("ERROR: %s binary component for %s in object %s - %s" % (errstr, attrname, dn, val))
-        controls=["extended_dn:1:1", "show_recycled:1"]
+        controls = ["extended_dn:1:1", "show_recycled:1"]
 
         if not self.confirm_all('Change DN to %s?' % str(dsdb_dn), 'fix_all_binary_dn'):
             self.report("Not fixing %s" % errstr)
@@ -631,7 +656,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
         m.dn = dn
         m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
         m['new_value'] = ldb.MessageElement(str(dsdb_dn), ldb.FLAG_MOD_ADD, attrname)
-        if self.do_modify(m, ["show_recycled:1"],
+        if self.do_modify(m, ["show_recycled:1",
+                              "local_oid:%s:1" % dsdb.DSDB_CONTROL_DBCHECK_FIX_LINK_DN_NAME],
                           "Failed to fix old DN string on attribute %s" % (attrname)):
             self.report("Fixed old DN string on attribute %s" % (attrname))
 
@@ -708,31 +734,44 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                           "Failed to fix incorrect RMD_FLAGS %u" % rmd_flags):
             self.report("Fixed incorrect RMD_FLAGS %u" % (rmd_flags))
 
-    def err_orphaned_backlink(self, obj, attrname, val, link_name, target_dn):
+    def err_orphaned_backlink(self, obj_dn, backlink_attr, backlink_val,
+                              target_dn, forward_attr, forward_syntax,
+                              check_duplicates=True):
         '''handle a orphaned backlink value'''
-        self.report("ERROR: orphaned backlink attribute '%s' in %s for link %s in %s" % (attrname, obj.dn, link_name, target_dn))
-        if not self.confirm_all('Remove orphaned backlink %s' % attrname, 'fix_all_orphaned_backlinks'):
-            self.report("Not removing orphaned backlink %s" % attrname)
+        if check_duplicates is True and self.has_duplicate_links(target_dn, forward_attr, forward_syntax):
+            self.report("WARNING: Keep orphaned backlink attribute " + \
+                        "'%s' in '%s' for link '%s' in '%s'" % (
+                            backlink_attr, obj_dn, forward_attr, target_dn))
+            return
+        self.report("ERROR: orphaned backlink attribute '%s' in %s for link %s in %s" % (backlink_attr, obj_dn, forward_attr, target_dn))
+        if not self.confirm_all('Remove orphaned backlink %s' % backlink_attr, 'fix_all_orphaned_backlinks'):
+            self.report("Not removing orphaned backlink %s" % backlink_attr)
             return
         m = ldb.Message()
-        m.dn = obj.dn
-        m['value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
+        m.dn = obj_dn
+        m['value'] = ldb.MessageElement(backlink_val, ldb.FLAG_MOD_DELETE, backlink_attr)
         if self.do_modify(m, ["show_recycled:1", "relax:0"],
-                          "Failed to fix orphaned backlink %s" % attrname):
-            self.report("Fixed orphaned backlink %s" % (attrname))
+                          "Failed to fix orphaned backlink %s" % backlink_attr):
+            self.report("Fixed orphaned backlink %s" % (backlink_attr))
 
-    def err_duplicate_links(self, obj, attrname, vals):
+    def err_recover_forward_links(self, obj, forward_attr, forward_vals):
         '''handle a duplicate links value'''
 
-        if not self.confirm_all("Remove duplicate links in attribute '%s'" % attrname, 'fix_all_duplicate_links'):
-            self.report("Not removing duplicate links in attribute '%s'" % attrname)
+        self.report("RECHECK: 'Missing/Duplicate/Correct link' lines above for attribute '%s' in '%s'" % (forward_attr, obj.dn))
+
+        if not self.confirm_all("Commit fixes for (missing/duplicate) forward links in attribute '%s'" % forward_attr, 'recover_all_forward_links'):
+            self.report("Not fixing corrupted (missing/duplicate) forward links in attribute '%s' of '%s'" % (
+                        forward_attr, obj.dn))
             return
         m = ldb.Message()
         m.dn = obj.dn
-        m['value'] = ldb.MessageElement(vals, ldb.FLAG_MOD_REPLACE, attrname)
-        if self.do_modify(m, ["local_oid:1.3.6.1.4.1.7165.4.3.19.2:1"],
-                "Failed to fix duplicate links in attribute '%s'" % attrname):
-            self.report("Fixed duplicate links in attribute '%s'" % (attrname))
+        m['value'] = ldb.MessageElement(forward_vals, ldb.FLAG_MOD_REPLACE, forward_attr)
+        if self.do_modify(m, ["local_oid:%s:1" % dsdb.DSDB_CONTROL_DBCHECK_FIX_DUPLICATE_LINKS],
+                          "Failed to fix duplicate links in attribute '%s'" % forward_attr):
+            self.report("Fixed duplicate links in attribute '%s'" % (forward_attr))
+            duplicate_cache_key = "%s:%s" % (str(obj.dn), forward_attr)
+            assert duplicate_cache_key in self.duplicate_link_cache
+            self.duplicate_link_cache[duplicate_cache_key] = False
 
     def err_no_fsmoRoleOwner(self, obj):
         '''handle a missing fSMORoleOwner'''
@@ -879,35 +918,33 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                                 controls=["show_deleted:0", "extended_dn:0", "reveal_internals:0"])
         syntax_oid = self.samdb_schema.get_syntax_oid_from_lDAPDisplayName(attrname)
         for val in res[0][attrname]:
-            dsdb_dn = dsdb_Dn(self.samdb, val, syntax_oid)
+            dsdb_dn = dsdb_Dn(self.samdb, val.decode('utf8'), syntax_oid)
             guid2 = dsdb_dn.dn.get_extended_component("GUID")
             if guid == guid2:
                 return dsdb_dn
         return None
 
-    def check_dn(self, obj, attrname, syntax_oid):
-        '''check a DN attribute for correctness'''
+    def check_duplicate_links(self, obj, forward_attr, forward_syntax, forward_linkID, backlink_attr):
+        '''check a linked values for duplicate forward links'''
         error_count = 0
-        obj_guid = obj['objectGUID'][0]
-
-        linkID, reverse_link_name = self.get_attr_linkID_and_reverse_name(attrname)
-        if reverse_link_name is not None:
-            reverse_syntax_oid = self.samdb_schema.get_syntax_oid_from_lDAPDisplayName(reverse_link_name)
-        else:
-            reverse_syntax_oid = None
 
         duplicate_dict = dict()
-        duplicate_list = list()
         unique_dict = dict()
-        unique_list = list()
-        for val in obj[attrname]:
-            if linkID & 1:
-                #
-                # Only cleanup forward links here,
-                # back links are handled below.
-                break
 
-            dsdb_dn = dsdb_Dn(self.samdb, val, syntax_oid)
+        # Only forward links can have this problem
+        if forward_linkID & 1:
+            # If we got the reverse, skip it
+            return (error_count, duplicate_dict, unique_dict)
+
+        if backlink_attr is None:
+            return (error_count, duplicate_dict, unique_dict)
+
+        duplicate_cache_key = "%s:%s" % (str(obj.dn), forward_attr)
+        if duplicate_cache_key not in self.duplicate_link_cache:
+            self.duplicate_link_cache[duplicate_cache_key] = False
+
+        for val in obj[forward_attr]:
+            dsdb_dn = dsdb_Dn(self.samdb, val.decode('utf8'), forward_syntax)
 
             # all DNs should have a GUID component
             guid = dsdb_dn.dn.get_extended_component("GUID")
@@ -917,14 +954,12 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             keystr = guidstr + dsdb_dn.prefix
             if keystr not in unique_dict:
                 unique_dict[keystr] = dsdb_dn
-                unique_list.append(keystr)
                 continue
             error_count += 1
             if keystr not in duplicate_dict:
                 duplicate_dict[keystr] = dict()
                 duplicate_dict[keystr]["keep"] = None
                 duplicate_dict[keystr]["delete"] = list()
-                duplicate_list.append(keystr)
 
             # Now check for the highest RMD_VERSION
             v1 = int(unique_dict[keystr].dn.get_extended_component("RMD_VERSION"))
@@ -949,31 +984,209 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             duplicate_dict[keystr]["delete"].append(unique_dict[keystr])
             unique_dict[keystr] = dsdb_dn
 
-        if len(duplicate_list) != 0:
-            self.report("ERROR: Duplicate link values for attribute '%s' in '%s'" % (attrname, obj.dn))
-            for keystr in duplicate_list:
+        if error_count != 0:
+            self.duplicate_link_cache[duplicate_cache_key] = True
+
+        return (error_count, duplicate_dict, unique_dict)
+
+    def has_duplicate_links(self, dn, forward_attr, forward_syntax):
+        '''check a linked values for duplicate forward links'''
+        error_count = 0
+
+        duplicate_cache_key = "%s:%s" % (str(dn), forward_attr)
+        if duplicate_cache_key in self.duplicate_link_cache:
+            return self.duplicate_link_cache[duplicate_cache_key]
+
+        forward_linkID, backlink_attr = self.get_attr_linkID_and_reverse_name(forward_attr)
+
+        attrs = [forward_attr]
+        controls = ["extended_dn:1:1", "reveal_internals:0"]
+
+        # check its the right GUID
+        try:
+            res = self.samdb.search(base=str(dn), scope=ldb.SCOPE_BASE,
+                                    attrs=attrs, controls=controls)
+        except ldb.LdbError as e8:
+            (enum, estr) = e8.args
+            if enum != ldb.ERR_NO_SUCH_OBJECT:
+                raise
+
+            return False
+
+        obj = res[0]
+        error_count, duplicate_dict, unique_dict = \
+            self.check_duplicate_links(obj, forward_attr, forward_syntax, forward_linkID, backlink_attr)
+
+        if duplicate_cache_key in self.duplicate_link_cache:
+            return self.duplicate_link_cache[duplicate_cache_key]
+
+        return False
+
+    def find_missing_forward_links_from_backlinks(self, obj,
+                                                  forward_attr,
+                                                  forward_syntax,
+                                                  backlink_attr,
+                                                  forward_unique_dict):
+        '''Find all backlinks linking to obj_guid_str not already in forward_unique_dict'''
+        missing_forward_links = []
+        error_count = 0
+
+        if backlink_attr is None:
+            return (missing_forward_links, error_count)
+
+        if forward_syntax != ldb.SYNTAX_DN:
+            self.report("Not checking for missing forward links for syntax: %s",
+                        forward_syntax)
+            return (missing_forward_links, error_count)
+
+        if "sortedLinks" in self.compatibleFeatures:
+            self.report("Not checking for missing forward links because the db " + \
+                        "has the sortedLinks feature")
+            return (missing_forward_links, error_count)
+
+        try:
+            obj_guid = obj['objectGUID'][0]
+            obj_guid_str = str(ndr_unpack(misc.GUID, obj_guid))
+            filter = "(%s=<GUID=%s>)" % (backlink_attr, obj_guid_str)
+
+            res = self.samdb.search(expression=filter,
+                                    scope=ldb.SCOPE_SUBTREE, attrs=["objectGUID"],
+                                    controls=["extended_dn:1:1",
+                                              "search_options:1:2",
+                                              "paged_results:1:1000"])
+        except ldb.LdbError as e9:
+            (enum, estr) = e9.args
+            raise
+
+        for r in res:
+            target_dn = dsdb_Dn(self.samdb, r.dn.extended_str(), forward_syntax)
+
+            guid = target_dn.dn.get_extended_component("GUID")
+            guidstr = str(misc.GUID(guid))
+            if guidstr in forward_unique_dict:
+                continue
+
+            # A valid forward link looks like this:
+            #
+            #    <GUID=9f92d30a-fc23-11e4-a5f6-30be15454808>;
+            #    <RMD_ADDTIME=131607546230000000>;
+            #    <RMD_CHANGETIME=131607546230000000>;
+            #    <RMD_FLAGS=0>;
+            #    <RMD_INVOCID=4e4496a3-7fb8-4f97-8a33-d238db8b5e2d>;
+            #    <RMD_LOCAL_USN=3765>;
+            #    <RMD_ORIGINATING_USN=3765>;
+            #    <RMD_VERSION=1>;
+            #    <SID=S-1-5-21-4177067393-1453636373-93818738-1124>;
+            #    CN=unsorted-u8,CN=Users,DC=release-4-5-0-pre1,DC=samba,DC=corp
+            #
+            # Note that versions older than Samba 4.8 create
+            # links with RMD_VERSION=0.
+            #
+            # Try to get the local_usn and time from objectClass
+            # if possible and fallback to any other one.
+            repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob,
+                              obj['replPropertyMetadata'][0])
+            for o in repl.ctr.array:
+                local_usn = o.local_usn
+                t = o.originating_change_time
+                if o.attid == drsuapi.DRSUAPI_ATTID_objectClass:
+                    break
+
+            # We use a magic invocationID for restoring missing
+            # forward links to recover from bug #13228.
+            # This should allow some more future magic to fix the
+            # problem.
+            #
+            # It also means it looses the conflict resolution
+            # against almost every real invocation, if the
+            # version is also 0.
+            originating_invocid = misc.GUID("ffffffff-4700-4700-4700-000000b13228")
+            originating_usn = 1
+
+            rmd_addtime = t
+            rmd_changetime = t
+            rmd_flags = 0
+            rmd_invocid = originating_invocid
+            rmd_originating_usn = originating_usn
+            rmd_local_usn = local_usn
+            rmd_version = 0
+
+            target_dn.dn.set_extended_component("RMD_ADDTIME", str(rmd_addtime))
+            target_dn.dn.set_extended_component("RMD_CHANGETIME", str(rmd_changetime))
+            target_dn.dn.set_extended_component("RMD_FLAGS", str(rmd_flags))
+            target_dn.dn.set_extended_component("RMD_INVOCID", ndr_pack(rmd_invocid))
+            target_dn.dn.set_extended_component("RMD_ORIGINATING_USN", str(rmd_originating_usn))
+            target_dn.dn.set_extended_component("RMD_LOCAL_USN", str(rmd_local_usn))
+            target_dn.dn.set_extended_component("RMD_VERSION", str(rmd_version))
+
+            error_count += 1
+            missing_forward_links.append(target_dn)
+
+        return (missing_forward_links, error_count)
+
+    def check_dn(self, obj, attrname, syntax_oid):
+        '''check a DN attribute for correctness'''
+        error_count = 0
+        obj_guid = obj['objectGUID'][0]
+
+        linkID, reverse_link_name = self.get_attr_linkID_and_reverse_name(attrname)
+        if reverse_link_name is not None:
+            reverse_syntax_oid = self.samdb_schema.get_syntax_oid_from_lDAPDisplayName(reverse_link_name)
+        else:
+            reverse_syntax_oid = None
+
+        error_count, duplicate_dict, unique_dict = \
+            self.check_duplicate_links(obj, attrname, syntax_oid, linkID, reverse_link_name)
+
+        if len(duplicate_dict) != 0:
+
+            missing_forward_links, missing_error_count = \
+                self.find_missing_forward_links_from_backlinks(obj,
+                                                               attrname, syntax_oid,
+                                                               reverse_link_name,
+                                                               unique_dict)
+            error_count += missing_error_count
+
+            forward_links = [dn for dn in unique_dict.values()]
+
+            if missing_error_count != 0:
+                self.report("ERROR: Missing and duplicate forward link values for attribute '%s' in '%s'" % (
+                            attrname, obj.dn))
+            else:
+                self.report("ERROR: Duplicate forward link values for attribute '%s' in '%s'" % (attrname, obj.dn))
+            for m in missing_forward_links:
+                self.report("Missing   link '%s'" % (m))
+                if not self.confirm_all("Schedule readding missing forward link for attribute %s" % attrname,
+                                        'fix_all_missing_forward_links'):
+                    self.err_orphaned_backlink(m.dn, reverse_link_name,
+                                               obj.dn.extended_str(), obj.dn,
+                                               attrname, syntax_oid,
+                                               check_duplicates=False)
+                    continue
+                forward_links += [m]
+            for keystr in duplicate_dict.keys():
                 d = duplicate_dict[keystr]
                 for dd in d["delete"]:
                     self.report("Duplicate link '%s'" % dd)
                 self.report("Correct   link '%s'" % d["keep"])
 
-            vals = []
-            for keystr in unique_list:
-                dsdb_dn = unique_dict[keystr]
-                vals.append(str(dsdb_dn))
-            self.err_duplicate_links(obj, attrname, vals)
+            # We now construct the sorted dn values.
+            # They're sorted by the objectGUID of the target
+            # See dsdb_Dn.__cmp__()
+            vals = [str(dn) for dn in sorted(forward_links)]
+            self.err_recover_forward_links(obj, attrname, vals)
             # We should continue with the fixed values
-            obj[attrname] = ldb.MessageElement(vals, ldb.FLAG_MOD_REPLACE, attrname)
+            obj[attrname] = ldb.MessageElement(vals, 0, attrname)
 
         for val in obj[attrname]:
-            dsdb_dn = dsdb_Dn(self.samdb, val, syntax_oid)
+            dsdb_dn = dsdb_Dn(self.samdb, val.decode('utf8'), syntax_oid)
 
             # all DNs should have a GUID component
             guid = dsdb_dn.dn.get_extended_component("GUID")
             if guid is None:
                 error_count += 1
                 self.err_missing_dn_GUID_component(obj.dn, attrname, val, dsdb_dn,
-                    "missing GUID")
+                                                   "missing GUID")
                 continue
 
             guidstr = str(misc.GUID(guid))
@@ -993,8 +1206,9 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                 res = self.samdb.search(base="<GUID=%s>" % guidstr, scope=ldb.SCOPE_BASE,
                                         attrs=attrs, controls=["extended_dn:1:1", "show_recycled:1",
                                                                "reveal_internals:0"
-                                        ])
-            except ldb.LdbError, (enum, estr):
+                                                               ])
+            except ldb.LdbError as e3:
+                (enum, estr) = e3.args
                 if enum != ldb.ERR_NO_SUCH_OBJECT:
                     raise
 
@@ -1010,7 +1224,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                 dsdb_dn.binary = "%08X" % int(res[0]['instanceType'][0])
 
                 if str(dsdb_dn) != val:
-                    error_count +=1
+                    error_count += 1
                     self.err_incorrect_binary_dn(obj.dn, attrname, val, dsdb_dn, "incorrect instanceType part of Binary DN")
                     continue
 
@@ -1085,21 +1299,29 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                                                       res[0].dn, "SID")
                 continue
 
+            # Only for non-links, not even forward-only links
+            # (otherwise this breaks repl_meta_data):
+            #
             # Now we have checked the GUID and SID, offer to fix old
-            # DN strings as a non-error (for forward links with no
+            # DN strings as a non-error (DNs, not links so no
             # backlink).  Samba does not maintain this string
             # otherwise, so we don't increment error_count.
             if reverse_link_name is None:
-                if str(res[0].dn) != str(dsdb_dn.dn):
-                    self.err_dn_string_component_old(obj.dn, attrname, val, dsdb_dn,
-                                                     res[0].dn)
+                if linkID == 0 and str(res[0].dn) != str(dsdb_dn.dn):
+                    # Pass in the old/bad DN without the <GUID=...> part,
+                    # otherwise the LDB code will correct it on the way through
+                    # (Note: we still want to preserve the DSDB DN prefix in the
+                    # case of binary DNs)
+                    bad_dn = dsdb_dn.prefix + dsdb_dn.dn.get_linearized()
+                    self.err_dn_string_component_old(obj.dn, attrname, bad_dn,
+                                                     dsdb_dn, res[0].dn)
                 continue
 
             # check the reverse_link is correct if there should be one
             match_count = 0
             if reverse_link_name in res[0]:
                 for v in res[0][reverse_link_name]:
-                    v_dn = dsdb_Dn(self.samdb, v)
+                    v_dn = dsdb_Dn(self.samdb, v.decode('utf8'))
                     v_guid = v_dn.dn.get_extended_component("GUID")
                     v_blob = v_dn.dn.get_extended_component("RMD_FLAGS")
                     v_rmd_flags = 0
@@ -1116,7 +1338,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                         # Forward binary multi-valued linked attribute
                         forward_count = 0
                         for w in obj[attrname]:
-                            w_guid = dsdb_Dn(self.samdb, w).dn.get_extended_component("GUID")
+                            w_guid = dsdb_Dn(self.samdb, w.decode('utf8')).dn.get_extended_component("GUID")
                             if w_guid == guid:
                                 forward_count += 1
 
@@ -1124,7 +1346,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                             continue
             expected_count = 0
             for v in obj[attrname]:
-                v_dn = dsdb_Dn(self.samdb, v)
+                v_dn = dsdb_Dn(self.samdb, v.decode('utf8'))
                 v_guid = v_dn.dn.get_extended_component("GUID")
                 v_blob = v_dn.dn.get_extended_component("RMD_FLAGS")
                 v_rmd_flags = 0
@@ -1146,9 +1368,10 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                 # UNLESS, there is no forward link detected.
                 if match_count == 0:
                     error_count += 1
-                    self.err_orphaned_backlink(obj, attrname,
-                                               val, reverse_link_name,
-                                               dsdb_dn.dn)
+                    self.err_orphaned_backlink(obj.dn, attrname,
+                                               val, dsdb_dn.dn,
+                                               reverse_link_name,
+                                               reverse_syntax_oid)
                     continue
                 # Only warn here and let the forward link logic fix it.
                 self.report("WARNING: Link (back) mismatch for '%s' (%d) on '%s' to '%s' (%d) on '%s'" % (
@@ -1178,9 +1401,9 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                                               dsdb_dn.dn)
                     diff_count -= 1
                 else:
-                    self.err_orphaned_backlink(res[0], reverse_link_name,
-                                               obj.dn.extended_str(), attrname,
-                                               obj.dn)
+                    self.err_orphaned_backlink(res[0].dn, reverse_link_name,
+                                               obj.dn.extended_str(), obj.dn,
+                                               attrname, syntax_oid)
                     diff_count += 1
 
 
@@ -1360,7 +1583,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
         cls = None
         try:
             cls = obj["objectClass"][-1]
-        except KeyError, e:
+        except KeyError as e:
             pass
 
         if cls is None:
@@ -1546,7 +1769,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
         # the correct values are above 0x80000000) values first and
         # remove the 'second' value we see.
         for o in reversed(ctr.array):
-            print "%s: 0x%08x" % (dn, o.attid)
+            print("%s: 0x%08x" % (dn, o.attid))
             att = self.samdb_schema.get_lDAPDisplayName_by_attid(o.attid)
             if att.lower() in set_att:
                 self.report('ERROR: duplicate attributeID values for %s in %s on %s\n' % (att, attr, dn))
@@ -1616,9 +1839,9 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
         nmsg.dn = dn
         nmsg[attr] = ldb.MessageElement(replBlob, ldb.FLAG_MOD_REPLACE, attr)
         if self.do_modify(nmsg, ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK_MODIFY_RO_REPLICA,
-                             "local_oid:1.3.6.1.4.1.7165.4.3.14:0",
-                             "local_oid:1.3.6.1.4.1.7165.4.3.25:0"],
-                      "Failed to fix attribute %s" % attr):
+                                 "local_oid:1.3.6.1.4.1.7165.4.3.14:0",
+                                 "local_oid:1.3.6.1.4.1.7165.4.3.25:0"],
+                          "Failed to fix attribute %s" % attr):
             self.report("Fixed attribute '%s' of '%s'\n" % (attr, dn))
 
 
@@ -1719,7 +1942,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             instancetype |= dsdb.INSTANCE_TYPE_IS_NC_HEAD
             try:
                 self.samdb.search(base=dn.parent(), scope=ldb.SCOPE_BASE, attrs=[], controls=["show_recycled:1"])
-            except ldb.LdbError, (enum, estr):
+            except ldb.LdbError as e4:
+                (enum, estr) = e4.args
                 if enum != ldb.ERR_NO_SUCH_OBJECT:
                     raise
             else:
@@ -1758,10 +1982,21 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             attrs.append(dn.get_rdn_name())
             attrs.append("isDeleted")
             attrs.append("systemFlags")
+        need_replPropertyMetaData = False
         if '*' in attrs:
-            attrs.append("replPropertyMetaData")
+            need_replPropertyMetaData = True
         else:
-            attrs.append("objectGUID")
+            for a in attrs:
+                linkID, _ = self.get_attr_linkID_and_reverse_name(a)
+                if linkID == 0:
+                    continue
+                if linkID & 1:
+                    continue
+                need_replPropertyMetaData = True
+                break
+        if need_replPropertyMetaData:
+            attrs.append("replPropertyMetaData")
+        attrs.append("objectGUID")
 
         try:
             sd_flags = 0
@@ -1779,7 +2014,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                                         "reveal_internals:0",
                                     ],
                                     attrs=attrs)
-        except ldb.LdbError, (enum, estr):
+        except ldb.LdbError as e10:
+            (enum, estr) = e10.args
             if enum == ldb.ERR_NO_SUCH_OBJECT:
                 if self.in_transaction:
                     self.report("ERROR: Object %s disappeared during check" % dn)
@@ -1861,7 +2097,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                 if len(set_attrs_from_md) < len(list_attid_from_md) \
                    or len(wrong_attids) > 0 \
                    or sorted(list_attid_from_md) != list_attid_from_md:
-                    error_count +=1
+                    error_count += 1
                     self.err_replmetadata_incorrect_attid(dn, attrname, obj[attrname], wrong_attids)
 
                 else:
@@ -1975,7 +2211,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             # special handling for some specific attribute types
             try:
                 syntax_oid = self.samdb_schema.get_syntax_oid_from_lDAPDisplayName(attrname)
-            except Exception, msg:
+            except Exception as msg:
                 self.err_unknown_attribute(obj, attrname)
                 error_count += 1
                 continue
@@ -1988,8 +2224,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                 and not linkID):
                 set_attrs_seen.add(str(attrname).lower())
 
-            if syntax_oid in [ dsdb.DSDB_SYNTAX_BINARY_DN, dsdb.DSDB_SYNTAX_OR_NAME,
-                               dsdb.DSDB_SYNTAX_STRING_DN, ldb.SYNTAX_DN ]:
+            if syntax_oid in [dsdb.DSDB_SYNTAX_BINARY_DN, dsdb.DSDB_SYNTAX_OR_NAME,
+                               dsdb.DSDB_SYNTAX_STRING_DN, ldb.SYNTAX_DN]:
                 # it's some form of DN, do specialised checking on those
                 error_count += self.check_dn(obj, attrname, syntax_oid)
             else:
@@ -2087,7 +2323,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             if dn != self.samdb.get_root_basedn() and str(dn.parent()) not in self.dn_set:
                 res = self.samdb.search(base=dn.parent(), scope=ldb.SCOPE_BASE,
                                         controls=["show_recycled:1", "show_deleted:1"])
-        except ldb.LdbError, (enum, estr):
+        except ldb.LdbError as e11:
+            (enum, estr) = e11.args
             if enum == ldb.ERR_NO_SUCH_OBJECT:
                 self.err_missing_parent(obj)
                 error_count += 1
@@ -2191,7 +2428,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                     try:
                         res = self.samdb.search(base="<SID=%s>" % sid, scope=ldb.SCOPE_BASE,
                                                 attrs=[])
-                    except ldb.LdbError, (enum, estr):
+                    except ldb.LdbError as e:
+                        (enum, estr) = e.args
                         if enum != ldb.ERR_NO_SUCH_OBJECT:
                             raise
                         res = None
@@ -2247,14 +2485,14 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
         # check that the dsServiceName is in GUID form
         if not 'dsServiceName' in obj:
             self.report('ERROR: dsServiceName missing in @ROOTDSE')
-            return error_count+1
+            return error_count + 1
 
         if not obj['dsServiceName'][0].startswith('<GUID='):
             self.report('ERROR: dsServiceName not in GUID form in @ROOTDSE')
             error_count += 1
             if not self.confirm('Change dsServiceName to GUID form?'):
                 return error_count
-            res = self.samdb.search(base=ldb.Dn(self.samdb, obj['dsServiceName'][0]),
+            res = self.samdb.search(base=ldb.Dn(self.samdb, obj['dsServiceName'][0].decode('utf8')),
                                     scope=ldb.SCOPE_BASE, attrs=['objectGUID'])
             guid_str = str(ndr_unpack(misc.GUID, res[0]['objectGUID'][0]))
             m = ldb.Message()