dbcheck: only calculate linked attribute helper variables once in check_dn()
[sfrench/samba-autobuild/.git] / python / samba / dbchecker.py
index 3d33c381efbee9ac4deacfdd8f6eb625102d1d39..dafa8443d24a97a638221eac8c6738e5da0db976 100644 (file)
@@ -31,6 +31,8 @@ from samba.common import dsdb_Dn
 from samba.dcerpc import security
 from samba.descriptor import get_wellknown_sds, get_diff_sds
 from samba.auth import system_session, admin_session
+from samba.netcmd import CommandError
+from samba.netcmd.fsmo import get_fsmo_roleowner
 
 
 class dbcheck(object):
@@ -52,10 +54,15 @@ class dbcheck(object):
         self.fix_all_duplicates = False
         self.fix_all_DN_GUIDs = False
         self.fix_all_binary_dn = False
-        self.remove_all_deleted_DN_links = False
-        self.fix_all_target_mismatch = False
+        self.remove_implausible_deleted_DN_links = False
+        self.remove_plausible_deleted_DN_links = False
+        self.fix_all_string_dn_component_mismatch = False
+        self.fix_all_GUID_dn_component_mismatch = False
+        self.fix_all_SID_dn_component_mismatch = False
+        self.fix_all_old_dn_string_component_mismatch = False
         self.fix_all_metadata = False
         self.fix_time_metadata = False
+        self.fix_undead_linked_attributes = False
         self.fix_all_missing_backlinks = False
         self.fix_all_orphaned_backlinks = False
         self.fix_rmd_flags = False
@@ -74,6 +81,7 @@ class dbcheck(object):
         self.fix_base64_userparameters = False
         self.fix_utf8_userparameters = False
         self.fix_doubled_userparameters = False
+        self.fix_sid_rid_set_conflict = False
         self.reset_well_known_acls = reset_well_known_acls
         self.reset_all_well_known_acls = False
         self.in_transaction = in_transaction
@@ -86,9 +94,11 @@ class dbcheck(object):
         self.wellknown_sds = get_wellknown_sds(self.samdb)
         self.fix_all_missing_objectclass = False
         self.fix_missing_deleted_objects = False
+        self.fix_replica_locations = False
+        self.fix_missing_rid_set_master = False
 
         self.dn_set = set()
-
+        self.link_id_cache = {}
         self.name_map = {}
         try:
             res = samdb.search(base="CN=DnsAdmins,CN=Users,%s" % samdb.domain_dn(), scope=ldb.SCOPE_BASE,
@@ -119,6 +129,7 @@ class dbcheck(object):
         res = self.samdb.search(base="", scope=ldb.SCOPE_BASE, attrs=['namingContexts'])
         self.deleted_objects_containers = []
         self.ncs_lacking_deleted_containers = []
+        self.dns_partitions = []
         try:
             self.ncs = res[0]["namingContexts"]
         except KeyError:
@@ -134,6 +145,44 @@ class dbcheck(object):
             except KeyError:
                 self.ncs_lacking_deleted_containers.append(ldb.Dn(self.samdb, nc))
 
+        domaindns_zone = 'DC=DomainDnsZones,%s' % self.samdb.get_default_basedn()
+        forestdns_zone = 'DC=ForestDnsZones,%s' % self.samdb.get_root_basedn()
+        domain = self.samdb.search(scope=ldb.SCOPE_ONELEVEL,
+                                   attrs=["msDS-NC-Replica-Locations", "msDS-NC-RO-Replica-Locations"],
+                                   base=self.samdb.get_partitions_dn(),
+                                   expression="(&(objectClass=crossRef)(ncName=%s))" % domaindns_zone)
+        if len(domain) == 1:
+            self.dns_partitions.append((ldb.Dn(self.samdb, forestdns_zone), domain[0]))
+
+        forest = self.samdb.search(scope=ldb.SCOPE_ONELEVEL,
+                                   attrs=["msDS-NC-Replica-Locations", "msDS-NC-RO-Replica-Locations"],
+                                   base=self.samdb.get_partitions_dn(),
+                                   expression="(&(objectClass=crossRef)(ncName=%s))" % forestdns_zone)
+        if len(forest) == 1:
+            self.dns_partitions.append((ldb.Dn(self.samdb, domaindns_zone), forest[0]))
+
+        fsmo_dn = ldb.Dn(self.samdb, "CN=RID Manager$,CN=System," + self.samdb.domain_dn())
+        rid_master = get_fsmo_roleowner(self.samdb, fsmo_dn, "rid")
+        if ldb.Dn(self.samdb, self.samdb.get_dsServiceName()) == rid_master:
+            self.is_rid_master = True
+        else:
+            self.is_rid_master = False
+
+        # To get your rid set
+        # 1. Get server name
+        res = self.samdb.search(base=ldb.Dn(self.samdb, self.samdb.get_serverName()),
+                                scope=ldb.SCOPE_BASE, attrs=["serverReference"])
+        # 2. Get server reference
+        self.server_ref_dn = ldb.Dn(self.samdb, res[0]['serverReference'][0])
+
+        # 3. Get RID Set
+        res = self.samdb.search(base=self.server_ref_dn,
+                                scope=ldb.SCOPE_BASE, attrs=['rIDSetReferences'])
+        if "rIDSetReferences" in res[0]:
+            self.rid_set_dn = ldb.Dn(self.samdb, res[0]['rIDSetReferences'][0])
+        else:
+            self.rid_set_dn = None
+
     def check_database(self, DN=None, scope=ldb.SCOPE_SUBTREE, controls=[], attrs=['*']):
         '''perform a database check, returning the number of errors found'''
         res = self.samdb.search(base=DN, scope=scope, attrs=['dn'], controls=controls)
@@ -157,7 +206,6 @@ class dbcheck(object):
         self.report('Checked %u objects (%u errors)' % (len(res), error_count))
         return error_count
 
-
     def check_deleted_objects_containers(self):
         """This function only fixes conflicts on the Deleted Objects
         containers, not the attributes"""
@@ -277,14 +325,14 @@ systemFlags: -1946157056%s""" % (dn, guid_suffix),
         '''confirm a change with support for "all" '''
         if not self.fix:
             return False
-        if self.quiet:
-            return self.yes
         if getattr(self, all_attr) == 'NONE':
             return False
         if getattr(self, all_attr) == 'ALL':
             forced = True
         else:
             forced = self.yes
+        if self.quiet:
+            return forced
         c = common.confirm(msg, forced=forced, allow_all=True)
         if c == 'ALL':
             setattr(self, all_attr, 'ALL')
@@ -302,6 +350,8 @@ systemFlags: -1946157056%s""" % (dn, guid_suffix),
             controls = controls + ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK]
             self.samdb.delete(dn, controls=controls)
         except Exception, err:
+            if self.in_transaction:
+                raise CommandError("%s : %s" % (msg, err))
             self.report("%s : %s" % (msg, err))
             return False
         return True
@@ -314,6 +364,8 @@ systemFlags: -1946157056%s""" % (dn, guid_suffix),
             controls = controls + ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK]
             self.samdb.modify(m, controls=controls, validate=validate)
         except Exception, err:
+            if self.in_transaction:
+                raise CommandError("%s : %s" % (msg, err))
             self.report("%s : %s" % (msg, err))
             return False
         return True
@@ -331,10 +383,23 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             controls = controls + ["local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK]
             self.samdb.rename(from_dn, to_dn, controls=controls)
         except Exception, err:
+            if self.in_transaction:
+                raise CommandError("%s : %s" % (msg, err))
             self.report("%s : %s" % (msg, err))
             return False
         return True
 
+    def get_attr_linkID_and_reverse_name(self, attrname):
+        if attrname in self.link_id_cache:
+            return self.link_id_cache[attrname]
+        linkID = self.samdb_schema.get_linkId_from_lDAPDisplayName(attrname)
+        if linkID:
+            revname = self.samdb_schema.get_backlink_from_lDAPDisplayName(attrname)
+        else:
+            revname = None
+        self.link_id_cache[attrname] = (linkID, revname)
+        return linkID, revname
+
     def err_empty_attribute(self, dn, attrname):
         '''fix empty attributes'''
         self.report("ERROR: Empty attribute %s in %s" % (attrname, dn))
@@ -431,30 +496,80 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                           "Failed to remove DN %s" % dn):
             self.report("Removed DN %s" % dn)
 
-    def err_deleted_dn(self, dn, attrname, val, dsdb_dn, correct_dn):
+    def err_deleted_dn(self, dn, attrname, val, dsdb_dn, correct_dn, remove_plausible=False):
         """handle a DN pointing to a deleted object"""
-        self.report("ERROR: target DN is deleted for %s in object %s - %s" % (attrname, dn, val))
-        self.report("Target GUID points at deleted DN %r" % str(correct_dn))
-        if not self.confirm_all('Remove DN link?', 'remove_all_deleted_DN_links'):
-            self.report("Not removing")
-            return
+        if not remove_plausible:
+            self.report("ERROR: target DN is deleted for %s in object %s - %s" % (attrname, dn, val))
+            self.report("Target GUID points at deleted DN %r" % str(correct_dn))
+            if not self.confirm_all('Remove DN link?', 'remove_implausible_deleted_DN_links'):
+                self.report("Not removing")
+                return
+        else:
+            self.report("WARNING: target DN is deleted for %s in object %s - %s" % (attrname, dn, val))
+            self.report("Target GUID points at deleted DN %r" % str(correct_dn))
+            if not self.confirm_all('Remove stale DN link?', 'remove_plausible_deleted_DN_links'):
+                self.report("Not removing")
+                return
+
         m = ldb.Message()
         m.dn = dn
         m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
-        if self.do_modify(m, ["show_recycled:1", "local_oid:%s:0" % dsdb.DSDB_CONTROL_DBCHECK],
+        if self.do_modify(m, ["show_recycled:1",
+                              "local_oid:%s:0" % dsdb.DSDB_CONTROL_REPLMD_VANISH_LINKS],
                           "Failed to remove deleted DN attribute %s" % attrname):
             self.report("Removed deleted DN on attribute %s" % attrname)
 
-    def err_missing_dn_GUID(self, dn, attrname, val, dsdb_dn):
-        """handle a missing target DN (both GUID and DN string form are missing)"""
+    def err_missing_target_dn_or_GUID(self, dn, attrname, val, dsdb_dn):
+        """handle a missing target DN (if specified, GUID form can't be found,
+        and otherwise DN string form can't be found)"""
         # check if its a backlink
-        linkID = self.samdb_schema.get_linkId_from_lDAPDisplayName(attrname)
+        linkID, _ = self.get_attr_linkID_and_reverse_name(attrname)
         if (linkID & 1 == 0) and str(dsdb_dn).find('\\0ADEL') == -1:
-            self.report("Not removing dangling forward link")
-            return
-        self.err_deleted_dn(dn, attrname, val, dsdb_dn, dsdb_dn)
 
-    def err_incorrect_dn_GUID(self, dn, attrname, val, dsdb_dn, errstr):
+            linkID, reverse_link_name \
+                = self.get_attr_linkID_and_reverse_name(attrname)
+            if reverse_link_name is not None:
+                self.report("WARNING: no target object found for GUID "
+                            "component for one-way forward link "
+                            "%s in object "
+                            "%s - %s" % (attrname, dn, val))
+                self.report("Not removing dangling forward link")
+                return 0
+
+            nc_root = self.samdb.get_nc_root(dn)
+            target_nc_root = self.samdb.get_nc_root(dsdb_dn.dn)
+            if nc_root != target_nc_root:
+                # We don't bump the error count as Samba produces these
+                # in normal operation
+                self.report("WARNING: no target object found for GUID "
+                            "component for cross-partition link "
+                            "%s in object "
+                            "%s - %s" % (attrname, dn, val))
+                self.report("Not removing dangling one-way "
+                            "cross-partition link "
+                            "(we might be mid-replication)")
+                return 0
+
+            # Due to our link handling one-way links pointing to
+            # missing objects are plausible.
+            #
+            # We don't bump the error count as Samba produces these
+            # in normal operation
+            self.report("WARNING: no target object found for GUID "
+                        "component for DN value %s in object "
+                        "%s - %s" % (attrname, dn, val))
+            self.err_deleted_dn(dn, attrname, val,
+                                dsdb_dn, dsdb_dn, True)
+            return 0
+
+        # We bump the error count here, as we should have deleted this
+        self.report("ERROR: no target object found for GUID "
+                    "component for link %s in object "
+                    "%s - %s" % (attrname, dn, val))
+        self.err_deleted_dn(dn, attrname, val, dsdb_dn, dsdb_dn, False)
+        return 1
+
+    def err_missing_dn_GUID_component(self, dn, attrname, val, dsdb_dn, errstr):
         """handle a missing GUID extended DN component"""
         self.report("ERROR: %s component for %s in object %s - %s" % (errstr, attrname, dn, val))
         controls=["extended_dn:1:1", "show_recycled:1"]
@@ -463,11 +578,13 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                                     attrs=[], controls=controls)
         except ldb.LdbError, (enum, estr):
             self.report("unable to find object for DN %s - (%s)" % (dsdb_dn.dn, estr))
-            self.err_missing_dn_GUID(dn, attrname, val, dsdb_dn)
+            if enum != ldb.ERR_NO_SUCH_OBJECT:
+                raise
+            self.err_missing_target_dn_or_GUID(dn, attrname, val, dsdb_dn)
             return
         if len(res) == 0:
             self.report("unable to find object for DN %s" % dsdb_dn.dn)
-            self.err_missing_dn_GUID(dn, attrname, val, dsdb_dn)
+            self.err_missing_target_dn_or_GUID(dn, attrname, val, dsdb_dn)
             return
         dsdb_dn.dn = res[0].dn
 
@@ -500,21 +617,39 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                           "Failed to fix %s on attribute %s" % (errstr, attrname)):
             self.report("Fixed %s on attribute %s" % (errstr, attrname))
 
-    def err_dn_target_mismatch(self, dn, attrname, val, dsdb_dn, correct_dn, errstr):
+    def err_dn_string_component_old(self, dn, attrname, val, dsdb_dn, correct_dn):
         """handle a DN string being incorrect"""
-        self.report("ERROR: incorrect DN string component for %s in object %s - %s" % (attrname, dn, val))
+        self.report("NOTE: old (due to rename or delete) DN string component for %s in object %s - %s" % (attrname, dn, val))
         dsdb_dn.dn = correct_dn
 
-        if not self.confirm_all('Change DN to %s?' % str(dsdb_dn), 'fix_all_target_mismatch'):
-            self.report("Not fixing %s" % errstr)
+        if not self.confirm_all('Change DN to %s?' % str(dsdb_dn),
+                                'fix_all_old_dn_string_component_mismatch'):
+            self.report("Not fixing old string component")
             return
         m = ldb.Message()
         m.dn = dn
         m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
         m['new_value'] = ldb.MessageElement(str(dsdb_dn), ldb.FLAG_MOD_ADD, attrname)
         if self.do_modify(m, ["show_recycled:1"],
-                          "Failed to fix incorrect DN string on attribute %s" % attrname):
-            self.report("Fixed incorrect DN string on attribute %s" % (attrname))
+                          "Failed to fix old DN string on attribute %s" % (attrname)):
+            self.report("Fixed old DN string on attribute %s" % (attrname))
+
+    def err_dn_component_target_mismatch(self, dn, attrname, val, dsdb_dn, correct_dn, mismatch_type):
+        """handle a DN string being incorrect"""
+        self.report("ERROR: incorrect DN %s component for %s in object %s - %s" % (mismatch_type, attrname, dn, val))
+        dsdb_dn.dn = correct_dn
+
+        if not self.confirm_all('Change DN to %s?' % str(dsdb_dn),
+                                'fix_all_%s_dn_component_mismatch' % mismatch_type):
+            self.report("Not fixing %s component mismatch" % mismatch_type)
+            return
+        m = ldb.Message()
+        m.dn = dn
+        m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
+        m['new_value'] = ldb.MessageElement(str(dsdb_dn), ldb.FLAG_MOD_ADD, attrname)
+        if self.do_modify(m, ["show_recycled:1"],
+                          "Failed to fix incorrect DN %s on attribute %s" % (mismatch_type, attrname)):
+            self.report("Fixed incorrect DN %s on attribute %s" % (mismatch_type, attrname))
 
     def err_unknown_attribute(self, obj, attrname):
         '''handle an unknown attribute error'''
@@ -529,6 +664,22 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                           "Failed to remove unknown attribute %s" % attrname):
             self.report("Removed unknown attribute %s" % (attrname))
 
+    def err_undead_linked_attribute(self, obj, attrname, val):
+        '''handle a link that should not be there on a deleted object'''
+        self.report("ERROR: linked attribute '%s' to '%s' is present on "
+                    "deleted object %s" % (attrname, val, obj.dn))
+        if not self.confirm_all('Remove linked attribute %s' % attrname, 'fix_undead_linked_attributes'):
+            self.report("Not removing linked attribute %s" % attrname)
+            return
+        m = ldb.Message()
+        m.dn = obj.dn
+        m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
+
+        if self.do_modify(m, ["show_recycled:1", "show_deleted:1", "reveal_internals:0",
+                              "local_oid:%s:0" % dsdb.DSDB_CONTROL_REPLMD_VANISH_LINKS],
+                          "Failed to delete forward link %s" % attrname):
+            self.report("Fixed undead forward link %s" % (attrname))
+
     def err_missing_backlink(self, obj, attrname, val, backlink_name, target_dn):
         '''handle a missing backlink value'''
         self.report("ERROR: missing backlink attribute '%s' in %s for link %s in %s" % (backlink_name, target_dn, attrname, obj.dn))
@@ -536,10 +687,9 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             self.report("Not fixing missing backlink %s" % backlink_name)
             return
         m = ldb.Message()
-        m.dn = obj.dn
-        m['old_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
-        m['new_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_ADD, attrname)
-        if self.do_modify(m, ["show_recycled:1"],
+        m.dn = target_dn
+        m['new_value'] = ldb.MessageElement(val, ldb.FLAG_MOD_ADD, backlink_name)
+        if self.do_modify(m, ["show_recycled:1", "relax:0"],
                           "Failed to fix missing backlink %s" % backlink_name):
             self.report("Fixed missing backlink %s" % (backlink_name))
 
@@ -560,15 +710,15 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
     def err_orphaned_backlink(self, obj, attrname, val, link_name, target_dn):
         '''handle a orphaned backlink value'''
         self.report("ERROR: orphaned backlink attribute '%s' in %s for link %s in %s" % (attrname, obj.dn, link_name, target_dn))
-        if not self.confirm_all('Remove orphaned backlink %s' % link_name, 'fix_all_orphaned_backlinks'):
-            self.report("Not removing orphaned backlink %s" % link_name)
+        if not self.confirm_all('Remove orphaned backlink %s' % attrname, 'fix_all_orphaned_backlinks'):
+            self.report("Not removing orphaned backlink %s" % attrname)
             return
         m = ldb.Message()
         m.dn = obj.dn
         m['value'] = ldb.MessageElement(val, ldb.FLAG_MOD_DELETE, attrname)
         if self.do_modify(m, ["show_recycled:1", "relax:0"],
-                          "Failed to fix orphaned backlink %s" % link_name):
-            self.report("Fixed orphaned backlink %s" % (link_name))
+                          "Failed to fix orphaned backlink %s" % attrname):
+            self.report("Fixed orphaned backlink %s" % (attrname))
 
     def err_no_fsmoRoleOwner(self, obj):
         '''handle a missing fSMORoleOwner'''
@@ -594,7 +744,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             self.report('Not moving object %s into LostAndFound' % (obj.dn))
             return
 
-        keep_transaction = True
+        keep_transaction = False
         self.samdb.transaction_start()
         try:
             nc_root = self.samdb.get_nc_root(obj.dn);
@@ -724,6 +874,14 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
     def check_dn(self, obj, attrname, syntax_oid):
         '''check a DN attribute for correctness'''
         error_count = 0
+        obj_guid = obj['objectGUID'][0]
+
+        linkID, reverse_link_name = self.get_attr_linkID_and_reverse_name(attrname)
+        if reverse_link_name is not None:
+            reverse_syntax_oid = self.samdb_schema.get_syntax_oid_from_lDAPDisplayName(reverse_link_name)
+        else:
+            reverse_syntax_oid = None
+
         for val in obj[attrname]:
             dsdb_dn = dsdb_Dn(self.samdb, val, syntax_oid)
 
@@ -731,13 +889,12 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             guid = dsdb_dn.dn.get_extended_component("GUID")
             if guid is None:
                 error_count += 1
-                self.err_incorrect_dn_GUID(obj.dn, attrname, val, dsdb_dn,
+                self.err_missing_dn_GUID_component(obj.dn, attrname, val, dsdb_dn,
                     "missing GUID")
                 continue
 
             guidstr = str(misc.GUID(guid))
-
-            attrs = ['isDeleted']
+            attrs = ['isDeleted', 'replPropertyMetaData']
 
             if (str(attrname).lower() == 'msds-hasinstantiatedncs') and (obj.dn == self.ntds_dsa):
                 fixing_msDS_HasInstantiatedNCs = True
@@ -745,18 +902,24 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             else:
                 fixing_msDS_HasInstantiatedNCs = False
 
-            linkID = self.samdb_schema.get_linkId_from_lDAPDisplayName(attrname)
-            reverse_link_name = self.samdb_schema.get_backlink_from_lDAPDisplayName(attrname)
             if reverse_link_name is not None:
                 attrs.append(reverse_link_name)
 
             # check its the right GUID
             try:
                 res = self.samdb.search(base="<GUID=%s>" % guidstr, scope=ldb.SCOPE_BASE,
-                                        attrs=attrs, controls=["extended_dn:1:1", "show_recycled:1"])
+                                        attrs=attrs, controls=["extended_dn:1:1", "show_recycled:1",
+                                                               "reveal_internals:0"
+                                        ])
             except ldb.LdbError, (enum, estr):
-                error_count += 1
-                self.err_incorrect_dn_GUID(obj.dn, attrname, val, dsdb_dn, "incorrect GUID")
+                if enum != ldb.ERR_NO_SUCH_OBJECT:
+                    raise
+
+                # We don't always want to
+                error_count += self.err_missing_target_dn_or_GUID(obj.dn,
+                                                                  attrname,
+                                                                  val,
+                                                                  dsdb_dn)
                 continue
 
             if fixing_msDS_HasInstantiatedNCs:
@@ -772,43 +935,154 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             is_deleted = 'isDeleted' in obj and obj['isDeleted'][0].upper() == 'TRUE'
             target_is_deleted = 'isDeleted' in res[0] and res[0]['isDeleted'][0].upper() == 'TRUE'
 
-            # the target DN is not allowed to be deleted, unless the target DN is the
-            # special Deleted Objects container
-            if target_is_deleted and not is_deleted and not self.is_deleted_objects_dn(dsdb_dn):
+
+            if is_deleted and not obj.dn in self.deleted_objects_containers and linkID:
+                # A fully deleted object should not have any linked
+                # attributes. (MS-ADTS 3.1.1.5.5.1.1 Tombstone
+                # Requirements and 3.1.1.5.5.1.3 Recycled-Object
+                # Requirements)
+                self.err_undead_linked_attribute(obj, attrname, val)
                 error_count += 1
-                self.err_deleted_dn(obj.dn, attrname, val, dsdb_dn, res[0].dn)
                 continue
-
-            # check the DN matches in string form
-            if res[0].dn.extended_str() != dsdb_dn.dn.extended_str():
+            elif target_is_deleted and not self.is_deleted_objects_dn(dsdb_dn) and linkID:
+                # the target DN is not allowed to be deleted, unless the target DN is the
+                # special Deleted Objects container
                 error_count += 1
-                self.err_dn_target_mismatch(obj.dn, attrname, val, dsdb_dn,
-                                            res[0].dn, "incorrect string version of DN")
+                local_usn = dsdb_dn.dn.get_extended_component("RMD_LOCAL_USN")
+                if local_usn:
+                    if 'replPropertyMetaData' in res[0]:
+                        repl = ndr_unpack(drsblobs.replPropertyMetaDataBlob,
+                                          str(res[0]['replPropertyMetadata']))
+                        found_data = False
+                        for o in repl.ctr.array:
+                            if o.attid == drsuapi.DRSUAPI_ATTID_isDeleted:
+                                deleted_usn = o.local_usn
+                                if deleted_usn >= int(local_usn):
+                                    # If the object was deleted after the link
+                                    # was last modified then, clean it up here
+                                    found_data = True
+                                    break
+
+                        if found_data:
+                            self.err_deleted_dn(obj.dn, attrname,
+                                                val, dsdb_dn, res[0].dn, True)
+                            continue
+
+                self.err_deleted_dn(obj.dn, attrname, val, dsdb_dn, res[0].dn, False)
                 continue
 
-            if is_deleted and not target_is_deleted and reverse_link_name is not None:
-                revealed_dn = self.find_revealed_link(obj.dn, attrname, guid)
-                rmd_flags = revealed_dn.dn.get_extended_component("RMD_FLAGS")
-                if rmd_flags is not None and (int(rmd_flags) & 1) == 0:
-                    # the RMD_FLAGS for this link should be 1, as the target is deleted
-                    self.err_incorrect_rmd_flags(obj, attrname, revealed_dn)
+            # We should not check for incorrect
+            # components on deleted links, as these are allowed to
+            # go stale (we just need the GUID, not the name)
+            rmd_blob = dsdb_dn.dn.get_extended_component("RMD_FLAGS")
+            if rmd_blob is not None:
+                rmd_flags = int(rmd_blob)
+                if rmd_flags & 1:
                     continue
 
-            # check the reverse_link is correct if there should be one
+            # assert the DN matches in string form, where a reverse
+            # link exists, otherwise (below) offer to fix it as a non-error.
+            # The string form is essentially only kept for forensics,
+            # as we always re-resolve by GUID in normal operations.
             if reverse_link_name is not None:
-                match_count = 0
-                if reverse_link_name in res[0]:
-                    for v in res[0][reverse_link_name]:
-                        if v == obj.dn.extended_str():
-                            match_count += 1
-                if match_count != 1:
+                if str(res[0].dn) != str(dsdb_dn.dn):
                     error_count += 1
-                    if linkID & 1:
-                        self.err_orphaned_backlink(obj, attrname, val, reverse_link_name, dsdb_dn.dn)
+                    self.err_dn_component_target_mismatch(obj.dn, attrname, val, dsdb_dn,
+                                                          res[0].dn, "string")
+                    continue
+
+            if res[0].dn.get_extended_component("GUID") != dsdb_dn.dn.get_extended_component("GUID"):
+                error_count += 1
+                self.err_dn_component_target_mismatch(obj.dn, attrname, val, dsdb_dn,
+                                                      res[0].dn, "GUID")
+                continue
+
+            if res[0].dn.get_extended_component("SID") != dsdb_dn.dn.get_extended_component("SID"):
+                error_count += 1
+                self.err_dn_component_target_mismatch(obj.dn, attrname, val, dsdb_dn,
+                                                      res[0].dn, "SID")
+                continue
+
+            # Now we have checked the GUID and SID, offer to fix old
+            # DN strings as a non-error (for forward links with no
+            # backlink).  Samba does not maintain this string
+            # otherwise, so we don't increment error_count.
+            if reverse_link_name is None:
+                if str(res[0].dn) != str(dsdb_dn.dn):
+                    self.err_dn_string_component_old(obj.dn, attrname, val, dsdb_dn,
+                                                     res[0].dn)
+                continue
+
+            # check the reverse_link is correct if there should be one
+            match_count = 0
+            if reverse_link_name in res[0]:
+                for v in res[0][reverse_link_name]:
+                    v_guid = dsdb_Dn(self.samdb, v).dn.get_extended_component("GUID")
+                    if v_guid == obj_guid:
+                        match_count += 1
+            if match_count != 1:
+                if syntax_oid == dsdb.DSDB_SYNTAX_BINARY_DN or reverse_syntax_oid == dsdb.DSDB_SYNTAX_BINARY_DN:
+                    if not linkID & 1:
+                        # Forward binary multi-valued linked attribute
+                        forward_count = 0
+                        for w in obj[attrname]:
+                            w_guid = dsdb_Dn(self.samdb, w).dn.get_extended_component("GUID")
+                            if w_guid == guid:
+                                forward_count += 1
+
+                        if match_count == forward_count:
+                            continue
+
+                        error_count += 1
+
+                        # Add or remove the missing number of backlinks
+                        diff_count = forward_count - match_count
+
+                        # Loop until the difference between the forward and
+                        # the backward links is resolved.
+                        while diff_count != 0:
+                            if diff_count > 0:
+                                # self.err_missing_backlink(obj, attrname,
+                                #                          obj.dn.extended_str(),
+                                #                          reverse_link_name,
+                                #                          dsdb_dn.dn)
+                                # diff_count -= 1
+                                # TODO no method to fix these right now
+                                self.report("ERROR: Can't fix missing "
+                                            "multi-valued backlinks on %s" % str(dsdb_dn.dn))
+                                break
+                            else:
+                                self.err_orphaned_backlink(res[0], reverse_link_name,
+                                                           obj.dn.extended_str(), attrname,
+                                                           dsdb_dn.dn)
+                                diff_count += 1
+
                     else:
-                        self.err_missing_backlink(obj, attrname, val, reverse_link_name, dsdb_dn.dn)
+                        # If there's a backward link on binary multi-valued linked attribute,
+                        # let the check on the forward link remedy the value.
+                        # UNLESS, there is no forward link detected.
+                        if match_count == 0:
+                            self.err_orphaned_backlink(obj, attrname,
+                                                       val, reverse_link_name,
+                                                       dsdb_dn.dn)
+
                     continue
 
+                error_count += 1
+                if linkID & 1:
+                    # Backlink exists, but forward link does not
+                    # Delete the hanging backlink
+                    self.err_orphaned_backlink(obj, attrname, val, reverse_link_name, dsdb_dn.dn)
+                else:
+                    # Forward link exists, but backlink does not
+                    # Add the missing backlink (if the target object is not Deleted Objects?)
+                    if not target_is_deleted:
+                        self.err_missing_backlink(obj, attrname, obj.dn.extended_str(), reverse_link_name, dsdb_dn.dn)
+                continue
+
+
+
+
         return error_count
 
 
@@ -852,11 +1126,14 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
         return (set_att, list_attid, wrong_attids)
 
 
-    def fix_metadata(self, dn, attr):
+    def fix_metadata(self, obj, attr):
         '''re-write replPropertyMetaData elements for a single attribute for a
         object. This is used to fix missing replPropertyMetaData elements'''
+        guid_str = str(ndr_unpack(misc.GUID, obj['objectGUID'][0]))
+        dn = ldb.Dn(self.samdb, "<GUID=%s>" % guid_str)
         res = self.samdb.search(base = dn, scope=ldb.SCOPE_BASE, attrs = [attr],
-                                controls = ["search_options:1:2", "show_recycled:1"])
+                                controls = ["search_options:1:2",
+                                            "show_recycled:1"])
         msg = res[0]
         nmsg = ldb.Message()
         nmsg.dn = dn
@@ -1302,6 +1579,23 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                           "Failed to fix Deleted Objects container  %s" % dn):
             self.report("Fixed Deleted Objects container '%s'\n" % (dn))
 
+    def err_replica_locations(self, obj, cross_ref, attr):
+        nmsg = ldb.Message()
+        nmsg.dn = cross_ref
+        target = self.samdb.get_dsServiceName()
+
+        if self.samdb.am_rodc():
+            self.report('Not fixing %s for the RODC' % (attr, obj.dn))
+            return
+
+        if not self.confirm_all('Add yourself to the replica locations for %s?'
+                                % (obj.dn), 'fix_replica_locations'):
+            self.report('Not fixing missing/incorrect attributes on %s\n' % (obj.dn))
+            return
+
+        nmsg[attr] = ldb.MessageElement(target, ldb.FLAG_MOD_ADD, attr)
+        if self.do_modify(nmsg, [], "Failed to add %s for %s" % (attr, obj.dn)):
+            self.report("Fixed %s for %s" % (attr, obj.dn))
 
     def is_fsmo_role(self, dn):
         if dn == self.samdb.domain_dn:
@@ -1365,6 +1659,8 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
             attrs.append("systemFlags")
         if '*' in attrs:
             attrs.append("replPropertyMetaData")
+        else:
+            attrs.append("objectGUID")
 
         try:
             sd_flags = 0
@@ -1379,6 +1675,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                                         "show_recycled:1",
                                         "show_deleted:1",
                                         "sd_flags:1:%d" % sd_flags,
+                                        "reveal_internals:0",
                                     ],
                                     attrs=attrs)
         except ldb.LdbError, (enum, estr):
@@ -1415,7 +1712,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
         systemFlags = 0
 
         for attrname in obj:
-            if attrname == 'dn':
+            if attrname == 'dn' or attrname == "distinguishedName":
                 continue
 
             if str(attrname).lower() == 'objectclass':
@@ -1468,8 +1765,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
 
                 else:
                     # Here we check that the first attid is 0
-                    # (objectClass) and that the last on is the RDN
-                    # from the DN.
+                    # (objectClass).
                     if list_attid_from_md[0] != 0:
                         error_count += 1
                         self.report("ERROR: Not fixing incorrect inital attributeID in '%s' on '%s', it should be objectClass" %
@@ -1583,33 +1879,36 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                 error_count += 1
                 continue
 
+            linkID, reverse_link_name = self.get_attr_linkID_and_reverse_name(attrname)
+
             flag = self.samdb_schema.get_systemFlags_from_lDAPDisplayName(attrname)
             if (not flag & dsdb.DS_FLAG_ATTR_NOT_REPLICATED
                 and not flag & dsdb.DS_FLAG_ATTR_IS_CONSTRUCTED
-                and not self.samdb_schema.get_linkId_from_lDAPDisplayName(attrname)):
+                and not linkID):
                 set_attrs_seen.add(str(attrname).lower())
 
             if syntax_oid in [ dsdb.DSDB_SYNTAX_BINARY_DN, dsdb.DSDB_SYNTAX_OR_NAME,
                                dsdb.DSDB_SYNTAX_STRING_DN, ldb.SYNTAX_DN ]:
                 # it's some form of DN, do specialised checking on those
                 error_count += self.check_dn(obj, attrname, syntax_oid)
+            else:
 
-            values = set()
-            # check for incorrectly normalised attributes
-            for val in obj[attrname]:
-                values.add(str(val))
+                values = set()
+                # check for incorrectly normalised attributes
+                for val in obj[attrname]:
+                    values.add(str(val))
+
+                    normalised = self.samdb.dsdb_normalise_attributes(self.samdb_schema, attrname, [val])
+                    if len(normalised) != 1 or normalised[0] != val:
+                        self.err_normalise_mismatch(dn, attrname, obj[attrname])
+                        error_count += 1
+                        break
 
-                normalised = self.samdb.dsdb_normalise_attributes(self.samdb_schema, attrname, [val])
-                if len(normalised) != 1 or normalised[0] != val:
-                    self.err_normalise_mismatch(dn, attrname, obj[attrname])
+                if len(obj[attrname]) != len(values):
+                    self.err_duplicate_values(dn, attrname, obj[attrname], list(values))
                     error_count += 1
                     break
 
-            if len(obj[attrname]) != len(values):
-                   self.err_duplicate_values(dn, attrname, obj[attrname], list(values))
-                   error_count += 1
-                   break
-
             if str(attrname).lower() == "instancetype":
                 calculated_instancetype = self.calculate_instancetype(dn)
                 if len(obj["instanceType"]) != 1 or obj["instanceType"][0] != str(calculated_instancetype):
@@ -1676,7 +1975,7 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                 if not self.confirm_all("Fix missing replPropertyMetaData element '%s'" % att, 'fix_all_metadata'):
                     self.report("Not fixing missing replPropertyMetaData element '%s'" % att)
                     continue
-                self.fix_metadata(dn, att)
+                self.fix_metadata(obj, att)
 
         if self.is_fsmo_role(dn):
             if "fSMORoleOwner" not in obj and ("*" in attrs or "fsmoroleowner" in map(str.lower, attrs)):
@@ -1699,6 +1998,135 @@ newSuperior: %s""" % (str(from_dn), str(to_rdn), str(to_base)))
                 self.err_deleted_deleted_objects(obj)
                 error_count += 1
 
+        for (dns_part, msg) in self.dns_partitions:
+            if dn == dns_part and 'repsFrom' in obj:
+                location = "msDS-NC-Replica-Locations"
+                if self.samdb.am_rodc():
+                    location = "msDS-NC-RO-Replica-Locations"
+
+                if location not in msg:
+                    # There are no replica locations!
+                    self.err_replica_locations(obj, msg.dn, location)
+                    error_count += 1
+                    continue
+
+                found = False
+                for loc in msg[location]:
+                    if loc == self.samdb.get_dsServiceName():
+                        found = True
+                if not found:
+                    # This DC is not in the replica locations
+                    self.err_replica_locations(obj, msg.dn, location)
+                    error_count += 1
+
+        if dn == self.server_ref_dn:
+            # Check we have a valid RID Set
+            if "*" in attrs or "rIDSetReferences" in attrs:
+                if "rIDSetReferences" not in obj:
+                    # NO RID SET reference
+                    # We are RID master, allocate it.
+                    error_count += 1
+
+                    if self.is_rid_master:
+                        # Allocate a RID Set
+                        if self.confirm_all('Allocate the missing RID set for RID master?',
+                                            'fix_missing_rid_set_master'):
+
+                            # We don't have auto-transaction logic on
+                            # extended operations, so we have to do it
+                            # here.
+
+                            self.samdb.transaction_start()
+
+                            try:
+                                self.samdb.create_own_rid_set()
+
+                            except:
+                                self.samdb.transaction_cancel()
+                                raise
+
+                            self.samdb.transaction_commit()
+
+
+                    elif not self.samdb.am_rodc():
+                        self.report("No RID Set found for this server: %s, and we are not the RID Master (so can not self-allocate)" % dn)
+
+
+        # Check some details of our own RID Set
+        if dn == self.rid_set_dn:
+            res = self.samdb.search(base=self.rid_set_dn, scope=ldb.SCOPE_BASE,
+                                    attrs=["rIDAllocationPool",
+                                           "rIDPreviousAllocationPool",
+                                           "rIDUsedPool",
+                                           "rIDNextRID"])
+            if "rIDAllocationPool" not in res[0]:
+                self.report("No rIDAllocationPool found in %s" % dn)
+                error_count += 1
+            else:
+                next_pool = int(res[0]["rIDAllocationPool"][0])
+
+                high = (0xFFFFFFFF00000000 & next_pool) >> 32
+                low = 0x00000000FFFFFFFF & next_pool
+
+                if high <= low:
+                    self.report("Invalid RID set %d-%s, %d > %d!" % (low, high, low, high))
+                    error_count += 1
+
+                if "rIDNextRID" in res[0]:
+                    next_free_rid = int(res[0]["rIDNextRID"][0])
+                else:
+                    next_free_rid = 0
+
+                if next_free_rid == 0:
+                    next_free_rid = low
+                else:
+                    next_free_rid += 1
+
+                # Check the remainder of this pool for conflicts.  If
+                # ridalloc_allocate_rid() moves to a new pool, this
+                # will be above high, so we will stop.
+                while next_free_rid <= high:
+                    sid = "%s-%d" % (self.samdb.get_domain_sid(), next_free_rid)
+                    try:
+                        res = self.samdb.search(base="<SID=%s>" % sid, scope=ldb.SCOPE_BASE,
+                                                attrs=[])
+                    except ldb.LdbError, (enum, estr):
+                        if enum != ldb.ERR_NO_SUCH_OBJECT:
+                            raise
+                        res = None
+                    if res is not None:
+                        self.report("SID %s for %s conflicts with our current RID set in %s" % (sid, res[0].dn, dn))
+                        error_count += 1
+
+                        if self.confirm_all('Fix conflict between SID %s and RID pool in %s by allocating a new RID?'
+                                            % (sid, dn),
+                                            'fix_sid_rid_set_conflict'):
+                            self.samdb.transaction_start()
+
+                            # This will burn RIDs, which will move
+                            # past the conflict.  We then check again
+                            # to see if the new RID conflicts, until
+                            # the end of the current pool.  We don't
+                            # look at the next pool to avoid burning
+                            # all RIDs in one go in some strange
+                            # failure case.
+                            try:
+                                while True:
+                                    allocated_rid = self.samdb.allocate_rid()
+                                    if allocated_rid >= next_free_rid:
+                                        next_free_rid = allocated_rid + 1
+                                        break
+                            except:
+                                self.samdb.transaction_cancel()
+                                raise
+
+                            self.samdb.transaction_commit()
+                        else:
+                            break
+                    else:
+                        next_free_rid += 1
+
+
         return error_count
 
     ################################################################