getncchanges.c: Send linked attributes in each chunk
[nivanova/samba-autobuild/.git] / source4 / rpc_server / drsuapi / getncchanges.c
index c8f2e91911fc5f0bf4cd19eadf4d6f05fd437af3..cd3f51fc2b0135b69b2c792fd87f3081420be63d 100644 (file)
 #include "libcli/security/session.h"
 #include "rpc_server/drsuapi/dcesrv_drsuapi.h"
 #include "rpc_server/dcerpc_server_proto.h"
+#include "rpc_server/common/sid_helper.h"
 #include "../libcli/drsuapi/drsuapi.h"
 #include "lib/util/binsearch.h"
 #include "lib/util/tsort.h"
 #include "auth/session.h"
 #include "dsdb/common/util.h"
+#include "lib/dbwrap/dbwrap.h"
+#include "lib/dbwrap/dbwrap_rbt.h"
+#include "librpc/gen_ndr/ndr_misc.h"
 
-/* state of a partially completed getncchanges call */
+#undef DBGC_CLASS
+#define DBGC_CLASS            DBGC_DRS_REPL
+
+#define DRS_GUID_SIZE       16
+
+/*
+ * state of a partially-completed replication cycle. This state persists
+ * over multiple calls to dcesrv_drsuapi_DsGetNCChanges()
+ */
 struct drsuapi_getncchanges_state {
+       struct db_context *obj_cache;
        struct GUID *guids;
        uint32_t num_records;
        uint32_t num_processed;
        struct ldb_dn *ncRoot_dn;
+       struct GUID ncRoot_guid;
        bool is_schema_nc;
+       bool is_get_anc;
        uint64_t min_usn;
        uint64_t max_usn;
        struct drsuapi_DsReplicaHighWaterMark last_hwm;
@@ -53,15 +68,29 @@ struct drsuapi_getncchanges_state {
        struct drsuapi_DsReplicaCursor2CtrEx *final_udv;
        struct drsuapi_DsReplicaLinkedAttribute *la_list;
        uint32_t la_count;
-       struct la_for_sorting *la_sorted;
        uint32_t la_idx;
+
+       /* these are just used for debugging the replication's progress */
+       uint32_t links_given;
+       uint32_t total_links;
 };
 
 /* We must keep the GUIDs in NDR form for sorting */
 struct la_for_sorting {
-       struct drsuapi_DsReplicaLinkedAttribute *link;
-       uint8_t target_guid[16];
-        uint8_t source_guid[16];
+       const struct drsuapi_DsReplicaLinkedAttribute *link;
+       uint8_t target_guid[DRS_GUID_SIZE];
+       uint8_t source_guid[DRS_GUID_SIZE];
+};
+
+/*
+ * stores the state for a chunk of replication data. This state information
+ * only exists for a single call to dcesrv_drsuapi_DsGetNCChanges()
+ */
+struct getncchanges_repl_chunk {
+       struct drsuapi_DsGetNCChangesCtr6 *ctr6;
+
+       /* the last object written to the response */
+       struct drsuapi_DsReplicaObjectListItemEx *last_object;
 };
 
 static int drsuapi_DsReplicaHighWaterMark_cmp(const struct drsuapi_DsReplicaHighWaterMark *h1,
@@ -88,7 +117,7 @@ static int drsuapi_DsReplicaHighWaterMark_cmp(const struct drsuapi_DsReplicaHigh
   build a DsReplicaObjectIdentifier from a ldb msg
  */
 static struct drsuapi_DsReplicaObjectIdentifier *get_object_identifier(TALLOC_CTX *mem_ctx,
-                                                                      struct ldb_message *msg)
+                                                                      const struct ldb_message *msg)
 {
        struct drsuapi_DsReplicaObjectIdentifier *identifier;
        struct dom_sid *sid;
@@ -130,7 +159,6 @@ static bool udv_filter(const struct drsuapi_DsReplicaCursorCtrEx *udv,
                return true;
        }
        return false;
-
 }
 
 static int uint32_t_cmp(uint32_t a1, uint32_t a2)
@@ -185,39 +213,291 @@ fail:
        }
 }
 
+/*
+ * Similar to function in repl_meta_data without the extra
+ * dependencies.
+ */
+static WERROR get_parsed_dns_trusted(TALLOC_CTX *mem_ctx, struct ldb_message_element *el,
+                                 struct parsed_dn **pdn)
+{
+       /* Here we get a list of 'struct parsed_dns' without the parsing */
+       int i;
+       *pdn = talloc_zero_array(mem_ctx, struct parsed_dn,
+                                el->num_values);
+       if (!*pdn) {
+               return WERR_DS_DRA_INTERNAL_ERROR;
+       }
+
+       for (i = 0; i < el->num_values; i++) {
+               (*pdn)[i].v = &el->values[i];
+       }
+
+       return WERR_OK;
+}
+
+static WERROR getncchanges_update_revealed_list(struct ldb_context *sam_ctx,
+                                               TALLOC_CTX *mem_ctx,
+                                               struct ldb_message **msg,
+                                               struct ldb_dn *object_dn,
+                                               const struct GUID *object_guid,
+                                               const struct dsdb_attribute *sa,
+                                               struct replPropertyMetaData1 *meta_data,
+                                               struct ldb_message *revealed_users)
+{
+       enum ndr_err_code ndr_err;
+       int ldb_err;
+       char *attr_str = NULL;
+       char *attr_hex = NULL;
+       DATA_BLOB attr_blob;
+       struct ldb_message_element *existing = NULL, *el_add = NULL, *el_del = NULL;
+       const char * const * secret_attributes = ldb_get_opaque(sam_ctx, "LDB_SECRET_ATTRIBUTE_LIST");
+
+       if (!ldb_attr_in_list(secret_attributes,
+                             sa->lDAPDisplayName)) {
+               return WERR_OK;
+       }
+
+
+       ndr_err = ndr_push_struct_blob(&attr_blob, mem_ctx, meta_data, (ndr_push_flags_fn_t)ndr_push_replPropertyMetaData1);
+       if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+               return WERR_DS_DRA_INTERNAL_ERROR;
+       }
+
+       attr_hex = hex_encode_talloc(mem_ctx, attr_blob.data, attr_blob.length);
+       if (attr_hex == NULL) {
+               return WERR_NOT_ENOUGH_MEMORY;
+       }
+
+       attr_str = talloc_asprintf(mem_ctx, "B:%zd:%s:%s", attr_blob.length*2, attr_hex, ldb_dn_get_linearized(object_dn));
+       if (attr_str == NULL) {
+               return WERR_NOT_ENOUGH_MEMORY;
+       }
+
+       existing = ldb_msg_find_element(revealed_users, "msDS-RevealedUsers");
+       if (existing != NULL) {
+               /* Replace the old value (if one exists) with the current one */
+               struct parsed_dn *link_dns;
+               struct parsed_dn *exact = NULL, *unused = NULL;
+               WERROR werr;
+               uint8_t attid[4];
+               DATA_BLOB partial_meta;
+
+               werr = get_parsed_dns_trusted(mem_ctx, existing, &link_dns);
+               if (!W_ERROR_IS_OK(werr)) {
+                       return werr;
+               }
+
+               /* Construct a partial metadata blob to match on in the DB */
+               SIVAL(attid, 0, sa->attributeID_id);
+               partial_meta.length = 4;
+               partial_meta.data = attid;
+
+               /* Binary search using GUID and attribute id for uniqueness */
+               ldb_err = parsed_dn_find(sam_ctx, link_dns, existing->num_values,
+                                        object_guid, object_dn,
+                                        partial_meta, 4,
+                                        &exact, &unused,
+                                        DSDB_SYNTAX_BINARY_DN, true);
+
+               if (ldb_err != LDB_SUCCESS) {
+                       DEBUG(0,(__location__ ": Failed parsed DN find - %s\n",
+                                ldb_errstring(sam_ctx)));
+                       return WERR_DS_DRA_INTERNAL_ERROR;
+               }
+
+               if (exact != NULL) {
+                       /* Perform some verification of the blob */
+                       struct replPropertyMetaData1 existing_meta_data;
+                       ndr_err = ndr_pull_struct_blob_all_noalloc(&exact->dsdb_dn->extra_part,
+                                                                  &existing_meta_data,
+                                                                  (ndr_pull_flags_fn_t)ndr_pull_replPropertyMetaData1);
+                       if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+                               return WERR_DS_DRA_INTERNAL_ERROR;
+                       }
+
+                       if (existing_meta_data.attid == sa->attributeID_id) {
+                               ldb_err = ldb_msg_add_empty(*msg, "msDS-RevealedUsers", LDB_FLAG_MOD_DELETE, &el_del);
+                               if (ldb_err != LDB_SUCCESS) {
+                                       return WERR_DS_DRA_INTERNAL_ERROR;
+                               }
+
+                               el_del->values = talloc_array((*msg)->elements, struct ldb_val, 1);
+                               if (el_del->values == NULL) {
+                                       return WERR_NOT_ENOUGH_MEMORY;
+                               }
+                               el_del->values[0] = *exact->v;
+                               el_del->num_values = 1;
+                       } else {
+                               return WERR_DS_DRA_INTERNAL_ERROR;
+                       }
+               }
+       }
+
+       ldb_err = ldb_msg_add_empty(*msg, "msDS-RevealedUsers", LDB_FLAG_MOD_ADD, &el_add);
+       if (ldb_err != LDB_SUCCESS) {
+               return WERR_DS_DRA_INTERNAL_ERROR;
+       }
+
+       el_add->values = talloc_array((*msg)->elements, struct ldb_val, 1);
+       if (el_add->values == NULL) {
+               return WERR_NOT_ENOUGH_MEMORY;
+
+       }
+
+       el_add->values[0] = data_blob_string_const(attr_str);
+       el_add->num_values = 1;
+
+       return WERR_OK;
+}
+
+/*
+ * This function filter attributes for build_object based on the
+ * uptodatenessvector and partial attribute set.
+ *
+ * Any secret attributes are forced here for REPL_SECRET, and audited at this
+ * point with msDS-RevealedUsers.
+ */
+static WERROR get_nc_changes_filter_attrs(struct drsuapi_DsReplicaObjectListItemEx *obj,
+                                         struct replPropertyMetaDataBlob md,
+                                         struct ldb_context *sam_ctx,
+                                         const struct ldb_message *msg,
+                                         const struct GUID *guid,
+                                         uint32_t *count,
+                                         uint64_t highest_usn,
+                                         const struct dsdb_attribute *rdn_sa,
+                                         struct dsdb_schema *schema,
+                                         struct drsuapi_DsReplicaCursorCtrEx *uptodateness_vector,
+                                         struct drsuapi_DsPartialAttributeSet *partial_attribute_set,
+                                         uint32_t *local_pas,
+                                         uint32_t *attids,
+                                         bool exop_secret,
+                                         struct ldb_message **revealed_list_msg,
+                                         struct ldb_message *existing_revealed_list_msg)
+{
+       uint32_t i, n;
+       WERROR werr;
+       for (n=i=0; i<md.ctr.ctr1.count; i++) {
+               const struct dsdb_attribute *sa;
+               bool force_attribute = false;
+
+               /* if the attribute has not changed, and it is not the
+                  instanceType then don't include it */
+               if (md.ctr.ctr1.array[i].local_usn < highest_usn &&
+                   !exop_secret &&
+                   md.ctr.ctr1.array[i].attid != DRSUAPI_ATTID_instanceType) continue;
+
+               /* don't include the rDN */
+               if (md.ctr.ctr1.array[i].attid == rdn_sa->attributeID_id) continue;
+
+               sa = dsdb_attribute_by_attributeID_id(schema, md.ctr.ctr1.array[i].attid);
+               if (!sa) {
+                       DEBUG(0,(__location__ ": Failed to find attribute in schema for attrid %u mentioned in replPropertyMetaData of %s\n",
+                                (unsigned int)md.ctr.ctr1.array[i].attid,
+                                ldb_dn_get_linearized(msg->dn)));
+                       return WERR_DS_DRA_INTERNAL_ERROR;
+               }
+
+               if (sa->linkID) {
+                       struct ldb_message_element *el;
+                       el = ldb_msg_find_element(msg, sa->lDAPDisplayName);
+                       if (el && el->num_values && dsdb_dn_is_upgraded_link_val(&el->values[0])) {
+                               /* don't send upgraded links inline */
+                               continue;
+                       }
+               }
+
+               if (exop_secret &&
+                   !dsdb_attr_in_rodc_fas(sa)) {
+                       force_attribute = true;
+                       DEBUG(4,("Forcing attribute %s in %s\n",
+                                sa->lDAPDisplayName, ldb_dn_get_linearized(msg->dn)));
+                       werr = getncchanges_update_revealed_list(sam_ctx, obj,
+                                                                revealed_list_msg,
+                                                                msg->dn, guid, sa,
+                                                                &md.ctr.ctr1.array[i],
+                                                                existing_revealed_list_msg);
+                       if (!W_ERROR_IS_OK(werr)) {
+                               return werr;
+                       }
+               }
+
+               /* filter by uptodateness_vector */
+               if (md.ctr.ctr1.array[i].attid != DRSUAPI_ATTID_instanceType &&
+                   !force_attribute &&
+                   udv_filter(uptodateness_vector,
+                              &md.ctr.ctr1.array[i].originating_invocation_id,
+                              md.ctr.ctr1.array[i].originating_usn)) {
+                       continue;
+               }
+
+               /* filter by partial_attribute_set */
+               if (partial_attribute_set && !force_attribute) {
+                       uint32_t *result = NULL;
+                       BINARY_ARRAY_SEARCH_V(local_pas, partial_attribute_set->num_attids, sa->attributeID_id,
+                                             uint32_t_cmp, result);
+                       if (result == NULL) {
+                               continue;
+                       }
+               }
+
+               obj->meta_data_ctr->meta_data[n].originating_change_time = md.ctr.ctr1.array[i].originating_change_time;
+               obj->meta_data_ctr->meta_data[n].version = md.ctr.ctr1.array[i].version;
+               obj->meta_data_ctr->meta_data[n].originating_invocation_id = md.ctr.ctr1.array[i].originating_invocation_id;
+               obj->meta_data_ctr->meta_data[n].originating_usn = md.ctr.ctr1.array[i].originating_usn;
+               attids[n] = md.ctr.ctr1.array[i].attid;
+
+               n++;
+       }
+
+       *count = n;
+
+       return WERR_OK;
+}
+
 /* 
   drsuapi_DsGetNCChanges for one object
 */
 static WERROR get_nc_changes_build_object(struct drsuapi_DsReplicaObjectListItemEx *obj,
-                                         struct ldb_message *msg,
+                                         const struct ldb_message *msg,
                                          struct ldb_context *sam_ctx,
-                                         struct ldb_dn *ncRoot_dn,
-                                         bool   is_schema_nc,
+                                         struct drsuapi_getncchanges_state *getnc_state,
                                          struct dsdb_schema *schema,
                                          DATA_BLOB *session_key,
-                                         uint64_t highest_usn,
-                                         uint32_t replica_flags,
-                                         struct drsuapi_DsPartialAttributeSet *partial_attribute_set,
-                                         struct drsuapi_DsReplicaCursorCtrEx *uptodateness_vector,
-                                         enum drsuapi_DsExtendedOperation extended_op,
+                                         struct drsuapi_DsGetNCChangesRequest10 *req10,
                                          bool force_object_return,
-                                         uint32_t *local_pas)
+                                         uint32_t *local_pas,
+                                         struct ldb_dn *machine_dn,
+                                         const struct GUID *guid)
 {
        const struct ldb_val *md_value;
        uint32_t i, n;
        struct replPropertyMetaDataBlob md;
        uint32_t rid = 0;
+       int ldb_err;
        enum ndr_err_code ndr_err;
        uint32_t *attids;
        const char *rdn;
        const struct dsdb_attribute *rdn_sa;
+       uint64_t uSNChanged;
        unsigned int instanceType;
        struct dsdb_syntax_ctx syntax_ctx;
+       struct ldb_result *res = NULL;
+       WERROR werr;
+       int ret;
+       uint32_t replica_flags = req10->replica_flags;
+       struct drsuapi_DsPartialAttributeSet *partial_attribute_set =
+                       req10->partial_attribute_set;
+       struct drsuapi_DsReplicaCursorCtrEx *uptodateness_vector =
+                       req10->uptodateness_vector;
+       enum drsuapi_DsExtendedOperation extended_op = req10->extended_op;
+       bool is_schema_nc = getnc_state->is_schema_nc;
+       uint64_t highest_usn = getnc_state->min_usn;
 
        /* make dsdb sytanx context for conversions */
        dsdb_syntax_ctx_init(&syntax_ctx, sam_ctx, schema);
        syntax_ctx.is_schema_nc = is_schema_nc;
 
+       uSNChanged = ldb_msg_find_attr_as_uint64(msg, "uSNChanged", 0);
        instanceType = ldb_msg_find_attr_as_uint(msg, "instanceType", 0);
        if (instanceType & INSTANCE_TYPE_IS_NC_HEAD) {
                obj->is_nc_prefix = true;
@@ -248,6 +528,11 @@ static WERROR get_nc_changes_build_object(struct drsuapi_DsReplicaObjectListItem
                return WERR_OK;
        }
 
+       if (uSNChanged <= highest_usn) {
+               /* nothing to send */
+               return WERR_OK;
+       }
+
        ndr_err = ndr_pull_struct_blob(md_value, obj, &md,
                                       (ndr_pull_flags_fn_t)ndr_pull_replPropertyMetaDataBlob);
        if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
@@ -281,68 +566,79 @@ static WERROR get_nc_changes_build_object(struct drsuapi_DsReplicaObjectListItem
        dom_sid_split_rid(NULL, &obj->object.identifier->sid, NULL, &rid);
 
        obj->meta_data_ctr->meta_data = talloc_array(obj, struct drsuapi_DsReplicaMetaData, md.ctr.ctr1.count);
-       for (n=i=0; i<md.ctr.ctr1.count; i++) {
-               const struct dsdb_attribute *sa;
-               bool force_attribute = false;
-
-               /* if the attribute has not changed, and it is not the
-                  instanceType then don't include it */
-               if (md.ctr.ctr1.array[i].local_usn < highest_usn &&
-                   extended_op != DRSUAPI_EXOP_REPL_SECRET &&
-                   md.ctr.ctr1.array[i].attid != DRSUAPI_ATTID_instanceType) continue;
 
-               /* don't include the rDN */
-               if (md.ctr.ctr1.array[i].attid == rdn_sa->attributeID_id) continue;
+       if (extended_op == DRSUAPI_EXOP_REPL_SECRET) {
+               /* Get the existing revealed users for the destination */
+               struct ldb_message *revealed_list_msg = NULL;
+               struct ldb_message *existing_revealed_list_msg = NULL;
+               const char *machine_attrs[] = {
+                       "msDS-RevealedUsers",
+                       NULL
+               };
+
+               revealed_list_msg = ldb_msg_new(sam_ctx);
+               if (revealed_list_msg == NULL) {
+                       return WERR_NOT_ENOUGH_MEMORY;
+               }
+               revealed_list_msg->dn = machine_dn;
 
-               sa = dsdb_attribute_by_attributeID_id(schema, md.ctr.ctr1.array[i].attid);
-               if (!sa) {
-                       DEBUG(0,(__location__ ": Failed to find attribute in schema for attrid %u mentioned in replPropertyMetaData of %s\n",
-                                (unsigned int)md.ctr.ctr1.array[i].attid,
-                                ldb_dn_get_linearized(msg->dn)));
+               ret = ldb_transaction_start(sam_ctx);
+               if (ret != LDB_SUCCESS) {
+                       DEBUG(0,(__location__ ": Failed transaction start - %s\n",
+                                ldb_errstring(sam_ctx)));
                        return WERR_DS_DRA_INTERNAL_ERROR;
                }
 
-               if (sa->linkID) {
-                       struct ldb_message_element *el;
-                       el = ldb_msg_find_element(msg, sa->lDAPDisplayName);
-                       if (el && el->num_values && dsdb_dn_is_upgraded_link_val(&el->values[0])) {
-                               /* don't send upgraded links inline */
-                               continue;
-                       }
+               ldb_err = dsdb_search_dn(sam_ctx, obj, &res, machine_dn, machine_attrs, DSDB_SEARCH_SHOW_EXTENDED_DN);
+               if (ldb_err != LDB_SUCCESS || res->count != 1) {
+                       ldb_transaction_cancel(sam_ctx);
+                       return WERR_DS_DRA_INTERNAL_ERROR;
                }
 
-               if (extended_op == DRSUAPI_EXOP_REPL_SECRET &&
-                   !dsdb_attr_in_rodc_fas(sa)) {
-                       force_attribute = true;
-                       DEBUG(4,("Forcing attribute %s in %s\n",
-                                sa->lDAPDisplayName, ldb_dn_get_linearized(msg->dn)));
-               }
+               existing_revealed_list_msg = res->msgs[0];
 
-               /* filter by uptodateness_vector */
-               if (md.ctr.ctr1.array[i].attid != DRSUAPI_ATTID_instanceType &&
-                   !force_attribute &&
-                   udv_filter(uptodateness_vector,
-                              &md.ctr.ctr1.array[i].originating_invocation_id,
-                              md.ctr.ctr1.array[i].originating_usn)) {
-                       continue;
+               werr = get_nc_changes_filter_attrs(obj, md, sam_ctx, msg,
+                                                  guid, &n, highest_usn,
+                                                  rdn_sa, schema,
+                                                  uptodateness_vector,
+                                                  partial_attribute_set, local_pas,
+                                                  attids,
+                                                  true,
+                                                  &revealed_list_msg,
+                                                  existing_revealed_list_msg);
+               if (!W_ERROR_IS_OK(werr)) {
+                       ldb_transaction_cancel(sam_ctx);
+                       return werr;
                }
 
-               /* filter by partial_attribute_set */
-               if (partial_attribute_set) {
-                       uint32_t *result = NULL;
-                       BINARY_ARRAY_SEARCH_V(local_pas, partial_attribute_set->num_attids, sa->attributeID_id,
-                                             uint32_t_cmp, result);
-                       if (result == NULL) {
-                               continue;
+               if (revealed_list_msg != NULL) {
+                       ret = ldb_modify(sam_ctx, revealed_list_msg);
+                       if (ret != LDB_SUCCESS) {
+                               DEBUG(0,(__location__ ": Failed to alter revealed links - %s\n",
+                                        ldb_errstring(sam_ctx)));
+                               ldb_transaction_cancel(sam_ctx);
+                               return WERR_DS_DRA_INTERNAL_ERROR;
                        }
                }
 
-               obj->meta_data_ctr->meta_data[n].originating_change_time = md.ctr.ctr1.array[i].originating_change_time;
-               obj->meta_data_ctr->meta_data[n].version = md.ctr.ctr1.array[i].version;
-               obj->meta_data_ctr->meta_data[n].originating_invocation_id = md.ctr.ctr1.array[i].originating_invocation_id;
-               obj->meta_data_ctr->meta_data[n].originating_usn = md.ctr.ctr1.array[i].originating_usn;
-               attids[n] = md.ctr.ctr1.array[i].attid;
-               n++;
+               ret = ldb_transaction_commit(sam_ctx);
+               if (ret != LDB_SUCCESS) {
+                       DEBUG(0,(__location__ ": Failed transaction commit - %s\n",
+                                ldb_errstring(sam_ctx)));
+                       return WERR_DS_DRA_INTERNAL_ERROR;
+               }
+       } else {
+               werr = get_nc_changes_filter_attrs(obj, md, sam_ctx, msg, guid,
+                                                  &n, highest_usn, rdn_sa,
+                                                  schema, uptodateness_vector,
+                                                  partial_attribute_set, local_pas,
+                                                  attids,
+                                                  false,
+                                                  NULL,
+                                                  NULL);
+               if (!W_ERROR_IS_OK(werr)) {
+                       return werr;
+               }
        }
 
        /* ignore it if its an empty change. Note that renames always
@@ -379,7 +675,6 @@ static WERROR get_nc_changes_build_object(struct drsuapi_DsReplicaObjectListItem
         */
        for (i=0; i<obj->object.attribute_ctr.num_attributes; i++) {
                struct ldb_message_element *el;
-               WERROR werr;
                const struct dsdb_attribute *sa;
 
                sa = dsdb_attribute_by_attributeID_id(schema, attids[i]);
@@ -446,7 +741,7 @@ static WERROR get_nc_changes_add_la(TALLOC_CTX *mem_ctx,
                                    struct ldb_context *sam_ctx,
                                    const struct dsdb_schema *schema,
                                    const struct dsdb_attribute *sa,
-                                   struct ldb_message *msg,
+                                   const struct ldb_message *msg,
                                    struct dsdb_dn *dsdb_dn,
                                    struct drsuapi_DsReplicaLinkedAttribute **la_list,
                                    uint32_t *la_count,
@@ -570,19 +865,34 @@ static WERROR get_nc_changes_add_la(TALLOC_CTX *mem_ctx,
  */
 static WERROR get_nc_changes_add_links(struct ldb_context *sam_ctx,
                                       TALLOC_CTX *mem_ctx,
-                                      struct ldb_dn *ncRoot_dn,
                                       bool is_schema_nc,
                                       struct dsdb_schema *schema,
                                       uint64_t highest_usn,
                                       uint32_t replica_flags,
-                                      struct ldb_message *msg,
+                                      const struct ldb_message *msg,
                                       struct drsuapi_DsReplicaLinkedAttribute **la_list,
                                       uint32_t *la_count,
                                       struct drsuapi_DsReplicaCursorCtrEx *uptodateness_vector)
 {
        unsigned int i;
-       TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
-       uint64_t uSNChanged = ldb_msg_find_attr_as_int(msg, "uSNChanged", -1);
+       TALLOC_CTX *tmp_ctx = NULL;
+       uint64_t uSNChanged = ldb_msg_find_attr_as_uint64(msg, "uSNChanged", 0);
+       bool is_critical = ldb_msg_find_attr_as_bool(msg, "isCriticalSystemObject", false);
+
+       if (replica_flags & DRSUAPI_DRS_CRITICAL_ONLY) {
+               if (!is_critical) {
+                       return WERR_OK;
+               }
+       }
+
+       if (uSNChanged <= highest_usn) {
+               return WERR_OK;
+       }
+
+       tmp_ctx = talloc_new(mem_ctx);
+       if (tmp_ctx == NULL) {
+               return WERR_NOT_ENOUGH_MEMORY;
+       }
 
        for (i=0; i<msg->num_elements; i++) {
                struct ldb_message_element *el = &msg->elements[i];
@@ -633,7 +943,7 @@ static WERROR get_nc_changes_add_links(struct ldb_context *sam_ctx,
                                return WERR_DS_DRA_INTERNAL_ERROR;
                        }
 
-                       if (local_usn < highest_usn) {
+                       if (local_usn <= highest_usn) {
                                continue;
                        }
 
@@ -723,16 +1033,6 @@ struct drsuapi_changed_objects {
        uint64_t usn;
 };
 
-/*
-  sort the objects we send by tree order
- */
-static int site_res_cmp_anc_order(struct drsuapi_changed_objects *m1,
-                                 struct drsuapi_changed_objects *m2,
-                                 struct drsuapi_getncchanges_state *getnc_state)
-{
-       return ldb_dn_compare(m2->dn, m1->dn);
-}
-
 /*
   sort the objects we send first by uSNChanged
  */
@@ -866,123 +1166,28 @@ static WERROR getncchanges_rid_alloc(struct drsuapi_bind_state *b_state,
 }
 
 /*
-  return an array of SIDs from a ldb_message given an attribute name
-  assumes the SIDs are in extended DN format
+  handle a DRSUAPI_EXOP_REPL_SECRET call
  */
-static WERROR samdb_result_sid_array_dn(struct ldb_context *sam_ctx,
-                                       struct ldb_message *msg,
-                                       TALLOC_CTX *mem_ctx,
-                                       const char *attr,
-                                       const struct dom_sid ***sids)
+static WERROR getncchanges_repl_secret(struct drsuapi_bind_state *b_state,
+                                      TALLOC_CTX *mem_ctx,
+                                      struct drsuapi_DsGetNCChangesRequest10 *req10,
+                                      struct dom_sid *user_sid,
+                                      struct drsuapi_DsGetNCChangesCtr6 *ctr6,
+                                      bool has_get_all_changes,
+                                      struct ldb_dn **machine_dn)
 {
-       struct ldb_message_element *el;
-       unsigned int i;
-
-       el = ldb_msg_find_element(msg, attr);
-       if (!el) {
-               *sids = NULL;
-               return WERR_OK;
-       }
-
-       (*sids) = talloc_array(mem_ctx, const struct dom_sid *, el->num_values + 1);
-       W_ERROR_HAVE_NO_MEMORY(*sids);
-
-       for (i=0; i<el->num_values; i++) {
-               struct ldb_dn *dn = ldb_dn_from_ldb_val(mem_ctx, sam_ctx, &el->values[i]);
-               NTSTATUS status;
-               struct dom_sid *sid;
-
-               sid = talloc(*sids, struct dom_sid);
-               W_ERROR_HAVE_NO_MEMORY(sid);
-               status = dsdb_get_extended_dn_sid(dn, sid, "SID");
-               if (!NT_STATUS_IS_OK(status)) {
-                       return WERR_INTERNAL_DB_CORRUPTION;
-               }
-               (*sids)[i] = sid;
-       }
-       (*sids)[i] = NULL;
-
-       return WERR_OK;
-}
-
-
-/*
-  return an array of SIDs from a ldb_message given an attribute name
-  assumes the SIDs are in NDR form
- */
-static WERROR samdb_result_sid_array_ndr(struct ldb_context *sam_ctx,
-                                        struct ldb_message *msg,
-                                        TALLOC_CTX *mem_ctx,
-                                        const char *attr,
-                                        const struct dom_sid ***sids)
-{
-       struct ldb_message_element *el;
-       unsigned int i;
-
-       el = ldb_msg_find_element(msg, attr);
-       if (!el) {
-               *sids = NULL;
-               return WERR_OK;
-       }
-
-       (*sids) = talloc_array(mem_ctx, const struct dom_sid *, el->num_values + 1);
-       W_ERROR_HAVE_NO_MEMORY(*sids);
-
-       for (i=0; i<el->num_values; i++) {
-               enum ndr_err_code ndr_err;
-               struct dom_sid *sid;
-
-               sid = talloc(*sids, struct dom_sid);
-               W_ERROR_HAVE_NO_MEMORY(sid);
-
-               ndr_err = ndr_pull_struct_blob(&el->values[i], sid, sid,
-                                              (ndr_pull_flags_fn_t)ndr_pull_dom_sid);
-               if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
-                       return WERR_INTERNAL_DB_CORRUPTION;
-               }
-               (*sids)[i] = sid;
-       }
-       (*sids)[i] = NULL;
-
-       return WERR_OK;
-}
-
-/*
-  see if any SIDs in list1 are in list2
- */
-static bool sid_list_match(const struct dom_sid **list1, const struct dom_sid **list2)
-{
-       unsigned int i, j;
-       /* do we ever have enough SIDs here to worry about O(n^2) ? */
-       for (i=0; list1[i]; i++) {
-               for (j=0; list2[j]; j++) {
-                       if (dom_sid_equal(list1[i], list2[j])) {
-                               return true;
-                       }
-               }
-       }
-       return false;
-}
-
-/*
-  handle a DRSUAPI_EXOP_REPL_SECRET call
- */
-static WERROR getncchanges_repl_secret(struct drsuapi_bind_state *b_state,
-                                      TALLOC_CTX *mem_ctx,
-                                      struct drsuapi_DsGetNCChangesRequest10 *req10,
-                                      struct dom_sid *user_sid,
-                                      struct drsuapi_DsGetNCChangesCtr6 *ctr6,
-                                      bool has_get_all_changes)
-{
-       struct drsuapi_DsReplicaObjectIdentifier *ncRoot = req10->naming_context;
-       struct ldb_dn *obj_dn = NULL;
-       struct ldb_dn *rodc_dn, *krbtgt_link_dn;
-       int ret;
-       const char *rodc_attrs[] = { "msDS-KrbTgtLink", "msDS-NeverRevealGroup", "msDS-RevealOnDemandGroup", NULL };
-       const char *obj_attrs[] = { "tokenGroups", "objectSid", "UserAccountControl", "msDS-KrbTgtLinkBL", NULL };
-       struct ldb_result *rodc_res, *obj_res;
-       const struct dom_sid **never_reveal_sids, **reveal_sids, **token_sids;
-       WERROR werr;
+       struct drsuapi_DsReplicaObjectIdentifier *ncRoot = req10->naming_context;
+       struct ldb_dn *obj_dn = NULL;
+       struct ldb_dn *ntds_dn = NULL, *server_dn = NULL;
+       struct ldb_dn *rodc_dn, *krbtgt_link_dn;
+       int ret;
+       const char *rodc_attrs[] = { "msDS-KrbTgtLink", "msDS-NeverRevealGroup", "msDS-RevealOnDemandGroup", "objectGUID", NULL };
+       const char *obj_attrs[] = { "tokenGroups", "objectSid", "UserAccountControl", "msDS-KrbTgtLinkBL", NULL };
+       struct ldb_result *rodc_res = NULL, *obj_res = NULL;
+       const struct dom_sid **never_reveal_sids, **reveal_sids, **token_sids;
+       const struct dom_sid *object_sid = NULL;
+       WERROR werr;
+       const struct dom_sid *additional_sids[] = { NULL, NULL };
 
        DEBUG(3,(__location__ ": DRSUAPI_EXOP_REPL_SECRET extended op on %s\n",
                 drs_ObjectIdentifier_to_string(mem_ctx, ncRoot)));
@@ -998,7 +1203,32 @@ static WERROR getncchanges_repl_secret(struct drsuapi_bind_state *b_state,
        if (b_state->sam_ctx_system == NULL) {
                /* this operation needs system level access */
                ctr6->extended_ret = DRSUAPI_EXOP_ERR_ACCESS_DENIED;
-               return WERR_DS_DRA_SOURCE_DISABLED;
+               return WERR_DS_DRA_ACCESS_DENIED;
+       }
+
+       /*
+        * Before we accept or deny, fetch the machine DN for the destination
+        * DSA GUID.
+        *
+        * If we are the RODC, we will check that this matches the SID.
+        */
+       ret = dsdb_find_dn_by_guid(b_state->sam_ctx_system, mem_ctx,
+                                  &req10->destination_dsa_guid, 0,
+                                  &ntds_dn);
+       if (ret != LDB_SUCCESS) {
+               goto failed;
+       }
+
+       server_dn = ldb_dn_get_parent(mem_ctx, ntds_dn);
+       if (server_dn == NULL) {
+               goto failed;
+       }
+
+       ret = samdb_reference_dn(b_state->sam_ctx_system, mem_ctx, server_dn,
+                                "serverReference", machine_dn);
+
+       if (ret != LDB_SUCCESS) {
+               goto failed;
        }
 
        /*
@@ -1021,13 +1251,13 @@ static WERROR getncchanges_repl_secret(struct drsuapi_bind_state *b_state,
         * Which basically means that if you have GET_ALL_CHANGES rights (~== RWDC)
         * then you can do EXOP_REPL_SECRETS
         */
+       obj_dn = drs_ObjectIdentifier_to_dn(mem_ctx, b_state->sam_ctx_system, ncRoot);
+       if (!ldb_dn_validate(obj_dn)) goto failed;
+
        if (has_get_all_changes) {
                goto allowed;
        }
 
-       obj_dn = drs_ObjectIdentifier_to_dn(mem_ctx, b_state->sam_ctx_system, ncRoot);
-       if (!ldb_dn_validate(obj_dn)) goto failed;
-
        rodc_dn = ldb_dn_new_fmt(mem_ctx, b_state->sam_ctx_system, "<SID=%s>",
                                 dom_sid_string(mem_ctx, user_sid));
        if (!ldb_dn_validate(rodc_dn)) goto failed;
@@ -1041,11 +1271,21 @@ static WERROR getncchanges_repl_secret(struct drsuapi_bind_state *b_state,
        if (ret != LDB_SUCCESS || obj_res->count != 1) goto failed;
 
        /* if the object SID is equal to the user_sid, allow */
-       if (dom_sid_equal(user_sid,
-                         samdb_result_dom_sid(mem_ctx, obj_res->msgs[0], "objectSid"))) {
+       object_sid = samdb_result_dom_sid(mem_ctx, obj_res->msgs[0], "objectSid");
+       if (dom_sid_equal(user_sid, object_sid)) {
                goto allowed;
        }
 
+       additional_sids[0] = object_sid;
+
+       /*
+        * Must be an RODC account at this point, verify machine DN matches the
+        * SID account
+        */
+       if (ldb_dn_compare(rodc_res->msgs[0]->dn, *machine_dn) != 0) {
+               goto denied;
+       }
+
        /* an RODC is allowed to get its own krbtgt account secrets */
        krbtgt_link_dn = samdb_result_dn(b_state->sam_ctx_system, mem_ctx,
                                         rodc_res->msgs[0], "msDS-KrbTgtLink", NULL);
@@ -1078,8 +1318,14 @@ static WERROR getncchanges_repl_secret(struct drsuapi_bind_state *b_state,
                goto denied;
        }
 
+       /*
+        * The SID list needs to include itself as well as the tokenGroups.
+        *
+        * TODO determine if sIDHistory is required for this check
+        */
        werr = samdb_result_sid_array_ndr(b_state->sam_ctx_system, obj_res->msgs[0],
-                                        mem_ctx, "tokenGroups", &token_sids);
+                                         mem_ctx, "tokenGroups", &token_sids,
+                                         additional_sids, 1);
        if (!W_ERROR_IS_OK(werr) || token_sids==NULL) {
                goto denied;
        }
@@ -1099,12 +1345,12 @@ denied:
        DEBUG(2,(__location__ ": Denied single object with secret replication for %s by RODC %s\n",
                 ldb_dn_get_linearized(obj_dn), ldb_dn_get_linearized(rodc_res->msgs[0]->dn)));
        ctr6->extended_ret = DRSUAPI_EXOP_ERR_NONE;
-       return WERR_DS_DRA_ACCESS_DENIED;
+       return WERR_DS_DRA_SECRETS_DENIED;
 
 allowed:
        DEBUG(2,(__location__ ": Allowed single object with secret replication for %s by %s %s\n",
                 ldb_dn_get_linearized(obj_dn), has_get_all_changes?"RWDC":"RODC",
-                ldb_dn_get_linearized(rodc_res->msgs[0]->dn)));
+                ldb_dn_get_linearized(*machine_dn)));
        ctr6->extended_ret = DRSUAPI_EXOP_ERR_SUCCESS;
        req10->highwatermark.highest_usn = 0;
        return WERR_OK;
@@ -1116,7 +1362,6 @@ failed:
        return WERR_DS_DRA_BAD_DN;
 }
 
-
 /*
   handle a DRSUAPI_EXOP_REPL_OBJ call
  */
@@ -1468,18 +1713,19 @@ static WERROR getncchanges_collect_objects(struct drsuapi_bind_state *b_state,
        int ret;
        char* search_filter;
        enum ldb_scope scope = LDB_SCOPE_SUBTREE;
-       //const char *extra_filter;
        struct drsuapi_getncchanges_state *getnc_state = b_state->getncchanges_state;
+       bool critical_only = false;
+
+       if (req10->replica_flags & DRSUAPI_DRS_CRITICAL_ONLY) {
+               critical_only = true;
+       }
 
        if (req10->extended_op == DRSUAPI_EXOP_REPL_OBJ ||
            req10->extended_op == DRSUAPI_EXOP_REPL_SECRET) {
                scope = LDB_SCOPE_BASE;
+               critical_only = false;
        }
 
-       //extra_filter = lpcfg_parm_string(dce_call->conn->dce_ctx->lp_ctx, NULL, "drs", "object filter");
-
-       //getnc_state->min_usn = req10->highwatermark.highest_usn;
-
        /* Construct response. */
        search_filter = talloc_asprintf(mem_ctx,
                                        "(uSNChanged>=%llu)",
@@ -1489,7 +1735,7 @@ static WERROR getncchanges_collect_objects(struct drsuapi_bind_state *b_state,
                search_filter = talloc_asprintf(mem_ctx, "(&%s(%s))", search_filter, extra_filter);
        }
 
-       if (req10->replica_flags & DRSUAPI_DRS_CRITICAL_ONLY) {
+       if (critical_only) {
                search_filter = talloc_asprintf(mem_ctx,
                                                "(&%s(isCriticalSystemObject=TRUE))",
                                                search_filter);
@@ -1684,7 +1930,424 @@ static WERROR getncchanges_collect_objects_exop(struct drsuapi_bind_state *b_sta
        }
 }
 
-/* 
+static void dcesrv_drsuapi_update_highwatermark(const struct ldb_message *msg,
+                                               uint64_t max_usn,
+                                               struct drsuapi_DsReplicaHighWaterMark *hwm)
+{
+       uint64_t uSN = ldb_msg_find_attr_as_uint64(msg, "uSNChanged", 0);
+
+       if (uSN > max_usn) {
+               /*
+                * Only report the max_usn we had at the start
+                * of the replication cycle.
+                *
+                * If this object has changed lately we better
+                * let the destination dsa refetch the change.
+                * This is better than the risk of loosing some
+                * objects or linked attributes.
+                */
+               return;
+       }
+
+       if (uSN <= hwm->tmp_highest_usn) {
+               return;
+       }
+
+       hwm->tmp_highest_usn = uSN;
+       hwm->reserved_usn = 0;
+}
+
+/**
+ * Adds an object's GUID to the cache of objects already sent.
+ * This avoids us sending the same object multiple times when
+ * the GetNCChanges request uses a flag like GET_ANC.
+ */
+static WERROR dcesrv_drsuapi_obj_cache_add(struct db_context *obj_cache,
+                                          const struct GUID *guid)
+{
+       enum ndr_err_code ndr_err;
+       uint8_t guid_buf[DRS_GUID_SIZE] = { 0, };
+       DATA_BLOB b = {
+               .data = guid_buf,
+               .length = sizeof(guid_buf),
+       };
+       TDB_DATA key = {
+               .dptr = b.data,
+               .dsize = b.length,
+       };
+       TDB_DATA val = {
+               .dptr = NULL,
+               .dsize = 0,
+       };
+       NTSTATUS status;
+
+       ndr_err = ndr_push_struct_into_fixed_blob(&b, guid,
+                       (ndr_push_flags_fn_t)ndr_push_GUID);
+       if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+               return WERR_DS_DRA_INTERNAL_ERROR;
+       }
+
+       status = dbwrap_store(obj_cache, key, val, TDB_REPLACE);
+       if (!NT_STATUS_IS_OK(status)) {
+               return WERR_DS_DRA_INTERNAL_ERROR;
+       }
+
+       return WERR_OK;
+}
+
+/**
+ * Checks if the object with the GUID specified already exists in the
+ * object cache, i.e. it's already been sent in a GetNCChanges response.
+ */
+static WERROR dcesrv_drsuapi_obj_cache_exists(struct db_context *obj_cache,
+                                             const struct GUID *guid)
+{
+       enum ndr_err_code ndr_err;
+       uint8_t guid_buf[DRS_GUID_SIZE] = { 0, };
+       DATA_BLOB b = {
+               .data = guid_buf,
+               .length = sizeof(guid_buf),
+       };
+       TDB_DATA key = {
+               .dptr = b.data,
+               .dsize = b.length,
+       };
+       bool exists;
+
+       ndr_err = ndr_push_struct_into_fixed_blob(&b, guid,
+                       (ndr_push_flags_fn_t)ndr_push_GUID);
+       if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+               return WERR_DS_DRA_INTERNAL_ERROR;
+       }
+
+       exists = dbwrap_exists(obj_cache, key);
+       if (!exists) {
+               return WERR_OBJECT_NOT_FOUND;
+       }
+
+       return WERR_OBJECT_NAME_EXISTS;
+}
+
+/**
+ * Copies the la_list specified into a sorted array, ready to be sent in a
+ * GetNCChanges response.
+ */
+static WERROR getncchanges_get_sorted_array(const struct drsuapi_DsReplicaLinkedAttribute *la_list,
+                                           const uint32_t link_count,
+                                           struct ldb_context *sam_ctx,
+                                           TALLOC_CTX *mem_ctx,
+                                           const struct dsdb_schema *schema,
+                                           struct la_for_sorting **ret_array)
+{
+       int j;
+       struct la_for_sorting *guid_array;
+       WERROR werr = WERR_OK;
+
+       *ret_array = NULL;
+       guid_array = talloc_array(mem_ctx, struct la_for_sorting, link_count);
+       if (guid_array == NULL) {
+               DEBUG(0, ("Out of memory allocating %u linked attributes for sorting", link_count));
+               return WERR_NOT_ENOUGH_MEMORY;
+       }
+
+       for (j = 0; j < link_count; j++) {
+
+               /* we need to get the target GUIDs to compare */
+               struct dsdb_dn *dn;
+               const struct drsuapi_DsReplicaLinkedAttribute *la = &la_list[j];
+               const struct dsdb_attribute *schema_attrib;
+               const struct ldb_val *target_guid;
+               DATA_BLOB source_guid;
+               TALLOC_CTX *frame = talloc_stackframe();
+               NTSTATUS status;
+
+               schema_attrib = dsdb_attribute_by_attributeID_id(schema, la->attid);
+
+               werr = dsdb_dn_la_from_blob(sam_ctx, schema_attrib, schema, frame, la->value.blob, &dn);
+               if (!W_ERROR_IS_OK(werr)) {
+                       DEBUG(0,(__location__ ": Bad la blob in sort\n"));
+                       TALLOC_FREE(frame);
+                       return werr;
+               }
+
+               /* Extract the target GUID in NDR form */
+               target_guid = ldb_dn_get_extended_component(dn->dn, "GUID");
+               if (target_guid == NULL
+                               || target_guid->length != sizeof(guid_array[0].target_guid)) {
+                       status = NT_STATUS_OBJECT_NAME_NOT_FOUND;
+               } else {
+                       /* Repack the source GUID as NDR for sorting */
+                       status = GUID_to_ndr_blob(&la->identifier->guid,
+                                                 frame,
+                                                 &source_guid);
+               }
+
+               if (!NT_STATUS_IS_OK(status)
+                               || source_guid.length != sizeof(guid_array[0].source_guid)) {
+                       DEBUG(0,(__location__ ": Bad la guid in sort\n"));
+                       TALLOC_FREE(frame);
+                       return ntstatus_to_werror(status);
+               }
+
+               guid_array[j].link = &la_list[j];
+               memcpy(guid_array[j].target_guid, target_guid->data,
+                      sizeof(guid_array[j].target_guid));
+               memcpy(guid_array[j].source_guid, source_guid.data,
+                      sizeof(guid_array[j].source_guid));
+               TALLOC_FREE(frame);
+       }
+
+       LDB_TYPESAFE_QSORT(guid_array, link_count, NULL, linked_attribute_compare);
+
+       *ret_array = guid_array;
+
+       return werr;
+}
+
+
+/**
+ * Adds any ancestor/parent objects of the child_obj specified.
+ * This is needed when the GET_ANC flag is specified in the request.
+ * @param new_objs if parents are added, this gets updated to point to a chain
+ * of parent objects (with the parents first and the child last)
+ */
+static WERROR getncchanges_add_ancestors(struct drsuapi_DsReplicaObjectListItemEx *child_obj,
+                                        struct ldb_dn *child_dn,
+                                        TALLOC_CTX *mem_ctx,
+                                        struct ldb_context *sam_ctx,
+                                        struct drsuapi_getncchanges_state *getnc_state,
+                                        struct dsdb_schema *schema,
+                                        DATA_BLOB *session_key,
+                                        struct drsuapi_DsGetNCChangesRequest10 *req10,
+                                        uint32_t *local_pas,
+                                        struct ldb_dn *machine_dn,
+                                        struct drsuapi_DsReplicaObjectListItemEx **new_objs)
+{
+       int ret;
+       const struct GUID *next_anc_guid = NULL;
+       WERROR werr = WERR_OK;
+       static const char * const msg_attrs[] = {
+                                           "*",
+                                           "nTSecurityDescriptor",
+                                           "parentGUID",
+                                           "replPropertyMetaData",
+                                           DSDB_SECRET_ATTRIBUTES,
+                                           NULL };
+
+       next_anc_guid = child_obj->parent_object_guid;
+
+       while (next_anc_guid != NULL) {
+               struct drsuapi_DsReplicaObjectListItemEx *anc_obj = NULL;
+               struct ldb_message *anc_msg = NULL;
+               struct ldb_result *anc_res = NULL;
+               struct ldb_dn *anc_dn = NULL;
+
+               /*
+                * Don't send an object twice. (If we've sent the object, then
+                * we've also sent all its parents as well)
+                */
+               werr = dcesrv_drsuapi_obj_cache_exists(getnc_state->obj_cache,
+                                                      next_anc_guid);
+               if (W_ERROR_EQUAL(werr, WERR_OBJECT_NAME_EXISTS)) {
+                       return WERR_OK;
+               }
+               if (W_ERROR_IS_OK(werr)) {
+                       return WERR_INTERNAL_ERROR;
+               }
+               if (!W_ERROR_EQUAL(werr, WERR_OBJECT_NOT_FOUND)) {
+                       return werr;
+               }
+
+               anc_obj = talloc_zero(mem_ctx,
+                                     struct drsuapi_DsReplicaObjectListItemEx);
+               if (anc_obj == NULL) {
+                       return WERR_NOT_ENOUGH_MEMORY;
+               }
+
+               anc_dn = ldb_dn_new_fmt(anc_obj, sam_ctx, "<GUID=%s>",
+                                       GUID_string(anc_obj, next_anc_guid));
+               if (anc_dn == NULL) {
+                       return WERR_NOT_ENOUGH_MEMORY;
+               }
+
+               ret = drsuapi_search_with_extended_dn(sam_ctx, anc_obj,
+                                                     &anc_res, anc_dn,
+                                                     LDB_SCOPE_BASE,
+                                                     msg_attrs, NULL);
+               if (ret != LDB_SUCCESS) {
+                       const char *anc_str = NULL;
+                       const char *obj_str = NULL;
+
+                       anc_str = ldb_dn_get_extended_linearized(anc_obj,
+                                                                anc_dn,
+                                                                1);
+                       obj_str = ldb_dn_get_extended_linearized(anc_obj,
+                                                                child_dn,
+                                                                1);
+
+                       DBG_ERR("getncchanges: failed to fetch ANC "
+                               "DN %s for DN %s - %s\n",
+                               anc_str, obj_str, ldb_errstring(sam_ctx));
+                       return WERR_DS_DRA_INCONSISTENT_DIT;
+               }
+
+               anc_msg = anc_res->msgs[0];
+
+               werr = get_nc_changes_build_object(anc_obj, anc_msg,
+                                                  sam_ctx,
+                                                  getnc_state,
+                                                  schema, session_key,
+                                                  req10,
+                                                  false, /* force_object_return */
+                                                  local_pas,
+                                                  machine_dn,
+                                                  next_anc_guid);
+               if (!W_ERROR_IS_OK(werr)) {
+                       return werr;
+               }
+
+               /*
+                * Regardless of whether we actually use it or not,
+                * we add it to the cache so we don't look at it again
+                */
+               werr = dcesrv_drsuapi_obj_cache_add(getnc_state->obj_cache,
+                                                   next_anc_guid);
+               if (!W_ERROR_IS_OK(werr)) {
+                       return werr;
+               }
+
+               /*
+                * Any ancestors which are below the highwatermark
+                * or uptodateness_vector shouldn't be added,
+                * but we still look further up the
+                * tree for ones which have been changed recently.
+                */
+               if (anc_obj->meta_data_ctr != NULL) {
+
+                       /*
+                        * prepend the parent to the list so that the client-side
+                        * adds the parent object before it adds the children
+                        */
+                       anc_obj->next_object = *new_objs;
+                       *new_objs = anc_obj;
+               }
+
+               anc_msg = NULL;
+               TALLOC_FREE(anc_res);
+               TALLOC_FREE(anc_dn);
+
+               /*
+                * We may need to resolve more parents...
+                */
+               next_anc_guid = anc_obj->parent_object_guid;
+       }
+       return werr;
+}
+
+/**
+ * Adds a list of new objects into the getNCChanges response message
+ */
+static void getncchanges_add_objs_to_resp(struct getncchanges_repl_chunk *repl_chunk,
+                                         struct drsuapi_DsReplicaObjectListItemEx *obj_list)
+{
+       struct drsuapi_DsReplicaObjectListItemEx *obj;
+
+       /*
+        * We track the last object added to the response message, so just add
+        * the new object-list onto the end
+        */
+       if (repl_chunk->last_object == NULL) {
+               repl_chunk->ctr6->first_object = obj_list;
+       } else {
+               repl_chunk->last_object->next_object = obj_list;
+       }
+
+       for (obj = obj_list; obj != NULL; obj = obj->next_object) {
+               repl_chunk->ctr6->object_count += 1;
+
+               /*
+                * Remember the last object in the response - we'll use this to
+                * link the next object(s) processed onto the existing list
+                */
+               if (obj->next_object == NULL) {
+                       repl_chunk->last_object = obj;
+               }
+       }
+}
+
+/**
+ * Gets the object to send, packed into an RPC struct ready to send. This also
+ * adds the object to the object cache, and adds any ancestors (if needed).
+ * @param msg - DB search result for the object to add
+ * @param guid - GUID of the object to add
+ * @param ret_obj_list - returns the object ready to be sent (in a list, along
+ * with any ancestors that might be needed). NULL if nothing to send.
+ */
+static WERROR getncchanges_get_obj_to_send(const struct ldb_message *msg,
+                                          TALLOC_CTX *mem_ctx,
+                                          struct ldb_context *sam_ctx,
+                                          struct drsuapi_getncchanges_state *getnc_state,
+                                          struct dsdb_schema *schema,
+                                          DATA_BLOB *session_key,
+                                          struct drsuapi_DsGetNCChangesRequest10 *req10,
+                                          bool force_object_return,
+                                          uint32_t *local_pas,
+                                          struct ldb_dn *machine_dn,
+                                          const struct GUID *guid,
+                                          struct drsuapi_DsReplicaObjectListItemEx **ret_obj_list)
+{
+       struct drsuapi_DsReplicaObjectListItemEx *obj;
+       WERROR werr;
+
+       *ret_obj_list = NULL;
+
+       obj = talloc_zero(mem_ctx, struct drsuapi_DsReplicaObjectListItemEx);
+       W_ERROR_HAVE_NO_MEMORY(obj);
+
+       werr = get_nc_changes_build_object(obj, msg, sam_ctx, getnc_state,
+                                          schema, session_key, req10,
+                                          force_object_return,
+                                          local_pas, machine_dn, guid);
+       if (!W_ERROR_IS_OK(werr)) {
+               return werr;
+       }
+
+       /*
+        * The object may get filtered out by the UTDV's USN and not actually
+        * sent, in which case there's nothing more to do here
+        */
+       if (obj->meta_data_ctr == NULL) {
+               TALLOC_FREE(obj);
+               return WERR_OK;
+       }
+
+       if (getnc_state->obj_cache != NULL) {
+               werr = dcesrv_drsuapi_obj_cache_add(getnc_state->obj_cache,
+                                                   guid);
+               if (!W_ERROR_IS_OK(werr)) {
+                       return werr;
+               }
+       }
+
+       *ret_obj_list = obj;
+
+       /*
+        * If required, also add any ancestors that the client may need to know
+        * about before it can resolve this object. These get prepended to the
+        * ret_obj_list so the client adds them first.
+        */
+       if (getnc_state->is_get_anc) {
+               werr = getncchanges_add_ancestors(obj, msg->dn, mem_ctx,
+                                                 sam_ctx, getnc_state,
+                                                 schema, session_key,
+                                                 req10, local_pas,
+                                                 machine_dn, ret_obj_list);
+       }
+
+       return werr;
+}
+
+/*
   drsuapi_DsGetNCChanges
 
   see MS-DRSR 4.1.10.5.2 for basic logic of this function
@@ -1697,7 +2360,7 @@ WERROR dcesrv_drsuapi_DsGetNCChanges(struct dcesrv_call_state *dce_call, TALLOC_
        uint32_t i, k;
        struct dsdb_schema *schema;
        struct drsuapi_DsReplicaOIDMapping_Ctr *ctr;
-       struct drsuapi_DsReplicaObjectListItemEx **currentObject;
+       struct getncchanges_repl_chunk repl_chunk = { 0 };
        NTSTATUS status;
        DATA_BLOB session_key;
        WERROR werr;
@@ -1709,10 +2372,8 @@ WERROR dcesrv_drsuapi_DsGetNCChanges(struct dcesrv_call_state *dce_call, TALLOC_
        uint32_t max_objects;
        uint32_t max_links;
        uint32_t link_count = 0;
-       uint32_t link_total = 0;
-       uint32_t link_given = 0;
        struct ldb_dn *search_dn = NULL;
-       bool am_rodc, null_scope=false;
+       bool am_rodc;
        enum security_user_level security_level;
        struct ldb_context *sam_ctx;
        struct dom_sid *user_sid;
@@ -1728,16 +2389,19 @@ WERROR dcesrv_drsuapi_DsGetNCChanges(struct dcesrv_call_state *dce_call, TALLOC_
        struct dsdb_schema_prefixmap *pfm_remote = NULL;
        bool full = true;
        uint32_t *local_pas = NULL;
+       struct ldb_dn *machine_dn = NULL; /* Only used for REPL SECRET EXOP */
+       bool immediate_link_sync;
 
        DCESRV_PULL_HANDLE_WERR(h, r->in.bind_handle, DRSUAPI_BIND_HANDLE);
        b_state = h->data;
 
+       /* sam_ctx_system is not present for non-administrator users */
        sam_ctx = b_state->sam_ctx_system?b_state->sam_ctx_system:b_state->sam_ctx;
 
        invocation_id = *(samdb_ntds_invocation_id(sam_ctx));
 
        *r->out.level_out = 6;
-       /* TODO: linked attributes*/
+
        r->out.ctr->ctr6.linked_attributes_count = 0;
        r->out.ctr->ctr6.linked_attributes = discard_const_p(struct drsuapi_DsReplicaLinkedAttribute, &no_linked_attr);
 
@@ -1796,7 +2460,7 @@ WERROR dcesrv_drsuapi_DsGetNCChanges(struct dcesrv_call_state *dce_call, TALLOC_
        user_sid = &dce_call->conn->auth_state.session_info->security_token->sids[PRIMARY_USER_SID_INDEX];
 
        /* all clients must have GUID_DRS_GET_CHANGES */
-       werr = drs_security_access_check_nc_root(b_state->sam_ctx,
+       werr = drs_security_access_check_nc_root(sam_ctx,
                                                 mem_ctx,
                                                 dce_call->conn->auth_state.session_info->security_token,
                                                 req10->naming_context,
@@ -1838,7 +2502,7 @@ WERROR dcesrv_drsuapi_DsGetNCChanges(struct dcesrv_call_state *dce_call, TALLOC_
                return werr;
        }
        if (is_gc_pas_request) {
-               werr = drs_security_access_check_nc_root(b_state->sam_ctx,
+               werr = drs_security_access_check_nc_root(sam_ctx,
                                                         mem_ctx,
                                                         dce_call->conn->auth_state.session_info->security_token,
                                                         req10->naming_context,
@@ -1854,14 +2518,17 @@ WERROR dcesrv_drsuapi_DsGetNCChanges(struct dcesrv_call_state *dce_call, TALLOC_
        if (!W_ERROR_IS_OK(werr)) {
                return werr;
        }
-       if (is_secret_request && req10->extended_op != DRSUAPI_EXOP_REPL_SECRET) {
-               werr = drs_security_access_check_nc_root(b_state->sam_ctx,
+       if (is_secret_request) {
+               werr = drs_security_access_check_nc_root(sam_ctx,
                                                         mem_ctx,
                                                         dce_call->conn->auth_state.session_info->security_token,
                                                         req10->naming_context,
                                                         GUID_DRS_GET_ALL_CHANGES);
                if (!W_ERROR_IS_OK(werr)) {
-                       return werr;
+                       /* Only bail if this is not a EXOP_REPL_SECRET */
+                       if (req10->extended_op != DRSUAPI_EXOP_REPL_SECRET) {
+                               return werr;
+                       }
                } else {
                        has_get_all_changes = true;
                }
@@ -1906,8 +2573,8 @@ allowed:
                                 ldb_dn_get_linearized(new_dn),
                                 ldb_dn_get_linearized(getnc_state->ncRoot_dn),
                                 ldb_dn_get_linearized(getnc_state->last_dn)));
-                       talloc_free(getnc_state);
-                       getnc_state = NULL;
+                       TALLOC_FREE(getnc_state);
+                       b_state->getncchanges_state = NULL;
                }
        }
 
@@ -1920,24 +2587,38 @@ allowed:
                                 ldb_dn_get_linearized(getnc_state->ncRoot_dn),
                                 (ret > 0) ? "older" : "newer",
                                 ldb_dn_get_linearized(getnc_state->last_dn)));
-                       talloc_free(getnc_state);
-                       getnc_state = NULL;
+                       TALLOC_FREE(getnc_state);
+                       b_state->getncchanges_state = NULL;
                }
        }
 
        if (getnc_state == NULL) {
-               getnc_state = talloc_zero(b_state, struct drsuapi_getncchanges_state);
-               if (getnc_state == NULL) {
+               struct ldb_result *res = NULL;
+               const char *attrs[] = {
+                       "instanceType",
+                       "objectGuID",
+                       NULL
+               };
+               uint32_t nc_instanceType;
+               struct ldb_dn *ncRoot_dn;
+
+               ncRoot_dn = drs_ObjectIdentifier_to_dn(mem_ctx, sam_ctx, ncRoot);
+               if (ncRoot_dn == NULL) {
                        return WERR_NOT_ENOUGH_MEMORY;
                }
-               b_state->getncchanges_state = getnc_state;
-               getnc_state->ncRoot_dn = drs_ObjectIdentifier_to_dn(getnc_state, sam_ctx, ncRoot);
 
-               /* find out if we are to replicate Schema NC */
-               ret = ldb_dn_compare_base(ldb_get_schema_basedn(b_state->sam_ctx),
-                                         getnc_state->ncRoot_dn);
-
-               getnc_state->is_schema_nc = (0 == ret);
+               ret = dsdb_search_dn(sam_ctx, mem_ctx, &res,
+                                    ncRoot_dn, attrs,
+                                    DSDB_SEARCH_SHOW_DELETED |
+                                    DSDB_SEARCH_SHOW_RECYCLED);
+               if (ret != LDB_SUCCESS) {
+                       DBG_WARNING("Failed to find ncRoot_dn %s\n",
+                                   ldb_dn_get_linearized(ncRoot_dn));
+                       return WERR_DS_DRA_BAD_DN;
+               }
+               nc_instanceType = ldb_msg_find_attr_as_int(res->msgs[0],
+                                                          "instanceType",
+                                                          0);
 
                if (req10->extended_op != DRSUAPI_EXOP_NONE) {
                        r->out.ctr->ctr6.extended_ret = DRSUAPI_EXOP_ERR_SUCCESS;
@@ -1951,6 +2632,16 @@ allowed:
                 */
                switch (req10->extended_op) {
                case DRSUAPI_EXOP_NONE:
+                       if ((nc_instanceType & INSTANCE_TYPE_IS_NC_HEAD) == 0) {
+                               const char *dn_str
+                                       = ldb_dn_get_linearized(ncRoot_dn);
+
+                               DBG_NOTICE("Rejecting full replication on "
+                                          "not NC %s", dn_str);
+
+                               return WERR_DS_CANT_FIND_EXPECTED_NC;
+                       }
+
                        break;
                case DRSUAPI_EXOP_FSMO_RID_ALLOC:
                        werr = getncchanges_rid_alloc(b_state, mem_ctx, req10, &r->out.ctr->ctr6, &search_dn);
@@ -1963,7 +2654,8 @@ allowed:
                        werr = getncchanges_repl_secret(b_state, mem_ctx, req10,
                                                        user_sid,
                                                        &r->out.ctr->ctr6,
-                                                       has_get_all_changes);
+                                                       has_get_all_changes,
+                                                       &machine_dn);
                        r->out.result = werr;
                        W_ERROR_NOT_OK_RETURN(werr);
                        break;
@@ -2000,6 +2692,27 @@ allowed:
                                 (unsigned)req10->extended_op));
                        return WERR_DS_DRA_NOT_SUPPORTED;
                }
+
+               /* Initialize the state we'll store over the replication cycle */
+               getnc_state = talloc_zero(b_state, struct drsuapi_getncchanges_state);
+               if (getnc_state == NULL) {
+                       return WERR_NOT_ENOUGH_MEMORY;
+               }
+               b_state->getncchanges_state = getnc_state;
+
+               getnc_state->ncRoot_dn = ncRoot_dn;
+               talloc_steal(getnc_state, ncRoot_dn);
+
+               getnc_state->ncRoot_guid = samdb_result_guid(res->msgs[0],
+                                                            "objectGUID");
+               ncRoot->guid = getnc_state->ncRoot_guid;
+
+               /* find out if we are to replicate Schema NC */
+               ret = ldb_dn_compare_base(ldb_get_schema_basedn(sam_ctx),
+                                         ncRoot_dn);
+               getnc_state->is_schema_nc = (0 == ret);
+
+               TALLOC_FREE(res);
        }
 
        if (!ldb_dn_validate(getnc_state->ncRoot_dn) ||
@@ -2009,6 +2722,8 @@ allowed:
                return WERR_DS_DRA_INVALID_PARAMETER;
        }
 
+       ncRoot->guid = getnc_state->ncRoot_guid;
+
        /* we need the session key for encrypting password attributes */
        status = dcesrv_inherited_session_key(dce_call->conn, &session_key);
        if (!NT_STATUS_IS_OK(status)) {
@@ -2023,10 +2738,40 @@ allowed:
        if (getnc_state->guids == NULL) {
                const char *extra_filter;
                struct ldb_result *search_res = NULL;
+               static const struct drsuapi_DsReplicaCursorCtrEx empty_udv;
+               const struct drsuapi_DsReplicaCursorCtrEx *udv = NULL;
 
                extra_filter = lpcfg_parm_string(dce_call->conn->dce_ctx->lp_ctx, NULL, "drs", "object filter");
 
-               getnc_state->min_usn = req10->highwatermark.highest_usn;
+               if (req10->extended_op == DRSUAPI_EXOP_NONE) {
+                       if (req10->uptodateness_vector != NULL) {
+                               udv = req10->uptodateness_vector;
+                       } else {
+                               udv = &empty_udv;
+                       }
+
+                       getnc_state->min_usn = req10->highwatermark.highest_usn;
+                       for (i = 0; i < udv->count; i++) {
+                               bool match;
+                               const struct drsuapi_DsReplicaCursor *cur =
+                                       &udv->cursors[i];
+
+                               match = GUID_equal(&invocation_id,
+                                                  &cur->source_dsa_invocation_id);
+                               if (!match) {
+                                       continue;
+                               }
+                               if (cur->highest_usn > getnc_state->min_usn) {
+                                       getnc_state->min_usn = cur->highest_usn;
+                               }
+                               break;
+                       }
+               } else {
+                       /* We do not want REPL_SECRETS or REPL_SINGLE to return empty-handed */
+                       udv = &empty_udv;
+                       getnc_state->min_usn = 0;
+               }
+
                getnc_state->max_usn = getnc_state->min_usn;
 
                getnc_state->final_udv = talloc_zero(getnc_state,
@@ -2075,11 +2820,6 @@ allowed:
                /* RID_ALLOC returns 3 objects in a fixed order */
                if (req10->extended_op == DRSUAPI_EXOP_FSMO_RID_ALLOC) {
                        /* Do nothing */
-               } else if (req10->replica_flags & DRSUAPI_DRS_GET_ANC) {
-                       LDB_TYPESAFE_QSORT(changes,
-                                          getnc_state->num_records,
-                                          getnc_state,
-                                          site_res_cmp_anc_order);
                } else {
                        LDB_TYPESAFE_QSORT(changes,
                                           getnc_state->num_records,
@@ -2102,6 +2842,16 @@ allowed:
 
                talloc_free(search_res);
                talloc_free(changes);
+
+               if (req10->extended_op != DRSUAPI_EXOP_NONE) {
+                       /* Do nothing */
+               } else if (req10->replica_flags & DRSUAPI_DRS_GET_ANC) {
+                       getnc_state->obj_cache = db_open_rbt(getnc_state);
+                       if (getnc_state->obj_cache == NULL) {
+                               return WERR_NOT_ENOUGH_MEMORY;
+                       }
+                       getnc_state->is_get_anc = true;
+               }
        }
 
        if (req10->uptodateness_vector) {
@@ -2119,14 +2869,10 @@ allowed:
        }
 
        r->out.ctr->ctr6.naming_context = talloc(mem_ctx, struct drsuapi_DsReplicaObjectIdentifier);
-       *r->out.ctr->ctr6.naming_context = *ncRoot;
-
-       if (dsdb_find_guid_by_dn(sam_ctx, getnc_state->ncRoot_dn,
-                                &r->out.ctr->ctr6.naming_context->guid) != LDB_SUCCESS) {
-               DEBUG(0,(__location__ ": Failed to find GUID of ncRoot_dn %s\n",
-                        ldb_dn_get_linearized(getnc_state->ncRoot_dn)));
-               return WERR_DS_DRA_INTERNAL_ERROR;
+       if (r->out.ctr->ctr6.naming_context == NULL) {
+               return WERR_NOT_ENOUGH_MEMORY;
        }
+       *r->out.ctr->ctr6.naming_context = *ncRoot;
 
        /* find the SID if there is one */
        dsdb_find_sid_by_dn(sam_ctx, getnc_state->ncRoot_dn, &r->out.ctr->ctr6.naming_context->sid);
@@ -2140,7 +2886,8 @@ allowed:
        r->out.ctr->ctr6.old_highwatermark = req10->highwatermark;
        r->out.ctr->ctr6.new_highwatermark = req10->highwatermark;
 
-       currentObject = &r->out.ctr->ctr6.first_object;
+       repl_chunk.ctr6 = &r->out.ctr->ctr6;
+       repl_chunk.last_object = NULL;
 
        max_objects = lpcfg_parm_int(dce_call->conn->dce_ctx->lp_ctx, NULL, "drs", "max object sync", 1000);
        /*
@@ -2161,6 +2908,9 @@ allowed:
         */
        max_links = lpcfg_parm_int(dce_call->conn->dce_ctx->lp_ctx, NULL, "drs", "max link sync", 1500);
 
+       immediate_link_sync = lpcfg_parm_bool(dce_call->conn->dce_ctx->lp_ctx, NULL,
+                                             "drs", "immediate link sync", false);
+
        /*
         * Maximum time that we can spend in a getncchanges
         * in order to avoid timeout of the other part.
@@ -2172,7 +2922,7 @@ allowed:
                struct dsdb_syntax_ctx syntax_ctx;
                uint32_t j = 0;
 
-               dsdb_syntax_ctx_init(&syntax_ctx, b_state->sam_ctx, schema);
+               dsdb_syntax_ctx_init(&syntax_ctx, sam_ctx, schema);
                syntax_ctx.pfm_remote = pfm_remote;
 
                local_pas = talloc_array(b_state, uint32_t, req10->partial_attribute_set->num_attids);
@@ -2193,12 +2943,10 @@ allowed:
 
        for (i=getnc_state->num_processed;
             i<getnc_state->num_records &&
-                    !null_scope &&
                     (r->out.ctr->ctr6.object_count < max_objects)
                     && !max_wait_reached;
            i++) {
-               int uSN;
-               struct drsuapi_DsReplicaObjectListItemEx *obj;
+               struct drsuapi_DsReplicaObjectListItemEx *new_objs = NULL;
                struct ldb_message *msg;
                static const char * const msg_attrs[] = {
                                            "*",
@@ -2209,49 +2957,86 @@ allowed:
                                            NULL };
                struct ldb_result *msg_res;
                struct ldb_dn *msg_dn;
+               bool obj_already_sent = false;
+               TALLOC_CTX *tmp_ctx = talloc_new(mem_ctx);
+               uint32_t old_la_index;
 
-               obj = talloc_zero(mem_ctx, struct drsuapi_DsReplicaObjectListItemEx);
-               W_ERROR_HAVE_NO_MEMORY(obj);
-
-               msg_dn = ldb_dn_new_fmt(obj, sam_ctx, "<GUID=%s>", GUID_string(obj, &getnc_state->guids[i]));
+               msg_dn = ldb_dn_new_fmt(tmp_ctx, sam_ctx, "<GUID=%s>",
+                                       GUID_string(tmp_ctx, &getnc_state->guids[i]));
                W_ERROR_HAVE_NO_MEMORY(msg_dn);
 
-
-               /* by re-searching here we avoid having a lot of full
-                * records in memory between calls to getncchanges
+               /*
+                * by re-searching here we avoid having a lot of full
+                * records in memory between calls to getncchanges.
+                *
+                * We expect that we may get some objects that vanish
+                * (tombstone expunge) between the first and second
+                * check.
                 */
-               ret = drsuapi_search_with_extended_dn(sam_ctx, obj, &msg_res,
+               ret = drsuapi_search_with_extended_dn(sam_ctx, tmp_ctx, &msg_res,
                                                      msg_dn,
                                                      LDB_SCOPE_BASE, msg_attrs, NULL);
                if (ret != LDB_SUCCESS) {
                        if (ret != LDB_ERR_NO_SUCH_OBJECT) {
                                DEBUG(1,("getncchanges: failed to fetch DN %s - %s\n",
-                                        ldb_dn_get_extended_linearized(obj, msg_dn, 1), ldb_errstring(sam_ctx)));
+                                        ldb_dn_get_extended_linearized(tmp_ctx, msg_dn, 1),
+                                        ldb_errstring(sam_ctx)));
                        }
-                       talloc_free(obj);
+                       TALLOC_FREE(tmp_ctx);
+                       continue;
+               }
+
+               if (msg_res->count == 0) {
+                       DEBUG(1,("getncchanges: got LDB_SUCCESS but failed"
+                                "to get any results in fetch of DN "
+                                "%s (race with tombstone expunge?)\n",
+                                ldb_dn_get_extended_linearized(tmp_ctx,
+                                                               msg_dn, 1)));
+                       TALLOC_FREE(tmp_ctx);
                        continue;
                }
 
                msg = msg_res->msgs[0];
 
-               max_wait_reached = (time(NULL) - start > max_wait);
-
-               werr = get_nc_changes_build_object(obj, msg,
-                                                  sam_ctx, getnc_state->ncRoot_dn,
-                                                  getnc_state->is_schema_nc,
-                                                  schema, &session_key, getnc_state->min_usn,
-                                                  req10->replica_flags,
-                                                  req10->partial_attribute_set,
-                                                  req10->uptodateness_vector,
-                                                  req10->extended_op,
-                                                  max_wait_reached,
-                                                  local_pas);
-               if (!W_ERROR_IS_OK(werr)) {
-                       return werr;
+               /*
+                * Check if we've already sent the object as an ancestor of
+                * another object. If so, we don't need to send it again
+                */
+               if (getnc_state->obj_cache != NULL) {
+                       werr = dcesrv_drsuapi_obj_cache_exists(getnc_state->obj_cache,
+                                                              &getnc_state->guids[i]);
+                       if (W_ERROR_EQUAL(werr, WERR_OBJECT_NAME_EXISTS)) {
+                               obj_already_sent = true;
+                       }
                }
 
+               if (!obj_already_sent) {
+                       max_wait_reached = (time(NULL) - start > max_wait);
+
+                       /*
+                        * Construct an object, ready to send (this will include
+                        * the object's ancestors as well, if needed)
+                        */
+                       werr = getncchanges_get_obj_to_send(msg, mem_ctx, sam_ctx,
+                                                           getnc_state, schema,
+                                                           &session_key, req10,
+                                                           max_wait_reached,
+                                                           local_pas, machine_dn,
+                                                           &getnc_state->guids[i],
+                                                           &new_objs);
+                       if (!W_ERROR_IS_OK(werr)) {
+                               return werr;
+                       }
+               }
+
+               old_la_index = getnc_state->la_count;
+
+               /*
+                * We've reached the USN where this object naturally occurs.
+                * Regardless of whether we've already sent the object (as an
+                * ancestor), we add its links and update the HWM at this point
+                */
                werr = get_nc_changes_add_links(sam_ctx, getnc_state,
-                                               getnc_state->ncRoot_dn,
                                                getnc_state->is_schema_nc,
                                                schema, getnc_state->min_usn,
                                                req10->replica_flags,
@@ -2263,48 +3048,37 @@ allowed:
                        return werr;
                }
 
-               uSN = ldb_msg_find_attr_as_int(msg, "uSNChanged", -1);
-               if (uSN > getnc_state->max_usn) {
+               dcesrv_drsuapi_update_highwatermark(msg,
+                                       getnc_state->max_usn,
+                                       &r->out.ctr->ctr6.new_highwatermark);
+
+               if (new_objs != NULL) {
+
                        /*
-                        * Only report the max_usn we had at the start
-                        * of the replication cycle.
-                        *
-                        * If this object has changed lately we better
-                        * let the destination dsa refetch the change.
-                        * This is better than the risk of loosing some
-                        * objects or linked attributes.
+                        * Add the object (and, if GET_ANC, any parents it may
+                        * have) into the current chunk of replication data
                         */
-                       uSN = 0;
-               }
-               if (uSN > r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn) {
-                       r->out.ctr->ctr6.new_highwatermark.tmp_highest_usn = uSN;
-                       r->out.ctr->ctr6.new_highwatermark.reserved_usn = 0;
-               }
+                       getncchanges_add_objs_to_resp(&repl_chunk, new_objs);
 
-               if (obj->meta_data_ctr == NULL) {
-                       DEBUG(8,(__location__ ": getncchanges skipping send of object %s\n",
-                                ldb_dn_get_linearized(msg->dn)));
-                       /* no attributes to send */
-                       talloc_free(obj);
-                       continue;
+                       talloc_free(getnc_state->last_dn);
+                       getnc_state->last_dn = talloc_move(getnc_state, &msg->dn);
                }
 
-               r->out.ctr->ctr6.object_count++;
-
-               *currentObject = obj;
-               currentObject = &obj->next_object;
+               DEBUG(8,(__location__ ": %s object %s\n",
+                        new_objs ? "replicating" : "skipping send of",
+                        ldb_dn_get_linearized(msg->dn)));
 
-               DEBUG(8,(__location__ ": replicating object %s\n", ldb_dn_get_linearized(msg->dn)));
+               getnc_state->total_links += (getnc_state->la_count - old_la_index);
 
-               talloc_free(getnc_state->last_dn);
-               getnc_state->last_dn = talloc_move(getnc_state, &msg->dn);
-
-               talloc_free(msg_res);
-               talloc_free(msg_dn);
+               TALLOC_FREE(tmp_ctx);
        }
 
        getnc_state->num_processed = i;
 
+       if (i < getnc_state->num_records) {
+               r->out.ctr->ctr6.more_data = true;
+       }
+
        /* the client can us to call UpdateRefs on its behalf to
           re-establish monitoring of the NC */
        if ((req10->replica_flags & (DRSUAPI_DRS_ADD_REF | DRSUAPI_DRS_REF_GCSPN)) &&
@@ -2313,7 +3087,7 @@ allowed:
                DEBUG(3,("UpdateRefs on getncchanges for %s\n",
                         GUID_string(mem_ctx, &req10->destination_dsa_guid)));
                ureq.naming_context = ncRoot;
-               ureq.dest_dsa_dns_name = samdb_ntds_msdcs_dns_name(b_state->sam_ctx, mem_ctx,
+               ureq.dest_dsa_dns_name = samdb_ntds_msdcs_dns_name(sam_ctx, mem_ctx,
                                                                   &req10->destination_dsa_guid);
                if (!ureq.dest_dsa_dns_name) {
                        return WERR_NOT_ENOUGH_MEMORY;
@@ -2328,7 +3102,9 @@ allowed:
                   to send notifies using the GC SPN */
                ureq.options |= (req10->replica_flags & DRSUAPI_DRS_REF_GCSPN);
 
-               werr = drsuapi_UpdateRefs(b_state, mem_ctx, &ureq);
+               werr = drsuapi_UpdateRefs(dce_call->msg_ctx,
+                                         dce_call->event_ctx, b_state,
+                                         mem_ctx, &ureq);
                if (!W_ERROR_IS_OK(werr)) {
                        DEBUG(0,(__location__ ": Failed UpdateRefs on %s for %s in DsGetNCChanges - %s\n",
                                 drs_ObjectIdentifier_to_string(mem_ctx, ncRoot), ureq.dest_dsa_dns_name,
@@ -2348,71 +3124,33 @@ allowed:
                max_links -= r->out.ctr->ctr6.object_count;
        }
 
-       link_total = getnc_state->la_count;
-
-       if (i < getnc_state->num_records) {
-               r->out.ctr->ctr6.more_data = true;
-       } else {
-               /* sort the whole array the first time */
-               if (getnc_state->la_sorted == NULL) {
-                       int j;
-                       struct la_for_sorting *guid_array = talloc_array(getnc_state, struct la_for_sorting, getnc_state->la_count);
-                       if (guid_array == NULL) {
-                               DEBUG(0, ("Out of memory allocating %u linked attributes for sorting", getnc_state->la_count));
-                               return WERR_NOT_ENOUGH_MEMORY;
-                       }
-                       for (j = 0; j < getnc_state->la_count; j++) {
-                               /* we need to get the target GUIDs to compare */
-                               struct dsdb_dn *dn;
-                               const struct drsuapi_DsReplicaLinkedAttribute *la = &getnc_state->la_list[j];
-                               const struct dsdb_attribute *schema_attrib;
-                               const struct ldb_val *target_guid;
-                               DATA_BLOB source_guid;
-                               TALLOC_CTX *frame = talloc_stackframe();
-
-                               schema_attrib = dsdb_attribute_by_attributeID_id(schema, la->attid);
-
-                               werr = dsdb_dn_la_from_blob(sam_ctx, schema_attrib, schema, frame, la->value.blob, &dn);
-                               if (!W_ERROR_IS_OK(werr)) {
-                                       DEBUG(0,(__location__ ": Bad la blob in sort\n"));
-                                       TALLOC_FREE(frame);
-                                       return werr;
-                               }
-
-                               /* Extract the target GUID in NDR form */
-                               target_guid = ldb_dn_get_extended_component(dn->dn, "GUID");
-                               if (target_guid == NULL
-                                   || target_guid->length != sizeof(guid_array[0].target_guid)) {
-                                       status = NT_STATUS_OBJECT_NAME_NOT_FOUND;
-                               } else {
-                                       /* Repack the source GUID as NDR for sorting */
-                                       status = GUID_to_ndr_blob(&la->identifier->guid,
-                                                                 frame,
-                                                                 &source_guid);
-                               }
-
-                               if (!NT_STATUS_IS_OK(status)
-                                   || source_guid.length != sizeof(guid_array[0].source_guid)) {
-                                       DEBUG(0,(__location__ ": Bad la guid in sort\n"));
-                                       TALLOC_FREE(frame);
-                                       return ntstatus_to_werror(status);
-                               }
+       /*
+        * Work out how many links we can send in this chunk. The default is to
+        * send all the links last, but there is a config option to send them
+        * immediately, in the same chunk as their source object
+        */
+       if (!r->out.ctr->ctr6.more_data || immediate_link_sync) {
+               link_count = getnc_state->la_count - getnc_state->la_idx;
+               link_count = MIN(max_links, link_count);
+       }
 
-                               guid_array[j].link = &getnc_state->la_list[j];
-                               memcpy(guid_array[j].target_guid, target_guid->data,
-                                      sizeof(guid_array[j].target_guid));
-                               memcpy(guid_array[j].source_guid, source_guid.data,
-                                      sizeof(guid_array[j].source_guid));
-                               TALLOC_FREE(frame);
-                       }
+       /* If we've got linked attributes to send, add them now */
+       if (link_count > 0) {
+               struct la_for_sorting *la_sorted;
 
-                       LDB_TYPESAFE_QSORT(guid_array, getnc_state->la_count, NULL, linked_attribute_compare);
-                       getnc_state->la_sorted = guid_array;
+               /*
+                * Grab a chunk of linked attributes off the list and put them
+                * in sorted array, ready to send
+                */
+               werr = getncchanges_get_sorted_array(&getnc_state->la_list[getnc_state->la_idx],
+                                                    link_count,
+                                                    sam_ctx, getnc_state,
+                                                    schema,
+                                                    &la_sorted);
+               if (!W_ERROR_IS_OK(werr)) {
+                       return werr;
                }
 
-               link_count = getnc_state->la_count - getnc_state->la_idx;
-               link_count = MIN(max_links, link_count);
-
                r->out.ctr->ctr6.linked_attributes_count = link_count;
                r->out.ctr->ctr6.linked_attributes = talloc_array(r->out.ctr, struct drsuapi_DsReplicaLinkedAttribute, link_count);
                if (r->out.ctr->ctr6.linked_attributes == NULL) {
@@ -2421,16 +3159,29 @@ allowed:
                }
 
                for (k = 0; k < link_count; k++) {
-                       r->out.ctr->ctr6.linked_attributes[k]
-                               = *getnc_state->la_sorted[getnc_state->la_idx + k].link;
+                       r->out.ctr->ctr6.linked_attributes[k] = *la_sorted[k].link;
                }
 
                getnc_state->la_idx += link_count;
-               link_given = getnc_state->la_idx;
+               getnc_state->links_given += link_count;
 
                if (getnc_state->la_idx < getnc_state->la_count) {
                        r->out.ctr->ctr6.more_data = true;
+               } else {
+
+                       /*
+                        * We've now sent all the links seen so far, so we can
+                        * reset la_list back to an empty list again. Note that
+                        * the steal means the linked attribute memory gets
+                        * freed after this RPC message is sent on the wire.
+                        */
+                       talloc_steal(mem_ctx, getnc_state->la_list);
+                       getnc_state->la_list = NULL;
+                       getnc_state->la_idx = 0;
+                       getnc_state->la_count = 0;
                }
+
+               TALLOC_FREE(la_sorted);
        }
 
        if (req10->replica_flags & DRSUAPI_DRS_GET_NC_SIZE) {
@@ -2446,17 +3197,23 @@ allowed:
                 * of links we found so far during the cycle.
                 */
                r->out.ctr->ctr6.nc_object_count = getnc_state->num_records;
-               r->out.ctr->ctr6.nc_linked_attributes_count = getnc_state->la_count;
+               r->out.ctr->ctr6.nc_linked_attributes_count = getnc_state->total_links;
        }
 
        if (!r->out.ctr->ctr6.more_data) {
-               talloc_steal(mem_ctx, getnc_state->la_list);
 
+               /* this is the last response in the replication cycle */
                r->out.ctr->ctr6.new_highwatermark = getnc_state->final_hwm;
                r->out.ctr->ctr6.uptodateness_vector = talloc_move(mem_ctx,
                                                        &getnc_state->final_udv);
 
-               talloc_free(getnc_state);
+               /*
+                * Free the state info stored for the replication cycle. Note
+                * that the RPC message we're sending contains links stored in
+                * getnc_state. mem_ctx is local to this RPC call, so the memory
+                * will get freed after the RPC message is sent on the wire.
+                */
+               talloc_steal(mem_ctx, getnc_state);
                b_state->getncchanges_state = NULL;
        } else {
                ret = drsuapi_DsReplicaHighWaterMark_cmp(&r->out.ctr->ctr6.old_highwatermark,
@@ -2492,7 +3249,7 @@ allowed:
               r->out.ctr->ctr6.object_count,
               i, r->out.ctr->ctr6.more_data?getnc_state->num_records:i,
               r->out.ctr->ctr6.linked_attributes_count,
-              link_given, link_total,
+              getnc_state->links_given, getnc_state->total_links,
               dom_sid_string(mem_ctx, user_sid)));
 
 #if 0