repl_meta_data: add some TODOs to replmd_modify_originating()
[ira/wip.git] / source / dsdb / samdb / ldb_modules / repl_meta_data.c
index b91d5e3db2f391d89eefa85bf970865a102e9408..441dbc95983a9167079140c2949352ecc552a3ba 100644 (file)
@@ -4,7 +4,7 @@
    Copyright (C) Simo Sorce  2004-2006
    Copyright (C) Andrew Bartlett <abartlet@samba.org> 2005
    Copyright (C) Andrew Tridgell 2005
-   Copyright (C) Stefan Metzmacher 2007
+   Copyright (C) Stefan Metzmacher <metze@samba.org> 2007
 
      ** NOTE! The following LGPL license applies to the ldb
      ** library. This does NOT imply that all of Samba is released
@@ -13,7 +13,7 @@
    This library is free software; you can redistribute it and/or
    modify it under the terms of the GNU Lesser General Public
    License as published by the Free Software Foundation; either
-   version 2 of the License, or (at your option) any later version.
+   version 3 of the License, or (at your option) any later version.
 
    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
@@ -21,8 +21,7 @@
    Lesser General Public License for more details.
 
    You should have received a copy of the GNU Lesser General Public
-   License along with this library; if not, write to the Free Software
-   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
 */
 
 /*
 #include "lib/ldb/include/ldb_errors.h"
 #include "lib/ldb/include/ldb_private.h"
 #include "dsdb/samdb/samdb.h"
+#include "dsdb/common/flags.h"
 #include "librpc/gen_ndr/ndr_misc.h"
 #include "librpc/gen_ndr/ndr_drsuapi.h"
 #include "librpc/gen_ndr/ndr_drsblobs.h"
+#include "param/param.h"
 
 struct replmd_replicated_request {
        struct ldb_module *module;
        struct ldb_handle *handle;
        struct ldb_request *orig_req;
 
+       const struct dsdb_schema *schema;
+
        struct dsdb_extended_replicated_objects *objs;
 
        uint32_t index_current;
@@ -73,6 +76,14 @@ static struct replmd_replicated_request *replmd_replicated_init_handle(struct ld
 {
        struct replmd_replicated_request *ar;
        struct ldb_handle *h;
+       const struct dsdb_schema *schema;
+
+       schema = dsdb_get_schema(module->ldb);
+       if (!schema) {
+               ldb_debug_set(module->ldb, LDB_DEBUG_FATAL,
+                             "replmd_replicated_init_handle: no loaded schema found\n");
+               return NULL;
+       }
 
        h = talloc_zero(req, struct ldb_handle);
        if (h == NULL) {
@@ -96,6 +107,7 @@ static struct replmd_replicated_request *replmd_replicated_init_handle(struct ld
        ar->module      = module;
        ar->handle      = h;
        ar->orig_req    = req;
+       ar->schema      = schema;
        ar->objs        = objs;
 
        req->handle = h;
@@ -103,19 +115,6 @@ static struct replmd_replicated_request *replmd_replicated_init_handle(struct ld
        return ar;
 }
 
-static struct ldb_message_element *replmd_find_attribute(const struct ldb_message *msg, const char *name)
-{
-       int i;
-
-       for (i = 0; i < msg->num_elements; i++) {
-               if (ldb_attr_cmp(name, msg->elements[i].name) == 0) {
-                       return &msg->elements[i];
-               }
-       }
-
-       return NULL;
-}
-
 /*
   add a time element to a record
 */
@@ -168,73 +167,302 @@ static int add_uint64_element(struct ldb_message *msg, const char *attr, uint64_
        return 0;
 }
 
+static int replmd_replPropertyMetaData1_attid_sort(const struct replPropertyMetaData1 *m1,
+                                                  const struct replPropertyMetaData1 *m2,
+                                                  const uint32_t *rdn_attid)
+{
+       if (m1->attid == m2->attid) {
+               return 0;
+       }
+
+       /*
+        * the rdn attribute should be at the end!
+        * so we need to return a value greater than zero
+        * which means m1 is greater than m2
+        */
+       if (m1->attid == *rdn_attid) {
+               return 1;
+       }
+
+       /*
+        * the rdn attribute should be at the end!
+        * so we need to return a value less than zero
+        * which means m2 is greater than m1
+        */
+       if (m2->attid == *rdn_attid) {
+               return -1;
+       }
+
+       return m1->attid - m2->attid;
+}
+
+static void replmd_replPropertyMetaDataCtr1_sort(struct replPropertyMetaDataCtr1 *ctr1,
+                                                const uint32_t *rdn_attid)
+{
+       ldb_qsort(ctr1->array, ctr1->count, sizeof(struct replPropertyMetaData1),
+                 discard_const_p(void, rdn_attid), (ldb_qsort_cmp_fn_t)replmd_replPropertyMetaData1_attid_sort);
+}
+
+static int replmd_ldb_message_element_attid_sort(const struct ldb_message_element *e1,
+                                                const struct ldb_message_element *e2,
+                                                const struct dsdb_schema *schema)
+{
+       const struct dsdb_attribute *a1;
+       const struct dsdb_attribute *a2;
+
+       /* 
+        * TODO: make this faster by caching the dsdb_attribute pointer
+        *       on the ldb_messag_element
+        */
+
+       a1 = dsdb_attribute_by_lDAPDisplayName(schema, e1->name);
+       a2 = dsdb_attribute_by_lDAPDisplayName(schema, e2->name);
+
+       /*
+        * TODO: remove this check, we should rely on e1 and e2 having valid attribute names
+        *       in the schema
+        */
+       if (!a1 || !a2) {
+               return strcasecmp(e1->name, e2->name);
+       }
+
+       return a1->attributeID_id - a2->attributeID_id;
+}
+
+static void replmd_ldb_message_sort(struct ldb_message *msg,
+                                   const struct dsdb_schema *schema)
+{
+       ldb_qsort(msg->elements, msg->num_elements, sizeof(struct ldb_message_element),
+                 discard_const_p(void, schema), (ldb_qsort_cmp_fn_t)replmd_ldb_message_element_attid_sort);
+}
+
+static int replmd_prepare_originating(struct ldb_module *module, struct ldb_request *req,
+                                     struct ldb_dn *dn, const char *fn_name,
+                                     int (*fn)(struct ldb_module *,
+                                               struct ldb_request *,
+                                               const struct dsdb_schema *))
+{
+       const struct dsdb_schema *schema;
+       /* do not manipulate our control entries */
+       if (ldb_dn_is_special(dn)) {
+               return ldb_next_request(module, req);
+       }
+
+       schema = dsdb_get_schema(module->ldb);
+       if (!schema) {
+               ldb_debug_set(module->ldb, LDB_DEBUG_FATAL,
+                             "%s: no dsdb_schema loaded",
+                             fn_name);
+               return LDB_ERR_CONSTRAINT_VIOLATION;
+       }
+
+       return fn(module, req, schema);
+}
+
 static int replmd_add_originating(struct ldb_module *module,
                                  struct ldb_request *req,
-                                 const struct dsdb_schema *schema,
-                                 const struct dsdb_control_current_partition *partition)
+                                 const struct dsdb_schema *schema)
 {
+       enum ndr_err_code ndr_err;
        struct ldb_request *down_req;
-       struct ldb_message_element *attribute;
        struct ldb_message *msg;
-       struct ldb_val v;
+       const struct dsdb_attribute *rdn_attr = NULL;
        struct GUID guid;
+       struct ldb_val guid_value;
+       struct replPropertyMetaDataBlob nmd;
+       struct ldb_val nmd_value;
        uint64_t seq_num;
-       NTSTATUS nt_status;
-       int ret;
+       const struct GUID *our_invocation_id;
        time_t t = time(NULL);
+       NTTIME now;
+       char *time_str;
+       int ret;
+       uint32_t i, ni=0;
 
        ldb_debug(module->ldb, LDB_DEBUG_TRACE, "replmd_add_originating\n");
 
-       if ((attribute = replmd_find_attribute(req->op.add.message, "objectGUID")) != NULL ) {
-               return ldb_next_request(module, req);
+       if (ldb_msg_find_element(req->op.add.message, "objectGUID")) {
+               ldb_debug_set(module->ldb, LDB_DEBUG_ERROR,
+                             "replmd_add_originating: it's not allowed to add an object with objectGUID\n");
+               return LDB_ERR_UNWILLING_TO_PERFORM;
+       }
+
+       /* Get a sequence number from the backend */
+       ret = ldb_sequence_number(module->ldb, LDB_SEQ_NEXT, &seq_num);
+       if (ret != LDB_SUCCESS) {
+               return ret;
+       }
+
+       /* a new GUID */
+       guid = GUID_random();
+
+       /* get our invicationId */
+       our_invocation_id = samdb_ntds_invocation_id(module->ldb);
+       if (!our_invocation_id) {
+               ldb_debug_set(module->ldb, LDB_DEBUG_ERROR,
+                             "replmd_add_originating: unable to find invocationId\n");
+               return LDB_ERR_OPERATIONS_ERROR;
        }
 
+       /* create a copy of the request */
        down_req = talloc(req, struct ldb_request);
        if (down_req == NULL) {
+               ldb_oom(module->ldb);
                return LDB_ERR_OPERATIONS_ERROR;
        }
-
        *down_req = *req;
 
        /* we have to copy the message as the caller might have it as a const */
        down_req->op.add.message = msg = ldb_msg_copy_shallow(down_req, req->op.add.message);
        if (msg == NULL) {
                talloc_free(down_req);
+               ldb_oom(module->ldb);
                return LDB_ERR_OPERATIONS_ERROR;
        }
 
-       /* a new GUID */
-       guid = GUID_random();
-
-       nt_status = ndr_push_struct_blob(&v, msg, &guid, 
-                                        (ndr_push_flags_fn_t)ndr_push_GUID);
-       if (!NT_STATUS_IS_OK(nt_status)) {
+       /* generated times */
+       unix_to_nt_time(&now, t);
+       time_str = ldb_timestring(msg, t);
+       if (!time_str) {
                talloc_free(down_req);
                return LDB_ERR_OPERATIONS_ERROR;
        }
 
-       ret = ldb_msg_add_value(msg, "objectGUID", &v, NULL);
-       if (ret) {
+       /* 
+        * remove autogenerated attributes
+        */
+       ldb_msg_remove_attr(msg, "whenCreated");
+       ldb_msg_remove_attr(msg, "whenChanged");
+       ldb_msg_remove_attr(msg, "uSNCreated");
+       ldb_msg_remove_attr(msg, "uSNChanged");
+       ldb_msg_remove_attr(msg, "replPropertyMetaData");
+
+       /*
+        * readd replicated attributes
+        */
+       ret = ldb_msg_add_string(msg, "whenCreated", time_str);
+       if (ret != LDB_SUCCESS) {
                talloc_free(down_req);
-               return ret;
+               ldb_oom(module->ldb);
+               return LDB_ERR_OPERATIONS_ERROR;
        }
-       
-       if (add_time_element(msg, "whenCreated", t) != 0 ||
-           add_time_element(msg, "whenChanged", t) != 0) {
+
+       /* build the replication meta_data */
+       ZERO_STRUCT(nmd);
+       nmd.version             = 1;
+       nmd.ctr.ctr1.count      = msg->num_elements;
+       nmd.ctr.ctr1.array      = talloc_array(msg,
+                                              struct replPropertyMetaData1,
+                                              nmd.ctr.ctr1.count);
+       if (!nmd.ctr.ctr1.array) {
                talloc_free(down_req);
+               ldb_oom(module->ldb);
                return LDB_ERR_OPERATIONS_ERROR;
        }
 
-       /* Get a sequence number from the backend */
-       ret = ldb_sequence_number(module->ldb, LDB_SEQ_NEXT, &seq_num);
-       if (ret == LDB_SUCCESS) {
-               if (add_uint64_element(msg, "uSNCreated", seq_num) != 0 ||
-                   add_uint64_element(msg, "uSNChanged", seq_num) != 0) {
+       for (i=0; i < msg->num_elements; i++) {
+               struct ldb_message_element *e = &msg->elements[i];
+               struct replPropertyMetaData1 *m = &nmd.ctr.ctr1.array[ni];
+               const struct dsdb_attribute *sa;
+
+               if (e->name[0] == '@') continue;
+
+               sa = dsdb_attribute_by_lDAPDisplayName(schema, e->name);
+               if (!sa) {
+                       ldb_debug_set(module->ldb, LDB_DEBUG_ERROR,
+                                     "replmd_add_originating: attribute '%s' not defined in schema\n",
+                                     e->name);
                        talloc_free(down_req);
-                       return LDB_ERR_OPERATIONS_ERROR;
+                       return LDB_ERR_NO_SUCH_ATTRIBUTE;
+               }
+
+               if ((sa->systemFlags & 0x00000001) || (sa->systemFlags & 0x00000004)) {
+                       /* if the attribute is not replicated (0x00000001)
+                        * or constructed (0x00000004) it has no metadata
+                        */
+                       continue;
+               }
+
+               m->attid                        = sa->attributeID_id;
+               m->version                      = 1;
+               m->originating_change_time      = now;
+               m->originating_invocation_id    = *our_invocation_id;
+               m->originating_usn              = seq_num;
+               m->local_usn                    = seq_num;
+               ni++;
+
+               if (ldb_attr_cmp(e->name, ldb_dn_get_rdn_name(msg->dn))) {
+                       rdn_attr = sa;
                }
        }
 
+       /* fix meta data count */
+       nmd.ctr.ctr1.count = ni;
+
+       /*
+        * sort meta data array, and move the rdn attribute entry to the end
+        */
+       replmd_replPropertyMetaDataCtr1_sort(&nmd.ctr.ctr1, &rdn_attr->attributeID_id);
+
+       /* generated NDR encoded values */
+       ndr_err = ndr_push_struct_blob(&guid_value, msg, 
+                                      NULL,
+                                      &guid,
+                                      (ndr_push_flags_fn_t)ndr_push_GUID);
+       if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+               ldb_oom(module->ldb);
+               return LDB_ERR_OPERATIONS_ERROR;
+       }
+       ndr_err = ndr_push_struct_blob(&nmd_value, msg, 
+                                      lp_iconv_convenience(ldb_get_opaque(module->ldb, "loadparm")),
+                                      &nmd,
+                                      (ndr_push_flags_fn_t)ndr_push_replPropertyMetaDataBlob);
+       if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+               talloc_free(down_req);
+               ldb_oom(module->ldb);
+               return LDB_ERR_OPERATIONS_ERROR;
+       }
+
+       /*
+        * add the autogenerated values
+        */
+       ret = ldb_msg_add_value(msg, "objectGUID", &guid_value, NULL);
+       if (ret != LDB_SUCCESS) {
+               talloc_free(down_req);
+               ldb_oom(module->ldb);
+               return LDB_ERR_OPERATIONS_ERROR;
+       }
+       ret = ldb_msg_add_string(msg, "whenChanged", time_str);
+       if (ret != LDB_SUCCESS) {
+               talloc_free(down_req);
+               ldb_oom(module->ldb);
+               return LDB_ERR_OPERATIONS_ERROR;
+       }
+       ret = samdb_msg_add_uint64(module->ldb, msg, msg, "uSNCreated", seq_num);
+       if (ret != LDB_SUCCESS) {
+               talloc_free(down_req);
+               ldb_oom(module->ldb);
+               return LDB_ERR_OPERATIONS_ERROR;
+       }
+       ret = samdb_msg_add_uint64(module->ldb, msg, msg, "uSNChanged", seq_num);
+       if (ret != LDB_SUCCESS) {
+               talloc_free(down_req);
+               ldb_oom(module->ldb);
+               return LDB_ERR_OPERATIONS_ERROR;
+       }
+       ret = ldb_msg_add_value(msg, "replPropertyMetaData", &nmd_value, NULL);
+       if (ret != LDB_SUCCESS) {
+               talloc_free(down_req);
+               ldb_oom(module->ldb);
+               return LDB_ERR_OPERATIONS_ERROR;
+       }
+
+       /*
+        * sort the attributes by attid before storing the object
+        */
+       replmd_ldb_message_sort(msg, schema);
+
        ldb_set_timeout_from_prev_req(module->ldb, req, down_req);
 
        /* go on with the call chain */
@@ -251,51 +479,13 @@ static int replmd_add_originating(struct ldb_module *module,
 
 static int replmd_add(struct ldb_module *module, struct ldb_request *req)
 {
-       const struct dsdb_schema *schema;
-       const struct ldb_control *partition_ctrl;
-       const struct dsdb_control_current_partition *partition;
-
-       /* do not manipulate our control entries */
-       if (ldb_dn_is_special(req->op.add.message->dn)) {
-               return ldb_next_request(module, req);
-       }
-
-       schema = dsdb_get_schema(module->ldb);
-       if (!schema) {
-               ldb_debug_set(module->ldb, LDB_DEBUG_FATAL,
-                             "replmd_add: no dsdb_schema loaded");
-               return LDB_ERR_CONSTRAINT_VIOLATION;
-       }
-
-       partition_ctrl = get_control_from_list(req->controls, DSDB_CONTROL_CURRENT_PARTITION_OID);
-       if (!partition_ctrl) {
-               ldb_debug_set(module->ldb, LDB_DEBUG_FATAL,
-                             "replmd_add: no current partition control found");
-               return LDB_ERR_CONSTRAINT_VIOLATION;
-       }
-
-       partition = talloc_get_type(partition_ctrl->data,
-                                   struct dsdb_control_current_partition);
-       if (!partition) {
-               ldb_debug_set(module->ldb, LDB_DEBUG_FATAL,
-                             "replmd_add: current partition control contains invalid data");
-               return LDB_ERR_CONSTRAINT_VIOLATION;
-       }
-
-       if (partition->version != DSDB_CONTROL_CURRENT_PARTITION_VERSION) {
-               ldb_debug_set(module->ldb, LDB_DEBUG_FATAL,
-                             "replmd_add: current partition control contains invalid version [%u != %u]\n",
-                             partition->version, DSDB_CONTROL_CURRENT_PARTITION_VERSION);
-               return LDB_ERR_CONSTRAINT_VIOLATION;
-       }
-
-       return replmd_add_originating(module, req, schema, partition);
+       return replmd_prepare_originating(module, req, req->op.add.message->dn,
+                                         "replmd_add", replmd_add_originating);
 }
 
 static int replmd_modify_originating(struct ldb_module *module,
                                     struct ldb_request *req,
-                                    const struct dsdb_schema *schema,
-                                    const struct dsdb_control_current_partition *partition)
+                                    const struct dsdb_schema *schema)
 {
        struct ldb_request *down_req;
        struct ldb_message *msg;
@@ -319,6 +509,18 @@ static int replmd_modify_originating(struct ldb_module *module,
                return LDB_ERR_OPERATIONS_ERROR;
        }
 
+       /* TODO:
+        * - get the whole old object
+        * - if the old object doesn't exist report an error
+        * - give an error when a readonly attribute should
+        *   be modified
+        * - merge the changed into the old object
+        *   if the caller set values to the same value
+        *   ignore the attribute, return success when no
+        *   attribute was changed
+        * - calculate the new replPropertyMetaData attribute
+        */
+
        if (add_time_element(msg, "whenChanged", t) != 0) {
                talloc_free(down_req);
                return LDB_ERR_OPERATIONS_ERROR;
@@ -333,6 +535,11 @@ static int replmd_modify_originating(struct ldb_module *module,
                }
        }
 
+       /* TODO:
+        * - sort the attributes by attid with replmd_ldb_message_sort()
+        * - replace the old object with the newly constructed one
+        */
+
        ldb_set_timeout_from_prev_req(module->ldb, req, down_req);
 
        /* go on with the call chain */
@@ -349,52 +556,8 @@ static int replmd_modify_originating(struct ldb_module *module,
 
 static int replmd_modify(struct ldb_module *module, struct ldb_request *req)
 {
-       const struct dsdb_schema *schema;
-       const struct ldb_control *partition_ctrl;
-       const struct dsdb_control_current_partition *partition;
-       /* do not manipulate our control entries */
-       if (ldb_dn_is_special(req->op.mod.message->dn)) {
-               return ldb_next_request(module, req);
-       }
-
-       schema = dsdb_get_schema(module->ldb);
-       if (!schema) {
-               ldb_debug_set(module->ldb, LDB_DEBUG_FATAL,
-                             "replmd_modify: no dsdb_schema loaded");
-               return LDB_ERR_CONSTRAINT_VIOLATION;
-       }
-
-       schema = dsdb_get_schema(module->ldb);
-       if (!schema) {
-               ldb_debug_set(module->ldb, LDB_DEBUG_FATAL,
-                             "replmd_modify: no dsdb_schema loaded");
-               return LDB_ERR_CONSTRAINT_VIOLATION;
-       }
-
-       partition_ctrl = get_control_from_list(req->controls, DSDB_CONTROL_CURRENT_PARTITION_OID);
-       if (!partition_ctrl) {
-               ldb_debug_set(module->ldb, LDB_DEBUG_FATAL,
-                             "replmd_modify: no current partition control found");
-               return LDB_ERR_CONSTRAINT_VIOLATION;
-       }
-
-       partition = talloc_get_type(partition_ctrl->data,
-                                   struct dsdb_control_current_partition);
-       if (!partition) {
-               ldb_debug_set(module->ldb, LDB_DEBUG_FATAL,
-                             "replmd_modify: current partition control contains invalid data");
-               return LDB_ERR_CONSTRAINT_VIOLATION;
-       }
-
-       if (partition->version != DSDB_CONTROL_CURRENT_PARTITION_VERSION) {
-               ldb_debug_set(module->ldb, LDB_DEBUG_FATAL,
-                             "replmd_modify: current partition control contains invalid version [%u != %u]\n",
-                             partition->version, DSDB_CONTROL_CURRENT_PARTITION_VERSION);
-               return LDB_ERR_CONSTRAINT_VIOLATION;
-       }
-
-       return replmd_modify_originating(module, req, schema, partition);
+       return replmd_prepare_originating(module, req, req->op.mod.message->dn,
+                                         "replmd_modify", replmd_modify_originating);
 }
 
 static int replmd_replicated_request_reply_helper(struct replmd_replicated_request *ar, int ret)
@@ -445,7 +608,7 @@ static int replmd_replicated_apply_add_callback(struct ldb_context *ldb,
                                                void *private_data,
                                                struct ldb_reply *ares)
 {
-#ifdef REPLMD_FULL_ASYNC /* TODO: active this code when ldb support full async code */ 
+#ifdef REPLMD_FULL_ASYNC /* TODO: activate this code when ldb support full async code */ 
        struct replmd_replicated_request *ar = talloc_get_type(private_data,
                                               struct replmd_replicated_request);
 
@@ -467,7 +630,7 @@ static int replmd_replicated_apply_add_callback(struct ldb_context *ldb,
 
 static int replmd_replicated_apply_add(struct replmd_replicated_request *ar)
 {
-       NTSTATUS nt_status;
+       enum ndr_err_code ndr_err;
        struct ldb_message *msg;
        struct replPropertyMetaDataBlob *md;
        struct ldb_val md_value;
@@ -512,13 +675,18 @@ static int replmd_replicated_apply_add(struct replmd_replicated_request *ar)
                return replmd_replicated_request_error(ar, ret);
        }
 
-       md = ar->objs->objects[ar->index_current].meta_data;
+       /*
+        * the meta data array is already sorted by the caller
+        */
        for (i=0; i < md->ctr.ctr1.count; i++) {
                md->ctr.ctr1.array[i].local_usn = seq_num;
        }
-       nt_status = ndr_push_struct_blob(&md_value, msg, md,
-                                        (ndr_push_flags_fn_t)ndr_push_replPropertyMetaDataBlob);
-       if (!NT_STATUS_IS_OK(nt_status)) {
+       ndr_err = ndr_push_struct_blob(&md_value, msg, 
+                                      lp_iconv_convenience(ldb_get_opaque(ar->module->ldb, "loadparm")),
+                                      md,
+                                      (ndr_push_flags_fn_t)ndr_push_replPropertyMetaDataBlob);
+       if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+               NTSTATUS nt_status = ndr_map_error2ntstatus(ndr_err);
                return replmd_replicated_request_werror(ar, ntstatus_to_werror(nt_status));
        }
        ret = ldb_msg_add_value(msg, "replPropertyMetaData", &md_value, NULL);
@@ -526,6 +694,8 @@ static int replmd_replicated_apply_add(struct replmd_replicated_request *ar)
                return replmd_replicated_request_error(ar, ret);
        }
 
+       replmd_ldb_message_sort(msg, ar->schema);
+
        ret = ldb_build_add_req(&ar->sub.change_req,
                                ar->module->ldb,
                                ar->sub.mem_ctx,
@@ -535,14 +705,20 @@ static int replmd_replicated_apply_add(struct replmd_replicated_request *ar)
                                replmd_replicated_apply_add_callback);
        if (ret != LDB_SUCCESS) return replmd_replicated_request_error(ar, ret);
 
-#ifdef REPLMD_FULL_ASYNC /* TODO: active this code when ldb support full async code */ 
+#ifdef REPLMD_FULL_ASYNC /* TODO: activate this code when ldb support full async code */ 
        return ldb_next_request(ar->module, ar->sub.change_req);
 #else
        ret = ldb_next_request(ar->module, ar->sub.change_req);
-       if (ret != LDB_SUCCESS) return replmd_replicated_request_error(ar, ret);
+       if (ret != LDB_SUCCESS) {
+               ldb_asprintf_errstring(ar->module->ldb, "Failed to add replicated object %s: %s", ldb_dn_get_linearized(ar->sub.change_req->op.add.message->dn), 
+                                      ldb_errstring(ar->module->ldb));
+               return replmd_replicated_request_error(ar, ret);
+       }
 
-       ar->sub.change_ret = ldb_wait(ar->sub.search_req->handle, LDB_WAIT_ALL);
+       ar->sub.change_ret = ldb_wait(ar->sub.change_req->handle, LDB_WAIT_ALL);
        if (ar->sub.change_ret != LDB_SUCCESS) {
+               ldb_asprintf_errstring(ar->module->ldb, "Failed while waiting on add replicated object %s: %s", ldb_dn_get_linearized(ar->sub.change_req->op.add.message->dn), 
+                                      ldb_errstring(ar->module->ldb));
                return replmd_replicated_request_error(ar, ar->sub.change_ret);
        }
 
@@ -555,12 +731,6 @@ static int replmd_replicated_apply_add(struct replmd_replicated_request *ar)
 #endif
 }
 
-static int replmd_replPropertyMetaData1_attid_compare(struct replPropertyMetaData1 *m1,
-                                                     struct replPropertyMetaData1 *m2)
-{
-       return m1->attid - m2->attid;
-}
-
 static int replmd_replPropertyMetaData1_conflict_compare(struct replPropertyMetaData1 *m1,
                                                         struct replPropertyMetaData1 *m2)
 {
@@ -570,23 +740,23 @@ static int replmd_replPropertyMetaData1_conflict_compare(struct replPropertyMeta
                return m1->version - m2->version;
        }
 
-       if (m1->orginating_time != m2->orginating_time) {
-               return m1->orginating_time - m2->orginating_time;
+       if (m1->originating_change_time != m2->originating_change_time) {
+               return m1->originating_change_time - m2->originating_change_time;
        }
 
-       ret = GUID_compare(&m1->orginating_invocation_id, &m2->orginating_invocation_id);
+       ret = GUID_compare(&m1->originating_invocation_id, &m2->originating_invocation_id);
        if (ret != 0) {
                return ret;
        }
 
-       return m1->orginating_usn - m2->orginating_usn;
+       return m1->originating_usn - m2->originating_usn;
 }
 
 static int replmd_replicated_apply_merge_callback(struct ldb_context *ldb,
                                                  void *private_data,
                                                  struct ldb_reply *ares)
 {
-#ifdef REPLMD_FULL_ASYNC /* TODO: active this code when ldb support full async code */ 
+#ifdef REPLMD_FULL_ASYNC /* TODO: activate this code when ldb support full async code */ 
        struct replmd_replicated_request *ar = talloc_get_type(private_data,
                                               struct replmd_replicated_request);
 
@@ -611,7 +781,7 @@ static int replmd_replicated_apply_merge_callback(struct ldb_context *ldb,
 
 static int replmd_replicated_apply_merge(struct replmd_replicated_request *ar)
 {
-       NTSTATUS nt_status;
+       enum ndr_err_code ndr_err;
        struct ldb_message *msg;
        struct replPropertyMetaDataBlob *rmd;
        struct replPropertyMetaDataBlob omd;
@@ -648,9 +818,11 @@ static int replmd_replicated_apply_merge(struct replmd_replicated_request *ar)
        /* find existing meta data */
        omd_value = ldb_msg_find_ldb_val(ar->sub.search_msg, "replPropertyMetaData");
        if (omd_value) {
-               nt_status = ndr_pull_struct_blob(omd_value, ar->sub.mem_ctx, &omd,
-                                                (ndr_pull_flags_fn_t)ndr_pull_replPropertyMetaDataBlob);
-               if (!NT_STATUS_IS_OK(nt_status)) {
+               ndr_err = ndr_pull_struct_blob(omd_value, ar->sub.mem_ctx, 
+                                              lp_iconv_convenience(ldb_get_opaque(ar->module->ldb, "loadparm")), &omd,
+                                              (ndr_pull_flags_fn_t)ndr_pull_replPropertyMetaDataBlob);
+               if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+                       NTSTATUS nt_status = ndr_map_error2ntstatus(ndr_err);
                        return replmd_replicated_request_werror(ar, ntstatus_to_werror(nt_status));
                }
 
@@ -719,35 +891,23 @@ static int replmd_replicated_apply_merge(struct replmd_replicated_request *ar)
         * 'cn' for most objects is the last entry in the meta data array
         * we have stored
         *
-        * as it should stay the last one in the new list, we move it to the end
+        * sort the new meta data array
         */
        {
-               struct replPropertyMetaData1 *rdn_p, rdn, *last_p;
+               struct replPropertyMetaData1 *rdn_p;
                uint32_t rdn_idx = omd.ctr.ctr1.count - 1;
-               uint32_t last_idx = ni - 1;
 
                rdn_p = &nmd.ctr.ctr1.array[rdn_idx];
-               rdn = *rdn_p;
-               last_p = &nmd.ctr.ctr1.array[last_idx];
-
-               if (last_idx > rdn_idx) {
-                       memmove(rdn_p, rdn_p+1, (last_idx - rdn_idx)*sizeof(rdn));
-                       *last_p = rdn;
-               }
+               replmd_replPropertyMetaDataCtr1_sort(&nmd.ctr.ctr1, &rdn_p->attid);
        }
 
-       /*
-        * sort the meta data entries by attid, but skip the last one containing
-        * the rdn attribute
-        */
-       qsort(nmd.ctr.ctr1.array, nmd.ctr.ctr1.count - 1,
-             sizeof(struct replPropertyMetaData1),
-             (comparison_fn_t)replmd_replPropertyMetaData1_attid_compare);
-
        /* create the meta data value */
-       nt_status = ndr_push_struct_blob(&nmd_value, msg, &nmd,
-                                        (ndr_push_flags_fn_t)ndr_push_replPropertyMetaDataBlob);
-       if (!NT_STATUS_IS_OK(nt_status)) {
+       ndr_err = ndr_push_struct_blob(&nmd_value, msg, 
+                                      lp_iconv_convenience(ldb_get_opaque(ar->module->ldb, "loadparm")),
+                                      &nmd,
+                                      (ndr_push_flags_fn_t)ndr_push_replPropertyMetaDataBlob);
+       if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+               NTSTATUS nt_status = ndr_map_error2ntstatus(ndr_err);
                return replmd_replicated_request_werror(ar, ntstatus_to_werror(nt_status));
        }
 
@@ -780,6 +940,8 @@ static int replmd_replicated_apply_merge(struct replmd_replicated_request *ar)
                return replmd_replicated_request_error(ar, ret);
        }
 
+       replmd_ldb_message_sort(msg, ar->schema);
+
        /* we want to replace the old values */
        for (i=0; i < msg->num_elements; i++) {
                msg->elements[i].flags = LDB_FLAG_MOD_REPLACE;
@@ -794,13 +956,13 @@ static int replmd_replicated_apply_merge(struct replmd_replicated_request *ar)
                                replmd_replicated_apply_merge_callback);
        if (ret != LDB_SUCCESS) return replmd_replicated_request_error(ar, ret);
 
-#ifdef REPLMD_FULL_ASYNC /* TODO: active this code when ldb support full async code */ 
+#ifdef REPLMD_FULL_ASYNC /* TODO: activate this code when ldb support full async code */ 
        return ldb_next_request(ar->module, ar->sub.change_req);
 #else
        ret = ldb_next_request(ar->module, ar->sub.change_req);
        if (ret != LDB_SUCCESS) return replmd_replicated_request_error(ar, ret);
 
-       ar->sub.change_ret = ldb_wait(ar->sub.search_req->handle, LDB_WAIT_ALL);
+       ar->sub.change_ret = ldb_wait(ar->sub.change_req->handle, LDB_WAIT_ALL);
        if (ar->sub.change_ret != LDB_SUCCESS) {
                return replmd_replicated_request_error(ar, ar->sub.change_ret);
        }
@@ -837,7 +999,7 @@ static int replmd_replicated_apply_search_callback(struct ldb_context *ldb,
 
        talloc_free(ares);
 
-#ifdef REPLMD_FULL_ASYNC /* TODO: active this code when ldb support full async code */ 
+#ifdef REPLMD_FULL_ASYNC /* TODO: activate this code when ldb support full async code */ 
        if (is_done) {
                ar->sub.search_ret = ldb_wait(ar->sub.search_req->handle, LDB_WAIT_ALL);
                if (ar->sub.search_ret != LDB_SUCCESS) {
@@ -877,14 +1039,14 @@ static int replmd_replicated_apply_search(struct replmd_replicated_request *ar)
                                   replmd_replicated_apply_search_callback);
        if (ret != LDB_SUCCESS) return replmd_replicated_request_error(ar, ret);
 
-#ifdef REPLMD_FULL_ASYNC /* TODO: active this code when ldb support full async code */ 
+#ifdef REPLMD_FULL_ASYNC /* TODO: activate this code when ldb support full async code */ 
        return ldb_next_request(ar->module, ar->sub.search_req);
 #else
        ret = ldb_next_request(ar->module, ar->sub.search_req);
        if (ret != LDB_SUCCESS) return replmd_replicated_request_error(ar, ret);
 
        ar->sub.search_ret = ldb_wait(ar->sub.search_req->handle, LDB_WAIT_ALL);
-       if (ar->sub.search_ret != LDB_SUCCESS) {
+       if (ar->sub.search_ret != LDB_SUCCESS && ar->sub.search_ret != LDB_ERR_NO_SUCH_OBJECT) {
                return replmd_replicated_request_error(ar, ar->sub.search_ret);
        }
        if (ar->sub.search_msg) {
@@ -897,7 +1059,7 @@ static int replmd_replicated_apply_search(struct replmd_replicated_request *ar)
 
 static int replmd_replicated_apply_next(struct replmd_replicated_request *ar)
 {
-#ifdef REPLMD_FULL_ASYNC /* TODO: active this code when ldb support full async code */ 
+#ifdef REPLMD_FULL_ASYNC /* TODO: activate this code when ldb support full async code */ 
        if (ar->index_current >= ar->objs->num_objects) {
                return replmd_replicated_uptodate_vector(ar);
        }
@@ -913,7 +1075,7 @@ static int replmd_replicated_uptodate_modify_callback(struct ldb_context *ldb,
                                                      void *private_data,
                                                      struct ldb_reply *ares)
 {
-#ifdef REPLMD_FULL_ASYNC /* TODO: active this code when ldb support full async code */ 
+#ifdef REPLMD_FULL_ASYNC /* TODO: activate this code when ldb support full async code */ 
        struct replmd_replicated_request *ar = talloc_get_type(private_data,
                                               struct replmd_replicated_request);
 
@@ -939,7 +1101,7 @@ static int replmd_drsuapi_DsReplicaCursor2_compare(const struct drsuapi_DsReplic
 
 static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *ar)
 {
-       NTSTATUS nt_status;
+       enum ndr_err_code ndr_err;
        struct ldb_message *msg;
        struct replUpToDateVectorBlob ouv;
        const struct ldb_val *ouv_value;
@@ -982,9 +1144,11 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a
         */
        ouv_value = ldb_msg_find_ldb_val(ar->sub.search_msg, "replUpToDateVector");
        if (ouv_value) {
-               nt_status = ndr_pull_struct_blob(ouv_value, ar->sub.mem_ctx, &ouv,
-                                                (ndr_pull_flags_fn_t)ndr_pull_replUpToDateVectorBlob);
-               if (!NT_STATUS_IS_OK(nt_status)) {
+               ndr_err = ndr_pull_struct_blob(ouv_value, ar->sub.mem_ctx, 
+                                              lp_iconv_convenience(ldb_get_opaque(ar->module->ldb, "loadparm")), &ouv,
+                                              (ndr_pull_flags_fn_t)ndr_pull_replUpToDateVectorBlob);
+               if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+                       NTSTATUS nt_status = ndr_map_error2ntstatus(ndr_err);
                        return replmd_replicated_request_werror(ar, ntstatus_to_werror(nt_status));
                }
 
@@ -995,11 +1159,11 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a
 
        /*
         * the new uptodateness vector will at least
-        * contain 2 entries, one for the source_dsa and one the local server
+        * contain 1 entry, one for the source_dsa
         *
         * plus optional values from our old vector and the one from the source_dsa
         */
-       nuv.ctr.ctr2.count = 2 + ouv.ctr.ctr2.count;
+       nuv.ctr.ctr2.count = 1 + ouv.ctr.ctr2.count;
        if (ruv) nuv.ctr.ctr2.count += ruv->count;
        nuv.ctr.ctr2.cursors = talloc_array(ar->sub.mem_ctx,
                                            struct drsuapi_DsReplicaCursor2,
@@ -1012,10 +1176,19 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a
                ni++;
        }
 
+       /* get our invocation_id if we have one already attached to the ldb */
+       our_invocation_id = samdb_ntds_invocation_id(ar->module->ldb);
+
        /* merge in the source_dsa vector is available */
        for (i=0; (ruv && i < ruv->count); i++) {
                found = false;
 
+               if (our_invocation_id &&
+                   GUID_equal(&ruv->cursors[i].source_dsa_invocation_id,
+                              our_invocation_id)) {
+                       continue;
+               }
+
                for (j=0; j < ni; j++) {
                        if (!GUID_equal(&ruv->cursors[i].source_dsa_invocation_id,
                                        &nuv.ctr.ctr2.cursors[j].source_dsa_invocation_id)) {
@@ -1078,41 +1251,6 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a
                ni++;
        }
 
-       /*
-        * merge our own current values if we have a invocation_id already
-        * attached to the ldb
-        */
-       our_invocation_id = samdb_ntds_invocation_id(ar->module->ldb);
-       if (our_invocation_id) {
-               found = false;
-               for (j=0; j < ni; j++) {
-                       if (!GUID_equal(our_invocation_id,
-                                       &nuv.ctr.ctr2.cursors[j].source_dsa_invocation_id)) {
-                               continue;
-                       }
-
-                       found = true;
-
-                       /*
-                        * here we update the highest_usn and last_sync_success time
-                        * because it's our own entry
-                        */
-                       nuv.ctr.ctr2.cursors[j].highest_usn             = seq_num;
-                       nuv.ctr.ctr2.cursors[j].last_sync_success       = now;
-                       break;
-               }
-               if (!found) {
-                       /*
-                        * here we update the highest_usn and last_sync_success time
-                        * because it's our own entry
-                        */
-                       nuv.ctr.ctr2.cursors[ni].source_dsa_invocation_id= *our_invocation_id;
-                       nuv.ctr.ctr2.cursors[ni].highest_usn            = seq_num;
-                       nuv.ctr.ctr2.cursors[ni].last_sync_success      = now;
-                       ni++;
-               }
-       }
-
        /*
         * finally correct the size of the cursors array
         */
@@ -1132,9 +1270,12 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a
        if (!msg) return replmd_replicated_request_werror(ar, WERR_NOMEM);
        msg->dn = ar->sub.search_msg->dn;
 
-       nt_status = ndr_push_struct_blob(&nuv_value, msg, &nuv,
-                                        (ndr_push_flags_fn_t)ndr_push_replUpToDateVectorBlob);
-       if (!NT_STATUS_IS_OK(nt_status)) {
+       ndr_err = ndr_push_struct_blob(&nuv_value, msg, 
+                                      lp_iconv_convenience(ldb_get_opaque(ar->module->ldb, "loadparm")), 
+                                      &nuv,
+                                      (ndr_push_flags_fn_t)ndr_push_replUpToDateVectorBlob);
+       if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+               NTSTATUS nt_status = ndr_map_error2ntstatus(ndr_err);
                return replmd_replicated_request_werror(ar, ntstatus_to_werror(nt_status));
        }
        ret = ldb_msg_add_value(msg, "replUpToDateVector", &nuv_value, &nuv_el);
@@ -1168,9 +1309,10 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a
                        trf = talloc(ar->sub.mem_ctx, struct repsFromToBlob);
                        if (!trf) return replmd_replicated_request_werror(ar, WERR_NOMEM);
 
-                       nt_status = ndr_pull_struct_blob(&orf_el->values[i], trf, trf,
-                                                        (ndr_pull_flags_fn_t)ndr_pull_repsFromToBlob);
-                       if (!NT_STATUS_IS_OK(nt_status)) {
+                       ndr_err = ndr_pull_struct_blob(&orf_el->values[i], trf, lp_iconv_convenience(ldb_get_opaque(ar->module->ldb, "loadparm")), trf,
+                                                      (ndr_pull_flags_fn_t)ndr_pull_repsFromToBlob);
+                       if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+                               NTSTATUS nt_status = ndr_map_error2ntstatus(ndr_err);
                                return replmd_replicated_request_werror(ar, ntstatus_to_werror(nt_status));
                        }
 
@@ -1217,9 +1359,12 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a
        }
 
        /* we now fill the value which is already attached to ldb_message */
-       nt_status = ndr_push_struct_blob(nrf_value, msg, &nrf,
-                                        (ndr_push_flags_fn_t)ndr_push_repsFromToBlob);
-       if (!NT_STATUS_IS_OK(nt_status)) {
+       ndr_err = ndr_push_struct_blob(nrf_value, msg, 
+                                      lp_iconv_convenience(ldb_get_opaque(ar->module->ldb, "loadparm")),
+                                      &nrf,
+                                      (ndr_push_flags_fn_t)ndr_push_repsFromToBlob);
+       if (!NDR_ERR_CODE_IS_SUCCESS(ndr_err)) {
+               NTSTATUS nt_status = ndr_map_error2ntstatus(ndr_err);
                return replmd_replicated_request_werror(ar, ntstatus_to_werror(nt_status));
        }
 
@@ -1239,7 +1384,7 @@ static int replmd_replicated_uptodate_modify(struct replmd_replicated_request *a
                                replmd_replicated_uptodate_modify_callback);
        if (ret != LDB_SUCCESS) return replmd_replicated_request_error(ar, ret);
 
-#ifdef REPLMD_FULL_ASYNC /* TODO: active this code when ldb support full async code */ 
+#ifdef REPLMD_FULL_ASYNC /* TODO: activate this code when ldb support full async code */ 
        return ldb_next_request(ar->module, ar->sub.change_req);
 #else
        ret = ldb_next_request(ar->module, ar->sub.change_req);
@@ -1279,7 +1424,7 @@ static int replmd_replicated_uptodate_search_callback(struct ldb_context *ldb,
 
        talloc_free(ares);
 
-#ifdef REPLMD_FULL_ASYNC /* TODO: active this code when ldb support full async code */ 
+#ifdef REPLMD_FULL_ASYNC /* TODO: activate this code when ldb support full async code */ 
        if (is_done) {
                ar->sub.search_ret = ldb_wait(ar->sub.search_req->handle, LDB_WAIT_ALL);
                if (ar->sub.search_ret != LDB_SUCCESS) {
@@ -1316,7 +1461,7 @@ static int replmd_replicated_uptodate_search(struct replmd_replicated_request *a
                                   replmd_replicated_uptodate_search_callback);
        if (ret != LDB_SUCCESS) return replmd_replicated_request_error(ar, ret);
 
-#ifdef REPLMD_FULL_ASYNC /* TODO: active this code when ldb support full async code */ 
+#ifdef REPLMD_FULL_ASYNC /* TODO: activate this code when ldb support full async code */ 
        return ldb_next_request(ar->module, ar->sub.search_req);
 #else
        ret = ldb_next_request(ar->module, ar->sub.search_req);
@@ -1366,7 +1511,7 @@ static int replmd_extended_replicated_objects(struct ldb_module *module, struct
                return LDB_ERR_OPERATIONS_ERROR;
        }
 
-#ifdef REPLMD_FULL_ASYNC /* TODO: active this code when ldb support full async code */ 
+#ifdef REPLMD_FULL_ASYNC /* TODO: activate this code when ldb support full async code */ 
        return replmd_replicated_apply_next(ar);
 #else
        while (ar->index_current < ar->objs->num_objects &&