s4:partition LDB module - change counter variables to "unsigned" where appropriate
[ira/wip.git] / source4 / dsdb / samdb / ldb_modules / partition.c
index c5bbdf8dce35b9c3fff78735026797b6ed5bd887..19bd03633a8b1340623b475ffa11a76bab109f50 100644 (file)
  *  Author: Stefan Metzmacher
  */
 
-#include "includes.h"
-#include "lib/ldb/include/ldb.h"
-#include "lib/ldb/include/ldb_errors.h"
-#include "lib/ldb/include/ldb_module.h"
-#include "lib/ldb/include/ldb_private.h"
-#include "dsdb/samdb/samdb.h"
-
-struct dsdb_partition {
-       struct ldb_module *module;
-       struct dsdb_control_current_partition *ctrl;
-};
-
-struct partition_private_data {
-       struct dsdb_partition **partitions;
-       struct ldb_dn **replicate;
-};
+#include "dsdb/samdb/ldb_modules/partition.h"
 
 struct part_request {
        struct ldb_module *module;
@@ -54,11 +39,12 @@ struct part_request {
 struct partition_context {
        struct ldb_module *module;
        struct ldb_request *req;
-       bool got_success;
 
        struct part_request *part_req;
-       int num_requests;
-       int finished_requests;
+       unsigned int num_requests;
+       unsigned int finished_requests;
+
+       const char **referrals;
 };
 
 static struct partition_context *partition_init_ctx(struct ldb_module *module, struct ldb_request *req)
@@ -77,74 +63,35 @@ static struct partition_context *partition_init_ctx(struct ldb_module *module, s
        return ac;
 }
 
-#define PARTITION_FIND_OP_NOERROR(module, op) do { \
-        while (module && module->ops->op == NULL) module = module->next; \
-} while (0)
-
-#define PARTITION_FIND_OP(module, op) do { \
-       PARTITION_FIND_OP_NOERROR(module, op); \
-        if (module == NULL) { \
-                ldb_asprintf_errstring(ldb_module_get_ctx(module), \
-                       "Unable to find backend operation for " #op ); \
-                return LDB_ERR_OPERATIONS_ERROR; \
-        } \
-} while (0)
-
 /*
- *    helper functions to call the next module in chain
- *    */
-
-static int partition_request(struct ldb_module *module, struct ldb_request *request)
+ * helper functions to call the next module in chain
+ */
+int partition_request(struct ldb_module *module, struct ldb_request *request)
 {
-       int ret;
-       switch (request->operation) {
-       case LDB_SEARCH:
-               PARTITION_FIND_OP(module, search);
-               ret = module->ops->search(module, request);
-               break;
-       case LDB_ADD:
-               PARTITION_FIND_OP(module, add);
-               ret = module->ops->add(module, request);
-               break;
-       case LDB_MODIFY:
-               PARTITION_FIND_OP(module, modify);
-               ret = module->ops->modify(module, request);
-               break;
-       case LDB_DELETE:
-               PARTITION_FIND_OP(module, del);
-               ret = module->ops->del(module, request);
-               break;
-       case LDB_RENAME:
-               PARTITION_FIND_OP(module, rename);
-               ret = module->ops->rename(module, request);
-               break;
-       case LDB_EXTENDED:
-               PARTITION_FIND_OP(module, extended);
-               ret = module->ops->extended(module, request);
-               break;
-       default:
-               PARTITION_FIND_OP(module, request);
-               ret = module->ops->request(module, request);
-               break;
-       }
-       if (ret == LDB_SUCCESS) {
-               return ret;
-       }
-       if (!ldb_errstring(ldb_module_get_ctx(module))) {
-               /* Set a default error string, to place the blame somewhere */
-               ldb_asprintf_errstring(ldb_module_get_ctx(module),
-                                       "error in module %s: %s (%d)",
-                                       module->ops->name,
-                                       ldb_strerror(ret), ret);
+       if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) { \
+               const struct dsdb_control_current_partition *partition = NULL;
+               struct ldb_control *partition_ctrl = ldb_request_get_control(request, DSDB_CONTROL_CURRENT_PARTITION_OID);
+               if (partition_ctrl) {
+                       partition = talloc_get_type(partition_ctrl->data,
+                                                   struct dsdb_control_current_partition);
+               }
+
+               if (partition != NULL) {
+                       ldb_debug(module->ldb, LDB_DEBUG_TRACE, "partition_request() -> %s", 
+                                 ldb_dn_get_linearized(partition->dn));                        
+               } else {
+                       ldb_debug(module->ldb, LDB_DEBUG_TRACE, "partition_request() -> (metadata partition)");                 
+               }
        }
-       return ret;
+
+       return ldb_next_request(module, request);
 }
 
 static struct dsdb_partition *find_partition(struct partition_private_data *data,
                                             struct ldb_dn *dn,
                                             struct ldb_request *req)
 {
-       int i;
+       unsigned int i;
        struct ldb_control *partition_ctrl;
 
        /* see if the request has the partition DN specified in a
@@ -186,8 +133,9 @@ static int partition_req_callback(struct ldb_request *req,
        struct partition_context *ac;
        struct ldb_module *module;
        struct ldb_request *nreq;
-       int ret, i;
+       int ret;
        struct partition_private_data *data;
+       struct ldb_control *partition_ctrl;
 
        ac = talloc_get_type(req->context, struct partition_context);
        data = talloc_get_type(ac->module->private_data, struct partition_private_data);
@@ -197,7 +145,22 @@ static int partition_req_callback(struct ldb_request *req,
                                        LDB_ERR_OPERATIONS_ERROR);
        }
 
-       if (ares->error != LDB_SUCCESS && !ac->got_success) {
+       partition_ctrl = ldb_request_get_control(req, DSDB_CONTROL_CURRENT_PARTITION_OID);
+       if (partition_ctrl && (ac->num_requests == 1 || ares->type == LDB_REPLY_ENTRY)) {
+               /* If we didn't fan this request out to mulitple partitions,
+                * or this is an individual search result, we can
+                * deterministily tell the caller what partition this was
+                * written to (repl_meta_data likes to know) */
+               ret = ldb_reply_add_control(ares,
+                                           DSDB_CONTROL_CURRENT_PARTITION_OID,
+                                           false, partition_ctrl->data);
+               if (ret != LDB_SUCCESS) {
+                       return ldb_module_done(ac->req, NULL, NULL,
+                                              ret);
+               }
+       }
+
+       if (ares->error != LDB_SUCCESS) {
                return ldb_module_done(ac->req, ares->controls,
                                        ares->response, ares->error);
        }
@@ -215,28 +178,10 @@ static int partition_req_callback(struct ldb_request *req,
                        return ldb_module_done(ac->req, NULL, NULL,
                                                LDB_ERR_OPERATIONS_ERROR);
                }
-               for (i=0; data && data->partitions && data->partitions[i]; i++) {
-                       if (ldb_dn_compare(ares->message->dn, data->partitions[i]->ctrl->dn) == 0) {
-                               struct ldb_control *part_control;
-                               /* this is a partition root message - make
-                                  sure it isn't one of our fake root
-                                  entries from a parent partition */
-                               part_control = ldb_request_get_control(req, DSDB_CONTROL_CURRENT_PARTITION_OID);
-                               if (part_control && part_control->data != data->partitions[i]->ctrl) {
-                                       DEBUG(6,(__location__ ": Discarding partition mount object %s\n",
-                                                ldb_dn_get_linearized(ares->message->dn)));
-                                       talloc_free(ares);
-                                       return LDB_SUCCESS;
-                               }
-                       }
-               }
                
                return ldb_module_send_entry(ac->req, ares->message, ares->controls);
 
        case LDB_REPLY_DONE:
-               if (ares->error == LDB_SUCCESS) {
-                       ac->got_success = true;
-               }
                if (ac->req->operation == LDB_EXTENDED) {
                        /* FIXME: check for ares->response, replmd does not fill it ! */
                        if (ares->response) {
@@ -254,10 +199,23 @@ static int partition_req_callback(struct ldb_request *req,
 
                ac->finished_requests++;
                if (ac->finished_requests == ac->num_requests) {
+                       /* Send back referrals if they do exist (search ops) */
+                       if (ac->referrals != NULL) {
+                               const char **ref;
+                               for (ref = ac->referrals; *ref != NULL; ++ref) {
+                                       ret = ldb_module_send_referral(ac->req,
+                                                                      talloc_strdup(ac->req, *ref));
+                                       if (ret != LDB_SUCCESS) {
+                                               return ldb_module_done(ac->req, NULL, NULL,
+                                                                      ret);
+                                       }
+                               }
+                       }
+
                        /* this was the last one, call callback */
                        return ldb_module_done(ac->req, ares->controls,
                                               ares->response, 
-                                              ac->got_success?LDB_SUCCESS:ares->error);
+                                              ares->error);
                }
 
                /* not the last, now call the next one */
@@ -385,9 +343,9 @@ static int partition_prep_request(struct partition_context *ac,
                }
 
        } else {
-               /* make sure you put the NEXT module here, or
-                * partition_request() will simply loop forever on itself */
-               ac->part_req[ac->num_requests].module = ac->module->next;
+               /* make sure you put the module here, or
+                * or ldb_next_request() will skip a module */
+               ac->part_req[ac->num_requests].module = ac->module;
        }
 
        ac->num_requests++;
@@ -407,7 +365,7 @@ static int partition_send_all(struct ldb_module *module,
                              struct partition_context *ac, 
                              struct ldb_request *req) 
 {
-       int i;
+       unsigned int i;
        struct partition_private_data *data = talloc_get_type(module->private_data, 
                                                              struct partition_private_data);
        int ret = partition_prep_request(ac, NULL);
@@ -432,7 +390,7 @@ static int partition_send_all(struct ldb_module *module,
 static int partition_replicate(struct ldb_module *module, struct ldb_request *req, struct ldb_dn *dn) 
 {
        struct partition_context *ac;
-       unsigned i;
+       unsigned int i;
        int ret;
        struct dsdb_partition *partition;
        struct partition_private_data *data = talloc_get_type(module->private_data, 
@@ -440,8 +398,8 @@ static int partition_replicate(struct ldb_module *module, struct ldb_request *re
        if (!data || !data->partitions) {
                return ldb_next_request(module, req);
        }
-       
-       if (req->operation != LDB_SEARCH) {
+
+       if (req->operation != LDB_SEARCH && ldb_dn_is_special(dn)) {
                /* Is this a special DN, we need to replicate to every backend? */
                for (i=0; data->replicate && data->replicate[i]; i++) {
                        if (ldb_dn_compare(data->replicate[i], 
@@ -495,18 +453,24 @@ static int partition_search(struct ldb_module *module, struct ldb_request *req)
        /* Find backend */
        struct partition_private_data *data = talloc_get_type(module->private_data, 
                                                              struct partition_private_data);
-
-       /* issue request */
-
-       /* (later) consider if we should be searching multiple
-        * partitions (for 'invisible' partition behaviour */
+       struct partition_context *ac;
+       struct ldb_context *ldb;
+       struct loadparm_context *lp_ctx;
 
        struct ldb_control *search_control = ldb_request_get_control(req, LDB_CONTROL_SEARCH_OPTIONS_OID);
        struct ldb_control *domain_scope_control = ldb_request_get_control(req, LDB_CONTROL_DOMAIN_SCOPE_OID);
        
        struct ldb_search_options_control *search_options = NULL;
        struct dsdb_partition *p;
+       unsigned int i, j;
+       int ret;
+       bool domain_scope = false, phantom_root = false;
        
+       ret = partition_reload_if_required(module, data);
+       if (ret != LDB_SUCCESS) {
+               return ret;
+       }
+
        p = find_partition(data, NULL, req);
        if (p != NULL) {
                /* the caller specified what partition they want the
@@ -515,111 +479,179 @@ static int partition_search(struct ldb_module *module, struct ldb_request *req)
                return ldb_next_request(p->module, req);                
        }
 
-
+       /* Get back the search options from the search control, and mark it as
+        * non-critical (to make backends and also dcpromo happy).
+        */
        if (search_control) {
                search_options = talloc_get_type(search_control->data, struct ldb_search_options_control);
+               search_control->critical = 0;
+
        }
 
-       /* Remove the domain_scope control, so we don't confuse a backend server */
+       /* Remove the "domain_scope" control, so we don't confuse a backend
+        * server */
        if (domain_scope_control && !save_controls(domain_scope_control, req, &saved_controls)) {
                ldb_oom(ldb_module_get_ctx(module));
                return LDB_ERR_OPERATIONS_ERROR;
        }
 
-       /*
-        * for now pass down the LDB_CONTROL_SEARCH_OPTIONS_OID control
-        * down as uncritical to make windows 2008 dcpromo happy.
-        */
-       if (search_control) {
-               search_control->critical = 0;
+       /* Locate the options */
+       domain_scope = (search_options
+               && (search_options->search_options & LDB_SEARCH_OPTION_DOMAIN_SCOPE))
+               || domain_scope_control;
+       phantom_root = search_options
+               && (search_options->search_options & LDB_SEARCH_OPTION_PHANTOM_ROOT);
+
+       /* Remove handled options from the search control flag */
+       if (search_options) {
+               search_options->search_options = search_options->search_options
+                       & ~LDB_SEARCH_OPTION_DOMAIN_SCOPE
+                       & ~LDB_SEARCH_OPTION_PHANTOM_ROOT;
        }
 
-       /* TODO:
-          Generate referrals (look for a partition under this DN) if we don't have the above control specified
-       */
-       
-       if (search_options && (search_options->search_options & LDB_SEARCH_OPTION_PHANTOM_ROOT)) {
-               int ret, i;
-               struct partition_context *ac;
-               if ((search_options->search_options & ~LDB_SEARCH_OPTION_PHANTOM_ROOT) == 0) {
-                       /* We have processed this flag, so we are done with this control now */
-
-                       /* Remove search control, so we don't confuse a backend server */
-                       if (search_control && !save_controls(search_control, req, &saved_controls)) {
-                               ldb_oom(ldb_module_get_ctx(module));
-                               return LDB_ERR_OPERATIONS_ERROR;
-                       }
-               }
-               ac = partition_init_ctx(module, req);
-               if (!ac) {
-                       return LDB_ERR_OPERATIONS_ERROR;
-               }
+       if (!data || !data->partitions) {
+               return ldb_next_request(module, req);
+       }
 
-               /* Search from the base DN */
-               if (!req->op.search.base || ldb_dn_is_null(req->op.search.base)) {
-                       return partition_send_all(module, ac, req);
-               }
-               for (i=0; data && data->partitions && data->partitions[i]; i++) {
-                       bool match = false, stop = false;
-                       /* Find all partitions under the search base 
-                          
-                          we match if:
-
-                             1) the DN we are looking for exactly matches the partition
-                            or
-                             2) the DN we are looking for is a parent of the partition and it isn't
-                                 a scope base search
-                             or
-                             3) the DN we are looking for is a child of the partition
+       ac = partition_init_ctx(module, req);
+       if (!ac) {
+               return LDB_ERR_OPERATIONS_ERROR;
+       }
+
+       ldb = ldb_module_get_ctx(ac->module);
+       lp_ctx = talloc_get_type(ldb_get_opaque(ldb, "loadparm"),
+                                               struct loadparm_context);
+
+       /* Search from the base DN */
+       if (ldb_dn_is_null(req->op.search.base)) {
+               return partition_send_all(module, ac, req);
+       }
+
+       for (i=0; data->partitions[i]; i++) {
+               bool match = false, stop = false;
+
+               if (phantom_root) {
+                       /* Phantom root: Find all partitions under the
+                        * search base. We match if:
+                        *
+                        * 1) the DN we are looking for exactly matches a
+                        *    certain partition and always stop
+                        * 2) the DN we are looking for is a parent of certain
+                        *    partitions and it isn't a scope base search
+                        * 3) the DN we are looking for is a child of a certain
+                        *    partition and always stop
+                        *    - we don't need to go any further up in the
+                        *    hierarchy!
                         */
-                       if (ldb_dn_compare(data->partitions[i]->ctrl->dn, req->op.search.base) == 0) {
+                       if (ldb_dn_compare(data->partitions[i]->ctrl->dn,
+                                          req->op.search.base) == 0) {
                                match = true;
-                               if (req->op.search.scope == LDB_SCOPE_BASE) {
-                                       stop = true;
-                               }
+                               stop = true;
                        }
-                       if (!match && 
-                           (ldb_dn_compare_base(req->op.search.base, data->partitions[i]->ctrl->dn) == 0 &&
+                       if (!match &&
+                           (ldb_dn_compare_base(req->op.search.base,
+                                                data->partitions[i]->ctrl->dn) == 0 &&
                             req->op.search.scope != LDB_SCOPE_BASE)) {
                                match = true;
                        }
                        if (!match &&
-                           ldb_dn_compare_base(data->partitions[i]->ctrl->dn, req->op.search.base) == 0) {
+                           ldb_dn_compare_base(data->partitions[i]->ctrl->dn,
+                                               req->op.search.base) == 0) {
                                match = true;
                                stop = true; /* note that this relies on partition ordering */
                        }
-                       if (match) {
-                               ret = partition_prep_request(ac, data->partitions[i]);
-                               if (ret != LDB_SUCCESS) {
-                                       return ret;
+               } else {
+                       /* Domain scope: Find all partitions under the search
+                        * base.
+                        *
+                        * We generate referral candidates if we haven't
+                        * specified the domain scope control, haven't a base
+                        * search* scope and the DN we are looking for is a real
+                        * predecessor of certain partitions. When a new
+                        * referral candidate is nearer to the DN than an
+                        * existing one delete the latter (we want to have only
+                        * the closest ones). When we checked this for all
+                        * candidates we have the final referrals.
+                        *
+                        * We match if the DN we are looking for is a child of
+                        * a certain partition or the partition
+                        * DN itself - we don't need to go any further
+                        * up in the hierarchy!
+                        */
+                       if ((!domain_scope) &&
+                           (req->op.search.scope != LDB_SCOPE_BASE) &&
+                           (ldb_dn_compare_base(req->op.search.base,
+                                                data->partitions[i]->ctrl->dn) == 0) &&
+                           (ldb_dn_compare(req->op.search.base,
+                                           data->partitions[i]->ctrl->dn) != 0)) {
+                               char *ref = talloc_asprintf(ac,
+                                                           "ldap://%s/%s%s",
+                                                           lp_dnsdomain(lp_ctx),
+                                                           ldb_dn_get_linearized(data->partitions[i]->ctrl->dn),
+                                                           req->op.search.scope == LDB_SCOPE_ONELEVEL ? "??base" : "");
+
+                               if (ref == NULL) {
+                                       ldb_oom(ldb);
+                                       return LDB_ERR_OPERATIONS_ERROR;
                                }
-                       }
-                       if (stop) break;
-               }
 
-               /* Perhaps we didn't match any partitions.  Try the main partition, only */
-               if (ac->num_requests == 0) {
-                       talloc_free(ac);
-                       return ldb_next_request(module, req);
-               }
+                               /* Initialise the referrals list */
+                               if (ac->referrals == NULL) {
+                                       ac->referrals = (const char **) str_list_make_empty(ac);
+                                       if (ac->referrals == NULL) {
+                                               ldb_oom(ldb);
+                                               return LDB_ERR_OPERATIONS_ERROR;
+                                       }
+                               }
 
-               /* fire the first one */
-               return partition_call_first(ac);
+                               /* Check if the new referral candidate is
+                                * closer to the base DN than already
+                                * saved ones and delete the latters */
+                               j = 0;
+                               while (ac->referrals[j] != NULL) {
+                                       if (strstr(ac->referrals[j],
+                                                  ldb_dn_get_linearized(data->partitions[i]->ctrl->dn)) != NULL) {
+                                               str_list_remove(ac->referrals,
+                                                               ac->referrals[j]);
+                                       } else {
+                                               ++j;
+                                       }
+                               }
 
-       } else {
-               /* Handle this like all other requests */
-               if (search_control && (search_options->search_options & ~LDB_SEARCH_OPTION_PHANTOM_ROOT) == 0) {
-                       /* We have processed this flag, so we are done with this control now */
+                               /* Add our new candidate */
+                               ac->referrals = str_list_add(ac->referrals, ref);
 
-                       /* Remove search control, so we don't confuse a backend server */
-                       if (search_control && !save_controls(search_control, req, &saved_controls)) {
-                               ldb_oom(ldb_module_get_ctx(module));
-                               return LDB_ERR_OPERATIONS_ERROR;
+                               talloc_free(ref);
+
+                               if (ac->referrals == NULL) {
+                                       ldb_oom(ldb);
+                                       return LDB_ERR_OPERATIONS_ERROR;
+                               }
+                       }
+                       if (ldb_dn_compare_base(data->partitions[i]->ctrl->dn, req->op.search.base) == 0) {
+                               match = true;
+                               stop = true; /* note that this relies on partition ordering */
                        }
                }
 
-               return partition_replicate(module, req, req->op.search.base);
+               if (match) {
+                       ret = partition_prep_request(ac, data->partitions[i]);
+                       if (ret != LDB_SUCCESS) {
+                               return ret;
+                       }
+               }
+
+               if (stop) break;
        }
+
+       /* Perhaps we didn't match any partitions. Try the main partition */
+       if (ac->num_requests == 0) {
+               talloc_free(ac);
+               return ldb_next_request(module, req);
+       }
+
+       /* fire the first one */
+       return partition_call_first(ac);
 }
 
 /* add */
@@ -678,59 +710,73 @@ static int partition_rename(struct ldb_module *module, struct ldb_request *req)
 /* start a transaction */
 static int partition_start_trans(struct ldb_module *module)
 {
-       int i, ret;
+       unsigned int i;
+       int ret;
        struct partition_private_data *data = talloc_get_type(module->private_data, 
                                                              struct partition_private_data);
        /* Look at base DN */
        /* Figure out which partition it is under */
        /* Skip the lot if 'data' isn't here yet (initialization) */
+       if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+               ldb_debug(module->ldb, LDB_DEBUG_TRACE, "partition_start_trans() -> (metadata partition)");
+       }
        ret = ldb_next_start_trans(module);
        if (ret != LDB_SUCCESS) {
                return ret;
        }
 
-       for (i=0; data && data->partitions && data->partitions[i]; i++) {
-               struct ldb_module *next = data->partitions[i]->module;
-               PARTITION_FIND_OP(next, start_transaction);
+       ret = partition_reload_if_required(module, data);
+       if (ret != LDB_SUCCESS) {
+               return ret;
+       }
 
-               ret = next->ops->start_transaction(next);
+       for (i=0; data && data->partitions && data->partitions[i]; i++) {
+               if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+                       ldb_debug(module->ldb, LDB_DEBUG_TRACE, "partition_start_trans() -> %s", 
+                                 ldb_dn_get_linearized(data->partitions[i]->ctrl->dn));
+               }
+               ret = ldb_next_start_trans(data->partitions[i]->module);
                if (ret != LDB_SUCCESS) {
                        /* Back it out, if it fails on one */
                        for (i--; i >= 0; i--) {
-                               next = data->partitions[i]->module;
-                               PARTITION_FIND_OP(next, del_transaction);
-
-                               next->ops->del_transaction(next);
+                               ldb_next_del_trans(data->partitions[i]->module);
                        }
                        ldb_next_del_trans(module);
                        return ret;
                }
        }
+
+       data->in_transaction++;
+
        return LDB_SUCCESS;
 }
 
 /* prepare for a commit */
 static int partition_prepare_commit(struct ldb_module *module)
 {
-       int i;
+       unsigned int i;
        struct partition_private_data *data = talloc_get_type(module->private_data, 
                                                              struct partition_private_data);
 
        for (i=0; data && data->partitions && data->partitions[i]; i++) {
-               struct ldb_module *next_prepare = data->partitions[i]->module;
                int ret;
 
-               PARTITION_FIND_OP_NOERROR(next_prepare, prepare_commit);
-               if (next_prepare == NULL) {
-                       continue;
+               if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+                       ldb_debug(module->ldb, LDB_DEBUG_TRACE, "partition_prepare_commit() -> %s", 
+                                 ldb_dn_get_linearized(data->partitions[i]->ctrl->dn));
                }
-
-               ret = next_prepare->ops->prepare_commit(next_prepare);
+               ret = ldb_next_prepare_commit(data->partitions[i]->module);
                if (ret != LDB_SUCCESS) {
+                       ldb_asprintf_errstring(module->ldb, "prepare_commit error on %s: %s",
+                                              ldb_dn_get_linearized(data->partitions[i]->ctrl->dn),
+                                              ldb_errstring(module->ldb));
                        return ret;
                }
        }
 
+       if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+               ldb_debug(module->ldb, LDB_DEBUG_TRACE, "partition_prepare_commit() -> (metadata partition)");
+       }
        return ldb_next_prepare_commit(module);
 }
 
@@ -738,40 +784,74 @@ static int partition_prepare_commit(struct ldb_module *module)
 /* end a transaction */
 static int partition_end_trans(struct ldb_module *module)
 {
-       int i;
+       int ret, ret2;
+       unsigned int i;
        struct partition_private_data *data = talloc_get_type(module->private_data, 
                                                              struct partition_private_data);
-       for (i=0; data && data->partitions && data->partitions[i]; i++) {
-               struct ldb_module *next_end = data->partitions[i]->module;
-               int ret;
 
-               PARTITION_FIND_OP(next_end, end_transaction);
+       ret = LDB_SUCCESS;
 
-               ret = next_end->ops->end_transaction(next_end);
-               if (ret != LDB_SUCCESS) {
-                       return ret;
+       if (data->in_transaction == 0) {
+               DEBUG(0,("partition end transaction mismatch\n"));
+               ret = LDB_ERR_OPERATIONS_ERROR;
+       } else {
+               data->in_transaction--;
+       }
+
+       for (i=0; data && data->partitions && data->partitions[i]; i++) {
+               if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+                       ldb_debug(module->ldb, LDB_DEBUG_TRACE, "partition_end_trans() -> %s", 
+                                 ldb_dn_get_linearized(data->partitions[i]->ctrl->dn));
+               }
+               ret2 = ldb_next_end_trans(data->partitions[i]->module);
+               if (ret2 != LDB_SUCCESS) {
+                       ldb_asprintf_errstring(module->ldb, "end_trans error on %s: %s",
+                                              ldb_dn_get_linearized(data->partitions[i]->ctrl->dn),
+                                              ldb_errstring(module->ldb));
+                       ret = ret2;
                }
        }
 
-       return ldb_next_end_trans(module);
+       if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+               ldb_debug(module->ldb, LDB_DEBUG_TRACE, "partition_end_trans() -> (metadata partition)");
+       }
+       ret2 = ldb_next_end_trans(module);
+       if (ret2 != LDB_SUCCESS) {
+               ret = ret2;
+       }
+       return ret;
 }
 
 /* delete a transaction */
 static int partition_del_trans(struct ldb_module *module)
 {
-       int i, ret, final_ret = LDB_SUCCESS;
+       int ret, final_ret = LDB_SUCCESS;
+       unsigned int i;
        struct partition_private_data *data = talloc_get_type(module->private_data, 
                                                              struct partition_private_data);
        for (i=0; data && data->partitions && data->partitions[i]; i++) {
-               struct ldb_module *next = data->partitions[i]->module;
-               PARTITION_FIND_OP(next, del_transaction);
-
-               ret = next->ops->del_transaction(next);
+               if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+                       ldb_debug(module->ldb, LDB_DEBUG_TRACE, "partition_del_trans() -> %s", 
+                                 ldb_dn_get_linearized(data->partitions[i]->ctrl->dn));
+               }
+               ret = ldb_next_del_trans(data->partitions[i]->module);
                if (ret != LDB_SUCCESS) {
+                       ldb_asprintf_errstring(module->ldb, "del_trans error on %s: %s",
+                                              ldb_dn_get_linearized(data->partitions[i]->ctrl->dn),
+                                              ldb_errstring(module->ldb));
                        final_ret = ret;
                }
        }       
 
+       if (data->in_transaction == 0) {
+               DEBUG(0,("partition del transaction mismatch\n"));
+               return LDB_ERR_OPERATIONS_ERROR;
+       }
+       data->in_transaction--;
+
+       if ((module && module->ldb->flags & LDB_FLG_ENABLE_TRACING)) {
+               ldb_debug(module->ldb, LDB_DEBUG_TRACE, "partition_del_trans() -> (metadata partition)");
+       }
        ret = ldb_next_del_trans(module);
        if (ret != LDB_SUCCESS) {
                final_ret = ret;
@@ -779,11 +859,67 @@ static int partition_del_trans(struct ldb_module *module)
        return final_ret;
 }
 
+int partition_primary_sequence_number(struct ldb_module *module, TALLOC_CTX *mem_ctx, 
+                                    enum ldb_sequence_type type, uint64_t *seq_number) 
+{
+       int ret;
+       struct ldb_result *res;
+       struct ldb_seqnum_request *tseq;
+       struct ldb_request *treq;
+       struct ldb_seqnum_result *seqr;
+       res = talloc_zero(mem_ctx, struct ldb_result);
+       if (res == NULL) {
+               return LDB_ERR_OPERATIONS_ERROR;
+       }
+       tseq = talloc_zero(res, struct ldb_seqnum_request);
+       if (tseq == NULL) {
+               talloc_free(res);
+               return LDB_ERR_OPERATIONS_ERROR;
+       }
+       tseq->type = type;
+       
+       ret = ldb_build_extended_req(&treq, ldb_module_get_ctx(module), res,
+                                    LDB_EXTENDED_SEQUENCE_NUMBER,
+                                    tseq,
+                                    NULL,
+                                    res,
+                                    ldb_extended_default_callback,
+                                    NULL);
+       if (ret != LDB_SUCCESS) {
+               talloc_free(res);
+               return ret;
+       }
+       
+       ret = ldb_next_request(module, treq);
+       if (ret != LDB_SUCCESS) {
+               talloc_free(res);
+               return ret;
+       }
+       ret = ldb_wait(treq->handle, LDB_WAIT_ALL);
+       if (ret != LDB_SUCCESS) {
+               talloc_free(res);
+               return ret;
+       }
+       
+       seqr = talloc_get_type(res->extended->data,
+                              struct ldb_seqnum_result);
+       if (seqr->flags & LDB_SEQ_TIMESTAMP_SEQUENCE) {
+               ret = LDB_ERR_OPERATIONS_ERROR;
+               ldb_set_errstring(ldb_module_get_ctx(module), "Primary backend in partitions module returned a timestamp based seq number (must return a normal number)");
+               talloc_free(res);
+               return ret;
+       } else {
+               *seq_number = seqr->seq_num;
+       }
+       talloc_free(res);
+       return LDB_SUCCESS;
+}
 
 /* FIXME: This function is still semi-async */
 static int partition_sequence_number(struct ldb_module *module, struct ldb_request *req)
 {
-       int i, ret;
+       int ret;
+       unsigned int i;
        uint64_t seq_number = 0;
        uint64_t timestamp_sequence = 0;
        uint64_t timestamp = 0;
@@ -811,49 +947,12 @@ static int partition_sequence_number(struct ldb_module *module, struct ldb_reque
        switch (seq->type) {
        case LDB_SEQ_NEXT:
        case LDB_SEQ_HIGHEST_SEQ:
-               res = talloc_zero(req, struct ldb_result);
-               if (res == NULL) {
-                       return LDB_ERR_OPERATIONS_ERROR;
-               }
-               tseq = talloc_zero(res, struct ldb_seqnum_request);
-               if (tseq == NULL) {
-                       talloc_free(res);
-                       return LDB_ERR_OPERATIONS_ERROR;
-               }
-               tseq->type = seq->type;
-
-               ret = ldb_build_extended_req(&treq, ldb_module_get_ctx(module), res,
-                                            LDB_EXTENDED_SEQUENCE_NUMBER,
-                                            tseq,
-                                            NULL,
-                                            res,
-                                            ldb_extended_default_callback,
-                                            NULL);
-               if (ret != LDB_SUCCESS) {
-                       talloc_free(res);
-                       return ret;
-               }
 
-               ret = ldb_next_request(module, treq);
-               if (ret != LDB_SUCCESS) {
-                       talloc_free(res);
-                       return ret;
-               }
-               ret = ldb_wait(treq->handle, LDB_WAIT_ALL);
+               ret = partition_primary_sequence_number(module, req, seq->type, &seq_number);
                if (ret != LDB_SUCCESS) {
-                       talloc_free(res);
                        return ret;
                }
 
-               seqr = talloc_get_type(res->extended->data,
-                                       struct ldb_seqnum_result);
-               if (seqr->flags & LDB_SEQ_TIMESTAMP_SEQUENCE) {
-                       timestamp_sequence = seqr->seq_num;
-               } else {
-                       seq_number += seqr->seq_num;
-               }
-               talloc_free(res);
-
                /* Skip the lot if 'data' isn't here yet (initialisation) */
                for (i=0; data && data->partitions && data->partitions[i]; i++) {
 
@@ -1055,71 +1154,30 @@ static int partition_sequence_number(struct ldb_module *module, struct ldb_reque
        return ldb_module_done(req, NULL, ext, LDB_SUCCESS);
 }
 
-static int partition_extended_schema_update_now(struct ldb_module *module, struct ldb_request *req)
+/* extended */
+static int partition_extended(struct ldb_module *module, struct ldb_request *req)
 {
-       struct dsdb_partition *partition;
        struct partition_private_data *data;
-       struct ldb_dn *schema_dn;
        struct partition_context *ac;
        int ret;
 
-       schema_dn = talloc_get_type(req->op.extended.data, struct ldb_dn);
-       if (!schema_dn) {
-               ldb_debug(ldb_module_get_ctx(module), LDB_DEBUG_FATAL, "partition_extended: invalid extended data\n");
-               return LDB_ERR_PROTOCOL_ERROR;
-       }
-
        data = talloc_get_type(module->private_data, struct partition_private_data);
        if (!data) {
-               return LDB_ERR_OPERATIONS_ERROR;
-       }
-       
-       partition = find_partition( data, schema_dn, req);
-       if (!partition) {
                return ldb_next_request(module, req);
        }
 
-       ac = partition_init_ctx(module, req);
-       if (!ac) {
-               return LDB_ERR_OPERATIONS_ERROR;
-       }
-
-       /* we need to add a control but we never touch the original request */
-       ret = partition_prep_request(ac, partition);
+       ret = partition_reload_if_required(module, data);
        if (ret != LDB_SUCCESS) {
                return ret;
        }
-
-       /* fire the first one */
-       ret = partition_call_first(ac);
-
-       if (ret != LDB_SUCCESS){
-               return ret;
-       }
-
-       return ldb_request_done(req, ret);
-}
-
-
-/* extended */
-static int partition_extended(struct ldb_module *module, struct ldb_request *req)
-{
-       struct partition_private_data *data;
-       struct partition_context *ac;
-
-       data = talloc_get_type(module->private_data, struct partition_private_data);
-       if (!data || !data->partitions) {
-               return ldb_next_request(module, req);
-       }
-
+       
        if (strcmp(req->op.extended.oid, LDB_EXTENDED_SEQUENCE_NUMBER) == 0) {
                return partition_sequence_number(module, req);
        }
 
-       /* forward schemaUpdateNow operation to schema_fsmo module*/
-       if (strcmp(req->op.extended.oid, DSDB_EXTENDED_SCHEMA_UPDATE_NOW_OID) == 0) {
-               return partition_extended_schema_update_now( module, req );
-       }       
+       if (strcmp(req->op.extended.oid, DSDB_EXTENDED_CREATE_PARTITION_OID) == 0) {
+               return partition_create(module, req);
+       }
 
        /* 
         * as the extended operation has no dn
@@ -1134,279 +1192,6 @@ static int partition_extended(struct ldb_module *module, struct ldb_request *req
        return partition_send_all(module, ac, req);
 }
 
-static int partition_sort_compare(const void *v1, const void *v2)
-{
-       const struct dsdb_partition *p1;
-       const struct dsdb_partition *p2;
-
-       p1 = *((struct dsdb_partition * const*)v1);
-       p2 = *((struct dsdb_partition * const*)v2);
-
-       return ldb_dn_compare(p1->ctrl->dn, p2->ctrl->dn);
-}
-
-static int partition_init(struct ldb_module *module)
-{
-       int ret, i;
-       TALLOC_CTX *mem_ctx = talloc_new(module);
-       const char *attrs[] = { "partition", "replicateEntries", "modules", NULL };
-       struct ldb_result *res;
-       struct ldb_message *msg;
-       struct ldb_message_element *partition_attributes;
-       struct ldb_message_element *replicate_attributes;
-       struct ldb_message_element *modules_attributes;
-
-       struct partition_private_data *data;
-
-       if (!mem_ctx) {
-               return LDB_ERR_OPERATIONS_ERROR;
-       }
-
-       data = talloc(mem_ctx, struct partition_private_data);
-       if (data == NULL) {
-               return LDB_ERR_OPERATIONS_ERROR;
-       }
-
-       ret = ldb_search(ldb_module_get_ctx(module), mem_ctx, &res,
-                        ldb_dn_new(mem_ctx, ldb_module_get_ctx(module), "@PARTITION"),
-                        LDB_SCOPE_BASE, attrs, NULL);
-       if (ret != LDB_SUCCESS) {
-               talloc_free(mem_ctx);
-               return ret;
-       }
-       if (res->count == 0) {
-               talloc_free(mem_ctx);
-               return ldb_next_init(module);
-       }
-
-       if (res->count > 1) {
-               talloc_free(mem_ctx);
-               return LDB_ERR_CONSTRAINT_VIOLATION;
-       }
-
-       msg = res->msgs[0];
-
-       partition_attributes = ldb_msg_find_element(msg, "partition");
-       if (!partition_attributes) {
-               ldb_set_errstring(ldb_module_get_ctx(module), "partition_init: no partitions specified");
-               talloc_free(mem_ctx);
-               return LDB_ERR_CONSTRAINT_VIOLATION;
-       }
-       data->partitions = talloc_array(data, struct dsdb_partition *, partition_attributes->num_values + 1);
-       if (!data->partitions) {
-               talloc_free(mem_ctx);
-               return LDB_ERR_OPERATIONS_ERROR;
-       }
-       for (i=0; i < partition_attributes->num_values; i++) {
-               char *base = talloc_strdup(data->partitions, (char *)partition_attributes->values[i].data);
-               char *p = strchr(base, ':');
-               const char *backend;
-
-               if (!p) {
-                       ldb_asprintf_errstring(ldb_module_get_ctx(module), 
-                                               "partition_init: "
-                                               "invalid form for partition record (missing ':'): %s", base);
-                       talloc_free(mem_ctx);
-                       return LDB_ERR_CONSTRAINT_VIOLATION;
-               }
-               p[0] = '\0';
-               p++;
-               if (!p[0]) {
-                       ldb_asprintf_errstring(ldb_module_get_ctx(module), 
-                                               "partition_init: "
-                                               "invalid form for partition record (missing backend database): %s", base);
-                       talloc_free(mem_ctx);
-                       return LDB_ERR_CONSTRAINT_VIOLATION;
-               }
-               data->partitions[i] = talloc(data->partitions, struct dsdb_partition);
-               if (!data->partitions[i]) {
-                       talloc_free(mem_ctx);
-                       return LDB_ERR_OPERATIONS_ERROR;
-               }
-               data->partitions[i]->ctrl = talloc(data->partitions[i], struct dsdb_control_current_partition);
-               if (!data->partitions[i]->ctrl) {
-                       talloc_free(mem_ctx);
-                       return LDB_ERR_OPERATIONS_ERROR;
-               }
-               data->partitions[i]->ctrl->version = DSDB_CONTROL_CURRENT_PARTITION_VERSION;
-               data->partitions[i]->ctrl->dn = ldb_dn_new(data->partitions[i], ldb_module_get_ctx(module), base);
-               if (!data->partitions[i]->ctrl->dn) {
-                       ldb_asprintf_errstring(ldb_module_get_ctx(module), 
-                                               "partition_init: invalid DN in partition record: %s", base);
-                       talloc_free(mem_ctx);
-                       return LDB_ERR_CONSTRAINT_VIOLATION;
-               }
-
-               backend = samdb_relative_path(ldb_module_get_ctx(module), 
-                                                                  data->partitions[i], 
-                                                                  p);
-               if (!backend) {
-                       ldb_asprintf_errstring(ldb_module_get_ctx(module), 
-                                               "partition_init: unable to determine an relative path for partition: %s", base);
-                       talloc_free(mem_ctx);                   
-               }
-               ret = ldb_connect_backend(ldb_module_get_ctx(module), backend, NULL, &data->partitions[i]->module);
-               if (ret != LDB_SUCCESS) {
-                       talloc_free(mem_ctx);
-                       return ret;
-               }
-       }
-       data->partitions[i] = NULL;
-
-       /* sort these into order, most to least specific */
-       qsort(data->partitions, partition_attributes->num_values,
-             sizeof(*data->partitions), partition_sort_compare);
-
-       for (i=0; data->partitions[i]; i++) {
-               struct ldb_request *req;
-               req = talloc_zero(mem_ctx, struct ldb_request);
-               if (req == NULL) {
-                       ldb_debug(ldb_module_get_ctx(module), LDB_DEBUG_ERROR, "partition: Out of memory!\n");
-                       talloc_free(mem_ctx);
-                       return LDB_ERR_OPERATIONS_ERROR;
-               }
-               
-               req->operation = LDB_REQ_REGISTER_PARTITION;
-               req->op.reg_partition.dn = data->partitions[i]->ctrl->dn;
-               req->callback = ldb_op_default_callback;
-
-               ldb_set_timeout(ldb_module_get_ctx(module), req, 0);
-
-               req->handle = ldb_handle_new(req, ldb_module_get_ctx(module));
-               if (req->handle == NULL) {
-                       return LDB_ERR_OPERATIONS_ERROR;
-               }
-               
-               ret = ldb_request(ldb_module_get_ctx(module), req);
-               if (ret == LDB_SUCCESS) {
-                       ret = ldb_wait(req->handle, LDB_WAIT_ALL);
-               }
-               if (ret != LDB_SUCCESS) {
-                       ldb_debug(ldb_module_get_ctx(module), LDB_DEBUG_ERROR, "partition: Unable to register partition with rootdse!\n");
-                       talloc_free(mem_ctx);
-                       return LDB_ERR_OTHER;
-               }
-               talloc_free(req);
-       }
-
-       replicate_attributes = ldb_msg_find_element(msg, "replicateEntries");
-       if (!replicate_attributes) {
-               data->replicate = NULL;
-       } else {
-               data->replicate = talloc_array(data, struct ldb_dn *, replicate_attributes->num_values + 1);
-               if (!data->replicate) {
-                       talloc_free(mem_ctx);
-                       return LDB_ERR_OPERATIONS_ERROR;
-               }
-
-               for (i=0; i < replicate_attributes->num_values; i++) {
-                       data->replicate[i] = ldb_dn_from_ldb_val(data->replicate, ldb_module_get_ctx(module), &replicate_attributes->values[i]);
-                       if (!ldb_dn_validate(data->replicate[i])) {
-                               ldb_asprintf_errstring(ldb_module_get_ctx(module), 
-                                                       "partition_init: "
-                                                       "invalid DN in partition replicate record: %s", 
-                                                       replicate_attributes->values[i].data);
-                               talloc_free(mem_ctx);
-                               return LDB_ERR_CONSTRAINT_VIOLATION;
-                       }
-               }
-               data->replicate[i] = NULL;
-       }
-
-       /* Make the private data available to any searches the modules may trigger in initialisation */
-       module->private_data = data;
-       talloc_steal(module, data);
-       
-       modules_attributes = ldb_msg_find_element(msg, "modules");
-       if (modules_attributes) {
-               for (i=0; i < modules_attributes->num_values; i++) {
-                       struct ldb_dn *base_dn;
-                       int partition_idx;
-                       struct dsdb_partition *partition = NULL;
-                       const char **modules = NULL;
-
-                       char *base = talloc_strdup(data->partitions, (char *)modules_attributes->values[i].data);
-                       char *p = strchr(base, ':');
-                       if (!p) {
-                               ldb_asprintf_errstring(ldb_module_get_ctx(module), 
-                                                       "partition_init: "
-                                                       "invalid form for partition module record (missing ':'): %s", base);
-                               talloc_free(mem_ctx);
-                               return LDB_ERR_CONSTRAINT_VIOLATION;
-                       }
-                       p[0] = '\0';
-                       p++;
-                       if (!p[0]) {
-                               ldb_asprintf_errstring(ldb_module_get_ctx(module), 
-                                                       "partition_init: "
-                                                       "invalid form for partition module record (missing backend database): %s", base);
-                               talloc_free(mem_ctx);
-                               return LDB_ERR_CONSTRAINT_VIOLATION;
-                       }
-
-                       modules = ldb_modules_list_from_string(ldb_module_get_ctx(module), mem_ctx,
-                                                              p);
-                       
-                       base_dn = ldb_dn_new(mem_ctx, ldb_module_get_ctx(module), base);
-                       if (!ldb_dn_validate(base_dn)) {
-                               talloc_free(mem_ctx);
-                               return LDB_ERR_OPERATIONS_ERROR;
-                       }
-                       
-                       for (partition_idx = 0; data->partitions[partition_idx]; partition_idx++) {
-                               if (ldb_dn_compare(data->partitions[partition_idx]->ctrl->dn, base_dn) == 0) {
-                                       partition = data->partitions[partition_idx];
-                                       break;
-                               }
-                       }
-                       
-                       if (!partition) {
-                               ldb_asprintf_errstring(ldb_module_get_ctx(module), 
-                                                       "partition_init: "
-                                                       "invalid form for partition module record (no such partition): %s", base);
-                               talloc_free(mem_ctx);
-                               return LDB_ERR_CONSTRAINT_VIOLATION;
-                       }
-                       
-                       ret = ldb_load_modules_list(ldb_module_get_ctx(module), modules, partition->module, &partition->module);
-                       if (ret != LDB_SUCCESS) {
-                               ldb_asprintf_errstring(ldb_module_get_ctx(module), 
-                                                      "partition_init: "
-                                                      "loading backend for %s failed: %s", 
-                                                      base, ldb_errstring(ldb_module_get_ctx(module)));
-                               talloc_free(mem_ctx);
-                               return ret;
-                       }
-                       ret = ldb_init_module_chain(ldb_module_get_ctx(module), partition->module);
-                       if (ret != LDB_SUCCESS) {
-                               ldb_asprintf_errstring(ldb_module_get_ctx(module), 
-                                                      "partition_init: "
-                                                      "initialising backend for %s failed: %s", 
-                                                      base, ldb_errstring(ldb_module_get_ctx(module)));
-                               talloc_free(mem_ctx);
-                               return ret;
-                       }
-               }
-       }
-
-       ret = ldb_mod_register_control(module, LDB_CONTROL_DOMAIN_SCOPE_OID);
-       if (ret != LDB_SUCCESS) {
-               ldb_debug(ldb_module_get_ctx(module), LDB_DEBUG_ERROR,
-                       "partition: Unable to register control with rootdse!\n");
-               return ret;
-       }
-
-       ret = ldb_mod_register_control(module, LDB_CONTROL_SEARCH_OPTIONS_OID);
-       if (ret != LDB_SUCCESS) {
-               ldb_debug(ldb_module_get_ctx(module), LDB_DEBUG_ERROR,
-                       "partition: Unable to register control with rootdse!\n");
-               return ret;
-       }
-
-       talloc_free(mem_ctx);
-       return ldb_next_init(module);
-}
-
 _PUBLIC_ const struct ldb_module_ops ldb_partition_module_ops = {
        .name              = "partition",
        .init_context      = partition_init,