s4-ldbwrap: added re-use of ldb contexts in ldb_wrap_connect()
authorAndrew Tridgell <tridge@samba.org>
Fri, 23 Oct 2009 03:27:00 +0000 (14:27 +1100)
committerAndrew Tridgell <tridge@samba.org>
Fri, 23 Oct 2009 03:52:17 +0000 (14:52 +1100)
This allows us to reuse a ldb context if it is open twice, instead
of going through the expensive process of a full ldb open. We can
reuse it if all of the parameters are the same.

The change relies on callers using talloc_unlink() or free of a parent
to close a ldb context.

20 files changed:
source4/auth/gensec/schannel.c
source4/dsdb/samdb/samdb_privilege.c
source4/ldap_server/ldap_server.c
source4/lib/ldb_wrap.c
source4/lib/ldb_wrap.h
source4/lib/registry/ldb.c
source4/libgpo/ads_convenience.c
source4/libnet/libnet_become_dc.c
source4/libnet/libnet_join.c
source4/libnet/libnet_samsync_ldb.c
source4/libnet/libnet_unbecome_dc.c
source4/nbt_server/wins/winsdb.c
source4/rpc_server/netlogon/dcerpc_netlogon.c
source4/rpc_server/samr/dcesrv_samr.c
source4/torture/ldap/ldap_sort.c
source4/torture/ldap/schema.c
source4/torture/ldap/uptodatevector.c
source4/torture/local/dbspeed.c
source4/torture/rpc/netlogon.c
source4/winbind/idmap.c

index 2fe97fede47abd9b6b257ab0289f8e253ddd1c59..58cbb6a593c58c6eecee9b794a7923943dde7eb5 100644 (file)
@@ -146,7 +146,7 @@ static NTSTATUS schannel_update(struct gensec_security *gensec_security, TALLOC_
                /* pull the session key for this client */
                status = schannel_fetch_session_key_ldb(schannel_ldb,
                                                        out_mem_ctx, workstation, &creds);
-               talloc_free(schannel_ldb);
+               talloc_unlink(out_mem_ctx, schannel_ldb);
                if (!NT_STATUS_IS_OK(status)) {
                        DEBUG(3, ("Could not find session key for attempted schannel connection from %s: %s\n",
                                  workstation, nt_errstr(status)));
index e7742c681cf6fab7338f133662ac64d9156c4f44..dcb96d8c0e84b9f6bf358af2e9eb23c40bfc10e2 100644 (file)
@@ -40,7 +40,7 @@ struct ldb_context *privilege_connect(TALLOC_CTX *mem_ctx,
        if (!path) return NULL;
 
        pdb = ldb_wrap_connect(mem_ctx, ev_ctx, lp_ctx, path, 
-                              NULL, NULL, 0, NULL);
+                              NULL, NULL, 0);
        talloc_free(path);
 
        return pdb;
index 0255c72086c05e7cf8ab78864d1d31fff0a5e1d4..c45a4168f91c9397b1667e2d0fb609803d08d3cf 100644 (file)
@@ -548,7 +548,7 @@ static NTSTATUS add_socket(struct tevent_context *event_context,
 
        /* And once we are bound, free the tempoary ldb, it will
         * connect again on each incoming LDAP connection */
-       talloc_free(ldb);
+       talloc_unlink(ldap_service, ldb);
 
        return status;
 }
index 74502afde283684c9c8282b9853b711dd8c16205..0427a9c37838ab2955bd1cb80e8288ea5049ee6b 100644 (file)
@@ -3,7 +3,7 @@
 
    LDB wrap functions
 
-   Copyright (C) Andrew Tridgell 2004
+   Copyright (C) Andrew Tridgell 2004-2009
    
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -34,6 +34,7 @@
 #include "ldb_wrap.h"
 #include "dsdb/samdb/samdb.h"
 #include "param/param.h"
+#include "dlinklist.h"
 
 /*
   this is used to catch debug messages from ldb
@@ -67,26 +68,53 @@ static void ldb_wrap_debug(void *context, enum ldb_debug_level level,
        free(s);
 }
 
-/* check for memory leaks on the ldb context */
-static int ldb_wrap_destructor(struct ldb_context *ldb)
+
+/*
+  connecting to a ldb can be a relatively expensive operation because
+  of the schema and partition loads. We keep a list of open ldb
+  contexts here, and try to re-use when possible. 
+
+  This means callers of ldb_wrap_connect() must use talloc_unlink() or
+  the free of a parent to destroy the context
+ */
+static struct ldb_wrap {
+       struct ldb_wrap *next, *prev;
+       struct ldb_wrap_context {
+               /* the context is what we use to tell if two ldb
+                * connections are exactly equivalent 
+                */              
+               const char *url;
+               struct tevent_context *ev;
+               struct loadparm_context *lp_ctx;
+               struct auth_session_info *session_info;
+               struct cli_credentials *credentials;
+               unsigned int flags;
+       } context;
+       struct ldb_context *ldb;
+} *ldb_wrap_list;
+
+/*
+  see if two database opens are equivalent
+ */
+static bool ldb_wrap_same_context(const struct ldb_wrap_context *c1,
+                                 const struct ldb_wrap_context *c2)
 {
-       size_t *startup_blocks = (size_t *)ldb_get_opaque(ldb, "startup_blocks");
-
-       if (startup_blocks &&
-           talloc_total_blocks(ldb) > *startup_blocks + 400) {
-               DEBUG(0,("WARNING: probable memory leak in ldb %s - %lu blocks (startup %lu) %lu bytes\n",
-                        (char *)ldb_get_opaque(ldb, "wrap_url"), 
-                        (unsigned long)talloc_total_blocks(ldb), 
-                        (unsigned long)*startup_blocks,
-                        (unsigned long)talloc_total_size(ldb)));
-#if 0
-               talloc_report_full(ldb, stdout);
-               call_backtrace();
-               smb_panic("probable memory leak in ldb");
-#endif
-       }
+       return (c1->ev == c2->ev &&
+               c1->lp_ctx == c2->lp_ctx &&
+               c1->session_info == c2->session_info &&
+               c1->credentials == c2->credentials &&
+               c1->flags == c2->flags &&
+               (c1->url == c2->url || strcmp(c1->url, c2->url) == 0));
+}
+
+/* 
+   free a ldb_wrap structure
+ */
+static int ldb_wrap_destructor(struct ldb_wrap *w)
+{
+       DLIST_REMOVE(ldb_wrap_list, w);
        return 0;
-}                               
+}
 
 /*
   wrapped connection to a ldb database
@@ -100,13 +128,27 @@ struct ldb_context *ldb_wrap_connect(TALLOC_CTX *mem_ctx,
                                     const char *url,
                                     struct auth_session_info *session_info,
                                     struct cli_credentials *credentials,
-                                    unsigned int flags,
-                                    const char *options[])
+                                    unsigned int flags)
 {
        struct ldb_context *ldb;
        int ret;
        char *real_url = NULL;
-       size_t *startup_blocks;
+       struct ldb_wrap *w;
+       struct ldb_wrap_context c;
+
+       c.url          = url;
+       c.ev           = ev;
+       c.lp_ctx       = lp_ctx;
+       c.session_info = session_info;
+       c.credentials  = credentials;
+       c.flags        = flags;
+
+       /* see if we can re-use an existing ldb */
+       for (w=ldb_wrap_list; w; w=w->next) {
+               if (ldb_wrap_same_context(&c, &w->context)) {
+                       return talloc_reference(mem_ctx, w->ldb);
+               }
+       }
 
        /* we want to use the existing event context if possible. This
           relies on the fact that in smbd, everything is a child of
@@ -178,7 +220,7 @@ struct ldb_context *ldb_wrap_connect(TALLOC_CTX *mem_ctx,
           ldb_wrap_connect() */
        ldb_set_create_perms(ldb, 0600);
        
-       ret = ldb_connect(ldb, real_url, flags, options);
+       ret = ldb_connect(ldb, real_url, flags, NULL);
        if (ret != LDB_SUCCESS) {
                talloc_free(ldb);
                return NULL;
@@ -186,11 +228,38 @@ struct ldb_context *ldb_wrap_connect(TALLOC_CTX *mem_ctx,
 
        /* setup for leak detection */
        ldb_set_opaque(ldb, "wrap_url", real_url);
-       startup_blocks = talloc(ldb, size_t);
-       *startup_blocks = talloc_total_blocks(ldb);
-       ldb_set_opaque(ldb, "startup_blocks", startup_blocks);
        
-       talloc_set_destructor(ldb, ldb_wrap_destructor);
+       /* add to the list of open ldb contexts */
+       w = talloc(ldb, struct ldb_wrap);
+       if (w == NULL) {
+               talloc_free(ldb);
+               return NULL;
+       }
+
+       w->context = c;
+       w->context.url = talloc_strdup(w, url);
+       if (w->context.url == NULL) {
+               talloc_free(ldb);
+               return NULL;
+       }
+
+       w->ldb = ldb;
+
+       DLIST_ADD(ldb_wrap_list, w);
+
+       DEBUG(3,("ldb_wrap open of %s\n", url));
+
+       talloc_set_destructor(w, ldb_wrap_destructor);
 
        return ldb;
 }
+
+/*
+  when we fork() we need to make sure that any open ldb contexts have
+  any open transactions cancelled
+ */
+void ldb_wrap_fork_hook(void)
+{
+       
+}
+
index f44ff8c5992fe94a947f3355bc68c4e29221c0f2..650f97d17d811c287e628c412e348370ab6d014d 100644 (file)
@@ -37,7 +37,7 @@ struct ldb_context *ldb_wrap_connect(TALLOC_CTX *mem_ctx,
                                     const char *url,
                                     struct auth_session_info *session_info,
                                     struct cli_credentials *credentials,
-                                    unsigned int flags,
-                                    const char *options[]);
+                                    unsigned int flags);
 
+void ldb_wrap_fork_hook(void);
 #endif /* _LDB_WRAP_H_ */
index c558805e049e11a2798d4d71541753918d0acc7b..033fdcb780cc3c295876023addeb4665c3cae0a0 100644 (file)
@@ -441,7 +441,7 @@ WERROR reg_open_ldb_file(TALLOC_CTX *parent_ctx, const char *location,
                return WERR_INVALID_PARAM;
 
        wrap = ldb_wrap_connect(parent_ctx, ev_ctx, lp_ctx,
-                               location, session_info, credentials, 0, NULL);
+                               location, session_info, credentials, 0);
 
        if (wrap == NULL) {
                DEBUG(1, (__FILE__": unable to connect\n"));
index 6e2dbca5381b77eef47798de3041b3cfce175ed1..09ceecc752e3ed14383e0cfd2b1ec88c72a3a43f 100644 (file)
@@ -56,7 +56,7 @@ static ADS_STATUS ads_connect(ADS_STRUCT *ads)
 
        url = talloc_asprintf(ads, "ldap://%s", io->out.dcs[0].name);
        ads->ldbctx = ldb_wrap_connect(ads, ads->netctx->event_ctx, ads->netctx->lp_ctx,
-                        url, NULL, ads->netctx->cred, 0, NULL);
+                        url, NULL, ads->netctx->cred, 0);
        if (ads->ldbctx == NULL) {
                return ADS_ERROR_NT(NT_STATUS_UNSUCCESSFUL);
        }
index 3fb308dcd7065788f0a7594d8fa9f0769c042876..6668be83ac8c619837b470f89f97c2b83acd766e 100644 (file)
@@ -814,7 +814,7 @@ static NTSTATUS becomeDC_ldap_connect(struct libnet_BecomeDC_state *s,
        ldap->ldb = ldb_wrap_connect(s, s->libnet->event_ctx, s->libnet->lp_ctx, url,
                                     NULL,
                                     s->libnet->cred,
-                                    0, NULL);
+                                    0);
        talloc_free(url);
        if (ldap->ldb == NULL) {
                return NT_STATUS_UNEXPECTED_NETWORK_ERROR;
index fc7de10506bc8258416521d6903c1e5d307aa02d..e60d45e31645f13ba193b8d9abd1a6a50c5a73c1 100644 (file)
@@ -247,7 +247,7 @@ static NTSTATUS libnet_JoinADSDomain(struct libnet_context *ctx, struct libnet_J
 
        remote_ldb = ldb_wrap_connect(tmp_ctx, ctx->event_ctx, ctx->lp_ctx, 
                                      remote_ldb_url, 
-                                     NULL, ctx->cred, 0, NULL);
+                                     NULL, ctx->cred, 0);
        if (!remote_ldb) {
                r->out.error_string = NULL;
                talloc_free(tmp_ctx);
index 164e806906fa0c162428fc8014ca62d402a9de92..4cd23cd8ed6d7b0d3521f37226247471d241ad9e 100644 (file)
@@ -1191,7 +1191,7 @@ static NTSTATUS libnet_samsync_ldb_init(TALLOC_CTX *mem_ctx,
                                                     state->samsync_state->machine_net_ctx->lp_ctx, 
                                                     ldap_url, 
                                                     NULL, state->samsync_state->machine_net_ctx->cred,
-                                                    0, NULL);
+                                                    0);
                if (!state->remote_ldb) {
                        *error_string = talloc_asprintf(mem_ctx, "Failed to connect to remote LDAP server at %s (used to extract additional data in SamSync replication)", ldap_url);
                        return NT_STATUS_NO_LOGON_SERVERS;
index 3bd7a4e28723f1aff89fb7a66fc600132a16fc74..6e1ecf072d26aeabdfe390635c2cda492af0d06f 100644 (file)
@@ -316,7 +316,7 @@ static NTSTATUS unbecomeDC_ldap_connect(struct libnet_UnbecomeDC_state *s)
        s->ldap.ldb = ldb_wrap_connect(s, s->libnet->event_ctx, s->libnet->lp_ctx, url,
                                       NULL,
                                       s->libnet->cred,
-                                      0, NULL);
+                                      0);
        talloc_free(url);
        if (s->ldap.ldb == NULL) {
                return NT_STATUS_UNEXPECTED_NETWORK_ERROR;
index a502050299eb86e2b32e73a54c4fea1a435841d3..56ba146be196794f1f4b75a4e9f4ca634a4e89db 100644 (file)
@@ -980,7 +980,7 @@ static bool winsdb_check_or_add_module_list(struct tevent_context *ev_ctx,
        }
 
        h->ldb = ldb_wrap_connect(h, ev_ctx, lp_ctx, lock_path(h, lp_ctx, lp_wins_url(lp_ctx)),
-                                 NULL, NULL, flags, NULL);
+                                 NULL, NULL, flags);
        if (!h->ldb) goto failed;
 
        talloc_free(tmp_ctx);
@@ -1016,7 +1016,7 @@ struct winsdb_handle *winsdb_connect(TALLOC_CTX *mem_ctx,
        }
 
        h->ldb = ldb_wrap_connect(h, ev_ctx, lp_ctx, lock_path(h, lp_ctx, lp_wins_url(lp_ctx)),
-                                 NULL, NULL, flags, NULL);
+                                 NULL, NULL, flags);
        if (!h->ldb) goto failed;       
 
        h->caller = caller;
index c3cdfb15817704fbca4a6cd9a47019ae465cea62..d59cb6fb823d480903d4a66195672e18d15f79c5 100644 (file)
@@ -256,7 +256,7 @@ static NTSTATUS dcesrv_netr_ServerAuthenticate3(struct dcesrv_call_state *dce_ca
        }
 
        nt_status = schannel_store_session_key_ldb(schannel_ldb, mem_ctx, creds);
-       talloc_free(schannel_ldb);
+       talloc_unlink(mem_ctx, schannel_ldb);
 
        return nt_status;
 }
@@ -343,7 +343,7 @@ static NTSTATUS dcesrv_netr_creds_server_step_check(struct dcesrv_call_state *dc
                                                         schannel_in_use,
                                                         received_authenticator,
                                                         return_authenticator, creds_out);
-       talloc_free(ldb);
+       talloc_unlink(mem_ctx, ldb);
        return nt_status;
 }
 
index f4ada2c7c7d180ddb23c28328b5a6a63cafa0355..08f14716657a564a70187e056b6b28e4a913c151 100644 (file)
@@ -4205,7 +4205,7 @@ static NTSTATUS dcesrv_samr_GetDomPwInfo(struct dcesrv_call_state *dce_call, TAL
                "pwdProperties", 1);
 
        talloc_free(msgs);
-       talloc_free(sam_ctx);
+       talloc_unlink(mem_ctx, sam_ctx);
 
        return NT_STATUS_OK;
 }
index 4d732cf997183d59349e250c174890b8c3c2e41e..03fcfd8482dffe511e6163cbda86676c5e47b22d 100644 (file)
@@ -62,7 +62,7 @@ bool torture_ldap_sort(struct torture_context *torture)
        ldb = ldb_wrap_connect(torture, torture->ev, torture->lp_ctx, url,
                                                 NULL,
                                                 cmdline_credentials,
-                                                0, NULL);
+                                                0);
        torture_assert(torture, ldb, "Failed to make LDB connection to target");
 
        ctx = talloc_zero(ldb, struct ldb_result);
index e746a32f2bfeaffb98ebb38606446de8b791f30d..7fe171691a656d63e4d96d09077417c6fc71c6ad 100644 (file)
@@ -387,7 +387,7 @@ bool torture_ldap_schema(struct torture_context *torture)
        ldb = ldb_wrap_connect(torture, torture->ev, torture->lp_ctx, url,
                               NULL,
                               cmdline_credentials,
-                              0, NULL);
+                              0);
        if (!ldb) goto failed;
 
        ret &= test_search_rootDSE(ldb, &rootDSE);
index c53459a58e0cced2bcb5b6dd4b46025d11a5c3e5..a4ca7e535769f763d73883d3e5b1e6d4c7124627 100644 (file)
@@ -159,7 +159,7 @@ bool torture_ldap_uptodatevector(struct torture_context *torture)
        ldb = ldb_wrap_connect(torture, torture->ev, torture->lp_ctx, url,
                               NULL,
                               cmdline_credentials,
-                              0, NULL);
+                              0);
        if (!ldb) goto failed;
 
        ret &= test_check_uptodatevector(torture, ldb, samdb_base_dn(ldb));
index 9547e9ff74ce03946fbb5fa5ee78c6eef14ef13f..236a52b5172c342fef63759539b026ee0c8140ab 100644 (file)
@@ -177,7 +177,7 @@ static bool test_ldb_speed(struct torture_context *torture, const void *_data)
        torture_comment(torture, "Testing ldb speed for sidmap\n");
 
        ldb = ldb_wrap_connect(tmp_ctx, torture->ev, torture->lp_ctx, "tdb://test.ldb", 
-                               NULL, NULL, LDB_FLG_NOSYNC, NULL);
+                               NULL, NULL, LDB_FLG_NOSYNC);
        if (!ldb) {
                unlink("./test.ldb");
                talloc_free(tmp_ctx);
index 1ef58fd3f702bbc4295f4caadbe76d1b274467b3..db949ada2a92fc34ddec07b3babd36de1dec32e5 100644 (file)
@@ -2330,7 +2330,7 @@ static bool test_GetDomainInfo(struct torture_context *tctx,
                sam_ctx = ldb_wrap_connect(tctx, tctx->ev, tctx->lp_ctx, url,
                                           NULL,
                                           cmdline_credentials,
-                                          0, NULL);
+                                          0);
                
                torture_assert(tctx, sam_ctx, "Connection to the SAMDB on DC failed!");
        }
index 2ef6c939708e381ffd1864b4a08e6e637ae2c703..153b5c363efd5fba6dfa5852e6b6e8cf63710b34 100644 (file)
@@ -167,8 +167,8 @@ struct idmap_context *idmap_init(TALLOC_CTX *mem_ctx,
 
        idmap_ctx->ldb_ctx = ldb_wrap_connect(mem_ctx, ev_ctx, lp_ctx,
                                              lp_idmap_url(lp_ctx),
-                                             system_session(mem_ctx, lp_ctx),
-                                             NULL, 0, NULL);
+                                             system_session(lp_ctx),
+                                             NULL, 0);
        if (idmap_ctx->ldb_ctx == NULL) {
                return NULL;
        }