r12434: implement database scavenging, the only missing part is the verifying of...

with the owning wins server, after the verify interval passes.

metze

diff --git a/source/wrepl_server/config.mk b/source/wrepl_server/config.mk
index dc9508dc5af70e67a108725e9d01e1d7bee7f889..74335cbf05166ecc36fbdb2e8fc872505ffe8ddf 100644 (file)
--- a/source/wrepl_server/config.mk
+++ b/source/wrepl_server/config.mk
@@ -9,6 +9,7 @@ INIT_OBJ_FILES = \
                wrepl_in_call.o \
                wrepl_apply_records.o \
                wrepl_periodic.o \
                wrepl_in_call.o \
                wrepl_apply_records.o \
                wrepl_periodic.o \

+               wrepl_scavenging.o \

                wrepl_out_pull.o \
                wrepl_out_push.o \
                wrepl_out_helpers.o
                wrepl_out_pull.o \
                wrepl_out_push.o \
                wrepl_out_helpers.o

diff --git a/source/wrepl_server/wrepl_periodic.c b/source/wrepl_server/wrepl_periodic.c
index 2cea3b8e7c3940c864a183bca5c07daf2ca2ae0e..09554d1659afa69988ac0621cb7e36129f70cfb8 100644 (file)
--- a/source/wrepl_server/wrepl_periodic.c
+++ b/source/wrepl_server/wrepl_periodic.c
@@ -39,6 +39,9 @@ static NTSTATUS wreplsrv_periodic_run(struct wreplsrv_service *service)
 {
        NTSTATUS status;
 
 {
        NTSTATUS status;
 

+       status = wreplsrv_scavenging_run(service);
+       NT_STATUS_NOT_OK_RETURN(status);
+

        status = wreplsrv_out_pull_run(service);
        NT_STATUS_NOT_OK_RETURN(status);
 
        status = wreplsrv_out_pull_run(service);
        NT_STATUS_NOT_OK_RETURN(status);
 

diff --git a/source/wrepl_server/wrepl_scavenging.c b/source/wrepl_server/wrepl_scavenging.c
new file mode 100644 (file)
index 0000000..a19604d
--- /dev/null
+++ b/source/wrepl_server/wrepl_scavenging.c
@@ -0,0 +1,383 @@
+/* 
+   Unix SMB/CIFS implementation.
+   
+   WINS Replication server
+   
+   Copyright (C) Stefan Metzmacher     2005
+   
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+*/
+
+#include "includes.h"
+#include "dlinklist.h"
+#include "lib/events/events.h"
+#include "lib/socket/socket.h"
+#include "smbd/service_task.h"
+#include "smbd/service_stream.h"
+#include "lib/messaging/irpc.h"
+#include "librpc/gen_ndr/ndr_winsrepl.h"
+#include "wrepl_server/wrepl_server.h"
+#include "nbt_server/wins/winsdb.h"
+#include "ldb/include/ldb.h"
+#include "ldb/include/ldb_errors.h"
+#include "libcli/composite/composite.h"
+#include "libcli/wrepl/winsrepl.h"
+#include "wrepl_server/wrepl_out_helpers.h"
+#include "system/time.h"
+
+static NTSTATUS wreplsrv_scavenging_owned_records(struct wreplsrv_service *service, TALLOC_CTX *tmp_mem)
+{
+       NTSTATUS status;
+       struct winsdb_record *rec = NULL;
+       struct ldb_result *res = NULL;
+       const char *filter;
+       uint32_t i;
+       int ret;
+       time_t now = time(NULL);
+       const char *now_timestr;
+       const char *action;
+       const char *old_state;
+       uint32_t modify_flags;
+       BOOL modify_record;
+       BOOL delete_record;
+       BOOL delete_tombstones;
+       struct timeval tombstone_extra_time;
+
+       now_timestr = ldb_timestring(tmp_mem, now);
+       NT_STATUS_HAVE_NO_MEMORY(now_timestr);
+       filter = talloc_asprintf(tmp_mem,
+                                "(&(winsOwner=%s)(objectClass=winsRecord)"
+                                "(expireTime<=%s)(!(isStatic=1)))",
+                                WINSDB_OWNER_LOCAL, now_timestr);
+       NT_STATUS_HAVE_NO_MEMORY(filter);
+       ret = ldb_search(service->wins_db, NULL, LDB_SCOPE_SUBTREE, filter, NULL, &res);
+       if (ret != LDB_SUCCESS) return NT_STATUS_INTERNAL_DB_CORRUPTION;
+       talloc_steal(tmp_mem, res);
+
+       tombstone_extra_time = timeval_add(&service->startup_time,
+                                          service->config.tombstone_extra_timeout,
+                                          0);
+       delete_tombstones = timeval_expired(&tombstone_extra_time);
+
+       for (i=0; i < res->count; i++) {
+               status = winsdb_record(res->msgs[i], tmp_mem, &rec);
+               NT_STATUS_NOT_OK_RETURN(status);
+
+               if (rec->is_static) {
+                       DEBUG(0,("%s: corrupted record: %s\n",
+                               __location__, nbt_name_string(rec, rec->name)));
+                       return NT_STATUS_INTERNAL_DB_CORRUPTION;
+               }
+
+               if (rec->expire_time > now) {
+                       DEBUG(0,("%s: corrupted record: %s\n",
+                               __location__, nbt_name_string(rec, rec->name)));
+                       return NT_STATUS_INTERNAL_DB_CORRUPTION;
+               }
+
+               modify_flags    = 0;
+               modify_record   = False;
+               delete_record   = False;
+
+               switch (rec->state) {
+               case WREPL_STATE_ACTIVE:
+                       old_state       = "active";
+                       rec->state      = WREPL_STATE_RELEASED;
+                       rec->expire_time= service->config.tombstone_interval + now;
+                       modify_flags    = 0;
+                       modify_record   = True;
+                       break;
+
+               case WREPL_STATE_RELEASED:
+                       old_state       = "released";
+                       rec->state      = WREPL_STATE_TOMBSTONE;
+                       rec->expire_time= service->config.tombstone_timeout + now;
+                       modify_flags    = WINSDB_FLAG_ALLOC_VERSION | WINSDB_FLAG_TAKE_OWNERSHIP;
+                       modify_record   = True;
+                       break;
+
+               case WREPL_STATE_TOMBSTONE:
+                       old_state       = "tombstone";
+                       if (!delete_tombstones) break;
+                       delete_record = True;
+                       break;
+
+               case WREPL_STATE_RESERVED:
+                       DEBUG(0,("%s: corrupted record: %s\n",
+                               __location__, nbt_name_string(rec, rec->name)));
+                       return NT_STATUS_INTERNAL_DB_CORRUPTION;
+               }
+
+               if (modify_record) {
+                       action = "modify";
+                       ret = winsdb_modify(service->wins_db, rec, modify_flags);
+               } else if (delete_record) {
+                       action = "delete";
+                       ret = winsdb_delete(service->wins_db, rec);
+               } else {
+                       action = "skip";
+                       ret = NBT_RCODE_OK;
+               }
+
+               if (ret != NBT_RCODE_OK) {
+                       DEBUG(1,("WINS scavenging: failed to %s name %s (owned:%s): error:%u\n",
+                               action, nbt_name_string(rec, rec->name), old_state, ret));
+               } else {
+                       DEBUG(4,("WINS scavenging: %s name: %s (owned:%s)\n",
+                               action, nbt_name_string(rec, rec->name), old_state));
+               }
+
+               talloc_free(rec);
+       }
+
+       return NT_STATUS_OK;
+}
+
+static NTSTATUS wreplsrv_scavenging_replica_non_active_records(struct wreplsrv_service *service, TALLOC_CTX *tmp_mem)
+{
+       NTSTATUS status;
+       struct winsdb_record *rec = NULL;
+       struct ldb_result *res = NULL;
+       const char *filter;
+       uint32_t i;
+       int ret;
+       time_t now = time(NULL);
+       const char *now_timestr;
+       const char *action;
+       const char *old_state;
+       uint32_t modify_flags;
+       BOOL modify_record;
+       BOOL delete_record;
+       BOOL delete_tombstones;
+       struct timeval tombstone_extra_time;
+
+       now_timestr = ldb_timestring(tmp_mem, now);
+       NT_STATUS_HAVE_NO_MEMORY(now_timestr);
+       filter = talloc_asprintf(tmp_mem,
+                                "(&(!(winsOwner=%s))(objectClass=winsRecord)"
+                                "(!(recordState=%u))(expireTime<=%s)(!(isStatic=1)))",
+                                WINSDB_OWNER_LOCAL, WREPL_STATE_ACTIVE, now_timestr);
+       NT_STATUS_HAVE_NO_MEMORY(filter);
+       ret = ldb_search(service->wins_db, NULL, LDB_SCOPE_SUBTREE, filter, NULL, &res);
+       if (ret != LDB_SUCCESS) return NT_STATUS_INTERNAL_DB_CORRUPTION;
+       talloc_steal(tmp_mem, res);
+
+       tombstone_extra_time = timeval_add(&service->startup_time,
+                                          service->config.tombstone_extra_timeout,
+                                          0);
+       delete_tombstones = timeval_expired(&tombstone_extra_time);
+
+       for (i=0; i < res->count; i++) {
+               status = winsdb_record(res->msgs[i], tmp_mem, &rec);
+               NT_STATUS_NOT_OK_RETURN(status);
+
+               if (rec->is_static) {
+                       DEBUG(0,("%s: corrupted record: %s\n",
+                               __location__, nbt_name_string(rec, rec->name)));
+                       return NT_STATUS_INTERNAL_DB_CORRUPTION;
+               }
+
+               if (rec->expire_time > now) {
+                       DEBUG(0,("%s: corrupted record: %s\n",
+                               __location__, nbt_name_string(rec, rec->name)));
+                       return NT_STATUS_INTERNAL_DB_CORRUPTION;
+               }
+
+               modify_flags    = 0;
+               modify_record   = False;
+               delete_record   = False;
+
+               switch (rec->state) {
+               case WREPL_STATE_ACTIVE:
+                       DEBUG(0,("%s: corrupted record: %s\n",
+                               __location__, nbt_name_string(rec, rec->name)));
+                       return NT_STATUS_INTERNAL_DB_CORRUPTION;
+
+               case WREPL_STATE_RELEASED:
+                       old_state       = "released";
+                       rec->state      = WREPL_STATE_TOMBSTONE;
+                       rec->expire_time= service->config.tombstone_timeout + now;
+                       modify_flags    = 0;
+                       modify_record   = True;
+                       break;
+
+               case WREPL_STATE_TOMBSTONE:
+                       old_state       = "tombstone";
+                       if (!delete_tombstones) break;
+                       delete_record = True;
+                       break;
+
+               case WREPL_STATE_RESERVED:
+                       DEBUG(0,("%s: corrupted record: %s\n",
+                               __location__, nbt_name_string(rec, rec->name)));
+                       return NT_STATUS_INTERNAL_DB_CORRUPTION;
+               }
+
+               if (modify_record) {
+                       action = "modify";
+                       ret = winsdb_modify(service->wins_db, rec, modify_flags);
+               } else if (delete_record) {
+                       action = "delete";
+                       ret = winsdb_delete(service->wins_db, rec);
+               } else {
+                       action = "skip";
+                       ret = NBT_RCODE_OK;
+               }
+
+               if (ret != NBT_RCODE_OK) {
+                       DEBUG(1,("WINS scavenging: failed to %s name %s (replica:%s): error:%u\n",
+                               action, nbt_name_string(rec, rec->name), old_state, ret));
+               } else {
+                       DEBUG(4,("WINS scavenging: %s name: %s (replica:%s)\n",
+                               action, nbt_name_string(rec, rec->name), old_state));
+               }
+
+               talloc_free(rec);
+       }
+
+       return NT_STATUS_OK;
+}
+
+static NTSTATUS wreplsrv_scavenging_replica_active_records(struct wreplsrv_service *service, TALLOC_CTX *tmp_mem)
+{
+       NTSTATUS status;
+       struct winsdb_record *rec = NULL;
+       struct ldb_result *res = NULL;
+       const char *filter;
+       uint32_t i;
+       int ret;
+       time_t now = time(NULL);
+       const char *now_timestr;
+       const char *action;
+       const char *old_state;
+       BOOL modify_flags;
+       BOOL modify_record;
+       BOOL delete_record;
+
+       now_timestr = ldb_timestring(tmp_mem, now);
+       NT_STATUS_HAVE_NO_MEMORY(now_timestr);
+       filter = talloc_asprintf(tmp_mem,
+                                "(&(!(winsOwner=%s))(objectClass=winsRecord)"
+                                "(recordState=%u)(expireTime<=%s)(!(isStatic=1)))",
+                                WINSDB_OWNER_LOCAL, WREPL_STATE_ACTIVE, now_timestr);
+       NT_STATUS_HAVE_NO_MEMORY(filter);
+       ret = ldb_search(service->wins_db, NULL, LDB_SCOPE_SUBTREE, filter, NULL, &res);
+       if (ret != LDB_SUCCESS) return NT_STATUS_INTERNAL_DB_CORRUPTION;
+       talloc_steal(tmp_mem, res);
+
+       for (i=0; i < res->count; i++) {
+               status = winsdb_record(res->msgs[i], tmp_mem, &rec);
+               NT_STATUS_NOT_OK_RETURN(status);
+
+               if (rec->is_static) {
+                       DEBUG(0,("%s: corrupted record: %s\n",
+                               __location__, nbt_name_string(rec, rec->name)));
+                       return NT_STATUS_INTERNAL_DB_CORRUPTION;
+               }
+
+               if (rec->expire_time > now) {
+                       DEBUG(0,("%s: corrupted record: %s\n",
+                               __location__, nbt_name_string(rec, rec->name)));
+                       return NT_STATUS_INTERNAL_DB_CORRUPTION;
+               }
+
+               if (rec->state != WREPL_STATE_ACTIVE) {
+                       DEBUG(0,("%s: corrupted record: %s\n",
+                               __location__, nbt_name_string(rec, rec->name)));
+                       return NT_STATUS_INTERNAL_DB_CORRUPTION;
+               }
+
+               old_state = "active";
+
+               modify_flags    = 0;
+               modify_record   = False;
+               delete_record   = False;
+
+               /* 
+                * TODO: ask the owning wins server if the record still exists,
+                *       if not delete the record
+                */
+               DEBUG(0,("TODO: ask wins server '%s' if '%s' with version_id:%llu still exists\n",
+                       rec->wins_owner, nbt_name_string(rec, rec->name), rec->version));
+
+               if (modify_record) {
+                       action = "modify";
+                       ret = winsdb_modify(service->wins_db, rec, modify_flags);
+               } else if (delete_record) {
+                       action = "delete";
+                       ret = winsdb_delete(service->wins_db, rec);
+               } else {
+                       action = "skip";
+                       ret = NBT_RCODE_OK;
+               }
+
+               if (ret != NBT_RCODE_OK) {
+                       DEBUG(1,("WINS scavenging: failed to %s name %s (replica:%s): error:%u\n",
+                               action, nbt_name_string(rec, rec->name), old_state, ret));
+               } else {
+                       DEBUG(4,("WINS scavenging: %s name: %s (replica:%s)\n",
+                               action, nbt_name_string(rec, rec->name), old_state));
+               }
+
+               talloc_free(rec);
+       }
+
+       return NT_STATUS_OK;
+}
+
+NTSTATUS wreplsrv_scavenging_run(struct wreplsrv_service *service)
+{
+       NTSTATUS status;
+       TALLOC_CTX *tmp_mem;
+
+       if (!timeval_expired(&service->scavenging.next_run)) {
+               return NT_STATUS_OK;
+       }
+
+       service->scavenging.next_run = timeval_current_ofs(service->config.scavenging_interval, 0);
+       status = wreplsrv_periodic_schedule(service, service->config.scavenging_interval);
+       NT_STATUS_NOT_OK_RETURN(status);
+
+       if (service->scavenging.processing) {
+               return NT_STATUS_OK;
+       }
+
+       DEBUG(4,("wreplsrv_scavenging_run(): start\n"));
+
+       tmp_mem = talloc_new(service);
+       service->scavenging.processing = True;
+       status = wreplsrv_scavenging_owned_records(service,tmp_mem);
+       service->scavenging.processing = False;
+       talloc_free(tmp_mem);
+       NT_STATUS_NOT_OK_RETURN(status);
+
+       tmp_mem = talloc_new(service);  
+       service->scavenging.processing = True;
+       status = wreplsrv_scavenging_replica_non_active_records(service, tmp_mem);
+       service->scavenging.processing = False;
+       talloc_free(tmp_mem);
+       NT_STATUS_NOT_OK_RETURN(status);
+
+       tmp_mem = talloc_new(service);
+       service->scavenging.processing = True;
+       status = wreplsrv_scavenging_replica_active_records(service, tmp_mem);
+       service->scavenging.processing = False;
+       talloc_free(tmp_mem);
+       NT_STATUS_NOT_OK_RETURN(status);
+
+       DEBUG(4,("wreplsrv_scavenging_run(): end\n"));
+
+       return NT_STATUS_OK;
+}

diff --git a/source/wrepl_server/wrepl_server.c b/source/wrepl_server/wrepl_server.c
index 3b800333edb9987cf281f5da58808b39b373c33c..ad1bca3e60c951ecfa32bb3fe89ef04ae40f7ad3 100644 (file)
--- a/source/wrepl_server/wrepl_server.c
+++ b/source/wrepl_server/wrepl_server.c
@@ -63,9 +63,16 @@ static NTSTATUS wreplsrv_open_winsdb(struct wreplsrv_service *service)
        /* the default tombstone (extinction) timeout is 1 day */
        service->config.tombstone_timeout = lp_parm_int(-1,"wreplsrv","tombstone_timeout", 1*24*60*60);
 
        /* the default tombstone (extinction) timeout is 1 day */
        service->config.tombstone_timeout = lp_parm_int(-1,"wreplsrv","tombstone_timeout", 1*24*60*60);
 

+       /* the default tombstone extra timeout is 3 days */
+       service->config.tombstone_extra_timeout = lp_parm_int(-1,"wreplsrv","tombstone_extra_timeout", 3*24*60*60);
+

        /* the default verify interval is 24 days */
        service->config.verify_interval   = lp_parm_int(-1,"wreplsrv","verify_interval", 24*24*60*60);
 
        /* the default verify interval is 24 days */
        service->config.verify_interval   = lp_parm_int(-1,"wreplsrv","verify_interval", 24*24*60*60);
 

+       /* the default scavenging interval is 'renew_interval/2' */
+       service->config.scavenging_interval=lp_parm_int(-1,"wreplsrv","scavenging_interval",
+                                                       service->config.renew_interval/2);
+

        /* the maximun interval to the next periodic processing event */
        service->config.periodic_interval = lp_parm_int(-1,"wreplsrv","periodic_interval", 60);
 
        /* the maximun interval to the next periodic processing event */
        service->config.periodic_interval = lp_parm_int(-1,"wreplsrv","periodic_interval", 60);
 


@@ -364,8 +371,9 @@ static void wreplsrv_task_init(struct task_server *task)
                task_server_terminate(task, "wreplsrv_task_init: out of memory");
                return;
        }
                task_server_terminate(task, "wreplsrv_task_init: out of memory");
                return;
        }

-       service->task = task;
-       task->private = service;
+       service->task           = task;
+       service->startup_time   = timeval_current();
+       task->private           = service;

 
        /*
         * setup up all partners, and open the winsdb
 
        /*
         * setup up all partners, and open the winsdb

diff --git a/source/wrepl_server/wrepl_server.h b/source/wrepl_server/wrepl_server.h
index 56751b6f296d2b53613e73194f34e00f52c036b3..125e04b84c0455e8e441b3a1839b6beb8fe39da3 100644 (file)
--- a/source/wrepl_server/wrepl_server.h
+++ b/source/wrepl_server/wrepl_server.h
@@ -209,6 +209,9 @@ struct wreplsrv_service {
        /* the whole wrepl service is in one task */
        struct task_server *task;
 
        /* the whole wrepl service is in one task */
        struct task_server *task;
 

+       /* the time the service was started */
+       struct timeval startup_time;
+

        /* the winsdb handle */
        struct ldb_context *wins_db;
 
        /* the winsdb handle */
        struct ldb_context *wins_db;
 


@@ -232,16 +235,31 @@ struct wreplsrv_service {
                /* 
                 * the interval (in secs) a record remains in TOMBSTONE state,
                 * before it will be removed from the database.
                /* 
                 * the interval (in secs) a record remains in TOMBSTONE state,
                 * before it will be removed from the database.

+                * See also 'tombstone_extra_timeout'.

                 * (also known as extinction timeout)
                 */
                uint32_t tombstone_timeout;
 
                 * (also known as extinction timeout)
                 */
                uint32_t tombstone_timeout;
 

+               /* 
+                * the interval (in secs) a record remains in TOMBSTONE state,
+                * even after 'tombstone_timeout' passes the current timestamp.
+                * this is the minimum uptime of the wrepl service, before
+                * we start delete tombstones. This is to prevent deletion of
+                * tombstones, without replacte them.
+                */
+               uint32_t tombstone_extra_timeout;
+

                /* 
                 * the interval (in secs) till a replica record will be verified
                 * with the owning wins server
                 */
                uint32_t verify_interval;
 
                /* 
                 * the interval (in secs) till a replica record will be verified
                 * with the owning wins server
                 */
                uint32_t verify_interval;
 

+               /* 
+                * the interval (in secs) till a do a database cleanup
+                */
+               uint32_t scavenging_interval;
+

                /* 
                 * the interval (in secs) to the next periodic processing
                 * (this is the maximun interval)
                /* 
                 * the interval (in secs) to the next periodic processing
                 * (this is the maximun interval)


@@ -269,4 +287,18 @@ struct wreplsrv_service {
                /* here we have a reference to the timed event the schedules the periodic stuff */
                struct timed_event *te;
        } periodic;
                /* here we have a reference to the timed event the schedules the periodic stuff */
                struct timed_event *te;
        } periodic;

+
+       /* some stuff for scavenging processing */
+       struct {
+               /*
+                * the timestamp for the next scavenging run,
+                * this is the timstamp passed to event_add_timed()
+                */
+               struct timeval next_run;
+
+               /*
+                * are we currently inside a scavenging run
+                */
+               BOOL processing;        
+       } scavenging;

 };
 };