1 Added some DB-access routines to help rsync keep extra filesystem info
2 about the files it is dealing with. This adds both the --db=CONFIG_FILE
3 option and the "db config" daemon parameter.
5 Future improvements may include:
7 - Updating of MD4 checksums when transferring any file, even w/o -c.
8 To make that work we'd need to make the sender force checksum_seed to
9 0 when using a DB and having the receiving side check to see if it
10 got a 0 checksum_seed.
12 - Caching of path info that allows for the finding of files to use for
13 moving/linking/copying/alternate-basis-use.
15 - Extend DB support beyond MySQL and SQLite (PostgreSQL?).
17 To use this patch, run these commands for a successful build:
19 patch -p1 <patches/db.diff
24 based-on: db5bfe67a5d022f9ad25340db6bc2cca2cbbdb65
25 diff --git a/.gitignore b/.gitignore
28 @@ -33,6 +33,7 @@ aclocal.m4
36 diff --git a/Makefile.in b/Makefile.in
39 @@ -4,6 +4,7 @@ prefix=@prefix@
40 datarootdir=@datarootdir@
41 exec_prefix=@exec_prefix@
46 with_rrsync=@with_rrsync@
47 @@ -36,7 +37,7 @@ MD5_ASM_x86_64=lib/md5-asm-x86_64.o
49 GENFILES=configure.sh aclocal.m4 config.h.in rsync.1 rsync.1.html \
50 rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html \
52 + rsyncdb.1 rsyncdb.1.html @GEN_RRSYNC@
53 HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \
54 lib/pool_alloc.h lib/mdigest.h lib/md-defines.h
55 LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
56 @@ -46,7 +47,7 @@ zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
57 OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
58 util1.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
59 OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
60 - usage.o fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
61 + usage.o fileio.o batch.o clientname.o chmod.o db.o acls.o xattrs.o
62 OBJS3=progress.o pipe.o @MD5_ASM@ @ROLL_SIMD@ @ROLL_ASM@
63 DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
64 popt_OBJS=popt/findme.o popt/popt.o popt/poptconfig.o \
65 @@ -78,10 +79,12 @@ install: all
66 -$(MKDIR_P) $(DESTDIR)$(bindir)
67 $(INSTALLCMD) $(INSTALL_STRIP) -m 755 rsync$(EXEEXT) $(DESTDIR)$(bindir)
68 $(INSTALLCMD) -m 755 $(srcdir)/rsync-ssl $(DESTDIR)$(bindir)
69 + rsync -ilt rsyncdb$(EXEEXT) $(DESTDIR)$(bindir)/
70 -$(MKDIR_P) $(DESTDIR)$(mandir)/man1
71 -$(MKDIR_P) $(DESTDIR)$(mandir)/man5
72 if test -f rsync.1; then $(INSTALLMAN) -m 644 rsync.1 $(DESTDIR)$(mandir)/man1; fi
73 if test -f rsync-ssl.1; then $(INSTALLMAN) -m 644 rsync-ssl.1 $(DESTDIR)$(mandir)/man1; fi
74 + if test -f rsyncdb.1; then $(INSTALLMAN) -m 644 rsyncdb.1 $(DESTDIR)$(mandir)/man1; fi
75 if test -f rsyncd.conf.5; then $(INSTALLMAN) -m 644 rsyncd.conf.5 $(DESTDIR)$(mandir)/man5; fi
76 if test "$(with_rrsync)" = yes; then \
77 $(INSTALLCMD) -m 755 rrsync $(DESTDIR)$(bindir); \
78 @@ -106,10 +109,13 @@ rsync$(EXEEXT): $(OBJS)
79 rrsync: support/rrsync
80 cp -p $(srcdir)/support/rrsync rrsync
82 +rsyncdb$(EXEEXT): rsync$(EXEEXT)
83 + ln -s rsync$(EXEEXT) rsyncdb$(EXEEXT)
86 $(CHECK_OBJS): $(HEADERS)
87 tls.o xattrs.o: lib/sysxattrs.h
88 -usage.o: version.h latest-year.h help-rsync.h help-rsyncd.h git-version.h default-cvsignore.h
89 +usage.o: version.h latest-year.h help-rsync.h help-rsyncd.h help-rsyncdb.h git-version.h default-cvsignore.h
90 loadparm.o: default-dont-compress.h daemon-parm.h
93 @@ -120,6 +126,9 @@ default-cvsignore.h default-dont-compress.h: rsync.1.md define-from-md.awk
94 help-rsync.h help-rsyncd.h: rsync.1.md help-from-md.awk
95 $(AWK) -f $(srcdir)/help-from-md.awk -v hfile=$@ $(srcdir)/rsync.1.md
97 +help-rsyncdb.h: rsyncdb.1.md help-from-md.awk
98 + awk -f $(srcdir)/help-from-md.awk -v hfile=$@ $(srcdir)/rsyncdb.1.md
100 daemon-parm.h: daemon-parm.txt daemon-parm.awk
101 $(AWK) -f $(srcdir)/daemon-parm.awk $(srcdir)/daemon-parm.txt
103 @@ -257,7 +266,7 @@ proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h
104 $(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h
107 -man: rsync.1 rsync-ssl.1 rsyncd.conf.5 @MAKE_RRSYNC_1@
108 +man: rsync.1 rsync-ssl.1 rsyncd.conf.5 @MAKE_RRSYNC_1@ rsyncdb.1
110 rsync.1: rsync.1.md md-convert version.h Makefile
111 @$(srcdir)/maybe-make-man rsync.1.md
112 @@ -271,11 +280,14 @@ rsyncd.conf.5: rsyncd.conf.5.md md-convert version.h Makefile
113 rrsync.1: support/rrsync.1.md md-convert Makefile
114 @$(srcdir)/maybe-make-man support/rrsync.1.md
116 +rsyncdb.1: rsyncdb.1.md md-convert NEWS.md Makefile
117 + @$(srcdir)/maybe-make-man rsyncdb.1.md
121 rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) @MAKE_RRSYNC@ \
122 git-version.h rounding rounding.h *.old rsync*.1 rsync*.5 @MAKE_RRSYNC_1@ \
123 - *.html daemon-parm.h help-*.h default-*.h proto.h proto.h-tstamp
124 + *.html daemon-parm.h help-*.h default-*.h proto.h proto.h-tstamp rsyncdb$(EXEEXT)
128 diff --git a/checksum.c b/checksum.c
131 @@ -40,6 +40,7 @@ extern int whole_file;
132 extern int checksum_seed;
133 extern int protocol_version;
134 extern int proper_seed_order;
136 extern const char *checksum_choice;
138 struct name_num_item valid_checksums_items[] = {
139 @@ -393,6 +394,8 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
140 md5_update(&m5, (uchar *)map_ptr(buf, i, remainder), remainder);
142 md5_result(&m5, (uchar *)sum);
144 + db_set_checksum(5, st_p, sum);
148 @@ -432,6 +435,8 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum)
149 mdfour_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder);
151 mdfour_result(&m, (uchar *)sum);
153 + db_set_checksum(4, st_p, sum);
157 diff --git a/cleanup.c b/cleanup.c
160 @@ -28,6 +28,7 @@ extern int am_daemon;
161 extern int am_receiver;
162 extern int am_sender;
165 extern int keep_partial;
166 extern int got_xfer_error;
167 extern int protocol_version;
168 @@ -143,6 +144,12 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
173 + db_disconnect(False);
178 if (cleanup_child_pid != -1) {
180 int pid = wait_process(cleanup_child_pid, &status, WNOHANG);
181 diff --git a/clientserver.c b/clientserver.c
184 @@ -44,6 +44,8 @@ extern int numeric_ids;
185 extern int filesfrom_fd;
186 extern int remote_protocol;
187 extern int protocol_version;
188 +extern int always_checksum;
190 extern int io_timeout;
191 extern int no_detach;
192 extern int write_batch;
193 @@ -51,6 +53,7 @@ extern int old_style_args;
194 extern int default_af_hint;
195 extern int logfile_format_has_i;
196 extern int logfile_format_has_o_or_i;
197 +extern char *db_config;
198 extern char *bind_address;
199 extern char *config_file;
200 extern char *logfile_format;
201 @@ -869,6 +872,11 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
205 + if (*lp_db_config(i)) {
206 + db_read_config(FLOG, lp_db_config(i));
207 + db_lax = lp_db_lax(i);
210 #if defined HAVE_SETENV || defined HAVE_PUTENV
211 if ((*lp_early_exec(module_id) || *lp_prexfer_exec(module_id)
212 || *lp_postxfer_exec(module_id) || *lp_name_converter(module_id))
213 @@ -1081,6 +1089,8 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
215 am_server = 1; /* Don't let someone try to be tricky. */
219 if (lp_ignore_errors(module_id))
222 diff --git a/configure.ac b/configure.ac
225 @@ -10,6 +10,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \
226 unistd.h utime.h compat.h sys/param.h ctype.h sys/wait.h sys/stat.h \
227 sys/ioctl.h sys/filio.h string.h stdlib.h sys/socket.h sys/mode.h grp.h \
228 sys/un.h sys/attr.h arpa/inet.h arpa/nameser.h locale.h sys/types.h \
229 + mysql/mysql.h sqlite3.h \
230 netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h mcheck.h \
231 sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h dl.h \
232 popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netgroup.h \
233 @@ -1423,6 +1424,48 @@ if test x"$enable_acl_support" = x"no" || test x"$enable_xattr_support" = x"no"
237 +AC_MSG_CHECKING([whether to include mysql DB support])
238 +AC_ARG_ENABLE(mysql,
239 + AC_HELP_STRING([--disable-mysql],
240 + [disable mysql DB support]))
242 +if test x"$enable_mysql" = x"no"; then
245 + AC_MSG_RESULT([yes])
246 + AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, 1, 0)
247 + if test x$MYSQL_CONFIG = x1; then
248 + AC_MSG_CHECKING(for mysql version >= 4)
249 + mysql_version=`mysql_config --version`
250 + mysql_major_version=`echo $mysql_version | sed 's/\..*//'`
251 + if test $mysql_major_version -lt 4; then
252 + AC_MSG_RESULT(no.. skipping MySQL)
256 + MYSQL_CFLAGS=`mysql_config --cflags`
257 + MYSQL_LIBS=`mysql_config --libs`
259 + CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS"
260 + LIBS="$MYSQL_LIBS $LIBS"
262 + AC_CHECK_LIB(mysqlclient, mysql_init)
267 +AC_MSG_CHECKING([whether to include sqlite DB support])
268 +AC_ARG_ENABLE(sqlite,
269 + AC_HELP_STRING([--disable-sqlite],
270 + [disable sqlite DB support]))
272 +if test x"$enable_sqlite" = x"no"; then
275 + AC_CHECK_LIB(sqlite3, sqlite3_open)
276 + AC_CHECK_FUNCS(sqlite3_open_v2 sqlite3_prepare_v2)
280 ' checker'*|checker*)
281 AC_DEFINE(FORCE_FD_ZERO_MEMSET, 1, [Used to make "checker" understand that FD_ZERO() clears memory.])
282 diff --git a/daemon-parm.txt b/daemon-parm.txt
283 --- a/daemon-parm.txt
284 +++ b/daemon-parm.txt
285 @@ -18,6 +18,7 @@ Locals: =================================================================
286 STRING auth_users NULL
289 +STRING db_config NULL
290 STRING dont_compress DEFAULT_DONT_COMPRESS
291 STRING early_exec NULL
293 @@ -51,6 +52,7 @@ INTEGER timeout 0
295 ENUM syslog_facility LOG_DAEMON
298 BOOL fake_super False
299 BOOL forward_lookup True
300 BOOL ignore_errors False
301 diff --git a/db.c b/db.c
307 + * Routines to access extended file info via DB.
309 + * Copyright (C) 2008-2013 Wayne Davison
311 + * This program is free software; you can redistribute it and/or modify
312 + * it under the terms of the GNU General Public License as published by
313 + * the Free Software Foundation; either version 3 of the License, or
314 + * (at your option) any later version.
316 + * This program is distributed in the hope that it will be useful,
317 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
318 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
319 + * GNU General Public License for more details.
321 + * You should have received a copy of the GNU General Public License along
322 + * with this program; if not, visit the http://fsf.org website.
330 +#include "openssl/md4.h"
331 +#include "openssl/md5.h"
336 +extern int am_receiver;
337 +extern int am_generator;
338 +extern int checksum_type;
339 +extern int db_clean, db_check, db_do_md4, db_do_md5, db_update, db_lax, db_init, db_mounts;
340 +extern int db_output_name, db_output_sum, db_output_info, db_output_unchanged, db_output_dirs, db_output_msgs;
341 +extern int saw_db_output_opt, saw_db_sum_opt;
342 +extern char *db_config;
344 +#define MOUNT_HELPER_SCRIPT "/usr/sbin/rsyncdb-mountinfo"
346 +#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
348 +#include <mysql/mysql.h>
349 +#include <mysql/errmsg.h>
352 +#if defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
354 +#include <sqlite3.h>
355 +#ifndef HAVE_SQLITE3_OPEN_V2
356 +#define sqlite3_open_v2(dbname, dbhptr, flags, vfs) \
357 + sqlite3_open(dbname, dbhptr)
359 +#ifndef HAVE_SQLITE3_PREPARE_V2
360 +#define sqlite3_prepare_v2 sqlite3_prepare
362 +#define MAX_LOCK_FAILURES 10
363 +#define LOCK_FAIL_MSLEEP 100
367 +#define MD5_CTX md_context
368 +#define MD5_Init md5_begin
369 +#define MD5_Update md5_update
370 +#define MD5_Final(digest, cptr) md5_result(cptr, digest)
373 +#define DB_TYPE_NONE 0
374 +#define DB_TYPE_MYSQL 1
375 +#define DB_TYPE_SQLITE 2
377 +int use_db = DB_TYPE_NONE;
378 +int select_many_sums = 0;
381 +#define PREP_MOUNT 1
383 +static const char *dbhost = NULL, *dbuser = NULL, *dbpass = NULL, *dbname = NULL;
384 +static unsigned int dbport = 0;
385 +static int transaction_state = -1;
402 +#define UPD_MOUNT 5 /* SQLite only */
406 +#define INS_PRESENT 9
407 +#define MAX_PREP_CNT 10
409 +#define MAX_BIND_CNT 7
410 +#define MAX_RESULT_BINDS 32
417 + sqlite3_stmt *sqlite;
420 +} statements[MAX_PREP_CNT];
423 +static enum logcode log_code;
426 +static unsigned int bind_disk_id, bind_mdnum;
427 +static int64 bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime;
428 +static char bind_sum[MAX_DIGEST_LEN];
429 +static unsigned long result_length[MAX_RESULT_BINDS];
430 +static bool result_is_null[MAX_RESULT_BINDS], result_error[MAX_RESULT_BINDS];
431 +#elif defined USE_SQLITE
432 +static int64 bind_mtime;
434 +static char bind_thishost[128+1];
435 +static unsigned long bind_thishost_len;
436 +static char *mount_helper_script = NULL;
438 +static char *error_log;
439 +#if defined USE_SQLITE && defined SQLITE_CONFIG_LOG
440 +static char bind_mount_uniq[128+1];
441 +static unsigned long bind_mount_uniq_len;
442 +static FILE *error_log_fp;
445 +#define PTR_SIZE (sizeof (struct file_struct *))
447 +#if defined USE_MYSQL || defined USE_SQLITE
448 +static void update_mounts(void);
452 + struct name_list *next;
456 +int db_read_config(enum logcode code, const char *config_file)
458 + char buf[2048], *cp;
464 + bind_thishost_len = strlcpy(bind_thishost, "localhost", sizeof bind_thishost);
466 + if (!(fp = fopen(config_file, "r"))) {
467 + rsyserr(log_code, errno, "unable to open %s", config_file);
470 + if (DEBUG_GTE(DB, 1))
471 + rprintf(FCLIENT, "[%s] Reading DB config from %s\n", who_am_i(), config_file);
472 + while (fgets(buf, sizeof buf, fp)) {
474 + if ((cp = strchr(buf, '#')) == NULL
475 + && (cp = strchr(buf, '\r')) == NULL
476 + && (cp = strchr(buf, '\n')) == NULL)
477 + cp = buf + strlen(buf);
478 + while (cp != buf && isSpace(cp-1)) cp--;
484 + if (!(cp = strchr(buf, ':')))
488 + while (isSpace(cp)) cp++;
489 + if (strcasecmp(buf, "dbhost") == 0)
490 + dbhost = strdup(cp);
491 + else if (strcasecmp(buf, "dbuser") == 0)
492 + dbuser = strdup(cp);
493 + else if (strcasecmp(buf, "dbpass") == 0)
494 + dbpass = strdup(cp);
495 + else if (strcasecmp(buf, "dbname") == 0)
496 + dbname = strdup(cp);
497 + else if (strcasecmp(buf, "dbport") == 0)
499 + else if (strcasecmp(buf, "transaction") == 0)
500 + transaction_state = atoi(cp) ? 0 : -1;
501 + else if (strcasecmp(buf, "mountHelper") == 0)
502 + mount_helper_script = strdup(cp);
503 + else if (strcasecmp(buf, "errlog") == 0)
504 + error_log = strdup(cp);
505 + else if (strcasecmp(buf, "thishost") == 0)
506 + bind_thishost_len = strlcpy(bind_thishost, cp, sizeof bind_thishost);
507 + else if (strcasecmp(buf, "dbtype") == 0) {
509 + if (strcasecmp(cp, "mysql") == 0) {
510 + use_db = DB_TYPE_MYSQL;
515 + if (strcasecmp(cp, "sqlite") == 0) {
516 + use_db = DB_TYPE_SQLITE;
521 + "Unsupported dbtype on line #%d in %s.\n",
522 + lineno, config_file);
523 + use_db = DB_TYPE_NONE;
527 + rprintf(log_code, "Invalid line #%d in %s\n",
528 + lineno, config_file);
529 + use_db = DB_TYPE_NONE;
535 + if (bind_thishost_len >= (int)sizeof bind_thishost)
536 + bind_thishost_len = sizeof bind_thishost - 1;
538 + if (!use_db || !dbname) {
539 + rprintf(log_code, "Please specify at least dbtype and dbname in %s.\n", config_file);
540 + use_db = DB_TYPE_NONE;
544 + md_num = checksum_type == 5 ? 5 : 4;
547 + if (use_db != DB_TYPE_SQLITE)
548 + rprintf(log_code, "Ignoring errlog setting for non-SQLite DB.\n");
549 +#ifndef SQLITE_CONFIG_LOG
551 + rprintf(log_code, "Your sqlite doesn't support SQLITE_CONFIG_LOG.\n");
555 + if (!mount_helper_script)
556 + mount_helper_script = MOUNT_HELPER_SCRIPT;
561 +#if defined USE_SQLITE && defined SQLITE_CONFIG_LOG
562 +static void errorLogCallback(UNUSED(void *pArg), int iErrCode, const char *zMsg)
564 + fprintf(error_log_fp, "[%d] %s (%d)\n", (int)getpid(), zMsg, iErrCode);
568 +static int run_sql(const char *fmt, ...)
575 + qlen = vasprintf(&query, fmt, ap);
578 + out_of_memory("run_sql");
579 + if (DEBUG_GTE(DB, 3))
580 + rprintf(FCLIENT, "[%s] SQL being run: %s\n", who_am_i(), query);
584 + case DB_TYPE_MYSQL:
585 + if (mysql_query(dbh.mysql, query) < 0) {
586 + rprintf(FERROR, "Failed to run sql: %s\n", mysql_error(dbh.mysql));
587 + rprintf(FERROR, "%s\n", query);
593 + case DB_TYPE_SQLITE: {
594 + int rc, lock_failures = 0;
596 + if ((rc = sqlite3_exec(dbh.sqlite, query, NULL, NULL, NULL)) == 0)
598 + if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
600 + if (++lock_failures > MAX_LOCK_FAILURES)
602 + msleep(LOCK_FAIL_MSLEEP);
605 + rprintf(FERROR, "[%s] Failed to run sql: %s\n", who_am_i(), sqlite3_errmsg(dbh.sqlite));
606 + rprintf(FERROR, "%s\n", query);
620 +static int prepare_mysql(int ndx, MYSQL_BIND *binds, int bind_cnt, const char *fmt, ...)
624 + int qlen, param_cnt;
625 + MYSQL_STMT *stmt = mysql_stmt_init(dbh.mysql);
628 + out_of_memory("prepare_mysql");
631 + qlen = vasprintf(&query, fmt, ap);
634 + out_of_memory("prepare_mysql");
635 + if (DEBUG_GTE(DB, 3))
636 + rprintf(FCLIENT, "[%s] SQL being prepared: %s\n", who_am_i(), query);
638 + if (mysql_stmt_prepare(stmt, query, qlen) != 0) {
639 + rprintf(log_code, "[%s] Prepare failed: %s\n", who_am_i(), mysql_stmt_error(stmt));
640 + rprintf(log_code, "%s\n", query);
645 + if ((param_cnt = mysql_stmt_param_count(stmt)) != bind_cnt) {
646 + rprintf(log_code, "[%s] Parameters in statement = %d, bind vars = %d\n",
647 + who_am_i(), param_cnt, bind_cnt);
648 + rprintf(log_code, "%s\n", query);
653 + mysql_stmt_bind_param(stmt, binds);
655 + statements[ndx].mysql = stmt;
663 +static int prepare_mysql_queries(int type)
665 + MYSQL_BIND binds[MAX_BIND_CNT];
670 + sql="SELECT disk_id"
672 + " WHERE host = ? AND devno = ?";
673 + memset(binds, 0, sizeof binds);
674 + binds[0].buffer_type = MYSQL_TYPE_STRING;
675 + binds[0].buffer = &bind_thishost;
676 + binds[0].buffer_length = bind_thishost_len;
677 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
678 + binds[1].buffer = &bind_devno;
679 + if (!prepare_mysql(SEL_DEV, binds, 2, sql))
682 + memset(binds, 0, sizeof binds);
683 + binds[0].buffer_type = MYSQL_TYPE_LONG;
684 + binds[0].buffer = &bind_disk_id;
685 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
686 + binds[1].buffer = &bind_ino;
687 + if (select_many_sums) {
688 + sql="SELECT checksum, sum_type, size, mtime, ctime"
690 + " WHERE disk_id = ? AND ino = ?";
691 + if (!prepare_mysql(SEL_SUM, binds, 2, sql))
694 + sql="SELECT checksum"
696 + " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
697 + " AND size = ? AND mtime = ? %s"; /* optional: AND ctime = ? */
698 + binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
699 + binds[2].buffer = &bind_size;
700 + binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
701 + binds[3].buffer = &bind_mtime;
703 + binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
704 + binds[4].buffer = &bind_ctime;
706 + if (!prepare_mysql(SEL_SUM, binds, 4 + !db_lax, sql, md_num, db_lax ? "" : "AND ctime = ?"))
710 + sql="INSERT INTO inode_map"
711 + " SET disk_id = ?, ino = ?, sum_type = ?,"
712 + " size = ?, mtime = ?, ctime = ?, checksum = ?"
713 + " ON DUPLICATE KEY"
714 + " UPDATE size = VALUES(size), mtime = VALUES(mtime),"
715 + " ctime = VALUES(ctime), checksum = VALUES(checksum)";
716 + memset(binds, 0, sizeof binds);
717 + binds[0].buffer_type = MYSQL_TYPE_LONG;
718 + binds[0].buffer = &bind_disk_id;
719 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
720 + binds[1].buffer = &bind_ino;
721 + binds[2].buffer_type = MYSQL_TYPE_LONG;
722 + binds[2].buffer = &bind_mdnum;
723 + binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
724 + binds[3].buffer = &bind_size;
725 + binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
726 + binds[4].buffer = &bind_mtime;
727 + binds[5].buffer_type = MYSQL_TYPE_LONGLONG;
728 + binds[5].buffer = &bind_ctime;
729 + binds[6].buffer_type = MYSQL_TYPE_BLOB;
730 + binds[6].buffer = &bind_sum;
731 + binds[6].buffer_length = MD5_DIGEST_LEN; /* Same as MD4_DIGEST_LEN */
732 + if (!prepare_mysql(REP_SUM, binds, 7, sql))
735 + sql="UPDATE inode_map"
737 + " WHERE disk_id = ? AND ino = ? AND sum_type = ? AND size = ? AND mtime = ?";
738 + memset(binds, 0, sizeof binds);
739 + binds[0].buffer_type = MYSQL_TYPE_LONGLONG;
740 + binds[0].buffer = &bind_ctime;
741 + binds[1].buffer_type = MYSQL_TYPE_LONG;
742 + binds[1].buffer = &bind_disk_id;
743 + binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
744 + binds[2].buffer = &bind_ino;
745 + binds[3].buffer_type = MYSQL_TYPE_LONG;
746 + binds[3].buffer = &bind_mdnum;
747 + binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
748 + binds[4].buffer = &bind_size;
749 + binds[5].buffer_type = MYSQL_TYPE_LONGLONG;
750 + binds[5].buffer = &bind_mtime;
751 + if (!prepare_mysql(UPD_CTIME, binds, 6, sql))
756 + sql="INSERT INTO disk"
757 + " SET host = ?, last_seen = ?, mount_uniq = ?, devno = ?"
758 + " ON DUPLICATE KEY"
759 + " UPDATE last_seen = VALUES(last_seen), devno = VALUES(devno)";
760 + memset(binds, 0, sizeof binds);
761 + binds[0].buffer_type = MYSQL_TYPE_STRING;
762 + binds[0].buffer = &bind_thishost;
763 + binds[0].buffer_length = bind_thishost_len;
764 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
765 + binds[1].buffer = &bind_mtime; /* we abuse mtime to hold the last_seen value */
766 + binds[2].buffer_type = MYSQL_TYPE_STRING;
767 + binds[2].buffer = &bind_mount_uniq;
768 + binds[2].buffer_length = sizeof bind_mount_uniq;
769 + binds[2].length = &bind_mount_uniq_len;
770 + binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
771 + binds[3].buffer = &bind_devno;
772 + if (!prepare_mysql(INS_MOUNT, binds, 4, sql))
775 + sql="SELECT mount_uniq"
777 + " WHERE host = ? AND last_seen < ? AND devno != 0";
778 + /* Reusing first 2 binds from INS_MOUNT */
779 + if (!prepare_mysql(SEL_MOUNT, binds, 2, sql))
784 + " WHERE host = ? AND last_seen < ? AND devno != 0";
785 + /* Reusing binds from SEL_MOUNT */
786 + if (!prepare_mysql(UN_MOUNT, binds, 2, sql))
796 +static int db_connect_mysql(void)
798 + const char *open_dbname = db_init ? "mysql" : dbname;
800 + if (!(dbh.mysql = mysql_init(NULL)))
801 + out_of_memory("db_read_config");
803 + if (DEBUG_GTE(DB, 1)) {
804 + rprintf(FCLIENT, "[%s] connecting: host=%s user=%s db=%s port=%d\n",
805 + who_am_i(), dbhost, dbuser, open_dbname, dbport);
807 + if (!mysql_real_connect(dbh.mysql, dbhost, dbuser, dbpass, open_dbname, dbport, NULL, 0)) {
808 + rprintf(log_code, "[%s] Unable to connect to DB: %s\n", who_am_i(), mysql_error(dbh.mysql));
813 + if (db_output_msgs)
814 + rprintf(FCLIENT, "Creating DB %s (if it does not exist)\n", dbname);
815 + if (!run_sql("CREATE DATABASE IF NOT EXISTS `%s`", dbname)
816 + || !run_sql("USE `%s`", dbname))
817 + exit_cleanup(RERR_IPC);
819 + if (db_output_msgs)
820 + rprintf(FCLIENT, "Dropping old tables (if they exist))\n");
821 + if (!run_sql("DROP TABLE IF EXISTS disk")
822 + || !run_sql("DROP TABLE IF EXISTS inode_map"))
823 + exit_cleanup(RERR_IPC);
825 + if (db_output_msgs)
826 + rprintf(FCLIENT, "Creating empty tables ...\n");
828 + "CREATE TABLE disk (\n"
829 + " disk_id integer unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT,\n"
830 + " host varchar(128) NOT NULL default 'localhost',\n"
831 + " mount_uniq varchar(128) default NULL,\n"
832 + " devno bigint unsigned NOT NULL,\n" /* This is 0 when not mounted */
833 + " last_seen bigint NOT NULL,\n"
834 + " UNIQUE KEY mount_lookup (host, mount_uniq),\n"
835 + " KEY dev_lookup (devno, host)\n"
837 + exit_cleanup(RERR_IPC);
840 + "CREATE TABLE inode_map (\n"
841 + " disk_id integer unsigned NOT NULL,\n"
842 + " ino bigint unsigned NOT NULL,\n"
843 + " sum_type tinyint NOT NULL default '0',\n"
844 + " size bigint unsigned NOT NULL,\n"
845 + " mtime bigint NOT NULL,\n"
846 + " ctime bigint NOT NULL,\n"
847 + " checksum binary(16) NOT NULL,\n"
848 + " PRIMARY KEY (disk_id,ino,sum_type)\n"
850 + exit_cleanup(RERR_IPC);
857 + if (!prepare_mysql_queries(PREP_MOUNT))
858 + exit_cleanup(RERR_IPC);
863 + if (!prepare_mysql_queries(PREP_NORM))
871 +static int prepare_sqlite(int ndx, const char *fmt, ...)
875 + int rc, qlen, lock_failures = 0;
878 + qlen = vasprintf(&query, fmt, ap);
881 + out_of_memory("prepare_sqlite");
882 + if (DEBUG_GTE(DB, 3))
883 + rprintf(FCLIENT, "[%s] SQL being prepared: %s\n", who_am_i(), query);
885 + while ((rc = sqlite3_prepare_v2(dbh.sqlite, query, -1, &statements[ndx].sqlite, NULL)) != 0) {
886 + if (DEBUG_GTE(DB, 4)) {
887 + rprintf(FCLIENT, "[%s] sqlite3_prepare_v2(,%s,,) returned %d\n",
888 + who_am_i(), query, rc);
890 + if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
892 + if (++lock_failures > MAX_LOCK_FAILURES)
894 + msleep(LOCK_FAIL_MSLEEP);
897 + rprintf(log_code, "[%s] Failed to prepare SQL: %s (%d)\n", who_am_i(), sqlite3_errmsg(dbh.sqlite), rc);
898 + rprintf(log_code, "%s\n", query);
909 +static int prepare_sqlite_queries(int type)
915 + sql="SELECT disk_id"
917 + " WHERE host = ? AND devno = ?";
918 + if (!prepare_sqlite(SEL_DEV, sql))
921 + if (select_many_sums) {
922 + sql="SELECT checksum, sum_type, size, mtime, ctime"
924 + " WHERE disk_id = ? AND ino = ?";
925 + if (!prepare_sqlite(SEL_SUM, sql))
928 + sql="SELECT checksum"
930 + " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
931 + " AND size = ? AND mtime = ? %s";
932 + if (!prepare_sqlite(SEL_SUM, sql, md_num, db_lax ? "" : "AND ctime = ?"))
936 + sql="INSERT OR REPLACE INTO inode_map"
937 + " (disk_id, ino, sum_type, size, mtime, ctime, checksum)"
938 + " VALUES (?, ?, ?, ?, ?, ?, ?)";
939 + if (!prepare_sqlite(REP_SUM, sql))
942 + sql="UPDATE inode_map"
944 + " WHERE disk_id = ? AND ino = ? AND sum_type = ? AND size = ? AND mtime = ?";
945 + if (!prepare_sqlite(UPD_CTIME, sql))
950 + sql="INSERT OR IGNORE INTO disk"
951 + " (host, last_seen, mount_uniq, devno)"
952 + " VALUES (?, ?, ?, ?)";
953 + if (!prepare_sqlite(INS_MOUNT, sql))
957 + " SET last_seen = ?, devno = ?"
958 + " WHERE host = ? AND mount_uniq = ?";
959 + if (!prepare_sqlite(UPD_MOUNT, sql))
962 + sql="SELECT mount_uniq"
964 + " WHERE host = ? AND last_seen < ? AND devno != 0";
965 + if (!prepare_sqlite(SEL_MOUNT, sql))
970 + " WHERE host = ? AND last_seen < ? AND devno != 0";
971 + if (!prepare_sqlite(UN_MOUNT, sql))
981 +static int db_connect_sqlite(void)
983 + int lock_failures = 0;
986 +#ifdef SQLITE_CONFIG_LOG
988 + if (DEBUG_GTE(DB, 1))
989 + rprintf(FCLIENT, "[%s] Setting sqlite errlog to %s\n", who_am_i(), error_log);
990 + if (!(error_log_fp = fopen(error_log, "a"))) {
991 + rsyserr(log_code, errno, "unable to append to logfile %s", error_log);
993 + } else if (sqlite3_config(SQLITE_CONFIG_LOG, errorLogCallback, NULL) != 0)
994 + rprintf(log_code, "Failed to set errorLogCallback: %s\n", sqlite3_errmsg(dbh.sqlite));
999 + int open_flags = SQLITE_OPEN_READWRITE;
1001 + open_flags |= SQLITE_OPEN_CREATE;
1002 + if (DEBUG_GTE(DB, 1))
1003 + rprintf(FCLIENT, "[%s] opening %s (%d)\n", who_am_i(), dbname, open_flags);
1004 + if ((rc = sqlite3_open_v2(dbname, &dbh.sqlite, open_flags, NULL)) == 0) {
1007 + if (DEBUG_GTE(DB, 4)) {
1008 + rprintf(FCLIENT, "[%s] sqlite3_open_v2(%s,,%d,NULL) returned %d\n",
1009 + who_am_i(), dbname, open_flags, rc);
1011 + if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
1013 + if (++lock_failures > MAX_LOCK_FAILURES)
1015 + msleep(LOCK_FAIL_MSLEEP);
1019 + rprintf(log_code, "Unable to connect to DB: %s (%d)\n", sqlite3_errmsg(dbh.sqlite), rc);
1025 + if (db_output_msgs)
1026 + rprintf(FCLIENT, "Dropping old tables (if they exist) ...\n");
1027 + if (!run_sql("DROP TABLE IF EXISTS disk")
1028 + || !run_sql("DROP TABLE IF EXISTS inode_map"))
1029 + exit_cleanup(RERR_IPC);
1031 + if (db_output_msgs)
1032 + rprintf(FCLIENT, "Creating empty tables ...\n");
1033 + sql="CREATE TABLE disk (\n"
1034 + " disk_id integer NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
1035 + " host varchar(128) NOT NULL default 'localhost',\n"
1036 + " mount_uniq varchar(128) default NULL,\n"
1037 + " devno bigint NOT NULL,\n" /* This is 0 when not mounted */
1038 + " last_seen bigint NOT NULL,\n"
1039 + " UNIQUE (host, mount_uniq)\n"
1041 + if (!run_sql(sql))
1042 + exit_cleanup(RERR_IPC);
1044 + sql="CREATE TABLE inode_map (\n"
1045 + " disk_id integer NOT NULL,\n"
1046 + " ino bigint NOT NULL,\n"
1047 + " size bigint NOT NULL,\n"
1048 + " mtime bigint NOT NULL,\n"
1049 + " ctime bigint NOT NULL,\n"
1050 + " sum_type tinyint NOT NULL default '0',\n"
1051 + " checksum binary(16) NOT NULL,\n"
1052 + " PRIMARY KEY (disk_id,ino,sum_type)\n"
1054 + if (!run_sql(sql))
1055 + exit_cleanup(RERR_IPC);
1057 +#if SQLITE_VERSION_NUMBER >= 3007000
1058 + /* Using WAL locking makes concurrency much better (requires sqlite 3.7.0). */
1059 + sql="PRAGMA journal_mode = wal";
1060 + run_sql(sql); /* We don't check this for success. */
1068 + if (!prepare_sqlite_queries(PREP_MOUNT))
1069 + exit_cleanup(RERR_IPC);
1074 + if (!prepare_sqlite_queries(PREP_NORM)) {
1075 + db_disconnect(False);
1083 +int db_connect(int select_many)
1085 + select_many_sums = select_many;
1089 + case DB_TYPE_MYSQL:
1090 + if (db_connect_mysql())
1095 + case DB_TYPE_SQLITE:
1096 + if (db_connect_sqlite())
1102 + db_disconnect(False);
1107 +void db_disconnect(BOOL commit)
1114 + if (transaction_state > 0) {
1115 + if (DEBUG_GTE(DB, 1)) {
1116 + rprintf(FCLIENT, "[%s] %s our DB transaction\n",
1117 + who_am_i(), commit ? "Committing" : "Rolling back");
1119 + transaction_state = 0;
1121 + run_sql("COMMIT");
1123 + run_sql("ROLLBACK");
1126 + if (DEBUG_GTE(DB, 1))
1127 + rprintf(FCLIENT, "[%s] Disconnecting from the DB\n", who_am_i());
1129 + for (ndx = 0; ndx < MAX_PREP_CNT; ndx++) {
1130 + if (statements[ndx].all) {
1133 + case DB_TYPE_MYSQL:
1134 + mysql_stmt_close(statements[ndx].mysql);
1138 + case DB_TYPE_SQLITE:
1139 + sqlite3_finalize(statements[ndx].sqlite);
1143 + statements[ndx].all = NULL;
1149 + case DB_TYPE_MYSQL:
1150 + mysql_close(dbh.mysql);
1154 + case DB_TYPE_SQLITE:
1155 + sqlite3_close(dbh.sqlite);
1161 + use_db = DB_TYPE_NONE;
1165 +static MYSQL_STMT *exec_mysql(int ndx)
1167 + MYSQL_STMT *stmt = statements[ndx].mysql;
1170 + if ((rc = mysql_stmt_execute(stmt)) == CR_SERVER_LOST) {
1171 + db_disconnect(False);
1172 + use_db = DB_TYPE_MYSQL;
1173 + if (db_connect(select_many_sums)) {
1174 + stmt = statements[ndx].mysql;
1175 + rc = mysql_stmt_execute(stmt);
1179 + rprintf(log_code, "SQL execute failed: %s\n", mysql_stmt_error(stmt));
1188 +/* This stores up to max_rows into the values pointed to by the bind data arrays.
1189 + * If max_rows is > 1, then all the buffer pointers MUST be set to an array long
1190 + * enough to hold the max count of rows. The buffer pointer will be incremented
1191 + * to read additional rows (but never past the end). If stmt_ptr is non-NULL, it
1192 + * will be set to the "stmt" pointer IFF we didn't run out of rows before hitting
1193 + * the max. In this case, the caller should call mysql_stmt_fetch() to read any
1194 + * remaining rows (the buffer pointers will point at the final array element) and
1195 + * then call mysql_stmt_free_result(). If *stmt_ptr is a NULL value, there were
1196 + * not enough rows to fill the max_rows arrays, and the stmt was already freed. */
1197 +static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx, int max_rows, MYSQL_STMT **stmt_ptr)
1200 + int i, rc, rows = 0;
1202 + if (bind_cnt > MAX_RESULT_BINDS) {
1203 + fprintf(stderr, "Internal error: MAX_RESULT_BINDS overflow\n");
1204 + exit_cleanup(RERR_UNSUPPORTED);
1207 + if ((stmt = exec_mysql(ndx)) == NULL)
1210 + for (i = 0; i < bind_cnt; i++) {
1211 + binds[i].is_null = &result_is_null[i];
1212 + binds[i].length = &result_length[i];
1213 + binds[i].error = &result_error[i];
1215 + mysql_stmt_bind_result(stmt, binds);
1217 + while (rows < max_rows) {
1218 + if ((rc = mysql_stmt_fetch(stmt)) != 0) {
1219 + if (rc != MYSQL_NO_DATA)
1220 + rprintf(log_code, "SELECT fetch failed: %s\n", mysql_stmt_error(stmt));
1223 + if (++rows >= max_rows)
1225 + for (i = 0; i < bind_cnt; i++) {
1226 + switch (binds[i].buffer_type) {
1227 + case MYSQL_TYPE_BLOB:
1228 + case MYSQL_TYPE_STRING:
1229 + binds[i].buffer += binds[i].buffer_length;
1231 + case MYSQL_TYPE_LONG:
1232 + binds[i].buffer += sizeof (int);
1234 + case MYSQL_TYPE_LONGLONG:
1235 + binds[i].buffer += sizeof (int64);
1238 + fprintf(stderr, "Unknown MYSQL_TYPE_* in multi-row read: %d.\n", binds[i].buffer_type);
1239 + exit_cleanup(RERR_UNSUPPORTED);
1244 + if (!stmt_ptr || rows < max_rows) {
1245 + mysql_stmt_free_result(stmt);
1255 +#if defined USE_MYSQL || defined USE_SQLITE
1256 +static void update_mounts(void)
1258 + char buf[2048], *argv[2];
1259 + int f_from, f_to, len;
1263 + if (DEBUG_GTE(DB, 2))
1264 + printf("Running %s to grab mount info\n", mount_helper_script);
1265 + argv[0] = mount_helper_script;
1267 + pid = piped_child(argv, &f_from, &f_to);
1270 + bind_mtime = time(NULL); /* abuse mtime slightly to hold our last_seen value */
1272 + /* Strict format has 2 items with one tab as separator: MOUNT_UNIQ\tPATH */
1273 + while ((len = read_line(f_from, buf, sizeof buf, 0)) > 0) {
1274 + char *mount_uniq, *path;
1276 + if (DEBUG_GTE(DB, 3))
1277 + printf("Parsing mount info: %s\n", buf);
1278 + mount_uniq = strtok(buf, "\t");
1279 + path = mount_uniq ? strtok(NULL, "\r\n") : NULL;
1281 + fprintf(stderr, "Failed to parse line from %s output\n", mount_helper_script);
1282 + exit_cleanup(RERR_SYNTAX);
1285 + if (lstat(path, &st) < 0) {
1286 + fprintf(stderr, "Failed to lstat(%s): %s\n", path, strerror(errno));
1287 + exit_cleanup(RERR_IPC);
1290 + bind_mount_uniq_len = strlcpy(bind_mount_uniq, mount_uniq, sizeof bind_mount_uniq);
1291 + if (bind_mount_uniq_len >= (int)sizeof bind_mount_uniq)
1292 + bind_mount_uniq_len = sizeof bind_mount_uniq - 1;
1294 + if (db_output_msgs) {
1295 + printf("Marking mount \"%s\" (%s) as a recent mount\n",
1296 + bind_mount_uniq, big_num(st.st_dev));
1300 + case DB_TYPE_MYSQL:
1301 + bind_devno = st.st_dev;
1302 + if (exec_mysql(INS_MOUNT) == NULL) {
1303 + fprintf(stderr, "Failed to update mount info for \"%s\" - %s\n",
1304 + bind_mount_uniq, mysql_error(dbh.mysql));
1305 + exit_cleanup(RERR_IPC);
1310 + case DB_TYPE_SQLITE: {
1311 + int rc, change_cnt;
1312 + sqlite3_stmt *stmt = statements[INS_MOUNT].sqlite;
1313 + sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1314 + sqlite3_bind_int64(stmt, 2, bind_mtime);
1315 + sqlite3_bind_text(stmt, 3, bind_mount_uniq, bind_mount_uniq_len, SQLITE_STATIC);
1316 + sqlite3_bind_int64(stmt, 4, st.st_dev);
1317 + rc = sqlite3_step(stmt);
1318 + if (rc != SQLITE_DONE) {
1319 + fprintf(stderr, "Failed to insert mount info for \"%s\" - %s (%d)\n",
1320 + bind_mount_uniq, sqlite3_errmsg(dbh.sqlite), rc);
1321 + exit_cleanup(RERR_IPC);
1323 + change_cnt = sqlite3_changes(dbh.sqlite);
1324 + sqlite3_reset(stmt);
1325 + if (change_cnt == 0) {
1326 + stmt = statements[UPD_MOUNT].sqlite;
1327 + sqlite3_bind_int64(stmt, 1, bind_mtime);
1328 + sqlite3_bind_int64(stmt, 2, st.st_dev);
1329 + sqlite3_bind_text(stmt, 3, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1330 + sqlite3_bind_text(stmt, 4, bind_mount_uniq, bind_mount_uniq_len, SQLITE_STATIC);
1331 + rc = sqlite3_step(stmt);
1332 + if (rc != SQLITE_DONE) {
1333 + fprintf(stderr, "Failed to update mount info for \"%s\" - %s (%d)\n",
1334 + bind_mount_uniq, sqlite3_errmsg(dbh.sqlite), rc);
1335 + exit_cleanup(RERR_IPC);
1337 + sqlite3_reset(stmt);
1346 + waitpid(pid, &status, 0);
1350 + case DB_TYPE_MYSQL: {
1351 + if (db_output_msgs) {
1352 + MYSQL_BIND binds[1];
1355 + binds[0].buffer_type = MYSQL_TYPE_BLOB;
1356 + binds[0].buffer = bind_mount_uniq;
1357 + binds[0].buffer_length = sizeof bind_mount_uniq;
1358 + if (fetch_mysql(binds, 1, SEL_MOUNT, 1, &stmt)) {
1360 + printf("Marking mount \"%s\" as unmounted.\n", bind_mount_uniq);
1361 + if (mysql_stmt_fetch(stmt) != 0)
1364 + mysql_stmt_free_result(stmt);
1368 + if (exec_mysql(UN_MOUNT) == NULL) {
1369 + fprintf(stderr, "Failed to update old mount info - %s\n",
1370 + mysql_error(dbh.mysql));
1371 + exit_cleanup(RERR_IPC);
1377 + case DB_TYPE_SQLITE: {
1378 + sqlite3_stmt *stmt;
1381 + if (db_output_msgs) {
1382 + stmt = statements[SEL_MOUNT].sqlite;
1383 + sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1384 + sqlite3_bind_int64(stmt, 2, bind_mtime);
1386 + if (sqlite3_step(stmt) != SQLITE_ROW)
1388 + printf("Marking mount \"%s\" as unmounted.\n", sqlite3_column_text(stmt, 0));
1390 + sqlite3_reset(stmt);
1393 + stmt = statements[UN_MOUNT].sqlite;
1394 + sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1395 + sqlite3_bind_int64(stmt, 2, bind_mtime);
1396 + rc = sqlite3_step(stmt);
1397 + sqlite3_reset(stmt);
1398 + if (rc != SQLITE_DONE) {
1399 + fprintf(stderr, "Failed to update old mount info - %s (%d)\n",
1400 + sqlite3_errmsg(dbh.sqlite), rc);
1401 + exit_cleanup(RERR_IPC);
1410 +static unsigned int get_disk_id(int64 devno)
1412 + static unsigned int prior_disk_id = 0;
1413 + static int64 prior_devno = 0;
1415 + if (prior_devno == devno && prior_disk_id) {
1416 + if (DEBUG_GTE(DB, 5))
1417 + rprintf(FCLIENT, "get_disk_id(%s,%s) = %d (cached)\n", bind_thishost, big_num(devno), prior_disk_id);
1418 + return prior_disk_id;
1420 + prior_devno = devno;
1424 + case DB_TYPE_MYSQL: {
1425 + MYSQL_BIND binds[1];
1427 + bind_devno = devno; /* The one changing SEL_DEV input value. */
1429 + /* Bind where to put the output. */
1430 + binds[0].buffer_type = MYSQL_TYPE_LONG;
1431 + binds[0].buffer = &prior_disk_id;
1432 + if (!fetch_mysql(binds, 1, SEL_DEV, 1, NULL))
1433 + prior_disk_id = 0;
1438 + case DB_TYPE_SQLITE: {
1439 + sqlite3_stmt *stmt = statements[SEL_DEV].sqlite;
1440 + sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1441 + sqlite3_bind_int64(stmt, 2, devno);
1442 + if (sqlite3_step(stmt) == SQLITE_ROW)
1443 + prior_disk_id = sqlite3_column_int(stmt, 0);
1445 + prior_disk_id = 0;
1446 + sqlite3_reset(stmt);
1452 + if (DEBUG_GTE(DB, 2))
1453 + rprintf(FCLIENT, "get_disk_id(%s,%s) = %d\n", bind_thishost, big_num(devno), prior_disk_id);
1454 + return prior_disk_id;
1457 +int db_get_checksum(const STRUCT_STAT *st_p, char *sum)
1459 + unsigned int disk_id = get_disk_id(st_p->st_dev);
1467 + case DB_TYPE_MYSQL: {
1468 + MYSQL_BIND binds[1];
1470 + bind_disk_id = disk_id;
1471 + bind_ino = st_p->st_ino;
1472 + bind_size = st_p->st_size;
1473 + bind_mtime = st_p->st_mtime;
1475 + bind_ctime = st_p->st_ctime;
1477 + binds[0].buffer_type = MYSQL_TYPE_BLOB;
1478 + binds[0].buffer = sum;
1479 + binds[0].buffer_length = MD5_DIGEST_LEN;
1480 + ok = fetch_mysql(binds, 1, SEL_SUM, 1, NULL);
1485 + case DB_TYPE_SQLITE: {
1486 + sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
1487 + sqlite3_bind_int(stmt, 1, disk_id);
1488 + sqlite3_bind_int64(stmt, 2, st_p->st_ino);
1489 + sqlite3_bind_int64(stmt, 3, st_p->st_size);
1490 + sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
1492 + sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
1493 + if (sqlite3_step(stmt) == SQLITE_ROW) {
1494 + int len = sqlite3_column_bytes(stmt, 0);
1495 + if (len > MAX_DIGEST_LEN)
1496 + len = MAX_DIGEST_LEN;
1497 + memcpy(sum, sqlite3_column_blob(stmt, 0), len);
1500 + sqlite3_reset(stmt);
1506 + if (DEBUG_GTE(DB, 2)) {
1508 + rprintf(FCLIENT, "[%s] Found DB checksum for %s,%s,%d: %s\n",
1509 + who_am_i(), big_num(st_p->st_dev),
1510 + big_num(st_p->st_ino), md_num, sum_as_hex(md_num, sum, 0));
1512 + rprintf(FCLIENT, "[%s] No DB checksum for %s,%s,%d\n",
1513 + who_am_i(), big_num(st_p->st_dev),
1514 + big_num(st_p->st_ino), md_num);
1521 +int db_get_both_checksums(const STRUCT_STAT *st_p, int *right_sum_cnt, int *wrong_sum_cnt, char **sum4, char **sum5)
1523 + static char dbsum[MD5_DIGEST_LEN*2];
1524 + int rows, j, sum_type[2];
1525 + int64 dbsize[2], dbmtime[2], dbctime[2];
1526 + unsigned int disk_id = get_disk_id(st_p->st_dev);
1533 + case DB_TYPE_MYSQL: {
1534 + MYSQL_BIND binds[5];
1536 + bind_disk_id = disk_id;
1537 + bind_ino = st_p->st_ino;
1539 + binds[0].buffer_type = MYSQL_TYPE_BLOB;
1540 + binds[0].buffer = dbsum;
1541 + binds[0].buffer_length = MD5_DIGEST_LEN;
1542 + binds[1].buffer_type = MYSQL_TYPE_LONG;
1543 + binds[1].buffer = (char*)sum_type;
1544 + binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
1545 + binds[2].buffer = (char*)dbsize;
1546 + binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
1547 + binds[3].buffer = (char*)dbmtime;
1548 + binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
1549 + binds[4].buffer = (char*)dbctime;
1550 + rows = fetch_mysql(binds, 5, SEL_SUM, 2, NULL);
1555 + case DB_TYPE_SQLITE: {
1556 + sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
1557 + sqlite3_bind_int(stmt, 1, disk_id);
1558 + sqlite3_bind_int64(stmt, 2, st_p->st_ino);
1559 + for (j = 0; j < 2; j++) {
1561 + if (sqlite3_step(stmt) != SQLITE_ROW)
1563 + len = sqlite3_column_bytes(stmt, 0);
1564 + if (len > MD5_DIGEST_LEN)
1565 + len = MD5_DIGEST_LEN;
1566 + memcpy(dbsum + MD5_DIGEST_LEN*j, sqlite3_column_blob(stmt, 0), len);
1567 + sum_type[j] = sqlite3_column_int(stmt, 1);
1568 + dbsize[j] = sqlite3_column_int(stmt, 2);
1569 + dbmtime[j] = sqlite3_column_int64(stmt, 3);
1570 + dbctime[j] = sqlite3_column_int64(stmt, 4);
1572 + sqlite3_reset(stmt);
1585 + *right_sum_cnt = *wrong_sum_cnt = 0;
1586 + for (j = 0; j < rows; j++) {
1587 + if (DEBUG_GTE(DB, 3)) {
1588 + rprintf(FCLIENT, "DB checksum for %s,%s,%d: %s\n",
1589 + big_num(st_p->st_dev), big_num(st_p->st_ino), sum_type[j],
1590 + sum_as_hex(sum_type[j], dbsum + MD5_DIGEST_LEN*j, 0));
1593 + if (sum_type[j] == 4) {
1596 + *sum4 = dbsum + MD5_DIGEST_LEN*j;
1600 + *sum5 = dbsum + MD5_DIGEST_LEN*j;
1602 + if (st_p->st_size == dbsize[j] && st_p->st_mtime == dbmtime[j] && (db_lax || st_p->st_ctime == dbctime[j]))
1611 +int db_set_checksum(int mdnum, const STRUCT_STAT *st_p, const char *sum)
1613 + unsigned int disk_id;
1614 + const char *errmsg = NULL;
1617 + if (am_receiver || (am_generator && same_db)) {
1618 + /* Forward the setting to a single process. The receiver always
1619 + * forwards to the generator, and the generator will forward to
1620 + * the sender ONLY if this is a local transfer. */
1621 + char data[MSG_CHECKSUM_LEN];
1622 + SIVAL64(data, 0, st_p->st_dev);
1623 + SIVAL64(data, 8, st_p->st_ino);
1624 + SIVAL64(data, 16, st_p->st_size);
1625 + SIVAL64(data, 24, st_p->st_mtime);
1626 + SIVAL64(data, 32, st_p->st_ctime);
1627 +#if MSG_CHECKSUM_LONGS != 5
1628 +#error Fix the setting of checksum long values
1630 + SIVAL(data, MSG_CHECKSUM_LONGS*8, mdnum);
1631 + memcpy(data + MSG_CHECKSUM_LONGS*8 + 4, sum, MAX_DIGEST_LEN);
1632 + return send_msg(MSG_CHECKSUM, data, sizeof data, 0);
1635 + if ((disk_id = get_disk_id(st_p->st_dev)) == 0)
1640 + case DB_TYPE_MYSQL:
1641 + if (transaction_state == 0) {
1642 + if (!run_sql("BEGIN"))
1644 + transaction_state = 1;
1647 + bind_disk_id = disk_id;
1648 + bind_ino = st_p->st_ino;
1649 + bind_mdnum = mdnum;
1650 + bind_size = st_p->st_size;
1651 + bind_mtime = st_p->st_mtime;
1652 + bind_ctime = st_p->st_ctime;
1653 + memcpy(bind_sum, sum, MD5_DIGEST_LEN);
1654 + if (exec_mysql(REP_SUM) == NULL)
1655 + errmsg = mysql_error(dbh.mysql);
1659 + case DB_TYPE_SQLITE: {
1660 + sqlite3_stmt *stmt = statements[REP_SUM].sqlite;
1661 + int lock_failures = 0;
1663 + if (transaction_state == 0) {
1664 + if (!run_sql("BEGIN"))
1666 + transaction_state = 1;
1669 + sqlite3_bind_int(stmt, 1, disk_id);
1670 + sqlite3_bind_int64(stmt, 2, st_p->st_ino);
1671 + sqlite3_bind_int(stmt, 3, mdnum);
1672 + sqlite3_bind_int64(stmt, 4, st_p->st_size);
1673 + sqlite3_bind_int64(stmt, 5, st_p->st_mtime);
1674 + sqlite3_bind_int64(stmt, 6, st_p->st_ctime);
1675 + sqlite3_bind_blob(stmt, 7, sum, MD5_DIGEST_LEN, SQLITE_TRANSIENT);
1677 + rc = sqlite3_step(stmt);
1678 + if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
1680 + if (++lock_failures > MAX_LOCK_FAILURES)
1682 + sqlite3_reset(stmt);
1683 + msleep(LOCK_FAIL_MSLEEP);
1685 + if (rc != SQLITE_DONE)
1686 + errmsg = sqlite3_errmsg(dbh.sqlite);
1687 + sqlite3_reset(stmt);
1694 + if (DEBUG_GTE(DB, 2)) {
1695 + rprintf(FCLIENT, "[%s] Set DB checksum for %s,%s,%d: %s\n",
1696 + who_am_i(), big_num(st_p->st_dev), big_num(st_p->st_ino),
1697 + md_num, sum_as_hex(md_num, sum, 0));
1700 + rprintf(log_code, "[%s] Failed to set checksum for %s,%s,%d: %s (%d) -- closing DB\n",
1701 + who_am_i(), big_num(st_p->st_dev), big_num(st_p->st_ino),
1702 + md_num, errmsg, rc);
1703 + db_disconnect(False);
1706 + return errmsg ? 0 : 1;
1709 +/* For a delayed-update copy, we set the checksum on the file when it was
1710 + * inside the partial-dir. Since renaming the file changes its ctime, we need
1711 + * to update the ctime to its new value (we can skip this in db_lax mode). */
1712 +int db_update_ctime(UNUSED(int mdnum), const STRUCT_STAT *st_p)
1714 + unsigned int disk_id = get_disk_id(st_p->st_dev);
1721 + case DB_TYPE_MYSQL:
1722 + bind_ctime = st_p->st_ctime;
1723 + bind_disk_id = disk_id;
1724 + bind_ino = st_p->st_ino;
1725 + bind_mdnum = mdnum;
1726 + bind_size = st_p->st_size;
1727 + bind_mtime = st_p->st_mtime;
1728 + return exec_mysql(UPD_CTIME) != NULL;
1731 + case DB_TYPE_SQLITE: {
1734 + sqlite3_stmt *stmt = statements[UPD_CTIME].sqlite;
1737 + sqlite3_bind_int64(stmt, 1, st_p->st_ctime);
1738 + sqlite3_bind_int(stmt, 2, disk_id);
1739 + sqlite3_bind_int64(stmt, 3, st_p->st_ino);
1740 + sqlite3_bind_int(stmt, 4, mdnum);
1741 + sqlite3_bind_int64(stmt, 5, st_p->st_size);
1742 + sqlite3_bind_int64(stmt, 6, st_p->st_mtime);
1743 + rc = sqlite3_step(stmt);
1744 + sqlite3_reset(stmt);
1745 + return rc == SQLITE_DONE;
1753 +static int db_clean_init(void)
1757 + case DB_TYPE_MYSQL: {
1758 + MYSQL_BIND binds[MAX_BIND_CNT];
1761 + mysql_query(dbh.mysql,
1762 + "CREATE TEMPORARY TABLE inode_present ("
1763 + " disk_id integer unsigned NOT NULL,"
1764 + " ino bigint unsigned NOT NULL,"
1765 + " PRIMARY KEY (disk_id,ino)"
1769 + sql="INSERT IGNORE INTO inode_present"
1770 + " SET disk_id = ?, ino = ?";
1771 + memset(binds, 0, sizeof binds);
1772 + binds[0].buffer_type = MYSQL_TYPE_LONG;
1773 + binds[0].buffer = &bind_disk_id;
1774 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
1775 + binds[1].buffer = &bind_ino;
1776 + if (!prepare_mysql(INS_PRESENT, binds, 2, sql))
1777 + exit_cleanup(RERR_SYNTAX);
1780 + " FROM inode_map AS m"
1781 + " LEFT JOIN inode_present AS p USING(disk_id, ino)"
1782 + " JOIN disk AS d ON(m.disk_id = d.disk_id)"
1783 + " WHERE host = ? AND devno != 0 AND p.disk_id IS NULL AND ctime < ?";
1784 + memset(binds, 0, sizeof binds);
1785 + binds[0].buffer_type = MYSQL_TYPE_STRING;
1786 + binds[0].buffer = &bind_thishost;
1787 + binds[0].buffer_length = bind_thishost_len;
1788 + binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
1789 + binds[1].buffer = &bind_ctime;
1790 + if (!prepare_mysql(DEL_SUMS, binds, 2, sql))
1791 + exit_cleanup(RERR_SYNTAX);
1797 + case DB_TYPE_SQLITE: {
1799 + sql="ATTACH DATABASE '' AS aux1;"; /* Private temp DB, probably in-memory */
1800 + if (!run_sql(sql))
1801 + exit_cleanup(RERR_IPC);
1803 + sql="CREATE TABLE aux1.inode_present ("
1804 + " disk_id integer NOT NULL,"
1805 + " ino bigint NOT NULL,"
1806 + " PRIMARY KEY (disk_id,ino)"
1808 + if (!run_sql(sql))
1809 + exit_cleanup(RERR_IPC);
1811 + sql="INSERT OR IGNORE INTO aux1.inode_present"
1814 + if (!prepare_sqlite(INS_PRESENT, sql))
1815 + exit_cleanup(RERR_IPC);
1817 + sql="DELETE FROM inode_map"
1818 + " WHERE ROWID IN ("
1820 + " FROM inode_map AS m"
1821 + " LEFT JOIN aux1.inode_present AS p USING(disk_id, ino)"
1822 + " JOIN disk AS d ON(m.disk_id = d.disk_id)"
1823 + " WHERE host = ? AND devno != 0 AND p.disk_id IS NULL AND ctime < ?"
1825 + if (!prepare_sqlite(DEL_SUMS, sql))
1826 + exit_cleanup(RERR_IPC);
1828 + transaction_state = -1; /* bug work-around -- force transaction off when cleaning XXX */
1838 +static int db_note_present(UNUSED(int disk_id), UNUSED(int64 ino))
1842 + case DB_TYPE_MYSQL:
1843 + bind_disk_id = disk_id;
1845 + return exec_mysql(INS_PRESENT) != NULL;
1848 + case DB_TYPE_SQLITE: {
1850 + sqlite3_stmt *stmt = statements[INS_PRESENT].sqlite;
1851 + sqlite3_bind_int(stmt, 1, disk_id);
1852 + sqlite3_bind_int64(stmt, 2, ino);
1853 + rc = sqlite3_step(stmt);
1854 + sqlite3_reset(stmt);
1855 + return rc == SQLITE_DONE;
1863 +/* This function requires the user to have populated all disk_id+inode pairs
1864 + * into the inode_present table. */
1865 +static int db_clean_inodes(UNUSED(time_t start_time))
1869 + /* The extra ctime < start_time check ensures that brand-new checksums that
1870 + * were added after the start of our cleaning run are not removed. */
1873 + case DB_TYPE_MYSQL: {
1875 + bind_ctime = start_time;
1876 + stmt = exec_mysql(DEL_SUMS);
1878 + del_cnt = mysql_affected_rows(dbh.mysql);
1883 + case DB_TYPE_SQLITE: {
1885 + sqlite3_stmt *stmt = statements[DEL_SUMS].sqlite;
1886 + sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1887 + sqlite3_bind_int64(stmt, 2, start_time);
1888 + rc = sqlite3_step(stmt);
1889 + if (rc == SQLITE_DONE)
1890 + del_cnt = sqlite3_changes(dbh.sqlite);
1891 + sqlite3_reset(stmt);
1900 +static int abs_path(char *buf, int bufsiz, const char *curdir, const char *dir)
1903 + strlcpy(buf, dir, bufsiz);
1905 + int len = snprintf(buf, bufsiz, "%s/%s", curdir, dir);
1906 + assert(len > 0); /* silence a compiler warning */
1909 + return clean_fname(buf, CFN_DROP_TRAILING_DOT_DIR | CFN_COLLAPSE_DOT_DOT_DIRS);
1912 +static struct name_list *new_name(const char *basename, const char *filename)
1914 + struct name_list *n;
1915 + int blen = strlen(basename);
1916 + int slen = filename ? (int)strlen(filename) : -1;
1917 + int len = blen + 1 + slen;
1919 + if (len >= MAXPATHLEN) {
1921 + rprintf(FERROR, "Filename too long: %s/%s\n", basename, filename);
1923 + rprintf(FERROR, "Filename too long: %s\n", basename);
1927 + n = (struct name_list *)new_array(char, sizeof (struct name_list) + len);
1929 + memcpy(n->name, basename, blen);
1931 + n->name[blen] = '/';
1932 + memcpy(n->name + 1 + blen, filename, slen);
1934 + n->name[len] = '\0';
1940 +static int name_compare(const void *n1, const void *n2)
1942 + struct name_list *p1 = *(struct name_list **)n1;
1943 + struct name_list *p2 = *(struct name_list **)n2;
1944 + return strcmp(p1->name, p2->name);
1947 +static struct name_list *get_sorted_names(const char *dir)
1949 + struct name_list *add, **sortbuf, *names = NULL, *prior_name = NULL;
1950 + struct dirent *di;
1954 + if (!(d = opendir("."))) {
1955 + rprintf(FERROR, "Unable to opendir %s: %s\n", dir, strerror(errno));
1958 + while ((di = readdir(d)) != NULL) {
1959 + char *dname = d_name(di);
1960 + if (dname[0] == '.' && (dname[1] == '\0' || (dname[1] == '.' && dname[2] == '\0')))
1962 + if (!(add = new_name(dname, NULL)))
1965 + prior_name->next = add;
1976 + sortbuf = new_array(struct name_list *, cnt);
1977 + for (j = 0; j < cnt; j++) {
1978 + sortbuf[j] = names;
1979 + names = names->next;
1982 + qsort(sortbuf, cnt, PTR_SIZE, name_compare);
1984 + names = prior_name = NULL;
1985 + for (j = 0; j < cnt; j++) {
1988 + prior_name->next = add;
1995 + prior_name->next = NULL;
2002 +static inline int sums_ne(const char *sum1, const char *sum2)
2004 + return memcmp(sum1, sum2, MD5_DIGEST_LEN) != 0;
2007 +/* Returns 1 if there is a checksum change, else 0. */
2008 +static int mention_file(const char *dir, const char *name, int right_cnt, int wrong_cnt,
2009 + const char *dbsum4, const char *dbsum5, const char *sum4, const char *sum5)
2011 + char *info_str = wrong_cnt && !right_cnt ? "!i " : " ";
2012 + char *md4_str = !db_do_md4 ? NULL : !dbsum4 ? "+4 " : !sum4 ? "?4 " : sums_ne(sum4, dbsum4) ? "!4 " : " ";
2013 + char *md5_str = !db_do_md5 ? NULL : !dbsum5 ? "+5 " : !sum5 ? "?5 " : sums_ne(sum5, dbsum5) ? "!5 " : " ";
2014 + int chg = *info_str != ' ' || (md4_str && *md4_str != ' ') || (md5_str && *md5_str != ' ');
2015 + if (chg || db_output_unchanged) {
2016 + if (db_output_info) {
2017 + fputs(info_str, stdout);
2019 + fputs(md4_str, stdout);
2021 + fputs(md5_str, stdout);
2023 + if (db_output_sum) {
2025 + printf("%s ", sum_as_hex(4, sum4, 0));
2027 + printf("%s ", sum_as_hex(5, sum5, 0));
2029 + if (db_output_name) {
2030 + if (db_output_sum)
2031 + putchar(' '); /* We want 2 spaces, like md5sum. */
2032 + if (*dir != '.' || dir[1]) {
2033 + fputs(dir, stdout);
2043 +NORETURN void run_dbonly(const char **args)
2045 + char start_dir[MAXPATHLEN], dirbuf[MAXPATHLEN];
2046 + int need_sum_cnt, start_dir_len;
2047 + struct name_list *prior_dir;
2048 + struct name_list *names;
2049 + time_t clean_start = 0;
2050 + int exit_code = 0;
2052 + checksum_type = 5;
2054 + need_sum_cnt = db_do_md4 + db_do_md5;
2056 + if (!db_read_config(FERROR, db_config) || !db_connect(1))
2057 + exit_cleanup(RERR_FILEIO);
2060 + clean_start = time(NULL);
2064 + if (getcwd(start_dir, sizeof start_dir - 1) == NULL) {
2065 + rsyserr(FERROR, errno, "getcwd()");
2066 + exit_cleanup(RERR_FILESELECT);
2068 + start_dir_len = strlen(start_dir);
2073 + struct name_list *add;
2074 + if (abs_path(dirbuf, sizeof dirbuf, start_dir, *args++) <= 0)
2076 + if (!(add = new_name(dirbuf, NULL)))
2079 + prior_dir->next = add;
2085 + dirs_list = new_name(start_dir, NULL);
2088 + while (dirs_list) {
2089 + struct name_list *subdirs, *prior_subdir, *prior_name;
2090 + const char *dir = dirs_list->name;
2091 + const char *reldir = dir;
2094 + free((void*)prior_dir);
2095 + prior_dir = dirs_list;
2096 + dirs_list = dirs_list->next;
2098 + if (strncmp(reldir, start_dir, start_dir_len) == 0) {
2099 + if (reldir[start_dir_len] == '\0')
2101 + else if (reldir[start_dir_len] == '/')
2102 + reldir += start_dir_len + 1;
2104 + if (db_output_dirs)
2105 + printf("... %s/ ...\n", reldir);
2107 + if (chdir(dir) < 0) {
2108 + rprintf(FERROR, "Unable to chdir to %s: %s\n", dir, strerror(errno));
2111 + if (!(names = get_sorted_names(dir)))
2114 + subdirs = prior_subdir = prior_name = NULL;
2117 + char *dbsum4, *sum4, sumbuf4[MD5_DIGEST_LEN];
2118 + char *dbsum5, *sum5, sumbuf5[MD5_DIGEST_LEN];
2119 + int right_sum_cnt, wrong_sum_cnt;
2120 + const char *name = names->name;
2121 + unsigned int disk_id;
2124 + free((void*)prior_name);
2125 + prior_name = names;
2126 + names = names->next;
2128 + dbsum4 = dbsum5 = sum4 = sum5 = NULL;
2130 + if (lstat(name, &st) < 0) {
2131 + rprintf(FERROR, "Failed to lstat(%s): %s\n", name, strerror(errno));
2134 + if (S_ISLNK(st.st_mode))
2136 + if (S_ISDIR(st.st_mode)) {
2137 + /* add optional excluding of things like /^(CVS|\.svn|\.git|\.bzr)$/; */
2139 + struct name_list *add = new_name(dir, name);
2142 + prior_subdir->next = add;
2145 + prior_subdir = add;
2150 + if (!S_ISREG(st.st_mode))
2153 + if (!(disk_id = get_disk_id(st.st_dev)))
2156 + db_note_present(disk_id, st.st_ino);
2157 + if (!db_update && !db_check)
2160 + db_get_both_checksums(&st, &right_sum_cnt, &wrong_sum_cnt,
2161 + db_do_md4 ? &dbsum4 : NULL, db_do_md5 ? &dbsum5 : NULL);
2163 + if (!db_check && right_sum_cnt == need_sum_cnt) {
2164 + mention_file(reldir, name, right_sum_cnt, wrong_sum_cnt, dbsum4, dbsum5, dbsum4, dbsum5);
2168 + if (db_update || (db_check && right_sum_cnt) || db_output_sum) {
2173 + struct map_struct *buf;
2174 + OFF_T off, len = st.st_size;
2175 + int fd = do_open(name, O_RDONLY, 0);
2178 + rprintf(FERROR, "ERROR: unable to read %s: %s\n", name, strerror(errno));
2183 + mdfour_begin(&m4);
2187 + buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
2189 + for (off = 0; off + CSUM_CHUNK <= len; off += CSUM_CHUNK) {
2190 + data = (uchar*)map_ptr(buf, off, CSUM_CHUNK);
2192 + mdfour_update(&m4, data, CSUM_CHUNK);
2194 + MD5_Update(&m5, data, CSUM_CHUNK);
2197 + remainder = (int32)(len - off);
2198 + data = (uchar*)map_ptr(buf, off, remainder);
2200 + mdfour_update(&m4, data, remainder);
2201 + mdfour_result(&m4, (uchar*)(sum4 = sumbuf4));
2204 + MD5_Update(&m5, data, remainder);
2205 + MD5_Final((uchar*)(sum5 = sumbuf5), &m5);
2212 + int chg = mention_file(reldir, name, right_sum_cnt, wrong_sum_cnt, dbsum4, dbsum5, sum4, sum5);
2214 + /* Only db_check should get here... */
2215 + } else if (!db_update) {
2219 + if (db_do_md4 && !db_set_checksum(4, &st, sum4))
2221 + if (db_do_md5 && !db_set_checksum(5, &st, sum5))
2224 + fprintf(stderr, "Failed to set checksum on %s/%s\n", reldir, name);
2225 + exit_cleanup(RERR_FILEIO);
2230 + free((void*)prior_name);
2232 + if (recurse && subdirs) {
2233 + prior_subdir->next = dirs_list;
2234 + dirs_list = subdirs;
2238 + free((void*)prior_dir);
2241 + int rows = db_clean_inodes(clean_start);
2242 + if (db_output_msgs)
2243 + printf("Cleaned out %d old inode%s.\n", rows, rows == 1 ? "" : "s");
2246 + db_disconnect(True);
2249 diff --git a/flist.c b/flist.c
2252 @@ -55,6 +55,7 @@ extern int preserve_devices;
2253 extern int preserve_specials;
2254 extern int delete_during;
2255 extern int missing_args;
2257 extern int eol_nulls;
2258 extern int atimes_ndx;
2259 extern int crtimes_ndx;
2260 @@ -1407,11 +1408,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
2261 extra_len += EXTRA_LEN;
2264 - if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
2265 - file_checksum(thisname, &st, tmp_sum);
2266 - if (sender_keeps_checksum)
2267 - extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
2269 + if (sender_keeps_checksum && S_ISREG(st.st_mode))
2270 + extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
2272 #if EXTRA_ROUNDING > 0
2273 if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
2274 @@ -1500,8 +1498,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
2278 - if (sender_keeps_checksum && S_ISREG(st.st_mode))
2279 - memcpy(F_SUM(file), tmp_sum, flist_csum_len);
2280 + if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
2281 + if (!use_db || !db_get_checksum(&st, tmp_sum))
2282 + file_checksum(thisname, &st, tmp_sum);
2283 + if (sender_keeps_checksum)
2284 + memcpy(F_SUM(file), tmp_sum, flist_csum_len);
2288 F_NDX(file) = stats.num_dirs;
2289 @@ -2185,6 +2187,9 @@ void send_extra_file_list(int f, int at_least)
2291 if (io_error != save_io_error && protocol_version == 30 && !ignore_errors)
2292 send_msg_int(MSG_IO_ERROR, io_error);
2294 + if (use_db && flist_eof)
2295 + db_disconnect(True);
2298 struct file_list *send_file_list(int f, int argc, char *argv[])
2299 @@ -2208,6 +2213,13 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
2300 | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0);
2301 int implied_dot_dir = 0;
2304 + if (always_checksum)
2305 + db_connect(0); /* Will reset use_db on error. */
2310 rprintf(FLOG, "building file list\n");
2311 if (show_filelist_progress)
2312 start_filelist_progress("building file list");
2313 @@ -2553,6 +2565,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
2314 rprintf(FINFO, "[%s] flist_eof=1\n", who_am_i());
2317 + if (use_db && (!inc_recurse || flist_eof))
2318 + db_disconnect(True);
2323 diff --git a/generator.c b/generator.c
2326 @@ -63,6 +63,7 @@ extern int ignore_non_existing;
2327 extern int want_xattr_optim;
2328 extern int modify_window;
2331 extern int append_mode;
2332 extern int make_backups;
2333 extern int csum_length;
2334 @@ -626,6 +627,8 @@ int quick_check_ok(enum filetype ftype, const char *fn, struct file_struct *file
2335 if (always_checksum > 0) {
2336 char sum[MAX_DIGEST_LEN];
2337 file_checksum(fn, st, sum);
2338 + if (!use_db || !db_get_checksum(st, sum))
2339 + file_checksum(fn, st, sum);
2340 return memcmp(sum, F_SUM(file), flist_csum_len) == 0;
2343 @@ -2270,6 +2273,13 @@ void generate_files(int f_out, const char *local_name)
2348 + if (always_checksum || (append_mode != 1 && protocol_version >= 30))
2349 + db_connect(0); /* Will reset use_db on error. */
2354 dflt_perms = (ACCESSPERMS & ~orig_umask);
2357 @@ -2395,6 +2405,9 @@ void generate_files(int f_out, const char *local_name)
2358 wait_for_receiver();
2362 + db_disconnect(True);
2364 info_levels[INFO_FLIST] = save_info_flist;
2365 info_levels[INFO_PROGRESS] = save_info_progress;
2367 diff --git a/io.c b/io.c
2370 @@ -44,6 +44,7 @@ extern int am_generator;
2371 extern int local_server;
2372 extern int msgs2stderr;
2373 extern int inc_recurse;
2374 +extern int same_db;
2375 extern int io_error;
2376 extern int batch_fd;
2377 extern int eol_nulls;
2378 @@ -1545,6 +1546,32 @@ static void read_a_msg(void)
2380 maybe_send_keepalive(time(NULL), MSK_ALLOW_FLUSH);
2382 + case MSG_CHECKSUM:
2383 + /* This receives some checksum info that we want to make a note of
2384 + * (which allows a single process to do all the writing to the db). */
2385 + if (msg_bytes != MSG_CHECKSUM_LEN)
2387 + raw_read_buf(data, MSG_CHECKSUM_LEN);
2388 + if (am_generator && same_db) {
2389 + iobuf.in_multiplexed = 1;
2390 + send_msg(MSG_CHECKSUM, data, MSG_CHECKSUM_LEN, 0);
2391 + } if (am_receiver || (am_sender && !local_server))
2394 + /* The received data is a set of numbers followed by the checksum. */
2396 + st.st_dev = IVAL64(data, 0);
2397 + st.st_ino = IVAL64(data, 8);
2398 + st.st_size = IVAL64(data, 16);
2399 + st.st_mtime = IVAL64(data, 24);
2400 + st.st_ctime = IVAL64(data, 32);
2401 +#if MSG_CHECKSUM_LONGS != 5
2402 +#error Fix the parsing of checksum long values
2404 + iobuf.in_multiplexed = 1;
2405 + db_set_checksum(IVAL(data, MSG_CHECKSUM_LONGS*8), &st, data + MSG_CHECKSUM_LONGS*8 + 4);
2409 if (msg_bytes >= sizeof data)
2411 @@ -1699,6 +1726,7 @@ static void read_a_msg(void)
2412 * with a duplicate exit message. */
2413 _exit_cleanup(val, __FILE__, 0 - __LINE__);
2416 rprintf(FERROR, "unexpected tag %d [%s%s]\n",
2417 tag, who_am_i(), inc_recurse ? "/inc" : "");
2418 exit_cleanup(RERR_STREAMIO);
2419 diff --git a/main.c b/main.c
2422 @@ -39,6 +39,7 @@ extern int am_root;
2423 extern int am_server;
2424 extern int am_sender;
2425 extern int am_daemon;
2426 +extern int am_dbadmin;
2427 extern int inc_recurse;
2428 extern int blocking_io;
2429 extern int always_checksum;
2430 @@ -58,6 +59,7 @@ extern int copy_unsafe_links;
2431 extern int keep_dirlinks;
2432 extern int preserve_hard_links;
2433 extern int protocol_version;
2434 +extern int always_checksum;
2435 extern int mkpath_dest_arg;
2436 extern int file_total;
2438 @@ -95,6 +97,7 @@ extern char *logfile_format;
2439 extern char *filesfrom_host;
2440 extern char *partial_dir;
2441 extern char *rsync_path;
2442 +extern char *db_config;
2443 extern char *shell_cmd;
2444 extern char *password_file;
2445 extern char *backup_dir;
2446 @@ -1238,6 +1241,9 @@ void start_server(int f_in, int f_out, int argc, char *argv[])
2447 if (am_daemon && io_timeout && protocol_version >= 31)
2448 send_msg_int(MSG_IO_TIMEOUT, io_timeout);
2451 + db_read_config(FERROR, db_config);
2454 keep_dirlinks = 0; /* Must be disabled on the sender. */
2455 if (need_messages_from_generator)
2456 @@ -1535,6 +1541,9 @@ static int start_client(int argc, char *argv[])
2458 env_port = rsync_port;
2461 + db_read_config(FERROR, db_config);
2463 if (daemon_connection < 0)
2464 return start_socket_client(shell_machine, remote_argc, remote_argv, argc, argv);
2466 diff --git a/options.c b/options.c
2469 @@ -90,6 +90,7 @@ int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
2472 int am_starting_up = 1;
2473 +int am_dbadmin = 0;
2474 int relative_paths = -1;
2475 int implied_dirs = 1;
2476 int missing_args = 0; /* 0 = FERROR_XFER, 1 = ignore, 2 = delete */
2477 @@ -104,6 +105,7 @@ int use_qsort = 0;
2478 char *files_from = NULL;
2479 int filesfrom_fd = -1;
2480 char *filesfrom_host = NULL;
2481 +char *db_config = NULL;
2483 int protect_args = -1;
2484 int old_style_args = -1;
2485 @@ -113,6 +115,9 @@ int mkpath_dest_arg = 0;
2486 int allow_inc_recurse = 1;
2489 +int db_clean, db_check, db_do_md4, db_do_md5, db_update = 1, db_lax, db_init, db_mounts;
2490 +int db_output_name, db_output_sum, db_output_info, db_output_unchanged, db_output_dirs, db_output_msgs;
2491 +int saw_db_output_opt, saw_db_sum_opt;
2492 int connect_timeout = 0;
2493 int keep_partial = 0;
2494 int safe_symlinks = 0;
2495 @@ -292,6 +297,7 @@ static struct output_struct debug_words[COUNT_DEBUG+1] = {
2496 DEBUG_WORD(CHDIR, W_CLI|W_SRV, "Debug when the current directory changes"),
2497 DEBUG_WORD(CONNECT, W_CLI, "Debug connection events (levels 1-2)"),
2498 DEBUG_WORD(CMD, W_CLI, "Debug commands+options that are issued (levels 1-2)"),
2499 + DEBUG_WORD(DB, W_SND|W_REC, "Debug DB operations (levels 1-5)"),
2500 DEBUG_WORD(DEL, W_REC, "Debug delete actions (levels 1-3)"),
2501 DEBUG_WORD(DELTASUM, W_SND|W_REC, "Debug delta-transfer checksumming (levels 1-4)"),
2502 DEBUG_WORD(DUP, W_REC, "Debug weeding of duplicate names"),
2503 @@ -581,6 +587,7 @@ enum {OPT_SERVER = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
2504 OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD,
2505 OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
2506 OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG, OPT_BLOCK_SIZE,
2507 + OPT_NO_DB, OPT_DBONLY,
2508 OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT, OPT_STDERR,
2509 OPT_OLD_COMPRESS, OPT_NEW_COMPRESS, OPT_NO_COMPRESS, OPT_OLD_ARGS,
2510 OPT_STOP_AFTER, OPT_STOP_AT,
2511 @@ -738,6 +745,10 @@ static struct poptOption long_options[] = {
2512 {"no-c", 0, POPT_ARG_VAL, &always_checksum, 0, 0, 0 },
2513 {"checksum-choice", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
2514 {"cc", 0, POPT_ARG_STRING, &checksum_choice, 0, 0, 0 },
2515 + {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
2516 + {"no-db", 0, POPT_ARG_NONE, 0, OPT_NO_DB, 0, 0 },
2517 + {"db-lax", 0, POPT_ARG_VAL, &db_lax, 1, 0, 0 },
2518 + {"no-db-lax", 0, POPT_ARG_VAL, &db_lax, 0, 0, 0 },
2519 {"block-size", 'B', POPT_ARG_STRING, 0, OPT_BLOCK_SIZE, 0, 0 },
2520 {"compare-dest", 0, POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
2521 {"copy-dest", 0, POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
2522 @@ -840,6 +851,9 @@ static struct poptOption long_options[] = {
2523 {"dparam", 0, POPT_ARG_STRING, 0, OPT_DAEMON, 0, 0 },
2524 {"detach", 0, POPT_ARG_NONE, 0, OPT_DAEMON, 0, 0 },
2525 {"no-detach", 0, POPT_ARG_NONE, 0, OPT_DAEMON, 0, 0 },
2526 + /* All the following options switch us into DB-admin option-parsing. */
2527 + {"db-help", 0, POPT_ARG_NONE, 0, OPT_DBONLY, 0, 0 },
2528 + {"db-only", 0, POPT_ARG_STRING, 0, OPT_DBONLY, 0, 0 },
2532 @@ -868,6 +882,31 @@ static struct poptOption long_daemon_options[] = {
2536 +static struct poptOption long_dbonly_options[] = {
2537 + /* longName, shortName, argInfo, argPtr, value, descrip, argDesc */
2538 + {"check", 'c', POPT_ARG_NONE, &db_check, 0, 0, 0},
2539 + {"clean", 0, POPT_ARG_NONE, &db_clean, 0, 0, 0},
2540 + {"db", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
2541 + {"db-only", 0, POPT_ARG_STRING, &db_config, 0, 0, 0 },
2542 + {"db-lax", 0, POPT_ARG_VAL, &db_lax, 1, 0, 0 },
2543 + {"no-db-lax", 0, POPT_ARG_VAL, &db_lax, 0, 0, 0 },
2544 + {"info", 0, POPT_ARG_STRING, 0, OPT_INFO, 0, 0 },
2545 + {"debug", 0, POPT_ARG_STRING, 0, OPT_DEBUG, 0, 0 },
2546 + {"update", 'u', POPT_ARG_VAL, &db_update, 1, 0, 0 },
2547 + {"no-update", 'N', POPT_ARG_VAL, &db_update, 0, 0, 0 },
2548 + {"no-u", 0, POPT_ARG_VAL, &db_update, 0, 0, 0 },
2549 + {"output", 'o', POPT_ARG_STRING, 0, 'o', 0, 0 },
2550 + {"recursive", 'r', POPT_ARG_VAL, &recurse, 1, 0, 0 },
2551 + {"no-recursive", 0, POPT_ARG_VAL, &recurse, 0, 0, 0 },
2552 + {"no-r", 0, POPT_ARG_VAL, &recurse, 0, 0, 0 },
2553 + {"sums", 's', POPT_ARG_STRING, 0, 's', 0, 0 },
2554 + {"init", 0, POPT_ARG_NONE, &db_init, 0, 0, 0 },
2555 + {"mounts", 0, POPT_ARG_NONE, &db_mounts, 0, 0, 0 },
2556 + {"quiet", 'q', POPT_ARG_NONE, &quiet, 0, 0, 0 },
2557 + {"help", 'h', POPT_ARG_NONE, 0, 'h', 0, 0 },
2558 + {"db-help", 0, POPT_ARG_NONE, 0, 'h', 0, 0 },
2559 + {0,0,0,0, 0, 0, 0}
2562 static char err_buf[200];
2564 @@ -995,6 +1034,8 @@ static void set_refuse_options(void)
2565 parse_one_refuse_match(0, "iconv", list_end);
2567 parse_one_refuse_match(0, "log-file*", list_end);
2568 + parse_one_refuse_match(0, "db", list_end);
2569 + parse_one_refuse_match(0, "db-lax", list_end);
2572 #ifndef SUPPORT_ATIMES
2573 @@ -1302,6 +1343,102 @@ static void create_refuse_error(int which)
2574 snprintf(err_buf + n, sizeof err_buf - n, " (-%c)\n", op->shortName);
2577 +static NORETURN void parse_dbonly_args(int argc, const char **argv)
2579 + poptContext pc = poptGetContext(RSYNC_NAME, argc, argv, long_dbonly_options, 0);
2586 + while ((opt = poptGetNextOpt(pc)) != -1) {
2590 + for (cp = poptGetOptArg(pc); *cp; cp++) {
2591 + switch (toLower(cp)) {
2593 + db_output_name = 1;
2597 + db_output_sum = db_output_name = 1;
2600 + db_output_info = db_output_name = 1;
2603 + db_output_unchanged = db_output_name = 1;
2606 + db_output_dirs = 1;
2610 + saw_db_output_opt = 1;
2614 + for (cp = poptGetOptArg(pc); *cp; cp++) {
2624 + saw_db_sum_opt = 1;
2628 + dbonly_usage(FINFO);
2632 + arg = poptGetOptArg(pc);
2633 + parse_output_words(info_words, info_levels, arg, USER_PRIORITY);
2637 + arg = poptGetOptArg(pc);
2638 + parse_output_words(debug_words, debug_levels, arg, USER_PRIORITY);
2643 + "rsyncdb: %s: %s\n",
2644 + poptBadOption(pc, POPT_BADOPTION_NOALIAS),
2645 + poptStrerror(opt));
2646 + goto dbonly_usage;
2651 + rprintf(FERROR, "You must specify the --db=FILE option.\n");
2654 + "(Type \"rsyncdb --help\" for assistance.)\n");
2655 + exit_cleanup(RERR_SYNTAX);
2658 + if (!saw_db_output_opt && !quiet) {
2659 + db_output_dirs = db_output_name = 1;
2661 + db_output_info = 1;
2664 + db_output_msgs = 1;
2665 + if (!saw_db_sum_opt)
2668 + am_starting_up = 0;
2669 + run_dbonly(poptGetArgs(pc));
2670 + exit(42); /* NOT REACHED */
2673 /* This is used to make sure that --daemon & --server cannot be aliased to
2674 * something else. These options have always disabled popt aliases for the
2675 * parsing of a daemon or server command-line, but we have to make sure that
2676 @@ -1358,6 +1495,12 @@ int parse_arguments(int *argc_p, const char ***argv_p)
2680 + arg = *argv + strlen(*argv);
2681 + if (arg - *argv > 2 && strcmp(arg-2, "db") == 0) {
2682 + parse_dbonly_args(argc, argv);
2686 set_refuse_options();
2689 @@ -1476,6 +1619,12 @@ int parse_arguments(int *argc_p, const char ***argv_p)
2695 + poptFreeContext(pc);
2696 + parse_dbonly_args(argc, argv);
2697 + break; /* NOT REACHED */
2699 case OPT_MODIFY_WINDOW:
2700 /* The value has already been set by popt, but
2701 * we need to remember that we're using a
2702 @@ -1548,6 +1697,10 @@ int parse_arguments(int *argc_p, const char ***argv_p)
2703 preserve_devices = preserve_specials = 0;
2713 diff --git a/pipe.c b/pipe.c
2716 @@ -27,11 +27,16 @@ extern int am_server;
2717 extern int blocking_io;
2718 extern int filesfrom_fd;
2719 extern int munge_symlinks;
2720 +extern int always_checksum;
2722 +extern char *db_config;
2723 extern char *logfile_name;
2724 extern int remote_option_cnt;
2725 extern const char **remote_options;
2726 extern struct chmod_mode_struct *chmod_modes;
2731 * Create a child connected to us via its stdin/stdout.
2733 @@ -141,13 +146,22 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
2736 if (remote_option_cnt) {
2737 + const char *db_config_save = db_config;
2738 int rc = remote_option_cnt + 1;
2739 const char **rv = remote_options;
2740 if (!parse_arguments(&rc, &rv)) {
2742 exit_cleanup(RERR_SYNTAX);
2745 + if (db_config == db_config_save)
2746 + same_db = db_config != NULL;
2747 + else if (!db_config || !db_config_save || strcmp(db_config, db_config_save) != 0) {
2750 + db_read_config(FERROR, db_config);
2752 + } else if (use_db)
2755 if (dup2(to_child_pipe[0], STDIN_FILENO) < 0
2756 || close(to_child_pipe[1]) < 0
2757 diff --git a/receiver.c b/receiver.c
2763 extern int do_xfers;
2767 extern int am_server;
2768 extern int inc_recurse;
2769 @@ -439,6 +441,11 @@ static void handle_delayed_updates(char *local_name)
2770 "rename failed for %s (from %s)",
2771 full_fname(fname), partialptr);
2773 + if (use_db && !db_lax) {
2775 + if (do_lstat(fname, &st) == 0)
2776 + db_update_ctime(5, &st);
2778 if (remove_source_files || (preserve_hard_links && F_IS_HLINKED(file)))
2779 send_msg_success(fname, ndx);
2780 handle_partial_dir(partialptr, PDIR_DELETE);
2781 @@ -547,6 +554,9 @@ int recv_files(int f_in, int f_out, char *local_name)
2785 + if (use_db && (append_mode == 1 || protocol_version < 30))
2786 + use_db = 0; /* We can't note finished md5 values */
2791 @@ -897,6 +907,8 @@ int recv_files(int f_in, int f_out, char *local_name)
2792 do_unlink(partialptr);
2793 handle_partial_dir(partialptr, PDIR_DELETE);
2795 + if (use_db && do_lstat(fname, &st) == 0)
2796 + db_set_checksum(5, &st, sender_file_sum);
2797 } else if (keep_partial && partialptr && (!one_inplace || delay_updates)) {
2798 if (!handle_partial_dir(partialptr, PDIR_CREATE)) {
2800 @@ -910,6 +922,8 @@ int recv_files(int f_in, int f_out, char *local_name)
2802 else if (delay_updates && recv_ok) {
2803 bitbag_set_bit(delayed_bits, ndx);
2804 + if (use_db && do_lstat(partialptr, &st) == 0)
2805 + db_set_checksum(5, &st, sender_file_sum);
2809 diff --git a/rsync.1.md b/rsync.1.md
2812 @@ -434,6 +434,9 @@ has its own detailed description later in this manpage.
2813 --dry-run, -n perform a trial run with no changes made
2814 --whole-file, -W copy files whole (w/o delta-xfer algorithm)
2815 --checksum-choice=STR choose the checksum algorithm (aka --cc)
2816 +--db=CONFIG_FILE specify a CONFIG_FILE for DB checksums
2817 +--db-only=CONFIG_FILE behave like rsyncdb
2818 +--db-lax ignore ctime changes (use with CAUTION)
2819 --one-file-system, -x don't cross filesystem boundaries
2820 --block-size=SIZE, -B force a fixed checksum block-size
2821 --rsh=COMMAND, -e specify the remote shell to use
2822 diff --git a/rsync.c b/rsync.c
2825 @@ -41,6 +41,7 @@ extern int am_daemon;
2826 extern int am_sender;
2827 extern int am_receiver;
2828 extern int am_generator;
2829 +extern int am_dbadmin;
2830 extern int am_starting_up;
2831 extern int allow_8bit_chars;
2832 extern int protocol_version;
2833 @@ -819,6 +820,8 @@ struct file_list *flist_for_ndx(int ndx, const char *fatal_error_loc)
2835 const char *who_am_i(void)
2840 return am_server ? "server" : "client";
2841 return am_sender ? "sender"
2842 diff --git a/rsync.h b/rsync.h
2845 @@ -271,6 +271,7 @@ enum msgcode {
2846 MSG_IO_ERROR=22,/* the sending side had an I/O error */
2847 MSG_IO_TIMEOUT=33,/* tell client about a daemon's timeout value */
2848 MSG_NOOP=42, /* a do-nothing message (legacy protocol-30 only) */
2849 + MSG_CHECKSUM=55,/* sent via rcvr -> gen pipe and local-host-only gen -> sender */
2850 MSG_ERROR_EXIT=86, /* synchronize an error exit (siblings and protocol >= 31) */
2851 MSG_SUCCESS=100,/* successfully updated indicated flist index */
2852 MSG_DELETED=101,/* successfully deleted a file on receiving side */
2853 @@ -281,6 +282,9 @@ enum filetype {
2854 FT_UNSUPPORTED, FT_REG, FT_DIR, FT_SYMLINK, FT_SPECIAL, FT_DEVICE
2857 +#define MSG_CHECKSUM_LONGS 5
2858 +#define MSG_CHECKSUM_LEN (MSG_CHECKSUM_LONGS*8 + 4 + MAX_DIGEST_LEN)
2861 #define NDX_FLIST_EOF -2
2862 #define NDX_DEL_STATS -3
2863 @@ -1441,7 +1445,8 @@ extern short info_levels[], debug_levels[];
2864 #define DEBUG_CHDIR (DEBUG_BIND+1)
2865 #define DEBUG_CONNECT (DEBUG_CHDIR+1)
2866 #define DEBUG_CMD (DEBUG_CONNECT+1)
2867 -#define DEBUG_DEL (DEBUG_CMD+1)
2868 +#define DEBUG_DB (DEBUG_CMD+1)
2869 +#define DEBUG_DEL (DEBUG_DB+1)
2870 #define DEBUG_DELTASUM (DEBUG_DEL+1)
2871 #define DEBUG_DUP (DEBUG_DELTASUM+1)
2872 #define DEBUG_EXIT (DEBUG_DUP+1)
2873 diff --git a/rsyncdb-mountinfo b/rsyncdb-mountinfo
2874 new file mode 100755
2876 +++ b/rsyncdb-mountinfo
2880 +# This script outputs data for rsyncdb --mounts. It must output a complete
2881 +# list of the mounts for the current host in a strict format -- 2 fields
2882 +# with a Tab between: $MOUNT_UNIQ\t$PATH
2884 +# The list of mounts MUST NOT contain any entry that has the same devnum
2885 +# (st_dev) as any other entry in the list (as checked via its PATH).
2887 +# MOUNT_UNIQ is a unique string that identifies the mount on this host.
2888 +# This cannot be the devnum (st_dev) because that can vary depending on the
2889 +# mount order or be reused for different mounts if they are not mounted at
2890 +# the same time. Ideally this would be its UUID value, if that is available
2891 +# on this OS. This script looks in /dev/disk/by-uuid for the current UUID
2892 +# mappings). If the UUID is not found, the fallback default is the string
2893 +# "Mount of $devname", which should be adequate for situations that don't
2894 +# use removable media (though you may need to take steps to weed-out removable
2897 +# You can override the MOUNT_UNIQ value by putting a .rsyncdb_mount_uniq
2898 +# file in the root directory of any mount, at which point it is up to you
2899 +# to make sure that the value stays unique (note that all sequences of
2900 +# whitespace are transformed into a single space, and leading/trailing
2901 +# whitespace is removed).
2903 +# MOUNT_UNIQ may never contain a Tab but it would be legal for PATH to have
2904 +# a Tab (just really weird). Neither may have a CR or LF in it.
2906 +# The maximum size for MOUNT_UNIQ is 256 characters.
2908 +# If this script doesn't meet your needs, feel free to edit/replace it and
2909 +# choose some other method of finding a unique value for each mount. If you
2910 +# come up with a good idiom that might be useful to others, please share it
2911 +# with the rsync mailing list.
2915 +use Cwd 'abs_path';
2917 +my @MOUNT_FILES = qw( /proc/mounts /etc/mtab );
2918 +my $VALID_DEVICE_REGEX = qr{^/dev|^rootfs$};
2919 +my $UUID_DIR = '/dev/disk/by-uuid';
2920 +my $OVERRIDE_FILE = '.rsyncdb_mount_uniq';
2924 +if (-d $UUID_DIR) {
2925 + foreach my $uuid (glob "$UUID_DIR/*") {
2926 + my $lnk = readlink($uuid);
2927 + if ($lnk !~ m{^/}) {
2928 + $lnk = abs_path("$UUID_DIR/$lnk");
2930 + $uuid =~ s{.*/}{};
2931 + $uuid{$lnk} = $uuid;
2935 +foreach my $mount_file (@MOUNT_FILES) {
2936 + if (open MOUNTS, $mount_file) {
2937 + while (<MOUNTS>) {
2938 + my ($devname, $path) = (split)[0,1];
2939 + next unless $devname =~ /$VALID_DEVICE_REGEX/;
2941 + my ($devno) = (stat($path))[0];
2942 + next unless defined $devno; # Skip if mount is invalid.
2943 + next if $hash{$devno}++; # SKip if we've seen this devno earlier.
2945 + my $mount_uniq = $uuid{$devname} ? $uuid{$devname} : "Mount of $devname";
2946 + if (open UNIQ, '<', "$path/$OVERRIDE_FILE") {
2947 + $mount_uniq = <UNIQ>;
2949 + $mount_uniq =~ s/\s+/ /g; # This ensures no tab, CR, nor LF.
2950 + $mount_uniq =~ s/^ | $//g; # .. and no leading or trailing whitespace.
2952 + print $mount_uniq, "\t", $path, "\n";
2959 +die "Failed to to open any mount files: @MOUNT_FILES\n";
2960 diff --git a/rsyncdb.1.md b/rsyncdb.1.md
2961 new file mode 100644
2967 +rsyncdb - Maintain an rsync checksum DB
2972 +rsyncdb --db=CONFIG [OPTION...] [DIR...]
2977 +Rsyncdb can maintain a checksum-caching DB that rsync can use to make its
2978 +`--checksum` option more optimal. You must specify a config file via
2979 +the `--db=CONFIG_FILE` option in order for rsyncdb to know what DB to
2980 +manipulate. See the rsync manpage's `--db` option for full details on
2983 +You can specify one or more directory args for rsyncdb to scan. If no
2984 +DIR args are specified, the current directory is assumed to be the spot
2987 +Note that the rsyncdb program is usually just a symlink to the rsync program.
2988 +You can force rsync to behave as rsyncdb either by having a symlink (or
2989 +hardlink) name that ends with "db" or by `starting` the rsync args with
2990 +`--db-only=CONFIG` (and that option works just like `--db=CONFIG` to
2991 +a program named rsyncdb).
2995 +The following command will update checksum information in the database
2996 +described in the /etc/db.conf file:
2998 +> rsyncdb --db=/etc/db.conf -o n --clean /dir1 /dir2
3000 +It scans 2 directory hierarchies (/dir1 & /dir2) and cleans out any
3001 +checksums whose inodes are no longer found in those directories (so that
3002 +directory args are presumed to be complete for this host's DB contents).
3004 +The following command will scan all the files in the /dir2 directory (without
3005 +recursive scanning, due to the `--no-r` option) and check them against
3008 +> rsyncdb --db=/etc/db.conf --check --no-r /dir2
3010 +Any errors found are output as well as being fixed in the DB. (See
3011 +`--no-update` for how to check without updating.)
3013 +The following command will output MD5 sums for all the files found in the
3014 +directories mentioned, even if they are unchanged (due to the
3015 +`--output=us` option):
3017 +> rsyncdb --db=/etc/db.conf -rous /dir* >/tmp/md5sums.txt
3019 +This is just like running md5sum, only faster. Unlike md5sum, you can't
3020 +specify a single file, so use `--no-r` and grep the output if you just
3021 +want to see a single file's value.
3023 +The following command initializes a new DB, and is required for any new DB:
3025 +> rsyncdb --db=/etc/db.conf --init --mounts
3027 +The `--init` option should only be used once (unless you want to
3028 +destroy existing data). The `--mounts` option may need to be used
3029 +periodically, and makes use of a helper script (see below).
3033 +Rsyncdb accepts the following options:
3035 +[comment]: # (help-rsyncdb.h)
3038 +--db=CONFIG Specify the CONFIG file to read for the DB info
3039 +--db-lax Ignore ctime changes (use with CAUTION)
3040 +--recursive, -r Scan files in subdirs (the default w/o --no-recursive)
3041 +--sums=SUMS, -s List which checksums to update (default: 4,5)
3042 +--output=STR, -o One or more letters of what to output (default: "")
3043 +--check, -c Check checksums (by reading the files) and fix any
3044 + issues. Makes --output default to "dni".
3045 +--clean Note all inodes in the DIRS and remove DB extras
3046 +--no-update, -N Avoids updating/adding info w/--check and/or --clean
3047 +--init Initialize a DB by (re-)creating its tables
3048 +--mounts Scan for mounted filesystems and update the DB
3049 +--quiet, -q Disable the default non-error output
3050 +--help, -h Display this help message
3055 +Rsyncdb accepts both long (double-dash + word) and short (single-dash + letter)
3056 +options. The full list of the available options are described below. If an
3057 +option can be specified in more than one way, the choices are comma-separated.
3058 +Some options only have a long variant, not a short. If the option takes a
3059 +parameter, the parameter is only listed after the long variant, even though it
3060 +must also be specified for the short. When specifying a parameter, you can
3061 +either use the form --option=param or replace the '=' with whitespace. The
3062 +parameter may need to be quoted in some manner for it to survive the shell's
3063 +command-line parsing.
3065 +0. `--db=CONFIG_FILE`
3067 + This tells rsyncdb what DB-config file to read for the DB setup. This is
3068 + the same as the option in rsync, so refer to that manpage for full details.
3072 + This option works just like it does in rsync, so refer to that manpage for
3075 +0. `--no-recursive, --no-r`
3077 + This disables the default recursive directory scan that is performed on the
3078 + listed directory args. The options `--recursive` and `-r` are also
3079 + accepted, if someone wants to override an earlier `--no-r` override.
3081 +0. `--sums=SUMS, -s`
3083 + Only output/update the listed checksum types. By default we deal with just
3084 + the newer md5 checksums (i.e. `--sums=5`).
3086 + Note that this option does NOT affect the order that checksums are output
3087 + if "-o s" is enabled, so `-s5,4` is the same as `-s4,5`.
3089 +0. `--output=STR, -o`
3091 + The output option lets you specify one or more letters indicating what
3092 + information should be output. If `--output` is not specified, the default
3093 + is either "dn" or (with `--check`) "dni".
3095 + The following letters are accepted in the string:
3097 + - `d` outputs "... dir_name ..." lines for each directory in our scan. if
3098 + "d" is omitted, then this progress indictor is not output.
3099 + - `n` includes the file's name in the per-file output. These lines are only
3100 + output for changed files unless "u" is given. The "n" option is implied
3101 + by every other output option letter except "d".
3102 + - `s` includes the checksum info in the per-file output.
3103 + - `c` is a synonym for 's'.
3104 + - `i` includes itemized change info in the per-file output.
3105 + - `!i` indicates that the time and/or size is wrong.
3106 + - `+4` indicates the MD4 sum is missing.
3107 + - `+5` indicates the MD5 sum is missing.
3108 + - `!4` indicates the MD4 sum is wrong.
3109 + - `!5` indicates the MD5 sum is wrong.
3110 + - `?4` indicates an unknown MD4 difference. This can happen if we didn't
3111 + need to read the file; i.e. if the time/size is wrong and no sum info
3113 + - `?5` indicates an unknown MD5 difference.
3114 + - `u` includes unchanged files in the per-file output lines.
3118 + Check the checksums (forcing the reading of all the files) and fix any
3119 + issues that are found. Makes `--output` default to "dni".
3123 + Makes a temp-DB of all the inodes that we find in all the listed
3124 + directories and removes any extraneous checksums from the DB. You will
3125 + need to specify all the mounted directories that are present (and listed as
3126 + mounted) in the DB on this host or else the checksums from the unvisited
3127 + directories will be discarded from the DB. If you want to just --clean
3128 + without adding or updating the info of new or changed files, specify
3129 + `--no-update` as well.
3131 +0. `--no-update, -N`
3133 + Avoids updating/adding info with `--check` and/or `--clean`.
3137 + Disable the default (non-error) output settings. This turns off the
3138 + messages that `--init`, `--mount`, and `--clean` output, and makes the
3139 + default for `--output` be nothing (though an explicit `--output` option is
3144 + Create the tables in the DB. If it is used on an existing DB, all the
3145 + existing tables are dropped and re-created.
3147 +This option cannot be combined with the updating or reporting of checksum
3148 +information, but may be combined with `--mounts`.
3152 + Populate the "disk" DB with the available device numbers and change any
3153 + mounted/unmount information for devices. This should be run every time a
3154 + mount-change happens that may affect a directory hierarchy in the DB.
3155 + Rsyncdb will not save any checksums for a device that is not listed in the
3158 + The helper script "rsyncdb-mountinfo" is used as the source of the mount
3159 + information on the host, which it derives from various system files and
3160 + UUID directories (if available). That script supports the use of an
3161 + override file named ".rsyncdb_mount_uniq" in the root of the mount as one
3162 + way to manually assign unique values to a shared (mountable) device's
3165 + Some advanced users may want to maintain the disk table themselves in order
3166 + to support mounting a drive in different (or multiple) locations, etc.
3168 + Specifying the `--mounts` option cannot be combined with updating or
3169 + reporting of checksum information, but may be combined with `--init`.
3173 + Display a summary of the options.
3181 +Rsyncdb was written by Wayne Davison.
3182 diff --git a/usage.c b/usage.c
3185 @@ -137,6 +137,16 @@ static void print_info_flags(enum logcode f)
3189 +#if !defined HAVE_MYSQL_MYSQL_H || !defined HAVE_LIBMYSQLCLIENT
3194 +#if !defined HAVE_SQLITE3_H || !defined HAVE_LIBSQLITE3
3201 #ifndef USE_ROLL_SIMD
3202 @@ -263,6 +273,14 @@ void daemon_usage(enum logcode F)
3203 rprintf(F,"daemon-specific rsync options. See also the rsyncd.conf(5) manpage.\n");
3206 +void dbonly_usage(enum logcode F)
3208 + rprintf(F,"Usage: rsyncdb --db=CONFIG_FILE [OPTIONS] [DIRS]\n");
3210 + rprintf(F,"Options:\n");
3211 +#include "help-rsyncdb.h"
3214 const char *rsync_version(void)