db.diff

   1 Added some DB-access routines to help rsync keep extra filesystem info
   2 about the files it is dealing with.  This adds both the --db=CONFIG_FILE
   3 option and the "db config" daemon parameter.
   4
   5 Future improvements may include:
   6
   7  - Updating of MD4 checksums when transferring any file, even w/o -c.
   8    To make that work we'd need to make the sender force checksum_seed to
   9    0 when using a DB and having the receiving side check to see if it
  10    got a 0 checksum_seed.
  11
  12  - Caching of path info that allows for the finding of files to use for
  13    moving/linking/copying/alternate-basis-use.
  14
  15  - Extend DB support beyond MySQL and SQLite (PostgreSQL?).
  16
  17 To use this patch, run these commands for a successful build:
  18
  19     patch -p1 <patches/db.diff
  20     ./prepare-source
  21     ./configure
  22     make
  23
  24 based-on: a3f852bd7864ffd61ae709bc756e73034d4c33a3
  25 diff --git a/.gitignore b/.gitignore
  26 --- a/.gitignore
  27 +++ b/.gitignore
  28 @@ -15,6 +15,7 @@ config.status
  29  /proto.h
  30  /proto.h-tstamp
  31  /rsync.1
  32 +/rsyncdb.1
  33  /rsyncd.conf.5
  34  /autom4te*.cache
  35  /confdefs.h
  36 @@ -23,6 +24,7 @@ config.status
  37  /getgroups
  38  /gmon.out
  39  /rsync
  40 +/rsyncdb
  41  /rsync-ssl
  42  /stunnel-rsync
  43  /stunnel-rsyncd.conf
  44 diff --git a/Makefile.in b/Makefile.in
  45 --- a/Makefile.in
  46 +++ b/Makefile.in
  47 @@ -6,6 +6,7 @@ datarootdir=@datarootdir@
  48  exec_prefix=@exec_prefix@
  49  stunnel4=@STUNNEL4@
  50  bindir=@bindir@
  51 +sbindir=@sbindir@
  52  mandir=@mandir@
  53
  54  LIBS=@LIBS@
  55 @@ -28,7 +29,7 @@ VERSION=@RSYNC_VERSION@
  56  .SUFFIXES:
  57  .SUFFIXES: .c .o
  58
  59 -GENFILES=configure.sh config.h.in proto.h proto.h-tstamp rsync.1 rsyncd.conf.5
  60 +GENFILES=configure.sh config.h.in proto.h proto.h-tstamp rsync.1 rsyncdb.1 rsyncd.conf.5
  61  HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \
  62         lib/pool_alloc.h
  63  LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \
  64 @@ -38,7 +39,7 @@ zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \
  65  OBJS1=flist.o rsync.o generator.o receiver.o cleanup.o sender.o exclude.o \
  66         util.o util2.o main.o checksum.o match.o syscall.o log.o backup.o delete.o
  67  OBJS2=options.o io.o compat.o hlink.o token.o uidlist.o socket.o hashtable.o \
  68 -       fileio.o batch.o clientname.o chmod.o acls.o xattrs.o
  69 +       fileio.o batch.o clientname.o chmod.o db.o acls.o xattrs.o
  70  OBJS3=progress.o pipe.o
  71  DAEMON_OBJ = params.o loadparm.o clientserver.o access.o connection.o authenticate.o
  72  popt_OBJS=popt/findme.o  popt/popt.o  popt/poptconfig.o \
  73 @@ -62,14 +63,17 @@ CHECK_OBJS=tls.o testrun.o getgroups.o getfsdev.o t_stub.o t_unsafe.o trimslash.
  74         $(CC) -I. -I$(srcdir) $(CFLAGS) $(CPPFLAGS) -c $< @CC_SHOBJ_FLAG@
  75  @OBJ_RESTORE@
  76
  77 -all: Makefile rsync$(EXEEXT) rsync-ssl stunnel-rsync stunnel-rsyncd.conf @MAKE_MAN@
  78 +all: Makefile rsync$(EXEEXT) rsyncdb$(EXEEXT) rsync-ssl stunnel-rsync stunnel-rsyncd.conf @MAKE_MAN@
  79
  80  install: all
  81 -       -${MKDIR_P} ${DESTDIR}${bindir}
  82 +       -${MKDIR_P} ${DESTDIR}${bindir} ${DESTDIR}${sbindir}
  83         ${INSTALLCMD} ${INSTALL_STRIP} -m 755 rsync$(EXEEXT) ${DESTDIR}${bindir}
  84 +       rsync -ilt rsyncdb$(EXEEXT) ${DESTDIR}${bindir}/
  85 +       ${INSTALLCMD} ${INSTALL_STRIP} -m 755 rsyncdb-mountinfo ${DESTDIR}${sbindir}
  86         -${MKDIR_P} ${DESTDIR}${mandir}/man1
  87         -${MKDIR_P} ${DESTDIR}${mandir}/man5
  88         if test -f rsync.1; then ${INSTALLMAN} -m 644 rsync.1 ${DESTDIR}${mandir}/man1; fi
  89 +       if test -f rsyncdb.1; then ${INSTALLMAN} -m 644 rsyncdb.1 ${DESTDIR}${mandir}/man1; fi
  90         if test -f rsyncd.conf.5; then ${INSTALLMAN} -m 644 rsyncd.conf.5 ${DESTDIR}${mandir}/man5; fi
  91
  92  install-ssl-client: rsync-ssl stunnel-rsync
  93 @@ -92,6 +96,9 @@ install-strip:
  94  rsync$(EXEEXT): $(OBJS)
  95         $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS)
  96
  97 +rsyncdb$(EXEEXT): rsync$(EXEEXT)
  98 +       ln -s rsync$(EXEEXT) rsyncdb$(EXEEXT)
  99 +
 100  $(OBJS): $(HEADERS)
 101  $(CHECK_OBJS): $(HEADERS)
 102
 103 @@ -208,22 +215,27 @@ proto.h: proto.h-tstamp
 104  proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c config.h
 105         perl $(srcdir)/mkproto.pl $(srcdir)/*.c $(srcdir)/lib/compat.c
 106
 107 -man: rsync.1 rsyncd.conf.5 man-copy
 108 +man: rsync.1 rsyncdb.1 rsyncd.conf.5 man-copy
 109
 110  man-copy:
 111         @-if test -f rsync.1; then :; else echo 'Copying srcdir rsync.1'; cp -p $(srcdir)/rsync.1 .; fi
 112 +       @-if test -f rsyncdb.1; then :; else echo 'Copying srcdir rsyncdb.1'; cp -p $(srcdir)/rsyncdb.1 .; fi
 113         @-if test -f rsyncd.conf.5; then :; else echo 'Copying srcdir rsyncd.conf.5'; cp -p $(srcdir)/rsyncd.conf.5 .; fi
 114
 115  rsync.1: rsync.yo
 116         yodl2man -o rsync.1 $(srcdir)/rsync.yo
 117         -$(srcdir)/tweak_manpage rsync.1
 118
 119 +rsyncdb.1: rsyncdb.yo
 120 +       yodl2man -o rsyncdb.1 $(srcdir)/rsyncdb.yo
 121 +       -$(srcdir)/tweak_manpage rsyncdb.1
 122 +
 123  rsyncd.conf.5: rsyncd.conf.yo
 124         yodl2man -o rsyncd.conf.5 $(srcdir)/rsyncd.conf.yo
 125         -$(srcdir)/tweak_manpage rsyncd.conf.5
 126
 127  clean: cleantests
 128 -       rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) \
 129 +       rm -f *~ $(OBJS) $(CHECK_PROGS) $(CHECK_OBJS) $(CHECK_SYMLINKS) rsyncdb$(EXEEXT) \
 130                 rounding rounding.h *.old
 131
 132  cleantests:
 133 diff --git a/checksum.c b/checksum.c
 134 --- a/checksum.c
 135 +++ b/checksum.c
 136 @@ -23,6 +23,7 @@
 137
 138  extern int checksum_seed;
 139  extern int protocol_version;
 140 +extern int use_db;
 141
 142  /*
 143    a simple 32 bit checksum that can be upadted from either end
 144 @@ -98,10 +99,10 @@ void get_checksum2(char *buf, int32 len, char *sum)
 145         }
 146  }
 147
 148 -void file_checksum(char *fname, char *sum, OFF_T size)
 149 +void file_checksum(const char *fname, STRUCT_STAT *st_p, char *sum)
 150  {
 151         struct map_struct *buf;
 152 -       OFF_T i, len = size;
 153 +       OFF_T i, len = st_p->st_size;
 154         md_context m;
 155         int32 remainder;
 156         int fd;
 157 @@ -112,7 +113,7 @@ void file_checksum(char *fname, char *sum, OFF_T size)
 158         if (fd == -1)
 159                 return;
 160
 161 -       buf = map_file(fd, size, MAX_MAP_SIZE, CSUM_CHUNK);
 162 +       buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
 163
 164         if (protocol_version >= 30) {
 165                 md5_begin(&m);
 166 @@ -127,6 +128,9 @@ void file_checksum(char *fname, char *sum, OFF_T size)
 167                         md5_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder);
 168
 169                 md5_result(&m, (uchar *)sum);
 170 +
 171 +               if (use_db)
 172 +                       db_set_checksum(5, st_p, sum);
 173         } else {
 174                 mdfour_begin(&m);
 175
 176 @@ -144,6 +148,9 @@ void file_checksum(char *fname, char *sum, OFF_T size)
 177                         mdfour_update(&m, (uchar *)map_ptr(buf, i, remainder), remainder);
 178
 179                 mdfour_result(&m, (uchar *)sum);
 180 +
 181 +               if (use_db)
 182 +                       db_set_checksum(4, st_p, sum);
 183         }
 184
 185         close(fd);
 186 diff --git a/cleanup.c b/cleanup.c
 187 --- a/cleanup.c
 188 +++ b/cleanup.c
 189 @@ -26,6 +26,7 @@ extern int am_server;
 190  extern int am_daemon;
 191  extern int am_receiver;
 192  extern int io_error;
 193 +extern int use_db;
 194  extern int keep_partial;
 195  extern int got_xfer_error;
 196  extern int protocol_version;
 197 @@ -141,6 +142,12 @@ NORETURN void _exit_cleanup(int code, const char *file, int line)
 198  #include "case_N.h"
 199                 switch_step++;
 200
 201 +               if (use_db)
 202 +                       db_disconnect(False);
 203 +
 204 +               /* FALLTHROUGH */
 205 +#include "case_N.h"
 206 +
 207                 if (cleanup_child_pid != -1) {
 208                         int status;
 209                         int pid = wait_process(cleanup_child_pid, &status, WNOHANG);
 210 diff --git a/clientserver.c b/clientserver.c
 211 --- a/clientserver.c
 212 +++ b/clientserver.c
 213 @@ -42,12 +42,15 @@ extern int numeric_ids;
 214  extern int filesfrom_fd;
 215  extern int remote_protocol;
 216  extern int protocol_version;
 217 +extern int always_checksum;
 218 +extern int db_lax;
 219  extern int io_timeout;
 220  extern int no_detach;
 221  extern int write_batch;
 222  extern int default_af_hint;
 223  extern int logfile_format_has_i;
 224  extern int logfile_format_has_o_or_i;
 225 +extern char *db_config;
 226  extern char *bind_address;
 227  extern char *config_file;
 228  extern char *logfile_format;
 229 @@ -692,6 +695,11 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 230
 231         log_init(1);
 232
 233 +       if (*lp_db_config(i)) {
 234 +               db_read_config(FLOG, lp_db_config(i));
 235 +               db_lax = lp_db_lax(i);
 236 +       }
 237 +
 238  #ifdef HAVE_PUTENV
 239         if (*lp_prexfer_exec(i) || *lp_postxfer_exec(i)) {
 240                 int status;
 241 @@ -896,6 +904,8 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
 242
 243         am_server = 1; /* Don't let someone try to be tricky. */
 244         quiet = 0;
 245 +       db_config = NULL;
 246 +
 247         if (lp_ignore_errors(module_id))
 248                 ignore_errors = 1;
 249         if (write_batch < 0)
 250 diff --git a/configure.ac b/configure.ac
 251 --- a/configure.ac
 252 +++ b/configure.ac
 253 @@ -345,6 +345,7 @@ AC_CHECK_HEADERS(sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h \
 254      unistd.h utime.h grp.h compat.h sys/param.h ctype.h sys/wait.h \
 255      sys/ioctl.h sys/filio.h string.h stdlib.h sys/socket.h sys/mode.h \
 256      sys/un.h sys/attr.h mcheck.h arpa/inet.h arpa/nameser.h locale.h \
 257 +    mysql/mysql.h sqlite3.h \
 258      netdb.h malloc.h float.h limits.h iconv.h libcharset.h langinfo.h \
 259      sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h \
 260      popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netinet/ip.h \
 261 @@ -1096,6 +1097,48 @@ if test x"$enable_acl_support" = x"no" -o x"$enable_xattr_support" = x"no" -o x"
 262      fi
 263  fi
 264
 265 +AC_MSG_CHECKING([whether to include mysql DB support])
 266 +AC_ARG_ENABLE(mysql,
 267 +       AC_HELP_STRING([--disable-mysql],
 268 +               [disable mysql DB support]))
 269 +
 270 +if test x"$enable_mysql" = x"no"; then
 271 +    AC_MSG_RESULT(no)
 272 +else
 273 +    AC_MSG_RESULT([yes])
 274 +    AC_CHECK_PROG(MYSQL_CONFIG, mysql_config, 1, 0)
 275 +    if test x$MYSQL_CONFIG = x1; then
 276 +       AC_MSG_CHECKING(for mysql version >= 4)
 277 +       mysql_version=`mysql_config --version`
 278 +       mysql_major_version=`echo $mysql_version | sed 's/\..*//'`
 279 +       if test $mysql_major_version -lt 4; then
 280 +           AC_MSG_RESULT(no.. skipping MySQL)
 281 +       else
 282 +           AC_MSG_RESULT(yes)
 283 +
 284 +           MYSQL_CFLAGS=`mysql_config --cflags`
 285 +           MYSQL_LIBS=`mysql_config --libs`
 286 +
 287 +           CPPFLAGS="$CPPFLAGS $MYSQL_CFLAGS"
 288 +           LIBS="$MYSQL_LIBS $LIBS"
 289 +
 290 +           AC_CHECK_LIB(mysqlclient, mysql_init)
 291 +       fi
 292 +    fi
 293 +fi
 294 +
 295 +AC_MSG_CHECKING([whether to include sqlite DB support])
 296 +AC_ARG_ENABLE(sqlite,
 297 +       AC_HELP_STRING([--disable-sqlite],
 298 +               [disable sqlite DB support]))
 299 +
 300 +if test x"$enable_sqlite" = x"no"; then
 301 +    AC_MSG_RESULT(no)
 302 +else
 303 +    AC_CHECK_LIB(sqlite3, sqlite3_open)
 304 +    AC_CHECK_FUNCS(sqlite3_open_v2 sqlite3_prepare_v2)
 305 +fi
 306 +
 307  case "$CC" in
 308  ' checker'*|checker*)
 309      AC_DEFINE(FORCE_FD_ZERO_MEMSET, 1, [Used to make "checker" understand that FD_ZERO() clears memory.])
 310 diff --git a/db.c b/db.c
 311 new file mode 100644
 312 --- /dev/null
 313 +++ b/db.c
 314 @@ -0,0 +1,1913 @@
 315 +/*
 316 + * Routines to access extended file info via DB.
 317 + *
 318 + * Copyright (C) 2008-2013 Wayne Davison
 319 + *
 320 + * This program is free software; you can redistribute it and/or modify
 321 + * it under the terms of the GNU General Public License as published by
 322 + * the Free Software Foundation; either version 3 of the License, or
 323 + * (at your option) any later version.
 324 + *
 325 + * This program is distributed in the hope that it will be useful,
 326 + * but WITHOUT ANY WARRANTY; without even the implied warranty of
 327 + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 328 + * GNU General Public License for more details.
 329 + *
 330 + * You should have received a copy of the GNU General Public License along
 331 + * with this program; if not, visit the http://fsf.org website.
 332 + */
 333 +
 334 +#include "rsync.h"
 335 +#include "ifuncs.h"
 336 +#include "itypes.h"
 337 +#include "inums.h"
 338 +
 339 +extern int protocol_version;
 340 +extern int recurse;
 341 +extern int same_db;
 342 +extern int am_receiver;
 343 +extern int am_generator;
 344 +extern int checksum_len;
 345 +extern int db_clean, db_check, db_do_md4, db_do_md5, db_update, db_lax, db_init, db_mounts;
 346 +extern int db_output_name, db_output_sum, db_output_info, db_output_unchanged, db_output_dirs, db_output_msgs;
 347 +extern int saw_db_output_opt, saw_db_sum_opt;
 348 +extern char *db_config;
 349 +
 350 +/* TODO: make this configurable */
 351 +#define RSYNCDB_MOUNTS "/usr/sbin/rsyncdb-mountinfo"
 352 +
 353 +#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
 354 +#define USE_MYSQL
 355 +#include <mysql/mysql.h>
 356 +#include <mysql/errmsg.h>
 357 +#endif
 358 +
 359 +#if defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
 360 +#define USE_SQLITE
 361 +#include <sqlite3.h>
 362 +#ifndef HAVE_SQLITE3_OPEN_V2
 363 +#define sqlite3_open_v2(dbname, dbhptr, flags, vfs) \
 364 +       sqlite3_open(dbname, dbhptr)
 365 +#endif
 366 +#ifndef HAVE_SQLITE3_PREPARE_V2
 367 +#define sqlite3_prepare_v2 sqlite3_prepare
 368 +#endif
 369 +#define MAX_LOCK_FAILURES 10
 370 +#define LOCK_FAIL_MSLEEP 100
 371 +#endif
 372 +
 373 +#define DB_TYPE_NONE 0
 374 +#define DB_TYPE_MYSQL 1
 375 +#define DB_TYPE_SQLITE 2
 376 +
 377 +int use_db = DB_TYPE_NONE;
 378 +int select_many_sums = 0;
 379 +
 380 +#define PREP_NORM 0
 381 +#define PREP_MOUNT 1
 382 +
 383 +static const char *dbhost = NULL, *dbuser = NULL, *dbpass = NULL, *dbname = NULL;
 384 +static unsigned int dbport = 0;
 385 +static int transaction_state = -1;
 386 +
 387 +static union {
 388 +#ifdef USE_MYSQL
 389 +    MYSQL *mysql;
 390 +#endif
 391 +#ifdef USE_SQLITE
 392 +    sqlite3 *sqlite;
 393 +#endif
 394 +    void *all;
 395 +} dbh;
 396 +
 397 +#define SEL_DEV 0
 398 +#define SEL_SUM 1
 399 +#define REP_SUM 2
 400 +#define UPD_CTIME 3
 401 +#define INS_MOUNT 4
 402 +#define UPD_MOUNT 5 /* SQLite only */
 403 +#define SEL_MOUNT 6
 404 +#define UN_MOUNT 7
 405 +#define DEL_SUMS 8
 406 +#define INS_PRESENT 9
 407 +#define MAX_PREP_CNT 10
 408 +
 409 +#define MAX_BIND_CNT 7
 410 +#define MAX_RESULT_BINDS 32
 411 +
 412 +static union {
 413 +#ifdef USE_MYSQL
 414 +    MYSQL_STMT *mysql;
 415 +#endif
 416 +#ifdef USE_SQLITE
 417 +    sqlite3_stmt *sqlite;
 418 +#endif
 419 +    void *all;
 420 +} statements[MAX_PREP_CNT];
 421 +
 422 +static int md_num;
 423 +static enum logcode log_code;
 424 +
 425 +#ifdef USE_MYSQL
 426 +static unsigned int bind_disk_id, bind_mdnum;
 427 +static int64 bind_devno, bind_ino, bind_size, bind_mtime, bind_ctime;
 428 +static char bind_sum[MAX_DIGEST_LEN];
 429 +static unsigned long result_length[MAX_RESULT_BINDS];
 430 +static my_bool result_is_null[MAX_RESULT_BINDS], result_error[MAX_RESULT_BINDS];
 431 +#else
 432 +static int64 bind_mtime;
 433 +#endif
 434 +static char bind_thishost[128+1], bind_mount_uniq[128+1];
 435 +static unsigned long bind_thishost_len, bind_mount_uniq_len;
 436 +
 437 +static char *error_log;
 438 +#if defined USE_SQLITE && defined SQLITE_CONFIG_LOG
 439 +static FILE *error_log_fp;
 440 +#endif
 441 +
 442 +#define PTR_SIZE (sizeof (struct file_struct *))
 443 +
 444 +static void update_mounts(void);
 445 +
 446 +struct name_list {
 447 +       struct name_list *next;
 448 +       char name[1];
 449 +} *dirs_list;
 450 +
 451 +int db_read_config(enum logcode code, const char *config_file)
 452 +{
 453 +       char buf[2048], *cp;
 454 +       FILE *fp;
 455 +       int lineno = 0;
 456 +
 457 +       log_code = code;
 458 +
 459 +       bind_thishost_len = strlcpy(bind_thishost, "localhost", sizeof bind_thishost);
 460 +
 461 +       if (!(fp = fopen(config_file, "r"))) {
 462 +               rsyserr(log_code, errno, "unable to open %s", config_file);
 463 +               return 0;
 464 +       }
 465 +       if (DEBUG_GTE(DB, 1))
 466 +               rprintf(FCLIENT, "[%s] Reading DB config from %s\n", who_am_i(), config_file);
 467 +       while (fgets(buf, sizeof buf, fp)) {
 468 +               lineno++;
 469 +               if ((cp = strchr(buf, '#')) == NULL
 470 +                && (cp = strchr(buf, '\r')) == NULL
 471 +                && (cp = strchr(buf, '\n')) == NULL)
 472 +                       cp = buf + strlen(buf);
 473 +               while (cp != buf && isSpace(cp-1)) cp--;
 474 +               *cp = '\0';
 475 +
 476 +               if (!*buf)
 477 +                       continue;
 478 +
 479 +               if (!(cp = strchr(buf, ':')))
 480 +                       goto invalid_line;
 481 +               *cp++ = '\0';
 482 +
 483 +               while (isSpace(cp)) cp++;
 484 +               if (strcasecmp(buf, "dbhost") == 0)
 485 +                       dbhost = strdup(cp);
 486 +               else if (strcasecmp(buf, "dbuser") == 0)
 487 +                       dbuser = strdup(cp);
 488 +               else if (strcasecmp(buf, "dbpass") == 0)
 489 +                       dbpass = strdup(cp);
 490 +               else if (strcasecmp(buf, "dbname") == 0)
 491 +                       dbname = strdup(cp);
 492 +               else if (strcasecmp(buf, "dbport") == 0)
 493 +                       dbport = atoi(cp);
 494 +               else if (strcasecmp(buf, "transaction") == 0)
 495 +                       transaction_state = atoi(cp) ? 0 : -1;
 496 +               else if (strcasecmp(buf, "errlog") == 0)
 497 +                       error_log = strdup(cp);
 498 +               else if (strcasecmp(buf, "thishost") == 0)
 499 +                       bind_thishost_len = strlcpy(bind_thishost, cp, sizeof bind_thishost);
 500 +               else if (strcasecmp(buf, "dbtype") == 0) {
 501 +#ifdef USE_MYSQL
 502 +                       if (strcasecmp(cp, "mysql") == 0) {
 503 +                               use_db = DB_TYPE_MYSQL;
 504 +                               continue;
 505 +                       }
 506 +#endif
 507 +#ifdef USE_SQLITE
 508 +                       if (strcasecmp(cp, "sqlite") == 0) {
 509 +                               use_db = DB_TYPE_SQLITE;
 510 +                               continue;
 511 +                       }
 512 +#endif
 513 +                       rprintf(log_code,
 514 +                           "Unsupported dbtype on line #%d in %s.\n",
 515 +                           lineno, config_file);
 516 +                       use_db = DB_TYPE_NONE;
 517 +                       return 0;
 518 +               } else {
 519 +                 invalid_line:
 520 +                       rprintf(log_code, "Invalid line #%d in %s\n",
 521 +                               lineno, config_file);
 522 +                       use_db = DB_TYPE_NONE;
 523 +                       return 0;
 524 +               }
 525 +       }
 526 +       fclose(fp);
 527 +
 528 +       if (bind_thishost_len >= (int)sizeof bind_thishost)
 529 +               bind_thishost_len = sizeof bind_thishost - 1;
 530 +
 531 +       if (!use_db || !dbname) {
 532 +               rprintf(log_code, "Please specify at least dbtype and dbname in %s.\n", config_file);
 533 +               use_db = DB_TYPE_NONE;
 534 +               return 0;
 535 +       }
 536 +
 537 +       md_num = protocol_version >= 30 ? 5 : 4;
 538 +
 539 +       if (error_log) {
 540 +               if (use_db != DB_TYPE_SQLITE)
 541 +                       rprintf(log_code, "Ignoring errlog setting for non-SQLite DB.\n");
 542 +#ifndef SQLITE_CONFIG_LOG
 543 +               else
 544 +                       rprintf(log_code, "Your sqlite doesn't support SQLITE_CONFIG_LOG.\n");
 545 +#endif
 546 +       }
 547 +
 548 +       return 1;
 549 +}
 550 +
 551 +#if defined USE_SQLITE && defined SQLITE_CONFIG_LOG
 552 +static void errorLogCallback(UNUSED(void *pArg), int iErrCode, const char *zMsg)
 553 +{
 554 +       fprintf(error_log_fp, "[%d] %s (%d)\n", (int)getpid(), zMsg, iErrCode);
 555 +}
 556 +#endif
 557 +
 558 +static int run_sql(const char *fmt, ...)
 559 +{
 560 +       va_list ap;
 561 +       char *query;
 562 +       int ok = 0, qlen;
 563 +
 564 +       va_start(ap, fmt);
 565 +       qlen = vasprintf(&query, fmt, ap);
 566 +       va_end(ap);
 567 +       if (qlen < 0)
 568 +               out_of_memory("run_sql");
 569 +       if (DEBUG_GTE(DB, 3))
 570 +               rprintf(FCLIENT, "[%s] SQL being run: %s\n", who_am_i(), query);
 571 +
 572 +       switch (use_db) {
 573 +       case DB_TYPE_MYSQL:
 574 +#ifdef USE_MYSQL
 575 +               if (mysql_query(dbh.mysql, query) < 0) {
 576 +                       rprintf(FERROR, "Failed to run sql: %s\n", mysql_error(dbh.mysql));
 577 +                       rprintf(FERROR, "%s\n", query);
 578 +               } else
 579 +                       ok = 1;
 580 +               break;
 581 +#endif
 582 +#ifdef USE_SQLITE
 583 +       case DB_TYPE_SQLITE: {
 584 +               int rc, lock_failures = 0;
 585 +               while (1) {
 586 +                       if ((rc = sqlite3_exec(dbh.sqlite, query, NULL, NULL, NULL)) == 0)
 587 +                               break;
 588 +                       if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
 589 +                               break;
 590 +                       if (++lock_failures > MAX_LOCK_FAILURES)
 591 +                               break;
 592 +                       msleep(LOCK_FAIL_MSLEEP);
 593 +               }
 594 +               if (rc) {
 595 +                       rprintf(FERROR, "[%s] Failed to run sql: %s\n", who_am_i(), sqlite3_errmsg(dbh.sqlite));
 596 +                       rprintf(FERROR, "%s\n", query);
 597 +               } else
 598 +                       ok = 1;
 599 +               break;
 600 +           }
 601 +#endif
 602 +       }
 603 +
 604 +       free(query);
 605 +
 606 +       return ok;
 607 +}
 608 +
 609 +#ifdef USE_MYSQL
 610 +static int prepare_mysql(int ndx, MYSQL_BIND *binds, int bind_cnt, const char *fmt, ...)
 611 +{
 612 +       va_list ap;
 613 +       char *query;
 614 +       int qlen, param_cnt;
 615 +       MYSQL_STMT *stmt = mysql_stmt_init(dbh.mysql);
 616 +
 617 +       if (stmt == NULL)
 618 +               out_of_memory("prepare_mysql");
 619 +
 620 +       va_start(ap, fmt);
 621 +       qlen = vasprintf(&query, fmt, ap);
 622 +       va_end(ap);
 623 +       if (qlen < 0)
 624 +               out_of_memory("prepare_mysql");
 625 +       if (DEBUG_GTE(DB, 3))
 626 +               rprintf(FCLIENT, "[%s] SQL being prepared: %s\n", who_am_i(), query);
 627 +
 628 +       if (mysql_stmt_prepare(stmt, query, qlen) != 0) {
 629 +               rprintf(log_code, "[%s] Prepare failed: %s\n", who_am_i(), mysql_stmt_error(stmt));
 630 +               rprintf(log_code, "%s\n", query);
 631 +               free(query);
 632 +               return 0;
 633 +       }
 634 +
 635 +       if ((param_cnt = mysql_stmt_param_count(stmt)) != bind_cnt) {
 636 +               rprintf(log_code, "[%s] Parameters in statement = %d, bind vars = %d\n",
 637 +                       who_am_i(), param_cnt, bind_cnt);
 638 +               rprintf(log_code, "%s\n", query);
 639 +               free(query);
 640 +               return 0;
 641 +       }
 642 +       if (bind_cnt)
 643 +               mysql_stmt_bind_param(stmt, binds);
 644 +
 645 +       statements[ndx].mysql = stmt;
 646 +       free(query);
 647 +
 648 +       return 1;
 649 +}
 650 +#endif
 651 +
 652 +#ifdef USE_MYSQL
 653 +static int prepare_mysql_queries(int type)
 654 +{
 655 +       MYSQL_BIND binds[MAX_BIND_CNT];
 656 +       char *sql;
 657 +
 658 +       switch (type) {
 659 +       case PREP_NORM:
 660 +               sql="SELECT disk_id"
 661 +                   " FROM disk"
 662 +                   " WHERE host = ? AND devno = ?";
 663 +               memset(binds, 0, sizeof binds);
 664 +               binds[0].buffer_type = MYSQL_TYPE_STRING;
 665 +               binds[0].buffer = &bind_thishost;
 666 +               binds[0].buffer_length = bind_thishost_len;
 667 +               binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
 668 +               binds[1].buffer = &bind_devno;
 669 +               if (!prepare_mysql(SEL_DEV, binds, 2, sql))
 670 +                       return 0;
 671 +
 672 +               memset(binds, 0, sizeof binds);
 673 +               binds[0].buffer_type = MYSQL_TYPE_LONG;
 674 +               binds[0].buffer = &bind_disk_id;
 675 +               binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
 676 +               binds[1].buffer = &bind_ino;
 677 +               if (select_many_sums) {
 678 +                       sql="SELECT checksum, sum_type, size, mtime, ctime"
 679 +                           " FROM inode_map"
 680 +                           " WHERE disk_id = ? AND ino = ?";
 681 +                       if (!prepare_mysql(SEL_SUM, binds, 2, sql))
 682 +                               return 0;
 683 +               } else {
 684 +                       sql="SELECT checksum"
 685 +                           " FROM inode_map"
 686 +                           " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
 687 +                           "   AND size = ? AND mtime = ? %s";
 688 +                       binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
 689 +                       binds[2].buffer = &bind_size;
 690 +                       binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
 691 +                       binds[3].buffer = &bind_mtime;
 692 +                       if (!db_lax) {
 693 +                               binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
 694 +                               binds[4].buffer = &bind_ctime;
 695 +                       }
 696 +                       if (!prepare_mysql(SEL_SUM, binds, 4 + !db_lax, sql,    md_num, db_lax ? "" : "AND ctime = ?"))
 697 +                               return 0;
 698 +               }
 699 +
 700 +               sql="INSERT INTO inode_map"
 701 +                   " SET disk_id = ?, ino = ?, sum_type = ?,"
 702 +                   "     size = ?, mtime = ?, ctime = ?, checksum = ?"
 703 +                   " ON DUPLICATE KEY"
 704 +                   " UPDATE size = VALUES(size), mtime = VALUES(mtime),"
 705 +                   "        ctime = VALUES(ctime), checksum = VALUES(checksum)";
 706 +               memset(binds, 0, sizeof binds);
 707 +               binds[0].buffer_type = MYSQL_TYPE_LONG;
 708 +               binds[0].buffer = &bind_disk_id;
 709 +               binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
 710 +               binds[1].buffer = &bind_ino;
 711 +               binds[2].buffer_type = MYSQL_TYPE_LONG;
 712 +               binds[2].buffer = &bind_mdnum;
 713 +               binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
 714 +               binds[3].buffer = &bind_size;
 715 +               binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
 716 +               binds[4].buffer = &bind_mtime;
 717 +               binds[5].buffer_type = MYSQL_TYPE_LONGLONG;
 718 +               binds[5].buffer = &bind_ctime;
 719 +               binds[6].buffer_type = MYSQL_TYPE_BLOB;
 720 +               binds[6].buffer = &bind_sum;
 721 +               binds[6].buffer_length = checksum_len;
 722 +               if (!prepare_mysql(REP_SUM, binds, 7, sql))
 723 +                       return 0;
 724 +
 725 +               sql="UPDATE inode_map"
 726 +                   " SET ctime = ?"
 727 +                   " WHERE disk_id = ? AND ino = ? AND sum_type = ? AND size = ? AND mtime = ?";
 728 +               memset(binds, 0, sizeof binds);
 729 +               binds[0].buffer_type = MYSQL_TYPE_LONGLONG;
 730 +               binds[0].buffer = &bind_ctime;
 731 +               binds[1].buffer_type = MYSQL_TYPE_LONG;
 732 +               binds[1].buffer = &bind_disk_id;
 733 +               binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
 734 +               binds[2].buffer = &bind_ino;
 735 +               binds[3].buffer_type = MYSQL_TYPE_LONG;
 736 +               binds[3].buffer = &bind_mdnum;
 737 +               binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
 738 +               binds[4].buffer = &bind_size;
 739 +               binds[5].buffer_type = MYSQL_TYPE_LONGLONG;
 740 +               binds[5].buffer = &bind_mtime;
 741 +               if (!prepare_mysql(UPD_CTIME, binds, 6, sql))
 742 +                       return 0;
 743 +               break;
 744 +
 745 +       case PREP_MOUNT:
 746 +               sql="INSERT INTO disk"
 747 +                   " SET host = ?, last_seen = ?, mount_uniq = ?, devno = ?"
 748 +                   " ON DUPLICATE KEY"
 749 +                   " UPDATE last_seen = VALUES(last_seen), devno = VALUES(devno)";
 750 +               memset(binds, 0, sizeof binds);
 751 +               binds[0].buffer_type = MYSQL_TYPE_STRING;
 752 +               binds[0].buffer = &bind_thishost;
 753 +               binds[0].buffer_length = bind_thishost_len;
 754 +               binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
 755 +               binds[1].buffer = &bind_mtime; /* we abuse mtime to hold the last_seen value */
 756 +               binds[2].buffer_type = MYSQL_TYPE_STRING;
 757 +               binds[2].buffer = &bind_mount_uniq;
 758 +               binds[2].buffer_length = sizeof bind_mount_uniq;
 759 +               binds[2].length = &bind_mount_uniq_len;
 760 +               binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
 761 +               binds[3].buffer = &bind_devno;
 762 +               if (!prepare_mysql(INS_MOUNT, binds, 4, sql))
 763 +                       return 0;
 764 +
 765 +               sql="SELECT mount_uniq"
 766 +                   " FROM disk"
 767 +                   " WHERE host = ? AND last_seen < ? AND devno != 0";
 768 +               /* Reusing first 2 binds from INS_MOUNT */
 769 +               if (!prepare_mysql(SEL_MOUNT, binds, 2, sql))
 770 +                       return 0;
 771 +
 772 +               sql="UPDATE disk"
 773 +                   " SET devno = 0"
 774 +                   " WHERE host = ? AND last_seen < ? AND devno != 0";
 775 +               /* Reusing binds from SEL_MOUNT */
 776 +               if (!prepare_mysql(UN_MOUNT, binds, 2, sql))
 777 +                       return 0;
 778 +               break;
 779 +       }
 780 +
 781 +       return 1;
 782 +}
 783 +#endif
 784 +
 785 +#ifdef USE_MYSQL
 786 +static int db_connect_mysql(void)
 787 +{
 788 +       const char *open_dbname = db_init ? "mysql" : dbname;
 789 +
 790 +       if (!(dbh.mysql = mysql_init(NULL)))
 791 +               out_of_memory("db_read_config");
 792 +
 793 +       if (DEBUG_GTE(DB, 1)) {
 794 +               rprintf(FCLIENT, "[%s] connecting: host=%s user=%s db=%s port=%d\n",
 795 +                       who_am_i(), dbhost, dbuser, open_dbname, dbport);
 796 +       }
 797 +       if (!mysql_real_connect(dbh.mysql, dbhost, dbuser, dbpass, open_dbname, dbport, NULL, 0)) {
 798 +               rprintf(log_code, "[%s] Unable to connect to DB: %s\n", who_am_i(), mysql_error(dbh.mysql));
 799 +               return 0;
 800 +       }
 801 +
 802 +       if (db_init) {
 803 +               if (db_output_msgs)
 804 +                       rprintf(FCLIENT, "Creating DB %s (if it does not exist)\n", dbname);
 805 +               if (!run_sql("CREATE DATABASE IF NOT EXISTS `%s`", dbname)
 806 +                || !run_sql("USE `%s`", dbname))
 807 +                       exit_cleanup(RERR_IPC);
 808 +
 809 +               if (db_output_msgs)
 810 +                       rprintf(FCLIENT, "Dropping old tables (if they exist))\n");
 811 +               if (!run_sql("DROP TABLE IF EXISTS disk")
 812 +                || !run_sql("DROP TABLE IF EXISTS inode_map"))
 813 +                       exit_cleanup(RERR_IPC);
 814 +
 815 +               if (db_output_msgs)
 816 +                       rprintf(FCLIENT, "Creating empty tables ...\n");
 817 +               if (!run_sql(
 818 +                   "CREATE TABLE disk (\n"
 819 +                   "  disk_id integer unsigned NOT NULL PRIMARY KEY AUTO_INCREMENT,\n"
 820 +                   "  host varchar(128) NOT NULL default 'localhost',\n"
 821 +                   "  mount_uniq varchar(128) default NULL,\n"
 822 +                   "  devno bigint unsigned NOT NULL,\n" /* This is 0 when not mounted */
 823 +                   "  last_seen bigint NOT NULL,\n"
 824 +                   "  UNIQUE KEY mount_lookup (host, mount_uniq),\n"
 825 +                   "  KEY dev_lookup (devno, host)\n"
 826 +                   ")"))
 827 +                       exit_cleanup(RERR_IPC);
 828 +
 829 +               if (!run_sql(
 830 +                   "CREATE TABLE inode_map (\n"
 831 +                   "  disk_id integer unsigned NOT NULL,\n"
 832 +                   "  ino bigint unsigned NOT NULL,\n"
 833 +                   "  sum_type tinyint NOT NULL default '0',\n"
 834 +                   "  size bigint unsigned NOT NULL,\n"
 835 +                   "  mtime bigint NOT NULL,\n"
 836 +                   "  ctime bigint NOT NULL,\n"
 837 +                   "  checksum binary(16) NOT NULL,\n"
 838 +                   "  PRIMARY KEY (disk_id,ino,sum_type)\n"
 839 +                   ")"))
 840 +                       exit_cleanup(RERR_IPC);
 841 +
 842 +               if (!db_mounts)
 843 +                       exit_cleanup(0);
 844 +       }
 845 +
 846 +       if (db_mounts) {
 847 +               if (!prepare_mysql_queries(PREP_MOUNT))
 848 +                       exit_cleanup(RERR_IPC);
 849 +               update_mounts();
 850 +               exit_cleanup(0);
 851 +       }
 852 +
 853 +       if (!prepare_mysql_queries(PREP_NORM))
 854 +               return 0;
 855 +
 856 +       return 1;
 857 +}
 858 +#endif
 859 +
 860 +#ifdef USE_SQLITE
 861 +static int prepare_sqlite(int ndx, const char *fmt, ...)
 862 +{
 863 +       va_list ap;
 864 +       char *query;
 865 +       int rc, qlen, lock_failures = 0;
 866 +
 867 +       va_start(ap, fmt);
 868 +       qlen = vasprintf(&query, fmt, ap);
 869 +       va_end(ap);
 870 +       if (qlen < 0)
 871 +               out_of_memory("prepare_sqlite");
 872 +       if (DEBUG_GTE(DB, 3))
 873 +               rprintf(FCLIENT, "[%s] SQL being prepared: %s\n", who_am_i(), query);
 874 +
 875 +       while ((rc = sqlite3_prepare_v2(dbh.sqlite, query, -1, &statements[ndx].sqlite, NULL)) != 0) {
 876 +               if (DEBUG_GTE(DB, 4)) {
 877 +                       rprintf(FCLIENT, "[%s] sqlite3_prepare_v2(,%s,,) returned %d\n",
 878 +                               who_am_i(), query, rc);
 879 +               }
 880 +               if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
 881 +                       break;
 882 +               if (++lock_failures > MAX_LOCK_FAILURES)
 883 +                       break;
 884 +               msleep(LOCK_FAIL_MSLEEP);
 885 +       }
 886 +       if (rc) {
 887 +               rprintf(log_code, "[%s] Failed to prepare SQL: %s (%d)\n", who_am_i(), sqlite3_errmsg(dbh.sqlite), rc);
 888 +               rprintf(log_code, "%s\n", query);
 889 +               free(query);
 890 +               return 0;
 891 +       }
 892 +       free(query);
 893 +
 894 +       return 1;
 895 +}
 896 +#endif
 897 +
 898 +#ifdef USE_SQLITE
 899 +static int prepare_sqlite_queries(int type)
 900 +{
 901 +       char *sql;
 902 +
 903 +       switch (type) {
 904 +       case PREP_NORM:
 905 +               sql="SELECT disk_id"
 906 +                   " FROM disk"
 907 +                   " WHERE host = ? AND devno = ?";
 908 +               if (!prepare_sqlite(SEL_DEV, sql))
 909 +                       return 0;
 910 +
 911 +               if (select_many_sums) {
 912 +                       sql="SELECT checksum, sum_type, size, mtime, ctime"
 913 +                           " FROM inode_map"
 914 +                           " WHERE disk_id = ? AND ino = ?";
 915 +                       if (!prepare_sqlite(SEL_SUM, sql))
 916 +                               return 0;
 917 +               } else {
 918 +                       sql="SELECT checksum"
 919 +                           " FROM inode_map"
 920 +                           " WHERE disk_id = ? AND ino = ? AND sum_type = %d"
 921 +                           "   AND size = ? AND mtime = ? %s";
 922 +                       if (!prepare_sqlite(SEL_SUM, sql, md_num, db_lax ? "" : "AND ctime = ?"))
 923 +                               return 0;
 924 +               }
 925 +
 926 +               sql="INSERT OR REPLACE INTO inode_map"
 927 +                   " (disk_id, ino, sum_type, size, mtime, ctime, checksum)"
 928 +                   " VALUES (?, ?, ?, ?, ?, ?, ?)";
 929 +               if (!prepare_sqlite(REP_SUM, sql))
 930 +                       return 0;
 931 +
 932 +               sql="UPDATE inode_map"
 933 +                   " SET ctime = ?"
 934 +                   " WHERE disk_id = ? AND ino = ? AND sum_type = ? AND size = ? AND mtime = ?";
 935 +               if (!prepare_sqlite(UPD_CTIME, sql))
 936 +                       return 0;
 937 +               break;
 938 +
 939 +       case PREP_MOUNT:
 940 +               sql="INSERT OR IGNORE INTO disk"
 941 +                   " (host, last_seen, mount_uniq, devno)"
 942 +                   " VALUES (?, ?, ?, ?)";
 943 +               if (!prepare_sqlite(INS_MOUNT, sql))
 944 +                       return 0;
 945 +
 946 +               sql="UPDATE disk"
 947 +                   " SET last_seen = ?, devno = ?"
 948 +                   " WHERE host = ? AND mount_uniq = ?";
 949 +               if (!prepare_sqlite(UPD_MOUNT, sql))
 950 +                       return 0;
 951 +
 952 +               sql="SELECT mount_uniq"
 953 +                   " FROM disk"
 954 +                   " WHERE host = ? AND last_seen < ? AND devno != 0";
 955 +               if (!prepare_sqlite(SEL_MOUNT, sql))
 956 +                       return 0;
 957 +
 958 +               sql="UPDATE disk"
 959 +                   " SET devno = 0"
 960 +                   " WHERE host = ? AND last_seen < ? AND devno != 0";
 961 +               if (!prepare_sqlite(UN_MOUNT, sql))
 962 +                       return 0;
 963 +               break;
 964 +       }
 965 +
 966 +       return 1;
 967 +}
 968 +#endif
 969 +
 970 +#ifdef USE_SQLITE
 971 +static int db_connect_sqlite(void)
 972 +{
 973 +       int lock_failures = 0;
 974 +       int rc;
 975 +
 976 +#ifdef SQLITE_CONFIG_LOG
 977 +       if (error_log) {
 978 +               if (DEBUG_GTE(DB, 1))
 979 +                       rprintf(FCLIENT, "[%s] Setting sqlite errlog to %s\n", who_am_i(), error_log);
 980 +               if (!(error_log_fp = fopen(error_log, "a"))) {
 981 +                       rsyserr(log_code, errno, "unable to append to logfile %s", error_log);
 982 +                       error_log = NULL;
 983 +               } else if (sqlite3_config(SQLITE_CONFIG_LOG, errorLogCallback, NULL) != 0)
 984 +                       rprintf(log_code, "Failed to set errorLogCallback: %s\n", sqlite3_errmsg(dbh.sqlite));
 985 +       }
 986 +#endif
 987 +
 988 +       while (1) {
 989 +               int open_flags = SQLITE_OPEN_READWRITE;
 990 +               if (db_init)
 991 +                       open_flags |= SQLITE_OPEN_CREATE;
 992 +               if (DEBUG_GTE(DB, 1))
 993 +                       rprintf(FCLIENT, "[%s] opening %s (%d)\n", who_am_i(), dbname, open_flags);
 994 +               if ((rc = sqlite3_open_v2(dbname, &dbh.sqlite, open_flags, NULL)) == 0) {
 995 +                       break;
 996 +               }
 997 +               if (DEBUG_GTE(DB, 4)) {
 998 +                       rprintf(FCLIENT, "[%s] sqlite3_open_v2(%s,,%d,NULL) returned %d\n",
 999 +                               who_am_i(), dbname, open_flags, rc);
1000 +               }
1001 +               if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
1002 +                       break;
1003 +               if (++lock_failures > MAX_LOCK_FAILURES)
1004 +                       break;
1005 +               msleep(LOCK_FAIL_MSLEEP);
1006 +       }
1007 +
1008 +       if (rc) {
1009 +               rprintf(log_code, "Unable to connect to DB: %s (%d)\n", sqlite3_errmsg(dbh.sqlite), rc);
1010 +               return 0;
1011 +       }
1012 +
1013 +       if (db_init) {
1014 +               char *sql;
1015 +               if (db_output_msgs)
1016 +                       rprintf(FCLIENT, "Dropping old tables (if they exist) ...\n");
1017 +               if (!run_sql("DROP TABLE IF EXISTS disk")
1018 +                || !run_sql("DROP TABLE IF EXISTS inode_map"))
1019 +                       exit_cleanup(RERR_IPC);
1020 +
1021 +               if (db_output_msgs)
1022 +                       rprintf(FCLIENT, "Creating empty tables ...\n");
1023 +               sql="CREATE TABLE disk (\n"
1024 +                   "  disk_id integer NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
1025 +                   "  host varchar(128) NOT NULL default 'localhost',\n"
1026 +                   "  mount_uniq varchar(128) default NULL,\n"
1027 +                   "  devno bigint NOT NULL,\n" /* This is 0 when not mounted */
1028 +                   "  last_seen bigint NOT NULL,\n"
1029 +                   "  UNIQUE (host, mount_uniq)\n"
1030 +                   ")";
1031 +               if (!run_sql(sql))
1032 +                       exit_cleanup(RERR_IPC);
1033 +
1034 +               sql="CREATE TABLE inode_map (\n"
1035 +                   "  disk_id integer NOT NULL,\n"
1036 +                   "  ino bigint NOT NULL,\n"
1037 +                   "  size bigint NOT NULL,\n"
1038 +                   "  mtime bigint NOT NULL,\n"
1039 +                   "  ctime bigint NOT NULL,\n"
1040 +                   "  sum_type tinyint NOT NULL default '0',\n"
1041 +                   "  checksum binary(16) NOT NULL,\n"
1042 +                   "  PRIMARY KEY (disk_id,ino,sum_type)\n"
1043 +                   ")";
1044 +               if (!run_sql(sql))
1045 +                       exit_cleanup(RERR_IPC);
1046 +
1047 +#if SQLITE_VERSION_NUMBER >= 3007000
1048 +               /* Using WAL locking makes concurrency much better (requires sqlite 3.7.0). */
1049 +               sql="PRAGMA journal_mode = wal";
1050 +               run_sql(sql); /* We don't check this for success. */
1051 +#endif
1052 +
1053 +               if (!db_mounts)
1054 +                       exit_cleanup(0);
1055 +       }
1056 +
1057 +       if (db_mounts) {
1058 +               if (!prepare_sqlite_queries(PREP_MOUNT))
1059 +                       exit_cleanup(RERR_IPC);
1060 +               update_mounts();
1061 +               exit_cleanup(0);
1062 +       }
1063 +
1064 +       if (!prepare_sqlite_queries(PREP_NORM)) {
1065 +               db_disconnect(False);
1066 +               return 0;
1067 +       }
1068 +
1069 +       return 1;
1070 +}
1071 +#endif
1072 +
1073 +int db_connect(int select_many)
1074 +{
1075 +       select_many_sums = select_many;
1076 +
1077 +       switch (use_db) {
1078 +#ifdef USE_MYSQL
1079 +       case DB_TYPE_MYSQL:
1080 +               if (db_connect_mysql())
1081 +                       return 1;
1082 +               break;
1083 +#endif
1084 +#ifdef USE_SQLITE
1085 +       case DB_TYPE_SQLITE:
1086 +               if (db_connect_sqlite())
1087 +                       return 1;
1088 +               break;
1089 +#endif
1090 +       }
1091 +
1092 +       db_disconnect(False);
1093 +
1094 +       return 0;
1095 +}
1096 +
1097 +void db_disconnect(BOOL commit)
1098 +{
1099 +       int ndx;
1100 +
1101 +       if (!dbh.all)
1102 +               return;
1103 +
1104 +       if (transaction_state > 0) {
1105 +               if (DEBUG_GTE(DB, 1)) {
1106 +                       rprintf(FCLIENT, "[%s] %s our DB transaction\n",
1107 +                               who_am_i(), commit ? "Committing" : "Rolling back");
1108 +               }
1109 +               transaction_state = 0;
1110 +               if (commit)
1111 +                       run_sql("COMMIT");
1112 +               else
1113 +                       run_sql("ROLLBACK");
1114 +       }
1115 +
1116 +       if (DEBUG_GTE(DB, 1))
1117 +               rprintf(FCLIENT, "[%s] Disconnecting from the DB\n", who_am_i());
1118 +
1119 +       for (ndx = 0; ndx < MAX_PREP_CNT; ndx++) {
1120 +               if (statements[ndx].all) {
1121 +                       switch (use_db) {
1122 +#ifdef USE_MYSQL
1123 +                       case DB_TYPE_MYSQL:
1124 +                               mysql_stmt_close(statements[ndx].mysql);
1125 +                               break;
1126 +#endif
1127 +#ifdef USE_SQLITE
1128 +                       case DB_TYPE_SQLITE:
1129 +                               sqlite3_finalize(statements[ndx].sqlite);
1130 +                               break;
1131 +#endif
1132 +                       }
1133 +                       statements[ndx].all = NULL;
1134 +               }
1135 +       }
1136 +
1137 +       switch (use_db) {
1138 +#ifdef USE_MYSQL
1139 +       case DB_TYPE_MYSQL:
1140 +               mysql_close(dbh.mysql);
1141 +               break;
1142 +#endif
1143 +#ifdef USE_SQLITE
1144 +       case DB_TYPE_SQLITE:
1145 +               sqlite3_close(dbh.sqlite);
1146 +               break;
1147 +#endif
1148 +       }
1149 +
1150 +       dbh.all = NULL;
1151 +       use_db = DB_TYPE_NONE;
1152 +}
1153 +
1154 +#ifdef USE_MYSQL
1155 +static MYSQL_STMT *exec_mysql(int ndx)
1156 +{
1157 +       MYSQL_STMT *stmt = statements[ndx].mysql;
1158 +       int rc;
1159 +
1160 +       if ((rc = mysql_stmt_execute(stmt)) == CR_SERVER_LOST) {
1161 +               db_disconnect(False);
1162 +               use_db = DB_TYPE_MYSQL;
1163 +               if (db_connect(select_many_sums)) {
1164 +                       stmt = statements[ndx].mysql;
1165 +                       rc = mysql_stmt_execute(stmt);
1166 +               }
1167 +       }
1168 +       if (rc != 0) {
1169 +               rprintf(log_code, "SQL execute failed: %s\n", mysql_stmt_error(stmt));
1170 +               return NULL;
1171 +       }
1172 +
1173 +       return stmt;
1174 +}
1175 +#endif
1176 +
1177 +#ifdef USE_MYSQL
1178 +/* This stores up to max_rows into the values pointed to by the bind data arrays.
1179 + * If max_rows is > 1, then all the buffer pointers MUST be set to an array long
1180 + * enough to hold the max count of rows.  The buffer pointer will be incremented
1181 + * to read additional rows (but never past the end).  If stmt_ptr is non-NULL, it
1182 + * will be set to the "stmt" pointer IFF we didn't run out of rows before hitting
1183 + * the max.  In this case, the caller should call mysql_stmt_fetch() to read any
1184 + * remaining rows (the buffer pointers will point at the final array element) and
1185 + * then call mysql_stmt_free_result().  If *stmt_ptr is a NULL value, there were
1186 + * not enough rows to fill the max_rows arrays, and the stmt was already freed. */
1187 +static int fetch_mysql(MYSQL_BIND *binds, int bind_cnt, int ndx, int max_rows, MYSQL_STMT **stmt_ptr)
1188 +{
1189 +       MYSQL_STMT *stmt;
1190 +       int i, rc, rows = 0;
1191 +
1192 +       if (bind_cnt > MAX_RESULT_BINDS) {
1193 +               fprintf(stderr, "Internal error: MAX_RESULT_BINDS overflow\n");
1194 +               exit_cleanup(RERR_UNSUPPORTED);
1195 +       }
1196 +
1197 +       if ((stmt = exec_mysql(ndx)) == NULL)
1198 +               return 0;
1199 +
1200 +       for (i = 0; i < bind_cnt; i++) {
1201 +               binds[i].is_null = &result_is_null[i];
1202 +               binds[i].length = &result_length[i];
1203 +               binds[i].error = &result_error[i];
1204 +       }
1205 +       mysql_stmt_bind_result(stmt, binds);
1206 +
1207 +       while (rows < max_rows) {
1208 +               if ((rc = mysql_stmt_fetch(stmt)) != 0) {
1209 +                       if (rc != MYSQL_NO_DATA)
1210 +                               rprintf(log_code, "SELECT fetch failed: %s\n", mysql_stmt_error(stmt));
1211 +                       break;
1212 +               }
1213 +               if (++rows >= max_rows)
1214 +                       break;
1215 +               for (i = 0; i < bind_cnt; i++) {
1216 +                       switch (binds[i].buffer_type) {
1217 +                       case MYSQL_TYPE_BLOB:
1218 +                       case MYSQL_TYPE_STRING:
1219 +                           binds[i].buffer += binds[i].buffer_length;
1220 +                           break;
1221 +                       case MYSQL_TYPE_LONG:
1222 +                           binds[i].buffer += sizeof (int);
1223 +                           break;
1224 +                       case MYSQL_TYPE_LONGLONG:
1225 +                           binds[i].buffer += sizeof (int64);
1226 +                           break;
1227 +                       default:
1228 +                           fprintf(stderr, "Unknown MYSQL_TYPE_* in multi-row read: %d.\n", binds[i].buffer_type);
1229 +                           exit_cleanup(RERR_UNSUPPORTED);
1230 +                       }
1231 +               }
1232 +       }
1233 +
1234 +       if (!stmt_ptr || rows < max_rows) {
1235 +               mysql_stmt_free_result(stmt);
1236 +               stmt = NULL;
1237 +       }
1238 +       if (stmt_ptr)
1239 +               *stmt_ptr = stmt;
1240 +
1241 +       return rows;
1242 +}
1243 +#endif
1244 +
1245 +static void update_mounts(void)
1246 +{
1247 +       char buf[2048], *argv[2];
1248 +       int f_from, f_to, len;
1249 +       STRUCT_STAT st;
1250 +       int pid, status;
1251 +
1252 +       if (DEBUG_GTE(DB, 2))
1253 +               printf("Running %s to grab mount info\n", RSYNCDB_MOUNTS);
1254 +       argv[0] = RSYNCDB_MOUNTS;
1255 +       argv[1] = NULL;
1256 +       pid = piped_child(argv, &f_from, &f_to);
1257 +       close(f_to);
1258 +
1259 +       bind_mtime = time(NULL); /* abuse mtime slightly to hold our last_seen value */
1260 +
1261 +       /* Strict format has 2 items with one tab as separator: MOUNT_UNIQ\tPATH */
1262 +       while ((len = read_line(f_from, buf, sizeof buf, 0)) > 0) {
1263 +               char *mount_uniq, *path;
1264 +
1265 +               if (DEBUG_GTE(DB, 3))
1266 +                       printf("Parsing mount info: %s\n", buf);
1267 +               mount_uniq = strtok(buf, "\t");
1268 +               path = mount_uniq ? strtok(NULL, "\r\n") : NULL;
1269 +               if (!path) {
1270 +                       fprintf(stderr, "Failed to parse line from %s output\n", RSYNCDB_MOUNTS);
1271 +                       exit_cleanup(RERR_SYNTAX);
1272 +               }
1273 +
1274 +               if (lstat(path, &st) < 0) {
1275 +                       fprintf(stderr, "Failed to lstat(%s): %s\n", path, strerror(errno));
1276 +                       exit_cleanup(RERR_IPC);
1277 +               }
1278 +
1279 +               bind_mount_uniq_len = strlcpy(bind_mount_uniq, mount_uniq, sizeof bind_mount_uniq);
1280 +               if (bind_mount_uniq_len >= (int)sizeof bind_mount_uniq)
1281 +                       bind_mount_uniq_len = sizeof bind_mount_uniq - 1;
1282 +
1283 +               if (db_output_msgs) {
1284 +                       printf("Marking mount \"%s\" (%s) as a recent mount\n",
1285 +                               bind_mount_uniq, big_num(st.st_dev));
1286 +               }
1287 +               switch (use_db) {
1288 +#ifdef USE_MYSQL
1289 +               case DB_TYPE_MYSQL:
1290 +                       bind_devno = st.st_dev;
1291 +                       if (exec_mysql(INS_MOUNT) == NULL) {
1292 +                               fprintf(stderr, "Failed to update mount info for \"%s\" - %s\n",
1293 +                                       bind_mount_uniq, mysql_error(dbh.mysql));
1294 +                               exit_cleanup(RERR_IPC);
1295 +                       }
1296 +                       break;
1297 +#endif
1298 +#ifdef USE_SQLITE
1299 +               case DB_TYPE_SQLITE: {
1300 +                       int rc, change_cnt;
1301 +                       sqlite3_stmt *stmt = statements[INS_MOUNT].sqlite;
1302 +                       sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1303 +                       sqlite3_bind_int64(stmt, 2, bind_mtime);
1304 +                       sqlite3_bind_text(stmt, 3, bind_mount_uniq, bind_mount_uniq_len, SQLITE_STATIC);
1305 +                       sqlite3_bind_int64(stmt, 4, st.st_dev);
1306 +                       rc = sqlite3_step(stmt);
1307 +                       if (rc != SQLITE_DONE) {
1308 +                               fprintf(stderr, "Failed to insert mount info for \"%s\" - %s (%d)\n",
1309 +                                       bind_mount_uniq, sqlite3_errmsg(dbh.sqlite), rc);
1310 +                               exit_cleanup(RERR_IPC);
1311 +                       }
1312 +                       change_cnt = sqlite3_changes(dbh.sqlite);
1313 +                       sqlite3_reset(stmt);
1314 +                       if (change_cnt == 0) {
1315 +                               stmt = statements[UPD_MOUNT].sqlite;
1316 +                               sqlite3_bind_int64(stmt, 1, bind_mtime);
1317 +                               sqlite3_bind_int64(stmt, 2, st.st_dev);
1318 +                               sqlite3_bind_text(stmt, 3, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1319 +                               sqlite3_bind_text(stmt, 4, bind_mount_uniq, bind_mount_uniq_len, SQLITE_STATIC);
1320 +                               rc = sqlite3_step(stmt);
1321 +                               if (rc != SQLITE_DONE) {
1322 +                                       fprintf(stderr, "Failed to update mount info for \"%s\" - %s (%d)\n",
1323 +                                               bind_mount_uniq, sqlite3_errmsg(dbh.sqlite), rc);
1324 +                                       exit_cleanup(RERR_IPC);
1325 +                               }
1326 +                               sqlite3_reset(stmt);
1327 +                       }
1328 +                       break;
1329 +                   }
1330 +#endif
1331 +               }
1332 +       }
1333 +       close(f_from);
1334 +
1335 +       waitpid(pid, &status, 0);
1336 +
1337 +       switch (use_db) {
1338 +#ifdef USE_MYSQL
1339 +       case DB_TYPE_MYSQL: {
1340 +               if (db_output_msgs) {
1341 +                       MYSQL_BIND binds[1];
1342 +                       MYSQL_STMT *stmt;
1343 +
1344 +                       binds[0].buffer_type = MYSQL_TYPE_BLOB;
1345 +                       binds[0].buffer = bind_mount_uniq;
1346 +                       binds[0].buffer_length = sizeof bind_mount_uniq;
1347 +                       if (fetch_mysql(binds, 1, SEL_MOUNT, 1, &stmt)) {
1348 +                               while (1) {
1349 +                                       printf("Marking mount \"%s\" as unmounted.\n", bind_mount_uniq);
1350 +                                       if (mysql_stmt_fetch(stmt) != 0)
1351 +                                               break;
1352 +                               }
1353 +                               mysql_stmt_free_result(stmt);
1354 +                       }
1355 +               }
1356 +
1357 +               if (exec_mysql(UN_MOUNT) == NULL) {
1358 +                       fprintf(stderr, "Failed to update old mount info - %s\n",
1359 +                               mysql_error(dbh.mysql));
1360 +                       exit_cleanup(RERR_IPC);
1361 +               }
1362 +               break;
1363 +           }
1364 +#endif
1365 +#ifdef USE_SQLITE
1366 +       case DB_TYPE_SQLITE: {
1367 +               sqlite3_stmt *stmt;
1368 +               int rc;
1369 +
1370 +               if (db_output_msgs) {
1371 +                       stmt = statements[SEL_MOUNT].sqlite;
1372 +                       sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1373 +                       sqlite3_bind_int64(stmt, 2, bind_mtime);
1374 +                       while (1) {
1375 +                               if (sqlite3_step(stmt) != SQLITE_ROW)
1376 +                                       break;
1377 +                               printf("Marking mount \"%s\" as unmounted.\n", sqlite3_column_text(stmt, 0));
1378 +                       }
1379 +                       sqlite3_reset(stmt);
1380 +               }
1381 +
1382 +               stmt = statements[UN_MOUNT].sqlite;
1383 +               sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1384 +               sqlite3_bind_int64(stmt, 2, bind_mtime);
1385 +               rc = sqlite3_step(stmt);
1386 +               sqlite3_reset(stmt);
1387 +               if (rc != SQLITE_DONE) {
1388 +                       fprintf(stderr, "Failed to update old mount info - %s (%d)\n",
1389 +                               sqlite3_errmsg(dbh.sqlite), rc);
1390 +                       exit_cleanup(RERR_IPC);
1391 +               }
1392 +               break;
1393 +           }
1394 +#endif
1395 +       }
1396 +}
1397 +
1398 +unsigned int get_disk_id(int64 devno)
1399 +{
1400 +       static unsigned int prior_disk_id = 0;
1401 +       static int64 prior_devno = 0;
1402 +
1403 +       if (prior_devno == devno && prior_disk_id) {
1404 +               if (DEBUG_GTE(DB, 5))
1405 +                       rprintf(FCLIENT, "get_disk_id(%s,%s) = %d (cached)\n", bind_thishost, big_num(devno), prior_disk_id);
1406 +               return prior_disk_id;
1407 +       }
1408 +       prior_devno = devno;
1409 +
1410 +       switch (use_db) {
1411 +#ifdef USE_MYSQL
1412 +       case DB_TYPE_MYSQL: {
1413 +               MYSQL_BIND binds[1];
1414 +
1415 +               bind_devno = devno; /* The one changing SEL_DEV input value. */
1416 +
1417 +               /* Bind where to put the output. */
1418 +               binds[0].buffer_type = MYSQL_TYPE_LONG;
1419 +               binds[0].buffer = &prior_disk_id;
1420 +               if (!fetch_mysql(binds, 1, SEL_DEV, 1, NULL))
1421 +                       prior_disk_id = 0;
1422 +               break;
1423 +           }
1424 +#endif
1425 +#ifdef USE_SQLITE
1426 +       case DB_TYPE_SQLITE: {
1427 +               sqlite3_stmt *stmt = statements[SEL_DEV].sqlite;
1428 +               sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1429 +               sqlite3_bind_int64(stmt, 2, devno);
1430 +               if (sqlite3_step(stmt) == SQLITE_ROW)
1431 +                       prior_disk_id = sqlite3_column_int(stmt, 0);
1432 +               else
1433 +                       prior_disk_id = 0;
1434 +               sqlite3_reset(stmt);
1435 +               break;
1436 +           }
1437 +#endif
1438 +       }
1439 +
1440 +       if (DEBUG_GTE(DB, 2))
1441 +               rprintf(FCLIENT, "get_disk_id(%s,%s) = %d\n", bind_thishost, big_num(devno), prior_disk_id);
1442 +       return prior_disk_id;
1443 +}
1444 +
1445 +int db_get_checksum(const STRUCT_STAT *st_p, char *sum)
1446 +{
1447 +       unsigned int disk_id = get_disk_id(st_p->st_dev);
1448 +       int ok = 0;
1449 +
1450 +       if (disk_id == 0)
1451 +               return 0;
1452 +
1453 +       switch (use_db) {
1454 +#ifdef USE_MYSQL
1455 +       case DB_TYPE_MYSQL: {
1456 +               MYSQL_BIND binds[1];
1457 +
1458 +               bind_disk_id = disk_id;
1459 +               bind_ino = st_p->st_ino;
1460 +               bind_size = st_p->st_size;
1461 +               bind_mtime = st_p->st_mtime;
1462 +               if (!db_lax)
1463 +                       bind_ctime = st_p->st_ctime;
1464 +
1465 +               binds[0].buffer_type = MYSQL_TYPE_BLOB;
1466 +               binds[0].buffer = sum;
1467 +               binds[0].buffer_length = checksum_len;
1468 +               ok = fetch_mysql(binds, 1, SEL_SUM, 1, NULL);
1469 +               break;
1470 +           }
1471 +#endif
1472 +#ifdef USE_SQLITE
1473 +       case DB_TYPE_SQLITE: {
1474 +               sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
1475 +               sqlite3_bind_int(stmt, 1, disk_id);
1476 +               sqlite3_bind_int64(stmt, 2, st_p->st_ino);
1477 +               sqlite3_bind_int64(stmt, 3, st_p->st_size);
1478 +               sqlite3_bind_int64(stmt, 4, st_p->st_mtime);
1479 +               if (!db_lax)
1480 +                       sqlite3_bind_int64(stmt, 5, st_p->st_ctime);
1481 +               if (sqlite3_step(stmt) == SQLITE_ROW) {
1482 +                       int len = sqlite3_column_bytes(stmt, 0);
1483 +                       if (len > MAX_DIGEST_LEN)
1484 +                               len = MAX_DIGEST_LEN;
1485 +                       memcpy(sum, sqlite3_column_blob(stmt, 0), len);
1486 +                       ok = 1;
1487 +               }
1488 +               sqlite3_reset(stmt);
1489 +               break;
1490 +           }
1491 +#endif
1492 +       }
1493 +
1494 +       if (DEBUG_GTE(DB, 2)) {
1495 +               if (ok) {
1496 +                       rprintf(FCLIENT, "[%s] Found DB checksum for %s,%s,%d: %s\n",
1497 +                               who_am_i(), big_num(st_p->st_dev),
1498 +                               big_num(st_p->st_ino), md_num, sum_as_hex(sum));
1499 +               } else {
1500 +                       rprintf(FCLIENT, "[%s] No DB checksum for %s,%s,%d\n",
1501 +                               who_am_i(), big_num(st_p->st_dev),
1502 +                               big_num(st_p->st_ino), md_num);
1503 +               }
1504 +       }
1505 +
1506 +       return ok;
1507 +}
1508 +
1509 +int db_get_both_checksums(const STRUCT_STAT *st_p, int *right_sum_cnt, int *wrong_sum_cnt, char **sum4, char **sum5)
1510 +{
1511 +       static char dbsum[MD5_DIGEST_LEN*2];
1512 +       int rows, j, sum_type[2];
1513 +       int64 dbsize[2], dbmtime[2], dbctime[2];
1514 +       unsigned int disk_id = get_disk_id(st_p->st_dev);
1515 +
1516 +       if (disk_id == 0)
1517 +               return 0;
1518 +
1519 +       switch (use_db) {
1520 +#ifdef USE_MYSQL
1521 +       case DB_TYPE_MYSQL: {
1522 +               MYSQL_BIND binds[5];
1523 +
1524 +               bind_disk_id = disk_id;
1525 +               bind_ino = st_p->st_ino;
1526 +
1527 +               binds[0].buffer_type = MYSQL_TYPE_BLOB;
1528 +               binds[0].buffer = dbsum;
1529 +               binds[0].buffer_length = checksum_len;
1530 +               binds[1].buffer_type = MYSQL_TYPE_LONG;
1531 +               binds[1].buffer = (char*)sum_type;
1532 +               binds[2].buffer_type = MYSQL_TYPE_LONGLONG;
1533 +               binds[2].buffer = (char*)dbsize;
1534 +               binds[3].buffer_type = MYSQL_TYPE_LONGLONG;
1535 +               binds[3].buffer = (char*)dbmtime;
1536 +               binds[4].buffer_type = MYSQL_TYPE_LONGLONG;
1537 +               binds[4].buffer = (char*)dbctime;
1538 +               rows = fetch_mysql(binds, 5, SEL_SUM, 2, NULL);
1539 +               break;
1540 +           }
1541 +#endif
1542 +#ifdef USE_SQLITE
1543 +       case DB_TYPE_SQLITE: {
1544 +               sqlite3_stmt *stmt = statements[SEL_SUM].sqlite;
1545 +               sqlite3_bind_int(stmt, 1, disk_id);
1546 +               sqlite3_bind_int64(stmt, 2, st_p->st_ino);
1547 +               for (j = 0; j < 2; j++) {
1548 +                       int len;
1549 +                       if (sqlite3_step(stmt) != SQLITE_ROW)
1550 +                               break;
1551 +                       len = sqlite3_column_bytes(stmt, 0);
1552 +                       if (len > checksum_len)
1553 +                               len = checksum_len;
1554 +                       memcpy(dbsum + checksum_len*j, sqlite3_column_blob(stmt, 0), len);
1555 +                       sum_type[j] = sqlite3_column_int(stmt, 1);
1556 +                       dbsize[j] = sqlite3_column_int(stmt, 2);
1557 +                       dbmtime[j] = sqlite3_column_int64(stmt, 3);
1558 +                       dbctime[j] = sqlite3_column_int64(stmt, 4);
1559 +               }
1560 +               sqlite3_reset(stmt);
1561 +               rows = j;
1562 +               break;
1563 +           }
1564 +#endif
1565 +       default:
1566 +               return 0;
1567 +       }
1568 +
1569 +       if (sum4)
1570 +               *sum4 = NULL;
1571 +       if (sum5)
1572 +               *sum5 = NULL;
1573 +       *right_sum_cnt = *wrong_sum_cnt = 0;
1574 +       for (j = 0; j < rows; j++) {
1575 +               if (DEBUG_GTE(DB, 3)) {
1576 +                       rprintf(FCLIENT, "DB checksum for %s,%s,%d: %s\n",
1577 +                               big_num(st_p->st_dev), big_num(st_p->st_ino), sum_type[j], sum_as_hex(dbsum + checksum_len*j));
1578 +               }
1579 +
1580 +               if (sum_type[j] == 4) {
1581 +                       if (!sum4)
1582 +                               continue;
1583 +                       *sum4 = dbsum + checksum_len*j;
1584 +               } else {
1585 +                       if (!sum5)
1586 +                               continue;
1587 +                       *sum5 = dbsum + checksum_len*j;
1588 +               }
1589 +               if (st_p->st_size == dbsize[j] && st_p->st_mtime == dbmtime[j] && (db_lax || st_p->st_ctime == dbctime[j]))
1590 +                       ++*right_sum_cnt;
1591 +               else
1592 +                       ++*wrong_sum_cnt;
1593 +       }
1594 +
1595 +       return rows;
1596 +}
1597 +
1598 +int db_set_checksum(int mdnum, const STRUCT_STAT *st_p, const char *sum)
1599 +{
1600 +       unsigned int disk_id;
1601 +       const char *errmsg = NULL;
1602 +       int rc = 0;
1603 +
1604 +       if (am_receiver || (am_generator && same_db)) {
1605 +               /* Forward the setting to a single process.  The receiver always
1606 +                * forward to the generator, and the generator will forward to
1607 +                * the receiver ONLY if this is a local transfer. */
1608 +               char data[MSG_CHECKSUM_LEN];
1609 +               SIVAL64(data, 0, st_p->st_dev);
1610 +               SIVAL64(data, 8, st_p->st_ino);
1611 +               SIVAL64(data, 16, st_p->st_size);
1612 +               SIVAL64(data, 24, st_p->st_mtime);
1613 +               SIVAL64(data, 32, st_p->st_ctime);
1614 +#if MSG_CHECKSUM_LONGS != 5
1615 +#error Fix the setting of checksum long values
1616 +#endif
1617 +               SIVAL(data, MSG_CHECKSUM_LONGS*8, mdnum);
1618 +               memcpy(data + MSG_CHECKSUM_LONGS*8 + 4, sum, MAX_DIGEST_LEN);
1619 +               return send_msg(MSG_CHECKSUM, data, sizeof data, 0);
1620 +       }
1621 +
1622 +       if ((disk_id = get_disk_id(st_p->st_dev)) == 0)
1623 +               return 0;
1624 +
1625 +       switch (use_db) {
1626 +#ifdef USE_MYSQL
1627 +       case DB_TYPE_MYSQL:
1628 +               if (transaction_state == 0) {
1629 +                       if (!run_sql("BEGIN"))
1630 +                               return 0;
1631 +                       transaction_state = 1;
1632 +               }
1633 +
1634 +               bind_disk_id = disk_id;
1635 +               bind_ino = st_p->st_ino;
1636 +               bind_mdnum = mdnum;
1637 +               bind_size = st_p->st_size;
1638 +               bind_mtime = st_p->st_mtime;
1639 +               bind_ctime = st_p->st_ctime;
1640 +               memcpy(bind_sum, sum, checksum_len);
1641 +               if (exec_mysql(REP_SUM) == NULL)
1642 +                       errmsg = mysql_error(dbh.mysql);
1643 +               break;
1644 +#endif
1645 +#ifdef USE_SQLITE
1646 +       case DB_TYPE_SQLITE: {
1647 +               sqlite3_stmt *stmt = statements[REP_SUM].sqlite;
1648 +               int lock_failures = 0;
1649 +
1650 +               if (transaction_state == 0) {
1651 +                       if (!run_sql("BEGIN"))
1652 +                               return 0;
1653 +                       transaction_state = 1;
1654 +               }
1655 +
1656 +               sqlite3_bind_int(stmt, 1, disk_id);
1657 +               sqlite3_bind_int64(stmt, 2, st_p->st_ino);
1658 +               sqlite3_bind_int(stmt, 3, mdnum);
1659 +               sqlite3_bind_int64(stmt, 4, st_p->st_size);
1660 +               sqlite3_bind_int64(stmt, 5, st_p->st_mtime);
1661 +               sqlite3_bind_int64(stmt, 6, st_p->st_ctime);
1662 +               sqlite3_bind_blob(stmt, 7, sum, checksum_len, SQLITE_TRANSIENT);
1663 +               while (1) {
1664 +                       rc = sqlite3_step(stmt);
1665 +                       if (rc != SQLITE_BUSY && rc != SQLITE_LOCKED)
1666 +                               break;
1667 +                       if (++lock_failures > MAX_LOCK_FAILURES)
1668 +                               break;
1669 +                       sqlite3_reset(stmt);
1670 +                       msleep(LOCK_FAIL_MSLEEP);
1671 +               }
1672 +               if (rc != SQLITE_DONE)
1673 +                       errmsg = sqlite3_errmsg(dbh.sqlite);
1674 +               sqlite3_reset(stmt);
1675 +               break;
1676 +           }
1677 +#endif
1678 +       }
1679 +
1680 +       if (!errmsg) {
1681 +               if (DEBUG_GTE(DB, 2)) {
1682 +                       rprintf(FCLIENT, "[%s] Set DB checksum for %s,%s,%d: %s\n",
1683 +                               who_am_i(), big_num(st_p->st_dev), big_num(st_p->st_ino),
1684 +                               md_num, sum_as_hex(sum));
1685 +               }
1686 +       } else {
1687 +               rprintf(log_code, "[%s] Failed to set checksum for %s,%s,%d: %s (%d) -- closing DB\n",
1688 +                       who_am_i(), big_num(st_p->st_dev), big_num(st_p->st_ino),
1689 +                       md_num, errmsg, rc);
1690 +               db_disconnect(False);
1691 +       }
1692 +
1693 +       return errmsg ? 0 : 1;
1694 +}
1695 +
1696 +/* For a delayed-update copy, we set the checksum on the file when it was
1697 + * inside the partial-dir.  Since renaming the file changes its ctime, we need
1698 + * to update the ctime to its new value (we can skip this in db_lax mode). */
1699 +int db_update_ctime(int mdnum, const STRUCT_STAT *st_p)
1700 +{
1701 +       unsigned int disk_id = get_disk_id(st_p->st_dev);
1702 +
1703 +       if (disk_id == 0)
1704 +               return 0;
1705 +
1706 +       switch (use_db) {
1707 +#ifdef USE_MYSQL
1708 +       case DB_TYPE_MYSQL:
1709 +               bind_ctime = st_p->st_ctime;
1710 +               bind_disk_id = disk_id;
1711 +               bind_ino = st_p->st_ino;
1712 +               bind_mdnum = mdnum;
1713 +               bind_size = st_p->st_size;
1714 +               bind_mtime = st_p->st_mtime;
1715 +               return exec_mysql(UPD_CTIME) != NULL;
1716 +#endif
1717 +#ifdef USE_SQLITE
1718 +       case DB_TYPE_SQLITE: {
1719 +               int rc;
1720 +
1721 +               sqlite3_stmt *stmt = statements[UPD_CTIME].sqlite;
1722 +               if (stmt == NULL)
1723 +                       return 0;
1724 +               sqlite3_bind_int64(stmt, 1, st_p->st_ctime);
1725 +               sqlite3_bind_int(stmt, 2, disk_id);
1726 +               sqlite3_bind_int64(stmt, 3, st_p->st_ino);
1727 +               sqlite3_bind_int(stmt, 4, mdnum);
1728 +               sqlite3_bind_int64(stmt, 5, st_p->st_size);
1729 +               sqlite3_bind_int64(stmt, 6, st_p->st_mtime);
1730 +               rc = sqlite3_step(stmt);
1731 +               sqlite3_reset(stmt);
1732 +               return rc == SQLITE_DONE;
1733 +           }
1734 +#endif
1735 +       }
1736 +
1737 +       return 0;
1738 +}
1739 +
1740 +int db_clean_init(void)
1741 +{
1742 +       switch (use_db) {
1743 +#ifdef USE_MYSQL
1744 +       case DB_TYPE_MYSQL: {
1745 +               MYSQL_BIND binds[MAX_BIND_CNT];
1746 +               char *sql;
1747 +
1748 +               mysql_query(dbh.mysql,
1749 +                       "CREATE TEMPORARY TABLE inode_present ("
1750 +                       " disk_id integer unsigned NOT NULL,"
1751 +                       " ino bigint unsigned NOT NULL,"
1752 +                       " present tinyint NOT NULL default '1',"
1753 +                       " PRIMARY KEY (disk_id,ino)"
1754 +                       ") ENGINE=MEMORY"
1755 +                       );
1756 +
1757 +               sql="INSERT IGNORE INTO inode_present"
1758 +                   " SET disk_id = ?, ino = ?, present = 1";
1759 +               memset(binds, 0, sizeof binds);
1760 +               binds[0].buffer_type = MYSQL_TYPE_LONG;
1761 +               binds[0].buffer = &bind_disk_id;
1762 +               binds[1].buffer_type = MYSQL_TYPE_LONGLONG;
1763 +               binds[1].buffer = &bind_ino;
1764 +               if (!prepare_mysql(INS_PRESENT, binds, 2, sql))
1765 +                       exit_cleanup(RERR_SYNTAX);
1766 +
1767 +               sql="DELETE m.*"
1768 +                   " FROM inode_map AS m"
1769 +                   " LEFT JOIN inode_present USING(disk_id, ino)"
1770 +                   " JOIN disk AS d ON(m.disk_id = d.disk_id)"
1771 +                   " WHERE host = ? AND devno != 0 AND present IS NULL";
1772 +               memset(binds, 0, sizeof binds);
1773 +               binds[0].buffer_type = MYSQL_TYPE_STRING;
1774 +               binds[0].buffer = &bind_thishost;
1775 +               binds[0].buffer_length = bind_thishost_len;
1776 +               if (!prepare_mysql(DEL_SUMS, binds, 1, sql))
1777 +                       exit_cleanup(RERR_SYNTAX);
1778 +
1779 +               return 1;
1780 +           }
1781 +#endif
1782 +#ifdef USE_SQLITE
1783 +       case DB_TYPE_SQLITE: {
1784 +               char *sql;
1785 +               sql="ATTACH DATABASE '' AS aux1;"; /* Private temp DB, probably in-memory */
1786 +               if (!run_sql(sql))
1787 +                       exit_cleanup(RERR_IPC);
1788 +
1789 +               sql="CREATE TABLE aux1.inode_present ("
1790 +                   " disk_id integer NOT NULL,"
1791 +                   " ino bigint NOT NULL,"
1792 +                   " present tinyint NOT NULL default '1',"
1793 +                   " PRIMARY KEY (disk_id,ino)"
1794 +                   ")";
1795 +               if (!run_sql(sql))
1796 +                       exit_cleanup(RERR_IPC);
1797 +
1798 +               sql="INSERT OR IGNORE INTO aux1.inode_present"
1799 +                   " (disk_id, ino, present)"
1800 +                   " VALUES (?, ?, 1)";
1801 +               if (!prepare_sqlite(INS_PRESENT, sql))
1802 +                       exit_cleanup(RERR_IPC);
1803 +
1804 +               sql="DELETE FROM inode_map"
1805 +                   " WHERE ROWID IN ("
1806 +                   "  SELECT m.ROWID"
1807 +                   "  FROM inode_map AS m"
1808 +                   "  LEFT JOIN aux1.inode_present USING(disk_id, ino)"
1809 +                   "  JOIN disk AS d ON(m.disk_id = d.disk_id)"
1810 +                   "  WHERE host = ? AND devno != 0 AND present IS NULL"
1811 +                   " )";
1812 +               if (!prepare_sqlite(DEL_SUMS, sql))
1813 +                       exit_cleanup(RERR_IPC);
1814 +
1815 +               transaction_state = -1; /* bug work-around -- force transaction off when cleaning XXX */
1816 +
1817 +               return 1;
1818 +           }
1819 +#endif
1820 +       }
1821 +
1822 +       return 0;
1823 +}
1824 +
1825 +int db_note_present(int disk_id, int64 ino)
1826 +{
1827 +       switch (use_db) {
1828 +#ifdef USE_MYSQL
1829 +       case DB_TYPE_MYSQL:
1830 +               bind_disk_id = disk_id;
1831 +               bind_ino = ino;
1832 +               return exec_mysql(INS_PRESENT) != NULL;
1833 +#endif
1834 +#ifdef USE_SQLITE
1835 +       case DB_TYPE_SQLITE: {
1836 +               int rc;
1837 +               sqlite3_stmt *stmt = statements[INS_PRESENT].sqlite;
1838 +               sqlite3_bind_int(stmt, 1, disk_id);
1839 +               sqlite3_bind_int64(stmt, 2, ino);
1840 +               rc = sqlite3_step(stmt);
1841 +               sqlite3_reset(stmt);
1842 +               return rc == SQLITE_DONE;
1843 +           }
1844 +#endif
1845 +       }
1846 +
1847 +       return 0;
1848 +}
1849 +
1850 +/* This function requires the user to have populated all disk_id+inode pairs
1851 + * into the inode_present table. */
1852 +int db_clean_inodes(void)
1853 +{
1854 +       int del_cnt = 0;
1855 +
1856 +       switch (use_db) {
1857 +#ifdef USE_MYSQL
1858 +       case DB_TYPE_MYSQL: {
1859 +               MYSQL_STMT *stmt = exec_mysql(DEL_SUMS);
1860 +               if (stmt != NULL)
1861 +                       del_cnt = mysql_affected_rows(dbh.mysql);
1862 +               break;
1863 +           }
1864 +#endif
1865 +#ifdef USE_SQLITE
1866 +       case DB_TYPE_SQLITE: {
1867 +               int rc;
1868 +               sqlite3_stmt *stmt = statements[DEL_SUMS].sqlite;
1869 +               sqlite3_bind_text(stmt, 1, bind_thishost, bind_thishost_len, SQLITE_STATIC);
1870 +               rc = sqlite3_step(stmt);
1871 +               if (rc == SQLITE_DONE)
1872 +                       del_cnt = sqlite3_changes(dbh.sqlite);
1873 +               sqlite3_reset(stmt);
1874 +               break;
1875 +           }
1876 +#endif
1877 +       }
1878 +
1879 +       return del_cnt;
1880 +}
1881 +
1882 +static int abs_path(char *buf, int bufsiz, const char *curdir, const char *dir)
1883 +{
1884 +       if (*dir == '/')
1885 +               strlcpy(buf, dir, bufsiz);
1886 +       else
1887 +               snprintf(buf, bufsiz, "%s/%s", curdir, dir);
1888 +
1889 +       return clean_fname(buf, CFN_DROP_TRAILING_DOT_DIR | CFN_COLLAPSE_DOT_DOT_DIRS);
1890 +}
1891 +
1892 +static struct name_list *new_name(const char *basename, const char *filename)
1893 +{
1894 +       struct name_list *n;
1895 +       int blen = strlen(basename);
1896 +       int slen = filename ? (int)strlen(filename) : -1;
1897 +       int len = blen + 1 + slen;
1898 +
1899 +       if (len >= MAXPATHLEN) {
1900 +               if (filename)
1901 +                       rprintf(FERROR, "Filename too long: %s/%s\n", basename, filename);
1902 +               else
1903 +                       rprintf(FERROR, "Filename too long: %s\n", basename);
1904 +               return NULL;
1905 +       }
1906 +
1907 +       if (!(n = (struct name_list *)malloc(sizeof (struct name_list) + len)))
1908 +               out_of_memory("new_name");
1909 +
1910 +       memcpy(n->name, basename, blen);
1911 +       if (filename) {
1912 +               n->name[blen] = '/';
1913 +               memcpy(n->name + 1 + blen, filename, slen);
1914 +       }
1915 +       n->name[len] = '\0';
1916 +       n->next = NULL;
1917 +
1918 +       return n;
1919 +}
1920 +
1921 +static int name_compare(const void *n1, const void *n2)
1922 +{
1923 +       struct name_list *p1 = *(struct name_list **)n1;
1924 +       struct name_list *p2 = *(struct name_list **)n2;
1925 +       return strcmp(p1->name, p2->name);
1926 +}
1927 +
1928 +static struct name_list *get_sorted_names(const char *dir)
1929 +{
1930 +       struct name_list *add, **sortbuf, *names = NULL, *prior_name = NULL;
1931 +       struct dirent *di;
1932 +       int cnt = 0;
1933 +       DIR *d;
1934 +
1935 +       if (!(d = opendir("."))) {
1936 +               rprintf(FERROR, "Unable to opendir %s: %s\n", dir, strerror(errno));
1937 +               return NULL;
1938 +       }
1939 +       while ((di = readdir(d)) != NULL) {
1940 +               char *dname = d_name(di);
1941 +               if (dname[0] == '.' && (dname[1] == '\0' || (dname[1] == '.' && dname[2] == '\0')))
1942 +                       continue;
1943 +               if (!(add = new_name(dname, NULL)))
1944 +                       continue;
1945 +               if (prior_name)
1946 +                       prior_name->next = add;
1947 +               else
1948 +                       names = add;
1949 +               prior_name = add;
1950 +               cnt++;
1951 +       }
1952 +       closedir(d);
1953 +
1954 +       if (cnt) {
1955 +               int j;
1956 +
1957 +               if (!(sortbuf = new_array(struct name_list *, cnt)))
1958 +                       out_of_memory("get_sorted_names");
1959 +               for (j = 0; j < cnt; j++) {
1960 +                       sortbuf[j] = names;
1961 +                       names = names->next;
1962 +               }
1963 +
1964 +               qsort(sortbuf, cnt, PTR_SIZE, name_compare);
1965 +
1966 +               names = prior_name = NULL;
1967 +               for (j = 0; j < cnt; j++) {
1968 +                       add = sortbuf[j];
1969 +                       if (prior_name)
1970 +                               prior_name->next = add;
1971 +                       else
1972 +                               names = add;
1973 +                       prior_name = add;
1974 +               }
1975 +
1976 +               if (prior_name)
1977 +                       prior_name->next = NULL;
1978 +               free(sortbuf);
1979 +       }
1980 +
1981 +       return names;
1982 +}
1983 +
1984 +static inline int sums_ne(const char *sum1, const char *sum2)
1985 +{
1986 +       return memcmp(sum1, sum2, checksum_len) != 0;
1987 +}
1988 +
1989 +/* Returns 1 if there is a checksum change, else 0. */
1990 +static int mention_file(const char *dir, const char *name, int right_cnt, int wrong_cnt,
1991 +                       const char *dbsum4, const char *dbsum5, const char *sum4, const char *sum5)
1992 +{
1993 +       char *info_str = wrong_cnt && !right_cnt ? "!i " : "   ";
1994 +       char *md4_str = !db_do_md4 ? NULL : !dbsum4 ? "+4 " : !sum4 ? "?4 " : sums_ne(sum4, dbsum4) ? "!4 " : "   ";
1995 +       char *md5_str = !db_do_md5 ? NULL : !dbsum5 ? "+5 " : !sum5 ? "?5 " : sums_ne(sum5, dbsum5) ? "!5 " : "   ";
1996 +       int chg = *info_str != ' ' || (md4_str && *md4_str != ' ') || (md5_str && *md5_str != ' ');
1997 +       if (chg || db_output_unchanged) {
1998 +               if (db_output_info) {
1999 +                       fputs(info_str, stdout);
2000 +                       if (md4_str)
2001 +                               fputs(md4_str, stdout);
2002 +                       if (md5_str)
2003 +                               fputs(md5_str, stdout);
2004 +               }
2005 +               if (db_output_sum) {
2006 +                       if (db_do_md4)
2007 +                               printf("%s ", sum_as_hex(sum4));
2008 +                       if (db_do_md5)
2009 +                               printf("%s ", sum_as_hex(sum5));
2010 +               }
2011 +               if (db_output_name) {
2012 +                       if (db_output_sum)
2013 +                               putchar(' '); /* We want 2 spaces, like md5sum. */
2014 +                       if (*dir != '.' || dir[1]) {
2015 +                               fputs(dir, stdout);
2016 +                               putchar('/');
2017 +                       }
2018 +                       puts(name);
2019 +               }
2020 +       }
2021 +
2022 +       return chg;
2023 +}
2024 +
2025 +NORETURN void run_dbonly(const char **args)
2026 +{
2027 +       char start_dir[MAXPATHLEN], dirbuf[MAXPATHLEN];
2028 +       int need_sum_cnt, start_dir_len;
2029 +       struct name_list *prior_dir;
2030 +       struct name_list *names;
2031 +       int exit_code = 0;
2032 +
2033 +       checksum_len = MD5_DIGEST_LEN; /* Same as MD4_DIGEST_LEN */
2034 +       protocol_version = 31;
2035 +
2036 +       need_sum_cnt = db_do_md4 + db_do_md5;
2037 +
2038 +       if (!db_read_config(FERROR, db_config) || !db_connect(1))
2039 +               exit_cleanup(RERR_FILEIO);
2040 +
2041 +       if (db_clean)
2042 +               db_clean_init();
2043 +
2044 +       if (getcwd(start_dir, sizeof start_dir - 1) == NULL) {
2045 +               rsyserr(FERROR, errno, "getcwd()");
2046 +               exit_cleanup(RERR_FILESELECT);
2047 +       }
2048 +       start_dir_len = strlen(start_dir);
2049 +
2050 +       if (args) {
2051 +               prior_dir = NULL;
2052 +               while (*args) {
2053 +                       struct name_list *add;
2054 +                       if (abs_path(dirbuf, sizeof dirbuf, start_dir, *args++) <= 0)
2055 +                               continue;
2056 +                       if (!(add = new_name(dirbuf, NULL)))
2057 +                               continue;
2058 +                       if (prior_dir)
2059 +                               prior_dir->next = add;
2060 +                       else
2061 +                               dirs_list = add;
2062 +                       prior_dir = add;
2063 +               }
2064 +       } else
2065 +               dirs_list = new_name(start_dir, NULL);
2066 +
2067 +       prior_dir = NULL;
2068 +       while (dirs_list) {
2069 +               struct name_list *subdirs, *prior_subdir, *prior_name;
2070 +               const char *dir = dirs_list->name;
2071 +               const char *reldir = dir;
2072 +
2073 +               if (prior_dir)
2074 +                       free((void*)prior_dir);
2075 +               prior_dir = dirs_list;
2076 +               dirs_list = dirs_list->next;
2077 +
2078 +               if (strncmp(reldir, start_dir, start_dir_len) == 0) {
2079 +                       if (reldir[start_dir_len] == '\0')
2080 +                               reldir = ".";
2081 +                       else if (reldir[start_dir_len] == '/')
2082 +                               reldir += start_dir_len + 1;
2083 +               }
2084 +               if (db_output_dirs)
2085 +                       printf("... %s/ ...\n", reldir);
2086 +
2087 +               if (chdir(dir) < 0) {
2088 +                       rprintf(FERROR, "Unable to chdir to %s: %s\n", dir, strerror(errno));
2089 +                       continue;
2090 +               }
2091 +               if (!(names = get_sorted_names(dir)))
2092 +                       continue;
2093 +
2094 +               subdirs = prior_subdir = prior_name = NULL;
2095 +               while (names) {
2096 +                       STRUCT_STAT st;
2097 +                       char *dbsum4, *sum4, sumbuf4[MD5_DIGEST_LEN];
2098 +                       char *dbsum5, *sum5, sumbuf5[MD5_DIGEST_LEN];
2099 +                       int right_sum_cnt, wrong_sum_cnt;
2100 +                       const char *name = names->name;
2101 +                       unsigned int disk_id;
2102 +
2103 +                       if (prior_name)
2104 +                               free((void*)prior_name);
2105 +                       prior_name = names;
2106 +                       names = names->next;
2107 +
2108 +                       dbsum4 = dbsum5 = sum4 = sum5 = NULL;
2109 +
2110 +                       if (lstat(name, &st) < 0) {
2111 +                               rprintf(FERROR, "Failed to lstat(%s): %s\n", name, strerror(errno));
2112 +                               continue;
2113 +                       }
2114 +                       if (S_ISLNK(st.st_mode))
2115 +                               continue;
2116 +                       if (S_ISDIR(st.st_mode)) {
2117 +                               /* add optional excluding of things like /^(CVS|\.svn|\.git|\.bzr)$/; */
2118 +                               if (recurse) {
2119 +                                       struct name_list *add = new_name(dir, name);
2120 +                                       if (add) {
2121 +                                               if (prior_subdir)
2122 +                                                       prior_subdir->next = add;
2123 +                                               else
2124 +                                                       subdirs = add;
2125 +                                               prior_subdir = add;
2126 +                                       }
2127 +                               }
2128 +                               continue;
2129 +                       }
2130 +                       if (!S_ISREG(st.st_mode))
2131 +                               continue;
2132 +
2133 +                       if (!(disk_id = get_disk_id(st.st_dev)))
2134 +                               continue;
2135 +                       if (db_clean) {
2136 +                               db_note_present(disk_id, st.st_ino);
2137 +                               if (!db_update && !db_check)
2138 +                                       continue;
2139 +                       }
2140 +                       db_get_both_checksums(&st, &right_sum_cnt, &wrong_sum_cnt,
2141 +                                             db_do_md4 ? &dbsum4 : NULL, db_do_md5 ? &dbsum5 : NULL);
2142 +
2143 +                       if (!db_check && right_sum_cnt == need_sum_cnt) {
2144 +                               mention_file(reldir, name, right_sum_cnt, wrong_sum_cnt, dbsum4, dbsum5, dbsum4, dbsum5);
2145 +                               continue;
2146 +                       }
2147 +
2148 +                       if (db_update || (db_check && right_sum_cnt) || db_output_sum) {
2149 +                               uchar *data;
2150 +                               int32 remainder;
2151 +                               md_context m4, m5;
2152 +                               struct map_struct *buf;
2153 +                               OFF_T off, len = st.st_size;
2154 +                               int fd = do_open(name, O_RDONLY, 0);
2155 +
2156 +                               if (fd < 0) {
2157 +                                       rprintf(FERROR, "ERROR: unable to read %s: %s\n", name, strerror(errno));
2158 +                                       continue;
2159 +                               }
2160 +
2161 +                               if (db_do_md4)
2162 +                                       mdfour_begin(&m4);
2163 +                               if (db_do_md5)
2164 +                                       md5_begin(&m5);
2165 +
2166 +                               buf = map_file(fd, len, MAX_MAP_SIZE, CSUM_CHUNK);
2167 +
2168 +                               for (off = 0; off + CSUM_CHUNK <= len; off += CSUM_CHUNK) {
2169 +                                       data = (uchar*)map_ptr(buf, off, CSUM_CHUNK);
2170 +                                       if (db_do_md4)
2171 +                                               mdfour_update(&m4, data, CSUM_CHUNK);
2172 +                                       if (db_do_md5)
2173 +                                               md5_update(&m5, data, CSUM_CHUNK);
2174 +                               }
2175 +
2176 +                               remainder = (int32)(len - off);
2177 +                               data = (uchar*)map_ptr(buf, off, remainder);
2178 +                               if (db_do_md4) {
2179 +                                       mdfour_update(&m4, data, remainder);
2180 +                                       mdfour_result(&m4, (uchar*)(sum4 = sumbuf4));
2181 +                               }
2182 +                               if (db_do_md5) {
2183 +                                       md5_update(&m5, data, remainder);
2184 +                                       md5_result(&m5, (uchar*)(sum5 = sumbuf5));
2185 +                               }
2186 +
2187 +                               close(fd);
2188 +                               unmap_file(buf);
2189 +                       }
2190 +
2191 +                       int chg = mention_file(reldir, name, right_sum_cnt, wrong_sum_cnt, dbsum4, dbsum5, sum4, sum5);
2192 +                       if (!chg) {
2193 +                               /* Only db_check should get here... */
2194 +                       } else if (!db_update) {
2195 +                               exit_code = 1;
2196 +                       } else {
2197 +                               int fail = 0;
2198 +                               if (db_do_md4 && !db_set_checksum(4, &st, sum4))
2199 +                                       fail = 1;
2200 +                               if (db_do_md5 && !db_set_checksum(5, &st, sum5))
2201 +                                       fail = 1;
2202 +                               if (fail) {
2203 +                                       fprintf(stderr, "Failed to set checksum on %s/%s\n", reldir, name);
2204 +                                       exit_cleanup(RERR_FILEIO);
2205 +                               }
2206 +                       }
2207 +               }
2208 +               if (prior_name)
2209 +                       free((void*)prior_name);
2210 +
2211 +               if (recurse && subdirs) {
2212 +                       prior_subdir->next = dirs_list;
2213 +                       dirs_list = subdirs;
2214 +               }
2215 +       }
2216 +       if (prior_dir)
2217 +               free((void*)prior_dir);
2218 +
2219 +       if (db_clean) {
2220 +               int rows = db_clean_inodes();
2221 +               if (db_output_msgs)
2222 +                       printf("Cleaned out %d old inode%s.\n", rows, rows == 1 ? "" : "s");
2223 +       }
2224 +
2225 +       db_disconnect(True);
2226 +       exit(exit_code);
2227 +}
2228 diff --git a/flist.c b/flist.c
2229 --- a/flist.c
2230 +++ b/flist.c
2231 @@ -52,6 +52,7 @@ extern int preserve_devices;
2232  extern int preserve_specials;
2233  extern int delete_during;
2234  extern int missing_args;
2235 +extern int use_db;
2236  extern int eol_nulls;
2237  extern int relative_paths;
2238  extern int implied_dirs;
2239 @@ -1302,11 +1303,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
2240                 extra_len += EXTRA_LEN;
2241  #endif
2242
2243 -       if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
2244 -               file_checksum(thisname, tmp_sum, st.st_size);
2245 -               if (sender_keeps_checksum)
2246 -                       extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
2247 -       }
2248 +       if (sender_keeps_checksum && S_ISREG(st.st_mode))
2249 +               extra_len += SUM_EXTRA_CNT * EXTRA_LEN;
2250
2251  #if EXTRA_ROUNDING > 0
2252         if (extra_len & (EXTRA_ROUNDING * EXTRA_LEN))
2253 @@ -1391,8 +1389,12 @@ struct file_struct *make_file(const char *fname, struct file_list *flist,
2254                 return NULL;
2255         }
2256
2257 -       if (sender_keeps_checksum && S_ISREG(st.st_mode))
2258 -               memcpy(F_SUM(file), tmp_sum, checksum_len);
2259 +       if (always_checksum && am_sender && S_ISREG(st.st_mode)) {
2260 +               if (!use_db || !db_get_checksum(&st, tmp_sum))
2261 +                       file_checksum(thisname, &st, tmp_sum);
2262 +               if (sender_keeps_checksum)
2263 +                       memcpy(F_SUM(file), tmp_sum, checksum_len);
2264 +       }
2265
2266         if (unsort_ndx)
2267                 F_NDX(file) = stats.num_dirs;
2268 @@ -2053,6 +2055,9 @@ void send_extra_file_list(int f, int at_least)
2269    finish:
2270         if (io_error != save_io_error && protocol_version == 30 && !ignore_errors)
2271                 send_msg_int(MSG_IO_ERROR, io_error);
2272 +
2273 +       if (use_db && flist_eof)
2274 +               db_disconnect(True);
2275  }
2276
2277  struct file_list *send_file_list(int f, int argc, char *argv[])
2278 @@ -2076,6 +2081,13 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
2279                      | (eol_nulls || reading_remotely ? RL_EOL_NULLS : 0);
2280         int implied_dot_dir = 0;
2281
2282 +       if (use_db) {
2283 +               if (always_checksum)
2284 +                       db_connect(0); /* Will reset use_db on error. */
2285 +               else
2286 +                       use_db = 0;
2287 +       }
2288 +
2289         rprintf(FLOG, "building file list\n");
2290         if (show_filelist_p())
2291                 start_filelist_progress("building file list");
2292 @@ -2422,6 +2434,9 @@ struct file_list *send_file_list(int f, int argc, char *argv[])
2293                         rprintf(FINFO, "[%s] flist_eof=1\n", who_am_i());
2294         }
2295
2296 +       if (use_db && (!inc_recurse || flist_eof))
2297 +               db_disconnect(True);
2298 +
2299         return flist;
2300  }
2301
2302 diff --git a/generator.c b/generator.c
2303 --- a/generator.c
2304 +++ b/generator.c
2305 @@ -58,6 +58,7 @@ extern int human_readable;
2306  extern int ignore_existing;
2307  extern int ignore_non_existing;
2308  extern int inplace;
2309 +extern int use_db;
2310  extern int append_mode;
2311  extern int make_backups;
2312  extern int csum_length;
2313 @@ -572,7 +573,8 @@ int unchanged_file(char *fn, struct file_struct *file, STRUCT_STAT *st)
2314            of the file time to determine whether to sync */
2315         if (always_checksum > 0 && S_ISREG(st->st_mode)) {
2316                 char sum[MAX_DIGEST_LEN];
2317 -               file_checksum(fn, sum, st->st_size);
2318 +               if (!use_db || !db_get_checksum(st, sum))
2319 +                       file_checksum(fn, st, sum);
2320                 return memcmp(sum, F_SUM(file), checksum_len) == 0;
2321         }
2322
2323 @@ -2195,6 +2197,13 @@ void generate_files(int f_out, const char *local_name)
2324                         : "enabled");
2325         }
2326
2327 +       if (use_db) {
2328 +               if (always_checksum)
2329 +                       db_connect(0); /* Will reset use_db on error. */
2330 +               else
2331 +                       use_db = 0;
2332 +       }
2333 +
2334         dflt_perms = (ACCESSPERMS & ~orig_umask);
2335
2336         do {
2337 @@ -2320,6 +2329,9 @@ void generate_files(int f_out, const char *local_name)
2338                         wait_for_receiver();
2339         }
2340
2341 +       if (use_db)
2342 +               db_disconnect(True);
2343 +
2344         info_levels[INFO_FLIST] = save_info_flist;
2345         info_levels[INFO_PROGRESS] = save_info_progress;
2346
2347 diff --git a/io.c b/io.c
2348 --- a/io.c
2349 +++ b/io.c
2350 @@ -41,8 +41,10 @@ extern int am_server;
2351  extern int am_sender;
2352  extern int am_receiver;
2353  extern int am_generator;
2354 +extern int local_server;
2355  extern int msgs2stderr;
2356  extern int inc_recurse;
2357 +extern int same_db;
2358  extern int io_error;
2359  extern int eol_nulls;
2360  extern int flist_eof;
2361 @@ -1481,6 +1483,32 @@ static void read_a_msg(void)
2362                 if (am_sender)
2363                         maybe_send_keepalive(time(NULL), MSK_ALLOW_FLUSH);
2364                 break;
2365 +       case MSG_CHECKSUM:
2366 +               /* This receives some checksum info that we want to make a note of
2367 +                * (which allows a single process to do all the writing to the db). */
2368 +               if (msg_bytes != MSG_CHECKSUM_LEN)
2369 +                       goto overflow;
2370 +               raw_read_buf(data, MSG_CHECKSUM_LEN);
2371 +               if (am_generator && same_db) {
2372 +                       iobuf.in_multiplexed = 1;
2373 +                       send_msg(MSG_CHECKSUM, data, MSG_CHECKSUM_LEN, 0);
2374 +               } if (am_receiver || (am_sender && !local_server))
2375 +                       goto unexpected;
2376 +               else {
2377 +                       /* The received data is a set of numbers followed by the checksum. */
2378 +                       STRUCT_STAT st;
2379 +                       st.st_dev = IVAL64(data, 0);
2380 +                       st.st_ino = IVAL64(data, 8);
2381 +                       st.st_size = IVAL64(data, 16);
2382 +                       st.st_mtime = IVAL64(data, 24);
2383 +                       st.st_ctime = IVAL64(data, 32);
2384 +#if MSG_CHECKSUM_LONGS != 5
2385 +#error Fix the parsing of checksum long values
2386 +#endif
2387 +                       iobuf.in_multiplexed = 1;
2388 +                       db_set_checksum(IVAL(data, MSG_CHECKSUM_LONGS*8), &st, data + MSG_CHECKSUM_LONGS*8 + 4);
2389 +               }
2390 +               break;
2391         case MSG_DELETED:
2392                 if (msg_bytes >= sizeof data)
2393                         goto overflow;
2394 @@ -1632,6 +1660,7 @@ static void read_a_msg(void)
2395                  * with a duplicate exit message. */
2396                 _exit_cleanup(val, __FILE__, 0 - __LINE__);
2397         default:
2398 +       unexpected:
2399                 rprintf(FERROR, "unexpected tag %d [%s%s]\n",
2400                         tag, who_am_i(), inc_recurse ? "/inc" : "");
2401                 exit_cleanup(RERR_STREAMIO);
2402 diff --git a/loadparm.c b/loadparm.c
2403 --- a/loadparm.c
2404 +++ b/loadparm.c
2405 @@ -109,6 +109,7 @@ typedef struct {
2406         char *auth_users;
2407         char *charset;
2408         char *comment;
2409 +       char *db_config;
2410         char *dont_compress;
2411         char *exclude;
2412         char *exclude_from;
2413 @@ -139,6 +140,7 @@ typedef struct {
2414         int syslog_facility;
2415         int timeout;
2416
2417 +       BOOL db_lax;
2418         BOOL fake_super;
2419         BOOL forward_lookup;
2420         BOOL ignore_errors;
2421 @@ -185,6 +187,7 @@ static const all_vars Defaults = {
2422   /* auth_users; */             NULL,
2423   /* charset; */                NULL,
2424   /* comment; */                NULL,
2425 + /* db_config; */              NULL,
2426   /* dont_compress; */          DEFAULT_DONT_COMPRESS,
2427   /* exclude; */                        NULL,
2428   /* exclude_from; */           NULL,
2429 @@ -213,6 +216,7 @@ static const all_vars Defaults = {
2430   /* syslog_facility; */                LOG_DAEMON,
2431   /* timeout; */                        0,
2432
2433 + /* db_lax; */                 False,
2434   /* fake_super; */             False,
2435   /* forward_lookup; */         True,
2436   /* ignore_errors; */          False,
2437 @@ -322,6 +326,8 @@ static struct parm_struct parm_table[] =
2438   {"auth users",        P_STRING, P_LOCAL, &Vars.l.auth_users,          NULL,0},
2439   {"charset",           P_STRING, P_LOCAL, &Vars.l.charset,             NULL,0},
2440   {"comment",           P_STRING, P_LOCAL, &Vars.l.comment,             NULL,0},
2441 + {"db config",         P_STRING, P_LOCAL, &Vars.l.db_config,           NULL,0},
2442 + {"db lax",            P_BOOL,   P_LOCAL, &Vars.l.db_lax,              NULL,0},
2443   {"dont compress",     P_STRING, P_LOCAL, &Vars.l.dont_compress,       NULL,0},
2444   {"exclude from",      P_STRING, P_LOCAL, &Vars.l.exclude_from,        NULL,0},
2445   {"exclude",           P_STRING, P_LOCAL, &Vars.l.exclude,             NULL,0},
2446 @@ -454,6 +460,7 @@ FN_GLOBAL_INTEGER(lp_rsync_port, &Vars.g.rsync_port)
2447  FN_LOCAL_STRING(lp_auth_users, auth_users)
2448  FN_LOCAL_STRING(lp_charset, charset)
2449  FN_LOCAL_STRING(lp_comment, comment)
2450 +FN_LOCAL_STRING(lp_db_config, db_config)
2451  FN_LOCAL_STRING(lp_dont_compress, dont_compress)
2452  FN_LOCAL_STRING(lp_exclude, exclude)
2453  FN_LOCAL_STRING(lp_exclude_from, exclude_from)
2454 @@ -482,6 +489,7 @@ FN_LOCAL_INTEGER(lp_max_verbosity, max_verbosity)
2455  FN_LOCAL_INTEGER(lp_syslog_facility, syslog_facility)
2456  FN_LOCAL_INTEGER(lp_timeout, timeout)
2457
2458 +FN_LOCAL_BOOL(lp_db_lax, db_lax)
2459  FN_LOCAL_BOOL(lp_fake_super, fake_super)
2460  FN_LOCAL_BOOL(lp_forward_lookup, forward_lookup)
2461  FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors)
2462 diff --git a/main.c b/main.c
2463 --- a/main.c
2464 +++ b/main.c
2465 @@ -34,6 +34,7 @@ extern int am_root;
2466  extern int am_server;
2467  extern int am_sender;
2468  extern int am_daemon;
2469 +extern int am_dbadmin;
2470  extern int inc_recurse;
2471  extern int blocking_io;
2472  extern int always_checksum;
2473 @@ -51,6 +52,7 @@ extern int copy_unsafe_links;
2474  extern int keep_dirlinks;
2475  extern int preserve_hard_links;
2476  extern int protocol_version;
2477 +extern int always_checksum;
2478  extern int file_total;
2479  extern int recurse;
2480  extern int xfer_dirs;
2481 @@ -84,6 +86,7 @@ extern char *filesfrom_host;
2482  extern char *partial_dir;
2483  extern char *dest_option;
2484  extern char *rsync_path;
2485 +extern char *db_config;
2486  extern char *shell_cmd;
2487  extern char *batch_name;
2488  extern char *password_file;
2489 @@ -1079,6 +1082,9 @@ void start_server(int f_in, int f_out, int argc, char *argv[])
2490         if (am_daemon && io_timeout && protocol_version >= 31)
2491                 send_msg_int(MSG_IO_TIMEOUT, io_timeout);
2492
2493 +       if (db_config)
2494 +               db_read_config(FERROR, db_config);
2495 +
2496         if (am_sender) {
2497                 keep_dirlinks = 0; /* Must be disabled on the sender. */
2498                 if (need_messages_from_generator)
2499 @@ -1360,6 +1366,9 @@ static int start_client(int argc, char *argv[])
2500                 }
2501         }
2502
2503 +       if (db_config)
2504 +               db_read_config(FERROR, db_config);
2505 +
2506         if (daemon_over_rsh < 0)
2507                 return start_socket_client(shell_machine, remote_argc, remote_argv, argc, argv);
2508
2509 diff --git a/mkproto.pl b/mkproto.pl
2510 --- a/mkproto.pl
2511 +++ b/mkproto.pl
2512 @@ -13,6 +13,8 @@ if (open(IN, 'proto.h')) {
2513      STRING => 'char *',
2514  );
2515
2516 +@ARGV = grep !m{/rsyncdb\.c$}, @ARGV;
2517 +
2518  $inheader = 0;
2519  $protos = qq|/* This file is automatically generated with "make proto". DO NOT EDIT */\n\n|;
2520
2521 diff --git a/options.c b/options.c
2522 --- a/options.c
2523 +++ b/options.c
2524 @@ -80,6 +80,7 @@ int am_root = 0; /* 0 = normal, 1 = root, 2 = --super, -1 = --fake-super */
2525  int am_server = 0;
2526  int am_sender = 0;
2527  int am_starting_up = 1;
2528 +int am_dbadmin = 0;
2529  int relative_paths = -1;
2530  int implied_dirs = 1;
2531  int missing_args = 0; /* 0 = FERROR_XFER, 1 = ignore, 2 = delete */
2532 @@ -93,6 +94,7 @@ int use_qsort = 0;
2533  char *files_from = NULL;
2534  int filesfrom_fd = -1;
2535  char *filesfrom_host = NULL;
2536 +char *db_config = NULL;
2537  int eol_nulls = 0;
2538  int protect_args = -1;
2539  int human_readable = 1;
2540 @@ -100,6 +102,9 @@ int recurse = 0;
2541  int allow_inc_recurse = 1;
2542  int xfer_dirs = -1;
2543  int am_daemon = 0;
2544 +int db_clean, db_check, db_do_md4, db_do_md5, db_update = 1, db_lax, db_init, db_mounts;
2545 +int db_output_name, db_output_sum, db_output_info, db_output_unchanged, db_output_dirs, db_output_msgs;
2546 +int saw_db_output_opt, saw_db_sum_opt;
2547  int connect_timeout = 0;
2548  int keep_partial = 0;
2549  int safe_symlinks = 0;
2550 @@ -268,6 +273,7 @@ static struct output_struct debug_words[COUNT_DEBUG+1] = {
2551         DEBUG_WORD(CHDIR, W_CLI|W_SRV, "Debug when the current directory changes"),
2552         DEBUG_WORD(CONNECT, W_CLI, "Debug connection events (levels 1-2)"),
2553         DEBUG_WORD(CMD, W_CLI, "Debug commands+options that are issued (levels 1-2)"),
2554 +       DEBUG_WORD(DB, W_SND|W_REC, "Debug DB operations (levels 1-5)"),
2555         DEBUG_WORD(DEL, W_REC, "Debug delete actions (levels 1-3)"),
2556         DEBUG_WORD(DELTASUM, W_SND|W_REC, "Debug delta-transfer checksumming (levels 1-4)"),
2557         DEBUG_WORD(DUP, W_REC, "Debug weeding of duplicate names"),
2558 @@ -572,6 +578,7 @@ static void print_rsync_version(enum logcode f)
2559         char const *links = "no ";
2560         char const *iconv = "no ";
2561         char const *ipv6 = "no ";
2562 +       char const *db = "no ";
2563         STRUCT_STAT *dumstat;
2564
2565  #if SUBPROTOCOL_VERSION != 0
2566 @@ -608,6 +615,11 @@ static void print_rsync_version(enum logcode f)
2567  #ifdef CAN_SET_SYMLINK_TIMES
2568         symtimes = "";
2569  #endif
2570 +#if defined HAVE_MYSQL_MYSQL_H && defined HAVE_LIBMYSQLCLIENT
2571 +       db = "";
2572 +#elif defined HAVE_SQLITE3_H && defined HAVE_LIBSQLITE3
2573 +       db = "";
2574 +#endif
2575
2576         rprintf(f, "%s  version %s  protocol version %d%s\n",
2577                 RSYNC_NAME, RSYNC_VERSION, PROTOCOL_VERSION, subprotocol);
2578 @@ -621,8 +633,8 @@ static void print_rsync_version(enum logcode f)
2579                 (int)(sizeof (int64) * 8));
2580         rprintf(f, "    %ssocketpairs, %shardlinks, %ssymlinks, %sIPv6, batchfiles, %sinplace,\n",
2581                 got_socketpair, hardlinks, links, ipv6, have_inplace);
2582 -       rprintf(f, "    %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sprealloc\n",
2583 -               have_inplace, acls, xattrs, iconv, symtimes, prealloc);
2584 +       rprintf(f, "    %sappend, %sACLs, %sxattrs, %siconv, %ssymtimes, %sprealloc, %sdb\n",
2585 +               have_inplace, acls, xattrs, iconv, symtimes, prealloc, db);
2586
2587  #ifdef MAINTAINER_MODE
2588         rprintf(f, "Panic Action: \"%s\"\n", get_panic_action());
2589 @@ -671,6 +683,9 @@ void usage(enum logcode F)
2590    rprintf(F," -q, --quiet                 suppress non-error messages\n");
2591    rprintf(F,"     --no-motd               suppress daemon-mode MOTD (see manpage caveat)\n");
2592    rprintf(F," -c, --checksum              skip based on checksum, not mod-time & size\n");
2593 +  rprintf(F,"     --db=CONFIG_FILE        specify a CONFIG_FILE for DB checksums\n");
2594 +  rprintf(F,"     --db-only=CONFIG_FILE   behave like rsyncdb\n");
2595 +  rprintf(F,"     --db-lax                ignore ctime changes (use with CAUTION)\n");
2596    rprintf(F," -a, --archive               archive mode; equals -rlptgoD (no -H,-A,-X)\n");
2597    rprintf(F,"     --no-OPTION             turn off an implied OPTION (e.g. --no-D)\n");
2598    rprintf(F," -r, --recursive             recurse into directories\n");
2599 @@ -818,6 +833,7 @@ enum {OPT_VERSION = 1000, OPT_DAEMON, OPT_SENDER, OPT_EXCLUDE, OPT_EXCLUDE_FROM,
2600        OPT_INCLUDE, OPT_INCLUDE_FROM, OPT_MODIFY_WINDOW, OPT_MIN_SIZE, OPT_CHMOD,
2601        OPT_READ_BATCH, OPT_WRITE_BATCH, OPT_ONLY_WRITE_BATCH, OPT_MAX_SIZE,
2602        OPT_NO_D, OPT_APPEND, OPT_NO_ICONV, OPT_INFO, OPT_DEBUG,
2603 +      OPT_NO_DB, OPT_DBONLY,
2604        OPT_USERMAP, OPT_GROUPMAP, OPT_CHOWN, OPT_BWLIMIT,
2605        OPT_SERVER, OPT_REFUSED_BASE = 9000};
2606
2607 @@ -957,6 +973,10 @@ static struct poptOption long_options[] = {
2608    {"checksum",        'c', POPT_ARG_VAL,    &always_checksum, 1, 0, 0 },
2609    {"no-checksum",      0,  POPT_ARG_VAL,    &always_checksum, 0, 0, 0 },
2610    {"no-c",             0,  POPT_ARG_VAL,    &always_checksum, 0, 0, 0 },
2611 +  {"db",               0,  POPT_ARG_STRING, &db_config, 0, 0, 0 },
2612 +  {"no-db",            0,  POPT_ARG_NONE,   0, OPT_NO_DB, 0, 0 },
2613 +  {"db-lax",           0,  POPT_ARG_VAL,    &db_lax, 1, 0, 0 },
2614 +  {"no-db-lax",        0,  POPT_ARG_VAL,    &db_lax, 0, 0, 0 },
2615    {"block-size",      'B', POPT_ARG_LONG,   &block_size, 0, 0, 0 },
2616    {"compare-dest",     0,  POPT_ARG_STRING, 0, OPT_COMPARE_DEST, 0, 0 },
2617    {"copy-dest",        0,  POPT_ARG_STRING, 0, OPT_COPY_DEST, 0, 0 },
2618 @@ -1045,6 +1065,9 @@ static struct poptOption long_options[] = {
2619    {"dparam",           0,  POPT_ARG_STRING, 0, OPT_DAEMON, 0, 0 },
2620    {"detach",           0,  POPT_ARG_NONE,   0, OPT_DAEMON, 0, 0 },
2621    {"no-detach",        0,  POPT_ARG_NONE,   0, OPT_DAEMON, 0, 0 },
2622 +  /* All the following options switch us into DB-admin option-parsing. */
2623 +  {"db-help",          0,  POPT_ARG_NONE,   0, OPT_DBONLY, 0, 0 },
2624 +  {"db-only",          0,  POPT_ARG_STRING, 0, OPT_DBONLY, 0, 0 },
2625    {0,0,0,0, 0, 0, 0}
2626  };
2627
2628 @@ -1098,6 +1121,50 @@ static struct poptOption long_daemon_options[] = {
2629    {0,0,0,0, 0, 0, 0}
2630  };
2631
2632 +static void dbonly_usage(enum logcode F)
2633 +{
2634 +  rprintf(F,"Usage: rsyncdb --db=CONFIG_FILE [OPTIONS] [DIRS]\n");
2635 +  rprintf(F,"\n");
2636 +  rprintf(F,"Options:\n");
2637 +  rprintf(F,"    --db=CONFIG   Specify the CONFIG file to read for the DB info.\n");
2638 +  rprintf(F,"    --db-lax      Ignore ctime changes (use with CAUTION).\n");
2639 +  rprintf(F,"-r, --recursive   Scan files in subdirs (the default w/o --no-recursive).\n");
2640 +  rprintf(F,"-s, --sums=SUMS   List which checksums to update (default: 4,5).\n");
2641 +  rprintf(F,"-o, --output=STR  One or more letters of what to output (default is nothing).\n");
2642 +  rprintf(F,"-c, --check       Check the checksums (by reading the files) and fix issues.\n");
2643 +  rprintf(F,"    --clean       Note all inodes in the DIRS and remove DB extras.\n");
2644 +  rprintf(F,"-N, --no-update   Avoids updating/adding info with --check and/or --clean.\n");
2645 +  rprintf(F,"    --init        Initialize a DB by (re-)creating its tables.\n");
2646 +  rprintf(F,"    --mounts      Scan for mounted filesystems and update the DB.\n");
2647 +  rprintf(F,"-q, --quiet       Disable the default non-error output.\n");
2648 +  rprintf(F,"-h, --help        Display this help message.\n");
2649 +}
2650 +
2651 +static struct poptOption long_dbonly_options[] = {
2652 +  /* longName, shortName, argInfo, argPtr, value, descrip, argDesc */
2653 +  {"check",           'c', POPT_ARG_NONE,   &db_check, 0, 0, 0},
2654 +  {"clean",            0,  POPT_ARG_NONE,   &db_clean, 0, 0, 0},
2655 +  {"db",               0,  POPT_ARG_STRING, &db_config, 0, 0, 0 },
2656 +  {"db-only",          0,  POPT_ARG_STRING, &db_config, 0, 0, 0 },
2657 +  {"db-lax",           0,  POPT_ARG_VAL,    &db_lax, 1, 0, 0 },
2658 +  {"no-db-lax",        0,  POPT_ARG_VAL,    &db_lax, 0, 0, 0 },
2659 +  {"info",             0,  POPT_ARG_STRING, 0, OPT_INFO, 0, 0 },
2660 +  {"debug",            0,  POPT_ARG_STRING, 0, OPT_DEBUG, 0, 0 },
2661 +  {"update",          'u', POPT_ARG_VAL,    &db_update, 1, 0, 0 },
2662 +  {"no-update",       'N', POPT_ARG_VAL,    &db_update, 0, 0, 0 },
2663 +  {"no-u",             0,  POPT_ARG_VAL,    &db_update, 0, 0, 0 },
2664 +  {"output",          'o', POPT_ARG_STRING, 0, 'o', 0, 0 },
2665 +  {"recursive",       'r', POPT_ARG_VAL,    &recurse, 1, 0, 0 },
2666 +  {"no-recursive",     0,  POPT_ARG_VAL,    &recurse, 0, 0, 0 },
2667 +  {"no-r",             0,  POPT_ARG_VAL,    &recurse, 0, 0, 0 },
2668 +  {"sums",            's', POPT_ARG_STRING, 0, 's', 0, 0 },
2669 +  {"init",             0,  POPT_ARG_NONE,   &db_init, 0, 0, 0 },
2670 +  {"mounts",           0,  POPT_ARG_NONE,   &db_mounts, 0, 0, 0 },
2671 +  {"quiet",           'q', POPT_ARG_NONE,   &quiet, 0, 0, 0 },
2672 +  {"help",            'h', POPT_ARG_NONE,   0, 'h', 0, 0 },
2673 +  {"db-help",          0,  POPT_ARG_NONE,   0, 'h', 0, 0 },
2674 +  {0,0,0,0, 0, 0, 0}
2675 +};
2676
2677  static char err_buf[200];
2678
2679 @@ -1276,6 +1343,100 @@ static void create_refuse_error(int which)
2680         }
2681  }
2682
2683 +static NORETURN void parse_dbonly_args(int argc, const char **argv)
2684 +{
2685 +       poptContext pc = poptGetContext(RSYNC_NAME, argc, argv, long_dbonly_options, 0);
2686 +       const char *arg;
2687 +       int opt;
2688 +
2689 +       recurse = 1;
2690 +       am_dbadmin = 1;
2691 +
2692 +       while ((opt = poptGetNextOpt(pc)) != -1) {
2693 +               const char *cp;
2694 +               switch (opt) {
2695 +               case 'o':
2696 +                       for (cp = poptGetOptArg(pc); *cp; cp++) {
2697 +                               switch (toLower(cp)) {
2698 +                               case 'n':
2699 +                                       db_output_name = 1;
2700 +                                       break;
2701 +                               case 's':
2702 +                               case 'c':
2703 +                                       db_output_sum = db_output_name = 1;
2704 +                                       break;
2705 +                               case 'i':
2706 +                                       db_output_info = db_output_name = 1;
2707 +                                       break;
2708 +                               case 'u':
2709 +                                       db_output_unchanged = db_output_name = 1;
2710 +                                       break;
2711 +                               case 'd':
2712 +                                       db_output_dirs = 1;
2713 +                                       break;
2714 +                               }
2715 +                       }
2716 +                       saw_db_output_opt = 1;
2717 +                       break;
2718 +
2719 +               case 's':
2720 +                       for (cp = poptGetOptArg(pc); *cp; cp++) {
2721 +                               switch (*cp) {
2722 +                               case '4':
2723 +                                       db_do_md4 = 1;
2724 +                                       break;
2725 +                               case '5':
2726 +                                       db_do_md5 = 1;
2727 +                                       break;
2728 +                               }
2729 +                       }
2730 +                       saw_db_sum_opt = 1;
2731 +                       break;
2732 +
2733 +               case 'h':
2734 +                       dbonly_usage(FINFO);
2735 +                       exit_cleanup(0);
2736 +
2737 +               case OPT_INFO:
2738 +                       arg = poptGetOptArg(pc);
2739 +                       parse_output_words(info_words, info_levels, arg, USER_PRIORITY);
2740 +                       break;
2741 +
2742 +               case OPT_DEBUG:
2743 +                       arg = poptGetOptArg(pc);
2744 +                       parse_output_words(debug_words, debug_levels, arg, USER_PRIORITY);
2745 +                       break;
2746 +
2747 +               default:
2748 +                       rprintf(FERROR,
2749 +                               "rsyncdb: %s: %s\n",
2750 +                               poptBadOption(pc, POPT_BADOPTION_NOALIAS),
2751 +                               poptStrerror(opt));
2752 +                       goto dbonly_usage;
2753 +               }
2754 +       }
2755 +
2756 +       if (!db_config) {
2757 +               rprintf(FERROR, "You must specify the --db=FILE option.\n");
2758 +         dbonly_usage:
2759 +               rprintf(FERROR,
2760 +                       "(Type \"rsyncdb --help\" for assistance.)\n");
2761 +               exit_cleanup(RERR_SYNTAX);
2762 +       }
2763 +
2764 +       if (db_check)
2765 +               db_output_info = 1;
2766 +       if (!saw_db_output_opt && !quiet)
2767 +               db_output_dirs = db_output_name = 1;
2768 +       if (!quiet)
2769 +               db_output_msgs = 1;
2770 +       if (!saw_db_sum_opt)
2771 +               db_do_md5 = 1;
2772 +
2773 +       am_starting_up = 0;
2774 +       run_dbonly(poptGetArgs(pc));
2775 +       exit(42); /* NOT REACHED */
2776 +}
2777
2778  /**
2779   * Process command line arguments.  Called on both local and remote.
2780 @@ -1293,10 +1454,18 @@ int parse_arguments(int *argc_p, const char ***argv_p)
2781         int argc = *argc_p;
2782         int opt;
2783
2784 +       arg = *argv + strlen(*argv);
2785 +       if (arg - *argv > 2 && strcmp(arg-2, "db") == 0) {
2786 +               parse_dbonly_args(argc, argv);
2787 +               /* NOT REACHED */
2788 +       }
2789 +
2790         if (ref && *ref)
2791                 set_refuse_options(ref);
2792         if (am_daemon) {
2793                 set_refuse_options("log-file*");
2794 +               set_refuse_options("db");
2795 +               set_refuse_options("db-lax");
2796  #ifdef ICONV_OPTION
2797                 if (!*lp_charset(module_id))
2798                         set_refuse_options("iconv");
2799 @@ -1419,6 +1588,12 @@ int parse_arguments(int *argc_p, const char ***argv_p)
2800                         am_daemon = 1;
2801                         return 1;
2802
2803 +               case OPT_DBONLY:
2804 +                       protect_args = 0;
2805 +                       poptFreeContext(pc);
2806 +                       parse_dbonly_args(argc, argv);
2807 +                       break; /* NOT REACHED */
2808 +
2809                 case OPT_MODIFY_WINDOW:
2810                         /* The value has already been set by popt, but
2811                          * we need to remember that we're using a
2812 @@ -1493,6 +1668,10 @@ int parse_arguments(int *argc_p, const char ***argv_p)
2813                         preserve_devices = preserve_specials = 0;
2814                         break;
2815
2816 +               case OPT_NO_DB:
2817 +                       db_config = NULL;
2818 +                       break;
2819 +
2820                 case 'h':
2821                         human_readable++;
2822                         break;
2823 diff --git a/pipe.c b/pipe.c
2824 --- a/pipe.c
2825 +++ b/pipe.c
2826 @@ -27,11 +27,16 @@ extern int am_server;
2827  extern int blocking_io;
2828  extern int filesfrom_fd;
2829  extern int munge_symlinks;
2830 +extern int always_checksum;
2831 +extern int use_db;
2832 +extern char *db_config;
2833  extern char *logfile_name;
2834  extern int remote_option_cnt;
2835  extern const char **remote_options;
2836  extern struct chmod_mode_struct *chmod_modes;
2837
2838 +int same_db = 0;
2839 +
2840  /**
2841   * Create a child connected to us via its stdin/stdout.
2842   *
2843 @@ -142,13 +147,22 @@ pid_t local_child(int argc, char **argv, int *f_in, int *f_out,
2844                 }
2845
2846                 if (remote_option_cnt) {
2847 +                       const char *db_config_save = db_config;
2848                         int rc = remote_option_cnt + 1;
2849                         const char **rv = remote_options;
2850                         if (!parse_arguments(&rc, &rv)) {
2851                                 option_error();
2852                                 exit_cleanup(RERR_SYNTAX);
2853                         }
2854 -               }
2855 +                       if (db_config == db_config_save)
2856 +                               same_db = db_config != NULL;
2857 +                       else if (!db_config || !db_config_save || strcmp(db_config, db_config_save) != 0) {
2858 +                               use_db = 0;
2859 +                               if (db_config)
2860 +                                       db_read_config(FERROR, db_config);
2861 +                       }
2862 +               } else if (use_db)
2863 +                       same_db = 1;
2864
2865                 if (dup2(to_child_pipe[0], STDIN_FILENO) < 0 ||
2866                     close(to_child_pipe[1]) < 0 ||
2867 diff --git a/receiver.c b/receiver.c
2868 --- a/receiver.c
2869 +++ b/receiver.c
2870 @@ -24,6 +24,8 @@
2871
2872  extern int dry_run;
2873  extern int do_xfers;
2874 +extern int use_db;
2875 +extern int db_lax;
2876  extern int am_root;
2877  extern int am_server;
2878  extern int inc_recurse;
2879 @@ -431,6 +433,11 @@ static void handle_delayed_updates(char *local_name)
2880                                         "rename failed for %s (from %s)",
2881                                         full_fname(fname), partialptr);
2882                         } else {
2883 +                               if (use_db && !db_lax) {
2884 +                                       STRUCT_STAT st;
2885 +                                       if (do_lstat(fname, &st) == 0)
2886 +                                               db_update_ctime(5, &st);
2887 +                               }
2888                                 if (remove_source_files
2889                                  || (preserve_hard_links && F_IS_HLINKED(file)))
2890                                         send_msg_int(MSG_SUCCESS, ndx);
2891 @@ -537,6 +544,9 @@ int recv_files(int f_in, int f_out, char *local_name)
2892         if (delay_updates)
2893                 delayed_bits = bitbag_create(cur_flist->used + 1);
2894
2895 +       if (use_db && (append_mode == 1 || protocol_version < 30))
2896 +               use_db = 0; /* We can't note finished md5 values */
2897 +
2898         while (1) {
2899                 cleanup_disable();
2900
2901 @@ -864,6 +874,8 @@ int recv_files(int f_in, int f_out, char *local_name)
2902                                 do_unlink(partialptr);
2903                                 handle_partial_dir(partialptr, PDIR_DELETE);
2904                         }
2905 +                       if (use_db && do_lstat(fname, &st) == 0)
2906 +                               db_set_checksum(5, &st, sender_file_sum);
2907                 } else if (keep_partial && partialptr) {
2908                         if (!handle_partial_dir(partialptr, PDIR_CREATE)) {
2909                                 rprintf(FERROR,
2910 @@ -877,6 +889,8 @@ int recv_files(int f_in, int f_out, char *local_name)
2911                                 recv_ok = -1;
2912                         else if (delay_updates && recv_ok) {
2913                                 bitbag_set_bit(delayed_bits, ndx);
2914 +                               if (use_db && do_lstat(partialptr, &st) == 0)
2915 +                                       db_set_checksum(5, &st, sender_file_sum);
2916                                 recv_ok = 2;
2917                         } else
2918                                 partialptr = NULL;
2919 diff --git a/rsync.c b/rsync.c
2920 --- a/rsync.c
2921 +++ b/rsync.c
2922 @@ -39,6 +39,7 @@ extern int am_daemon;
2923  extern int am_sender;
2924  extern int am_receiver;
2925  extern int am_generator;
2926 +extern int am_dbadmin;
2927  extern int am_starting_up;
2928  extern int allow_8bit_chars;
2929  extern int protocol_version;
2930 @@ -738,6 +739,8 @@ struct file_list *flist_for_ndx(int ndx, const char *fatal_error_loc)
2931
2932  const char *who_am_i(void)
2933  {
2934 +       if (am_dbadmin)
2935 +               return "rsyncdb";
2936         if (am_starting_up)
2937                 return am_server ? "server" : "client";
2938         return am_sender ? "sender"
2939 diff --git a/rsync.h b/rsync.h
2940 --- a/rsync.h
2941 +++ b/rsync.h
2942 @@ -240,12 +240,16 @@ enum msgcode {
2943         MSG_IO_ERROR=22,/* the sending side had an I/O error */
2944         MSG_IO_TIMEOUT=33,/* tell client about a daemon's timeout value */
2945         MSG_NOOP=42,    /* a do-nothing message (legacy protocol-30 only) */
2946 +       MSG_CHECKSUM=55,/* sent via rcvr -> gen pipe and local-host-only gen -> sender */
2947         MSG_ERROR_EXIT=86, /* synchronize an error exit (siblings and protocol >= 31) */
2948         MSG_SUCCESS=100,/* successfully updated indicated flist index */
2949         MSG_DELETED=101,/* successfully deleted a file on receiving side */
2950         MSG_NO_SEND=102,/* sender failed to open a file we wanted */
2951  };
2952
2953 +#define MSG_CHECKSUM_LONGS 5
2954 +#define MSG_CHECKSUM_LEN (MSG_CHECKSUM_LONGS*8 + 4 + MAX_DIGEST_LEN)
2955 +
2956  #define NDX_DONE -1
2957  #define NDX_FLIST_EOF -2
2958  #define NDX_DEL_STATS -3
2959 @@ -1256,7 +1260,8 @@ extern short info_levels[], debug_levels[];
2960  #define DEBUG_CHDIR (DEBUG_BIND+1)
2961  #define DEBUG_CONNECT (DEBUG_CHDIR+1)
2962  #define DEBUG_CMD (DEBUG_CONNECT+1)
2963 -#define DEBUG_DEL (DEBUG_CMD+1)
2964 +#define DEBUG_DB (DEBUG_CMD+1)
2965 +#define DEBUG_DEL (DEBUG_DB+1)
2966  #define DEBUG_DELTASUM (DEBUG_DEL+1)
2967  #define DEBUG_DUP (DEBUG_DELTASUM+1)
2968  #define DEBUG_EXIT (DEBUG_DUP+1)
2969 diff --git a/rsync.yo b/rsync.yo
2970 --- a/rsync.yo
2971 +++ b/rsync.yo
2972 @@ -337,6 +337,9 @@ to the detailed description below for a complete description.  verb(
2973   -q, --quiet                 suppress non-error messages
2974       --no-motd               suppress daemon-mode MOTD (see caveat)
2975   -c, --checksum              skip based on checksum, not mod-time & size
2976 +     --db=CONFIG_FILE        specify a CONFIG_FILE for DB checksums
2977 +     --db-only=CONFIG_FILE   Behave like rsyncdb (see that manpage).
2978 +     --db-lax                Ignore ctime changes (use with CAUTION).
2979   -a, --archive               archive mode; equals -rlptgoD (no -H,-A,-X)
2980       --no-OPTION             turn off an implied OPTION (e.g. --no-D)
2981   -r, --recursive             recurse into directories
2982 @@ -625,6 +628,67 @@ option's before-the-transfer "Does this file need to be updated?" check.
2983  For protocol 30 and beyond (first supported in 3.0.0), the checksum used is
2984  MD5.  For older protocols, the checksum used is MD4.
2985
2986 +dit(bf(--db=CONFIG_FILE))  This option specifies a CONFIG_FILE to read
2987 +that holds connection details for a database of checksum information.
2988 +When combined with the bf(--checksum) (bf(-c)) option, rsync will try to
2989 +use cached checksum information from the DB, and will update it if it is
2990 +missing.
2991 +
2992 +The currently supported DB choices are MySQL and SQLite.  For example, a
2993 +MySQL configuration might look like this:
2994 +
2995 +verb(    dbtype: mysql
2996 +    dbhost: 127.0.0.1
2997 +    dbname: rsyncdb
2998 +    dbuser: rsyncuser
2999 +    dbpass: somepass
3000 +    port: 3306
3001 +    thishost: hostname )
3002 +
3003 +And a SQLite configuration might look like this:
3004 +
3005 +verb(    dbtype: SQLite
3006 +    dbname: /var/cache/rsync/sum.db
3007 +    transaction: 1)
3008 +
3009 +Both the bf(--db) and bf(--db-lax) options only affect the side where the
3010 +option is used.  To affect the remote side of a remote-shell connection,
3011 +use the bf(--remote-option) (bf(-M)) option.  For example, to specify the
3012 +same options on both sides, you could specify something like this:
3013 +
3014 +verb(    rsync -avc {-M,}--db=/etc/rsyncdb.conf src/ host:dest/ )
3015 +
3016 +For a local copy, this option affects both the source and the destination.
3017 +If you wish a local copy to enable this option just for the destination
3018 +files, specify bf(-M--db=CONFIG) (the same for bf(-M--db-lax).  If you wish
3019 +a local copy to enable this option just for the source files, combine
3020 +bf(--db=CONFIG) with bf(-M--no-db) (similarly use bf(-M--no-db-lax)).
3021 +
3022 +See the perl script "rsyncdb" in the support directory of the source code
3023 +(which may also be installed in /usr/bin) for a way to create the tables,
3024 +populate the mounted-disk information, check files against their checksums,
3025 +and update both the MD4 and MD5 checksums for files at the same time (since
3026 +an rsync copy will only update one or the other).
3027 +
3028 +You can use a single MySQL DB for all your hosts if you give each one
3029 +their own "thishost" name and setup their device-mapping data.  Or feel
3030 +free to use separate databases, separate servers, etc.  See the rsync
3031 +daemon's "db config" parameter for how to configure a daemon to use a DB
3032 +(since a client cannot control this parameter on a daemon).
3033 +
3034 +dit(bf(--db-lax)) This option can be used to modify the inode-matching
3035 +algorithm used by bf(--db) to one that ignores the ctime.  This can be very
3036 +DANGEROUS unless your files are known to ALWAYS be updated in a safe manner.
3037 +If unsure, don't use it.
3038 +
3039 +The reason you might want to use it is that the ctime (inode change time) is
3040 +changed by an added hard-link, or the file being moving around.  To use this
3041 +option safely you must be CERTAIN that either rsync w/--db is the only program
3042 +adding files into the cached hierarchies, OR that all new files will have new
3043 +modify times (never a historical mtime that might match an orphaned inode).
3044 +So, for certain applications, such as mirrors of new tar releases, this option
3045 +can save a lot of unneeded checksum re-computation due to ctime changes.
3046 +
3047  dit(bf(-a, --archive)) This is equivalent to bf(-rlptgoD). It is a quick
3048  way of saying you want recursion and want to preserve almost
3049  everything (with -H being a notable omission).
3050 diff --git a/rsyncd.conf.yo b/rsyncd.conf.yo
3051 --- a/rsyncd.conf.yo
3052 +++ b/rsyncd.conf.yo
3053 @@ -313,6 +313,22 @@ is daemon.  This setting has no effect if the "log file" setting is a
3054  non-empty string (either set in the per-modules settings, or inherited
3055  from the global settings).
3056
3057 +dit(bf(db config)) This parameter specifies a config file to read that
3058 +holds connection details for a database of checksum information.
3059 +
3060 +The config file will be read-in prior to any chroot restrictions, but
3061 +the connection occurs from inside the chroot.  This means that you
3062 +should use a socket connection (e.g. 127.0.0.1 rather than localhost)
3063 +for a MySQL config from inside a chroot.  For SQLite, the DB file must
3064 +be placed inside the chroot (though it can be placed outside the
3065 +transfer dir if you configured an inside-chroot path).
3066 +
3067 +See the bf(--db=CONFIG_FILE) option for full details.
3068 +
3069 +dit(bf(db lax)) This parameter specifies that a "db config" setup should use
3070 +lax (no ctime) lookups.  See the rsync manpage's section -n bf(--db-lax) for
3071 +some warnings about using this setting.
3072 +
3073  dit(bf(max verbosity)) This parameter allows you to control
3074  the maximum amount of verbose information that you'll allow the daemon to
3075  generate (since the information goes into the log file). The default is 1,
3076 diff --git a/rsyncdb-mountinfo b/rsyncdb-mountinfo
3077 new file mode 100755
3078 --- /dev/null
3079 +++ b/rsyncdb-mountinfo
3080 @@ -0,0 +1,60 @@
3081 +#!/usr/bin/perl
3082 +
3083 +# This script outputs data for rsyncdb --mounts.  It must output a complete
3084 +# list of the mounts for the current host in a strict format -- 2 fields
3085 +# with a Tab between:  $MOUNT_UNIQ\t$PATH
3086 +#
3087 +# The list of mounts MUST NOT contain any entry that has the same devnum
3088 +# (st_dev) as any other entry in the list (as checked via its PATH).
3089 +#
3090 +# MOUNT_UNIQ is a unique string that identifies the mount on this host.
3091 +# This cannot be the devnum (st_dev) because that can vary depending on the
3092 +# mount order or be reused for different mounts if they are not mounted at
3093 +# the same time.  By default the value is "Mount of $devname", which should
3094 +# be adequate for situations that don't want removable media in the DB
3095 +# (though you may need to take steps to weed-out removable media from the
3096 +# list to ensure that such inodes stay out of the DB).
3097 +#
3098 +# You can override the MOUNT_UNIQ value by putting a .rsyndb_mount_uniq
3099 +# file in the root directory of any mount, at which point it is up to you
3100 +# to make sure that the value stays unique (note that all sequences of
3101 +# whitespace are transformed into a single space, and leading/trailing
3102 +# whitespace is removed).
3103 +#
3104 +# MOUNT_UNIQ may never contain a Tab but it would be legal for PATH to have
3105 +# a Tab (just really weird).  Neither may have a CR or LF in it.
3106 +#
3107 +# The maximum size for MOUNT_UNIQ is 256 characters.
3108 +#
3109 +# If this script doesn't meet your needs, feel free to edit it and choose
3110 +# some other method of finding a unique value for each mount.  If you come
3111 +# up with a good idiom that might be useful to others, please share it back
3112 +# to me.
3113 +
3114 +use strict;
3115 +use warnings;
3116 +
3117 +my $MOUNT_FILE = '/etc/mtab';
3118 +my $VALID_DEVICE_REGEX = qr{^/dev};
3119 +
3120 +my %hash;
3121 +
3122 +open MOUNTS, $MOUNT_FILE or die "Unable to open $MOUNT_FILE: $!\n";
3123 +while (<MOUNTS>) {
3124 +    my ($devname, $path) = (split)[0,1];
3125 +    next unless $devname =~ /$VALID_DEVICE_REGEX/;
3126 +
3127 +    my ($devno) = (stat($path))[0];
3128 +    next unless defined $devno; # Skip if mount is invalid.
3129 +    next if $hash{$devno}++; # SKip if we've seen this devno earlier.
3130 +
3131 +    my $mount_uniq = "Mount of $devname";
3132 +    if (open UNIQ, '<', "$path/.rsyndb_mount_uniq") {
3133 +       $mount_uniq = <UNIQ>;
3134 +       close UNIQ;
3135 +       $mount_uniq =~ s/\s+/ /g; # This ensures no tab, CR, nor LF.
3136 +       $mount_uniq =~ s/^ | $//g; # .. and no leading or trailing whitespace.
3137 +    }
3138 +    print $mount_uniq, "\t", $path, "\n";
3139 +}
3140 +close MOUNTS;
3141 diff --git a/rsyncdb.yo b/rsyncdb.yo
3142 new file mode 100644
3143 --- /dev/null
3144 +++ b/rsyncdb.yo
3145 @@ -0,0 +1,186 @@
3146 +mailto(rsync-bugs@samba.org)
3147 +manpage(rsync)(1)(23 Jun 2013)()()
3148 +manpagename(rsyncdb)(Maintain an rsync checksum DB)
3149 +manpagesynopsis()
3150 +
3151 +verb(rsyncdb --db=CONFIG [OPTION...] [DIR...])
3152 +
3153 +manpagedescription()
3154 +
3155 +Rsyncdb can maintain a checksum-caching DB that rsync can use to make its
3156 +bf(--checksum) option more optimal.  You must specify a config file via
3157 +the bf(--db=CONFIG_FILE) option in order for rsyncdb to know what DB to
3158 +manipulate.  See the rsync manpage's bf(--db) option for full details on
3159 +the file's format.
3160 +
3161 +You can specify one or more directory args for rsyncdb to scan.  If no
3162 +DIR args are specified, the current directory is assumed to be the spot
3163 +to start scanning.
3164 +
3165 +Note that the rsyncdb program is usually just a symlink to the rsync program.
3166 +You can force rsync to behave as rsyncdb either by having a symlink (or
3167 +hardlink) name that ends with "db" or by bf(starting) the rsync args with
3168 +bf(--db-only=CONFIG) (and that option works just like bf(--db=CONFIG) to
3169 +a program named rsyncdb).
3170 +
3171 +manpagesection(EXAMPLES)
3172 +
3173 +The following command will update checksum information in the database
3174 +described in the /etc/db.conf file:
3175 +
3176 +verb(    rsyncdb --db=/etc/db.conf -o n --clean /dir1 /dir2)
3177 +
3178 +It scans 2 directory hierarchies (/dir1 & /dir2) and cleans out any
3179 +checksums whose inodes are no longer found in those directories (so that
3180 +directory args are presumed to be complete for this host's DB contents).
3181 +
3182 +The following command will scan all the files in the /dir2 directory (without
3183 +recursive scanning, due to the bf(--no-r) option) and check them against
3184 +the DB:
3185 +
3186 +verb(    rsyncdb --db=/etc/db.conf --check --no-r /dir2)
3187 +
3188 +Any errors found are output as well as being fixed in the DB.  (See
3189 +bf(--no-update) for how to check without updating.)
3190 +
3191 +The following command will output MD5 sums for all the files found in the
3192 +directories mentioned, even if they are unchanged (due to the
3193 +bf(--output=u) option):
3194 +
3195 +verb(    rsyncdb --db=/etc/db.conf -rous /dir* >/tmp/md5sums.txt)
3196 +
3197 +This is just like running md5sum, only faster.  Unlike md5sum, you can't
3198 +specify a single file, so use bf(--no-r) and grep the output if you just
3199 +want to see a single file's value.
3200 +
3201 +The following command initializes a new DB, and is required for any new DB:
3202 +
3203 +verb(    rsyncdb --db=/etc/db.conf --init --mounts)
3204 +
3205 +The bf(--init) option should only be used once (unless you want to
3206 +destroy existing data).  The bf(--mounts) option may need to be used
3207 +periodically.
3208 +
3209 +manpagesection(OPTIONS SUMMARY)
3210 +
3211 +Rsyncdb accepts the following options: verb(
3212 +     --db=CONFIG     Specify the CONFIG file to read for the DB info.
3213 +     --db-lax        Ignore ctime changes (use with CAUTION).
3214 +     --no-recursive  Avoid the default --recursive (-r) scanning behavior.
3215 + -s, --sums=SUMS     List which checksums to update (default: md5).
3216 + -o, --output=STR    One or more letters of what to output (default is nothing).
3217 + -c, --check         Check the checksums (by reading the files) and fix any
3218 +                     issues.  Enables --output=i.
3219 +     --clean         Note all inodes in the DIRS and remove DB extras.
3220 + -N, --no-update     Avoids updating/adding info with --check and/or --clean.
3221 +     --init          Initialize a DB by (re-)creating its tables.
3222 +     --mounts        Scan for mounted filesystems and update the DB.
3223 + -q, --quiet         Disables the default non-error output.
3224 + -h, --help          Display this help message.)
3225 +
3226 +manpageoptions()
3227 +
3228 +Rsyncdb accepts both long (double-dash + word) and short (single-dash + letter)
3229 +options.  The full list of the available options are described below.  If an
3230 +option can be specified in more than one way, the choices are comma-separated.
3231 +Some options only have a long variant, not a short.  If the option takes a
3232 +parameter, the parameter is only listed after the long variant, even though it
3233 +must also be specified for the short.  When specifying a parameter, you can
3234 +either use the form --option=param or replace the '=' with whitespace.  The
3235 +parameter may need to be quoted in some manner for it to survive the shell's
3236 +command-line parsing.
3237 +
3238 +startdit()
3239 +dit(bf(--db=CONFIG_FILE)) This tells rsyncdb what DB-config file to read
3240 +for the DB setup.  This is the same as the option in rsync, so refer to
3241 +that manpage for full details.
3242 +
3243 +dit(bf(--db-lax)) This option works just like it does in rsync, so refer to
3244 +that manpage for full details.
3245 +
3246 +dit(bf(--no-recursive, --no-r)) This disables the default recursive
3247 +directory scan that is performed on the listed directory args.  The
3248 +options bf(--recursive) and bf(-r) are also accepted, if someone wants
3249 +to override an earlier bf(--no-r) override.
3250 +
3251 +dit(bf(--sums=SUMS, -s)) Only output/update the listed checksum types. By
3252 +default we deal with just the newer md5 checksums (i.e.  bf(--sums=5)).
3253 +
3254 +Note that this option does NOT affect the order that checksums are output
3255 +if "-o s" is enabled, so bf(-s5,4) is the same as bf(-s4,5).
3256 +
3257 +dit(bf(--output=STR, -o)) The output option lets you specify one or more
3258 +letters indicating what information should be output.  The default is to
3259 +output "d" and "n" if bf(--output) is not specified.
3260 +
3261 +The following letters are accepted:
3262 +
3263 +quote(itemization(
3264 +  it() bf(d) outputs "... dir_name ..." lines for each directory in our scan.
3265 +  it() bf(n) outputs the names of files with changes (implied by all but "d").
3266 +  it() bf(s) outputs checksum info for changes (implies bf(n)).
3267 +  it() bf(u) outputs unchanged files too (implies bf(n)).
3268 +  it() bf(i) outputs prefixed change info.  The output strings are:
3269 +  quote(itemization(
3270 +    it() bf(!i) indicates that the time and/or size is wrong.
3271 +    it() bf(+4) indicates the MD4 sum is missing.
3272 +    it() bf(+5) indicates the MD5 sum is missing.
3273 +    it() bf(!4) indicates the MD4 sum is wrong.
3274 +    it() bf(!5) indicates the MD5 sum is wrong.
3275 +    it() bf(?4) indicates an unknown MD4 difference.  This can happen if we
3276 +    didn't need to read the file; i.e. if the time/size is wrong and no sum
3277 +    info was requested.
3278 +    it() bf(?5) indicates an unknown MD5 difference.
3279 +  ))
3280 +))
3281 +
3282 +dit(bf(--check, -c)) Check the checksums (forcing the reading of all the
3283 +files) and fix any issues that are found.  Forces bf(--output=ni) on.
3284 +
3285 +dit(bf(--clean)) Makes a temp-DB of all the inodes that we find in all the
3286 +listed directories and removes any extraneous checksums from the DB.  You
3287 +will need to specify all the mounted directories that are present (and
3288 +listed as mounted) in the DB on this host or else the checksums from the
3289 +unvisited directories will be discarded from the DB.  If you want to just
3290 +--clean without adding or updating the info of new or changed files,
3291 +specify bf(--no-update) as well.
3292 +
3293 +See the bf(--mount)
3294 +
3295 +dit(bf(--no-update, -N)) Avoids updating/adding info with bf(--check)
3296 +and/or bf(--clean).
3297 +
3298 +dit(bf(--quiet, -q)) Disable the default (non-error) output settings.  This
3299 +turns off the messages that bf(--init), bf(--mount), and bf(--clean) output,
3300 +and makes the default for bf(--output) be nothing (though an explicit
3301 +bf(--output) option is not affected).
3302 +
3303 +dit(bf(--init)) Create the tables in the DB.  If it is used on an existing
3304 +DB, all the existing tables are dropped and re-created.
3305 +
3306 +This option disables scanning for checksum information, but may be combined
3307 +with bf(--mounts).
3308 +
3309 +dit(bf(--mounts)) Populate the "disk" DB with the available device numbers
3310 +and change any mounted/unmount information for devices.  This should be run
3311 +every time a mount-change happens that may affect a directory hierarchy in
3312 +the DB.  Rsyncdb will not save any checksums for a device that is not
3313 +listed in the "disk" table.
3314 +
3315 +Some advanced users may want to maintain the disk table themselves in order
3316 +to support mounting a drive in different (or multiple) locations, etc.
3317 +
3318 +This option disables scanning for checksum information, but may be combined
3319 +with bf(--init).
3320 +
3321 +dit(bf(--help, -h)) Display a summary of the options.
3322 +
3323 +enddit()
3324 +
3325 +manpageseealso()
3326 +
3327 +bf(rsync)(1)
3328 +
3329 +manpageauthor()
3330 +
3331 +Rsyncdb was written by Wayne Davison.