Committing Uri Simchoni's improved ssl patch.
authorWayne Davison <wayned@samba.org>
Sat, 19 Jun 2010 18:37:37 +0000 (11:37 -0700)
committerWayne Davison <wayned@samba.org>
Sat, 19 Jun 2010 18:37:37 +0000 (11:37 -0700)
(With some cleanup by Wayne.)

openssl-support.diff

index 7bffb4b..dd8ff4d 100644 (file)
@@ -35,7 +35,7 @@ To use this patch, run these commands for a successful build:
     ./configure
     make
 
-based-on: 25082d1ef6712a15c52a1dacb36b7f0642c23ac8
+based-on: ce571e64b6c4ab2947c05a21ebc254729933ea0c
 diff --git a/Makefile.in b/Makefile.in
 --- a/Makefile.in
 +++ b/Makefile.in
@@ -164,17 +164,28 @@ diff --git a/clientserver.c b/clientserver.c
                        return -1;
                }
  
-@@ -1030,6 +1077,9 @@ int start_daemon(int f_in, int f_out)
+@@ -692,6 +739,10 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char
+                               set_env_num("RSYNC_PID", (long)pid);
+                               if (wait_process(pid, &status, 0) < 0)
+                                       status = -1;
++#ifdef HAVE_OPENSSL
++                              if (use_ssl)
++                                      end_tls();
++#endif
+                               set_env_num("RSYNC_RAW_STATUS", status);
+                               if (WIFEXITED(status))
+                                       status = WEXITSTATUS(status);
+@@ -1033,6 +1084,9 @@ int start_daemon(int f_in, int f_out)
        if (exchange_protocols(f_in, f_out, line, sizeof line, 0) < 0)
                return -1;
  
 +#ifdef HAVE_OPENSSL
-+retry:
++  retry:
 +#endif
        line[0] = 0;
        if (!read_line_old(f_in, line, sizeof line))
                return -1;
-@@ -1041,6 +1091,20 @@ int start_daemon(int f_in, int f_out)
+@@ -1044,6 +1098,20 @@ int start_daemon(int f_in, int f_out)
                return -1;
        }
  
@@ -198,7 +209,7 @@ diff --git a/clientserver.c b/clientserver.c
 diff --git a/configure.ac b/configure.ac
 --- a/configure.ac
 +++ b/configure.ac
-@@ -318,6 +318,21 @@ if test x"$enable_locale" != x"no"; then
+@@ -318,6 +318,25 @@ if test x"$enable_locale" != x"no"; then
        AC_DEFINE(CONFIG_LOCALE)
  fi
  
@@ -207,6 +218,8 @@ diff --git a/configure.ac b/configure.ac
 +
 +if test "x$enable_openssl" != xno
 +then
++      save_LIBS=$LIBS
++      LIBS="$LIBS -lcrypto"
 +      have_ssl=yes
 +      AC_CHECK_LIB(ssl, SSL_library_init, , [have_ssl=no])
 +      if test "x$have_ssl" = xyes
@@ -214,12 +227,114 @@ diff --git a/configure.ac b/configure.ac
 +              AC_DEFINE(HAVE_OPENSSL, 1, [true if you want to use SSL.])
 +              SSL_OBJS=ssl.o
 +              AC_SUBST(SSL_OBJS)
++      else
++              LIBS=$save_LIBS
 +      fi
 +fi
 +
  AC_MSG_CHECKING([whether to call shutdown on all sockets])
  case $host_os in
        *cygwin* ) AC_MSG_RESULT(yes)
+diff --git a/main.c b/main.c
+--- a/main.c
++++ b/main.c
+@@ -91,6 +91,9 @@ extern char backup_dir_buf[MAXPATHLEN];
+ extern char *basis_dir[MAX_BASIS_DIRS+1];
+ extern struct file_list *first_flist;
+ extern filter_rule_list daemon_filter_list;
++#ifdef HAVE_OPENSSL
++extern int use_ssl;
++#endif
+ uid_t our_uid;
+ gid_t our_gid;
+@@ -151,6 +154,52 @@ pid_t wait_process(pid_t pid, int *status_ptr, int flags)
+       return waited_pid;
+ }
++/* Sends signal "signo", waits for the process to die, and if it doesn't, sends
++ * a SIGKILL.  If "graceful" is set, the initial "signo" signal is delayed by a
++ * second to try to let the process exit on its own first. */
++pid_t terminate_process(pid_t pid, int *status_ptr, int signo, int graceful)
++{
++      pid_t waited_pid;
++      int timeout = graceful ? 1000 : 3000;
++      if (!graceful)
++              kill(pid, signo);
++      while (1) {
++              waited_pid = wait_process(pid, status_ptr, timeout >= 0 ? WNOHANG : 0);
++              if (waited_pid)
++                      break;
++              if (timeout == 0) {
++                      if (graceful) {
++                              graceful = 0;
++                              timeout = 3000;
++                      } else {
++                              signo = SIGKILL;
++                              timeout = -1;
++                      }
++                      rprintf(FINFO, "%s:%s shutdown didn't work - sending signal %d\n",
++                              __FUNCTION__, graceful ? " graceful" : "", signo);
++                      kill(pid, signo);
++              }
++
++              if (timeout > 0) {
++                      /* interruptible wait and calculate the time left for waiting */
++                      struct timeval tval, t1, t2;
++
++                      gettimeofday(&t1, NULL);
++
++                      tval.tv_sec = timeout/1000;
++                      tval.tv_usec = (timeout%1000)*1000;
++                      select(0, NULL, NULL, NULL, &tval);
++                      gettimeofday(&t2, NULL);
++
++                      timeout -= (t2.tv_sec-t1.tv_sec)*1000 + (t2.tv_usec-t1.tv_usec)/1000;
++                      if (timeout < 0)
++                              timeout = 0;
++              }
++      }
++
++      return waited_pid;
++}
++
+ /* Wait for a process to exit, calling io_flush while waiting. */
+ static void wait_process_with_flush(pid_t pid, int *exit_code_ptr)
+ {
+@@ -802,6 +851,11 @@ static void do_server_sender(int f_in, int f_out, int argc, char *argv[])
+               argv[0] = ".";
+       }
++#ifdef HAVE_OPENSSL
++      if (use_ssl)
++              start_tls_buffering();
++#endif
++
+       flist = send_file_list(f_out,argc,argv);
+       if (!flist || flist->used == 0)
+               exit_cleanup(0);
+@@ -922,6 +976,10 @@ static int do_recv(int f_in, int f_out, char *local_name)
+       io_start_buffering_out(f_out);
+       io_start_multiplex_in(f_in);
++#ifdef HAVE_OPENSSL
++      if (use_ssl)
++              start_tls_buffering();
++#endif
+ #ifdef SUPPORT_HARD_LINKS
+       if (preserve_hard_links && inc_recurse) {
+@@ -1126,6 +1184,10 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[])
+                       io_start_multiplex_in(f_in);
+               else
+                       io_start_buffering_in(f_in);
++#ifdef HAVE_OPENSSL
++              if (use_ssl)
++                      start_tls_buffering();
++#endif
+               send_filter_list(f_out);
+               if (filesfrom_host)
+                       filesfrom_fd = f_in;
 diff --git a/options.c b/options.c
 --- a/options.c
 +++ b/options.c
@@ -443,7 +558,7 @@ diff --git a/ssl.c b/ssl.c
 new file mode 100644
 --- /dev/null
 +++ b/ssl.c
-@@ -0,0 +1,369 @@
+@@ -0,0 +1,603 @@
 +/* -*- c-file-style: "linux" -*-
 + * ssl.c: operations for negotiating SSL rsync connections.
 + *
@@ -475,7 +590,8 @@ new file mode 100644
 +#endif
 +#include <string.h>
 +
-+#define BUF_SIZE 1024
++#define SSL_MAX_RECORD_SIZE (1024*16) /* TLS record size */
++#define MAX_BUFFERING_TIME_MS 100
 +
 +extern int am_daemon;
 +extern int am_server;
@@ -490,12 +606,20 @@ new file mode 100644
 +static int tls_read[2] = { -1, -1 };
 +static int tls_write[2] = { -1, -1 };
 +static int ssl_running;
++static int ssl_min_send_size;
 +static int ssl_pid = -1;
 +
 +#ifdef HAVE_SIGACTION
 +static struct sigaction sigact;
 +#endif
 +
++/* copied from progress.c */
++static unsigned long msdiff(struct timeval *t1, struct timeval *t2)
++{
++      return (t2->tv_sec - t1->tv_sec) * 1000L
++              + (t2->tv_usec - t1->tv_usec) / 1000;
++}
++
 +/**
 + * A non-interactive callback to be passed to SSL_CTX_set_default_password_cb,
 + * which merely copies the value of ssl_key_passwd into buf. This is
@@ -587,6 +711,8 @@ new file mode 100644
 +              return 1;
 +      SSL_CTX_set_info_callback(ssl_ctx, info_callback);
 +
++      SSL_CTX_set_options(ssl_ctx, SSL_OP_DONT_INSERT_EMPTY_FRAGMENTS|SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3);
++
 +      /* Sets the certificate sent to the other party. */
 +      if (ssl_cert_path != NULL
 +          && SSL_CTX_use_certificate_file(ssl_ctx, ssl_cert_path,
@@ -640,28 +766,59 @@ new file mode 100644
 + */
 +static RETSIGTYPE tls_sigusr1(int UNUSED(val))
 +{
-+      if (ssl) {
-+              SSL_shutdown(ssl);
-+              SSL_free(ssl);
-+              ssl = NULL;
-+      }
 +      ssl_running = 0;
 +}
 +
++/* Signal handler that starts record-size buffering. */
++static RETSIGTYPE tls_sigusr2(int UNUSED(val))
++{
++      ssl_min_send_size = SSL_MAX_RECORD_SIZE;
++}
++
 +/**
 + * Negotiates the TLS connection, creates a socket pair for communicating
 + * with the rsync process, then forks into a new process that will handle
 + * the communication.
 + *
 + * 0 is returned on success.
++ * rsync to network algorithm:
++ * A buffer ring of two buffers is maintained. Data is read from the local
++ * socket into the current buffer. The buffer is eligible for sending if
++ * enough data has been buffered, or enough time has passed. Once the buffer
++ * becomes eligible for sending it is marked "ready" and buffering continues
++ * on the other buffer (assuming it's "unready"). This algorithm tries to
++ * maximize SSL record size.
++ *
++ * network to rsync algorithm:
++ * One receive buffer twice the size of the SSL max record size. Reading
++ * from the network is attempted only if there's enough space for a whole
++ * record (this is done to optimize OpenSSL per-call overhead).
++ * As bytes enter the buffer sending to rsync begins, when the buffer becomes
++ * full we wait for sending to rsync to finish.
++ *
++ * A graceful stop is attempted on both directions. So if the pipe from rsync
++ * is gracefully shut down, we try to send all data to the network before closing
++ * the connection and if the SSL connection is gracefully shut down we try to
++ * send everything to rsync before closing the rsync pipe.
 + */
 +int start_tls(int f_in, int f_out)
 +{
-+      int tls_fd;
 +      int n = 0, r;
-+      unsigned char buf1[BUF_SIZE], buf2[BUF_SIZE];
-+      int avail1 = 0, avail2 = 0, write1 = 0, write2 = 0;
++      unsigned char buf_tonetwork[2][SSL_MAX_RECORD_SIZE], buf_fromnetwork[2*SSL_MAX_RECORD_SIZE];
++      int tonet_ready[2] = {0}, tonet_size[2] = {0}, write_tonet = 0, tonet_fill_idx = 0, tonet_send_idx = 0;
++      struct timeval buffering_start, now, timeout, *tp;
++      int avail_fromnet = 0, write_fromnet = 0;
++      int want_read_fromnet = 1, want_write_tonet = 0;
++      int want_read_fromrsync = 1, want_write_torsync = 0;
++      int write_tonet_blocks = 0, read_fromnet_blocks = 0;
++      int write_torsync_blocks = 0, read_fromrsync_blocks = 0;
++      int ssl_readfd_blocks = 0, ssl_writefd_blocks = 0;
++      int ssl_write_count = 0;
++      int want_more_io;
++      int net_closed = 0, rsync_closed = 0;
++      int loopcount;
 +      fd_set rd, wd;
++      const char *ssl_ciphers;
 +
 +      if (fd_pair(tls_read))
 +              return 1;
@@ -669,11 +826,11 @@ new file mode 100644
 +              return 1;
 +
 +      set_blocking(tls_read[0]);
-+      set_blocking(tls_read[1]);
-+      set_blocking(tls_write[0]);
++      set_nonblocking(tls_read[1]);
++      set_nonblocking(tls_write[0]);
 +      set_blocking(tls_write[1]);
-+      set_blocking(f_in);
-+      set_blocking(f_out);
++      set_nonblocking(f_in);
++      set_nonblocking(f_out);
 +
 +      ssl_pid = do_fork();
 +      if (ssl_pid < 0)
@@ -681,58 +838,114 @@ new file mode 100644
 +      if (ssl_pid != 0) {
 +              close(tls_write[0]);
 +              close(tls_read[1]);
++              close(f_in);
++              close(f_out);
 +              return 0;
 +      }
 +
++      close(tls_write[1]);
++      close(tls_read[0]);
++
 +      SIGACTION(SIGUSR1, tls_sigusr1);
++      SIGACTION(SIGUSR2, tls_sigusr2);
 +      ssl = SSL_new(ssl_ctx);
 +      if (!ssl)
-+              goto closed;
++              goto out;
 +      if (am_daemon || am_server)
 +              SSL_set_accept_state(ssl);
-+      else
++      else {
 +              SSL_set_connect_state(ssl);
++              ssl_ciphers = getenv("RSYNC_SSL_CIPHERS");
++              if (ssl_ciphers)
++                      SSL_set_cipher_list(ssl, ssl_ciphers);
++      }
++
 +      SSL_set_rfd(ssl, f_in);
 +      SSL_set_wfd(ssl, f_out);
 +
-+      tls_fd = SSL_get_fd(ssl);
 +      n = tls_write[0];
 +      n = MAX(tls_read[1], n);
-+      n = MAX(tls_fd, n) + 1;
++      n = MAX(f_in, n);
++      n = MAX(f_out, n) + 1;
 +
 +      ssl_running = 1;
++      loopcount = 0;
 +      while (ssl_running) {
-+              FD_ZERO(&rd);
-+              FD_ZERO(&wd);
-+              FD_SET(tls_write[0], &rd);
-+              FD_SET(tls_read[1], &wd);
-+              FD_SET(tls_fd, &rd);
-+              FD_SET(tls_fd, &wd);
-+
-+              r = select(n, &rd, &wd, NULL, NULL);
-+
-+              if (r == -1 && errno == EINTR)
-+                      continue;
-+              if (FD_ISSET(tls_write[0], &rd)) {
-+                      r = read(tls_write[0], buf1+avail1, BUF_SIZE-avail1);
-+                      if (r >= 0)
-+                              avail1 += r;
-+                      else {
++              ++loopcount; /* the loopcount prevents starvation of one transfer direction by the other */
++              want_more_io = 0;
++
++              if (want_read_fromrsync && !read_fromrsync_blocks) {
++                      int cur_buf_size = tonet_size[tonet_fill_idx];
++                      r = read(tls_write[0], buf_tonetwork[tonet_fill_idx]+cur_buf_size, sizeof(buf_tonetwork[0])-cur_buf_size);
++                      if (r > 0) {
++                              want_more_io = 1;
++                              if (!net_closed) {
++                                      if (cur_buf_size == 0)
++                                              gettimeofday(&buffering_start, NULL);
++                                      cur_buf_size += r;
++                                      tonet_size[tonet_fill_idx] = cur_buf_size;
++                                      if (cur_buf_size >= ssl_min_send_size) {
++                                              want_write_tonet = 1;
++                                              tonet_ready[tonet_fill_idx] = 1;
++                                              tonet_fill_idx = !tonet_fill_idx;
++                                              if (tonet_ready[tonet_fill_idx])
++                                                      want_read_fromrsync = 0;
++                                      }
++                              }
++                      } else if (r < 0 && errno == EWOULDBLOCK)
++                              read_fromrsync_blocks = 1;
++                      else if (r < 0) {
 +                              rprintf(FERROR, "pipe read error: %s\n",
 +                                      strerror(errno));
 +                              break;
++                      } else {
++                              rsync_closed = 1;
++                              if (cur_buf_size) {
++                                      want_write_tonet = 1;
++                                      tonet_ready[tonet_fill_idx] = 1; /* close current buffer */
++                              }
++                              want_read_fromrsync = 0; /* don't read */
++                              want_write_torsync = 0; /* don't write */
++                              read_fromrsync_blocks = 0; /* don't select */
++                              write_torsync_blocks = 0;
++                              want_read_fromnet = 1; /* purge incoming data from network */
++                              avail_fromnet = 0;
++                              if (net_closed || !want_write_tonet)
++                                      break;
 +                      }
 +              }
-+              if (FD_ISSET(tls_fd, &rd)) {
-+                      r = SSL_read(ssl, buf2+avail2, BUF_SIZE-avail2);
-+                      if (r > 0)
-+                              avail2 += r;
-+                      else {
++
++              if (want_read_fromnet && !read_fromnet_blocks) {
++                      r = SSL_read(ssl, buf_fromnetwork+avail_fromnet, SSL_MAX_RECORD_SIZE);
++                      if (r > 0) {
++                              want_more_io = 1;
++                              if (!rsync_closed) {
++                                      avail_fromnet += r;
++                                      want_write_torsync = 1;
++                                      if (avail_fromnet >= (int)(sizeof(buf_fromnetwork)-SSL_MAX_RECORD_SIZE))
++                                              want_read_fromnet = 0;
++                              }
++                      } else {
 +                              switch (SSL_get_error(ssl, r)) {
 +                              case SSL_ERROR_ZERO_RETURN:
-+                                      goto closed;
++                                      net_closed = 1;
++                                      want_read_fromnet = 0; /* don't read */
++                                      want_write_tonet = 0; /* don't write */
++                                      ssl_readfd_blocks = 0; /* and for heaven's sake - don't select() */
++                                      ssl_writefd_blocks = 0;
++                                      want_read_fromrsync = 1; /* and purge stuff from rsync side */
++                                      tonet_size[tonet_fill_idx] = 0;
++                                      tonet_ready[tonet_fill_idx] = 0;
++                                      if (rsync_closed || !want_write_torsync)
++                                              goto graceful_stop;
++                                      break;
 +                              case SSL_ERROR_WANT_READ:
++                                      ssl_readfd_blocks = 1;
++                                      read_fromnet_blocks = 1;
++                                      break;
 +                              case SSL_ERROR_WANT_WRITE:
++                                      ssl_writefd_blocks = 1;
++                                      read_fromnet_blocks = 1;
 +                                      break;
 +                              case SSL_ERROR_SYSCALL:
 +                                      if (r == 0)
@@ -742,7 +955,7 @@ new file mode 100644
 +                                                      strerror(errno));
 +                                      goto closed;
 +                              case SSL_ERROR_SSL:
-+                                      rprintf(FERROR, "SSL: %s\n",
++                                      rprintf(FERROR, "SSL %s\n",
 +                                              ERR_error_string(ERR_get_error(), NULL));
 +                                      goto closed;
 +                              default:
@@ -751,33 +964,70 @@ new file mode 100644
 +                              }
 +                      }
 +              }
-+              if (FD_ISSET(tls_read[1], &wd) && write2 < avail2) {
-+                      r = write(tls_read[1], buf2+write2, avail2-write2);
-+                      if (r >= 0)
-+                              write2 += r;
++
++              if (want_write_torsync && !write_torsync_blocks) {
++                      r = write(tls_read[1], buf_fromnetwork+write_fromnet, avail_fromnet-write_fromnet);
++                      if (r > 0) {
++                              want_more_io = 1;
++                              write_fromnet += r;
++                              if (write_fromnet >= avail_fromnet) {
++                                      write_fromnet = 0;
++                                      avail_fromnet = 0;
++                                      if (net_closed)
++                                              break;
++                                      want_read_fromnet = 1;
++                                      want_write_torsync = 0;
++                              }
++                      }
++                      else if (errno == EWOULDBLOCK)
++                              write_torsync_blocks = 1;
 +                      else {
-+                              rprintf(FERROR, "pipe write error: %s\n",
-+                                      strerror(errno));
++                              if (errno != EPIPE)
++                                      rprintf(FERROR, "pipe write error: %s\n", strerror(errno));
 +                              break;
 +                      }
 +              }
-+              if (FD_ISSET(tls_fd, &wd) && write1 < avail1) {
-+                      r = SSL_write(ssl, buf1+write1, avail1-write1);
-+                      if (r > 0)
-+                              write1 += r;
-+                      else {
++
++              if (want_write_tonet && !write_tonet_blocks) {
++                      /* lock the write count, 'cause if SSL_write fails on non-blocking IO, it
++                       * must be repeated with same parameters. */
++                      if (ssl_write_count == 0)
++                              ssl_write_count = tonet_size[tonet_send_idx] - write_tonet;
++                      r = SSL_write(ssl, buf_tonetwork[tonet_send_idx]+write_tonet, ssl_write_count);
++                      if (r > 0) {
++                              want_more_io = 1;
++                              write_tonet += r;
++                              ssl_write_count = 0;
++                              if (write_tonet >= tonet_size[tonet_send_idx]) {
++                                      write_tonet = 0;
++                                      tonet_size[tonet_send_idx] = 0;
++                                      tonet_ready[tonet_send_idx] = 0;
++                                      if (!rsync_closed)
++                                              want_read_fromrsync = 1;
++                                      tonet_send_idx = !tonet_send_idx;
++                                      if (!tonet_ready[tonet_send_idx]) {
++                                              want_write_tonet = 0;
++                                              if (rsync_closed)
++                                                      break;
++                                      }
++                              }
++                      } else {
 +                              switch (SSL_get_error(ssl, r)) {
 +                              case SSL_ERROR_ZERO_RETURN:
-+                                      goto closed;
++                                      goto graceful_stop;
 +                              case SSL_ERROR_WANT_READ:
++                                      ssl_readfd_blocks = 1;
++                                      write_tonet_blocks = 1;
++                                      break;
 +                              case SSL_ERROR_WANT_WRITE:
++                                      ssl_writefd_blocks = 1;
++                                      write_tonet_blocks = 1;
 +                                      break;
 +                              case SSL_ERROR_SYSCALL:
 +                                      if (r == 0)
 +                                              rprintf(FERROR, "SSL: spurious EOF\n");
 +                                      else
-+                                              rprintf(FERROR, "SSL: I/O error: %s\n",
-+                                                      strerror(errno));
++                                              rprintf(FERROR, "SSL: I/O error: %s\n", strerror(errno));
 +                                      goto closed;
 +                              case SSL_ERROR_SSL:
 +                                      rprintf(FERROR, "SSL: %s\n",
@@ -789,20 +1039,110 @@ new file mode 100644
 +                              }
 +                      }
 +              }
-+              if (avail1 == write1)
-+                      avail1 = write1 = 0;
-+              if (avail2 == write2)
-+                      avail2 = write2 = 0;
++
++              if (!want_more_io || (loopcount&31) == 0) {
++                      int has_buffered = 0;
++                      if (!net_closed && !tonet_ready[tonet_fill_idx] && tonet_size[tonet_fill_idx]) {
++                              has_buffered = 1;
++                              unsigned long buffering_time;
++                              gettimeofday(&now,NULL);
++                              buffering_time = msdiff(&buffering_start,&now);
++                              if (buffering_time >= MAX_BUFFERING_TIME_MS) {
++                                      want_write_tonet = 1;
++                                      want_more_io = 1;
++                                      tonet_ready[tonet_fill_idx] = 1;
++                                      tonet_fill_idx = !tonet_fill_idx;
++                                      if (tonet_ready[tonet_fill_idx])
++                                              want_read_fromrsync = 0;
++                              }
++                      }
++
++                      int waiting_on_something = 0;
++                      FD_ZERO(&rd);
++                      FD_ZERO(&wd);
++                      if (read_fromrsync_blocks) {
++                              waiting_on_something = 1;
++                              FD_SET(tls_write[0], &rd);
++                      }
++                      if (write_torsync_blocks) {
++                              waiting_on_something = 1;
++                              FD_SET(tls_read[1], &wd);
++                      }
++                      if (ssl_readfd_blocks) {
++                              waiting_on_something = 1;
++                              FD_SET(f_in, &rd);
++                      }
++                      if (ssl_writefd_blocks) {
++                              waiting_on_something = 1;
++                              FD_SET(f_out, &wd);
++                      }
++
++                      if (want_more_io) {
++                              /* just poll to see if some sockets became non-blocking*/
++                              timeout.tv_sec = 0;
++                              timeout.tv_usec = 0;
++                              tp = &timeout;
++                      } else if (waiting_on_something && !has_buffered)
++                              tp = NULL; /*infinite wait until a socket becomes available*/
++                      else {
++                              timeout.tv_sec = MAX_BUFFERING_TIME_MS/1000;
++                              timeout.tv_usec = (MAX_BUFFERING_TIME_MS%1000)*1000;
++                              tp = &timeout;
++                      }
++
++                      if (waiting_on_something || !want_more_io) {
++                              r = select(n, &rd, &wd, NULL, tp);
++
++                              if (r == -1) {
++                                      if (errno != EINTR) {
++                                              rprintf(FERROR, "select error: %s\n",
++                                                      strerror(errno));
++                                              break;
++                                      }
++                              }
++
++                              if (r > 0) {
++                                      if (FD_ISSET(tls_write[0], &rd))
++                                              read_fromrsync_blocks = 0;
++                                      if (FD_ISSET(f_in,&rd)) {
++                                              read_fromnet_blocks = 0;
++                                              write_tonet_blocks = 0;
++                                              ssl_readfd_blocks = 0;
++                                      }
++                                      if (FD_ISSET(tls_read[1],&wd))
++                                              write_torsync_blocks = 0;
++                                      if (FD_ISSET(f_out,&wd)) {
++                                              read_fromnet_blocks = 0;
++                                              write_tonet_blocks = 0;
++                                              ssl_writefd_blocks = 0;
++                                      }
++                              }
++                      }
++              }
 +      }
 +
-+      /* XXX I'm pretty sure that there is a lot that I am not considering
-+         here. Bugs? Yes, probably. */
++  graceful_stop:
++      /* The SSL shutdown API looks pretty broken with respect to corner cases
++       * where the underlying socket is blocking. This is a "best effort". */
++      if (SSL_shutdown(ssl) != 1)
++              SSL_shutdown(ssl);
++  closed:
++      SSL_free(ssl);
 +
 +      /* We're finished. */
-+    closed:
++  out:
 +      close(tls_read[1]);
 +      close(tls_write[0]);
-+      exit(0);
++      close(f_in);
++      if (f_out != f_in)
++              close(f_out);
++      _exit(0);
++}
++
++void start_tls_buffering(void)
++{
++      if (ssl_pid > 0)
++              kill(ssl_pid, SIGUSR2);
 +}
 +
 +/**
@@ -810,6 +1150,15 @@ new file mode 100644
 + */
 +void end_tls(void)
 +{
-+      if (ssl_pid > 0)
-+              kill(ssl_pid, SIGUSR1);
++      if (ssl_pid > 0) {
++              int status;
++              /* try a graceful stop - this makes sure all data is sent, and
++               * also causes our side to send a close_notify alert, as required
++               * by TLS specs. */
++              close(get_tls_rfd());
++              if (get_tls_wfd() != get_tls_rfd())
++                      close(get_tls_wfd());
++              terminate_process(ssl_pid, &status, SIGUSR1, 1);
++              ssl_pid = -1;
++      }
 +}