r8746: replace opendir/readdir/telldir/seekdir/closedir on systems where they
authorAndrew Tridgell <tridge@samba.org>
Mon, 25 Jul 2005 04:03:01 +0000 (04:03 +0000)
committerGerald (Jerry) Carter <jerry@samba.org>
Wed, 10 Oct 2007 18:30:01 +0000 (13:30 -0500)
are broken (apparently all BSD systems). This breakage leads to unlink
on files in an open directory causing a later seekdir to miss
files. The bug happens due to a block boundary bug in the BSD libc
implementation of these calls.

This replacement code also fixes a severe memory usage problem with
telldir that can cause closedir() to take an arbitrary amount of time.

I have reported the bug in readdir to Greg Lehey (a FreeBSD maintainer)
(This used to be commit e1bf7c4279fbc03a52497d24cea375e75059cba1)

source4/build/tests/os2_delete.c [new file with mode: 0644]
source4/configure.in
source4/lib/replace/config.mk
source4/lib/replace/repdir/config.m4 [new file with mode: 0644]
source4/lib/replace/repdir/repdir.c [new file with mode: 0644]

diff --git a/source4/build/tests/os2_delete.c b/source4/build/tests/os2_delete.c
new file mode 100644 (file)
index 0000000..dad27e8
--- /dev/null
@@ -0,0 +1,110 @@
+/*
+  test readdir/unlink pattern that OS/2 uses
+  tridge@samba.org July 2005
+*/
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <dirent.h>
+#include <errno.h>
+#include <string.h>
+#include <fcntl.h>
+#ifdef REPLACE_READDIR
+#include "lib/replace/repdir/repdir.h"
+#endif
+
+#define NUM_FILES 700
+#define READDIR_SIZE 100
+#define DELETE_SIZE 4
+
+#define TESTDIR "test.dir"
+
+#define FAILED() (fprintf(stderr, "Failed at %s:%d - %s\n", __FUNCTION__, __LINE__, strerror(errno)), exit(1), 1)
+
+#ifndef MIN
+#define MIN(a,b) ((a)<(b)?(a):(b))
+#endif
+
+static void cleanup(void)
+{
+       /* I'm a lazy bastard */
+       system("rm -rf " TESTDIR);
+       mkdir(TESTDIR, 0700) == 0 || FAILED();
+}
+
+static void create_files()
+{
+       int i;
+       for (i=0;i<NUM_FILES;i++) {
+               char fname[40];
+               sprintf(fname, TESTDIR "/test%u.txt", i);
+               close(open(fname, O_CREAT|O_RDWR, 0600)) == 0 || FAILED();
+       }
+}
+
+static int os2_delete(DIR *d)
+{
+       off_t offsets[READDIR_SIZE];
+       int i, j;
+       struct dirent *de;
+       char names[READDIR_SIZE][30];
+
+       /* scan, remembering offsets */
+       for (i=0, de=readdir(d); 
+            de && i < READDIR_SIZE; 
+            de=readdir(d), i++) {
+               offsets[i] = telldir(d);
+               strcpy(names[i], de->d_name);
+       }
+
+       if (i == 0) {
+               return 0;
+       }
+
+       /* delete the first few */
+       for (j=0; j<MIN(i, DELETE_SIZE); j++) {
+               char fname[40];
+               sprintf(fname, TESTDIR "/%s", names[j]);
+               unlink(fname) == 0 || FAILED();
+       }
+
+       /* seek to just after the deletion */
+       seekdir(d, offsets[j-1]);
+
+       /* return number deleted */
+       return j;
+}
+
+int main(void)
+{
+       int total_deleted = 0;
+       DIR *d;
+       struct dirent *de;
+
+       cleanup();
+       create_files();
+       
+       d = opendir(TESTDIR);
+
+       /* skip past . and .. */
+       de = readdir(d);
+       strcmp(de->d_name, ".") == 0 || FAILED();
+       de = readdir(d);
+       strcmp(de->d_name, "..") == 0 || FAILED();
+
+       while (1) {
+               int n = os2_delete(d);
+               if (n == 0) break;
+               total_deleted += n;
+       }
+       closedir(d);
+
+       printf("Deleted %d files of %d\n", total_deleted, NUM_FILES);
+
+       rmdir(TESTDIR) == 0 || FAILED();
+
+       return 0;
+}
index 11bcc1211b0d974c1a2e950148e2a1e4e3a4cacf..51589e9fe80dc21ead88ffb2f2a2d7724a0e2f65 100644 (file)
@@ -11,6 +11,7 @@ AC_CONFIG_HEADER(include/config.h)
 sinclude(build/m4/env.m4)
 sinclude(build/m4/rewrite.m4)
 sinclude(lib/replace/win32/config.m4)
+sinclude(lib/replace/repdir/config.m4)
 sinclude(heimdal_build/config.m4)
 sinclude(lib/popt/config.m4)
 sinclude(lib/iconv.m4)
index bd60402ba331cbbe7d465223e8c667f97f8d6fb1..16952ea32d594587612f1e6b6a6a9ff1decbdcae 100644 (file)
@@ -1,9 +1,19 @@
+##############################
+# Start SUBSYSTEM REPLACE_READDIR
+[SUBSYSTEM::REPLACE_READDIR]
+ADD_OBJ_FILES = \
+               lib/replace/repdir/repdir.o
+NOPROTO = YES
+# End SUBSYSTEM REPLACE_READDIR
+##############################
+
+
 ##############################
 # Start SUBSYSTEM LIBREPLACE
 [SUBSYSTEM::LIBREPLACE]
 INIT_OBJ_FILES = lib/replace/replace.o
 ADD_OBJ_FILES = \
                lib/replace/snprintf.o
+REQUIRED_SUBSYSTEMS = REPLACE_READDIR
 # End SUBSYSTEM LIBREPLACE
 ##############################
-
diff --git a/source4/lib/replace/repdir/config.m4 b/source4/lib/replace/repdir/config.m4
new file mode 100644 (file)
index 0000000..0ef40ad
--- /dev/null
@@ -0,0 +1,17 @@
+AC_CACHE_CHECK([for broken readdir],samba_cv_HAVE_BROKEN_READDIR,[
+       AC_TRY_RUN([#include "${srcdir-.}/build/tests/os2_delete.c"],
+                  samba_cv_HAVE_BROKEN_READDIR=no,samba_cv_HAVE_BROKEN_READDIR=yes)])
+
+if test x"$samba_cv_HAVE_BROKEN_READDIR" = x"yes"; then
+AC_CACHE_CHECK([for replacing readdir],samba_cv_REPLACE_READDIR,[
+       AC_TRY_RUN([
+#include "${srcdir-.}/lib/replace/repdir/repdir.c"
+#include "${srcdir-.}/build/tests/os2_delete.c"],
+                  samba_cv_REPLACE_READDIR=yes,samba_cv_REPLACE_READDIR=no)])
+fi
+
+SMB_SUBSYSTEM_ENABLE(REPLACE_READDIR, NO)
+if test x"$samba_cv_REPLACE_READDIR" = x"yes"; then
+       AC_DEFINE(REPLACE_READDIR,1,[replace readdir])
+       SMB_SUBSYSTEM_ENABLE(REPLACE_READDIR, YES)
+fi
diff --git a/source4/lib/replace/repdir/repdir.c b/source4/lib/replace/repdir/repdir.c
new file mode 100644 (file)
index 0000000..9c81355
--- /dev/null
@@ -0,0 +1,138 @@
+/* 
+   Unix SMB/CIFS implementation.
+
+   Copyright (C) Andrew Tridgell 2005
+   
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+   
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+   
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+*/
+/*
+  a replacement for opendir/readdir/telldir/seekdir/closedir for BSD systems
+
+  This is needed because the existing directory handling in FreeBSD
+  and OpenBSD (and possibly NetBSD) doesn't correctly handle unlink()
+  on files in a directory where telldir() has been used. On a block
+  boundary it will occasionally miss a file when seekdir() is used to
+  return to a position previously recorded with telldir().
+
+  This also fixes a severe performance and memory usage problem with
+  telldir() on BSD systems. Each call to telldir() in BSD adds an
+  entry to a linked list, and those entries are cleaned up on
+  closedir(). This means with a large directory closedir() can take an
+  arbitrary amount of time, causing network timeouts as millions of
+  telldir() entries are freed
+
+  Note! This replacement code is not portable. It relies on getdents()
+  always leaving the file descriptor at a seek offset that is a
+  multiple of DIR_BUF_SIZE. If the code detects that this doesn't
+  happen then it will abort(). It also does not handle directories
+  with offsets larger than can be stored in a long,
+
+  This code is available under other free software licenses as
+  well. Contact the author.
+*/
+
+#include <stdlib.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <dirent.h>
+
+#define DIR_BUF_BITS 9
+#define DIR_BUF_SIZE (1<<DIR_BUF_BITS)
+
+struct dir_buf {
+       int fd;
+       int nbytes, ofs;
+       off_t seekpos;
+       char buf[DIR_BUF_SIZE];
+};
+
+DIR *opendir(const char *dname)
+{
+       struct dir_buf *d;
+       d = malloc(sizeof(*d));
+       if (d == NULL) {
+               errno = ENOMEM;
+               return NULL;
+       }
+       d->fd = open(dname, O_RDONLY);
+       if (d->fd == -1) {
+               free(d);
+               return NULL;
+       }
+       d->ofs = 0;
+       d->seekpos = 0;
+       d->nbytes = 0;
+       return (DIR *)d;
+}
+
+struct dirent *readdir(DIR *dir)
+{
+       struct dir_buf *d = (struct dir_buf *)dir;
+       struct dirent *de;
+
+       if (d->ofs >= d->nbytes) {
+               d->seekpos = lseek(d->fd, 0, SEEK_CUR);
+               d->nbytes = getdents(d->fd, d->buf, DIR_BUF_SIZE);
+               d->ofs = 0;
+       }
+       if (d->ofs >= d->nbytes) {
+               return NULL;
+       }
+       de = (struct dirent *)&d->buf[d->ofs];
+       d->ofs += de->d_reclen;
+       return de;
+}
+
+long telldir(DIR *dir)
+{
+       struct dir_buf *d = (struct dir_buf *)dir;
+       if (d->ofs >= d->nbytes) {
+               d->seekpos = lseek(d->fd, 0, SEEK_CUR);
+               d->ofs = 0;
+               d->nbytes = 0;
+       }
+       /* this relies on seekpos always being a multiple of
+          DIR_BUF_SIZE. Is that always true on BSD systems? */
+       if (d->seekpos & (DIR_BUF_SIZE-1)) {
+               abort();
+       }
+       return d->seekpos + d->ofs;
+}
+
+void seekdir(DIR *dir, long ofs)
+{
+       struct dir_buf *d = (struct dir_buf *)dir;
+       d->seekpos = lseek(d->fd, ofs & ~(DIR_BUF_SIZE-1), SEEK_SET);
+       d->nbytes = getdents(d->fd, d->buf, DIR_BUF_SIZE);
+       d->ofs = 0;
+       while (d->ofs < (ofs & (DIR_BUF_SIZE-1))) {
+               if (readdir(dir) == NULL) break;
+       }
+}
+
+int closedir(DIR *dir)
+{
+       struct dir_buf *d = (struct dir_buf *)dir;
+       int r = close(d->fd);
+       if (r != 0) {
+               return r;
+       }
+       free(d);
+       return 0;
+}
+