From f92c000fc903803e1c2817dd407974ec44390593 Mon Sep 17 00:00:00 2001 From: Andrew Tridgell Date: Mon, 25 Jul 2005 04:03:01 +0000 Subject: [PATCH] r8746: replace opendir/readdir/telldir/seekdir/closedir on systems where they are broken (apparently all BSD systems). This breakage leads to unlink on files in an open directory causing a later seekdir to miss files. The bug happens due to a block boundary bug in the BSD libc implementation of these calls. This replacement code also fixes a severe memory usage problem with telldir that can cause closedir() to take an arbitrary amount of time. I have reported the bug in readdir to Greg Lehey (a FreeBSD maintainer) (This used to be commit e1bf7c4279fbc03a52497d24cea375e75059cba1) --- source4/build/tests/os2_delete.c | 110 +++++++++++++++++++++ source4/configure.in | 1 + source4/lib/replace/config.mk | 12 ++- source4/lib/replace/repdir/config.m4 | 17 ++++ source4/lib/replace/repdir/repdir.c | 138 +++++++++++++++++++++++++++ 5 files changed, 277 insertions(+), 1 deletion(-) create mode 100644 source4/build/tests/os2_delete.c create mode 100644 source4/lib/replace/repdir/config.m4 create mode 100644 source4/lib/replace/repdir/repdir.c diff --git a/source4/build/tests/os2_delete.c b/source4/build/tests/os2_delete.c new file mode 100644 index 00000000000..dad27e8f2c6 --- /dev/null +++ b/source4/build/tests/os2_delete.c @@ -0,0 +1,110 @@ +/* + test readdir/unlink pattern that OS/2 uses + tridge@samba.org July 2005 +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef REPLACE_READDIR +#include "lib/replace/repdir/repdir.h" +#endif + +#define NUM_FILES 700 +#define READDIR_SIZE 100 +#define DELETE_SIZE 4 + +#define TESTDIR "test.dir" + +#define FAILED() (fprintf(stderr, "Failed at %s:%d - %s\n", __FUNCTION__, __LINE__, strerror(errno)), exit(1), 1) + +#ifndef MIN +#define MIN(a,b) ((a)<(b)?(a):(b)) +#endif + +static void cleanup(void) +{ + /* I'm a lazy bastard */ + system("rm -rf " TESTDIR); + mkdir(TESTDIR, 0700) == 0 || FAILED(); +} + +static void create_files() +{ + int i; + for (i=0;id_name); + } + + if (i == 0) { + return 0; + } + + /* delete the first few */ + for (j=0; jd_name, ".") == 0 || FAILED(); + de = readdir(d); + strcmp(de->d_name, "..") == 0 || FAILED(); + + while (1) { + int n = os2_delete(d); + if (n == 0) break; + total_deleted += n; + } + closedir(d); + + printf("Deleted %d files of %d\n", total_deleted, NUM_FILES); + + rmdir(TESTDIR) == 0 || FAILED(); + + return 0; +} diff --git a/source4/configure.in b/source4/configure.in index 11bcc1211b0..51589e9fe80 100644 --- a/source4/configure.in +++ b/source4/configure.in @@ -11,6 +11,7 @@ AC_CONFIG_HEADER(include/config.h) sinclude(build/m4/env.m4) sinclude(build/m4/rewrite.m4) sinclude(lib/replace/win32/config.m4) +sinclude(lib/replace/repdir/config.m4) sinclude(heimdal_build/config.m4) sinclude(lib/popt/config.m4) sinclude(lib/iconv.m4) diff --git a/source4/lib/replace/config.mk b/source4/lib/replace/config.mk index bd60402ba33..16952ea32d5 100644 --- a/source4/lib/replace/config.mk +++ b/source4/lib/replace/config.mk @@ -1,9 +1,19 @@ +############################## +# Start SUBSYSTEM REPLACE_READDIR +[SUBSYSTEM::REPLACE_READDIR] +ADD_OBJ_FILES = \ + lib/replace/repdir/repdir.o +NOPROTO = YES +# End SUBSYSTEM REPLACE_READDIR +############################## + + ############################## # Start SUBSYSTEM LIBREPLACE [SUBSYSTEM::LIBREPLACE] INIT_OBJ_FILES = lib/replace/replace.o ADD_OBJ_FILES = \ lib/replace/snprintf.o +REQUIRED_SUBSYSTEMS = REPLACE_READDIR # End SUBSYSTEM LIBREPLACE ############################## - diff --git a/source4/lib/replace/repdir/config.m4 b/source4/lib/replace/repdir/config.m4 new file mode 100644 index 00000000000..0ef40ad8cb6 --- /dev/null +++ b/source4/lib/replace/repdir/config.m4 @@ -0,0 +1,17 @@ +AC_CACHE_CHECK([for broken readdir],samba_cv_HAVE_BROKEN_READDIR,[ + AC_TRY_RUN([#include "${srcdir-.}/build/tests/os2_delete.c"], + samba_cv_HAVE_BROKEN_READDIR=no,samba_cv_HAVE_BROKEN_READDIR=yes)]) + +if test x"$samba_cv_HAVE_BROKEN_READDIR" = x"yes"; then +AC_CACHE_CHECK([for replacing readdir],samba_cv_REPLACE_READDIR,[ + AC_TRY_RUN([ +#include "${srcdir-.}/lib/replace/repdir/repdir.c" +#include "${srcdir-.}/build/tests/os2_delete.c"], + samba_cv_REPLACE_READDIR=yes,samba_cv_REPLACE_READDIR=no)]) +fi + +SMB_SUBSYSTEM_ENABLE(REPLACE_READDIR, NO) +if test x"$samba_cv_REPLACE_READDIR" = x"yes"; then + AC_DEFINE(REPLACE_READDIR,1,[replace readdir]) + SMB_SUBSYSTEM_ENABLE(REPLACE_READDIR, YES) +fi diff --git a/source4/lib/replace/repdir/repdir.c b/source4/lib/replace/repdir/repdir.c new file mode 100644 index 00000000000..9c81355c687 --- /dev/null +++ b/source4/lib/replace/repdir/repdir.c @@ -0,0 +1,138 @@ +/* + Unix SMB/CIFS implementation. + + Copyright (C) Andrew Tridgell 2005 + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. +*/ +/* + a replacement for opendir/readdir/telldir/seekdir/closedir for BSD systems + + This is needed because the existing directory handling in FreeBSD + and OpenBSD (and possibly NetBSD) doesn't correctly handle unlink() + on files in a directory where telldir() has been used. On a block + boundary it will occasionally miss a file when seekdir() is used to + return to a position previously recorded with telldir(). + + This also fixes a severe performance and memory usage problem with + telldir() on BSD systems. Each call to telldir() in BSD adds an + entry to a linked list, and those entries are cleaned up on + closedir(). This means with a large directory closedir() can take an + arbitrary amount of time, causing network timeouts as millions of + telldir() entries are freed + + Note! This replacement code is not portable. It relies on getdents() + always leaving the file descriptor at a seek offset that is a + multiple of DIR_BUF_SIZE. If the code detects that this doesn't + happen then it will abort(). It also does not handle directories + with offsets larger than can be stored in a long, + + This code is available under other free software licenses as + well. Contact the author. +*/ + +#include +#include +#include +#include +#include +#include +#include + +#define DIR_BUF_BITS 9 +#define DIR_BUF_SIZE (1<fd = open(dname, O_RDONLY); + if (d->fd == -1) { + free(d); + return NULL; + } + d->ofs = 0; + d->seekpos = 0; + d->nbytes = 0; + return (DIR *)d; +} + +struct dirent *readdir(DIR *dir) +{ + struct dir_buf *d = (struct dir_buf *)dir; + struct dirent *de; + + if (d->ofs >= d->nbytes) { + d->seekpos = lseek(d->fd, 0, SEEK_CUR); + d->nbytes = getdents(d->fd, d->buf, DIR_BUF_SIZE); + d->ofs = 0; + } + if (d->ofs >= d->nbytes) { + return NULL; + } + de = (struct dirent *)&d->buf[d->ofs]; + d->ofs += de->d_reclen; + return de; +} + +long telldir(DIR *dir) +{ + struct dir_buf *d = (struct dir_buf *)dir; + if (d->ofs >= d->nbytes) { + d->seekpos = lseek(d->fd, 0, SEEK_CUR); + d->ofs = 0; + d->nbytes = 0; + } + /* this relies on seekpos always being a multiple of + DIR_BUF_SIZE. Is that always true on BSD systems? */ + if (d->seekpos & (DIR_BUF_SIZE-1)) { + abort(); + } + return d->seekpos + d->ofs; +} + +void seekdir(DIR *dir, long ofs) +{ + struct dir_buf *d = (struct dir_buf *)dir; + d->seekpos = lseek(d->fd, ofs & ~(DIR_BUF_SIZE-1), SEEK_SET); + d->nbytes = getdents(d->fd, d->buf, DIR_BUF_SIZE); + d->ofs = 0; + while (d->ofs < (ofs & (DIR_BUF_SIZE-1))) { + if (readdir(dir) == NULL) break; + } +} + +int closedir(DIR *dir) +{ + struct dir_buf *d = (struct dir_buf *)dir; + int r = close(d->fd); + if (r != 0) { + return r; + } + free(d); + return 0; +} + -- 2.34.1