2 * Unix SMB/CIFS implementation.
4 * Support for OneFS bulk directory enumeration API
6 * Copyright (C) Steven Danneman, 2009
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 3 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, see <http://www.gnu.org/licenses/>.
24 #include <ifs/ifs_syscalls.h>
26 /* The OneFS filesystem provides a readdirplus() syscall, equivalent to the
27 * NFSv3 PDU, which retrieves bulk directory listings with stat information
28 * in a single syscall.
30 * This file hides this bulk interface underneath Samba's very POSIX like
31 * opendir/readdir/telldir VFS interface. This is done to provide a
32 * significant performance improvement when listing the contents of large
33 * directories, which also require file meta information. ie a typical
34 * Windows Explorer request.
37 #define RDP_RESUME_KEY_START 0x1
39 #define RDP_BATCH_SIZE 128
40 #define RDP_DIRENTRIES_SIZE ((size_t)(RDP_BATCH_SIZE * sizeof(struct dirent)))
42 static char *rdp_direntries = NULL;
43 static SMB_STRUCT_STAT *rdp_stats = NULL;
44 static uint64_t *rdp_cookies = NULL;
46 struct rdp_dir_state {
47 struct rdp_dir_state *next, *prev;
49 char *direntries_cursor; /* cursor to current direntry in the cache */
50 size_t stat_count; /* number of entries stored in the cache */
51 size_t stat_cursor; /* cursor to current stat in the cache */
52 uint64_t resume_cookie; /* last cookie returned from the cache */
53 long location; /* absolute location of direnty in DIR */
56 static struct rdp_dir_state *dirstatelist = NULL;
58 SMB_STRUCT_DIR *rdp_last_dirp = NULL;
61 * Given a DIR pointer, return our internal state.
63 * This function also tells us whether the given DIR is the same as we saw
64 * during the last call. Because we use a single globally allocated buffer
65 * for readdirplus entries we must check every call into this API to see if
66 * it's for the same directory listing, or a new one. If it's the same we can
67 * maintain our current cached entries, otherwise we must go to the kernel.
69 * @return 0 on success, 1 on failure
72 rdp_retrieve_dir_state(SMB_STRUCT_DIR *dirp, struct rdp_dir_state **dir_state,
75 struct rdp_dir_state *dsp;
77 /* Is this directory the same as the last call */
78 *same_as_last = (dirp == rdp_last_dirp);
80 for(dsp = dirstatelist; dsp; dsp = dsp->next)
81 if (dsp->dirp == dirp) {
86 /* Couldn't find existing dir_state for the given directory
92 * Initialize the global readdirplus buffers.
94 * These same buffers are used for all calls into readdirplus.
96 * @return 0 on success, errno value on failure
99 rdp_init(struct rdp_dir_state *dsp)
101 /* Unfortunately, there is no good way to free these buffers. If we
102 * allocated and freed for every DIR handle performance would be
103 * adversely affected. For now these buffers will be leaked and only
104 * freed when the smbd process dies. */
105 if (!rdp_direntries) {
106 rdp_direntries = SMB_MALLOC(RDP_DIRENTRIES_SIZE);
113 SMB_MALLOC(RDP_BATCH_SIZE * sizeof(SMB_STRUCT_STAT));
119 rdp_cookies = SMB_MALLOC(RDP_BATCH_SIZE * sizeof(uint64_t));
124 dsp->direntries_cursor = rdp_direntries + RDP_DIRENTRIES_SIZE;
125 dsp->stat_count = RDP_BATCH_SIZE;
126 dsp->stat_cursor = RDP_BATCH_SIZE;
127 dsp->resume_cookie = RDP_RESUME_KEY_START;
134 * Call into readdirplus() to refill our global dirent cache.
136 * This function also resets all cursors back to the beginning of the cache.
137 * All stat buffers are retrieved by following symlinks.
139 * @return number of entries retrieved, -1 on error
142 rdp_fill_cache(struct rdp_dir_state *dsp)
146 dirfd = dirfd(dsp->dirp);
148 DEBUG(1, ("Could not retrieve fd for DIR\n"));
152 /* Resize the stat_count to grab as many entries as possible */
153 dsp->stat_count = RDP_BATCH_SIZE;
155 DEBUG(9, ("Calling readdirplus() with DIR %p, dirfd: %d, "
156 "resume_cookie 0x%llx, location %u, size_to_read: %zu, "
157 "direntries_size: %zu, stat_count: %u\n",
158 dsp->dirp, dirfd, dsp->resume_cookie, dsp->location,
159 RDP_BATCH_SIZE, RDP_DIRENTRIES_SIZE, dsp->stat_count));
161 nread = readdirplus(dirfd,
171 DEBUG(1, ("Error calling readdirplus(): %s\n",
176 DEBUG(9, ("readdirplus() returned %u entries from DIR %p\n",
177 dsp->stat_count, dsp->dirp));
179 dsp->direntries_cursor = rdp_direntries;
180 dsp->stat_cursor = 0;
186 * Open a directory for enumeration.
188 * Create a state struct to track the state of this directory for the life
191 * @param[in] handle vfs handle given in most VFS calls
192 * @param[in] fname filename of the directory to open
193 * @param[in] mask unused
194 * @param[in] attr unused
196 * @return DIR pointer, NULL if directory does not exist, NULL on error
199 onefs_opendir(vfs_handle_struct *handle, const char *fname, const char *mask,
203 SMB_STRUCT_DIR *ret_dirp;
204 struct rdp_dir_state *dsp = NULL;
206 /* Fallback to default system routines if readdirplus is disabled */
207 if (!lp_parm_bool(SNUM(handle->conn), PARM_ONEFS_TYPE,
208 PARM_USE_READDIRPLUS, PARM_USE_READDIRPLUS_DEFAULT))
210 return SMB_VFS_NEXT_OPENDIR(handle, fname, mask, attr);
213 /* Create a struct dir_state */
214 dsp = SMB_MALLOC_P(struct rdp_dir_state);
216 DEBUG(0, ("Error allocating struct rdp_dir_state.\n"));
220 /* Open the directory */
221 ret_dirp = SMB_VFS_NEXT_OPENDIR(handle, fname, mask, attr);
223 DEBUG(3, ("Unable to open directory: %s\n", fname));
227 /* Initialize the dir_state structure and add it to the list */
230 DEBUG(0, ("Error initializing readdirplus() buffers: %s\n",
235 /* Set the SMB_STRUCT_DIR in the dsp */
236 dsp->dirp = ret_dirp;
238 DLIST_ADD(dirstatelist, dsp);
240 DEBUG(9, ("Opened handle on directory: \"%s\", DIR %p\n",
247 * Retrieve one direntry and optional stat buffer from our readdir cache.
249 * Increment the internal resume cookie, and refresh the cache from the
250 * kernel if necessary.
252 * @param[in] handle vfs handle given in most VFS calls
253 * @param[in] dirp system DIR handle to retrieve direntries from
254 * @param[in/out] sbuf optional stat buffer to fill, this can be NULL
256 * @return dirent structure, NULL if at the end of the directory, NULL on error
259 onefs_readdir(vfs_handle_struct *handle, SMB_STRUCT_DIR *dirp,
260 SMB_STRUCT_STAT *sbuf)
262 struct rdp_dir_state *dsp = NULL;
263 SMB_STRUCT_DIRENT *ret_direntp;
267 /* Set stat invalid in-case we error out */
269 SET_STAT_INVALID(*sbuf);
271 /* Fallback to default system routines if readdirplus is disabled */
272 if (!lp_parm_bool(SNUM(handle->conn), PARM_ONEFS_TYPE,
273 PARM_USE_READDIRPLUS, PARM_USE_READDIRPLUS_DEFAULT))
275 return sys_readdir(dirp);
278 /* Retrieve state based off DIR handle */
279 ret = rdp_retrieve_dir_state(dirp, &dsp, &same_as_last);
281 DEBUG(1, ("Could not retrieve dir_state struct for "
282 "SMB_STRUCT_DIR pointer.\n"));
287 /* DIR is the same, current buffer and cursors are valid.
288 * Grab the next direntry from our cache. */
290 if ((dsp->direntries_cursor >=
291 rdp_direntries + RDP_DIRENTRIES_SIZE) ||
292 (dsp->stat_cursor == dsp->stat_count))
294 /* Cache is empty, refill from kernel */
295 ret = rdp_fill_cache(dsp);
302 /* DIR is different from last call, reset all buffers and
303 * cursors, and refill the global cache from the new DIR */
304 ret = rdp_fill_cache(dsp);
309 DEBUG(8, ("Switched global rdp cache to new DIR entry.\n"));
312 /* Return next entry from cache */
313 ret_direntp = ((SMB_STRUCT_DIRENT *)dsp->direntries_cursor);
314 dsp->direntries_cursor +=
315 ((SMB_STRUCT_DIRENT *)dsp->direntries_cursor)->d_reclen;
317 *sbuf = rdp_stats[dsp->stat_cursor];
318 /* readdirplus() sets st_ino field to 0, if it was
319 * unable to retrieve stat information for that
320 * particular directory entry. */
321 if (sbuf->st_ino == 0)
322 SET_STAT_INVALID(*sbuf);
325 DEBUG(9, ("Read from DIR %p, direntry: \"%s\", location: %ld, "
326 "resume cookie: 0x%llx, cache cursor: %zu, cache count: %zu\n",
327 dsp->dirp, ret_direntp->d_name, dsp->location,
328 dsp->resume_cookie, dsp->stat_cursor, dsp->stat_count));
330 dsp->resume_cookie = rdp_cookies[dsp->stat_cursor];
336 /* Set rdp_last_dirp at the end of every VFS call where the cache was
338 rdp_last_dirp = dirp;
343 * Set the location of the next direntry to be read via onefs_readdir().
345 * This function should only pass in locations retrieved from onefs_telldir().
347 * Ideally the seek point will still be in the readdirplus cache, and we'll
348 * just update our cursors. If the seek location is outside of the current
349 * cache we must do an expensive re-enumeration of the entire directory up
352 * @param[in] handle vfs handle given in most VFS calls
353 * @param[in] dirp system DIR handle to set offset on
354 * @param[in] offset from the start of the directory where the next read
357 * @return no return value
360 onefs_seekdir(vfs_handle_struct *handle, SMB_STRUCT_DIR *dirp, long offset)
362 struct rdp_dir_state *dsp = NULL;
364 bool outside_cache = false;
367 /* Fallback to default system routines if readdirplus is disabled */
368 if (!lp_parm_bool(SNUM(handle->conn), PARM_ONEFS_TYPE,
369 PARM_USE_READDIRPLUS, PARM_USE_READDIRPLUS_DEFAULT))
371 return sys_seekdir(dirp, offset);
374 /* Validate inputs */
376 DEBUG(1, ("Invalid offset %ld passed.\n", offset));
380 /* Retrieve state based off DIR handle */
381 ret = rdp_retrieve_dir_state(dirp, &dsp, &same_as_last);
383 DEBUG(1, ("Could not retrieve dir_state struct for "
384 "SMB_STRUCT_DIR pointer.\n"));
385 /* XXX: we can't return an error, should we ABORT rather than
386 * return without actually seeking? */
390 /* Short cut if no work needs to be done */
391 if (offset == dsp->location)
394 /* If DIR is different from last call, reset all buffers and cursors,
395 * and refill the global cache from the new DIR */
397 ret = rdp_fill_cache(dsp);
400 DEBUG(8, ("Switched global rdp cache to new DIR entry.\n"));
403 /* Check if location is outside the currently cached entries */
404 if (offset < dsp->location - dsp->stat_cursor) {
405 /* offset is before the current cache */
406 /* reset to the beginning of the directory */
409 DEBUG(0, ("Error initializing readdirplus() buffers: "
410 "%s\n", strerror(ret)));
413 outside_cache = true;
415 dsp->location + (dsp->stat_count - 1 - dsp->stat_cursor))
417 /* offset is after the current cache
418 * advance the cookie to the end of the cache */
419 dsp->resume_cookie = rdp_cookies[dsp->stat_count - 1];
420 outside_cache = true;
424 /* start reading from the directory, until we have the
425 * specified offset in our cache */
427 dsp->location += dsp->stat_count - dsp->stat_cursor;
428 ret = rdp_fill_cache(dsp);
430 DEBUG(1, ("Error seeking to offset outside the "
431 "cached directory entries. Offset "
432 "%ld \n", dsp->location));
435 dsp->resume_cookie = rdp_cookies[dsp->stat_count - 1];
436 } while (offset >= dsp->location + dsp->stat_count);
439 /* Location should be within the currently cached entries */
440 if (offset < dsp->location &&
441 offset >= dsp->location - dsp->stat_cursor)
443 /* offset is within the current cache, before the cursor.
444 * update cursors to the new location */
445 int new_cursor = dsp->stat_cursor - (dsp->location - offset);
447 dsp->direntries_cursor = rdp_direntries;
448 for (i=0; i < new_cursor; i++) {
449 dsp->direntries_cursor +=
450 ((SMB_STRUCT_DIRENT *)
451 dsp->direntries_cursor)->d_reclen;
453 dsp->stat_cursor = new_cursor;
454 dsp->resume_cookie = rdp_cookies[dsp->stat_cursor];
455 dsp->location = offset;
456 } else if (offset >= dsp->location &&
457 offset <= dsp->location + (dsp->stat_count - 1 - dsp->stat_cursor))
459 /* offset is within the current cache, at or after the cursor.
460 * update cursors to the new location */
461 int add_to_cursor = offset - dsp->location - 1;
463 for (i=0; i < add_to_cursor; i++) {
464 dsp->direntries_cursor +=
465 ((SMB_STRUCT_DIRENT *)
466 dsp->direntries_cursor)->d_reclen;
468 dsp->stat_cursor += add_to_cursor;
469 dsp->resume_cookie = rdp_cookies[dsp->stat_cursor];
470 dsp->location = offset;
473 DEBUG(9, ("Seek DIR %p, location: %ld, cache cursor: %zu\n",
474 dsp->dirp, dsp->location, dsp->stat_cursor));
478 /* Set rdp_last_dirp at the end of every VFS call where the cache was
480 rdp_last_dirp = dirp;
485 * Returns the location of the next direntry to be read via onefs_readdir().
487 * This value can be passed into onefs_seekdir().
489 * @param[in] handle vfs handle given in most VFS calls
490 * @param[in] dirp system DIR handle to set offset on
492 * @return offset from the start of the directory where the next read
496 onefs_telldir(vfs_handle_struct *handle, SMB_STRUCT_DIR *dirp)
498 struct rdp_dir_state *dsp = NULL;
502 /* Fallback to default system routines if readdirplus is disabled */
503 if (!lp_parm_bool(SNUM(handle->conn), PARM_ONEFS_TYPE,
504 PARM_USE_READDIRPLUS, PARM_USE_READDIRPLUS_DEFAULT))
506 return sys_telldir(dirp);
509 /* Retrieve state based off DIR handle */
510 ret = rdp_retrieve_dir_state(dirp, &dsp, &same_as_last);
512 DEBUG(1, ("Could not retrieve dir_state struct for "
513 "SMB_STRUCT_DIR pointer.\n"));
517 DEBUG(9, ("Tell DIR %p, location: %ld, cache cursor: %zu\n",
518 dsp->dirp, dsp->location, dsp->stat_cursor));
520 return dsp->location;
524 * Set the next direntry to be read via onefs_readdir() to the beginning of the
527 * @param[in] handle vfs handle given in most VFS calls
528 * @param[in] dirp system DIR handle to set offset on
530 * @return no return value
533 onefs_rewinddir(vfs_handle_struct *handle, SMB_STRUCT_DIR *dirp)
535 struct rdp_dir_state *dsp = NULL;
539 /* Fallback to default system routines if readdirplus is disabled */
540 if (!lp_parm_bool(SNUM(handle->conn), PARM_ONEFS_TYPE,
541 PARM_USE_READDIRPLUS, PARM_USE_READDIRPLUS_DEFAULT))
543 return sys_rewinddir(dirp);
546 /* Retrieve state based off DIR handle */
547 ret = rdp_retrieve_dir_state(dirp, &dsp, &same_as_last);
549 DEBUG(1, ("Could not retrieve dir_state struct for "
550 "SMB_STRUCT_DIR pointer.\n"));
554 /* Reset location and resume key to beginning */
557 DEBUG(0, ("Error re-initializing rdp cursors: %s\n",
562 DEBUG(9, ("Rewind DIR: %p, to location: %ld\n", dsp->dirp,
569 * Close DIR pointer and remove all state for that directory open.
571 * @param[in] handle vfs handle given in most VFS calls
572 * @param[in] dirp system DIR handle to set offset on
574 * @return -1 on failure, setting errno
577 onefs_closedir(vfs_handle_struct *handle, SMB_STRUCT_DIR *dirp)
579 struct rdp_dir_state *dsp = NULL;
584 /* Fallback to default system routines if readdirplus is disabled */
585 if (!lp_parm_bool(SNUM(handle->conn), PARM_ONEFS_TYPE,
586 PARM_USE_READDIRPLUS, PARM_USE_READDIRPLUS_DEFAULT))
588 return SMB_VFS_NEXT_CLOSEDIR(handle, dirp);
591 /* Retrieve state based off DIR handle */
592 ret = rdp_retrieve_dir_state(dirp, &dsp, &same_as_last);
594 DEBUG(1, ("Could not retrieve dir_state struct for "
595 "SMB_STRUCT_DIR pointer.\n"));
600 /* Close DIR pointer */
601 ret_val = SMB_VFS_NEXT_CLOSEDIR(handle, dsp->dirp);
603 DEBUG(9, ("Closed handle on DIR %p\n", dsp->dirp));
605 /* Tear down state struct */
606 DLIST_REMOVE(dirstatelist, dsp);
609 /* Set lastp to NULL, as cache is no longer valid */
610 rdp_last_dirp = NULL;
616 * Initialize cache data at the beginning of every SMB search operation
618 * Since filesystem operations, such as delete files or meta data
619 * updates can occur to files in the directory we're searching
620 * between FIND_FIRST and FIND_NEXT calls we must refresh the cache
621 * from the kernel on every new search SMB.
623 * @param[in] handle vfs handle given in most VFS calls
624 * @param[in] dirp system DIR handle for the current search
629 onefs_init_search_op(vfs_handle_struct *handle, SMB_STRUCT_DIR *dirp)
631 /* Setting the rdp_last_dirp to NULL will cause the next readdir operation
632 * to refill the cache. */
633 rdp_last_dirp = NULL;