2 * Unix SMB/CIFS implementation.
3 * Support for OneFS kernel oplocks
5 * Copyright (C) Volker Lendecke 2007
6 * Copyright (C) Tim Prouty, 2009
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License as published by
10 * the Free Software Foundation; either version 3 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU General Public License for more details.
18 * You should have received a copy of the GNU General Public License
19 * along with this program; if not, see <http://www.gnu.org/licenses/>.
22 #define DBGC_CLASS DBGC_LOCKING
27 #include "oplock_onefs.h"
28 #include "smbd/globals.h"
30 #include <ifs/ifs_syscalls.h>
31 #include <isi_ecs/isi_ecs_oplocks.h>
34 struct onefs_oplocks_context {
35 struct kernel_oplocks *ctx;
36 const struct oplocks_event_ops *onefs_ops;
38 struct fd_event *read_fde;
41 enum onefs_callback_state {
43 ONEFS_WAITING_FOR_OPLOCK
46 struct onefs_callback_record {
47 struct onefs_callback_record *prev, *next;
49 enum onefs_callback_state state;
51 files_struct *fsp; /* ONEFS_OPEN_FILE */
52 uint64_t mid; /* ONEFS_WAITING_FOR_OPLOCK */
57 * Internal list of files (along with additional state) that have outstanding
58 * oplocks or requests for oplocks.
60 struct onefs_callback_record *callback_recs;
63 * Convert a onefs_callback_record to a debug string using the dbg_ctx().
65 const char *onefs_cb_record_str_dbg(const struct onefs_callback_record *r)
70 result = talloc_strdup(talloc_tos(), "NULL callback record");
76 result = talloc_asprintf(talloc_tos(), "cb record %llu for "
78 fsp_str_dbg(r->data.fsp));
79 case ONEFS_WAITING_FOR_OPLOCK:
80 result = talloc_asprintf(talloc_tos(), "cb record %llu for "
81 "pending mid %llu", r->id,
82 (unsigned long long)r->data.mid);
85 result = talloc_asprintf(talloc_tos(), "cb record %llu unknown "
86 "state %d", r->id, r->state);
94 * Traverse the list of onefs_callback_records and print all entries.
96 static void debug_cb_records(const char *fn)
98 struct onefs_callback_record *rec;
103 DEBUG(10, ("cb records (%s):\n", fn));
105 for (rec = callback_recs; rec; rec = rec->next) {
106 DEBUGADD(10, ("%s\n", onefs_cb_record_str_dbg(rec)));
111 * Find a callback record in the list of outstanding oplock operations.
113 * Once n ifs_createfile requests an oplock on a file, the kernel communicates
114 * with samba via the oplock event channel by sending events that reference an
115 * id. This function maps that id to the onefs_callback_record that was
116 * created for it during the initial setup on open (onefs_oplock_wait_record).
117 * When a matching id is found in the onefs_callback_record list, the
118 * callback_type is checked to make sure the record is in in the correct
121 static struct onefs_callback_record *onefs_find_cb(uint64_t id,
122 enum onefs_callback_state expected_state)
124 struct onefs_callback_record *rec;
126 debug_cb_records("onefs_find_cb");
128 for (rec = callback_recs; rec; rec = rec->next) {
130 DEBUG(10, ("found %s\n",
131 onefs_cb_record_str_dbg(rec)));
137 DEBUG(5, ("Could not find callback record for id %llu\n", id));
141 if (rec->state != expected_state) {
142 DEBUG(0, ("Expected cb type %d, got %s", expected_state,
143 onefs_cb_record_str_dbg(rec)));
152 * Remove and free a callback record from the callback record list.
154 void destroy_onefs_callback_record(uint64_t id)
156 struct onefs_callback_record *rec;
158 debug_cb_records("destroy_onefs_callback_record");
161 DEBUG(10, ("destroy_onefs_callback_record: Nothing to "
166 for (rec = callback_recs; rec; rec = rec->next) {
168 DLIST_REMOVE(callback_recs, rec);
170 DEBUG(10, ("removed cb rec %llu\n", id));
175 DEBUG(0, ("Could not find cb rec %llu to delete", id));
180 * Initialize a callback record and add it to the list of outstanding callback
183 * This is called in the open path before ifs_createfile so an id can be
184 * passed in. Each callback record can be in one of two states:
186 * 1. WAITING_FOR_OPLOCK: This is the initial state for all callback
187 * records. If ifs_createfile can be completed syncronously without needing
188 * to break any level I oplocks, the state is transitioned to OPEN_FILE.
189 * Otherwise ifs_createfile will finish asynchronously and the open is
190 * deferred. When the necessary level I opocks have been broken, and the
191 * open can be done, an event is sent by the kernel on the oplock event
192 * channel, which is handled by semlock_available_handler. At this point
193 * the deferred open is retried. Unless a level I oplock was acquired by
194 * another client, ifs_createfile will now complete synchronously.
196 * 2. OPEN_FILE: Once ifs_createfile completes, the callback record is
197 * transitioned to this state via onefs_set_oplock_callback.
199 uint64_t onefs_oplock_wait_record(uint64_t mid)
201 struct onefs_callback_record *result;
202 static uint64_t id_generator = 0;
204 if (!(result = SMB_MALLOC_P(struct onefs_callback_record))) {
205 DEBUG(0, ("talloc failed\n"));
209 memset(result, '\0', sizeof(result));
212 if (id_generator == 0) {
213 /* Wow, that's a long-running smbd... */
217 result->id = id_generator;
219 result->state = ONEFS_WAITING_FOR_OPLOCK;
220 result->data.mid = mid;
221 DLIST_ADD(callback_recs, result);
223 DEBUG(10, ("New cb rec %llu created\n", result->id));
229 * Transition the callback record state to OPEN_FILE.
231 * This is called after the file is opened and an fsp struct has been
232 * allocated. The mid is dropped in favor of storing the fsp.
234 void onefs_set_oplock_callback(uint64_t id, files_struct *fsp)
236 struct onefs_callback_record *cb;
239 DEBUG(10, ("onefs_set_oplock_callback called for cb rec %llu\n", id));
241 if (!(cb = onefs_find_cb(id, ONEFS_WAITING_FOR_OPLOCK))) {
242 if (asprintf(&msg, "Got invalid callback %lld\n", id) != -1) {
245 smb_panic("Got invalid callback id\n");
251 if (open_was_deferred(cb->data.mid)) {
252 if (asprintf(&msg, "Trying to upgrade callback for deferred "
253 "open mid=%llu\n", (unsigned long long)cb->data.mid) != -1) {
256 smb_panic("Trying to upgrade callback for deferred open "
260 cb->state = ONEFS_OPEN_FILE;
265 * Using a callback record, initialize a share mode entry to pass to
266 * share_mode_entry_to_message to send samba IPC messages.
268 static void init_share_mode_entry(struct share_mode_entry *sme,
269 struct onefs_callback_record *cb,
274 sme->pid = procid_self();
275 sme->op_type = op_type;
276 sme->id = cb->data.fsp->file_id;
277 sme->share_file_id = cb->data.fsp->fh->gen_id;
281 * Callback when a break-to-none event is received from the kernel.
283 * On OneFS level 1 oplocks are always broken to level 2 first, therefore an
284 * async level 2 break message is always sent when breaking to none. The
285 * downside of this is that OneFS currently has no way to express breaking
286 * directly from level 1 to none.
288 static void oplock_break_to_none_handler(uint64_t id)
290 struct onefs_callback_record *cb;
291 struct share_mode_entry sme;
292 char msg[MSG_SMB_SHARE_MODE_ENTRY_SIZE];
294 DEBUG(10, ("oplock_break_to_none_handler called for id %llu\n", id));
296 if (!(cb = onefs_find_cb(id, ONEFS_OPEN_FILE))) {
297 DEBUG(3, ("oplock_break_to_none_handler: could not find "
298 "callback id %llu\n", id));
302 DEBUG(10, ("oplock_break_to_none_handler called for file %s\n",
303 fsp_str_dbg(cb->data.fsp)));
305 init_share_mode_entry(&sme, cb, FORCE_OPLOCK_BREAK_TO_NONE);
306 share_mode_entry_to_message(msg, &sme);
307 messaging_send_buf(smbd_messaging_context(),
309 MSG_SMB_ASYNC_LEVEL2_BREAK,
311 MSG_SMB_SHARE_MODE_ENTRY_SIZE);
314 * We could still receive an OPLOCK_REVOKED message, so keep the
315 * oplock_callback_id around.
320 * Callback when a break-to-level2 event is received from the kernel.
322 * Breaks from level 1 to level 2.
324 static void oplock_break_to_level_two_handler(uint64_t id)
326 struct onefs_callback_record *cb;
327 struct share_mode_entry sme;
328 char msg[MSG_SMB_SHARE_MODE_ENTRY_SIZE];
330 DEBUG(10, ("oplock_break_to_level_two_handler called for id %llu\n",
333 if (!(cb = onefs_find_cb(id, ONEFS_OPEN_FILE))) {
334 DEBUG(3, ("oplock_break_to_level_two_handler: could not find "
335 "callback id %llu\n", id));
339 DEBUG(10, ("oplock_break_to_level_two_handler called for file %s\n",
340 fsp_str_dbg(cb->data.fsp)));
342 init_share_mode_entry(&sme, cb, LEVEL_II_OPLOCK);
343 share_mode_entry_to_message(msg, &sme);
344 messaging_send_buf(smbd_messaging_context(),
346 MSG_SMB_BREAK_REQUEST,
348 MSG_SMB_SHARE_MODE_ENTRY_SIZE);
351 * We could still receive an OPLOCK_REVOKED or OPLOCK_BREAK_TO_NONE
352 * message, so keep the oplock_callback_id around.
357 * Revoke an oplock from an unresponsive client.
359 * The kernel will send this message when it times out waiting for a level 1
360 * oplock break to be acknowledged by the client. The oplock is then
361 * immediately removed.
363 static void oplock_revoked_handler(uint64_t id)
365 struct onefs_callback_record *cb;
366 files_struct *fsp = NULL;
368 DEBUG(10, ("oplock_revoked_handler called for id %llu\n", id));
370 if (!(cb = onefs_find_cb(id, ONEFS_OPEN_FILE))) {
371 DEBUG(3, ("oplock_revoked_handler: could not find "
372 "callback id %llu\n", id));
378 SMB_ASSERT(fsp->oplock_timeout == NULL);
380 DEBUG(0,("Level 1 oplock break failed for file %s. Forcefully "
381 "revoking oplock\n", fsp_str_dbg(fsp)));
386 * cb record is cleaned up in fsp ext data destructor on close, so
387 * leave it in the list.
392 * Asynchronous ifs_createfile callback
394 * If ifs_createfile had to asynchronously break any oplocks, this function is
395 * called when the kernel sends an event that the open can be retried.
397 static void semlock_available_handler(uint64_t id)
399 struct onefs_callback_record *cb;
401 DEBUG(10, ("semlock_available_handler called: %llu\n", id));
403 if (!(cb = onefs_find_cb(id, ONEFS_WAITING_FOR_OPLOCK))) {
404 DEBUG(5, ("semlock_available_handler: Did not find callback "
409 DEBUG(10, ("Got semlock available for mid %llu\n",
410 (unsigned long long)cb->data.mid));
413 if (!(open_was_deferred(cb->data.mid))) {
415 if (asprintf(&msg, "Semlock available on an open that wasn't "
417 onefs_cb_record_str_dbg(cb)) != -1) {
420 smb_panic("Semlock available on an open that wasn't "
424 schedule_deferred_open_smb_message(cb->data.mid);
426 /* Cleanup the callback record since the open will be retried. */
427 destroy_onefs_callback_record(id);
433 * Asynchronous ifs_createfile failure callback
435 * If ifs_createfile had to asynchronously break any oplocks, but an error was
436 * encountered in the kernel, the open will be retried with the state->failed
437 * set to true. This will prompt the open path to send an INTERNAL_ERROR
438 * error message to the client.
440 static void semlock_async_failure_handler(uint64_t id)
442 struct onefs_callback_record *cb;
443 struct deferred_open_record *state;
445 DEBUG(1, ("semlock_async_failure_handler called: %llu\n", id));
447 if (!(cb = onefs_find_cb(id, ONEFS_WAITING_FOR_OPLOCK))) {
448 DEBUG(5, ("semlock_async_failure_handler: Did not find callback "
453 DEBUG(1, ("Got semlock_async_failure message for mid %llu\n",
454 (unsigned long long)cb->data.mid));
457 if (!(open_was_deferred(cb->data.mid))) {
459 if (asprintf(&msg, "Semlock failure on an open that wasn't "
461 onefs_cb_record_str_dbg(cb)) != -1) {
464 smb_panic("Semlock failure on an open that wasn't deferred\n");
467 /* Find the actual deferred open record. */
468 if (!get_open_deferred_message_state(cb->data.mid, NULL, &state)) {
469 DEBUG(0, ("Could not find deferred request for "
470 "mid %d\n", cb->data.mid));
471 destroy_onefs_callback_record(id);
475 /* Update to failed so the client can be notified on retried open. */
476 state->failed = true;
478 /* Schedule deferred open for immediate retry. */
479 schedule_deferred_open_smb_message(cb->data.mid);
481 /* Cleanup the callback record here since the open will be retried. */
482 destroy_onefs_callback_record(id);
488 * OneFS acquires all oplocks via ifs_createfile, so this is a no-op.
490 static bool onefs_set_kernel_oplock(struct kernel_oplocks *_ctx,
491 files_struct *fsp, int oplock_type) {
496 * Release the kernel oplock.
498 static void onefs_release_kernel_oplock(struct kernel_oplocks *_ctx,
499 files_struct *fsp, int oplock_type)
501 enum oplock_type oplock = onefs_samba_oplock_to_oplock(oplock_type);
503 DEBUG(10, ("onefs_release_kernel_oplock: Releasing %s to type %s\n",
504 fsp_str_dbg(fsp), onefs_oplock_str(oplock)));
506 if (fsp->fh->fd == -1) {
507 DEBUG(1, ("no fd\n"));
511 /* Downgrade oplock to either SHARED or NONE. */
512 if (ifs_oplock_downgrade(fsp->fh->fd, oplock)) {
513 DEBUG(1,("ifs_oplock_downgrade failed: %s\n",
519 * Wrap ifs_semlock_write so it is only called on operations that aren't
520 * already contended in the kernel.
522 static void onefs_semlock_write(int fd, enum level2_contention_type type,
523 enum semlock_operation semlock_op)
528 case LEVEL2_CONTEND_ALLOC_GROW:
529 case LEVEL2_CONTEND_POSIX_BRL:
530 DEBUG(10, ("Taking %d write semlock for cmd %d on fd: %d\n",
531 semlock_op, type, fd));
532 ret = ifs_semlock_write(fd, semlock_op);
534 DEBUG(0,("ifs_semlock_write failed taking %d write "
535 "semlock for cmd %d on fd: %d: %s",
536 semlock_op, type, fd, strerror(errno)));
540 DEBUG(10, ("Skipping write semlock for cmd %d on fd: %d\n",
546 * Contend level 2 oplocks in the kernel and smbd.
548 * Taking a write semlock will contend all level 2 oplocks in all smbds across
549 * the cluster except the fsp's own level 2 oplock. This lack of
550 * self-contention is a limitation of the current OneFS kernel oplocks
551 * implementation. Luckily it is easy to contend our own level 2 oplock by
552 * checking the the fsp's oplock_type. If it's a level2, send a break message
553 * to the client and remove the oplock.
555 static void onefs_contend_level2_oplocks_begin(files_struct *fsp,
556 enum level2_contention_type type)
558 /* Take care of level 2 kernel contention. */
559 onefs_semlock_write(fsp->fh->fd, type, SEMLOCK_LOCK);
561 /* Take care of level 2 self contention. */
562 if (LEVEL_II_OPLOCK_TYPE(fsp->oplock_type))
563 break_level2_to_none_async(fsp);
567 * Unlock the write semlock when the level 2 contending operation ends.
569 static void onefs_contend_level2_oplocks_end(files_struct *fsp,
570 enum level2_contention_type type)
572 /* Take care of level 2 kernel contention. */
573 onefs_semlock_write(fsp->fh->fd, type, SEMLOCK_UNLOCK);
577 * Return string value of onefs oplock types.
579 const char *onefs_oplock_str(enum oplock_type onefs_oplock_type)
581 switch (onefs_oplock_type) {
583 return "OPLOCK_NONE";
584 case OPLOCK_EXCLUSIVE:
585 return "OPLOCK_EXCLUSIVE";
587 return "OPLOCK_BATCH";
589 return "OPLOCK_SHARED";
597 * Convert from onefs to samba oplock.
599 int onefs_oplock_to_samba_oplock(enum oplock_type onefs_oplock)
601 switch (onefs_oplock) {
604 case OPLOCK_EXCLUSIVE:
605 return EXCLUSIVE_OPLOCK;
609 return LEVEL_II_OPLOCK;
611 DEBUG(0, ("unknown oplock type %d found\n", onefs_oplock));
618 * Convert from samba to onefs oplock.
620 enum oplock_type onefs_samba_oplock_to_oplock(int samba_oplock_type)
622 if (BATCH_OPLOCK_TYPE(samba_oplock_type)) return OPLOCK_BATCH;
623 if (EXCLUSIVE_OPLOCK_TYPE(samba_oplock_type)) return OPLOCK_EXCLUSIVE;
624 if (LEVEL_II_OPLOCK_TYPE(samba_oplock_type)) return OPLOCK_SHARED;
629 * Oplock event handler.
631 * Call into the event system dispatcher to handle each event.
633 static void onefs_oplocks_read_fde_handler(struct event_context *ev,
634 struct fd_event *fde,
638 struct onefs_oplocks_context *ctx =
639 talloc_get_type(private_data, struct onefs_oplocks_context);
641 if (oplocks_event_dispatcher(ctx->onefs_ops)) {
642 DEBUG(0, ("oplocks_event_dispatcher failed: %s\n",
648 * Setup kernel oplocks
650 static const struct kernel_oplocks_ops onefs_koplocks_ops = {
651 .set_oplock = onefs_set_kernel_oplock,
652 .release_oplock = onefs_release_kernel_oplock,
653 .contend_level2_oplocks_begin = onefs_contend_level2_oplocks_begin,
654 .contend_level2_oplocks_end = onefs_contend_level2_oplocks_end,
657 static const struct oplocks_event_ops onefs_dispatch_ops = {
658 .oplock_break_to_none = oplock_break_to_none_handler,
659 .oplock_break_to_level_two = oplock_break_to_level_two_handler,
660 .oplock_revoked = oplock_revoked_handler,
661 .semlock_available = semlock_available_handler,
662 .semlock_async_failure = semlock_async_failure_handler,
665 struct kernel_oplocks *onefs_init_kernel_oplocks(TALLOC_CTX *mem_ctx)
667 struct kernel_oplocks *_ctx = NULL;
668 struct onefs_oplocks_context *ctx = NULL;
669 struct procoptions po = PROCOPTIONS_INIT;
671 DEBUG(10, ("onefs_init_kernel_oplocks called\n"));
673 /* Set the non-blocking proc flag */
674 po.po_flags_on |= P_NON_BLOCKING_SEMLOCK;
675 if (setprocoptions(&po) != 0) {
676 DEBUG(0, ("setprocoptions failed: %s.\n", strerror(errno)));
680 /* Setup the oplock contexts */
681 _ctx = talloc_zero(mem_ctx, struct kernel_oplocks);
686 ctx = talloc_zero(_ctx, struct onefs_oplocks_context);
691 _ctx->ops = &onefs_koplocks_ops;
692 _ctx->flags = (KOPLOCKS_LEVEL2_SUPPORTED |
693 KOPLOCKS_DEFERRED_OPEN_NOTIFICATION |
694 KOPLOCKS_TIMEOUT_NOTIFICATION |
695 KOPLOCKS_OPLOCK_BROKEN_NOTIFICATION);
696 _ctx->private_data = ctx;
698 ctx->onefs_ops = &onefs_dispatch_ops;
700 /* Register an kernel event channel for oplocks */
701 ctx->onefs_event_fd = oplocks_event_register();
702 if (ctx->onefs_event_fd == -1) {
703 DEBUG(0, ("oplocks_event_register failed: %s\n",
708 DEBUG(10, ("oplock event_fd = %d\n", ctx->onefs_event_fd));
710 /* Register the oplock event_fd with samba's event system */
711 ctx->read_fde = event_add_fd(smbd_event_context(),
715 onefs_oplocks_read_fde_handler,
725 void oplock_onefs_dummy(void);
726 void oplock_onefs_dummy(void) {}
727 #endif /* HAVE_ONEFS */