btrfs: fix deadlock with fsync+fiemap+transaction commit
authorJosef Bacik <josef@toxicpanda.com>
Mon, 13 Jun 2022 19:09:49 +0000 (15:09 -0400)
committerDavid Sterba <dsterba@suse.com>
Tue, 21 Jun 2022 12:47:08 +0000 (14:47 +0200)
We are hitting the following deadlock in production occasionally

Task 1 Task 2 Task 3 Task 4 Task 5
fsync(A)
 start trans
start commit
falloc(A)
 lock 5m-10m
 start trans
  wait for commit
fiemap(A)
 lock 0-10m
  wait for 5m-10m
   (have 0-5m locked)

 have btrfs_need_log_full_commit
  !full_sync
  wait_ordered_extents
finish_ordered_io(A)
lock 0-5m
DEADLOCK

We have an existing dependency of file extent lock -> transaction.
However in fsync if we tried to do the fast logging, but then had to
fall back to committing the transaction, we will be forced to call
btrfs_wait_ordered_range() to make sure all of our extents are updated.

This creates a dependency of transaction -> file extent lock, because
btrfs_finish_ordered_io() will need to take the file extent lock in
order to run the ordered extents.

Fix this by stopping the transaction if we have to do the full commit
and we attempted to do the fast logging.  Then attach to the transaction
and commit it if we need to.

CC: stable@vger.kernel.org # 5.15+
Reviewed-by: Filipe Manana <fdmanana@suse.com>
Signed-off-by: Josef Bacik <josef@toxicpanda.com>
Signed-off-by: David Sterba <dsterba@suse.com>
fs/btrfs/file.c

index dd30639ecac23b55ebda26bc861c12e9e19effaa..af2f2b71d2df756abd6021c56c90339ae30eaa66 100644 (file)
@@ -2322,25 +2322,62 @@ int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
         */
        btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
 
-       if (ret != BTRFS_NO_LOG_SYNC) {
+       if (ret == BTRFS_NO_LOG_SYNC) {
+               ret = btrfs_end_transaction(trans);
+               goto out;
+       }
+
+       /* We successfully logged the inode, attempt to sync the log. */
+       if (!ret) {
+               ret = btrfs_sync_log(trans, root, &ctx);
                if (!ret) {
-                       ret = btrfs_sync_log(trans, root, &ctx);
-                       if (!ret) {
-                               ret = btrfs_end_transaction(trans);
-                               goto out;
-                       }
-               }
-               if (!full_sync) {
-                       ret = btrfs_wait_ordered_range(inode, start, len);
-                       if (ret) {
-                               btrfs_end_transaction(trans);
-                               goto out;
-                       }
+                       ret = btrfs_end_transaction(trans);
+                       goto out;
                }
-               ret = btrfs_commit_transaction(trans);
-       } else {
+       }
+
+       /*
+        * At this point we need to commit the transaction because we had
+        * btrfs_need_log_full_commit() or some other error.
+        *
+        * If we didn't do a full sync we have to stop the trans handle, wait on
+        * the ordered extents, start it again and commit the transaction.  If
+        * we attempt to wait on the ordered extents here we could deadlock with
+        * something like fallocate() that is holding the extent lock trying to
+        * start a transaction while some other thread is trying to commit the
+        * transaction while we (fsync) are currently holding the transaction
+        * open.
+        */
+       if (!full_sync) {
                ret = btrfs_end_transaction(trans);
+               if (ret)
+                       goto out;
+               ret = btrfs_wait_ordered_range(inode, start, len);
+               if (ret)
+                       goto out;
+
+               /*
+                * This is safe to use here because we're only interested in
+                * making sure the transaction that had the ordered extents is
+                * committed.  We aren't waiting on anything past this point,
+                * we're purely getting the transaction and committing it.
+                */
+               trans = btrfs_attach_transaction_barrier(root);
+               if (IS_ERR(trans)) {
+                       ret = PTR_ERR(trans);
+
+                       /*
+                        * We committed the transaction and there's no currently
+                        * running transaction, this means everything we care
+                        * about made it to disk and we are done.
+                        */
+                       if (ret == -ENOENT)
+                               ret = 0;
+                       goto out;
+               }
        }
+
+       ret = btrfs_commit_transaction(trans);
 out:
        ASSERT(list_empty(&ctx.list));
        err = file_check_and_advance_wb_err(file);