fs/btrfs/locking.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2008 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/sched.h>
   7 #include <linux/pagemap.h>
   8 #include <linux/spinlock.h>
   9 #include <linux/page-flags.h>
  10 #include <asm/bug.h>
  11 #include "ctree.h"
  12 #include "extent_io.h"
  13 #include "locking.h"
  14
  15 #ifdef CONFIG_BTRFS_DEBUG
  16 static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb)
  17 {
  18         WARN_ON(atomic_read(&eb->spinning_writers));
  19         atomic_inc(&eb->spinning_writers);
  20 }
  21
  22 static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb)
  23 {
  24         WARN_ON(atomic_read(&eb->spinning_writers) != 1);
  25         atomic_dec(&eb->spinning_writers);
  26 }
  27
  28 static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb)
  29 {
  30         WARN_ON(atomic_read(&eb->spinning_writers));
  31 }
  32
  33 static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb)
  34 {
  35         atomic_inc(&eb->spinning_readers);
  36 }
  37
  38 static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb)
  39 {
  40         WARN_ON(atomic_read(&eb->spinning_readers) == 0);
  41         atomic_dec(&eb->spinning_readers);
  42 }
  43
  44 static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb)
  45 {
  46         atomic_inc(&eb->read_locks);
  47 }
  48
  49 static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb)
  50 {
  51         atomic_dec(&eb->read_locks);
  52 }
  53
  54 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb)
  55 {
  56         BUG_ON(!atomic_read(&eb->read_locks));
  57 }
  58
  59 #else
  60 static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb) { }
  61 static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb) { }
  62 static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb) { }
  63 static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb) { }
  64 static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb) { }
  65 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb) { }
  66 static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb) { }
  67 static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb) { }
  68 #endif
  69
  70 void btrfs_set_lock_blocking_read(struct extent_buffer *eb)
  71 {
  72         /*
  73          * No lock is required.  The lock owner may change if we have a read
  74          * lock, but it won't change to or away from us.  If we have the write
  75          * lock, we are the owner and it'll never change.
  76          */
  77         if (eb->lock_nested && current->pid == eb->lock_owner)
  78                 return;
  79         btrfs_assert_tree_read_locked(eb);
  80         atomic_inc(&eb->blocking_readers);
  81         btrfs_assert_spinning_readers_put(eb);
  82         read_unlock(&eb->lock);
  83 }
  84
  85 void btrfs_set_lock_blocking_write(struct extent_buffer *eb)
  86 {
  87         /*
  88          * No lock is required.  The lock owner may change if we have a read
  89          * lock, but it won't change to or away from us.  If we have the write
  90          * lock, we are the owner and it'll never change.
  91          */
  92         if (eb->lock_nested && current->pid == eb->lock_owner)
  93                 return;
  94         if (atomic_read(&eb->blocking_writers) == 0) {
  95                 btrfs_assert_spinning_writers_put(eb);
  96                 btrfs_assert_tree_locked(eb);
  97                 atomic_inc(&eb->blocking_writers);
  98                 write_unlock(&eb->lock);
  99         }
 100 }
 101
 102 void btrfs_clear_lock_blocking_read(struct extent_buffer *eb)
 103 {
 104         /*
 105          * No lock is required.  The lock owner may change if we have a read
 106          * lock, but it won't change to or away from us.  If we have the write
 107          * lock, we are the owner and it'll never change.
 108          */
 109         if (eb->lock_nested && current->pid == eb->lock_owner)
 110                 return;
 111         BUG_ON(atomic_read(&eb->blocking_readers) == 0);
 112         read_lock(&eb->lock);
 113         btrfs_assert_spinning_readers_get(eb);
 114         /* atomic_dec_and_test implies a barrier */
 115         if (atomic_dec_and_test(&eb->blocking_readers))
 116                 cond_wake_up_nomb(&eb->read_lock_wq);
 117 }
 118
 119 void btrfs_clear_lock_blocking_write(struct extent_buffer *eb)
 120 {
 121         /*
 122          * no lock is required.  The lock owner may change if
 123          * we have a read lock, but it won't change to or away
 124          * from us.  If we have the write lock, we are the owner
 125          * and it'll never change.
 126          */
 127         if (eb->lock_nested && current->pid == eb->lock_owner)
 128                 return;
 129         BUG_ON(atomic_read(&eb->blocking_writers) != 1);
 130         write_lock(&eb->lock);
 131         btrfs_assert_spinning_writers_get(eb);
 132         /* atomic_dec_and_test implies a barrier */
 133         if (atomic_dec_and_test(&eb->blocking_writers))
 134                 cond_wake_up_nomb(&eb->write_lock_wq);
 135 }
 136
 137 /*
 138  * take a spinning read lock.  This will wait for any blocking
 139  * writers
 140  */
 141 void btrfs_tree_read_lock(struct extent_buffer *eb)
 142 {
 143 again:
 144         BUG_ON(!atomic_read(&eb->blocking_writers) &&
 145                current->pid == eb->lock_owner);
 146
 147         read_lock(&eb->lock);
 148         if (atomic_read(&eb->blocking_writers) &&
 149             current->pid == eb->lock_owner) {
 150                 /*
 151                  * This extent is already write-locked by our thread. We allow
 152                  * an additional read lock to be added because it's for the same
 153                  * thread. btrfs_find_all_roots() depends on this as it may be
 154                  * called on a partly (write-)locked tree.
 155                  */
 156                 BUG_ON(eb->lock_nested);
 157                 eb->lock_nested = 1;
 158                 read_unlock(&eb->lock);
 159                 return;
 160         }
 161         if (atomic_read(&eb->blocking_writers)) {
 162                 read_unlock(&eb->lock);
 163                 wait_event(eb->write_lock_wq,
 164                            atomic_read(&eb->blocking_writers) == 0);
 165                 goto again;
 166         }
 167         atomic_inc(&eb->read_locks);
 168         btrfs_assert_spinning_readers_get(eb);
 169 }
 170
 171 /*
 172  * take a spinning read lock.
 173  * returns 1 if we get the read lock and 0 if we don't
 174  * this won't wait for blocking writers
 175  */
 176 int btrfs_tree_read_lock_atomic(struct extent_buffer *eb)
 177 {
 178         if (atomic_read(&eb->blocking_writers))
 179                 return 0;
 180
 181         read_lock(&eb->lock);
 182         if (atomic_read(&eb->blocking_writers)) {
 183                 read_unlock(&eb->lock);
 184                 return 0;
 185         }
 186         atomic_inc(&eb->read_locks);
 187         btrfs_assert_spinning_readers_get(eb);
 188         return 1;
 189 }
 190
 191 /*
 192  * returns 1 if we get the read lock and 0 if we don't
 193  * this won't wait for blocking writers
 194  */
 195 int btrfs_try_tree_read_lock(struct extent_buffer *eb)
 196 {
 197         if (atomic_read(&eb->blocking_writers))
 198                 return 0;
 199
 200         if (!read_trylock(&eb->lock))
 201                 return 0;
 202
 203         if (atomic_read(&eb->blocking_writers)) {
 204                 read_unlock(&eb->lock);
 205                 return 0;
 206         }
 207         atomic_inc(&eb->read_locks);
 208         btrfs_assert_spinning_readers_get(eb);
 209         return 1;
 210 }
 211
 212 /*
 213  * returns 1 if we get the read lock and 0 if we don't
 214  * this won't wait for blocking writers or readers
 215  */
 216 int btrfs_try_tree_write_lock(struct extent_buffer *eb)
 217 {
 218         if (atomic_read(&eb->blocking_writers) ||
 219             atomic_read(&eb->blocking_readers))
 220                 return 0;
 221
 222         write_lock(&eb->lock);
 223         if (atomic_read(&eb->blocking_writers) ||
 224             atomic_read(&eb->blocking_readers)) {
 225                 write_unlock(&eb->lock);
 226                 return 0;
 227         }
 228         atomic_inc(&eb->write_locks);
 229         btrfs_assert_spinning_writers_get(eb);
 230         eb->lock_owner = current->pid;
 231         return 1;
 232 }
 233
 234 /*
 235  * drop a spinning read lock
 236  */
 237 void btrfs_tree_read_unlock(struct extent_buffer *eb)
 238 {
 239         /*
 240          * if we're nested, we have the write lock.  No new locking
 241          * is needed as long as we are the lock owner.
 242          * The write unlock will do a barrier for us, and the lock_nested
 243          * field only matters to the lock owner.
 244          */
 245         if (eb->lock_nested && current->pid == eb->lock_owner) {
 246                 eb->lock_nested = 0;
 247                 return;
 248         }
 249         btrfs_assert_tree_read_locked(eb);
 250         btrfs_assert_spinning_readers_put(eb);
 251         atomic_dec(&eb->read_locks);
 252         read_unlock(&eb->lock);
 253 }
 254
 255 /*
 256  * drop a blocking read lock
 257  */
 258 void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb)
 259 {
 260         /*
 261          * if we're nested, we have the write lock.  No new locking
 262          * is needed as long as we are the lock owner.
 263          * The write unlock will do a barrier for us, and the lock_nested
 264          * field only matters to the lock owner.
 265          */
 266         if (eb->lock_nested && current->pid == eb->lock_owner) {
 267                 eb->lock_nested = 0;
 268                 return;
 269         }
 270         btrfs_assert_tree_read_locked(eb);
 271         WARN_ON(atomic_read(&eb->blocking_readers) == 0);
 272         /* atomic_dec_and_test implies a barrier */
 273         if (atomic_dec_and_test(&eb->blocking_readers))
 274                 cond_wake_up_nomb(&eb->read_lock_wq);
 275         atomic_dec(&eb->read_locks);
 276 }
 277
 278 /*
 279  * take a spinning write lock.  This will wait for both
 280  * blocking readers or writers
 281  */
 282 void btrfs_tree_lock(struct extent_buffer *eb)
 283 {
 284         WARN_ON(eb->lock_owner == current->pid);
 285 again:
 286         wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0);
 287         wait_event(eb->write_lock_wq, atomic_read(&eb->blocking_writers) == 0);
 288         write_lock(&eb->lock);
 289         if (atomic_read(&eb->blocking_readers) ||
 290             atomic_read(&eb->blocking_writers)) {
 291                 write_unlock(&eb->lock);
 292                 goto again;
 293         }
 294         btrfs_assert_spinning_writers_get(eb);
 295         atomic_inc(&eb->write_locks);
 296         eb->lock_owner = current->pid;
 297 }
 298
 299 /*
 300  * drop a spinning or a blocking write lock.
 301  */
 302 void btrfs_tree_unlock(struct extent_buffer *eb)
 303 {
 304         int blockers = atomic_read(&eb->blocking_writers);
 305
 306         BUG_ON(blockers > 1);
 307
 308         btrfs_assert_tree_locked(eb);
 309         eb->lock_owner = 0;
 310         atomic_dec(&eb->write_locks);
 311
 312         if (blockers) {
 313                 btrfs_assert_no_spinning_writers(eb);
 314                 atomic_dec(&eb->blocking_writers);
 315                 /* Use the lighter barrier after atomic */
 316                 smp_mb__after_atomic();
 317                 cond_wake_up_nomb(&eb->write_lock_wq);
 318         } else {
 319                 btrfs_assert_spinning_writers_put(eb);
 320                 write_unlock(&eb->lock);
 321         }
 322 }
 323
 324 void btrfs_assert_tree_locked(struct extent_buffer *eb)
 325 {
 326         BUG_ON(!atomic_read(&eb->write_locks));
 327 }