fs/btrfs/locking.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2008 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/sched.h>
   7 #include <linux/pagemap.h>
   8 #include <linux/spinlock.h>
   9 #include <linux/page-flags.h>
  10 #include <asm/bug.h>
  11 #include "ctree.h"
  12 #include "extent_io.h"
  13 #include "locking.h"
  14
  15 #ifdef CONFIG_BTRFS_DEBUG
  16 static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb)
  17 {
  18         WARN_ON(atomic_read(&eb->spinning_writers));
  19         atomic_inc(&eb->spinning_writers);
  20 }
  21
  22 static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb)
  23 {
  24         WARN_ON(atomic_read(&eb->spinning_writers) != 1);
  25         atomic_dec(&eb->spinning_writers);
  26 }
  27
  28 static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb)
  29 {
  30         WARN_ON(atomic_read(&eb->spinning_writers));
  31 }
  32
  33 static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb)
  34 {
  35         atomic_inc(&eb->spinning_readers);
  36 }
  37
  38 static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb)
  39 {
  40         WARN_ON(atomic_read(&eb->spinning_readers) == 0);
  41         atomic_dec(&eb->spinning_readers);
  42 }
  43
  44 static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb)
  45 {
  46         atomic_inc(&eb->read_locks);
  47 }
  48
  49 static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb)
  50 {
  51         atomic_dec(&eb->read_locks);
  52 }
  53
  54 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb)
  55 {
  56         BUG_ON(!atomic_read(&eb->read_locks));
  57 }
  58
  59 static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb)
  60 {
  61         atomic_inc(&eb->write_locks);
  62 }
  63
  64 static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb)
  65 {
  66         atomic_dec(&eb->write_locks);
  67 }
  68
  69 void btrfs_assert_tree_locked(struct extent_buffer *eb)
  70 {
  71         BUG_ON(!atomic_read(&eb->write_locks));
  72 }
  73
  74 #else
  75 static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb) { }
  76 static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb) { }
  77 static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb) { }
  78 static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb) { }
  79 static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb) { }
  80 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb) { }
  81 static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb) { }
  82 static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb) { }
  83 void btrfs_assert_tree_locked(struct extent_buffer *eb) { }
  84 static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb) { }
  85 static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb) { }
  86 #endif
  87
  88 void btrfs_set_lock_blocking_read(struct extent_buffer *eb)
  89 {
  90         /*
  91          * No lock is required.  The lock owner may change if we have a read
  92          * lock, but it won't change to or away from us.  If we have the write
  93          * lock, we are the owner and it'll never change.
  94          */
  95         if (eb->lock_nested && current->pid == eb->lock_owner)
  96                 return;
  97         btrfs_assert_tree_read_locked(eb);
  98         atomic_inc(&eb->blocking_readers);
  99         btrfs_assert_spinning_readers_put(eb);
 100         read_unlock(&eb->lock);
 101 }
 102
 103 void btrfs_set_lock_blocking_write(struct extent_buffer *eb)
 104 {
 105         /*
 106          * No lock is required.  The lock owner may change if we have a read
 107          * lock, but it won't change to or away from us.  If we have the write
 108          * lock, we are the owner and it'll never change.
 109          */
 110         if (eb->lock_nested && current->pid == eb->lock_owner)
 111                 return;
 112         if (atomic_read(&eb->blocking_writers) == 0) {
 113                 btrfs_assert_spinning_writers_put(eb);
 114                 btrfs_assert_tree_locked(eb);
 115                 atomic_inc(&eb->blocking_writers);
 116                 write_unlock(&eb->lock);
 117         }
 118 }
 119
 120 void btrfs_clear_lock_blocking_read(struct extent_buffer *eb)
 121 {
 122         /*
 123          * No lock is required.  The lock owner may change if we have a read
 124          * lock, but it won't change to or away from us.  If we have the write
 125          * lock, we are the owner and it'll never change.
 126          */
 127         if (eb->lock_nested && current->pid == eb->lock_owner)
 128                 return;
 129         BUG_ON(atomic_read(&eb->blocking_readers) == 0);
 130         read_lock(&eb->lock);
 131         btrfs_assert_spinning_readers_get(eb);
 132         /* atomic_dec_and_test implies a barrier */
 133         if (atomic_dec_and_test(&eb->blocking_readers))
 134                 cond_wake_up_nomb(&eb->read_lock_wq);
 135 }
 136
 137 void btrfs_clear_lock_blocking_write(struct extent_buffer *eb)
 138 {
 139         /*
 140          * no lock is required.  The lock owner may change if
 141          * we have a read lock, but it won't change to or away
 142          * from us.  If we have the write lock, we are the owner
 143          * and it'll never change.
 144          */
 145         if (eb->lock_nested && current->pid == eb->lock_owner)
 146                 return;
 147         BUG_ON(atomic_read(&eb->blocking_writers) != 1);
 148         write_lock(&eb->lock);
 149         btrfs_assert_spinning_writers_get(eb);
 150         /* atomic_dec_and_test implies a barrier */
 151         if (atomic_dec_and_test(&eb->blocking_writers))
 152                 cond_wake_up_nomb(&eb->write_lock_wq);
 153 }
 154
 155 /*
 156  * take a spinning read lock.  This will wait for any blocking
 157  * writers
 158  */
 159 void btrfs_tree_read_lock(struct extent_buffer *eb)
 160 {
 161 again:
 162         BUG_ON(!atomic_read(&eb->blocking_writers) &&
 163                current->pid == eb->lock_owner);
 164
 165         read_lock(&eb->lock);
 166         if (atomic_read(&eb->blocking_writers) &&
 167             current->pid == eb->lock_owner) {
 168                 /*
 169                  * This extent is already write-locked by our thread. We allow
 170                  * an additional read lock to be added because it's for the same
 171                  * thread. btrfs_find_all_roots() depends on this as it may be
 172                  * called on a partly (write-)locked tree.
 173                  */
 174                 BUG_ON(eb->lock_nested);
 175                 eb->lock_nested = true;
 176                 read_unlock(&eb->lock);
 177                 return;
 178         }
 179         if (atomic_read(&eb->blocking_writers)) {
 180                 read_unlock(&eb->lock);
 181                 wait_event(eb->write_lock_wq,
 182                            atomic_read(&eb->blocking_writers) == 0);
 183                 goto again;
 184         }
 185         btrfs_assert_tree_read_locks_get(eb);
 186         btrfs_assert_spinning_readers_get(eb);
 187 }
 188
 189 /*
 190  * take a spinning read lock.
 191  * returns 1 if we get the read lock and 0 if we don't
 192  * this won't wait for blocking writers
 193  */
 194 int btrfs_tree_read_lock_atomic(struct extent_buffer *eb)
 195 {
 196         if (atomic_read(&eb->blocking_writers))
 197                 return 0;
 198
 199         read_lock(&eb->lock);
 200         if (atomic_read(&eb->blocking_writers)) {
 201                 read_unlock(&eb->lock);
 202                 return 0;
 203         }
 204         btrfs_assert_tree_read_locks_get(eb);
 205         btrfs_assert_spinning_readers_get(eb);
 206         return 1;
 207 }
 208
 209 /*
 210  * returns 1 if we get the read lock and 0 if we don't
 211  * this won't wait for blocking writers
 212  */
 213 int btrfs_try_tree_read_lock(struct extent_buffer *eb)
 214 {
 215         if (atomic_read(&eb->blocking_writers))
 216                 return 0;
 217
 218         if (!read_trylock(&eb->lock))
 219                 return 0;
 220
 221         if (atomic_read(&eb->blocking_writers)) {
 222                 read_unlock(&eb->lock);
 223                 return 0;
 224         }
 225         btrfs_assert_tree_read_locks_get(eb);
 226         btrfs_assert_spinning_readers_get(eb);
 227         return 1;
 228 }
 229
 230 /*
 231  * returns 1 if we get the read lock and 0 if we don't
 232  * this won't wait for blocking writers or readers
 233  */
 234 int btrfs_try_tree_write_lock(struct extent_buffer *eb)
 235 {
 236         if (atomic_read(&eb->blocking_writers) ||
 237             atomic_read(&eb->blocking_readers))
 238                 return 0;
 239
 240         write_lock(&eb->lock);
 241         if (atomic_read(&eb->blocking_writers) ||
 242             atomic_read(&eb->blocking_readers)) {
 243                 write_unlock(&eb->lock);
 244                 return 0;
 245         }
 246         btrfs_assert_tree_write_locks_get(eb);
 247         btrfs_assert_spinning_writers_get(eb);
 248         eb->lock_owner = current->pid;
 249         return 1;
 250 }
 251
 252 /*
 253  * drop a spinning read lock
 254  */
 255 void btrfs_tree_read_unlock(struct extent_buffer *eb)
 256 {
 257         /*
 258          * if we're nested, we have the write lock.  No new locking
 259          * is needed as long as we are the lock owner.
 260          * The write unlock will do a barrier for us, and the lock_nested
 261          * field only matters to the lock owner.
 262          */
 263         if (eb->lock_nested && current->pid == eb->lock_owner) {
 264                 eb->lock_nested = false;
 265                 return;
 266         }
 267         btrfs_assert_tree_read_locked(eb);
 268         btrfs_assert_spinning_readers_put(eb);
 269         btrfs_assert_tree_read_locks_put(eb);
 270         read_unlock(&eb->lock);
 271 }
 272
 273 /*
 274  * drop a blocking read lock
 275  */
 276 void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb)
 277 {
 278         /*
 279          * if we're nested, we have the write lock.  No new locking
 280          * is needed as long as we are the lock owner.
 281          * The write unlock will do a barrier for us, and the lock_nested
 282          * field only matters to the lock owner.
 283          */
 284         if (eb->lock_nested && current->pid == eb->lock_owner) {
 285                 eb->lock_nested = false;
 286                 return;
 287         }
 288         btrfs_assert_tree_read_locked(eb);
 289         WARN_ON(atomic_read(&eb->blocking_readers) == 0);
 290         /* atomic_dec_and_test implies a barrier */
 291         if (atomic_dec_and_test(&eb->blocking_readers))
 292                 cond_wake_up_nomb(&eb->read_lock_wq);
 293         btrfs_assert_tree_read_locks_put(eb);
 294 }
 295
 296 /*
 297  * take a spinning write lock.  This will wait for both
 298  * blocking readers or writers
 299  */
 300 void btrfs_tree_lock(struct extent_buffer *eb)
 301 {
 302         WARN_ON(eb->lock_owner == current->pid);
 303 again:
 304         wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0);
 305         wait_event(eb->write_lock_wq, atomic_read(&eb->blocking_writers) == 0);
 306         write_lock(&eb->lock);
 307         if (atomic_read(&eb->blocking_readers) ||
 308             atomic_read(&eb->blocking_writers)) {
 309                 write_unlock(&eb->lock);
 310                 goto again;
 311         }
 312         btrfs_assert_spinning_writers_get(eb);
 313         btrfs_assert_tree_write_locks_get(eb);
 314         eb->lock_owner = current->pid;
 315 }
 316
 317 /*
 318  * drop a spinning or a blocking write lock.
 319  */
 320 void btrfs_tree_unlock(struct extent_buffer *eb)
 321 {
 322         int blockers = atomic_read(&eb->blocking_writers);
 323
 324         BUG_ON(blockers > 1);
 325
 326         btrfs_assert_tree_locked(eb);
 327         eb->lock_owner = 0;
 328         btrfs_assert_tree_write_locks_put(eb);
 329
 330         if (blockers) {
 331                 btrfs_assert_no_spinning_writers(eb);
 332                 atomic_dec(&eb->blocking_writers);
 333                 /* Use the lighter barrier after atomic */
 334                 smp_mb__after_atomic();
 335                 cond_wake_up_nomb(&eb->write_lock_wq);
 336         } else {
 337                 btrfs_assert_spinning_writers_put(eb);
 338                 write_unlock(&eb->lock);
 339         }
 340 }