lib/lockref.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <linux/export.h>
   3 #include <linux/lockref.h>
   4
   5 #if USE_CMPXCHG_LOCKREF
   6
   7 /*
   8  * Note that the "cmpxchg()" reloads the "old" value for the
   9  * failure case.
  10  */
  11 #define CMPXCHG_LOOP(CODE, SUCCESS) do {                                        \
  12         struct lockref old;                                                     \
  13         BUILD_BUG_ON(sizeof(old) != 8);                                         \
  14         old.lock_count = READ_ONCE(lockref->lock_count);                        \
  15         while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) {     \
  16                 struct lockref new = old, prev = old;                           \
  17                 CODE                                                            \
  18                 old.lock_count = cmpxchg64_relaxed(&lockref->lock_count,        \
  19                                                    old.lock_count,              \
  20                                                    new.lock_count);             \
  21                 if (likely(old.lock_count == prev.lock_count)) {                \
  22                         SUCCESS;                                                \
  23                 }                                                               \
  24                 cpu_relax();                                                    \
  25         }                                                                       \
  26 } while (0)
  27
  28 #else
  29
  30 #define CMPXCHG_LOOP(CODE, SUCCESS) do { } while (0)
  31
  32 #endif
  33
  34 /**
  35  * lockref_get - Increments reference count unconditionally
  36  * @lockref: pointer to lockref structure
  37  *
  38  * This operation is only valid if you already hold a reference
  39  * to the object, so you know the count cannot be zero.
  40  */
  41 void lockref_get(struct lockref *lockref)
  42 {
  43         CMPXCHG_LOOP(
  44                 new.count++;
  45         ,
  46                 return;
  47         );
  48
  49         spin_lock(&lockref->lock);
  50         lockref->count++;
  51         spin_unlock(&lockref->lock);
  52 }
  53 EXPORT_SYMBOL(lockref_get);
  54
  55 /**
  56  * lockref_get_not_zero - Increments count unless the count is 0 or dead
  57  * @lockref: pointer to lockref structure
  58  * Return: 1 if count updated successfully or 0 if count was zero
  59  */
  60 int lockref_get_not_zero(struct lockref *lockref)
  61 {
  62         int retval;
  63
  64         CMPXCHG_LOOP(
  65                 new.count++;
  66                 if (old.count <= 0)
  67                         return 0;
  68         ,
  69                 return 1;
  70         );
  71
  72         spin_lock(&lockref->lock);
  73         retval = 0;
  74         if (lockref->count > 0) {
  75                 lockref->count++;
  76                 retval = 1;
  77         }
  78         spin_unlock(&lockref->lock);
  79         return retval;
  80 }
  81 EXPORT_SYMBOL(lockref_get_not_zero);
  82
  83 /**
  84  * lockref_put_not_zero - Decrements count unless count <= 1 before decrement
  85  * @lockref: pointer to lockref structure
  86  * Return: 1 if count updated successfully or 0 if count would become zero
  87  */
  88 int lockref_put_not_zero(struct lockref *lockref)
  89 {
  90         int retval;
  91
  92         CMPXCHG_LOOP(
  93                 new.count--;
  94                 if (old.count <= 1)
  95                         return 0;
  96         ,
  97                 return 1;
  98         );
  99
 100         spin_lock(&lockref->lock);
 101         retval = 0;
 102         if (lockref->count > 1) {
 103                 lockref->count--;
 104                 retval = 1;
 105         }
 106         spin_unlock(&lockref->lock);
 107         return retval;
 108 }
 109 EXPORT_SYMBOL(lockref_put_not_zero);
 110
 111 /**
 112  * lockref_get_or_lock - Increments count unless the count is 0 or dead
 113  * @lockref: pointer to lockref structure
 114  * Return: 1 if count updated successfully or 0 if count was zero
 115  * and we got the lock instead.
 116  */
 117 int lockref_get_or_lock(struct lockref *lockref)
 118 {
 119         CMPXCHG_LOOP(
 120                 new.count++;
 121                 if (old.count <= 0)
 122                         break;
 123         ,
 124                 return 1;
 125         );
 126
 127         spin_lock(&lockref->lock);
 128         if (lockref->count <= 0)
 129                 return 0;
 130         lockref->count++;
 131         spin_unlock(&lockref->lock);
 132         return 1;
 133 }
 134 EXPORT_SYMBOL(lockref_get_or_lock);
 135
 136 /**
 137  * lockref_put_return - Decrement reference count if possible
 138  * @lockref: pointer to lockref structure
 139  *
 140  * Decrement the reference count and return the new value.
 141  * If the lockref was dead or locked, return an error.
 142  */
 143 int lockref_put_return(struct lockref *lockref)
 144 {
 145         CMPXCHG_LOOP(
 146                 new.count--;
 147                 if (old.count <= 0)
 148                         return -1;
 149         ,
 150                 return new.count;
 151         );
 152         return -1;
 153 }
 154 EXPORT_SYMBOL(lockref_put_return);
 155
 156 /**
 157  * lockref_put_or_lock - decrements count unless count <= 1 before decrement
 158  * @lockref: pointer to lockref structure
 159  * Return: 1 if count updated successfully or 0 if count <= 1 and lock taken
 160  */
 161 int lockref_put_or_lock(struct lockref *lockref)
 162 {
 163         CMPXCHG_LOOP(
 164                 new.count--;
 165                 if (old.count <= 1)
 166                         break;
 167         ,
 168                 return 1;
 169         );
 170
 171         spin_lock(&lockref->lock);
 172         if (lockref->count <= 1)
 173                 return 0;
 174         lockref->count--;
 175         spin_unlock(&lockref->lock);
 176         return 1;
 177 }
 178 EXPORT_SYMBOL(lockref_put_or_lock);
 179
 180 /**
 181  * lockref_mark_dead - mark lockref dead
 182  * @lockref: pointer to lockref structure
 183  */
 184 void lockref_mark_dead(struct lockref *lockref)
 185 {
 186         assert_spin_locked(&lockref->lock);
 187         lockref->count = -128;
 188 }
 189 EXPORT_SYMBOL(lockref_mark_dead);
 190
 191 /**
 192  * lockref_get_not_dead - Increments count unless the ref is dead
 193  * @lockref: pointer to lockref structure
 194  * Return: 1 if count updated successfully or 0 if lockref was dead
 195  */
 196 int lockref_get_not_dead(struct lockref *lockref)
 197 {
 198         int retval;
 199
 200         CMPXCHG_LOOP(
 201                 new.count++;
 202                 if (old.count < 0)
 203                         return 0;
 204         ,
 205                 return 1;
 206         );
 207
 208         spin_lock(&lockref->lock);
 209         retval = 0;
 210         if (lockref->count >= 0) {
 211                 lockref->count++;
 212                 retval = 1;
 213         }
 214         spin_unlock(&lockref->lock);
 215         return retval;
 216 }
 217 EXPORT_SYMBOL(lockref_get_not_dead);