kernel/locking/rwsem.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* kernel/rwsem.c: R/W semaphores, public implementation
   3  *
   4  * Written by David Howells (dhowells@redhat.com).
   5  * Derived from asm-i386/semaphore.h
   6  */
   7
   8 #include <linux/types.h>
   9 #include <linux/kernel.h>
  10 #include <linux/sched.h>
  11 #include <linux/sched/debug.h>
  12 #include <linux/export.h>
  13 #include <linux/rwsem.h>
  14 #include <linux/atomic.h>
  15
  16 #include "rwsem.h"
  17
  18 /*
  19  * lock for reading
  20  */
  21 void __sched down_read(struct rw_semaphore *sem)
  22 {
  23         might_sleep();
  24         rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
  25
  26         LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
  27         rwsem_set_reader_owned(sem);
  28 }
  29
  30 EXPORT_SYMBOL(down_read);
  31
  32 int __sched down_read_killable(struct rw_semaphore *sem)
  33 {
  34         might_sleep();
  35         rwsem_acquire_read(&sem->dep_map, 0, 0, _RET_IP_);
  36
  37         if (LOCK_CONTENDED_RETURN(sem, __down_read_trylock, __down_read_killable)) {
  38                 rwsem_release(&sem->dep_map, 1, _RET_IP_);
  39                 return -EINTR;
  40         }
  41
  42         rwsem_set_reader_owned(sem);
  43         return 0;
  44 }
  45
  46 EXPORT_SYMBOL(down_read_killable);
  47
  48 /*
  49  * trylock for reading -- returns 1 if successful, 0 if contention
  50  */
  51 int down_read_trylock(struct rw_semaphore *sem)
  52 {
  53         int ret = __down_read_trylock(sem);
  54
  55         if (ret == 1) {
  56                 rwsem_acquire_read(&sem->dep_map, 0, 1, _RET_IP_);
  57                 rwsem_set_reader_owned(sem);
  58         }
  59         return ret;
  60 }
  61
  62 EXPORT_SYMBOL(down_read_trylock);
  63
  64 /*
  65  * lock for writing
  66  */
  67 void __sched down_write(struct rw_semaphore *sem)
  68 {
  69         might_sleep();
  70         rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
  71
  72         LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
  73         rwsem_set_owner(sem);
  74 }
  75
  76 EXPORT_SYMBOL(down_write);
  77
  78 /*
  79  * lock for writing
  80  */
  81 int __sched down_write_killable(struct rw_semaphore *sem)
  82 {
  83         might_sleep();
  84         rwsem_acquire(&sem->dep_map, 0, 0, _RET_IP_);
  85
  86         if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) {
  87                 rwsem_release(&sem->dep_map, 1, _RET_IP_);
  88                 return -EINTR;
  89         }
  90
  91         rwsem_set_owner(sem);
  92         return 0;
  93 }
  94
  95 EXPORT_SYMBOL(down_write_killable);
  96
  97 /*
  98  * trylock for writing -- returns 1 if successful, 0 if contention
  99  */
 100 int down_write_trylock(struct rw_semaphore *sem)
 101 {
 102         int ret = __down_write_trylock(sem);
 103
 104         if (ret == 1) {
 105                 rwsem_acquire(&sem->dep_map, 0, 1, _RET_IP_);
 106                 rwsem_set_owner(sem);
 107         }
 108
 109         return ret;
 110 }
 111
 112 EXPORT_SYMBOL(down_write_trylock);
 113
 114 /*
 115  * release a read lock
 116  */
 117 void up_read(struct rw_semaphore *sem)
 118 {
 119         rwsem_release(&sem->dep_map, 1, _RET_IP_);
 120         DEBUG_RWSEMS_WARN_ON(!((unsigned long)sem->owner & RWSEM_READER_OWNED));
 121
 122         rwsem_clear_reader_owned(sem);
 123         __up_read(sem);
 124 }
 125
 126 EXPORT_SYMBOL(up_read);
 127
 128 /*
 129  * release a write lock
 130  */
 131 void up_write(struct rw_semaphore *sem)
 132 {
 133         rwsem_release(&sem->dep_map, 1, _RET_IP_);
 134         DEBUG_RWSEMS_WARN_ON(sem->owner != current);
 135
 136         rwsem_clear_owner(sem);
 137         __up_write(sem);
 138 }
 139
 140 EXPORT_SYMBOL(up_write);
 141
 142 /*
 143  * downgrade write lock to read lock
 144  */
 145 void downgrade_write(struct rw_semaphore *sem)
 146 {
 147         lock_downgrade(&sem->dep_map, _RET_IP_);
 148         DEBUG_RWSEMS_WARN_ON(sem->owner != current);
 149
 150         rwsem_set_reader_owned(sem);
 151         __downgrade_write(sem);
 152 }
 153
 154 EXPORT_SYMBOL(downgrade_write);
 155
 156 #ifdef CONFIG_DEBUG_LOCK_ALLOC
 157
 158 void down_read_nested(struct rw_semaphore *sem, int subclass)
 159 {
 160         might_sleep();
 161         rwsem_acquire_read(&sem->dep_map, subclass, 0, _RET_IP_);
 162
 163         LOCK_CONTENDED(sem, __down_read_trylock, __down_read);
 164         rwsem_set_reader_owned(sem);
 165 }
 166
 167 EXPORT_SYMBOL(down_read_nested);
 168
 169 void _down_write_nest_lock(struct rw_semaphore *sem, struct lockdep_map *nest)
 170 {
 171         might_sleep();
 172         rwsem_acquire_nest(&sem->dep_map, 0, 0, nest, _RET_IP_);
 173
 174         LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
 175         rwsem_set_owner(sem);
 176 }
 177
 178 EXPORT_SYMBOL(_down_write_nest_lock);
 179
 180 void down_read_non_owner(struct rw_semaphore *sem)
 181 {
 182         might_sleep();
 183
 184         __down_read(sem);
 185         __rwsem_set_reader_owned(sem, NULL);
 186 }
 187
 188 EXPORT_SYMBOL(down_read_non_owner);
 189
 190 void down_write_nested(struct rw_semaphore *sem, int subclass)
 191 {
 192         might_sleep();
 193         rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
 194
 195         LOCK_CONTENDED(sem, __down_write_trylock, __down_write);
 196         rwsem_set_owner(sem);
 197 }
 198
 199 EXPORT_SYMBOL(down_write_nested);
 200
 201 int __sched down_write_killable_nested(struct rw_semaphore *sem, int subclass)
 202 {
 203         might_sleep();
 204         rwsem_acquire(&sem->dep_map, subclass, 0, _RET_IP_);
 205
 206         if (LOCK_CONTENDED_RETURN(sem, __down_write_trylock, __down_write_killable)) {
 207                 rwsem_release(&sem->dep_map, 1, _RET_IP_);
 208                 return -EINTR;
 209         }
 210
 211         rwsem_set_owner(sem);
 212         return 0;
 213 }
 214
 215 EXPORT_SYMBOL(down_write_killable_nested);
 216
 217 void up_read_non_owner(struct rw_semaphore *sem)
 218 {
 219         DEBUG_RWSEMS_WARN_ON(!((unsigned long)sem->owner & RWSEM_READER_OWNED));
 220         __up_read(sem);
 221 }
 222
 223 EXPORT_SYMBOL(up_read_non_owner);
 224
 225 #endif