1 /* rwsem-spinlock.c: R/W semaphores: contention handling functions for
2 * generic spinlock implementation
4 * Copyright (c) 2001 David Howells (dhowells@redhat.com).
5 * - Derived partially from idea by Andrea Arcangeli <andrea@suse.de>
6 * - Derived also from comments by Linus
8 #include <linux/rwsem.h>
9 #include <linux/sched/signal.h>
10 #include <linux/sched/debug.h>
11 #include <linux/export.h>
13 enum rwsem_waiter_type {
14 RWSEM_WAITING_FOR_WRITE,
15 RWSEM_WAITING_FOR_READ
19 struct list_head list;
20 struct task_struct *task;
21 enum rwsem_waiter_type type;
24 int rwsem_is_locked(struct rw_semaphore *sem)
29 if (raw_spin_trylock_irqsave(&sem->wait_lock, flags)) {
30 ret = (sem->count != 0);
31 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
35 EXPORT_SYMBOL(rwsem_is_locked);
38 * initialise the semaphore
40 void __init_rwsem(struct rw_semaphore *sem, const char *name,
41 struct lock_class_key *key)
43 #ifdef CONFIG_DEBUG_LOCK_ALLOC
45 * Make sure we are not reinitializing a held semaphore:
47 debug_check_no_locks_freed((void *)sem, sizeof(*sem));
48 lockdep_init_map(&sem->dep_map, name, key, 0);
51 raw_spin_lock_init(&sem->wait_lock);
52 INIT_LIST_HEAD(&sem->wait_list);
54 EXPORT_SYMBOL(__init_rwsem);
57 * handle the lock release when processes blocked on it that can now run
58 * - if we come here, then:
59 * - the 'active count' _reached_ zero
60 * - the 'waiting count' is non-zero
61 * - the spinlock must be held by the caller
62 * - woken process blocks are discarded from the list after having task zeroed
63 * - writers are only woken if wakewrite is non-zero
65 static inline struct rw_semaphore *
66 __rwsem_do_wake(struct rw_semaphore *sem, int wakewrite)
68 struct rwsem_waiter *waiter;
69 struct task_struct *tsk;
72 waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
74 if (waiter->type == RWSEM_WAITING_FOR_WRITE) {
76 /* Wake up a writer. Note that we do not grant it the
77 * lock - it will have to acquire it when it runs. */
78 wake_up_process(waiter->task);
82 /* grant an infinite number of read locks to the front of the queue */
85 struct list_head *next = waiter->list.next;
87 list_del(&waiter->list);
90 * Make sure we do not wakeup the next reader before
91 * setting the nil condition to grant the next reader;
92 * otherwise we could miss the wakeup on the other
93 * side and end up sleeping again. See the pairing
94 * in rwsem_down_read_failed().
101 if (next == &sem->wait_list)
103 waiter = list_entry(next, struct rwsem_waiter, list);
104 } while (waiter->type != RWSEM_WAITING_FOR_WRITE);
113 * wake a single writer
115 static inline struct rw_semaphore *
116 __rwsem_wake_one_writer(struct rw_semaphore *sem)
118 struct rwsem_waiter *waiter;
120 waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
121 wake_up_process(waiter->task);
127 * get a read lock on the semaphore
129 int __sched __down_read_common(struct rw_semaphore *sem, int state)
131 struct rwsem_waiter waiter;
134 raw_spin_lock_irqsave(&sem->wait_lock, flags);
136 if (sem->count >= 0 && list_empty(&sem->wait_list)) {
139 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
143 /* set up my own style of waitqueue */
144 waiter.task = current;
145 waiter.type = RWSEM_WAITING_FOR_READ;
146 get_task_struct(current);
148 list_add_tail(&waiter.list, &sem->wait_list);
150 /* wait to be given the lock */
154 if (signal_pending_state(state, current))
156 set_current_state(state);
157 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
159 raw_spin_lock_irqsave(&sem->wait_lock, flags);
162 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
168 * We didn't take the lock, so that there is a writer, which
169 * is owner or the first waiter of the sem. If it's a waiter,
170 * it will be woken by current owner. Not need to wake anybody.
172 list_del(&waiter.list);
173 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
177 void __sched __down_read(struct rw_semaphore *sem)
179 __down_read_common(sem, TASK_UNINTERRUPTIBLE);
182 int __sched __down_read_killable(struct rw_semaphore *sem)
184 return __down_read_common(sem, TASK_KILLABLE);
188 * trylock for reading -- returns 1 if successful, 0 if contention
190 int __down_read_trylock(struct rw_semaphore *sem)
196 raw_spin_lock_irqsave(&sem->wait_lock, flags);
198 if (sem->count >= 0 && list_empty(&sem->wait_list)) {
204 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
210 * get a write lock on the semaphore
212 int __sched __down_write_common(struct rw_semaphore *sem, int state)
214 struct rwsem_waiter waiter;
218 raw_spin_lock_irqsave(&sem->wait_lock, flags);
220 /* set up my own style of waitqueue */
221 waiter.task = current;
222 waiter.type = RWSEM_WAITING_FOR_WRITE;
223 list_add_tail(&waiter.list, &sem->wait_list);
225 /* wait for someone to release the lock */
228 * That is the key to support write lock stealing: allows the
229 * task already on CPU to get the lock soon rather than put
230 * itself into sleep and waiting for system woke it or someone
231 * else in the head of the wait list up.
235 if (signal_pending_state(state, current))
238 set_current_state(state);
239 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
241 raw_spin_lock_irqsave(&sem->wait_lock, flags);
245 list_del(&waiter.list);
247 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
252 list_del(&waiter.list);
253 if (!list_empty(&sem->wait_list) && sem->count >= 0)
254 __rwsem_do_wake(sem, 0);
255 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
260 void __sched __down_write(struct rw_semaphore *sem)
262 __down_write_common(sem, TASK_UNINTERRUPTIBLE);
265 int __sched __down_write_killable(struct rw_semaphore *sem)
267 return __down_write_common(sem, TASK_KILLABLE);
271 * trylock for writing -- returns 1 if successful, 0 if contention
273 int __down_write_trylock(struct rw_semaphore *sem)
278 raw_spin_lock_irqsave(&sem->wait_lock, flags);
280 if (sem->count == 0) {
286 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
292 * release a read lock on the semaphore
294 void __up_read(struct rw_semaphore *sem)
298 raw_spin_lock_irqsave(&sem->wait_lock, flags);
300 if (--sem->count == 0 && !list_empty(&sem->wait_list))
301 sem = __rwsem_wake_one_writer(sem);
303 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
307 * release a write lock on the semaphore
309 void __up_write(struct rw_semaphore *sem)
313 raw_spin_lock_irqsave(&sem->wait_lock, flags);
316 if (!list_empty(&sem->wait_list))
317 sem = __rwsem_do_wake(sem, 1);
319 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
323 * downgrade a write lock into a read lock
324 * - just wake up any readers at the front of the queue
326 void __downgrade_write(struct rw_semaphore *sem)
330 raw_spin_lock_irqsave(&sem->wait_lock, flags);
333 if (!list_empty(&sem->wait_list))
334 sem = __rwsem_do_wake(sem, 0);
336 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);