1 /* SPDX-License-Identifier: GPL-2.0 */
5 * Linux wait queue related types and methods
7 #include <linux/list.h>
8 #include <linux/stddef.h>
9 #include <linux/spinlock.h>
11 #include <asm/current.h>
12 #include <uapi/linux/wait.h>
14 typedef struct wait_queue_entry wait_queue_entry_t;
16 typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
17 int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
19 /* wait_queue_entry::flags */
20 #define WQ_FLAG_EXCLUSIVE 0x01
21 #define WQ_FLAG_WOKEN 0x02
22 #define WQ_FLAG_BOOKMARK 0x04
25 * A single wait-queue entry structure:
27 struct wait_queue_entry {
30 wait_queue_func_t func;
31 struct list_head entry;
34 struct wait_queue_head {
36 struct list_head head;
38 typedef struct wait_queue_head wait_queue_head_t;
43 * Macros for declaration and initialisaton of the datatypes
46 #define __WAITQUEUE_INITIALIZER(name, tsk) { \
48 .func = default_wake_function, \
49 .entry = { NULL, NULL } }
51 #define DECLARE_WAITQUEUE(name, tsk) \
52 struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)
54 #define __WAIT_QUEUE_HEAD_INITIALIZER(name) { \
55 .lock = __SPIN_LOCK_UNLOCKED(name.lock), \
56 .head = { &(name).head, &(name).head } }
58 #define DECLARE_WAIT_QUEUE_HEAD(name) \
59 struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
61 extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);
63 #define init_waitqueue_head(wq_head) \
65 static struct lock_class_key __key; \
67 __init_waitqueue_head((wq_head), #wq_head, &__key); \
71 # define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
72 ({ init_waitqueue_head(&name); name; })
73 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
74 struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
76 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
79 static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
82 wq_entry->private = p;
83 wq_entry->func = default_wake_function;
87 init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
90 wq_entry->private = NULL;
91 wq_entry->func = func;
95 * waitqueue_active -- locklessly test for waiters on the queue
96 * @wq_head: the waitqueue to test for waiters
98 * returns true if the wait list is not empty
100 * NOTE: this function is lockless and requires care, incorrect usage _will_
101 * lead to sporadic and non-obvious failure.
103 * Use either while holding wait_queue_head::lock or when used for wakeups
104 * with an extra smp_mb() like::
106 * CPU0 - waker CPU1 - waiter
109 * @cond = true; prepare_to_wait(&wq_head, &wait, state);
110 * smp_mb(); // smp_mb() from set_current_state()
111 * if (waitqueue_active(wq_head)) if (@cond)
112 * wake_up(wq_head); break;
115 * finish_wait(&wq_head, &wait);
117 * Because without the explicit smp_mb() it's possible for the
118 * waitqueue_active() load to get hoisted over the @cond store such that we'll
119 * observe an empty wait list while the waiter might not observe @cond.
121 * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
122 * which (when the lock is uncontended) are of roughly equal cost.
124 static inline int waitqueue_active(struct wait_queue_head *wq_head)
126 return !list_empty(&wq_head->head);
130 * wq_has_single_sleeper - check if there is only one sleeper
131 * @wq_head: wait queue head
133 * Returns true of wq_head has only one sleeper on the list.
135 * Please refer to the comment for waitqueue_active.
137 static inline bool wq_has_single_sleeper(struct wait_queue_head *wq_head)
139 return list_is_singular(&wq_head->head);
143 * wq_has_sleeper - check if there are any waiting processes
144 * @wq_head: wait queue head
146 * Returns true if wq_head has waiting processes
148 * Please refer to the comment for waitqueue_active.
150 static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
153 * We need to be sure we are in sync with the
154 * add_wait_queue modifications to the wait queue.
156 * This memory barrier should be paired with one on the
160 return waitqueue_active(wq_head);
163 extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
164 extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
165 extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
167 static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
169 list_add(&wq_entry->entry, &wq_head->head);
173 * Used for wake-one threads:
176 __add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
178 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
179 __add_wait_queue(wq_head, wq_entry);
182 static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
184 list_add_tail(&wq_entry->entry, &wq_head->head);
188 __add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
190 wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
191 __add_wait_queue_entry_tail(wq_head, wq_entry);
195 __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
197 list_del(&wq_entry->entry);
200 void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
201 void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
202 void __wake_up_locked_key_bookmark(struct wait_queue_head *wq_head,
203 unsigned int mode, void *key, wait_queue_entry_t *bookmark);
204 void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
205 void __wake_up_locked_sync_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
206 void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
207 void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode);
209 #define wake_up(x) __wake_up(x, TASK_NORMAL, 1, NULL)
210 #define wake_up_nr(x, nr) __wake_up(x, TASK_NORMAL, nr, NULL)
211 #define wake_up_all(x) __wake_up(x, TASK_NORMAL, 0, NULL)
212 #define wake_up_locked(x) __wake_up_locked((x), TASK_NORMAL, 1)
213 #define wake_up_all_locked(x) __wake_up_locked((x), TASK_NORMAL, 0)
215 #define wake_up_interruptible(x) __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
216 #define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
217 #define wake_up_interruptible_all(x) __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
218 #define wake_up_interruptible_sync(x) __wake_up_sync((x), TASK_INTERRUPTIBLE)
221 * Wakeup macros to be used to report events to the targets.
223 #define poll_to_key(m) ((void *)(__force uintptr_t)(__poll_t)(m))
224 #define key_to_poll(m) ((__force __poll_t)(uintptr_t)(void *)(m))
225 #define wake_up_poll(x, m) \
226 __wake_up(x, TASK_NORMAL, 1, poll_to_key(m))
227 #define wake_up_locked_poll(x, m) \
228 __wake_up_locked_key((x), TASK_NORMAL, poll_to_key(m))
229 #define wake_up_interruptible_poll(x, m) \
230 __wake_up(x, TASK_INTERRUPTIBLE, 1, poll_to_key(m))
231 #define wake_up_interruptible_sync_poll(x, m) \
232 __wake_up_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
233 #define wake_up_interruptible_sync_poll_locked(x, m) \
234 __wake_up_locked_sync_key((x), TASK_INTERRUPTIBLE, poll_to_key(m))
236 #define ___wait_cond_timeout(condition) \
238 bool __cond = (condition); \
239 if (__cond && !__ret) \
244 #define ___wait_is_interruptible(state) \
245 (!__builtin_constant_p(state) || \
246 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE) \
248 extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);
251 * The below macro ___wait_event() has an explicit shadow of the __ret
252 * variable when used from the wait_event_*() macros.
254 * This is so that both can use the ___wait_cond_timeout() construct
255 * to wrap the condition.
257 * The type inconsistency of the wait_event_*() __ret variable is also
258 * on purpose; we use long where we can return timeout values and int
262 #define ___wait_event(wq_head, condition, state, exclusive, ret, cmd) \
265 struct wait_queue_entry __wq_entry; \
266 long __ret = ret; /* explicit shadow */ \
268 init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0); \
270 long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
275 if (___wait_is_interruptible(state) && __int) { \
282 finish_wait(&wq_head, &__wq_entry); \
286 #define __wait_event(wq_head, condition) \
287 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
291 * wait_event - sleep until a condition gets true
292 * @wq_head: the waitqueue to wait on
293 * @condition: a C expression for the event to wait for
295 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
296 * @condition evaluates to true. The @condition is checked each time
297 * the waitqueue @wq_head is woken up.
299 * wake_up() has to be called after changing any variable that could
300 * change the result of the wait condition.
302 #define wait_event(wq_head, condition) \
307 __wait_event(wq_head, condition); \
310 #define __io_wait_event(wq_head, condition) \
311 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
315 * io_wait_event() -- like wait_event() but with io_schedule()
317 #define io_wait_event(wq_head, condition) \
322 __io_wait_event(wq_head, condition); \
325 #define __wait_event_freezable(wq_head, condition) \
326 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
327 freezable_schedule())
330 * wait_event_freezable - sleep (or freeze) until a condition gets true
331 * @wq_head: the waitqueue to wait on
332 * @condition: a C expression for the event to wait for
334 * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
335 * to system load) until the @condition evaluates to true. The
336 * @condition is checked each time the waitqueue @wq_head is woken up.
338 * wake_up() has to be called after changing any variable that could
339 * change the result of the wait condition.
341 #define wait_event_freezable(wq_head, condition) \
346 __ret = __wait_event_freezable(wq_head, condition); \
350 #define __wait_event_timeout(wq_head, condition, timeout) \
351 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
352 TASK_UNINTERRUPTIBLE, 0, timeout, \
353 __ret = schedule_timeout(__ret))
356 * wait_event_timeout - sleep until a condition gets true or a timeout elapses
357 * @wq_head: the waitqueue to wait on
358 * @condition: a C expression for the event to wait for
359 * @timeout: timeout, in jiffies
361 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
362 * @condition evaluates to true. The @condition is checked each time
363 * the waitqueue @wq_head is woken up.
365 * wake_up() has to be called after changing any variable that could
366 * change the result of the wait condition.
369 * 0 if the @condition evaluated to %false after the @timeout elapsed,
370 * 1 if the @condition evaluated to %true after the @timeout elapsed,
371 * or the remaining jiffies (at least 1) if the @condition evaluated
372 * to %true before the @timeout elapsed.
374 #define wait_event_timeout(wq_head, condition, timeout) \
376 long __ret = timeout; \
378 if (!___wait_cond_timeout(condition)) \
379 __ret = __wait_event_timeout(wq_head, condition, timeout); \
383 #define __wait_event_freezable_timeout(wq_head, condition, timeout) \
384 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
385 TASK_INTERRUPTIBLE, 0, timeout, \
386 __ret = freezable_schedule_timeout(__ret))
389 * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
390 * increasing load and is freezable.
392 #define wait_event_freezable_timeout(wq_head, condition, timeout) \
394 long __ret = timeout; \
396 if (!___wait_cond_timeout(condition)) \
397 __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
401 #define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
402 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0, \
403 cmd1; schedule(); cmd2)
405 * Just like wait_event_cmd(), except it sets exclusive flag
407 #define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2) \
411 __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2); \
414 #define __wait_event_cmd(wq_head, condition, cmd1, cmd2) \
415 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
416 cmd1; schedule(); cmd2)
419 * wait_event_cmd - sleep until a condition gets true
420 * @wq_head: the waitqueue to wait on
421 * @condition: a C expression for the event to wait for
422 * @cmd1: the command will be executed before sleep
423 * @cmd2: the command will be executed after sleep
425 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
426 * @condition evaluates to true. The @condition is checked each time
427 * the waitqueue @wq_head is woken up.
429 * wake_up() has to be called after changing any variable that could
430 * change the result of the wait condition.
432 #define wait_event_cmd(wq_head, condition, cmd1, cmd2) \
436 __wait_event_cmd(wq_head, condition, cmd1, cmd2); \
439 #define __wait_event_interruptible(wq_head, condition) \
440 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
444 * wait_event_interruptible - sleep until a condition gets true
445 * @wq_head: the waitqueue to wait on
446 * @condition: a C expression for the event to wait for
448 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
449 * @condition evaluates to true or a signal is received.
450 * The @condition is checked each time the waitqueue @wq_head is woken up.
452 * wake_up() has to be called after changing any variable that could
453 * change the result of the wait condition.
455 * The function will return -ERESTARTSYS if it was interrupted by a
456 * signal and 0 if @condition evaluated to true.
458 #define wait_event_interruptible(wq_head, condition) \
463 __ret = __wait_event_interruptible(wq_head, condition); \
467 #define __wait_event_interruptible_timeout(wq_head, condition, timeout) \
468 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
469 TASK_INTERRUPTIBLE, 0, timeout, \
470 __ret = schedule_timeout(__ret))
473 * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
474 * @wq_head: the waitqueue to wait on
475 * @condition: a C expression for the event to wait for
476 * @timeout: timeout, in jiffies
478 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
479 * @condition evaluates to true or a signal is received.
480 * The @condition is checked each time the waitqueue @wq_head is woken up.
482 * wake_up() has to be called after changing any variable that could
483 * change the result of the wait condition.
486 * 0 if the @condition evaluated to %false after the @timeout elapsed,
487 * 1 if the @condition evaluated to %true after the @timeout elapsed,
488 * the remaining jiffies (at least 1) if the @condition evaluated
489 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
490 * interrupted by a signal.
492 #define wait_event_interruptible_timeout(wq_head, condition, timeout) \
494 long __ret = timeout; \
496 if (!___wait_cond_timeout(condition)) \
497 __ret = __wait_event_interruptible_timeout(wq_head, \
498 condition, timeout); \
502 #define __wait_event_hrtimeout(wq_head, condition, timeout, state) \
505 struct hrtimer_sleeper __t; \
507 hrtimer_init_sleeper_on_stack(&__t, CLOCK_MONOTONIC, \
509 if ((timeout) != KTIME_MAX) \
510 hrtimer_start_range_ns(&__t.timer, timeout, \
511 current->timer_slack_ns, \
514 __ret = ___wait_event(wq_head, condition, state, 0, 0, \
521 hrtimer_cancel(&__t.timer); \
522 destroy_hrtimer_on_stack(&__t.timer); \
527 * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
528 * @wq_head: the waitqueue to wait on
529 * @condition: a C expression for the event to wait for
530 * @timeout: timeout, as a ktime_t
532 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
533 * @condition evaluates to true or a signal is received.
534 * The @condition is checked each time the waitqueue @wq_head is woken up.
536 * wake_up() has to be called after changing any variable that could
537 * change the result of the wait condition.
539 * The function returns 0 if @condition became true, or -ETIME if the timeout
542 #define wait_event_hrtimeout(wq_head, condition, timeout) \
547 __ret = __wait_event_hrtimeout(wq_head, condition, timeout, \
548 TASK_UNINTERRUPTIBLE); \
553 * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
554 * @wq: the waitqueue to wait on
555 * @condition: a C expression for the event to wait for
556 * @timeout: timeout, as a ktime_t
558 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
559 * @condition evaluates to true or a signal is received.
560 * The @condition is checked each time the waitqueue @wq is woken up.
562 * wake_up() has to be called after changing any variable that could
563 * change the result of the wait condition.
565 * The function returns 0 if @condition became true, -ERESTARTSYS if it was
566 * interrupted by a signal, or -ETIME if the timeout elapsed.
568 #define wait_event_interruptible_hrtimeout(wq, condition, timeout) \
573 __ret = __wait_event_hrtimeout(wq, condition, timeout, \
574 TASK_INTERRUPTIBLE); \
578 #define __wait_event_interruptible_exclusive(wq, condition) \
579 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
582 #define wait_event_interruptible_exclusive(wq, condition) \
587 __ret = __wait_event_interruptible_exclusive(wq, condition); \
591 #define __wait_event_killable_exclusive(wq, condition) \
592 ___wait_event(wq, condition, TASK_KILLABLE, 1, 0, \
595 #define wait_event_killable_exclusive(wq, condition) \
600 __ret = __wait_event_killable_exclusive(wq, condition); \
605 #define __wait_event_freezable_exclusive(wq, condition) \
606 ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0, \
607 freezable_schedule())
609 #define wait_event_freezable_exclusive(wq, condition) \
614 __ret = __wait_event_freezable_exclusive(wq, condition); \
619 * wait_event_idle - wait for a condition without contributing to system load
620 * @wq_head: the waitqueue to wait on
621 * @condition: a C expression for the event to wait for
623 * The process is put to sleep (TASK_IDLE) until the
624 * @condition evaluates to true.
625 * The @condition is checked each time the waitqueue @wq_head is woken up.
627 * wake_up() has to be called after changing any variable that could
628 * change the result of the wait condition.
631 #define wait_event_idle(wq_head, condition) \
635 ___wait_event(wq_head, condition, TASK_IDLE, 0, 0, schedule()); \
639 * wait_event_idle_exclusive - wait for a condition with contributing to system load
640 * @wq_head: the waitqueue to wait on
641 * @condition: a C expression for the event to wait for
643 * The process is put to sleep (TASK_IDLE) until the
644 * @condition evaluates to true.
645 * The @condition is checked each time the waitqueue @wq_head is woken up.
647 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
648 * set thus if other processes wait on the same list, when this
649 * process is woken further processes are not considered.
651 * wake_up() has to be called after changing any variable that could
652 * change the result of the wait condition.
655 #define wait_event_idle_exclusive(wq_head, condition) \
659 ___wait_event(wq_head, condition, TASK_IDLE, 1, 0, schedule()); \
662 #define __wait_event_idle_timeout(wq_head, condition, timeout) \
663 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
664 TASK_IDLE, 0, timeout, \
665 __ret = schedule_timeout(__ret))
668 * wait_event_idle_timeout - sleep without load until a condition becomes true or a timeout elapses
669 * @wq_head: the waitqueue to wait on
670 * @condition: a C expression for the event to wait for
671 * @timeout: timeout, in jiffies
673 * The process is put to sleep (TASK_IDLE) until the
674 * @condition evaluates to true. The @condition is checked each time
675 * the waitqueue @wq_head is woken up.
677 * wake_up() has to be called after changing any variable that could
678 * change the result of the wait condition.
681 * 0 if the @condition evaluated to %false after the @timeout elapsed,
682 * 1 if the @condition evaluated to %true after the @timeout elapsed,
683 * or the remaining jiffies (at least 1) if the @condition evaluated
684 * to %true before the @timeout elapsed.
686 #define wait_event_idle_timeout(wq_head, condition, timeout) \
688 long __ret = timeout; \
690 if (!___wait_cond_timeout(condition)) \
691 __ret = __wait_event_idle_timeout(wq_head, condition, timeout); \
695 #define __wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
696 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
697 TASK_IDLE, 1, timeout, \
698 __ret = schedule_timeout(__ret))
701 * wait_event_idle_exclusive_timeout - sleep without load until a condition becomes true or a timeout elapses
702 * @wq_head: the waitqueue to wait on
703 * @condition: a C expression for the event to wait for
704 * @timeout: timeout, in jiffies
706 * The process is put to sleep (TASK_IDLE) until the
707 * @condition evaluates to true. The @condition is checked each time
708 * the waitqueue @wq_head is woken up.
710 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
711 * set thus if other processes wait on the same list, when this
712 * process is woken further processes are not considered.
714 * wake_up() has to be called after changing any variable that could
715 * change the result of the wait condition.
718 * 0 if the @condition evaluated to %false after the @timeout elapsed,
719 * 1 if the @condition evaluated to %true after the @timeout elapsed,
720 * or the remaining jiffies (at least 1) if the @condition evaluated
721 * to %true before the @timeout elapsed.
723 #define wait_event_idle_exclusive_timeout(wq_head, condition, timeout) \
725 long __ret = timeout; \
727 if (!___wait_cond_timeout(condition)) \
728 __ret = __wait_event_idle_exclusive_timeout(wq_head, condition, timeout);\
732 extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
733 extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);
735 #define __wait_event_interruptible_locked(wq, condition, exclusive, fn) \
738 DEFINE_WAIT(__wait); \
740 __wait.flags |= WQ_FLAG_EXCLUSIVE; \
742 __ret = fn(&(wq), &__wait); \
745 } while (!(condition)); \
746 __remove_wait_queue(&(wq), &__wait); \
747 __set_current_state(TASK_RUNNING); \
753 * wait_event_interruptible_locked - sleep until a condition gets true
754 * @wq: the waitqueue to wait on
755 * @condition: a C expression for the event to wait for
757 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
758 * @condition evaluates to true or a signal is received.
759 * The @condition is checked each time the waitqueue @wq is woken up.
761 * It must be called with wq.lock being held. This spinlock is
762 * unlocked while sleeping but @condition testing is done while lock
763 * is held and when this macro exits the lock is held.
765 * The lock is locked/unlocked using spin_lock()/spin_unlock()
766 * functions which must match the way they are locked/unlocked outside
769 * wake_up_locked() has to be called after changing any variable that could
770 * change the result of the wait condition.
772 * The function will return -ERESTARTSYS if it was interrupted by a
773 * signal and 0 if @condition evaluated to true.
775 #define wait_event_interruptible_locked(wq, condition) \
777 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
780 * wait_event_interruptible_locked_irq - sleep until a condition gets true
781 * @wq: the waitqueue to wait on
782 * @condition: a C expression for the event to wait for
784 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
785 * @condition evaluates to true or a signal is received.
786 * The @condition is checked each time the waitqueue @wq is woken up.
788 * It must be called with wq.lock being held. This spinlock is
789 * unlocked while sleeping but @condition testing is done while lock
790 * is held and when this macro exits the lock is held.
792 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
793 * functions which must match the way they are locked/unlocked outside
796 * wake_up_locked() has to be called after changing any variable that could
797 * change the result of the wait condition.
799 * The function will return -ERESTARTSYS if it was interrupted by a
800 * signal and 0 if @condition evaluated to true.
802 #define wait_event_interruptible_locked_irq(wq, condition) \
804 ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
807 * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
808 * @wq: the waitqueue to wait on
809 * @condition: a C expression for the event to wait for
811 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
812 * @condition evaluates to true or a signal is received.
813 * The @condition is checked each time the waitqueue @wq is woken up.
815 * It must be called with wq.lock being held. This spinlock is
816 * unlocked while sleeping but @condition testing is done while lock
817 * is held and when this macro exits the lock is held.
819 * The lock is locked/unlocked using spin_lock()/spin_unlock()
820 * functions which must match the way they are locked/unlocked outside
823 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
824 * set thus when other process waits process on the list if this
825 * process is awaken further processes are not considered.
827 * wake_up_locked() has to be called after changing any variable that could
828 * change the result of the wait condition.
830 * The function will return -ERESTARTSYS if it was interrupted by a
831 * signal and 0 if @condition evaluated to true.
833 #define wait_event_interruptible_exclusive_locked(wq, condition) \
835 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
838 * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
839 * @wq: the waitqueue to wait on
840 * @condition: a C expression for the event to wait for
842 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
843 * @condition evaluates to true or a signal is received.
844 * The @condition is checked each time the waitqueue @wq is woken up.
846 * It must be called with wq.lock being held. This spinlock is
847 * unlocked while sleeping but @condition testing is done while lock
848 * is held and when this macro exits the lock is held.
850 * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
851 * functions which must match the way they are locked/unlocked outside
854 * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
855 * set thus when other process waits process on the list if this
856 * process is awaken further processes are not considered.
858 * wake_up_locked() has to be called after changing any variable that could
859 * change the result of the wait condition.
861 * The function will return -ERESTARTSYS if it was interrupted by a
862 * signal and 0 if @condition evaluated to true.
864 #define wait_event_interruptible_exclusive_locked_irq(wq, condition) \
866 ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
869 #define __wait_event_killable(wq, condition) \
870 ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
873 * wait_event_killable - sleep until a condition gets true
874 * @wq_head: the waitqueue to wait on
875 * @condition: a C expression for the event to wait for
877 * The process is put to sleep (TASK_KILLABLE) until the
878 * @condition evaluates to true or a signal is received.
879 * The @condition is checked each time the waitqueue @wq_head is woken up.
881 * wake_up() has to be called after changing any variable that could
882 * change the result of the wait condition.
884 * The function will return -ERESTARTSYS if it was interrupted by a
885 * signal and 0 if @condition evaluated to true.
887 #define wait_event_killable(wq_head, condition) \
892 __ret = __wait_event_killable(wq_head, condition); \
896 #define __wait_event_killable_timeout(wq_head, condition, timeout) \
897 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
898 TASK_KILLABLE, 0, timeout, \
899 __ret = schedule_timeout(__ret))
902 * wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
903 * @wq_head: the waitqueue to wait on
904 * @condition: a C expression for the event to wait for
905 * @timeout: timeout, in jiffies
907 * The process is put to sleep (TASK_KILLABLE) until the
908 * @condition evaluates to true or a kill signal is received.
909 * The @condition is checked each time the waitqueue @wq_head is woken up.
911 * wake_up() has to be called after changing any variable that could
912 * change the result of the wait condition.
915 * 0 if the @condition evaluated to %false after the @timeout elapsed,
916 * 1 if the @condition evaluated to %true after the @timeout elapsed,
917 * the remaining jiffies (at least 1) if the @condition evaluated
918 * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
919 * interrupted by a kill signal.
921 * Only kill signals interrupt this process.
923 #define wait_event_killable_timeout(wq_head, condition, timeout) \
925 long __ret = timeout; \
927 if (!___wait_cond_timeout(condition)) \
928 __ret = __wait_event_killable_timeout(wq_head, \
929 condition, timeout); \
934 #define __wait_event_lock_irq(wq_head, condition, lock, cmd) \
935 (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0, \
936 spin_unlock_irq(&lock); \
939 spin_lock_irq(&lock))
942 * wait_event_lock_irq_cmd - sleep until a condition gets true. The
943 * condition is checked under the lock. This
944 * is expected to be called with the lock
946 * @wq_head: the waitqueue to wait on
947 * @condition: a C expression for the event to wait for
948 * @lock: a locked spinlock_t, which will be released before cmd
949 * and schedule() and reacquired afterwards.
950 * @cmd: a command which is invoked outside the critical section before
953 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
954 * @condition evaluates to true. The @condition is checked each time
955 * the waitqueue @wq_head is woken up.
957 * wake_up() has to be called after changing any variable that could
958 * change the result of the wait condition.
960 * This is supposed to be called while holding the lock. The lock is
961 * dropped before invoking the cmd and going to sleep and is reacquired
964 #define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd) \
968 __wait_event_lock_irq(wq_head, condition, lock, cmd); \
972 * wait_event_lock_irq - sleep until a condition gets true. The
973 * condition is checked under the lock. This
974 * is expected to be called with the lock
976 * @wq_head: the waitqueue to wait on
977 * @condition: a C expression for the event to wait for
978 * @lock: a locked spinlock_t, which will be released before schedule()
979 * and reacquired afterwards.
981 * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
982 * @condition evaluates to true. The @condition is checked each time
983 * the waitqueue @wq_head is woken up.
985 * wake_up() has to be called after changing any variable that could
986 * change the result of the wait condition.
988 * This is supposed to be called while holding the lock. The lock is
989 * dropped before going to sleep and is reacquired afterwards.
991 #define wait_event_lock_irq(wq_head, condition, lock) \
995 __wait_event_lock_irq(wq_head, condition, lock, ); \
999 #define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd) \
1000 ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0, \
1001 spin_unlock_irq(&lock); \
1004 spin_lock_irq(&lock))
1007 * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
1008 * The condition is checked under the lock. This is expected to
1009 * be called with the lock taken.
1010 * @wq_head: the waitqueue to wait on
1011 * @condition: a C expression for the event to wait for
1012 * @lock: a locked spinlock_t, which will be released before cmd and
1013 * schedule() and reacquired afterwards.
1014 * @cmd: a command which is invoked outside the critical section before
1017 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1018 * @condition evaluates to true or a signal is received. The @condition is
1019 * checked each time the waitqueue @wq_head is woken up.
1021 * wake_up() has to be called after changing any variable that could
1022 * change the result of the wait condition.
1024 * This is supposed to be called while holding the lock. The lock is
1025 * dropped before invoking the cmd and going to sleep and is reacquired
1028 * The macro will return -ERESTARTSYS if it was interrupted by a signal
1029 * and 0 if @condition evaluated to true.
1031 #define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd) \
1035 __ret = __wait_event_interruptible_lock_irq(wq_head, \
1036 condition, lock, cmd); \
1041 * wait_event_interruptible_lock_irq - sleep until a condition gets true.
1042 * The condition is checked under the lock. This is expected
1043 * to be called with the lock taken.
1044 * @wq_head: the waitqueue to wait on
1045 * @condition: a C expression for the event to wait for
1046 * @lock: a locked spinlock_t, which will be released before schedule()
1047 * and reacquired afterwards.
1049 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1050 * @condition evaluates to true or signal is received. The @condition is
1051 * checked each time the waitqueue @wq_head is woken up.
1053 * wake_up() has to be called after changing any variable that could
1054 * change the result of the wait condition.
1056 * This is supposed to be called while holding the lock. The lock is
1057 * dropped before going to sleep and is reacquired afterwards.
1059 * The macro will return -ERESTARTSYS if it was interrupted by a signal
1060 * and 0 if @condition evaluated to true.
1062 #define wait_event_interruptible_lock_irq(wq_head, condition, lock) \
1066 __ret = __wait_event_interruptible_lock_irq(wq_head, \
1067 condition, lock,); \
1071 #define __wait_event_lock_irq_timeout(wq_head, condition, lock, timeout, state) \
1072 ___wait_event(wq_head, ___wait_cond_timeout(condition), \
1073 state, 0, timeout, \
1074 spin_unlock_irq(&lock); \
1075 __ret = schedule_timeout(__ret); \
1076 spin_lock_irq(&lock));
1079 * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
1080 * true or a timeout elapses. The condition is checked under
1081 * the lock. This is expected to be called with the lock taken.
1082 * @wq_head: the waitqueue to wait on
1083 * @condition: a C expression for the event to wait for
1084 * @lock: a locked spinlock_t, which will be released before schedule()
1085 * and reacquired afterwards.
1086 * @timeout: timeout, in jiffies
1088 * The process is put to sleep (TASK_INTERRUPTIBLE) until the
1089 * @condition evaluates to true or signal is received. The @condition is
1090 * checked each time the waitqueue @wq_head is woken up.
1092 * wake_up() has to be called after changing any variable that could
1093 * change the result of the wait condition.
1095 * This is supposed to be called while holding the lock. The lock is
1096 * dropped before going to sleep and is reacquired afterwards.
1098 * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
1099 * was interrupted by a signal, and the remaining jiffies otherwise
1100 * if the condition evaluated to true before the timeout elapsed.
1102 #define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock, \
1105 long __ret = timeout; \
1106 if (!___wait_cond_timeout(condition)) \
1107 __ret = __wait_event_lock_irq_timeout( \
1108 wq_head, condition, lock, timeout, \
1109 TASK_INTERRUPTIBLE); \
1113 #define wait_event_lock_irq_timeout(wq_head, condition, lock, timeout) \
1115 long __ret = timeout; \
1116 if (!___wait_cond_timeout(condition)) \
1117 __ret = __wait_event_lock_irq_timeout( \
1118 wq_head, condition, lock, timeout, \
1119 TASK_UNINTERRUPTIBLE); \
1124 * Waitqueues which are removed from the waitqueue_head at wakeup time
1126 void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1127 void prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1128 long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
1129 void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
1130 long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
1131 int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
1132 int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
1134 #define DEFINE_WAIT_FUNC(name, function) \
1135 struct wait_queue_entry name = { \
1136 .private = current, \
1138 .entry = LIST_HEAD_INIT((name).entry), \
1141 #define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
1143 #define init_wait(wait) \
1145 (wait)->private = current; \
1146 (wait)->func = autoremove_wake_function; \
1147 INIT_LIST_HEAD(&(wait)->entry); \
1148 (wait)->flags = 0; \
1151 #endif /* _LINUX_WAIT_H */