include/linux/wait.h

   1 #ifndef _LINUX_WAIT_H
   2 #define _LINUX_WAIT_H
   3 /*
   4  * Linux wait queue related types and methods
   5  */
   6 #include <linux/list.h>
   7 #include <linux/stddef.h>
   8 #include <linux/spinlock.h>
   9
  10 #include <asm/current.h>
  11 #include <uapi/linux/wait.h>
  12
  13 typedef struct wait_queue_entry wait_queue_entry_t;
  14
  15 typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
  16 int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
  17
  18 /* wait_queue_entry::flags */
  19 #define WQ_FLAG_EXCLUSIVE       0x01
  20 #define WQ_FLAG_WOKEN           0x02
  21 #define WQ_FLAG_BOOKMARK        0x04
  22
  23 /*
  24  * A single wait-queue entry structure:
  25  */
  26 struct wait_queue_entry {
  27         unsigned int            flags;
  28         void                    *private;
  29         wait_queue_func_t       func;
  30         struct list_head        entry;
  31 };
  32
  33 struct wait_queue_head {
  34         spinlock_t              lock;
  35         struct list_head        head;
  36 };
  37 typedef struct wait_queue_head wait_queue_head_t;
  38
  39 struct task_struct;
  40
  41 /*
  42  * Macros for declaration and initialisaton of the datatypes
  43  */
  44
  45 #define __WAITQUEUE_INITIALIZER(name, tsk) {                                    \
  46         .private        = tsk,                                                  \
  47         .func           = default_wake_function,                                \
  48         .entry          = { NULL, NULL } }
  49
  50 #define DECLARE_WAITQUEUE(name, tsk)                                            \
  51         struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)
  52
  53 #define __WAIT_QUEUE_HEAD_INITIALIZER(name) {                                   \
  54         .lock           = __SPIN_LOCK_UNLOCKED(name.lock),                      \
  55         .head           = { &(name).head, &(name).head } }
  56
  57 #define DECLARE_WAIT_QUEUE_HEAD(name) \
  58         struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
  59
  60 extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);
  61
  62 #define init_waitqueue_head(wq_head)                                            \
  63         do {                                                                    \
  64                 static struct lock_class_key __key;                             \
  65                                                                                 \
  66                 __init_waitqueue_head((wq_head), #wq_head, &__key);             \
  67         } while (0)
  68
  69 #ifdef CONFIG_LOCKDEP
  70 # define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
  71         ({ init_waitqueue_head(&name); name; })
  72 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
  73         struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
  74 #else
  75 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
  76 #endif
  77
  78 static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
  79 {
  80         wq_entry->flags         = 0;
  81         wq_entry->private       = p;
  82         wq_entry->func          = default_wake_function;
  83 }
  84
  85 static inline void
  86 init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
  87 {
  88         wq_entry->flags         = 0;
  89         wq_entry->private       = NULL;
  90         wq_entry->func          = func;
  91 }
  92
  93 /**
  94  * waitqueue_active -- locklessly test for waiters on the queue
  95  * @wq_head: the waitqueue to test for waiters
  96  *
  97  * returns true if the wait list is not empty
  98  *
  99  * NOTE: this function is lockless and requires care, incorrect usage _will_
 100  * lead to sporadic and non-obvious failure.
 101  *
 102  * Use either while holding wait_queue_head::lock or when used for wakeups
 103  * with an extra smp_mb() like:
 104  *
 105  *      CPU0 - waker                    CPU1 - waiter
 106  *
 107  *                                      for (;;) {
 108  *      @cond = true;                     prepare_to_wait(&wq_head, &wait, state);
 109  *      smp_mb();                         // smp_mb() from set_current_state()
 110  *      if (waitqueue_active(wq_head))         if (@cond)
 111  *        wake_up(wq_head);                      break;
 112  *                                        schedule();
 113  *                                      }
 114  *                                      finish_wait(&wq_head, &wait);
 115  *
 116  * Because without the explicit smp_mb() it's possible for the
 117  * waitqueue_active() load to get hoisted over the @cond store such that we'll
 118  * observe an empty wait list while the waiter might not observe @cond.
 119  *
 120  * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
 121  * which (when the lock is uncontended) are of roughly equal cost.
 122  */
 123 static inline int waitqueue_active(struct wait_queue_head *wq_head)
 124 {
 125         return !list_empty(&wq_head->head);
 126 }
 127
 128 /**
 129  * wq_has_sleeper - check if there are any waiting processes
 130  * @wq_head: wait queue head
 131  *
 132  * Returns true if wq_head has waiting processes
 133  *
 134  * Please refer to the comment for waitqueue_active.
 135  */
 136 static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
 137 {
 138         /*
 139          * We need to be sure we are in sync with the
 140          * add_wait_queue modifications to the wait queue.
 141          *
 142          * This memory barrier should be paired with one on the
 143          * waiting side.
 144          */
 145         smp_mb();
 146         return waitqueue_active(wq_head);
 147 }
 148
 149 extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
 150 extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
 151 extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
 152
 153 static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 154 {
 155         list_add(&wq_entry->entry, &wq_head->head);
 156 }
 157
 158 /*
 159  * Used for wake-one threads:
 160  */
 161 static inline void
 162 __add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 163 {
 164         wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
 165         __add_wait_queue(wq_head, wq_entry);
 166 }
 167
 168 static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 169 {
 170         list_add_tail(&wq_entry->entry, &wq_head->head);
 171 }
 172
 173 static inline void
 174 __add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 175 {
 176         wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
 177         __add_wait_queue_entry_tail(wq_head, wq_entry);
 178 }
 179
 180 static inline void
 181 __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
 182 {
 183         list_del(&wq_entry->entry);
 184 }
 185
 186 void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
 187 void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
 188 void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
 189 void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
 190 void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode, int nr);
 191
 192 #define wake_up(x)                      __wake_up(x, TASK_NORMAL, 1, NULL)
 193 #define wake_up_nr(x, nr)               __wake_up(x, TASK_NORMAL, nr, NULL)
 194 #define wake_up_all(x)                  __wake_up(x, TASK_NORMAL, 0, NULL)
 195 #define wake_up_locked(x)               __wake_up_locked((x), TASK_NORMAL, 1)
 196 #define wake_up_all_locked(x)           __wake_up_locked((x), TASK_NORMAL, 0)
 197
 198 #define wake_up_interruptible(x)        __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
 199 #define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
 200 #define wake_up_interruptible_all(x)    __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
 201 #define wake_up_interruptible_sync(x)   __wake_up_sync((x), TASK_INTERRUPTIBLE, 1)
 202
 203 /*
 204  * Wakeup macros to be used to report events to the targets.
 205  */
 206 #define wake_up_poll(x, m)                                                      \
 207         __wake_up(x, TASK_NORMAL, 1, (void *) (m))
 208 #define wake_up_locked_poll(x, m)                                               \
 209         __wake_up_locked_key((x), TASK_NORMAL, (void *) (m))
 210 #define wake_up_interruptible_poll(x, m)                                        \
 211         __wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m))
 212 #define wake_up_interruptible_sync_poll(x, m)                                   \
 213         __wake_up_sync_key((x), TASK_INTERRUPTIBLE, 1, (void *) (m))
 214
 215 #define ___wait_cond_timeout(condition)                                         \
 216 ({                                                                              \
 217         bool __cond = (condition);                                              \
 218         if (__cond && !__ret)                                                   \
 219                 __ret = 1;                                                      \
 220         __cond || !__ret;                                                       \
 221 })
 222
 223 #define ___wait_is_interruptible(state)                                         \
 224         (!__builtin_constant_p(state) ||                                        \
 225                 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE)          \
 226
 227 extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);
 228
 229 /*
 230  * The below macro ___wait_event() has an explicit shadow of the __ret
 231  * variable when used from the wait_event_*() macros.
 232  *
 233  * This is so that both can use the ___wait_cond_timeout() construct
 234  * to wrap the condition.
 235  *
 236  * The type inconsistency of the wait_event_*() __ret variable is also
 237  * on purpose; we use long where we can return timeout values and int
 238  * otherwise.
 239  */
 240
 241 #define ___wait_event(wq_head, condition, state, exclusive, ret, cmd)           \
 242 ({                                                                              \
 243         __label__ __out;                                                        \
 244         struct wait_queue_entry __wq_entry;                                     \
 245         long __ret = ret;       /* explicit shadow */                           \
 246                                                                                 \
 247         init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0);        \
 248         for (;;) {                                                              \
 249                 long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
 250                                                                                 \
 251                 if (condition)                                                  \
 252                         break;                                                  \
 253                                                                                 \
 254                 if (___wait_is_interruptible(state) && __int) {                 \
 255                         __ret = __int;                                          \
 256                         goto __out;                                             \
 257                 }                                                               \
 258                                                                                 \
 259                 cmd;                                                            \
 260         }                                                                       \
 261         finish_wait(&wq_head, &__wq_entry);                                     \
 262 __out:  __ret;                                                                  \
 263 })
 264
 265 #define __wait_event(wq_head, condition)                                        \
 266         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
 267                             schedule())
 268
 269 /**
 270  * wait_event - sleep until a condition gets true
 271  * @wq_head: the waitqueue to wait on
 272  * @condition: a C expression for the event to wait for
 273  *
 274  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 275  * @condition evaluates to true. The @condition is checked each time
 276  * the waitqueue @wq_head is woken up.
 277  *
 278  * wake_up() has to be called after changing any variable that could
 279  * change the result of the wait condition.
 280  */
 281 #define wait_event(wq_head, condition)                                          \
 282 do {                                                                            \
 283         might_sleep();                                                          \
 284         if (condition)                                                          \
 285                 break;                                                          \
 286         __wait_event(wq_head, condition);                                       \
 287 } while (0)
 288
 289 #define __io_wait_event(wq_head, condition)                                     \
 290         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
 291                             io_schedule())
 292
 293 /*
 294  * io_wait_event() -- like wait_event() but with io_schedule()
 295  */
 296 #define io_wait_event(wq_head, condition)                                       \
 297 do {                                                                            \
 298         might_sleep();                                                          \
 299         if (condition)                                                          \
 300                 break;                                                          \
 301         __io_wait_event(wq_head, condition);                                    \
 302 } while (0)
 303
 304 #define __wait_event_freezable(wq_head, condition)                              \
 305         ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,             \
 306                             schedule(); try_to_freeze())
 307
 308 /**
 309  * wait_event_freezable - sleep (or freeze) until a condition gets true
 310  * @wq_head: the waitqueue to wait on
 311  * @condition: a C expression for the event to wait for
 312  *
 313  * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
 314  * to system load) until the @condition evaluates to true. The
 315  * @condition is checked each time the waitqueue @wq_head is woken up.
 316  *
 317  * wake_up() has to be called after changing any variable that could
 318  * change the result of the wait condition.
 319  */
 320 #define wait_event_freezable(wq_head, condition)                                \
 321 ({                                                                              \
 322         int __ret = 0;                                                          \
 323         might_sleep();                                                          \
 324         if (!(condition))                                                       \
 325                 __ret = __wait_event_freezable(wq_head, condition);             \
 326         __ret;                                                                  \
 327 })
 328
 329 #define __wait_event_timeout(wq_head, condition, timeout)                       \
 330         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 331                       TASK_UNINTERRUPTIBLE, 0, timeout,                         \
 332                       __ret = schedule_timeout(__ret))
 333
 334 /**
 335  * wait_event_timeout - sleep until a condition gets true or a timeout elapses
 336  * @wq_head: the waitqueue to wait on
 337  * @condition: a C expression for the event to wait for
 338  * @timeout: timeout, in jiffies
 339  *
 340  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 341  * @condition evaluates to true. The @condition is checked each time
 342  * the waitqueue @wq_head is woken up.
 343  *
 344  * wake_up() has to be called after changing any variable that could
 345  * change the result of the wait condition.
 346  *
 347  * Returns:
 348  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 349  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 350  * or the remaining jiffies (at least 1) if the @condition evaluated
 351  * to %true before the @timeout elapsed.
 352  */
 353 #define wait_event_timeout(wq_head, condition, timeout)                         \
 354 ({                                                                              \
 355         long __ret = timeout;                                                   \
 356         might_sleep();                                                          \
 357         if (!___wait_cond_timeout(condition))                                   \
 358                 __ret = __wait_event_timeout(wq_head, condition, timeout);      \
 359         __ret;                                                                  \
 360 })
 361
 362 #define __wait_event_freezable_timeout(wq_head, condition, timeout)             \
 363         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 364                       TASK_INTERRUPTIBLE, 0, timeout,                           \
 365                       __ret = schedule_timeout(__ret); try_to_freeze())
 366
 367 /*
 368  * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
 369  * increasing load and is freezable.
 370  */
 371 #define wait_event_freezable_timeout(wq_head, condition, timeout)               \
 372 ({                                                                              \
 373         long __ret = timeout;                                                   \
 374         might_sleep();                                                          \
 375         if (!___wait_cond_timeout(condition))                                   \
 376                 __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
 377         __ret;                                                                  \
 378 })
 379
 380 #define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)              \
 381         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0,     \
 382                             cmd1; schedule(); cmd2)
 383 /*
 384  * Just like wait_event_cmd(), except it sets exclusive flag
 385  */
 386 #define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)                \
 387 do {                                                                            \
 388         if (condition)                                                          \
 389                 break;                                                          \
 390         __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2);             \
 391 } while (0)
 392
 393 #define __wait_event_cmd(wq_head, condition, cmd1, cmd2)                        \
 394         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
 395                             cmd1; schedule(); cmd2)
 396
 397 /**
 398  * wait_event_cmd - sleep until a condition gets true
 399  * @wq_head: the waitqueue to wait on
 400  * @condition: a C expression for the event to wait for
 401  * @cmd1: the command will be executed before sleep
 402  * @cmd2: the command will be executed after sleep
 403  *
 404  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 405  * @condition evaluates to true. The @condition is checked each time
 406  * the waitqueue @wq_head is woken up.
 407  *
 408  * wake_up() has to be called after changing any variable that could
 409  * change the result of the wait condition.
 410  */
 411 #define wait_event_cmd(wq_head, condition, cmd1, cmd2)                          \
 412 do {                                                                            \
 413         if (condition)                                                          \
 414                 break;                                                          \
 415         __wait_event_cmd(wq_head, condition, cmd1, cmd2);                       \
 416 } while (0)
 417
 418 #define __wait_event_interruptible(wq_head, condition)                          \
 419         ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,             \
 420                       schedule())
 421
 422 /**
 423  * wait_event_interruptible - sleep until a condition gets true
 424  * @wq_head: the waitqueue to wait on
 425  * @condition: a C expression for the event to wait for
 426  *
 427  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 428  * @condition evaluates to true or a signal is received.
 429  * The @condition is checked each time the waitqueue @wq_head is woken up.
 430  *
 431  * wake_up() has to be called after changing any variable that could
 432  * change the result of the wait condition.
 433  *
 434  * The function will return -ERESTARTSYS if it was interrupted by a
 435  * signal and 0 if @condition evaluated to true.
 436  */
 437 #define wait_event_interruptible(wq_head, condition)                            \
 438 ({                                                                              \
 439         int __ret = 0;                                                          \
 440         might_sleep();                                                          \
 441         if (!(condition))                                                       \
 442                 __ret = __wait_event_interruptible(wq_head, condition);         \
 443         __ret;                                                                  \
 444 })
 445
 446 #define __wait_event_interruptible_timeout(wq_head, condition, timeout)         \
 447         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 448                       TASK_INTERRUPTIBLE, 0, timeout,                           \
 449                       __ret = schedule_timeout(__ret))
 450
 451 /**
 452  * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
 453  * @wq_head: the waitqueue to wait on
 454  * @condition: a C expression for the event to wait for
 455  * @timeout: timeout, in jiffies
 456  *
 457  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 458  * @condition evaluates to true or a signal is received.
 459  * The @condition is checked each time the waitqueue @wq_head is woken up.
 460  *
 461  * wake_up() has to be called after changing any variable that could
 462  * change the result of the wait condition.
 463  *
 464  * Returns:
 465  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 466  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 467  * the remaining jiffies (at least 1) if the @condition evaluated
 468  * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 469  * interrupted by a signal.
 470  */
 471 #define wait_event_interruptible_timeout(wq_head, condition, timeout)           \
 472 ({                                                                              \
 473         long __ret = timeout;                                                   \
 474         might_sleep();                                                          \
 475         if (!___wait_cond_timeout(condition))                                   \
 476                 __ret = __wait_event_interruptible_timeout(wq_head,             \
 477                                                 condition, timeout);            \
 478         __ret;                                                                  \
 479 })
 480
 481 #define __wait_event_hrtimeout(wq_head, condition, timeout, state)              \
 482 ({                                                                              \
 483         int __ret = 0;                                                          \
 484         struct hrtimer_sleeper __t;                                             \
 485                                                                                 \
 486         hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);   \
 487         hrtimer_init_sleeper(&__t, current);                                    \
 488         if ((timeout) != KTIME_MAX)                                             \
 489                 hrtimer_start_range_ns(&__t.timer, timeout,                     \
 490                                        current->timer_slack_ns,                 \
 491                                        HRTIMER_MODE_REL);                       \
 492                                                                                 \
 493         __ret = ___wait_event(wq_head, condition, state, 0, 0,                  \
 494                 if (!__t.task) {                                                \
 495                         __ret = -ETIME;                                         \
 496                         break;                                                  \
 497                 }                                                               \
 498                 schedule());                                                    \
 499                                                                                 \
 500         hrtimer_cancel(&__t.timer);                                             \
 501         destroy_hrtimer_on_stack(&__t.timer);                                   \
 502         __ret;                                                                  \
 503 })
 504
 505 /**
 506  * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
 507  * @wq_head: the waitqueue to wait on
 508  * @condition: a C expression for the event to wait for
 509  * @timeout: timeout, as a ktime_t
 510  *
 511  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 512  * @condition evaluates to true or a signal is received.
 513  * The @condition is checked each time the waitqueue @wq_head is woken up.
 514  *
 515  * wake_up() has to be called after changing any variable that could
 516  * change the result of the wait condition.
 517  *
 518  * The function returns 0 if @condition became true, or -ETIME if the timeout
 519  * elapsed.
 520  */
 521 #define wait_event_hrtimeout(wq_head, condition, timeout)                       \
 522 ({                                                                              \
 523         int __ret = 0;                                                          \
 524         might_sleep();                                                          \
 525         if (!(condition))                                                       \
 526                 __ret = __wait_event_hrtimeout(wq_head, condition, timeout,     \
 527                                                TASK_UNINTERRUPTIBLE);           \
 528         __ret;                                                                  \
 529 })
 530
 531 /**
 532  * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
 533  * @wq: the waitqueue to wait on
 534  * @condition: a C expression for the event to wait for
 535  * @timeout: timeout, as a ktime_t
 536  *
 537  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 538  * @condition evaluates to true or a signal is received.
 539  * The @condition is checked each time the waitqueue @wq is woken up.
 540  *
 541  * wake_up() has to be called after changing any variable that could
 542  * change the result of the wait condition.
 543  *
 544  * The function returns 0 if @condition became true, -ERESTARTSYS if it was
 545  * interrupted by a signal, or -ETIME if the timeout elapsed.
 546  */
 547 #define wait_event_interruptible_hrtimeout(wq, condition, timeout)              \
 548 ({                                                                              \
 549         long __ret = 0;                                                         \
 550         might_sleep();                                                          \
 551         if (!(condition))                                                       \
 552                 __ret = __wait_event_hrtimeout(wq, condition, timeout,          \
 553                                                TASK_INTERRUPTIBLE);             \
 554         __ret;                                                                  \
 555 })
 556
 557 #define __wait_event_interruptible_exclusive(wq, condition)                     \
 558         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,                  \
 559                       schedule())
 560
 561 #define wait_event_interruptible_exclusive(wq, condition)                       \
 562 ({                                                                              \
 563         int __ret = 0;                                                          \
 564         might_sleep();                                                          \
 565         if (!(condition))                                                       \
 566                 __ret = __wait_event_interruptible_exclusive(wq, condition);    \
 567         __ret;                                                                  \
 568 })
 569
 570 #define __wait_event_killable_exclusive(wq, condition)                          \
 571         ___wait_event(wq, condition, TASK_KILLABLE, 1, 0,                       \
 572                       schedule())
 573
 574 #define wait_event_killable_exclusive(wq, condition)                            \
 575 ({                                                                              \
 576         int __ret = 0;                                                          \
 577         might_sleep();                                                          \
 578         if (!(condition))                                                       \
 579                 __ret = __wait_event_killable_exclusive(wq, condition);         \
 580         __ret;                                                                  \
 581 })
 582
 583
 584 #define __wait_event_freezable_exclusive(wq, condition)                         \
 585         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,                  \
 586                         schedule(); try_to_freeze())
 587
 588 #define wait_event_freezable_exclusive(wq, condition)                           \
 589 ({                                                                              \
 590         int __ret = 0;                                                          \
 591         might_sleep();                                                          \
 592         if (!(condition))                                                       \
 593                 __ret = __wait_event_freezable_exclusive(wq, condition);        \
 594         __ret;                                                                  \
 595 })
 596
 597 extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
 598 extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);
 599
 600 #define __wait_event_interruptible_locked(wq, condition, exclusive, fn)         \
 601 ({                                                                              \
 602         int __ret;                                                              \
 603         DEFINE_WAIT(__wait);                                                    \
 604         if (exclusive)                                                          \
 605                 __wait.flags |= WQ_FLAG_EXCLUSIVE;                              \
 606         do {                                                                    \
 607                 __ret = fn(&(wq), &__wait);                                     \
 608                 if (__ret)                                                      \
 609                         break;                                                  \
 610         } while (!(condition));                                                 \
 611         __remove_wait_queue(&(wq), &__wait);                                    \
 612         __set_current_state(TASK_RUNNING);                                      \
 613         __ret;                                                                  \
 614 })
 615
 616
 617 /**
 618  * wait_event_interruptible_locked - sleep until a condition gets true
 619  * @wq: the waitqueue to wait on
 620  * @condition: a C expression for the event to wait for
 621  *
 622  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 623  * @condition evaluates to true or a signal is received.
 624  * The @condition is checked each time the waitqueue @wq is woken up.
 625  *
 626  * It must be called with wq.lock being held.  This spinlock is
 627  * unlocked while sleeping but @condition testing is done while lock
 628  * is held and when this macro exits the lock is held.
 629  *
 630  * The lock is locked/unlocked using spin_lock()/spin_unlock()
 631  * functions which must match the way they are locked/unlocked outside
 632  * of this macro.
 633  *
 634  * wake_up_locked() has to be called after changing any variable that could
 635  * change the result of the wait condition.
 636  *
 637  * The function will return -ERESTARTSYS if it was interrupted by a
 638  * signal and 0 if @condition evaluated to true.
 639  */
 640 #define wait_event_interruptible_locked(wq, condition)                          \
 641         ((condition)                                                            \
 642          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
 643
 644 /**
 645  * wait_event_interruptible_locked_irq - sleep until a condition gets true
 646  * @wq: the waitqueue to wait on
 647  * @condition: a C expression for the event to wait for
 648  *
 649  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 650  * @condition evaluates to true or a signal is received.
 651  * The @condition is checked each time the waitqueue @wq is woken up.
 652  *
 653  * It must be called with wq.lock being held.  This spinlock is
 654  * unlocked while sleeping but @condition testing is done while lock
 655  * is held and when this macro exits the lock is held.
 656  *
 657  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 658  * functions which must match the way they are locked/unlocked outside
 659  * of this macro.
 660  *
 661  * wake_up_locked() has to be called after changing any variable that could
 662  * change the result of the wait condition.
 663  *
 664  * The function will return -ERESTARTSYS if it was interrupted by a
 665  * signal and 0 if @condition evaluated to true.
 666  */
 667 #define wait_event_interruptible_locked_irq(wq, condition)                      \
 668         ((condition)                                                            \
 669          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
 670
 671 /**
 672  * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
 673  * @wq: the waitqueue to wait on
 674  * @condition: a C expression for the event to wait for
 675  *
 676  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 677  * @condition evaluates to true or a signal is received.
 678  * The @condition is checked each time the waitqueue @wq is woken up.
 679  *
 680  * It must be called with wq.lock being held.  This spinlock is
 681  * unlocked while sleeping but @condition testing is done while lock
 682  * is held and when this macro exits the lock is held.
 683  *
 684  * The lock is locked/unlocked using spin_lock()/spin_unlock()
 685  * functions which must match the way they are locked/unlocked outside
 686  * of this macro.
 687  *
 688  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 689  * set thus when other process waits process on the list if this
 690  * process is awaken further processes are not considered.
 691  *
 692  * wake_up_locked() has to be called after changing any variable that could
 693  * change the result of the wait condition.
 694  *
 695  * The function will return -ERESTARTSYS if it was interrupted by a
 696  * signal and 0 if @condition evaluated to true.
 697  */
 698 #define wait_event_interruptible_exclusive_locked(wq, condition)                \
 699         ((condition)                                                            \
 700          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
 701
 702 /**
 703  * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
 704  * @wq: the waitqueue to wait on
 705  * @condition: a C expression for the event to wait for
 706  *
 707  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 708  * @condition evaluates to true or a signal is received.
 709  * The @condition is checked each time the waitqueue @wq is woken up.
 710  *
 711  * It must be called with wq.lock being held.  This spinlock is
 712  * unlocked while sleeping but @condition testing is done while lock
 713  * is held and when this macro exits the lock is held.
 714  *
 715  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
 716  * functions which must match the way they are locked/unlocked outside
 717  * of this macro.
 718  *
 719  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
 720  * set thus when other process waits process on the list if this
 721  * process is awaken further processes are not considered.
 722  *
 723  * wake_up_locked() has to be called after changing any variable that could
 724  * change the result of the wait condition.
 725  *
 726  * The function will return -ERESTARTSYS if it was interrupted by a
 727  * signal and 0 if @condition evaluated to true.
 728  */
 729 #define wait_event_interruptible_exclusive_locked_irq(wq, condition)            \
 730         ((condition)                                                            \
 731          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
 732
 733
 734 #define __wait_event_killable(wq, condition)                                    \
 735         ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
 736
 737 /**
 738  * wait_event_killable - sleep until a condition gets true
 739  * @wq_head: the waitqueue to wait on
 740  * @condition: a C expression for the event to wait for
 741  *
 742  * The process is put to sleep (TASK_KILLABLE) until the
 743  * @condition evaluates to true or a signal is received.
 744  * The @condition is checked each time the waitqueue @wq_head is woken up.
 745  *
 746  * wake_up() has to be called after changing any variable that could
 747  * change the result of the wait condition.
 748  *
 749  * The function will return -ERESTARTSYS if it was interrupted by a
 750  * signal and 0 if @condition evaluated to true.
 751  */
 752 #define wait_event_killable(wq_head, condition)                                 \
 753 ({                                                                              \
 754         int __ret = 0;                                                          \
 755         might_sleep();                                                          \
 756         if (!(condition))                                                       \
 757                 __ret = __wait_event_killable(wq_head, condition);              \
 758         __ret;                                                                  \
 759 })
 760
 761 #define __wait_event_killable_timeout(wq_head, condition, timeout)              \
 762         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 763                       TASK_KILLABLE, 0, timeout,                                \
 764                       __ret = schedule_timeout(__ret))
 765
 766 /**
 767  * wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
 768  * @wq_head: the waitqueue to wait on
 769  * @condition: a C expression for the event to wait for
 770  * @timeout: timeout, in jiffies
 771  *
 772  * The process is put to sleep (TASK_KILLABLE) until the
 773  * @condition evaluates to true or a kill signal is received.
 774  * The @condition is checked each time the waitqueue @wq_head is woken up.
 775  *
 776  * wake_up() has to be called after changing any variable that could
 777  * change the result of the wait condition.
 778  *
 779  * Returns:
 780  * 0 if the @condition evaluated to %false after the @timeout elapsed,
 781  * 1 if the @condition evaluated to %true after the @timeout elapsed,
 782  * the remaining jiffies (at least 1) if the @condition evaluated
 783  * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
 784  * interrupted by a kill signal.
 785  *
 786  * Only kill signals interrupt this process.
 787  */
 788 #define wait_event_killable_timeout(wq_head, condition, timeout)                \
 789 ({                                                                              \
 790         long __ret = timeout;                                                   \
 791         might_sleep();                                                          \
 792         if (!___wait_cond_timeout(condition))                                   \
 793                 __ret = __wait_event_killable_timeout(wq_head,                  \
 794                                                 condition, timeout);            \
 795         __ret;                                                                  \
 796 })
 797
 798
 799 #define __wait_event_lock_irq(wq_head, condition, lock, cmd)                    \
 800         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
 801                             spin_unlock_irq(&lock);                             \
 802                             cmd;                                                \
 803                             schedule();                                         \
 804                             spin_lock_irq(&lock))
 805
 806 /**
 807  * wait_event_lock_irq_cmd - sleep until a condition gets true. The
 808  *                           condition is checked under the lock. This
 809  *                           is expected to be called with the lock
 810  *                           taken.
 811  * @wq_head: the waitqueue to wait on
 812  * @condition: a C expression for the event to wait for
 813  * @lock: a locked spinlock_t, which will be released before cmd
 814  *        and schedule() and reacquired afterwards.
 815  * @cmd: a command which is invoked outside the critical section before
 816  *       sleep
 817  *
 818  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 819  * @condition evaluates to true. The @condition is checked each time
 820  * the waitqueue @wq_head is woken up.
 821  *
 822  * wake_up() has to be called after changing any variable that could
 823  * change the result of the wait condition.
 824  *
 825  * This is supposed to be called while holding the lock. The lock is
 826  * dropped before invoking the cmd and going to sleep and is reacquired
 827  * afterwards.
 828  */
 829 #define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd)                  \
 830 do {                                                                            \
 831         if (condition)                                                          \
 832                 break;                                                          \
 833         __wait_event_lock_irq(wq_head, condition, lock, cmd);                   \
 834 } while (0)
 835
 836 /**
 837  * wait_event_lock_irq - sleep until a condition gets true. The
 838  *                       condition is checked under the lock. This
 839  *                       is expected to be called with the lock
 840  *                       taken.
 841  * @wq_head: the waitqueue to wait on
 842  * @condition: a C expression for the event to wait for
 843  * @lock: a locked spinlock_t, which will be released before schedule()
 844  *        and reacquired afterwards.
 845  *
 846  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
 847  * @condition evaluates to true. The @condition is checked each time
 848  * the waitqueue @wq_head is woken up.
 849  *
 850  * wake_up() has to be called after changing any variable that could
 851  * change the result of the wait condition.
 852  *
 853  * This is supposed to be called while holding the lock. The lock is
 854  * dropped before going to sleep and is reacquired afterwards.
 855  */
 856 #define wait_event_lock_irq(wq_head, condition, lock)                           \
 857 do {                                                                            \
 858         if (condition)                                                          \
 859                 break;                                                          \
 860         __wait_event_lock_irq(wq_head, condition, lock, );                      \
 861 } while (0)
 862
 863
 864 #define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd)      \
 865         ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,             \
 866                       spin_unlock_irq(&lock);                                   \
 867                       cmd;                                                      \
 868                       schedule();                                               \
 869                       spin_lock_irq(&lock))
 870
 871 /**
 872  * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
 873  *              The condition is checked under the lock. This is expected to
 874  *              be called with the lock taken.
 875  * @wq_head: the waitqueue to wait on
 876  * @condition: a C expression for the event to wait for
 877  * @lock: a locked spinlock_t, which will be released before cmd and
 878  *        schedule() and reacquired afterwards.
 879  * @cmd: a command which is invoked outside the critical section before
 880  *       sleep
 881  *
 882  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 883  * @condition evaluates to true or a signal is received. The @condition is
 884  * checked each time the waitqueue @wq_head is woken up.
 885  *
 886  * wake_up() has to be called after changing any variable that could
 887  * change the result of the wait condition.
 888  *
 889  * This is supposed to be called while holding the lock. The lock is
 890  * dropped before invoking the cmd and going to sleep and is reacquired
 891  * afterwards.
 892  *
 893  * The macro will return -ERESTARTSYS if it was interrupted by a signal
 894  * and 0 if @condition evaluated to true.
 895  */
 896 #define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd)    \
 897 ({                                                                              \
 898         int __ret = 0;                                                          \
 899         if (!(condition))                                                       \
 900                 __ret = __wait_event_interruptible_lock_irq(wq_head,            \
 901                                                 condition, lock, cmd);          \
 902         __ret;                                                                  \
 903 })
 904
 905 /**
 906  * wait_event_interruptible_lock_irq - sleep until a condition gets true.
 907  *              The condition is checked under the lock. This is expected
 908  *              to be called with the lock taken.
 909  * @wq_head: the waitqueue to wait on
 910  * @condition: a C expression for the event to wait for
 911  * @lock: a locked spinlock_t, which will be released before schedule()
 912  *        and reacquired afterwards.
 913  *
 914  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 915  * @condition evaluates to true or signal is received. The @condition is
 916  * checked each time the waitqueue @wq_head is woken up.
 917  *
 918  * wake_up() has to be called after changing any variable that could
 919  * change the result of the wait condition.
 920  *
 921  * This is supposed to be called while holding the lock. The lock is
 922  * dropped before going to sleep and is reacquired afterwards.
 923  *
 924  * The macro will return -ERESTARTSYS if it was interrupted by a signal
 925  * and 0 if @condition evaluated to true.
 926  */
 927 #define wait_event_interruptible_lock_irq(wq_head, condition, lock)             \
 928 ({                                                                              \
 929         int __ret = 0;                                                          \
 930         if (!(condition))                                                       \
 931                 __ret = __wait_event_interruptible_lock_irq(wq_head,            \
 932                                                 condition, lock,);              \
 933         __ret;                                                                  \
 934 })
 935
 936 #define __wait_event_interruptible_lock_irq_timeout(wq_head, condition,         \
 937                                                     lock, timeout)              \
 938         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
 939                       TASK_INTERRUPTIBLE, 0, timeout,                           \
 940                       spin_unlock_irq(&lock);                                   \
 941                       __ret = schedule_timeout(__ret);                          \
 942                       spin_lock_irq(&lock));
 943
 944 /**
 945  * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
 946  *              true or a timeout elapses. The condition is checked under
 947  *              the lock. This is expected to be called with the lock taken.
 948  * @wq_head: the waitqueue to wait on
 949  * @condition: a C expression for the event to wait for
 950  * @lock: a locked spinlock_t, which will be released before schedule()
 951  *        and reacquired afterwards.
 952  * @timeout: timeout, in jiffies
 953  *
 954  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
 955  * @condition evaluates to true or signal is received. The @condition is
 956  * checked each time the waitqueue @wq_head is woken up.
 957  *
 958  * wake_up() has to be called after changing any variable that could
 959  * change the result of the wait condition.
 960  *
 961  * This is supposed to be called while holding the lock. The lock is
 962  * dropped before going to sleep and is reacquired afterwards.
 963  *
 964  * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
 965  * was interrupted by a signal, and the remaining jiffies otherwise
 966  * if the condition evaluated to true before the timeout elapsed.
 967  */
 968 #define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock,     \
 969                                                   timeout)                      \
 970 ({                                                                              \
 971         long __ret = timeout;                                                   \
 972         if (!___wait_cond_timeout(condition))                                   \
 973                 __ret = __wait_event_interruptible_lock_irq_timeout(            \
 974                                         wq_head, condition, lock, timeout);     \
 975         __ret;                                                                  \
 976 })
 977
 978 /*
 979  * Waitqueues which are removed from the waitqueue_head at wakeup time
 980  */
 981 void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
 982 void prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
 983 long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
 984 void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
 985 long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
 986 int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
 987 int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
 988
 989 #define DEFINE_WAIT_FUNC(name, function)                                        \
 990         struct wait_queue_entry name = {                                        \
 991                 .private        = current,                                      \
 992                 .func           = function,                                     \
 993                 .entry          = LIST_HEAD_INIT((name).entry),                 \
 994         }
 995
 996 #define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
 997
 998 #define init_wait(wait)                                                         \
 999         do {                                                                    \
1000                 (wait)->private = current;                                      \
1001                 (wait)->func = autoremove_wake_function;                        \
1002                 INIT_LIST_HEAD(&(wait)->entry);                                 \
1003                 (wait)->flags = 0;                                              \
1004         } while (0)
1005
1006 #endif /* _LINUX_WAIT_H */