sched/wait: Break up long wake list walk
[sfrench/cifs-2.6.git] / include / linux / wait.h
1 #ifndef _LINUX_WAIT_H
2 #define _LINUX_WAIT_H
3 /*
4  * Linux wait queue related types and methods
5  */
6 #include <linux/list.h>
7 #include <linux/stddef.h>
8 #include <linux/spinlock.h>
9
10 #include <asm/current.h>
11 #include <uapi/linux/wait.h>
12
13 typedef struct wait_queue_entry wait_queue_entry_t;
14
15 typedef int (*wait_queue_func_t)(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
16 int default_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int flags, void *key);
17
18 /* wait_queue_entry::flags */
19 #define WQ_FLAG_EXCLUSIVE       0x01
20 #define WQ_FLAG_WOKEN           0x02
21 #define WQ_FLAG_BOOKMARK        0x04
22
23 /*
24  * A single wait-queue entry structure:
25  */
26 struct wait_queue_entry {
27         unsigned int            flags;
28         void                    *private;
29         wait_queue_func_t       func;
30         struct list_head        entry;
31 };
32
33 struct wait_queue_head {
34         spinlock_t              lock;
35         struct list_head        head;
36 };
37 typedef struct wait_queue_head wait_queue_head_t;
38
39 struct task_struct;
40
41 /*
42  * Macros for declaration and initialisaton of the datatypes
43  */
44
45 #define __WAITQUEUE_INITIALIZER(name, tsk) {                                    \
46         .private        = tsk,                                                  \
47         .func           = default_wake_function,                                \
48         .entry          = { NULL, NULL } }
49
50 #define DECLARE_WAITQUEUE(name, tsk)                                            \
51         struct wait_queue_entry name = __WAITQUEUE_INITIALIZER(name, tsk)
52
53 #define __WAIT_QUEUE_HEAD_INITIALIZER(name) {                                   \
54         .lock           = __SPIN_LOCK_UNLOCKED(name.lock),                      \
55         .head           = { &(name).head, &(name).head } }
56
57 #define DECLARE_WAIT_QUEUE_HEAD(name) \
58         struct wait_queue_head name = __WAIT_QUEUE_HEAD_INITIALIZER(name)
59
60 extern void __init_waitqueue_head(struct wait_queue_head *wq_head, const char *name, struct lock_class_key *);
61
62 #define init_waitqueue_head(wq_head)                                            \
63         do {                                                                    \
64                 static struct lock_class_key __key;                             \
65                                                                                 \
66                 __init_waitqueue_head((wq_head), #wq_head, &__key);             \
67         } while (0)
68
69 #ifdef CONFIG_LOCKDEP
70 # define __WAIT_QUEUE_HEAD_INIT_ONSTACK(name) \
71         ({ init_waitqueue_head(&name); name; })
72 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) \
73         struct wait_queue_head name = __WAIT_QUEUE_HEAD_INIT_ONSTACK(name)
74 #else
75 # define DECLARE_WAIT_QUEUE_HEAD_ONSTACK(name) DECLARE_WAIT_QUEUE_HEAD(name)
76 #endif
77
78 static inline void init_waitqueue_entry(struct wait_queue_entry *wq_entry, struct task_struct *p)
79 {
80         wq_entry->flags         = 0;
81         wq_entry->private       = p;
82         wq_entry->func          = default_wake_function;
83 }
84
85 static inline void
86 init_waitqueue_func_entry(struct wait_queue_entry *wq_entry, wait_queue_func_t func)
87 {
88         wq_entry->flags         = 0;
89         wq_entry->private       = NULL;
90         wq_entry->func          = func;
91 }
92
93 /**
94  * waitqueue_active -- locklessly test for waiters on the queue
95  * @wq_head: the waitqueue to test for waiters
96  *
97  * returns true if the wait list is not empty
98  *
99  * NOTE: this function is lockless and requires care, incorrect usage _will_
100  * lead to sporadic and non-obvious failure.
101  *
102  * Use either while holding wait_queue_head::lock or when used for wakeups
103  * with an extra smp_mb() like:
104  *
105  *      CPU0 - waker                    CPU1 - waiter
106  *
107  *                                      for (;;) {
108  *      @cond = true;                     prepare_to_wait(&wq_head, &wait, state);
109  *      smp_mb();                         // smp_mb() from set_current_state()
110  *      if (waitqueue_active(wq_head))         if (@cond)
111  *        wake_up(wq_head);                      break;
112  *                                        schedule();
113  *                                      }
114  *                                      finish_wait(&wq_head, &wait);
115  *
116  * Because without the explicit smp_mb() it's possible for the
117  * waitqueue_active() load to get hoisted over the @cond store such that we'll
118  * observe an empty wait list while the waiter might not observe @cond.
119  *
120  * Also note that this 'optimization' trades a spin_lock() for an smp_mb(),
121  * which (when the lock is uncontended) are of roughly equal cost.
122  */
123 static inline int waitqueue_active(struct wait_queue_head *wq_head)
124 {
125         return !list_empty(&wq_head->head);
126 }
127
128 /**
129  * wq_has_sleeper - check if there are any waiting processes
130  * @wq_head: wait queue head
131  *
132  * Returns true if wq_head has waiting processes
133  *
134  * Please refer to the comment for waitqueue_active.
135  */
136 static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
137 {
138         /*
139          * We need to be sure we are in sync with the
140          * add_wait_queue modifications to the wait queue.
141          *
142          * This memory barrier should be paired with one on the
143          * waiting side.
144          */
145         smp_mb();
146         return waitqueue_active(wq_head);
147 }
148
149 extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
150 extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
151 extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
152
153 static inline void __add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
154 {
155         list_add(&wq_entry->entry, &wq_head->head);
156 }
157
158 /*
159  * Used for wake-one threads:
160  */
161 static inline void
162 __add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
163 {
164         wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
165         __add_wait_queue(wq_head, wq_entry);
166 }
167
168 static inline void __add_wait_queue_entry_tail(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
169 {
170         list_add_tail(&wq_entry->entry, &wq_head->head);
171 }
172
173 static inline void
174 __add_wait_queue_entry_tail_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
175 {
176         wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
177         __add_wait_queue_entry_tail(wq_head, wq_entry);
178 }
179
180 static inline void
181 __remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
182 {
183         list_del(&wq_entry->entry);
184 }
185
186 void __wake_up(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
187 void __wake_up_locked_key(struct wait_queue_head *wq_head, unsigned int mode, void *key);
188 void __wake_up_sync_key(struct wait_queue_head *wq_head, unsigned int mode, int nr, void *key);
189 void __wake_up_locked(struct wait_queue_head *wq_head, unsigned int mode, int nr);
190 void __wake_up_sync(struct wait_queue_head *wq_head, unsigned int mode, int nr);
191
192 #define wake_up(x)                      __wake_up(x, TASK_NORMAL, 1, NULL)
193 #define wake_up_nr(x, nr)               __wake_up(x, TASK_NORMAL, nr, NULL)
194 #define wake_up_all(x)                  __wake_up(x, TASK_NORMAL, 0, NULL)
195 #define wake_up_locked(x)               __wake_up_locked((x), TASK_NORMAL, 1)
196 #define wake_up_all_locked(x)           __wake_up_locked((x), TASK_NORMAL, 0)
197
198 #define wake_up_interruptible(x)        __wake_up(x, TASK_INTERRUPTIBLE, 1, NULL)
199 #define wake_up_interruptible_nr(x, nr) __wake_up(x, TASK_INTERRUPTIBLE, nr, NULL)
200 #define wake_up_interruptible_all(x)    __wake_up(x, TASK_INTERRUPTIBLE, 0, NULL)
201 #define wake_up_interruptible_sync(x)   __wake_up_sync((x), TASK_INTERRUPTIBLE, 1)
202
203 /*
204  * Wakeup macros to be used to report events to the targets.
205  */
206 #define wake_up_poll(x, m)                                                      \
207         __wake_up(x, TASK_NORMAL, 1, (void *) (m))
208 #define wake_up_locked_poll(x, m)                                               \
209         __wake_up_locked_key((x), TASK_NORMAL, (void *) (m))
210 #define wake_up_interruptible_poll(x, m)                                        \
211         __wake_up(x, TASK_INTERRUPTIBLE, 1, (void *) (m))
212 #define wake_up_interruptible_sync_poll(x, m)                                   \
213         __wake_up_sync_key((x), TASK_INTERRUPTIBLE, 1, (void *) (m))
214
215 #define ___wait_cond_timeout(condition)                                         \
216 ({                                                                              \
217         bool __cond = (condition);                                              \
218         if (__cond && !__ret)                                                   \
219                 __ret = 1;                                                      \
220         __cond || !__ret;                                                       \
221 })
222
223 #define ___wait_is_interruptible(state)                                         \
224         (!__builtin_constant_p(state) ||                                        \
225                 state == TASK_INTERRUPTIBLE || state == TASK_KILLABLE)          \
226
227 extern void init_wait_entry(struct wait_queue_entry *wq_entry, int flags);
228
229 /*
230  * The below macro ___wait_event() has an explicit shadow of the __ret
231  * variable when used from the wait_event_*() macros.
232  *
233  * This is so that both can use the ___wait_cond_timeout() construct
234  * to wrap the condition.
235  *
236  * The type inconsistency of the wait_event_*() __ret variable is also
237  * on purpose; we use long where we can return timeout values and int
238  * otherwise.
239  */
240
241 #define ___wait_event(wq_head, condition, state, exclusive, ret, cmd)           \
242 ({                                                                              \
243         __label__ __out;                                                        \
244         struct wait_queue_entry __wq_entry;                                     \
245         long __ret = ret;       /* explicit shadow */                           \
246                                                                                 \
247         init_wait_entry(&__wq_entry, exclusive ? WQ_FLAG_EXCLUSIVE : 0);        \
248         for (;;) {                                                              \
249                 long __int = prepare_to_wait_event(&wq_head, &__wq_entry, state);\
250                                                                                 \
251                 if (condition)                                                  \
252                         break;                                                  \
253                                                                                 \
254                 if (___wait_is_interruptible(state) && __int) {                 \
255                         __ret = __int;                                          \
256                         goto __out;                                             \
257                 }                                                               \
258                                                                                 \
259                 cmd;                                                            \
260         }                                                                       \
261         finish_wait(&wq_head, &__wq_entry);                                     \
262 __out:  __ret;                                                                  \
263 })
264
265 #define __wait_event(wq_head, condition)                                        \
266         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
267                             schedule())
268
269 /**
270  * wait_event - sleep until a condition gets true
271  * @wq_head: the waitqueue to wait on
272  * @condition: a C expression for the event to wait for
273  *
274  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
275  * @condition evaluates to true. The @condition is checked each time
276  * the waitqueue @wq_head is woken up.
277  *
278  * wake_up() has to be called after changing any variable that could
279  * change the result of the wait condition.
280  */
281 #define wait_event(wq_head, condition)                                          \
282 do {                                                                            \
283         might_sleep();                                                          \
284         if (condition)                                                          \
285                 break;                                                          \
286         __wait_event(wq_head, condition);                                       \
287 } while (0)
288
289 #define __io_wait_event(wq_head, condition)                                     \
290         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
291                             io_schedule())
292
293 /*
294  * io_wait_event() -- like wait_event() but with io_schedule()
295  */
296 #define io_wait_event(wq_head, condition)                                       \
297 do {                                                                            \
298         might_sleep();                                                          \
299         if (condition)                                                          \
300                 break;                                                          \
301         __io_wait_event(wq_head, condition);                                    \
302 } while (0)
303
304 #define __wait_event_freezable(wq_head, condition)                              \
305         ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,             \
306                             schedule(); try_to_freeze())
307
308 /**
309  * wait_event_freezable - sleep (or freeze) until a condition gets true
310  * @wq_head: the waitqueue to wait on
311  * @condition: a C expression for the event to wait for
312  *
313  * The process is put to sleep (TASK_INTERRUPTIBLE -- so as not to contribute
314  * to system load) until the @condition evaluates to true. The
315  * @condition is checked each time the waitqueue @wq_head is woken up.
316  *
317  * wake_up() has to be called after changing any variable that could
318  * change the result of the wait condition.
319  */
320 #define wait_event_freezable(wq_head, condition)                                \
321 ({                                                                              \
322         int __ret = 0;                                                          \
323         might_sleep();                                                          \
324         if (!(condition))                                                       \
325                 __ret = __wait_event_freezable(wq_head, condition);             \
326         __ret;                                                                  \
327 })
328
329 #define __wait_event_timeout(wq_head, condition, timeout)                       \
330         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
331                       TASK_UNINTERRUPTIBLE, 0, timeout,                         \
332                       __ret = schedule_timeout(__ret))
333
334 /**
335  * wait_event_timeout - sleep until a condition gets true or a timeout elapses
336  * @wq_head: the waitqueue to wait on
337  * @condition: a C expression for the event to wait for
338  * @timeout: timeout, in jiffies
339  *
340  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
341  * @condition evaluates to true. The @condition is checked each time
342  * the waitqueue @wq_head is woken up.
343  *
344  * wake_up() has to be called after changing any variable that could
345  * change the result of the wait condition.
346  *
347  * Returns:
348  * 0 if the @condition evaluated to %false after the @timeout elapsed,
349  * 1 if the @condition evaluated to %true after the @timeout elapsed,
350  * or the remaining jiffies (at least 1) if the @condition evaluated
351  * to %true before the @timeout elapsed.
352  */
353 #define wait_event_timeout(wq_head, condition, timeout)                         \
354 ({                                                                              \
355         long __ret = timeout;                                                   \
356         might_sleep();                                                          \
357         if (!___wait_cond_timeout(condition))                                   \
358                 __ret = __wait_event_timeout(wq_head, condition, timeout);      \
359         __ret;                                                                  \
360 })
361
362 #define __wait_event_freezable_timeout(wq_head, condition, timeout)             \
363         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
364                       TASK_INTERRUPTIBLE, 0, timeout,                           \
365                       __ret = schedule_timeout(__ret); try_to_freeze())
366
367 /*
368  * like wait_event_timeout() -- except it uses TASK_INTERRUPTIBLE to avoid
369  * increasing load and is freezable.
370  */
371 #define wait_event_freezable_timeout(wq_head, condition, timeout)               \
372 ({                                                                              \
373         long __ret = timeout;                                                   \
374         might_sleep();                                                          \
375         if (!___wait_cond_timeout(condition))                                   \
376                 __ret = __wait_event_freezable_timeout(wq_head, condition, timeout); \
377         __ret;                                                                  \
378 })
379
380 #define __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)              \
381         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 1, 0,     \
382                             cmd1; schedule(); cmd2)
383 /*
384  * Just like wait_event_cmd(), except it sets exclusive flag
385  */
386 #define wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2)                \
387 do {                                                                            \
388         if (condition)                                                          \
389                 break;                                                          \
390         __wait_event_exclusive_cmd(wq_head, condition, cmd1, cmd2);             \
391 } while (0)
392
393 #define __wait_event_cmd(wq_head, condition, cmd1, cmd2)                        \
394         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
395                             cmd1; schedule(); cmd2)
396
397 /**
398  * wait_event_cmd - sleep until a condition gets true
399  * @wq_head: the waitqueue to wait on
400  * @condition: a C expression for the event to wait for
401  * @cmd1: the command will be executed before sleep
402  * @cmd2: the command will be executed after sleep
403  *
404  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
405  * @condition evaluates to true. The @condition is checked each time
406  * the waitqueue @wq_head is woken up.
407  *
408  * wake_up() has to be called after changing any variable that could
409  * change the result of the wait condition.
410  */
411 #define wait_event_cmd(wq_head, condition, cmd1, cmd2)                          \
412 do {                                                                            \
413         if (condition)                                                          \
414                 break;                                                          \
415         __wait_event_cmd(wq_head, condition, cmd1, cmd2);                       \
416 } while (0)
417
418 #define __wait_event_interruptible(wq_head, condition)                          \
419         ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,             \
420                       schedule())
421
422 /**
423  * wait_event_interruptible - sleep until a condition gets true
424  * @wq_head: the waitqueue to wait on
425  * @condition: a C expression for the event to wait for
426  *
427  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
428  * @condition evaluates to true or a signal is received.
429  * The @condition is checked each time the waitqueue @wq_head is woken up.
430  *
431  * wake_up() has to be called after changing any variable that could
432  * change the result of the wait condition.
433  *
434  * The function will return -ERESTARTSYS if it was interrupted by a
435  * signal and 0 if @condition evaluated to true.
436  */
437 #define wait_event_interruptible(wq_head, condition)                            \
438 ({                                                                              \
439         int __ret = 0;                                                          \
440         might_sleep();                                                          \
441         if (!(condition))                                                       \
442                 __ret = __wait_event_interruptible(wq_head, condition);         \
443         __ret;                                                                  \
444 })
445
446 #define __wait_event_interruptible_timeout(wq_head, condition, timeout)         \
447         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
448                       TASK_INTERRUPTIBLE, 0, timeout,                           \
449                       __ret = schedule_timeout(__ret))
450
451 /**
452  * wait_event_interruptible_timeout - sleep until a condition gets true or a timeout elapses
453  * @wq_head: the waitqueue to wait on
454  * @condition: a C expression for the event to wait for
455  * @timeout: timeout, in jiffies
456  *
457  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
458  * @condition evaluates to true or a signal is received.
459  * The @condition is checked each time the waitqueue @wq_head is woken up.
460  *
461  * wake_up() has to be called after changing any variable that could
462  * change the result of the wait condition.
463  *
464  * Returns:
465  * 0 if the @condition evaluated to %false after the @timeout elapsed,
466  * 1 if the @condition evaluated to %true after the @timeout elapsed,
467  * the remaining jiffies (at least 1) if the @condition evaluated
468  * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
469  * interrupted by a signal.
470  */
471 #define wait_event_interruptible_timeout(wq_head, condition, timeout)           \
472 ({                                                                              \
473         long __ret = timeout;                                                   \
474         might_sleep();                                                          \
475         if (!___wait_cond_timeout(condition))                                   \
476                 __ret = __wait_event_interruptible_timeout(wq_head,             \
477                                                 condition, timeout);            \
478         __ret;                                                                  \
479 })
480
481 #define __wait_event_hrtimeout(wq_head, condition, timeout, state)              \
482 ({                                                                              \
483         int __ret = 0;                                                          \
484         struct hrtimer_sleeper __t;                                             \
485                                                                                 \
486         hrtimer_init_on_stack(&__t.timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);   \
487         hrtimer_init_sleeper(&__t, current);                                    \
488         if ((timeout) != KTIME_MAX)                                             \
489                 hrtimer_start_range_ns(&__t.timer, timeout,                     \
490                                        current->timer_slack_ns,                 \
491                                        HRTIMER_MODE_REL);                       \
492                                                                                 \
493         __ret = ___wait_event(wq_head, condition, state, 0, 0,                  \
494                 if (!__t.task) {                                                \
495                         __ret = -ETIME;                                         \
496                         break;                                                  \
497                 }                                                               \
498                 schedule());                                                    \
499                                                                                 \
500         hrtimer_cancel(&__t.timer);                                             \
501         destroy_hrtimer_on_stack(&__t.timer);                                   \
502         __ret;                                                                  \
503 })
504
505 /**
506  * wait_event_hrtimeout - sleep until a condition gets true or a timeout elapses
507  * @wq_head: the waitqueue to wait on
508  * @condition: a C expression for the event to wait for
509  * @timeout: timeout, as a ktime_t
510  *
511  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
512  * @condition evaluates to true or a signal is received.
513  * The @condition is checked each time the waitqueue @wq_head is woken up.
514  *
515  * wake_up() has to be called after changing any variable that could
516  * change the result of the wait condition.
517  *
518  * The function returns 0 if @condition became true, or -ETIME if the timeout
519  * elapsed.
520  */
521 #define wait_event_hrtimeout(wq_head, condition, timeout)                       \
522 ({                                                                              \
523         int __ret = 0;                                                          \
524         might_sleep();                                                          \
525         if (!(condition))                                                       \
526                 __ret = __wait_event_hrtimeout(wq_head, condition, timeout,     \
527                                                TASK_UNINTERRUPTIBLE);           \
528         __ret;                                                                  \
529 })
530
531 /**
532  * wait_event_interruptible_hrtimeout - sleep until a condition gets true or a timeout elapses
533  * @wq: the waitqueue to wait on
534  * @condition: a C expression for the event to wait for
535  * @timeout: timeout, as a ktime_t
536  *
537  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
538  * @condition evaluates to true or a signal is received.
539  * The @condition is checked each time the waitqueue @wq is woken up.
540  *
541  * wake_up() has to be called after changing any variable that could
542  * change the result of the wait condition.
543  *
544  * The function returns 0 if @condition became true, -ERESTARTSYS if it was
545  * interrupted by a signal, or -ETIME if the timeout elapsed.
546  */
547 #define wait_event_interruptible_hrtimeout(wq, condition, timeout)              \
548 ({                                                                              \
549         long __ret = 0;                                                         \
550         might_sleep();                                                          \
551         if (!(condition))                                                       \
552                 __ret = __wait_event_hrtimeout(wq, condition, timeout,          \
553                                                TASK_INTERRUPTIBLE);             \
554         __ret;                                                                  \
555 })
556
557 #define __wait_event_interruptible_exclusive(wq, condition)                     \
558         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,                  \
559                       schedule())
560
561 #define wait_event_interruptible_exclusive(wq, condition)                       \
562 ({                                                                              \
563         int __ret = 0;                                                          \
564         might_sleep();                                                          \
565         if (!(condition))                                                       \
566                 __ret = __wait_event_interruptible_exclusive(wq, condition);    \
567         __ret;                                                                  \
568 })
569
570 #define __wait_event_killable_exclusive(wq, condition)                          \
571         ___wait_event(wq, condition, TASK_KILLABLE, 1, 0,                       \
572                       schedule())
573
574 #define wait_event_killable_exclusive(wq, condition)                            \
575 ({                                                                              \
576         int __ret = 0;                                                          \
577         might_sleep();                                                          \
578         if (!(condition))                                                       \
579                 __ret = __wait_event_killable_exclusive(wq, condition);         \
580         __ret;                                                                  \
581 })
582
583
584 #define __wait_event_freezable_exclusive(wq, condition)                         \
585         ___wait_event(wq, condition, TASK_INTERRUPTIBLE, 1, 0,                  \
586                         schedule(); try_to_freeze())
587
588 #define wait_event_freezable_exclusive(wq, condition)                           \
589 ({                                                                              \
590         int __ret = 0;                                                          \
591         might_sleep();                                                          \
592         if (!(condition))                                                       \
593                 __ret = __wait_event_freezable_exclusive(wq, condition);        \
594         __ret;                                                                  \
595 })
596
597 extern int do_wait_intr(wait_queue_head_t *, wait_queue_entry_t *);
598 extern int do_wait_intr_irq(wait_queue_head_t *, wait_queue_entry_t *);
599
600 #define __wait_event_interruptible_locked(wq, condition, exclusive, fn)         \
601 ({                                                                              \
602         int __ret;                                                              \
603         DEFINE_WAIT(__wait);                                                    \
604         if (exclusive)                                                          \
605                 __wait.flags |= WQ_FLAG_EXCLUSIVE;                              \
606         do {                                                                    \
607                 __ret = fn(&(wq), &__wait);                                     \
608                 if (__ret)                                                      \
609                         break;                                                  \
610         } while (!(condition));                                                 \
611         __remove_wait_queue(&(wq), &__wait);                                    \
612         __set_current_state(TASK_RUNNING);                                      \
613         __ret;                                                                  \
614 })
615
616
617 /**
618  * wait_event_interruptible_locked - sleep until a condition gets true
619  * @wq: the waitqueue to wait on
620  * @condition: a C expression for the event to wait for
621  *
622  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
623  * @condition evaluates to true or a signal is received.
624  * The @condition is checked each time the waitqueue @wq is woken up.
625  *
626  * It must be called with wq.lock being held.  This spinlock is
627  * unlocked while sleeping but @condition testing is done while lock
628  * is held and when this macro exits the lock is held.
629  *
630  * The lock is locked/unlocked using spin_lock()/spin_unlock()
631  * functions which must match the way they are locked/unlocked outside
632  * of this macro.
633  *
634  * wake_up_locked() has to be called after changing any variable that could
635  * change the result of the wait condition.
636  *
637  * The function will return -ERESTARTSYS if it was interrupted by a
638  * signal and 0 if @condition evaluated to true.
639  */
640 #define wait_event_interruptible_locked(wq, condition)                          \
641         ((condition)                                                            \
642          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr))
643
644 /**
645  * wait_event_interruptible_locked_irq - sleep until a condition gets true
646  * @wq: the waitqueue to wait on
647  * @condition: a C expression for the event to wait for
648  *
649  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
650  * @condition evaluates to true or a signal is received.
651  * The @condition is checked each time the waitqueue @wq is woken up.
652  *
653  * It must be called with wq.lock being held.  This spinlock is
654  * unlocked while sleeping but @condition testing is done while lock
655  * is held and when this macro exits the lock is held.
656  *
657  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
658  * functions which must match the way they are locked/unlocked outside
659  * of this macro.
660  *
661  * wake_up_locked() has to be called after changing any variable that could
662  * change the result of the wait condition.
663  *
664  * The function will return -ERESTARTSYS if it was interrupted by a
665  * signal and 0 if @condition evaluated to true.
666  */
667 #define wait_event_interruptible_locked_irq(wq, condition)                      \
668         ((condition)                                                            \
669          ? 0 : __wait_event_interruptible_locked(wq, condition, 0, do_wait_intr_irq))
670
671 /**
672  * wait_event_interruptible_exclusive_locked - sleep exclusively until a condition gets true
673  * @wq: the waitqueue to wait on
674  * @condition: a C expression for the event to wait for
675  *
676  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
677  * @condition evaluates to true or a signal is received.
678  * The @condition is checked each time the waitqueue @wq is woken up.
679  *
680  * It must be called with wq.lock being held.  This spinlock is
681  * unlocked while sleeping but @condition testing is done while lock
682  * is held and when this macro exits the lock is held.
683  *
684  * The lock is locked/unlocked using spin_lock()/spin_unlock()
685  * functions which must match the way they are locked/unlocked outside
686  * of this macro.
687  *
688  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
689  * set thus when other process waits process on the list if this
690  * process is awaken further processes are not considered.
691  *
692  * wake_up_locked() has to be called after changing any variable that could
693  * change the result of the wait condition.
694  *
695  * The function will return -ERESTARTSYS if it was interrupted by a
696  * signal and 0 if @condition evaluated to true.
697  */
698 #define wait_event_interruptible_exclusive_locked(wq, condition)                \
699         ((condition)                                                            \
700          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr))
701
702 /**
703  * wait_event_interruptible_exclusive_locked_irq - sleep until a condition gets true
704  * @wq: the waitqueue to wait on
705  * @condition: a C expression for the event to wait for
706  *
707  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
708  * @condition evaluates to true or a signal is received.
709  * The @condition is checked each time the waitqueue @wq is woken up.
710  *
711  * It must be called with wq.lock being held.  This spinlock is
712  * unlocked while sleeping but @condition testing is done while lock
713  * is held and when this macro exits the lock is held.
714  *
715  * The lock is locked/unlocked using spin_lock_irq()/spin_unlock_irq()
716  * functions which must match the way they are locked/unlocked outside
717  * of this macro.
718  *
719  * The process is put on the wait queue with an WQ_FLAG_EXCLUSIVE flag
720  * set thus when other process waits process on the list if this
721  * process is awaken further processes are not considered.
722  *
723  * wake_up_locked() has to be called after changing any variable that could
724  * change the result of the wait condition.
725  *
726  * The function will return -ERESTARTSYS if it was interrupted by a
727  * signal and 0 if @condition evaluated to true.
728  */
729 #define wait_event_interruptible_exclusive_locked_irq(wq, condition)            \
730         ((condition)                                                            \
731          ? 0 : __wait_event_interruptible_locked(wq, condition, 1, do_wait_intr_irq))
732
733
734 #define __wait_event_killable(wq, condition)                                    \
735         ___wait_event(wq, condition, TASK_KILLABLE, 0, 0, schedule())
736
737 /**
738  * wait_event_killable - sleep until a condition gets true
739  * @wq_head: the waitqueue to wait on
740  * @condition: a C expression for the event to wait for
741  *
742  * The process is put to sleep (TASK_KILLABLE) until the
743  * @condition evaluates to true or a signal is received.
744  * The @condition is checked each time the waitqueue @wq_head is woken up.
745  *
746  * wake_up() has to be called after changing any variable that could
747  * change the result of the wait condition.
748  *
749  * The function will return -ERESTARTSYS if it was interrupted by a
750  * signal and 0 if @condition evaluated to true.
751  */
752 #define wait_event_killable(wq_head, condition)                                 \
753 ({                                                                              \
754         int __ret = 0;                                                          \
755         might_sleep();                                                          \
756         if (!(condition))                                                       \
757                 __ret = __wait_event_killable(wq_head, condition);              \
758         __ret;                                                                  \
759 })
760
761 #define __wait_event_killable_timeout(wq_head, condition, timeout)              \
762         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
763                       TASK_KILLABLE, 0, timeout,                                \
764                       __ret = schedule_timeout(__ret))
765
766 /**
767  * wait_event_killable_timeout - sleep until a condition gets true or a timeout elapses
768  * @wq_head: the waitqueue to wait on
769  * @condition: a C expression for the event to wait for
770  * @timeout: timeout, in jiffies
771  *
772  * The process is put to sleep (TASK_KILLABLE) until the
773  * @condition evaluates to true or a kill signal is received.
774  * The @condition is checked each time the waitqueue @wq_head is woken up.
775  *
776  * wake_up() has to be called after changing any variable that could
777  * change the result of the wait condition.
778  *
779  * Returns:
780  * 0 if the @condition evaluated to %false after the @timeout elapsed,
781  * 1 if the @condition evaluated to %true after the @timeout elapsed,
782  * the remaining jiffies (at least 1) if the @condition evaluated
783  * to %true before the @timeout elapsed, or -%ERESTARTSYS if it was
784  * interrupted by a kill signal.
785  *
786  * Only kill signals interrupt this process.
787  */
788 #define wait_event_killable_timeout(wq_head, condition, timeout)                \
789 ({                                                                              \
790         long __ret = timeout;                                                   \
791         might_sleep();                                                          \
792         if (!___wait_cond_timeout(condition))                                   \
793                 __ret = __wait_event_killable_timeout(wq_head,                  \
794                                                 condition, timeout);            \
795         __ret;                                                                  \
796 })
797
798
799 #define __wait_event_lock_irq(wq_head, condition, lock, cmd)                    \
800         (void)___wait_event(wq_head, condition, TASK_UNINTERRUPTIBLE, 0, 0,     \
801                             spin_unlock_irq(&lock);                             \
802                             cmd;                                                \
803                             schedule();                                         \
804                             spin_lock_irq(&lock))
805
806 /**
807  * wait_event_lock_irq_cmd - sleep until a condition gets true. The
808  *                           condition is checked under the lock. This
809  *                           is expected to be called with the lock
810  *                           taken.
811  * @wq_head: the waitqueue to wait on
812  * @condition: a C expression for the event to wait for
813  * @lock: a locked spinlock_t, which will be released before cmd
814  *        and schedule() and reacquired afterwards.
815  * @cmd: a command which is invoked outside the critical section before
816  *       sleep
817  *
818  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
819  * @condition evaluates to true. The @condition is checked each time
820  * the waitqueue @wq_head is woken up.
821  *
822  * wake_up() has to be called after changing any variable that could
823  * change the result of the wait condition.
824  *
825  * This is supposed to be called while holding the lock. The lock is
826  * dropped before invoking the cmd and going to sleep and is reacquired
827  * afterwards.
828  */
829 #define wait_event_lock_irq_cmd(wq_head, condition, lock, cmd)                  \
830 do {                                                                            \
831         if (condition)                                                          \
832                 break;                                                          \
833         __wait_event_lock_irq(wq_head, condition, lock, cmd);                   \
834 } while (0)
835
836 /**
837  * wait_event_lock_irq - sleep until a condition gets true. The
838  *                       condition is checked under the lock. This
839  *                       is expected to be called with the lock
840  *                       taken.
841  * @wq_head: the waitqueue to wait on
842  * @condition: a C expression for the event to wait for
843  * @lock: a locked spinlock_t, which will be released before schedule()
844  *        and reacquired afterwards.
845  *
846  * The process is put to sleep (TASK_UNINTERRUPTIBLE) until the
847  * @condition evaluates to true. The @condition is checked each time
848  * the waitqueue @wq_head is woken up.
849  *
850  * wake_up() has to be called after changing any variable that could
851  * change the result of the wait condition.
852  *
853  * This is supposed to be called while holding the lock. The lock is
854  * dropped before going to sleep and is reacquired afterwards.
855  */
856 #define wait_event_lock_irq(wq_head, condition, lock)                           \
857 do {                                                                            \
858         if (condition)                                                          \
859                 break;                                                          \
860         __wait_event_lock_irq(wq_head, condition, lock, );                      \
861 } while (0)
862
863
864 #define __wait_event_interruptible_lock_irq(wq_head, condition, lock, cmd)      \
865         ___wait_event(wq_head, condition, TASK_INTERRUPTIBLE, 0, 0,             \
866                       spin_unlock_irq(&lock);                                   \
867                       cmd;                                                      \
868                       schedule();                                               \
869                       spin_lock_irq(&lock))
870
871 /**
872  * wait_event_interruptible_lock_irq_cmd - sleep until a condition gets true.
873  *              The condition is checked under the lock. This is expected to
874  *              be called with the lock taken.
875  * @wq_head: the waitqueue to wait on
876  * @condition: a C expression for the event to wait for
877  * @lock: a locked spinlock_t, which will be released before cmd and
878  *        schedule() and reacquired afterwards.
879  * @cmd: a command which is invoked outside the critical section before
880  *       sleep
881  *
882  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
883  * @condition evaluates to true or a signal is received. The @condition is
884  * checked each time the waitqueue @wq_head is woken up.
885  *
886  * wake_up() has to be called after changing any variable that could
887  * change the result of the wait condition.
888  *
889  * This is supposed to be called while holding the lock. The lock is
890  * dropped before invoking the cmd and going to sleep and is reacquired
891  * afterwards.
892  *
893  * The macro will return -ERESTARTSYS if it was interrupted by a signal
894  * and 0 if @condition evaluated to true.
895  */
896 #define wait_event_interruptible_lock_irq_cmd(wq_head, condition, lock, cmd)    \
897 ({                                                                              \
898         int __ret = 0;                                                          \
899         if (!(condition))                                                       \
900                 __ret = __wait_event_interruptible_lock_irq(wq_head,            \
901                                                 condition, lock, cmd);          \
902         __ret;                                                                  \
903 })
904
905 /**
906  * wait_event_interruptible_lock_irq - sleep until a condition gets true.
907  *              The condition is checked under the lock. This is expected
908  *              to be called with the lock taken.
909  * @wq_head: the waitqueue to wait on
910  * @condition: a C expression for the event to wait for
911  * @lock: a locked spinlock_t, which will be released before schedule()
912  *        and reacquired afterwards.
913  *
914  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
915  * @condition evaluates to true or signal is received. The @condition is
916  * checked each time the waitqueue @wq_head is woken up.
917  *
918  * wake_up() has to be called after changing any variable that could
919  * change the result of the wait condition.
920  *
921  * This is supposed to be called while holding the lock. The lock is
922  * dropped before going to sleep and is reacquired afterwards.
923  *
924  * The macro will return -ERESTARTSYS if it was interrupted by a signal
925  * and 0 if @condition evaluated to true.
926  */
927 #define wait_event_interruptible_lock_irq(wq_head, condition, lock)             \
928 ({                                                                              \
929         int __ret = 0;                                                          \
930         if (!(condition))                                                       \
931                 __ret = __wait_event_interruptible_lock_irq(wq_head,            \
932                                                 condition, lock,);              \
933         __ret;                                                                  \
934 })
935
936 #define __wait_event_interruptible_lock_irq_timeout(wq_head, condition,         \
937                                                     lock, timeout)              \
938         ___wait_event(wq_head, ___wait_cond_timeout(condition),                 \
939                       TASK_INTERRUPTIBLE, 0, timeout,                           \
940                       spin_unlock_irq(&lock);                                   \
941                       __ret = schedule_timeout(__ret);                          \
942                       spin_lock_irq(&lock));
943
944 /**
945  * wait_event_interruptible_lock_irq_timeout - sleep until a condition gets
946  *              true or a timeout elapses. The condition is checked under
947  *              the lock. This is expected to be called with the lock taken.
948  * @wq_head: the waitqueue to wait on
949  * @condition: a C expression for the event to wait for
950  * @lock: a locked spinlock_t, which will be released before schedule()
951  *        and reacquired afterwards.
952  * @timeout: timeout, in jiffies
953  *
954  * The process is put to sleep (TASK_INTERRUPTIBLE) until the
955  * @condition evaluates to true or signal is received. The @condition is
956  * checked each time the waitqueue @wq_head is woken up.
957  *
958  * wake_up() has to be called after changing any variable that could
959  * change the result of the wait condition.
960  *
961  * This is supposed to be called while holding the lock. The lock is
962  * dropped before going to sleep and is reacquired afterwards.
963  *
964  * The function returns 0 if the @timeout elapsed, -ERESTARTSYS if it
965  * was interrupted by a signal, and the remaining jiffies otherwise
966  * if the condition evaluated to true before the timeout elapsed.
967  */
968 #define wait_event_interruptible_lock_irq_timeout(wq_head, condition, lock,     \
969                                                   timeout)                      \
970 ({                                                                              \
971         long __ret = timeout;                                                   \
972         if (!___wait_cond_timeout(condition))                                   \
973                 __ret = __wait_event_interruptible_lock_irq_timeout(            \
974                                         wq_head, condition, lock, timeout);     \
975         __ret;                                                                  \
976 })
977
978 /*
979  * Waitqueues which are removed from the waitqueue_head at wakeup time
980  */
981 void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
982 void prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
983 long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
984 void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
985 long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
986 int woken_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
987 int autoremove_wake_function(struct wait_queue_entry *wq_entry, unsigned mode, int sync, void *key);
988
989 #define DEFINE_WAIT_FUNC(name, function)                                        \
990         struct wait_queue_entry name = {                                        \
991                 .private        = current,                                      \
992                 .func           = function,                                     \
993                 .entry          = LIST_HEAD_INIT((name).entry),                 \
994         }
995
996 #define DEFINE_WAIT(name) DEFINE_WAIT_FUNC(name, autoremove_wake_function)
997
998 #define init_wait(wait)                                                         \
999         do {                                                                    \
1000                 (wait)->private = current;                                      \
1001                 (wait)->func = autoremove_wake_function;                        \
1002                 INIT_LIST_HEAD(&(wait)->entry);                                 \
1003                 (wait)->flags = 0;                                              \
1004         } while (0)
1005
1006 #endif /* _LINUX_WAIT_H */