1 /*-
2 * Copyright (c) 2010 Isilon Systems, Inc.
3 * Copyright (c) 2010 iX Systems, Inc.
4 * Copyright (c) 2010 Panasas, Inc.
5 * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd.
6 * Copyright (c) 2017 Mark Johnston <markj@FreeBSD.org>
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice unmodified, this list of conditions, and the following
14 * disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30
31 #ifndef _LINUXKPI_LINUX_WAIT_H_
32 #define _LINUXKPI_LINUX_WAIT_H_
33
34 #include <linux/compiler.h>
35 #include <linux/list.h>
36 #include <linux/spinlock.h>
37 #include <linux/sched.h>
38
39 #include <asm/atomic.h>
40
41 #include <sys/param.h>
42 #include <sys/systm.h>
43
44 #define SKIP_SLEEP() (SCHEDULER_STOPPED() || kdb_active)
45
46 struct wait_queue;
47 struct wait_queue_head;
48
49 #define wait_queue_entry wait_queue
50
51 typedef struct wait_queue wait_queue_t;
52 typedef struct wait_queue_entry wait_queue_entry_t;
53 typedef struct wait_queue_head wait_queue_head_t;
54
55 typedef int wait_queue_func_t(wait_queue_t *, unsigned int, int, void *);
56
57 #define WQ_FLAG_WOKEN 0x02
58
59 /*
60 * Many API consumers directly reference these fields and those of
61 * wait_queue_head.
62 */
63 struct wait_queue {
64 unsigned int flags;
65 void *private;
66 wait_queue_func_t *func;
67 union {
68 struct list_head task_list; /* < v4.13 */
69 struct list_head entry; /* >= v4.13 */
70 };
71 };
72
73 struct wait_queue_head {
74 spinlock_t lock;
75 union {
76 struct list_head task_list; /* < v4.13 */
77 struct list_head head; /* >= v4.13 */
78 };
79 };
80
81 /*
82 * This function is referenced by at least one DRM driver, so it may not be
83 * renamed and furthermore must be the default wait queue callback.
84 */
85 wait_queue_func_t autoremove_wake_function;
86 wait_queue_func_t default_wake_function;
87 wait_queue_func_t woken_wake_function;
88
89 long linux_wait_woken(wait_queue_t *wq, unsigned state, long timeout);
90
91 #define wait_woken(wq, state, timeout) \
92 linux_wait_woken((wq), (state), (timeout))
93
94 #define DEFINE_WAIT_FUNC(name, function) \
95 wait_queue_t name = { \
96 .private = current, \
97 .func = function, \
98 .task_list = LINUX_LIST_HEAD_INIT(name.task_list) \
99 }
100
101 #define DEFINE_WAIT(name) \
102 DEFINE_WAIT_FUNC(name, autoremove_wake_function)
103
104 #define DECLARE_WAITQUEUE(name, task) \
105 wait_queue_t name = { \
106 .private = task, \
107 .task_list = LINUX_LIST_HEAD_INIT(name.task_list) \
108 }
109
110 #define DECLARE_WAIT_QUEUE_HEAD(name) \
111 wait_queue_head_t name = { \
112 .task_list = LINUX_LIST_HEAD_INIT(name.task_list), \
113 }; \
114 MTX_SYSINIT(name, &(name).lock, spin_lock_name("wqhead"), MTX_DEF)
115
116 #define init_waitqueue_head(wqh) do { \
117 mtx_init(&(wqh)->lock, spin_lock_name("wqhead"), \
118 NULL, MTX_DEF | MTX_NEW | MTX_NOWITNESS); \
119 INIT_LIST_HEAD(&(wqh)->task_list); \
120 } while (0)
121
122 #define __init_waitqueue_head(wqh, name, lk) init_waitqueue_head(wqh)
123
124 void linux_init_wait_entry(wait_queue_t *, int);
125 void linux_wake_up(wait_queue_head_t *, unsigned int, int, bool);
126
127 #define init_wait_entry(wq, flags) \
128 linux_init_wait_entry(wq, flags)
129 #define wake_up(wqh) \
130 linux_wake_up(wqh, TASK_NORMAL, 1, false)
131 #define wake_up_all(wqh) \
132 linux_wake_up(wqh, TASK_NORMAL, 0, false)
133 #define wake_up_locked(wqh) \
134 linux_wake_up(wqh, TASK_NORMAL, 1, true)
135 #define wake_up_all_locked(wqh) \
136 linux_wake_up(wqh, TASK_NORMAL, 0, true)
137 #define wake_up_interruptible(wqh) \
138 linux_wake_up(wqh, TASK_INTERRUPTIBLE, 1, false)
139 #define wake_up_interruptible_all(wqh) \
140 linux_wake_up(wqh, TASK_INTERRUPTIBLE, 0, false)
141
142 int linux_wait_event_common(wait_queue_head_t *, wait_queue_t *, long,
143 unsigned int, spinlock_t *);
144
145 /*
146 * Returns -ERESTARTSYS for a signal, 0 if cond is false after timeout, 1 if
147 * cond is true after timeout, remaining jiffies (> 0) if cond is true before
148 * timeout.
149 */
150 #define __wait_event_common(wqh, cond, timeout, state, lock) ({ \
151 DEFINE_WAIT(__wq); \
152 const long __timeout = ((long)(timeout)) < 1 ? 1 : (timeout); \
153 long __start = jiffies; \
154 long __ret = 0; \
155 \
156 for (;;) { \
157 linux_prepare_to_wait(&(wqh), &__wq, state); \
158 if (cond) \
159 break; \
160 __ret = linux_wait_event_common(&(wqh), &__wq, \
161 __timeout, state, lock); \
162 if (__ret != 0) \
163 break; \
164 } \
165 linux_finish_wait(&(wqh), &__wq); \
166 if (__timeout != MAX_SCHEDULE_TIMEOUT) { \
167 if (__ret == -EWOULDBLOCK) \
168 __ret = !!(cond); \
169 else if (__ret != -ERESTARTSYS) { \
170 __ret = __timeout + __start - jiffies; \
171 /* range check return value */ \
172 if (__ret < 1) \
173 __ret = 1; \
174 else if (__ret > __timeout) \
175 __ret = __timeout; \
176 } \
177 } \
178 __ret; \
179 })
180
181 #define wait_event(wqh, cond) do { \
182 (void) __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \
183 TASK_UNINTERRUPTIBLE, NULL); \
184 } while (0)
185
186 #define wait_event_timeout(wqh, cond, timeout) ({ \
187 __wait_event_common(wqh, cond, timeout, TASK_UNINTERRUPTIBLE, \
188 NULL); \
189 })
190
191 #define wait_event_killable(wqh, cond) ({ \
192 __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \
193 TASK_INTERRUPTIBLE, NULL); \
194 })
195
196 #define wait_event_interruptible(wqh, cond) ({ \
197 __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \
198 TASK_INTERRUPTIBLE, NULL); \
199 })
200
201 #define wait_event_interruptible_timeout(wqh, cond, timeout) ({ \
202 __wait_event_common(wqh, cond, timeout, TASK_INTERRUPTIBLE, \
203 NULL); \
204 })
205
206 /*
207 * Wait queue is already locked.
208 */
209 #define wait_event_interruptible_locked(wqh, cond) ({ \
210 int __ret; \
211 \
212 spin_unlock(&(wqh).lock); \
213 __ret = __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \
214 TASK_INTERRUPTIBLE, NULL); \
215 spin_lock(&(wqh).lock); \
216 __ret; \
217 })
218
219 /*
220 * The passed spinlock is held when testing the condition.
221 */
222 #define wait_event_interruptible_lock_irq(wqh, cond, lock) ({ \
223 __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \
224 TASK_INTERRUPTIBLE, &(lock)); \
225 })
226
227 /*
228 * The passed spinlock is held when testing the condition.
229 */
230 #define wait_event_lock_irq(wqh, cond, lock) ({ \
231 __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \
232 TASK_UNINTERRUPTIBLE, &(lock)); \
233 })
234
235 static inline void
__add_wait_queue(wait_queue_head_t * wqh,wait_queue_t * wq)236 __add_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq)
237 {
238 list_add(&wq->task_list, &wqh->task_list);
239 }
240
241 static inline void
add_wait_queue(wait_queue_head_t * wqh,wait_queue_t * wq)242 add_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq)
243 {
244
245 spin_lock(&wqh->lock);
246 __add_wait_queue(wqh, wq);
247 spin_unlock(&wqh->lock);
248 }
249
250 static inline void
__add_wait_queue_tail(wait_queue_head_t * wqh,wait_queue_t * wq)251 __add_wait_queue_tail(wait_queue_head_t *wqh, wait_queue_t *wq)
252 {
253 list_add_tail(&wq->task_list, &wqh->task_list);
254 }
255
256 static inline void
__add_wait_queue_entry_tail(wait_queue_head_t * wqh,wait_queue_entry_t * wq)257 __add_wait_queue_entry_tail(wait_queue_head_t *wqh, wait_queue_entry_t *wq)
258 {
259 list_add_tail(&wq->entry, &wqh->head);
260 }
261
262 static inline void
__remove_wait_queue(wait_queue_head_t * wqh,wait_queue_t * wq)263 __remove_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq)
264 {
265 list_del(&wq->task_list);
266 }
267
268 static inline void
remove_wait_queue(wait_queue_head_t * wqh,wait_queue_t * wq)269 remove_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq)
270 {
271
272 spin_lock(&wqh->lock);
273 __remove_wait_queue(wqh, wq);
274 spin_unlock(&wqh->lock);
275 }
276
277 bool linux_waitqueue_active(wait_queue_head_t *);
278
279 #define waitqueue_active(wqh) linux_waitqueue_active(wqh)
280
281 void linux_prepare_to_wait(wait_queue_head_t *, wait_queue_t *, int);
282 void linux_finish_wait(wait_queue_head_t *, wait_queue_t *);
283
284 #define prepare_to_wait(wqh, wq, state) linux_prepare_to_wait(wqh, wq, state)
285 #define finish_wait(wqh, wq) linux_finish_wait(wqh, wq)
286
287 void linux_wake_up_bit(void *, int);
288 int linux_wait_on_bit_timeout(unsigned long *, int, unsigned int, long);
289 void linux_wake_up_atomic_t(atomic_t *);
290 int linux_wait_on_atomic_t(atomic_t *, unsigned int);
291
292 #define wake_up_bit(word, bit) linux_wake_up_bit(word, bit)
293 #define wait_on_bit(word, bit, state) \
294 linux_wait_on_bit_timeout(word, bit, state, MAX_SCHEDULE_TIMEOUT)
295 #define wait_on_bit_timeout(word, bit, state, timeout) \
296 linux_wait_on_bit_timeout(word, bit, state, timeout)
297 #define wake_up_atomic_t(a) linux_wake_up_atomic_t(a)
298 /*
299 * All existing callers have a cb that just schedule()s. To avoid adding
300 * complexity, just emulate that internally. The prototype is different so that
301 * callers must be manually modified; a cb that does something other than call
302 * schedule() will require special treatment.
303 */
304 #define wait_on_atomic_t(a, state) linux_wait_on_atomic_t(a, state)
305
306 struct task_struct;
307 bool linux_wake_up_state(struct task_struct *, unsigned int);
308
309 #define wake_up_process(task) linux_wake_up_state(task, TASK_NORMAL)
310 #define wake_up_state(task, state) linux_wake_up_state(task, state)
311
312 #endif /* _LINUXKPI_LINUX_WAIT_H_ */
313