1 /*- 2 * Copyright (c) 2010 Isilon Systems, Inc. 3 * Copyright (c) 2010 iX Systems, Inc. 4 * Copyright (c) 2010 Panasas, Inc. 5 * Copyright (c) 2013, 2014 Mellanox Technologies, Ltd. 6 * Copyright (c) 2017 Mark Johnston <markj@FreeBSD.org> 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #ifndef _LINUXKPI_LINUX_WAIT_H_ 32 #define _LINUXKPI_LINUX_WAIT_H_ 33 34 #include <linux/compiler.h> 35 #include <linux/list.h> 36 #include <linux/spinlock.h> 37 #include <linux/sched.h> 38 39 #include <asm/atomic.h> 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 44 #define SKIP_SLEEP() (SCHEDULER_STOPPED() || kdb_active) 45 46 struct wait_queue; 47 struct wait_queue_head; 48 49 #define wait_queue_entry wait_queue 50 51 typedef struct wait_queue wait_queue_t; 52 typedef struct wait_queue_entry wait_queue_entry_t; 53 typedef struct wait_queue_head wait_queue_head_t; 54 55 typedef int wait_queue_func_t(wait_queue_t *, unsigned int, int, void *); 56 57 #define WQ_FLAG_WOKEN 0x02 58 59 /* 60 * Many API consumers directly reference these fields and those of 61 * wait_queue_head. 62 */ 63 struct wait_queue { 64 unsigned int flags; 65 void *private; 66 wait_queue_func_t *func; 67 union { 68 struct list_head task_list; /* < v4.13 */ 69 struct list_head entry; /* >= v4.13 */ 70 }; 71 }; 72 73 struct wait_queue_head { 74 spinlock_t lock; 75 union { 76 struct list_head task_list; /* < v4.13 */ 77 struct list_head head; /* >= v4.13 */ 78 }; 79 }; 80 81 /* 82 * This function is referenced by at least one DRM driver, so it may not be 83 * renamed and furthermore must be the default wait queue callback. 84 */ 85 wait_queue_func_t autoremove_wake_function; 86 wait_queue_func_t default_wake_function; 87 wait_queue_func_t woken_wake_function; 88 89 long linux_wait_woken(wait_queue_t *wq, unsigned state, long timeout); 90 91 #define wait_woken(wq, state, timeout) \ 92 linux_wait_woken((wq), (state), (timeout)) 93 94 #define DEFINE_WAIT_FUNC(name, function) \ 95 wait_queue_t name = { \ 96 .private = current, \ 97 .func = function, \ 98 .task_list = LINUX_LIST_HEAD_INIT(name.task_list) \ 99 } 100 101 #define DEFINE_WAIT(name) \ 102 DEFINE_WAIT_FUNC(name, autoremove_wake_function) 103 104 #define DECLARE_WAITQUEUE(name, task) \ 105 wait_queue_t name = { \ 106 .private = task, \ 107 .task_list = LINUX_LIST_HEAD_INIT(name.task_list) \ 108 } 109 110 #define DECLARE_WAIT_QUEUE_HEAD(name) \ 111 wait_queue_head_t name = { \ 112 .task_list = LINUX_LIST_HEAD_INIT(name.task_list), \ 113 }; \ 114 MTX_SYSINIT(name, &(name).lock, spin_lock_name("wqhead"), MTX_DEF) 115 116 #define init_waitqueue_head(wqh) do { \ 117 mtx_init(&(wqh)->lock, spin_lock_name("wqhead"), \ 118 NULL, MTX_DEF | MTX_NEW | MTX_NOWITNESS); \ 119 INIT_LIST_HEAD(&(wqh)->task_list); \ 120 } while (0) 121 122 #define __init_waitqueue_head(wqh, name, lk) init_waitqueue_head(wqh) 123 124 void linux_init_wait_entry(wait_queue_t *, int); 125 void linux_wake_up(wait_queue_head_t *, unsigned int, int, bool); 126 127 #define init_wait_entry(wq, flags) \ 128 linux_init_wait_entry(wq, flags) 129 #define wake_up(wqh) \ 130 linux_wake_up(wqh, TASK_NORMAL, 1, false) 131 #define wake_up_all(wqh) \ 132 linux_wake_up(wqh, TASK_NORMAL, 0, false) 133 #define wake_up_locked(wqh) \ 134 linux_wake_up(wqh, TASK_NORMAL, 1, true) 135 #define wake_up_all_locked(wqh) \ 136 linux_wake_up(wqh, TASK_NORMAL, 0, true) 137 #define wake_up_interruptible(wqh) \ 138 linux_wake_up(wqh, TASK_INTERRUPTIBLE, 1, false) 139 #define wake_up_interruptible_all(wqh) \ 140 linux_wake_up(wqh, TASK_INTERRUPTIBLE, 0, false) 141 142 int linux_wait_event_common(wait_queue_head_t *, wait_queue_t *, long, 143 unsigned int, spinlock_t *); 144 145 /* 146 * Returns -ERESTARTSYS for a signal, 0 if cond is false after timeout, 1 if 147 * cond is true after timeout, remaining jiffies (> 0) if cond is true before 148 * timeout. 149 */ 150 #define __wait_event_common(wqh, cond, timeout, state, lock) ({ \ 151 DEFINE_WAIT(__wq); \ 152 const long __timeout = ((long)(timeout)) < 1 ? 1 : (timeout); \ 153 long __start = jiffies; \ 154 long __ret = 0; \ 155 \ 156 for (;;) { \ 157 linux_prepare_to_wait(&(wqh), &__wq, state); \ 158 if (cond) \ 159 break; \ 160 __ret = linux_wait_event_common(&(wqh), &__wq, \ 161 __timeout, state, lock); \ 162 if (__ret != 0) \ 163 break; \ 164 } \ 165 linux_finish_wait(&(wqh), &__wq); \ 166 if (__timeout != MAX_SCHEDULE_TIMEOUT) { \ 167 if (__ret == -EWOULDBLOCK) \ 168 __ret = !!(cond); \ 169 else if (__ret != -ERESTARTSYS) { \ 170 __ret = __timeout + __start - jiffies; \ 171 /* range check return value */ \ 172 if (__ret < 1) \ 173 __ret = 1; \ 174 else if (__ret > __timeout) \ 175 __ret = __timeout; \ 176 } \ 177 } \ 178 __ret; \ 179 }) 180 181 #define wait_event(wqh, cond) do { \ 182 (void) __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ 183 TASK_UNINTERRUPTIBLE, NULL); \ 184 } while (0) 185 186 #define wait_event_timeout(wqh, cond, timeout) ({ \ 187 __wait_event_common(wqh, cond, timeout, TASK_UNINTERRUPTIBLE, \ 188 NULL); \ 189 }) 190 191 #define wait_event_killable(wqh, cond) ({ \ 192 __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ 193 TASK_INTERRUPTIBLE, NULL); \ 194 }) 195 196 #define wait_event_interruptible(wqh, cond) ({ \ 197 __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ 198 TASK_INTERRUPTIBLE, NULL); \ 199 }) 200 201 #define wait_event_interruptible_timeout(wqh, cond, timeout) ({ \ 202 __wait_event_common(wqh, cond, timeout, TASK_INTERRUPTIBLE, \ 203 NULL); \ 204 }) 205 206 /* 207 * Wait queue is already locked. 208 */ 209 #define wait_event_interruptible_locked(wqh, cond) ({ \ 210 int __ret; \ 211 \ 212 spin_unlock(&(wqh).lock); \ 213 __ret = __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ 214 TASK_INTERRUPTIBLE, NULL); \ 215 spin_lock(&(wqh).lock); \ 216 __ret; \ 217 }) 218 219 /* 220 * The passed spinlock is held when testing the condition. 221 */ 222 #define wait_event_interruptible_lock_irq(wqh, cond, lock) ({ \ 223 __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ 224 TASK_INTERRUPTIBLE, &(lock)); \ 225 }) 226 227 /* 228 * The passed spinlock is held when testing the condition. 229 */ 230 #define wait_event_lock_irq(wqh, cond, lock) ({ \ 231 __wait_event_common(wqh, cond, MAX_SCHEDULE_TIMEOUT, \ 232 TASK_UNINTERRUPTIBLE, &(lock)); \ 233 }) 234 235 static inline void 236 __add_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq) 237 { 238 list_add(&wq->task_list, &wqh->task_list); 239 } 240 241 static inline void 242 add_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq) 243 { 244 245 spin_lock(&wqh->lock); 246 __add_wait_queue(wqh, wq); 247 spin_unlock(&wqh->lock); 248 } 249 250 static inline void 251 __add_wait_queue_tail(wait_queue_head_t *wqh, wait_queue_t *wq) 252 { 253 list_add_tail(&wq->task_list, &wqh->task_list); 254 } 255 256 static inline void 257 __add_wait_queue_entry_tail(wait_queue_head_t *wqh, wait_queue_entry_t *wq) 258 { 259 list_add_tail(&wq->entry, &wqh->head); 260 } 261 262 static inline void 263 __remove_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq) 264 { 265 list_del(&wq->task_list); 266 } 267 268 static inline void 269 remove_wait_queue(wait_queue_head_t *wqh, wait_queue_t *wq) 270 { 271 272 spin_lock(&wqh->lock); 273 __remove_wait_queue(wqh, wq); 274 spin_unlock(&wqh->lock); 275 } 276 277 bool linux_waitqueue_active(wait_queue_head_t *); 278 279 #define waitqueue_active(wqh) linux_waitqueue_active(wqh) 280 281 void linux_prepare_to_wait(wait_queue_head_t *, wait_queue_t *, int); 282 void linux_finish_wait(wait_queue_head_t *, wait_queue_t *); 283 284 #define prepare_to_wait(wqh, wq, state) linux_prepare_to_wait(wqh, wq, state) 285 #define finish_wait(wqh, wq) linux_finish_wait(wqh, wq) 286 287 void linux_wake_up_bit(void *, int); 288 int linux_wait_on_bit_timeout(unsigned long *, int, unsigned int, long); 289 void linux_wake_up_atomic_t(atomic_t *); 290 int linux_wait_on_atomic_t(atomic_t *, unsigned int); 291 292 #define wake_up_bit(word, bit) linux_wake_up_bit(word, bit) 293 #define wait_on_bit(word, bit, state) \ 294 linux_wait_on_bit_timeout(word, bit, state, MAX_SCHEDULE_TIMEOUT) 295 #define wait_on_bit_timeout(word, bit, state, timeout) \ 296 linux_wait_on_bit_timeout(word, bit, state, timeout) 297 #define wake_up_atomic_t(a) linux_wake_up_atomic_t(a) 298 /* 299 * All existing callers have a cb that just schedule()s. To avoid adding 300 * complexity, just emulate that internally. The prototype is different so that 301 * callers must be manually modified; a cb that does something other than call 302 * schedule() will require special treatment. 303 */ 304 #define wait_on_atomic_t(a, state) linux_wait_on_atomic_t(a, state) 305 306 struct task_struct; 307 bool linux_wake_up_state(struct task_struct *, unsigned int); 308 309 #define wake_up_process(task) linux_wake_up_state(task, TASK_NORMAL) 310 #define wake_up_state(task, state) linux_wake_up_state(task, state) 311 312 #endif /* _LINUXKPI_LINUX_WAIT_H_ */ 313