1 /* 2 * Generic wait-for-completion handler; 3 * 4 * It differs from semaphores in that their default case is the opposite, 5 * wait_for_completion default blocks whereas semaphore default non-block. The 6 * interface also makes it easy to 'complete' multiple waiting threads, 7 * something which isn't entirely natural for semaphores. 8 * 9 * But more importantly, the primitive documents the usage. Semaphores would 10 * typically be used for exclusion which gives rise to priority inversion. 11 * Waiting for completion is a typically sync point, but not an exclusion point. 12 */ 13 14 #include <linux/sched/signal.h> 15 #include <linux/sched/debug.h> 16 #include <linux/completion.h> 17 18 /** 19 * complete: - signals a single thread waiting on this completion 20 * @x: holds the state of this particular completion 21 * 22 * This will wake up a single thread waiting on this completion. Threads will be 23 * awakened in the same order in which they were queued. 24 * 25 * See also complete_all(), wait_for_completion() and related routines. 26 * 27 * It may be assumed that this function implies a write memory barrier before 28 * changing the task state if and only if any tasks are woken up. 29 */ 30 void complete(struct completion *x) 31 { 32 unsigned long flags; 33 34 spin_lock_irqsave(&x->wait.lock, flags); 35 if (x->done != UINT_MAX) 36 x->done++; 37 __wake_up_locked(&x->wait, TASK_NORMAL, 1); 38 spin_unlock_irqrestore(&x->wait.lock, flags); 39 } 40 EXPORT_SYMBOL(complete); 41 42 /** 43 * complete_all: - signals all threads waiting on this completion 44 * @x: holds the state of this particular completion 45 * 46 * This will wake up all threads waiting on this particular completion event. 47 * 48 * It may be assumed that this function implies a write memory barrier before 49 * changing the task state if and only if any tasks are woken up. 50 */ 51 void complete_all(struct completion *x) 52 { 53 unsigned long flags; 54 55 spin_lock_irqsave(&x->wait.lock, flags); 56 x->done = UINT_MAX; 57 __wake_up_locked(&x->wait, TASK_NORMAL, 0); 58 spin_unlock_irqrestore(&x->wait.lock, flags); 59 } 60 EXPORT_SYMBOL(complete_all); 61 62 static inline long __sched 63 do_wait_for_common(struct completion *x, 64 long (*action)(long), long timeout, int state) 65 { 66 if (!x->done) { 67 DECLARE_WAITQUEUE(wait, current); 68 69 __add_wait_queue_tail_exclusive(&x->wait, &wait); 70 do { 71 if (signal_pending_state(state, current)) { 72 timeout = -ERESTARTSYS; 73 break; 74 } 75 __set_current_state(state); 76 spin_unlock_irq(&x->wait.lock); 77 timeout = action(timeout); 78 spin_lock_irq(&x->wait.lock); 79 } while (!x->done && timeout); 80 __remove_wait_queue(&x->wait, &wait); 81 if (!x->done) 82 return timeout; 83 } 84 if (x->done != UINT_MAX) 85 x->done--; 86 return timeout ?: 1; 87 } 88 89 static inline long __sched 90 __wait_for_common(struct completion *x, 91 long (*action)(long), long timeout, int state) 92 { 93 might_sleep(); 94 95 spin_lock_irq(&x->wait.lock); 96 timeout = do_wait_for_common(x, action, timeout, state); 97 spin_unlock_irq(&x->wait.lock); 98 return timeout; 99 } 100 101 static long __sched 102 wait_for_common(struct completion *x, long timeout, int state) 103 { 104 return __wait_for_common(x, schedule_timeout, timeout, state); 105 } 106 107 static long __sched 108 wait_for_common_io(struct completion *x, long timeout, int state) 109 { 110 return __wait_for_common(x, io_schedule_timeout, timeout, state); 111 } 112 113 /** 114 * wait_for_completion: - waits for completion of a task 115 * @x: holds the state of this particular completion 116 * 117 * This waits to be signaled for completion of a specific task. It is NOT 118 * interruptible and there is no timeout. 119 * 120 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout 121 * and interrupt capability. Also see complete(). 122 */ 123 void __sched wait_for_completion(struct completion *x) 124 { 125 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); 126 } 127 EXPORT_SYMBOL(wait_for_completion); 128 129 /** 130 * wait_for_completion_timeout: - waits for completion of a task (w/timeout) 131 * @x: holds the state of this particular completion 132 * @timeout: timeout value in jiffies 133 * 134 * This waits for either a completion of a specific task to be signaled or for a 135 * specified timeout to expire. The timeout is in jiffies. It is not 136 * interruptible. 137 * 138 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left 139 * till timeout) if completed. 140 */ 141 unsigned long __sched 142 wait_for_completion_timeout(struct completion *x, unsigned long timeout) 143 { 144 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); 145 } 146 EXPORT_SYMBOL(wait_for_completion_timeout); 147 148 /** 149 * wait_for_completion_io: - waits for completion of a task 150 * @x: holds the state of this particular completion 151 * 152 * This waits to be signaled for completion of a specific task. It is NOT 153 * interruptible and there is no timeout. The caller is accounted as waiting 154 * for IO (which traditionally means blkio only). 155 */ 156 void __sched wait_for_completion_io(struct completion *x) 157 { 158 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); 159 } 160 EXPORT_SYMBOL(wait_for_completion_io); 161 162 /** 163 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout) 164 * @x: holds the state of this particular completion 165 * @timeout: timeout value in jiffies 166 * 167 * This waits for either a completion of a specific task to be signaled or for a 168 * specified timeout to expire. The timeout is in jiffies. It is not 169 * interruptible. The caller is accounted as waiting for IO (which traditionally 170 * means blkio only). 171 * 172 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left 173 * till timeout) if completed. 174 */ 175 unsigned long __sched 176 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout) 177 { 178 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE); 179 } 180 EXPORT_SYMBOL(wait_for_completion_io_timeout); 181 182 /** 183 * wait_for_completion_interruptible: - waits for completion of a task (w/intr) 184 * @x: holds the state of this particular completion 185 * 186 * This waits for completion of a specific task to be signaled. It is 187 * interruptible. 188 * 189 * Return: -ERESTARTSYS if interrupted, 0 if completed. 190 */ 191 int __sched wait_for_completion_interruptible(struct completion *x) 192 { 193 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); 194 if (t == -ERESTARTSYS) 195 return t; 196 return 0; 197 } 198 EXPORT_SYMBOL(wait_for_completion_interruptible); 199 200 /** 201 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr)) 202 * @x: holds the state of this particular completion 203 * @timeout: timeout value in jiffies 204 * 205 * This waits for either a completion of a specific task to be signaled or for a 206 * specified timeout to expire. It is interruptible. The timeout is in jiffies. 207 * 208 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, 209 * or number of jiffies left till timeout) if completed. 210 */ 211 long __sched 212 wait_for_completion_interruptible_timeout(struct completion *x, 213 unsigned long timeout) 214 { 215 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE); 216 } 217 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); 218 219 /** 220 * wait_for_completion_killable: - waits for completion of a task (killable) 221 * @x: holds the state of this particular completion 222 * 223 * This waits to be signaled for completion of a specific task. It can be 224 * interrupted by a kill signal. 225 * 226 * Return: -ERESTARTSYS if interrupted, 0 if completed. 227 */ 228 int __sched wait_for_completion_killable(struct completion *x) 229 { 230 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); 231 if (t == -ERESTARTSYS) 232 return t; 233 return 0; 234 } 235 EXPORT_SYMBOL(wait_for_completion_killable); 236 237 /** 238 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable)) 239 * @x: holds the state of this particular completion 240 * @timeout: timeout value in jiffies 241 * 242 * This waits for either a completion of a specific task to be 243 * signaled or for a specified timeout to expire. It can be 244 * interrupted by a kill signal. The timeout is in jiffies. 245 * 246 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, 247 * or number of jiffies left till timeout) if completed. 248 */ 249 long __sched 250 wait_for_completion_killable_timeout(struct completion *x, 251 unsigned long timeout) 252 { 253 return wait_for_common(x, timeout, TASK_KILLABLE); 254 } 255 EXPORT_SYMBOL(wait_for_completion_killable_timeout); 256 257 /** 258 * try_wait_for_completion - try to decrement a completion without blocking 259 * @x: completion structure 260 * 261 * Return: 0 if a decrement cannot be done without blocking 262 * 1 if a decrement succeeded. 263 * 264 * If a completion is being used as a counting completion, 265 * attempt to decrement the counter without blocking. This 266 * enables us to avoid waiting if the resource the completion 267 * is protecting is not available. 268 */ 269 bool try_wait_for_completion(struct completion *x) 270 { 271 unsigned long flags; 272 int ret = 1; 273 274 /* 275 * Since x->done will need to be locked only 276 * in the non-blocking case, we check x->done 277 * first without taking the lock so we can 278 * return early in the blocking case. 279 */ 280 if (!READ_ONCE(x->done)) 281 return 0; 282 283 spin_lock_irqsave(&x->wait.lock, flags); 284 if (!x->done) 285 ret = 0; 286 else if (x->done != UINT_MAX) 287 x->done--; 288 spin_unlock_irqrestore(&x->wait.lock, flags); 289 return ret; 290 } 291 EXPORT_SYMBOL(try_wait_for_completion); 292 293 /** 294 * completion_done - Test to see if a completion has any waiters 295 * @x: completion structure 296 * 297 * Return: 0 if there are waiters (wait_for_completion() in progress) 298 * 1 if there are no waiters. 299 * 300 */ 301 bool completion_done(struct completion *x) 302 { 303 if (!READ_ONCE(x->done)) 304 return false; 305 306 /* 307 * If ->done, we need to wait for complete() to release ->wait.lock 308 * otherwise we can end up freeing the completion before complete() 309 * is done referencing it. 310 * 311 * The RMB pairs with complete()'s RELEASE of ->wait.lock and orders 312 * the loads of ->done and ->wait.lock such that we cannot observe 313 * the lock before complete() acquires it while observing the ->done 314 * after it's acquired the lock. 315 */ 316 smp_rmb(); 317 spin_unlock_wait(&x->wait.lock); 318 return true; 319 } 320 EXPORT_SYMBOL(completion_done); 321