1 // SPDX-License-Identifier: GPL-2.0 2 3 /* 4 * Generic wait-for-completion handler; 5 * 6 * It differs from semaphores in that their default case is the opposite, 7 * wait_for_completion default blocks whereas semaphore default non-block. The 8 * interface also makes it easy to 'complete' multiple waiting threads, 9 * something which isn't entirely natural for semaphores. 10 * 11 * But more importantly, the primitive documents the usage. Semaphores would 12 * typically be used for exclusion which gives rise to priority inversion. 13 * Waiting for completion is a typically sync point, but not an exclusion point. 14 */ 15 16 static void complete_with_flags(struct completion *x, int wake_flags) 17 { 18 unsigned long flags; 19 20 raw_spin_lock_irqsave(&x->wait.lock, flags); 21 22 if (x->done != UINT_MAX) 23 x->done++; 24 swake_up_locked(&x->wait, wake_flags); 25 raw_spin_unlock_irqrestore(&x->wait.lock, flags); 26 } 27 28 void complete_on_current_cpu(struct completion *x) 29 { 30 return complete_with_flags(x, WF_CURRENT_CPU); 31 } 32 33 /** 34 * complete: - signals a single thread waiting on this completion 35 * @x: holds the state of this particular completion 36 * 37 * This will wake up a single thread waiting on this completion. Threads will be 38 * awakened in the same order in which they were queued. 39 * 40 * See also complete_all(), wait_for_completion() and related routines. 41 * 42 * If this function wakes up a task, it executes a full memory barrier before 43 * accessing the task state. 44 */ 45 void complete(struct completion *x) 46 { 47 complete_with_flags(x, 0); 48 } 49 EXPORT_SYMBOL(complete); 50 51 /** 52 * complete_all: - signals all threads waiting on this completion 53 * @x: holds the state of this particular completion 54 * 55 * This will wake up all threads waiting on this particular completion event. 56 * 57 * If this function wakes up a task, it executes a full memory barrier before 58 * accessing the task state. 59 * 60 * Since complete_all() sets the completion of @x permanently to done 61 * to allow multiple waiters to finish, a call to reinit_completion() 62 * must be used on @x if @x is to be used again. The code must make 63 * sure that all waiters have woken and finished before reinitializing 64 * @x. Also note that the function completion_done() can not be used 65 * to know if there are still waiters after complete_all() has been called. 66 */ 67 void complete_all(struct completion *x) 68 { 69 unsigned long flags; 70 71 lockdep_assert_RT_in_threaded_ctx(); 72 73 raw_spin_lock_irqsave(&x->wait.lock, flags); 74 x->done = UINT_MAX; 75 swake_up_all_locked(&x->wait); 76 raw_spin_unlock_irqrestore(&x->wait.lock, flags); 77 } 78 EXPORT_SYMBOL(complete_all); 79 80 static inline long __sched 81 do_wait_for_common(struct completion *x, 82 long (*action)(long), long timeout, int state) 83 { 84 if (!x->done) { 85 DECLARE_SWAITQUEUE(wait); 86 87 do { 88 if (signal_pending_state(state, current)) { 89 timeout = -ERESTARTSYS; 90 break; 91 } 92 __prepare_to_swait(&x->wait, &wait); 93 __set_current_state(state); 94 raw_spin_unlock_irq(&x->wait.lock); 95 timeout = action(timeout); 96 raw_spin_lock_irq(&x->wait.lock); 97 } while (!x->done && timeout); 98 __finish_swait(&x->wait, &wait); 99 if (!x->done) 100 return timeout; 101 } 102 if (x->done != UINT_MAX) 103 x->done--; 104 return timeout ?: 1; 105 } 106 107 static inline long __sched 108 __wait_for_common(struct completion *x, 109 long (*action)(long), long timeout, int state) 110 { 111 might_sleep(); 112 113 complete_acquire(x); 114 115 raw_spin_lock_irq(&x->wait.lock); 116 timeout = do_wait_for_common(x, action, timeout, state); 117 raw_spin_unlock_irq(&x->wait.lock); 118 119 complete_release(x); 120 121 return timeout; 122 } 123 124 static long __sched 125 wait_for_common(struct completion *x, long timeout, int state) 126 { 127 return __wait_for_common(x, schedule_timeout, timeout, state); 128 } 129 130 static long __sched 131 wait_for_common_io(struct completion *x, long timeout, int state) 132 { 133 return __wait_for_common(x, io_schedule_timeout, timeout, state); 134 } 135 136 /** 137 * wait_for_completion: - waits for completion of a task 138 * @x: holds the state of this particular completion 139 * 140 * This waits to be signaled for completion of a specific task. It is NOT 141 * interruptible and there is no timeout. 142 * 143 * See also similar routines (i.e. wait_for_completion_timeout()) with timeout 144 * and interrupt capability. Also see complete(). 145 */ 146 void __sched wait_for_completion(struct completion *x) 147 { 148 wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); 149 } 150 EXPORT_SYMBOL(wait_for_completion); 151 152 /** 153 * wait_for_completion_timeout: - waits for completion of a task (w/timeout) 154 * @x: holds the state of this particular completion 155 * @timeout: timeout value in jiffies 156 * 157 * This waits for either a completion of a specific task to be signaled or for a 158 * specified timeout to expire. The timeout is in jiffies. It is not 159 * interruptible. 160 * 161 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left 162 * till timeout) if completed. 163 */ 164 unsigned long __sched 165 wait_for_completion_timeout(struct completion *x, unsigned long timeout) 166 { 167 return wait_for_common(x, timeout, TASK_UNINTERRUPTIBLE); 168 } 169 EXPORT_SYMBOL(wait_for_completion_timeout); 170 171 /** 172 * wait_for_completion_io: - waits for completion of a task 173 * @x: holds the state of this particular completion 174 * 175 * This waits to be signaled for completion of a specific task. It is NOT 176 * interruptible and there is no timeout. The caller is accounted as waiting 177 * for IO (which traditionally means blkio only). 178 */ 179 void __sched wait_for_completion_io(struct completion *x) 180 { 181 wait_for_common_io(x, MAX_SCHEDULE_TIMEOUT, TASK_UNINTERRUPTIBLE); 182 } 183 EXPORT_SYMBOL(wait_for_completion_io); 184 185 /** 186 * wait_for_completion_io_timeout: - waits for completion of a task (w/timeout) 187 * @x: holds the state of this particular completion 188 * @timeout: timeout value in jiffies 189 * 190 * This waits for either a completion of a specific task to be signaled or for a 191 * specified timeout to expire. The timeout is in jiffies. It is not 192 * interruptible. The caller is accounted as waiting for IO (which traditionally 193 * means blkio only). 194 * 195 * Return: 0 if timed out, and positive (at least 1, or number of jiffies left 196 * till timeout) if completed. 197 */ 198 unsigned long __sched 199 wait_for_completion_io_timeout(struct completion *x, unsigned long timeout) 200 { 201 return wait_for_common_io(x, timeout, TASK_UNINTERRUPTIBLE); 202 } 203 EXPORT_SYMBOL(wait_for_completion_io_timeout); 204 205 /** 206 * wait_for_completion_interruptible: - waits for completion of a task (w/intr) 207 * @x: holds the state of this particular completion 208 * 209 * This waits for completion of a specific task to be signaled. It is 210 * interruptible. 211 * 212 * Return: -ERESTARTSYS if interrupted, 0 if completed. 213 */ 214 int __sched wait_for_completion_interruptible(struct completion *x) 215 { 216 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_INTERRUPTIBLE); 217 218 if (t == -ERESTARTSYS) 219 return t; 220 return 0; 221 } 222 EXPORT_SYMBOL(wait_for_completion_interruptible); 223 224 /** 225 * wait_for_completion_interruptible_timeout: - waits for completion (w/(to,intr)) 226 * @x: holds the state of this particular completion 227 * @timeout: timeout value in jiffies 228 * 229 * This waits for either a completion of a specific task to be signaled or for a 230 * specified timeout to expire. It is interruptible. The timeout is in jiffies. 231 * 232 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, 233 * or number of jiffies left till timeout) if completed. 234 */ 235 long __sched 236 wait_for_completion_interruptible_timeout(struct completion *x, 237 unsigned long timeout) 238 { 239 return wait_for_common(x, timeout, TASK_INTERRUPTIBLE); 240 } 241 EXPORT_SYMBOL(wait_for_completion_interruptible_timeout); 242 243 /** 244 * wait_for_completion_killable: - waits for completion of a task (killable) 245 * @x: holds the state of this particular completion 246 * 247 * This waits to be signaled for completion of a specific task. It can be 248 * interrupted by a kill signal. 249 * 250 * Return: -ERESTARTSYS if interrupted, 0 if completed. 251 */ 252 int __sched wait_for_completion_killable(struct completion *x) 253 { 254 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, TASK_KILLABLE); 255 256 if (t == -ERESTARTSYS) 257 return t; 258 return 0; 259 } 260 EXPORT_SYMBOL(wait_for_completion_killable); 261 262 int __sched wait_for_completion_state(struct completion *x, unsigned int state) 263 { 264 long t = wait_for_common(x, MAX_SCHEDULE_TIMEOUT, state); 265 266 if (t == -ERESTARTSYS) 267 return t; 268 return 0; 269 } 270 EXPORT_SYMBOL(wait_for_completion_state); 271 272 /** 273 * wait_for_completion_killable_timeout: - waits for completion of a task (w/(to,killable)) 274 * @x: holds the state of this particular completion 275 * @timeout: timeout value in jiffies 276 * 277 * This waits for either a completion of a specific task to be 278 * signaled or for a specified timeout to expire. It can be 279 * interrupted by a kill signal. The timeout is in jiffies. 280 * 281 * Return: -ERESTARTSYS if interrupted, 0 if timed out, positive (at least 1, 282 * or number of jiffies left till timeout) if completed. 283 */ 284 long __sched 285 wait_for_completion_killable_timeout(struct completion *x, 286 unsigned long timeout) 287 { 288 return wait_for_common(x, timeout, TASK_KILLABLE); 289 } 290 EXPORT_SYMBOL(wait_for_completion_killable_timeout); 291 292 /** 293 * try_wait_for_completion - try to decrement a completion without blocking 294 * @x: completion structure 295 * 296 * Return: 0 if a decrement cannot be done without blocking 297 * 1 if a decrement succeeded. 298 * 299 * If a completion is being used as a counting completion, 300 * attempt to decrement the counter without blocking. This 301 * enables us to avoid waiting if the resource the completion 302 * is protecting is not available. 303 */ 304 bool try_wait_for_completion(struct completion *x) 305 { 306 unsigned long flags; 307 bool ret = true; 308 309 /* 310 * Since x->done will need to be locked only 311 * in the non-blocking case, we check x->done 312 * first without taking the lock so we can 313 * return early in the blocking case. 314 */ 315 if (!READ_ONCE(x->done)) 316 return false; 317 318 raw_spin_lock_irqsave(&x->wait.lock, flags); 319 if (!x->done) 320 ret = false; 321 else if (x->done != UINT_MAX) 322 x->done--; 323 raw_spin_unlock_irqrestore(&x->wait.lock, flags); 324 return ret; 325 } 326 EXPORT_SYMBOL(try_wait_for_completion); 327 328 /** 329 * completion_done - Test to see if a completion has any waiters 330 * @x: completion structure 331 * 332 * Return: 0 if there are waiters (wait_for_completion() in progress) 333 * 1 if there are no waiters. 334 * 335 * Note, this will always return true if complete_all() was called on @X. 336 */ 337 bool completion_done(struct completion *x) 338 { 339 unsigned long flags; 340 341 if (!READ_ONCE(x->done)) 342 return false; 343 344 /* 345 * If ->done, we need to wait for complete() to release ->wait.lock 346 * otherwise we can end up freeing the completion before complete() 347 * is done referencing it. 348 */ 349 raw_spin_lock_irqsave(&x->wait.lock, flags); 350 raw_spin_unlock_irqrestore(&x->wait.lock, flags); 351 return true; 352 } 353 EXPORT_SYMBOL(completion_done); 354