1 // SPDX-License-Identifier: GPL-2.0
37 		cl->closure_get_happened = false;  in closure_put_after_sub()
39 		if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) {  in closure_put_after_sub()
40 			atomic_set(&cl->remaining,  in closure_put_after_sub()
44 			struct closure *parent = cl->parent;  in closure_put_after_sub()
45 			closure_fn *destructor = cl->fn;  in closure_put_after_sub()
50 				destructor(&cl->work);  in closure_put_after_sub()
61 	closure_put_after_sub(cl, atomic_sub_return_release(v, &cl->remaining));  in closure_sub()
66  * closure_put - decrement a closure's refcount
70 	closure_put_after_sub(cl, atomic_dec_return_release(&cl->remaining));  in closure_put()
75  * closure_wake_up - wake up all closures on a wait list, without memory barrier
81 	struct llist_node *reverse = NULL;  in __closure_wake_up()  local
83 	list = llist_del_all(&wait_list->list);  in __closure_wake_up()
85 	/* We first reverse the list to preserve FIFO ordering and fairness */  in __closure_wake_up()
86 	reverse = llist_reverse_order(list);  in __closure_wake_up()
89 	llist_for_each_entry_safe(cl, t, reverse, list) {  in __closure_wake_up()
97  * closure_wait - add a closure to a waitlist
105 	if (atomic_read(&cl->remaining) & CLOSURE_WAITING)  in closure_wait()
108 	cl->closure_get_happened = true;  in closure_wait()
110 	atomic_add(CLOSURE_WAITING + 1, &cl->remaining);  in closure_wait()
111 	llist_add(&cl->list, &waitlist->list);  in closure_wait()
125 	struct closure_syncer *s = cl->s;  in CLOSURE_CALLBACK()
129 	p = READ_ONCE(s->task);  in CLOSURE_CALLBACK()
130 	s->done = 1;  in CLOSURE_CALLBACK()
139 	cl->s = &s;  in __closure_sync()
154  * closure_return_sync - finish running a closure, synchronously (i.e. waiting
164 	cl->s = &s;  in closure_return_sync()
167 	unsigned flags = atomic_sub_return_release(1 + CLOSURE_RUNNING - CLOSURE_DESTRUCTOR,  in closure_return_sync()
168 						   &cl->remaining);  in closure_return_sync()
183 	if (cl->parent)  in closure_return_sync()
184 		closure_put(cl->parent);  in closure_return_sync()
193 	cl->s = &s;  in __closure_sync_timeout()
202 			 * Carefully undo the continue_at() - but only if it  in __closure_sync_timeout()
206 			unsigned old, new, v = atomic_read(&cl->remaining);  in __closure_sync_timeout()
213 			} while ((v = atomic_cmpxchg(&cl->remaining, old, new)) != old);  in __closure_sync_timeout()
214 			ret = -ETIME;  in __closure_sync_timeout()
234 	BUG_ON(cl->magic == CLOSURE_MAGIC_ALIVE);  in closure_debug_create()
235 	cl->magic = CLOSURE_MAGIC_ALIVE;  in closure_debug_create()
238 	list_add(&cl->all, &closure_list);  in closure_debug_create()
247 	if (cl->magic == CLOSURE_MAGIC_STACK)  in closure_debug_destroy()
250 	BUG_ON(cl->magic != CLOSURE_MAGIC_ALIVE);  in closure_debug_destroy()
251 	cl->magic = CLOSURE_MAGIC_DEAD;  in closure_debug_destroy()
254 	list_del(&cl->all);  in closure_debug_destroy()
259 static int debug_show(struct seq_file *f, void *data)  in debug_show()  argument
266 		int r = atomic_read(&cl->remaining);  in debug_show()
268 		seq_printf(f, "%p: %pS -> %pS p %p r %i ",  in debug_show()
269 			   cl, (void *) cl->ip, cl->fn, cl->parent,  in debug_show()
274 				    work_data_bits(&cl->work)) ? "Q" : "",  in debug_show()
279 				   (void *) cl->waiting_on);  in debug_show()