30 static inline void closure_put_after_sub(struct closure *cl, int flags)  in closure_put_after_sub()  argument
37 		cl->closure_get_happened = false;  in closure_put_after_sub()
39 		if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) {  in closure_put_after_sub()
40 			atomic_set(&cl->remaining,  in closure_put_after_sub()
42 			closure_queue(cl);  in closure_put_after_sub()
44 			struct closure *parent = cl->parent;  in closure_put_after_sub()
45 			closure_fn *destructor = cl->fn;  in closure_put_after_sub()
47 			closure_debug_destroy(cl);  in closure_put_after_sub()
50 				destructor(&cl->work);  in closure_put_after_sub()
59 void closure_sub(struct closure *cl, int v)  in closure_sub()  argument
61 	closure_put_after_sub(cl, atomic_sub_return_release(v, &cl->remaining));  in closure_sub()
68 void closure_put(struct closure *cl)  in closure_put()  argument
70 	closure_put_after_sub(cl, atomic_dec_return_release(&cl->remaining));  in closure_put()
80 	struct closure *cl, *t;  in __closure_wake_up()  local
89 	llist_for_each_entry_safe(cl, t, reverse, list) {  in __closure_wake_up()
90 		closure_set_waiting(cl, 0);  in __closure_wake_up()
91 		closure_sub(cl, CLOSURE_WAITING + 1);  in __closure_wake_up()
103 bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl)  in closure_wait()  argument
105 	if (atomic_read(&cl->remaining) & CLOSURE_WAITING)  in closure_wait()
108 	cl->closure_get_happened = true;  in closure_wait()
109 	closure_set_waiting(cl, _RET_IP_);  in closure_wait()
110 	atomic_add(CLOSURE_WAITING + 1, &cl->remaining);  in closure_wait()
111 	llist_add(&cl->list, &waitlist->list);  in closure_wait()
124 	struct closure *cl = container_of(ws, struct closure, work);  in CLOSURE_CALLBACK()  local
125 	struct closure_syncer *s = cl->s;  in CLOSURE_CALLBACK()
135 void __sched __closure_sync(struct closure *cl)  in __closure_sync()  argument
139 	cl->s = &s;  in __closure_sync()
140 	continue_at(cl, closure_sync_fn, NULL);  in __closure_sync()
160 void __sched closure_return_sync(struct closure *cl)  in closure_return_sync()  argument
164 	cl->s = &s;  in closure_return_sync()
165 	set_closure_fn(cl, closure_sync_fn, NULL);  in closure_return_sync()
168 						   &cl->remaining);  in closure_return_sync()
183 	if (cl->parent)  in closure_return_sync()
184 		closure_put(cl->parent);  in closure_return_sync()
188 int __sched __closure_sync_timeout(struct closure *cl, unsigned long timeout)  in __closure_sync_timeout()  argument
193 	cl->s = &s;  in __closure_sync_timeout()
194 	continue_at(cl, closure_sync_fn, NULL);  in __closure_sync_timeout()
206 			unsigned old, new, v = atomic_read(&cl->remaining);  in __closure_sync_timeout()
213 			} while ((v = atomic_cmpxchg(&cl->remaining, old, new)) != old);  in __closure_sync_timeout()
230 void closure_debug_create(struct closure *cl)  in closure_debug_create()  argument
234 	BUG_ON(cl->magic == CLOSURE_MAGIC_ALIVE);  in closure_debug_create()
235 	cl->magic = CLOSURE_MAGIC_ALIVE;  in closure_debug_create()
238 	list_add(&cl->all, &closure_list);  in closure_debug_create()
243 void closure_debug_destroy(struct closure *cl)  in closure_debug_destroy()  argument
247 	if (cl->magic == CLOSURE_MAGIC_STACK)  in closure_debug_destroy()
250 	BUG_ON(cl->magic != CLOSURE_MAGIC_ALIVE);  in closure_debug_destroy()
251 	cl->magic = CLOSURE_MAGIC_DEAD;  in closure_debug_destroy()
254 	list_del(&cl->all);  in closure_debug_destroy()
261 	struct closure *cl;  in debug_show()  local
265 	list_for_each_entry(cl, &closure_list, all) {  in debug_show()
266 		int r = atomic_read(&cl->remaining);  in debug_show()
269 			   cl, (void *) cl->ip, cl->fn, cl->parent,  in debug_show()
274 				    work_data_bits(&cl->work)) ? "Q" : "",  in debug_show()
279 				   (void *) cl->waiting_on);  in debug_show()