51 static void fuse_uring_flush_bg(struct fuse_ring_queue *queue)  in fuse_uring_flush_bg()  argument
53 	struct fuse_ring *ring = queue->ring;  in fuse_uring_flush_bg()
56 	lockdep_assert_held(&queue->lock);  in fuse_uring_flush_bg()
66 		!queue->active_background) &&  in fuse_uring_flush_bg()
67 	       (!list_empty(&queue->fuse_req_bg_queue))) {  in fuse_uring_flush_bg()
70 		req = list_first_entry(&queue->fuse_req_bg_queue,  in fuse_uring_flush_bg()
73 		queue->active_background++;  in fuse_uring_flush_bg()
75 		list_move_tail(&req->list, &queue->fuse_req_queue);  in fuse_uring_flush_bg()
82 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_req_end()  local
83 	struct fuse_ring *ring = queue->ring;  in fuse_uring_req_end()
86 	lockdep_assert_not_held(&queue->lock);  in fuse_uring_req_end()
87 	spin_lock(&queue->lock);  in fuse_uring_req_end()
91 		queue->active_background--;  in fuse_uring_req_end()
93 		fuse_uring_flush_bg(queue);  in fuse_uring_req_end()
97 	spin_unlock(&queue->lock);  in fuse_uring_req_end()
107 static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue)  in fuse_uring_abort_end_queue_requests()  argument
112 	spin_lock(&queue->lock);  in fuse_uring_abort_end_queue_requests()
113 	list_for_each_entry(req, &queue->fuse_req_queue, list)  in fuse_uring_abort_end_queue_requests()
115 	list_splice_init(&queue->fuse_req_queue, &req_list);  in fuse_uring_abort_end_queue_requests()
116 	spin_unlock(&queue->lock);  in fuse_uring_abort_end_queue_requests()
125 	struct fuse_ring_queue *queue;  in fuse_uring_abort_end_requests()  local
129 		queue = READ_ONCE(ring->queues[qid]);  in fuse_uring_abort_end_requests()
130 		if (!queue)  in fuse_uring_abort_end_requests()
133 		queue->stopped = true;  in fuse_uring_abort_end_requests()
136 		spin_lock(&queue->lock);  in fuse_uring_abort_end_requests()
138 		fuse_uring_flush_bg(queue);  in fuse_uring_abort_end_requests()
140 		spin_unlock(&queue->lock);  in fuse_uring_abort_end_requests()
141 		fuse_uring_abort_end_queue_requests(queue);  in fuse_uring_abort_end_requests()
163 	struct fuse_ring_queue *queue;  in fuse_uring_request_expired()  local
170 		queue = READ_ONCE(ring->queues[qid]);  in fuse_uring_request_expired()
171 		if (!queue)  in fuse_uring_request_expired()
174 		spin_lock(&queue->lock);  in fuse_uring_request_expired()
175 		if (fuse_request_expired(fc, &queue->fuse_req_queue) ||  in fuse_uring_request_expired()
176 		    fuse_request_expired(fc, &queue->fuse_req_bg_queue) ||  in fuse_uring_request_expired()
177 		    ent_list_request_expired(fc, &queue->ent_w_req_queue) ||  in fuse_uring_request_expired()
178 		    ent_list_request_expired(fc, &queue->ent_in_userspace)) {  in fuse_uring_request_expired()
179 			spin_unlock(&queue->lock);  in fuse_uring_request_expired()
182 		spin_unlock(&queue->lock);  in fuse_uring_request_expired()
197 		struct fuse_ring_queue *queue = ring->queues[qid];  in fuse_uring_destruct()  local
200 		if (!queue)  in fuse_uring_destruct()
203 		WARN_ON(!list_empty(&queue->ent_avail_queue));  in fuse_uring_destruct()
204 		WARN_ON(!list_empty(&queue->ent_w_req_queue));  in fuse_uring_destruct()
205 		WARN_ON(!list_empty(&queue->ent_commit_queue));  in fuse_uring_destruct()
206 		WARN_ON(!list_empty(&queue->ent_in_userspace));  in fuse_uring_destruct()
208 		list_for_each_entry_safe(ent, next, &queue->ent_released,  in fuse_uring_destruct()
214 		kfree(queue->fpq.processing);  in fuse_uring_destruct()
215 		kfree(queue);  in fuse_uring_destruct()
274 	struct fuse_ring_queue *queue;  in fuse_uring_create_queue()  local
277 	queue = kzalloc(sizeof(*queue), GFP_KERNEL_ACCOUNT);  in fuse_uring_create_queue()
278 	if (!queue)  in fuse_uring_create_queue()
282 		kfree(queue);  in fuse_uring_create_queue()
286 	queue->qid = qid;  in fuse_uring_create_queue()
287 	queue->ring = ring;  in fuse_uring_create_queue()
288 	spin_lock_init(&queue->lock);  in fuse_uring_create_queue()
290 	INIT_LIST_HEAD(&queue->ent_avail_queue);  in fuse_uring_create_queue()
291 	INIT_LIST_HEAD(&queue->ent_commit_queue);  in fuse_uring_create_queue()
292 	INIT_LIST_HEAD(&queue->ent_w_req_queue);  in fuse_uring_create_queue()
293 	INIT_LIST_HEAD(&queue->ent_in_userspace);  in fuse_uring_create_queue()
294 	INIT_LIST_HEAD(&queue->fuse_req_queue);  in fuse_uring_create_queue()
295 	INIT_LIST_HEAD(&queue->fuse_req_bg_queue);  in fuse_uring_create_queue()
296 	INIT_LIST_HEAD(&queue->ent_released);  in fuse_uring_create_queue()
298 	queue->fpq.processing = pq;  in fuse_uring_create_queue()
299 	fuse_pqueue_init(&queue->fpq);  in fuse_uring_create_queue()
304 		kfree(queue->fpq.processing);  in fuse_uring_create_queue()
305 		kfree(queue);  in fuse_uring_create_queue()
312 	WRITE_ONCE(ring->queues[qid], queue);  in fuse_uring_create_queue()
315 	return queue;  in fuse_uring_create_queue()
333 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_entry_teardown()  local
335 	spin_lock(&queue->lock);  in fuse_uring_entry_teardown()
351 	list_move(&ent->list, &queue->ent_released);  in fuse_uring_entry_teardown()
353 	spin_unlock(&queue->lock);  in fuse_uring_entry_teardown()
363 					 struct fuse_ring_queue *queue,  in fuse_uring_stop_list_entries()  argument
366 	struct fuse_ring *ring = queue->ring;  in fuse_uring_stop_list_entries()
371 	spin_lock(&queue->lock);  in fuse_uring_stop_list_entries()
375 				queue->qid, ent->state, exp_state);  in fuse_uring_stop_list_entries()
382 	spin_unlock(&queue->lock);  in fuse_uring_stop_list_entries()
392 static void fuse_uring_teardown_entries(struct fuse_ring_queue *queue)  in fuse_uring_teardown_entries()  argument
394 	fuse_uring_stop_list_entries(&queue->ent_in_userspace, queue,  in fuse_uring_teardown_entries()
396 	fuse_uring_stop_list_entries(&queue->ent_avail_queue, queue,  in fuse_uring_teardown_entries()
409 		struct fuse_ring_queue *queue = ring->queues[qid];  in fuse_uring_log_ent_state()  local
411 		if (!queue)  in fuse_uring_log_ent_state()
414 		spin_lock(&queue->lock);  in fuse_uring_log_ent_state()
419 		list_for_each_entry(ent, &queue->ent_w_req_queue, list) {  in fuse_uring_log_ent_state()
423 		list_for_each_entry(ent, &queue->ent_commit_queue, list) {  in fuse_uring_log_ent_state()
427 		spin_unlock(&queue->lock);  in fuse_uring_log_ent_state()
440 		struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]);  in fuse_uring_async_stop_queues()  local
442 		if (!queue)  in fuse_uring_async_stop_queues()
445 		fuse_uring_teardown_entries(queue);  in fuse_uring_async_stop_queues()
475 		struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]);  in fuse_uring_stop_queues()  local
477 		if (!queue)  in fuse_uring_stop_queues()
480 		fuse_uring_teardown_entries(queue);  in fuse_uring_stop_queues()
504 	struct fuse_ring_queue *queue;  in fuse_uring_cancel()  local
511 	queue = ent->queue;  in fuse_uring_cancel()
512 	spin_lock(&queue->lock);  in fuse_uring_cancel()
515 		list_move_tail(&ent->list, &queue->ent_in_userspace);  in fuse_uring_cancel()
519 	spin_unlock(&queue->lock);  in fuse_uring_cancel()
670 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_copy_to_ring()  local
671 	struct fuse_ring *ring = queue->ring;  in fuse_uring_copy_to_ring()
677 		       queue->qid, ent, ent->state);  in fuse_uring_copy_to_ring()
726 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_send_next_to_ring()  local
734 	spin_lock(&queue->lock);  in fuse_uring_send_next_to_ring()
738 	list_move_tail(&ent->list, &queue->ent_in_userspace);  in fuse_uring_send_next_to_ring()
739 	spin_unlock(&queue->lock);  in fuse_uring_send_next_to_ring()
749 				 struct fuse_ring_queue *queue)  in fuse_uring_ent_avail()  argument
752 	list_move(&ent->list, &queue->ent_avail_queue);  in fuse_uring_ent_avail()
760 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_add_to_pq()  local
761 	struct fuse_pqueue *fpq = &queue->fpq;  in fuse_uring_add_to_pq()
775 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_add_req_to_ring_ent()  local
777 	lockdep_assert_held(&queue->lock);  in fuse_uring_add_req_to_ring_ent()
781 		pr_warn("%s qid=%d state=%d\n", __func__, ent->queue->qid,  in fuse_uring_add_req_to_ring_ent()
788 	list_move_tail(&ent->list, &queue->ent_w_req_queue);  in fuse_uring_add_req_to_ring_ent()
794 	__must_hold(&queue->lock)  in fuse_uring_ent_assign_req()
797 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_ent_assign_req()  local
798 	struct list_head *req_queue = &queue->fuse_req_queue;  in fuse_uring_ent_assign_req()
800 	lockdep_assert_held(&queue->lock);  in fuse_uring_ent_assign_req()
818 	struct fuse_ring *ring = ent->queue->ring;  in fuse_uring_commit()
844 				     struct fuse_ring_queue *queue,  in fuse_uring_next_fuse_req()  argument
851 	spin_lock(&queue->lock);  in fuse_uring_next_fuse_req()
852 	fuse_uring_ent_avail(ent, queue);  in fuse_uring_next_fuse_req()
854 	spin_unlock(&queue->lock);  in fuse_uring_next_fuse_req()
865 	struct fuse_ring_queue *queue = ent->queue;  in fuse_ring_ent_set_commit()  local
867 	lockdep_assert_held(&queue->lock);  in fuse_ring_ent_set_commit()
873 	list_move(&ent->list, &queue->ent_commit_queue);  in fuse_ring_ent_set_commit()
886 	struct fuse_ring_queue *queue;  in fuse_uring_commit_fetch()  local
899 	queue = ring->queues[qid];  in fuse_uring_commit_fetch()
900 	if (!queue)  in fuse_uring_commit_fetch()
902 	fpq = &queue->fpq;  in fuse_uring_commit_fetch()
904 	if (!READ_ONCE(fc->connected) || READ_ONCE(queue->stopped))  in fuse_uring_commit_fetch()
907 	spin_lock(&queue->lock);  in fuse_uring_commit_fetch()
916 		pr_info("qid=%d commit_id %llu not found\n", queue->qid,  in fuse_uring_commit_fetch()
918 		spin_unlock(&queue->lock);  in fuse_uring_commit_fetch()
928 				    queue->qid, commit_id, ent->state);  in fuse_uring_commit_fetch()
929 		spin_unlock(&queue->lock);  in fuse_uring_commit_fetch()
937 	spin_unlock(&queue->lock);  in fuse_uring_commit_fetch()
949 	fuse_uring_next_fuse_req(ent, queue, issue_flags);  in fuse_uring_commit_fetch()
956 	struct fuse_ring_queue *queue;  in is_ring_ready()  local
963 		queue = ring->queues[qid];  in is_ring_ready()
964 		if (!queue) {  in is_ring_ready()
969 		spin_lock(&queue->lock);  in is_ring_ready()
970 		if (list_empty(&queue->ent_avail_queue))  in is_ring_ready()
972 		spin_unlock(&queue->lock);  in is_ring_ready()
985 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_do_register()  local
986 	struct fuse_ring *ring = queue->ring;  in fuse_uring_do_register()
992 	spin_lock(&queue->lock);  in fuse_uring_do_register()
994 	fuse_uring_ent_avail(ent, queue);  in fuse_uring_do_register()
995 	spin_unlock(&queue->lock);  in fuse_uring_do_register()
998 		bool ready = is_ring_ready(ring, queue->qid);  in fuse_uring_do_register()
1036 			   struct fuse_ring_queue *queue)  in fuse_uring_create_ring_ent()  argument
1038 	struct fuse_ring *ring = queue->ring;  in fuse_uring_create_ring_ent()
1071 	ent->queue = queue;  in fuse_uring_create_ring_ent()
1088 	struct fuse_ring_queue *queue;  in fuse_uring_register()  local
1105 	queue = ring->queues[qid];  in fuse_uring_register()
1106 	if (!queue) {  in fuse_uring_register()
1107 		queue = fuse_uring_create_queue(ring, qid);  in fuse_uring_register()
1108 		if (!queue)  in fuse_uring_register()
1117 	ent = fuse_uring_create_ring_ent(cmd, queue);  in fuse_uring_register()
1200 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_send()  local
1202 	spin_lock(&queue->lock);  in fuse_uring_send()
1204 	list_move_tail(&ent->list, &queue->ent_in_userspace);  in fuse_uring_send()
1206 	spin_unlock(&queue->lock);  in fuse_uring_send()
1221 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_send_in_task()  local
1227 			fuse_uring_next_fuse_req(ent, queue, issue_flags);  in fuse_uring_send_in_task()
1240 	struct fuse_ring_queue *queue;  in fuse_uring_task_to_queue()  local
1249 	queue = ring->queues[qid];  in fuse_uring_task_to_queue()
1250 	WARN_ONCE(!queue, "Missing queue for qid %d\n", qid);  in fuse_uring_task_to_queue()
1252 	return queue;  in fuse_uring_task_to_queue()
1268 	struct fuse_ring_queue *queue;  in fuse_uring_queue_fuse_req()  local
1273 	queue = fuse_uring_task_to_queue(ring);  in fuse_uring_queue_fuse_req()
1274 	if (!queue)  in fuse_uring_queue_fuse_req()
1279 	spin_lock(&queue->lock);  in fuse_uring_queue_fuse_req()
1281 	if (unlikely(queue->stopped))  in fuse_uring_queue_fuse_req()
1285 	req->ring_queue = queue;  in fuse_uring_queue_fuse_req()
1286 	ent = list_first_entry_or_null(&queue->ent_avail_queue,  in fuse_uring_queue_fuse_req()
1291 		list_add_tail(&req->list, &queue->fuse_req_queue);  in fuse_uring_queue_fuse_req()
1292 	spin_unlock(&queue->lock);  in fuse_uring_queue_fuse_req()
1300 	spin_unlock(&queue->lock);  in fuse_uring_queue_fuse_req()
1311 	struct fuse_ring_queue *queue;  in fuse_uring_queue_bq_req()  local
1314 	queue = fuse_uring_task_to_queue(ring);  in fuse_uring_queue_bq_req()
1315 	if (!queue)  in fuse_uring_queue_bq_req()
1318 	spin_lock(&queue->lock);  in fuse_uring_queue_bq_req()
1319 	if (unlikely(queue->stopped)) {  in fuse_uring_queue_bq_req()
1320 		spin_unlock(&queue->lock);  in fuse_uring_queue_bq_req()
1325 	req->ring_queue = queue;  in fuse_uring_queue_bq_req()
1326 	list_add_tail(&req->list, &queue->fuse_req_bg_queue);  in fuse_uring_queue_bq_req()
1328 	ent = list_first_entry_or_null(&queue->ent_avail_queue,  in fuse_uring_queue_bq_req()
1334 	fuse_uring_flush_bg(queue);  in fuse_uring_queue_bq_req()
1342 	req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req,  in fuse_uring_queue_bq_req()
1346 		spin_unlock(&queue->lock);  in fuse_uring_queue_bq_req()
1350 		spin_unlock(&queue->lock);  in fuse_uring_queue_bq_req()
1358 	struct fuse_ring_queue *queue = req->ring_queue;  in fuse_uring_remove_pending_req()  local
1360 	return fuse_remove_pending_req(req, &queue->lock);  in fuse_uring_remove_pending_req()