29 	struct fuse_ring_ent *ent;  member
40 	pdu->ent = ring_ent;  in uring_cmd_set_ring_ent()
48 	return pdu->ent;  in uring_cmd_to_ring_ent()
79 static void fuse_uring_req_end(struct fuse_ring_ent *ent, struct fuse_req *req,  in fuse_uring_req_end()  argument
82 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_req_end()
88 	ent->fuse_req = NULL;  in fuse_uring_req_end()
146 	struct fuse_ring_ent *ent;  in ent_list_request_expired()  local
149 	ent = list_first_entry_or_null(list, struct fuse_ring_ent, list);  in ent_list_request_expired()
150 	if (!ent)  in ent_list_request_expired()
153 	req = ent->fuse_req;  in ent_list_request_expired()
197 		struct fuse_ring_ent *ent, *next;  in fuse_uring_destruct()  local
207 		list_for_each_entry_safe(ent, next, &queue->ent_released,  in fuse_uring_destruct()
209 			list_del_init(&ent->list);  in fuse_uring_destruct()
210 			kfree(ent);  in fuse_uring_destruct()
327 static void fuse_uring_entry_teardown(struct fuse_ring_ent *ent)  in fuse_uring_entry_teardown()  argument
332 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_entry_teardown()
335 	cmd = ent->cmd;  in fuse_uring_entry_teardown()
336 	ent->cmd = NULL;  in fuse_uring_entry_teardown()
337 	req = ent->fuse_req;  in fuse_uring_entry_teardown()
338 	ent->fuse_req = NULL;  in fuse_uring_entry_teardown()
350 	list_move(&ent->list, &queue->ent_released);  in fuse_uring_entry_teardown()
351 	ent->state = FRRS_RELEASED;  in fuse_uring_entry_teardown()
366 	struct fuse_ring_ent *ent, *next;  in fuse_uring_stop_list_entries()  local
371 	list_for_each_entry_safe(ent, next, head, list) {  in fuse_uring_stop_list_entries()
372 		if (ent->state != exp_state) {  in fuse_uring_stop_list_entries()
374 				queue->qid, ent->state, exp_state);  in fuse_uring_stop_list_entries()
378 		ent->state = FRRS_TEARDOWN;  in fuse_uring_stop_list_entries()
379 		list_move(&ent->list, &to_teardown);  in fuse_uring_stop_list_entries()
384 	list_for_each_entry_safe(ent, next, &to_teardown, list) {  in fuse_uring_stop_list_entries()
385 		fuse_uring_entry_teardown(ent);  in fuse_uring_stop_list_entries()
405 	struct fuse_ring_ent *ent;  in fuse_uring_log_ent_state()  local
418 		list_for_each_entry(ent, &queue->ent_w_req_queue, list) {  in fuse_uring_log_ent_state()
419 			pr_info(" ent-req-queue ring=%p qid=%d ent=%p state=%d\n",  in fuse_uring_log_ent_state()
420 				ring, qid, ent, ent->state);  in fuse_uring_log_ent_state()
422 		list_for_each_entry(ent, &queue->ent_commit_queue, list) {  in fuse_uring_log_ent_state()
423 			pr_info(" ent-commit-queue ring=%p qid=%d ent=%p state=%d\n",  in fuse_uring_log_ent_state()
424 				ring, qid, ent, ent->state);  in fuse_uring_log_ent_state()
502 	struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd);  in fuse_uring_cancel()  local
507 	 * direct access on ent - it must not be destructed as long as  in fuse_uring_cancel()
510 	queue = ent->queue;  in fuse_uring_cancel()
512 	if (ent->state == FRRS_AVAILABLE) {  in fuse_uring_cancel()
513 		ent->state = FRRS_USERSPACE;  in fuse_uring_cancel()
514 		list_move_tail(&ent->list, &queue->ent_in_userspace);  in fuse_uring_cancel()
516 		ent->cmd = NULL;  in fuse_uring_cancel()
579 				     struct fuse_ring_ent *ent)  in fuse_uring_copy_from_ring()  argument
587 	err = copy_from_user(&ring_in_out, &ent->headers->ring_ent_in_out,  in fuse_uring_copy_from_ring()
592 	err = import_ubuf(ITER_SOURCE, ent->payload, ring->max_payload_sz,  in fuse_uring_copy_from_ring()
608 				   struct fuse_ring_ent *ent)  in fuse_uring_args_to_ring()  argument
621 	err = import_ubuf(ITER_DEST, ent->payload, ring->max_payload_sz, &iter);  in fuse_uring_args_to_ring()
637 			err = copy_to_user(&ent->headers->op_in, in_args->value,  in fuse_uring_args_to_ring()
658 	err = copy_to_user(&ent->headers->ring_ent_in_out, &ent_in_out,  in fuse_uring_args_to_ring()
663 static int fuse_uring_copy_to_ring(struct fuse_ring_ent *ent,  in fuse_uring_copy_to_ring()  argument
666 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_copy_to_ring()
671 	if (WARN_ON(ent->state != FRRS_FUSE_REQ)) {  in fuse_uring_copy_to_ring()
673 		       queue->qid, ent, ent->state);  in fuse_uring_copy_to_ring()
682 	err = fuse_uring_args_to_ring(ring, req, ent);  in fuse_uring_copy_to_ring()
689 	err = copy_to_user(&ent->headers->in_out, &req->in.h,  in fuse_uring_copy_to_ring()
699 static int fuse_uring_prepare_send(struct fuse_ring_ent *ent,  in fuse_uring_prepare_send()  argument
704 	err = fuse_uring_copy_to_ring(ent, req);  in fuse_uring_prepare_send()
708 		fuse_uring_req_end(ent, req, err);  in fuse_uring_prepare_send()
718 static int fuse_uring_send_next_to_ring(struct fuse_ring_ent *ent,  in fuse_uring_send_next_to_ring()  argument
722 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_send_next_to_ring()
726 	err = fuse_uring_prepare_send(ent, req);  in fuse_uring_send_next_to_ring()
731 	cmd = ent->cmd;  in fuse_uring_send_next_to_ring()
732 	ent->cmd = NULL;  in fuse_uring_send_next_to_ring()
733 	ent->state = FRRS_USERSPACE;  in fuse_uring_send_next_to_ring()
734 	list_move_tail(&ent->list, &queue->ent_in_userspace);  in fuse_uring_send_next_to_ring()
744 static void fuse_uring_ent_avail(struct fuse_ring_ent *ent,  in fuse_uring_ent_avail()  argument
747 	WARN_ON_ONCE(!ent->cmd);  in fuse_uring_ent_avail()
748 	list_move(&ent->list, &queue->ent_avail_queue);  in fuse_uring_ent_avail()
749 	ent->state = FRRS_AVAILABLE;  in fuse_uring_ent_avail()
753 static void fuse_uring_add_to_pq(struct fuse_ring_ent *ent,  in fuse_uring_add_to_pq()  argument
756 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_add_to_pq()
760 	req->ring_entry = ent;  in fuse_uring_add_to_pq()
768 static void fuse_uring_add_req_to_ring_ent(struct fuse_ring_ent *ent,  in fuse_uring_add_req_to_ring_ent()  argument
771 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_add_req_to_ring_ent()
775 	if (WARN_ON_ONCE(ent->state != FRRS_AVAILABLE &&  in fuse_uring_add_req_to_ring_ent()
776 			 ent->state != FRRS_COMMIT)) {  in fuse_uring_add_req_to_ring_ent()
777 		pr_warn("%s qid=%d state=%d\n", __func__, ent->queue->qid,  in fuse_uring_add_req_to_ring_ent()
778 			ent->state);  in fuse_uring_add_req_to_ring_ent()
782 	ent->fuse_req = req;  in fuse_uring_add_req_to_ring_ent()
783 	ent->state = FRRS_FUSE_REQ;  in fuse_uring_add_req_to_ring_ent()
784 	list_move_tail(&ent->list, &queue->ent_w_req_queue);  in fuse_uring_add_req_to_ring_ent()
785 	fuse_uring_add_to_pq(ent, req);  in fuse_uring_add_req_to_ring_ent()
789 static struct fuse_req *fuse_uring_ent_assign_req(struct fuse_ring_ent *ent)  in fuse_uring_ent_assign_req()  argument
793 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_ent_assign_req()
801 		fuse_uring_add_req_to_ring_ent(ent, req);  in fuse_uring_ent_assign_req()
811 static void fuse_uring_commit(struct fuse_ring_ent *ent, struct fuse_req *req,  in fuse_uring_commit()  argument
814 	struct fuse_ring *ring = ent->queue->ring;  in fuse_uring_commit()
818 	err = copy_from_user(&req->out.h, &ent->headers->in_out,  in fuse_uring_commit()
831 	err = fuse_uring_copy_from_ring(ring, req, ent);  in fuse_uring_commit()
833 	fuse_uring_req_end(ent, req, err);  in fuse_uring_commit()
839 static void fuse_uring_next_fuse_req(struct fuse_ring_ent *ent,  in fuse_uring_next_fuse_req()  argument
848 	fuse_uring_ent_avail(ent, queue);  in fuse_uring_next_fuse_req()
849 	req = fuse_uring_ent_assign_req(ent);  in fuse_uring_next_fuse_req()
853 		err = fuse_uring_send_next_to_ring(ent, req, issue_flags);  in fuse_uring_next_fuse_req()
859 static int fuse_ring_ent_set_commit(struct fuse_ring_ent *ent)  in fuse_ring_ent_set_commit()  argument
861 	struct fuse_ring_queue *queue = ent->queue;  in fuse_ring_ent_set_commit()
865 	if (WARN_ON_ONCE(ent->state != FRRS_USERSPACE))  in fuse_ring_ent_set_commit()
868 	ent->state = FRRS_COMMIT;  in fuse_ring_ent_set_commit()
869 	list_move(&ent->list, &queue->ent_commit_queue);  in fuse_ring_ent_set_commit()
879 	struct fuse_ring_ent *ent;  in fuse_uring_commit_fetch()  local
918 	ent = req->ring_entry;  in fuse_uring_commit_fetch()
921 	err = fuse_ring_ent_set_commit(ent);  in fuse_uring_commit_fetch()
924 				    queue->qid, commit_id, ent->state);  in fuse_uring_commit_fetch()
932 	ent->cmd = cmd;  in fuse_uring_commit_fetch()
936 	fuse_uring_prepare_cancel(cmd, issue_flags, ent);  in fuse_uring_commit_fetch()
937 	fuse_uring_commit(ent, req, issue_flags);  in fuse_uring_commit_fetch()
945 	fuse_uring_next_fuse_req(ent, queue, issue_flags);  in fuse_uring_commit_fetch()
977 static void fuse_uring_do_register(struct fuse_ring_ent *ent,  in fuse_uring_do_register()  argument
981 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_do_register()
986 	fuse_uring_prepare_cancel(cmd, issue_flags, ent);  in fuse_uring_do_register()
989 	ent->cmd = cmd;  in fuse_uring_do_register()
990 	fuse_uring_ent_avail(ent, queue);  in fuse_uring_do_register()
1035 	struct fuse_ring_ent *ent;  in fuse_uring_create_ring_ent()  local
1061 	ent = kzalloc(sizeof(*ent), GFP_KERNEL_ACCOUNT);  in fuse_uring_create_ring_ent()
1062 	if (!ent)  in fuse_uring_create_ring_ent()
1065 	INIT_LIST_HEAD(&ent->list);  in fuse_uring_create_ring_ent()
1067 	ent->queue = queue;  in fuse_uring_create_ring_ent()
1068 	ent->headers = iov[0].iov_base;  in fuse_uring_create_ring_ent()
1069 	ent->payload = iov[1].iov_base;  in fuse_uring_create_ring_ent()
1072 	return ent;  in fuse_uring_create_ring_ent()
1085 	struct fuse_ring_ent *ent;  in fuse_uring_register()  local
1113 	ent = fuse_uring_create_ring_ent(cmd, queue);  in fuse_uring_register()
1114 	if (IS_ERR(ent))  in fuse_uring_register()
1115 		return PTR_ERR(ent);  in fuse_uring_register()
1117 	fuse_uring_do_register(ent, cmd, issue_flags);  in fuse_uring_register()
1193 static void fuse_uring_send(struct fuse_ring_ent *ent, struct io_uring_cmd *cmd,  in fuse_uring_send()  argument
1196 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_send()
1199 	ent->state = FRRS_USERSPACE;  in fuse_uring_send()
1200 	list_move_tail(&ent->list, &queue->ent_in_userspace);  in fuse_uring_send()
1201 	ent->cmd = NULL;  in fuse_uring_send()
1215 	struct fuse_ring_ent *ent = uring_cmd_to_ring_ent(cmd);  in fuse_uring_send_in_task()  local
1216 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_send_in_task()
1220 		err = fuse_uring_prepare_send(ent, ent->fuse_req);  in fuse_uring_send_in_task()
1222 			fuse_uring_next_fuse_req(ent, queue, issue_flags);  in fuse_uring_send_in_task()
1229 	fuse_uring_send(ent, cmd, err, issue_flags);  in fuse_uring_send_in_task()
1250 static void fuse_uring_dispatch_ent(struct fuse_ring_ent *ent)  in fuse_uring_dispatch_ent()  argument
1252 	struct io_uring_cmd *cmd = ent->cmd;  in fuse_uring_dispatch_ent()
1254 	uring_cmd_set_ring_ent(cmd, ent);  in fuse_uring_dispatch_ent()
1264 	struct fuse_ring_ent *ent = NULL;  in fuse_uring_queue_fuse_req()  local
1281 	ent = list_first_entry_or_null(&queue->ent_avail_queue,  in fuse_uring_queue_fuse_req()
1283 	if (ent)  in fuse_uring_queue_fuse_req()
1284 		fuse_uring_add_req_to_ring_ent(ent, req);  in fuse_uring_queue_fuse_req()
1289 	if (ent)  in fuse_uring_queue_fuse_req()
1290 		fuse_uring_dispatch_ent(ent);  in fuse_uring_queue_fuse_req()
1307 	struct fuse_ring_ent *ent = NULL;  in fuse_uring_queue_bq_req()  local
1323 	ent = list_first_entry_or_null(&queue->ent_avail_queue,  in fuse_uring_queue_bq_req()
1339 	if (ent && req) {  in fuse_uring_queue_bq_req()
1340 		fuse_uring_add_req_to_ring_ent(ent, req);  in fuse_uring_queue_bq_req()
1343 		fuse_uring_dispatch_ent(ent);  in fuse_uring_queue_bq_req()