534 		struct io_uring_cqe *cqe;  in __io_cqring_overflow_flush()  local
540 		if (ocqe->cqe.flags & IORING_CQE_F_32 ||  in __io_cqring_overflow_flush()
549 			if (!io_get_cqe_overflow(ctx, &cqe, true, is_cqe32))  in __io_cqring_overflow_flush()
551 			memcpy(cqe, &ocqe->cqe, cqe_size);  in __io_cqring_overflow_flush()
661 					     struct io_cqe *cqe,  in io_alloc_ocqe()  argument
668 	if (cqe->flags & IORING_CQE_F_32 || ctx->flags & IORING_SETUP_CQE32) {  in io_alloc_ocqe()
674 	trace_io_uring_cqe_overflow(ctx, cqe->user_data, cqe->res, cqe->flags, ocqe);  in io_alloc_ocqe()
676 		ocqe->cqe.user_data = cqe->user_data;  in io_alloc_ocqe()
677 		ocqe->cqe.res = cqe->res;  in io_alloc_ocqe()
678 		ocqe->cqe.flags = cqe->flags;  in io_alloc_ocqe()
680 			ocqe->cqe.big_cqe[0] = big_cqe->extra1;  in io_alloc_ocqe()
681 			ocqe->cqe.big_cqe[1] = big_cqe->extra2;  in io_alloc_ocqe()
710 		struct io_uring_cqe *cqe = &ctx->rings->cqes[off];  in io_fill_nop_cqe()  local
712 		cqe->user_data = 0;  in io_fill_nop_cqe()
713 		cqe->res = 0;  in io_fill_nop_cqe()
714 		cqe->flags = IORING_CQE_F_SKIP;  in io_fill_nop_cqe()
769 	struct io_uring_cqe *cqe;  in io_fill_cqe_aux32()  local
773 	if (unlikely(!io_get_cqe(ctx, &cqe, true)))  in io_fill_cqe_aux32()
776 	memcpy(cqe, src_cqe, 2 * sizeof(*cqe));  in io_fill_cqe_aux32()
777 	trace_io_uring_complete(ctx, NULL, cqe);  in io_fill_cqe_aux32()
785 	struct io_uring_cqe *cqe;  in io_fill_cqe_aux()  local
787 	if (likely(io_get_cqe(ctx, &cqe, cqe32))) {  in io_fill_cqe_aux()
788 		WRITE_ONCE(cqe->user_data, user_data);  in io_fill_cqe_aux()
789 		WRITE_ONCE(cqe->res, res);  in io_fill_cqe_aux()
790 		WRITE_ONCE(cqe->flags, cflags);  in io_fill_cqe_aux()
793 			WRITE_ONCE(cqe->big_cqe[0], 0);  in io_fill_cqe_aux()
794 			WRITE_ONCE(cqe->big_cqe[1], 0);  in io_fill_cqe_aux()
797 		trace_io_uring_complete(ctx, NULL, cqe);  in io_fill_cqe_aux()
808 static __cold void io_cqe_overflow(struct io_ring_ctx *ctx, struct io_cqe *cqe,  in io_cqe_overflow()  argument
813 	ocqe = io_alloc_ocqe(ctx, cqe, big_cqe, GFP_KERNEL);  in io_cqe_overflow()
820 					  struct io_cqe *cqe,  in io_cqe_overflow_locked()  argument
825 	ocqe = io_alloc_ocqe(ctx, cqe, big_cqe, GFP_NOWAIT);  in io_cqe_overflow_locked()
836 		struct io_cqe cqe = io_init_cqe(user_data, res, cflags);  in io_post_aux_cqe()  local
838 		filled = io_cqe_overflow_locked(ctx, &cqe, NULL);  in io_post_aux_cqe()
854 		struct io_cqe cqe = io_init_cqe(user_data, res, cflags);  in io_add_aux_cqe()  local
856 		io_cqe_overflow(ctx, &cqe, NULL);  in io_add_aux_cqe()
883 		posted = io_fill_cqe_aux(ctx, req->cqe.user_data, res, cflags);  in io_req_post_cqe()
886 		posted = io_fill_cqe_aux(ctx, req->cqe.user_data, res, cflags);  in io_req_post_cqe()
897 bool io_req_post_cqe32(struct io_kiocb *req, struct io_uring_cqe cqe[2])  in io_req_post_cqe32()
905 	cqe[0].user_data = req->cqe.user_data;  in io_req_post_cqe32()
908 		posted = io_fill_cqe_aux32(ctx, cqe);  in io_req_post_cqe32()
911 		posted = io_fill_cqe_aux32(ctx, cqe);  in io_req_post_cqe32()
1048 	io_req_defer_failed(req, req->cqe.res);  in io_req_task_cancel()
1159 				io_cqe_overflow(ctx, &req->cqe, &req->big_cqe);  in __io_submit_flush_completions()
1161 				io_cqe_overflow_locked(ctx, &req->cqe, &req->big_cqe);  in __io_submit_flush_completions()
1372 		req->file = io_file_get_fixed(req, req->cqe.fd, issue_flags);  in io_assign_file()
1374 		req->file = io_file_get_normal(req, req->cqe.fd);  in io_assign_file()
1673 		io_req_defer_failed(req, req->cqe.res);  in io_queue_sqe_fallback()
1747 	req->cqe.user_data = READ_ONCE(sqe->user_data);  in io_init_req()
1817 		req->cqe.fd = READ_ONCE(sqe->fd);  in io_init_req()