Lines Matching +full:sr +full:- +full:iov
1 // SPDX-License-Identifier: GPL-2.0
79 /* per-invocation mshot limit */
89 * The UAPI flags are the lower 8 bits, as that's all sqe->ioprio will hold
128 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags ||
129 sqe->buf_index || sqe->splice_fd_in))
130 return -EINVAL;
132 shutdown->how = READ_ONCE(sqe->len);
133 req->flags |= REQ_F_FORCE_ASYNC;
145 sock = sock_from_file(req->file);
147 return -ENOTSOCK;
149 ret = __sys_shutdown_sock(sock, shutdown->how);
158 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET;
163 if (kmsg->vec.iovec)
164 io_vec_free(&kmsg->vec);
169 struct io_async_msghdr *hdr = req->async_data;
178 io_alloc_cache_vec_kasan(&hdr->vec);
179 if (hdr->vec.nr > IO_VEC_CACHE_SOFT_CAP)
180 io_vec_free(&hdr->vec);
182 if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr))
188 struct io_ring_ctx *ctx = req->ctx;
191 hdr = io_uring_alloc_async_data(&ctx->netmsg_cache, req);
195 /* If the async data was cached, we might have an iov cached inside. */
196 if (hdr->vec.iovec)
197 req->flags |= REQ_F_NEED_CLEANUP;
204 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
206 req->flags &= ~REQ_F_BL_EMPTY;
207 sr->done_io = 0;
208 sr->flags &= ~IORING_RECV_RETRY_CLEAR;
209 sr->len = sr->mshot_len;
216 struct iovec *iov;
219 if (iomsg->vec.iovec) {
220 nr_segs = iomsg->vec.nr;
221 iov = iomsg->vec.iovec;
224 iov = &iomsg->fast_iov;
227 ret = __import_iovec(ddir, uiov, uvec_seg, nr_segs, &iov,
228 &iomsg->msg.msg_iter, io_is_compat(req->ctx));
232 if (iov) {
233 req->flags |= REQ_F_NEED_CLEANUP;
234 io_vec_reset_iovec(&iomsg->vec, iov, iomsg->msg.msg_iter.nr_segs);
244 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
248 if (copy_from_user(msg, sr->umsg_compat, sizeof(*msg)))
249 return -EFAULT;
251 ret = __get_compat_msghdr(&iomsg->msg, msg, save_addr);
255 uiov = compat_ptr(msg->msg_iov);
256 if (req->flags & REQ_F_BUFFER_SELECT) {
257 if (msg->msg_iovlen == 0) {
258 sr->len = 0;
259 } else if (msg->msg_iovlen > 1) {
260 return -EINVAL;
265 return -EFAULT;
266 sr->len = tmp_iov.iov_len;
276 return -EFAULT;
277 unsafe_get_user(msg->msg_name, &umsg->msg_name, ua_end);
278 unsafe_get_user(msg->msg_namelen, &umsg->msg_namelen, ua_end);
279 unsafe_get_user(msg->msg_iov, &umsg->msg_iov, ua_end);
280 unsafe_get_user(msg->msg_iovlen, &umsg->msg_iovlen, ua_end);
281 unsafe_get_user(msg->msg_control, &umsg->msg_control, ua_end);
282 unsafe_get_user(msg->msg_controllen, &umsg->msg_controllen, ua_end);
287 return -EFAULT;
294 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
295 struct user_msghdr __user *umsg = sr->umsg;
298 iomsg->msg.msg_name = &iomsg->addr;
299 iomsg->msg.msg_iter.nr_segs = 0;
301 if (io_is_compat(req->ctx)) {
309 msg->msg_namelen = cmsg.msg_namelen;
310 msg->msg_controllen = cmsg.msg_controllen;
311 msg->msg_iov = compat_ptr(cmsg.msg_iov);
312 msg->msg_iovlen = cmsg.msg_iovlen;
320 msg->msg_flags = 0;
322 ret = __copy_msghdr(&iomsg->msg, msg, save_addr);
326 if (req->flags & REQ_F_BUFFER_SELECT) {
327 if (msg->msg_iovlen == 0) {
328 sr->len = 0;
329 } else if (msg->msg_iovlen > 1) {
330 return -EINVAL;
332 struct iovec __user *uiov = msg->msg_iov;
336 return -EFAULT;
337 sr->len = tmp_iov.iov_len;
345 struct io_async_msghdr *io = req->async_data;
352 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
353 struct io_async_msghdr *kmsg = req->async_data;
358 sr->buf = u64_to_user_ptr(READ_ONCE(sqe->addr));
360 if (READ_ONCE(sqe->__pad3[0]))
361 return -EINVAL;
363 kmsg->msg.msg_name = NULL;
364 kmsg->msg.msg_namelen = 0;
365 kmsg->msg.msg_control = NULL;
366 kmsg->msg.msg_controllen = 0;
367 kmsg->msg.msg_ubuf = NULL;
369 addr = u64_to_user_ptr(READ_ONCE(sqe->addr2));
370 addr_len = READ_ONCE(sqe->addr_len);
372 ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr);
375 kmsg->msg.msg_name = &kmsg->addr;
376 kmsg->msg.msg_namelen = addr_len;
378 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
379 req->flags |= REQ_F_IMPORT_BUFFER;
382 if (req->flags & REQ_F_BUFFER_SELECT)
385 if (sr->flags & IORING_SEND_VECTORIZED)
386 return io_net_import_vec(req, kmsg, sr->buf, sr->len, ITER_SOURCE);
388 return import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter);
393 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
394 struct io_async_msghdr *kmsg = req->async_data;
398 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
403 sr->msg_control = kmsg->msg.msg_control_user;
405 if (sr->flags & IORING_RECVSEND_FIXED_BUF) {
406 kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen;
407 return io_prep_reg_iovec(req, &kmsg->vec, msg.msg_iov,
410 if (req->flags & REQ_F_BUFFER_SELECT)
419 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
421 sr->done_io = 0;
422 sr->len = READ_ONCE(sqe->len);
423 sr->flags = READ_ONCE(sqe->ioprio);
424 if (sr->flags & ~SENDMSG_FLAGS)
425 return -EINVAL;
426 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL;
427 if (sr->msg_flags & MSG_DONTWAIT)
428 req->flags |= REQ_F_NOWAIT;
429 if (req->flags & REQ_F_BUFFER_SELECT)
430 sr->buf_group = req->buf_index;
431 if (sr->flags & IORING_RECVSEND_BUNDLE) {
432 if (req->opcode == IORING_OP_SENDMSG)
433 return -EINVAL;
434 sr->msg_flags |= MSG_WAITALL;
435 req->flags |= REQ_F_MULTISHOT;
438 if (io_is_compat(req->ctx))
439 sr->msg_flags |= MSG_CMSG_COMPAT;
442 return -ENOMEM;
443 if (req->opcode != IORING_OP_SENDMSG)
445 if (unlikely(sqe->addr2 || sqe->file_index))
446 return -EINVAL;
466 struct iovec *iov;
472 if (iter_is_ubuf(&kmsg->msg.msg_iter))
475 iov = kmsg->vec.iovec;
476 if (!iov)
477 iov = &kmsg->fast_iov;
480 if (!iov_iter_count(&kmsg->msg.msg_iter))
481 return iter_iov(&kmsg->msg.msg_iter) - iov;
486 int this_len = min_t(int, iov[nbufs].iov_len, ret);
489 ret -= this_len;
498 req->flags |= REQ_F_BL_NO_RECYCLE;
499 if (req->flags & REQ_F_BUFFERS_COMMIT)
508 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
509 bool bundle_finished = sel->val <= 0;
512 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) {
513 cflags = io_put_kbuf(req, sel->val, sel->buf_list);
517 cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val));
519 if (bundle_finished || req->flags & REQ_F_BL_EMPTY)
526 if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) {
533 io_req_set_res(req, sel->val, cflags);
534 sel->val = IOU_COMPLETE;
540 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
541 struct io_async_msghdr *kmsg = req->async_data;
547 sock = sock_from_file(req->file);
549 return -ENOTSOCK;
551 if (!(req->flags & REQ_F_POLLED) &&
552 (sr->flags & IORING_RECVSEND_POLL_FIRST))
553 return -EAGAIN;
555 flags = sr->msg_flags;
559 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
561 kmsg->msg.msg_control_user = sr->msg_control;
563 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
566 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
567 return -EAGAIN;
569 kmsg->msg.msg_controllen = 0;
570 kmsg->msg.msg_control = NULL;
571 sr->done_io += ret;
572 return -EAGAIN;
574 if (ret == -ERESTARTSYS)
575 ret = -EINTR;
580 ret += sr->done_io;
581 else if (sr->done_io)
582 ret = sr->done_io;
590 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
592 .iovs = &kmsg->fast_iov,
593 .max_len = min_not_zero(sr->len, INT_MAX),
595 .buf_group = sr->buf_group,
599 if (kmsg->vec.iovec) {
600 arg.nr_iovs = kmsg->vec.nr;
601 arg.iovs = kmsg->vec.iovec;
605 if (!(sr->flags & IORING_RECVSEND_BUNDLE))
614 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) {
615 kmsg->vec.nr = ret;
616 kmsg->vec.iovec = arg.iovs;
617 req->flags |= REQ_F_NEED_CLEANUP;
619 sr->len = arg.out_len;
622 sr->buf = arg.iovs[0].iov_base;
623 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len,
624 &kmsg->msg.msg_iter);
628 iov_iter_init(&kmsg->msg.msg_iter, ITER_SOURCE,
637 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
638 struct io_async_msghdr *kmsg = req->async_data;
645 sock = sock_from_file(req->file);
647 return -ENOTSOCK;
649 if (!(req->flags & REQ_F_POLLED) &&
650 (sr->flags & IORING_RECVSEND_POLL_FIRST))
651 return -EAGAIN;
653 flags = sr->msg_flags;
670 if (flags & MSG_WAITALL || sr->flags & IORING_RECVSEND_BUNDLE)
671 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
674 kmsg->msg.msg_flags = flags;
675 ret = sock_sendmsg(sock, &kmsg->msg);
677 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
678 return -EAGAIN;
681 sr->len -= ret;
682 sr->buf += ret;
683 sr->done_io += ret;
686 if (ret == -ERESTARTSYS)
687 ret = -EINTR;
691 ret += sr->done_io;
692 else if (sr->done_io)
693 ret = sr->done_io;
707 if ((req->flags & (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) ==
712 return -EOVERFLOW;
715 return -EOVERFLOW;
717 return -EOVERFLOW;
719 iomsg->namelen = namelen;
720 iomsg->controllen = controllen;
733 ret = io_msg_copy_hdr(req, iomsg, &msg, ITER_DEST, &iomsg->uaddr);
737 if (!(req->flags & REQ_F_BUFFER_SELECT)) {
749 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
754 return -ENOMEM;
756 if (req->opcode == IORING_OP_RECV) {
757 kmsg->msg.msg_name = NULL;
758 kmsg->msg.msg_namelen = 0;
759 kmsg->msg.msg_inq = 0;
760 kmsg->msg.msg_control = NULL;
761 kmsg->msg.msg_get_inq = 1;
762 kmsg->msg.msg_controllen = 0;
763 kmsg->msg.msg_iocb = NULL;
764 kmsg->msg.msg_ubuf = NULL;
766 if (req->flags & REQ_F_BUFFER_SELECT)
768 return import_ubuf(ITER_DEST, sr->buf, sr->len,
769 &kmsg->msg.msg_iter);
780 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
782 sr->done_io = 0;
784 if (unlikely(sqe->addr2))
785 return -EINVAL;
787 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr));
788 sr->len = READ_ONCE(sqe->len);
789 sr->flags = READ_ONCE(sqe->ioprio);
790 if (sr->flags & ~RECVMSG_FLAGS)
791 return -EINVAL;
792 sr->msg_flags = READ_ONCE(sqe->msg_flags);
793 if (sr->msg_flags & MSG_DONTWAIT)
794 req->flags |= REQ_F_NOWAIT;
795 if (sr->msg_flags & MSG_ERRQUEUE)
796 req->flags |= REQ_F_CLEAR_POLLIN;
797 if (req->flags & REQ_F_BUFFER_SELECT)
798 sr->buf_group = req->buf_index;
799 sr->mshot_total_len = sr->mshot_len = 0;
800 if (sr->flags & IORING_RECV_MULTISHOT) {
801 if (!(req->flags & REQ_F_BUFFER_SELECT))
802 return -EINVAL;
803 if (sr->msg_flags & MSG_WAITALL)
804 return -EINVAL;
805 if (req->opcode == IORING_OP_RECV) {
806 sr->mshot_len = sr->len;
807 sr->mshot_total_len = READ_ONCE(sqe->optlen);
808 if (sr->mshot_total_len)
809 sr->flags |= IORING_RECV_MSHOT_LIM;
810 } else if (sqe->optlen) {
811 return -EINVAL;
813 req->flags |= REQ_F_APOLL_MULTISHOT;
814 } else if (sqe->optlen) {
815 return -EINVAL;
818 if (sr->flags & IORING_RECVSEND_BUNDLE) {
819 if (req->opcode == IORING_OP_RECVMSG)
820 return -EINVAL;
823 if (io_is_compat(req->ctx))
824 sr->msg_flags |= MSG_CMSG_COMPAT;
826 sr->nr_multishot_loops = 0;
844 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
847 if (kmsg->msg.msg_inq > 0)
850 if (sel->val > 0 && sr->flags & IORING_RECV_MSHOT_LIM) {
852 * If sr->len hits zero, the limit has been reached. Mark
856 sr->mshot_total_len -= min_t(int, sel->val, sr->mshot_total_len);
857 if (!sr->mshot_total_len) {
858 sr->flags |= IORING_RECV_MSHOT_DONE;
863 if (sr->flags & IORING_RECVSEND_BUNDLE) {
864 size_t this_ret = sel->val - sr->done_io;
866 cflags |= io_put_kbufs(req, this_ret, sel->buf_list, io_bundle_nbufs(kmsg, this_ret));
867 if (sr->flags & IORING_RECV_RETRY)
868 cflags = req->cqe.flags | (cflags & CQE_F_MASK);
869 if (sr->mshot_len && sel->val >= sr->mshot_len)
870 sr->flags |= IORING_RECV_MSHOT_CAP;
872 if (req->flags & REQ_F_BL_EMPTY)
878 if (!(sr->flags & IORING_RECV_NO_RETRY) &&
879 kmsg->msg.msg_inq > 1 && this_ret > 0 &&
880 !iov_iter_count(&kmsg->msg.msg_iter)) {
881 req->cqe.flags = cflags & ~CQE_F_MASK;
882 sr->len = kmsg->msg.msg_inq;
883 sr->done_io += this_ret;
884 sr->flags |= IORING_RECV_RETRY;
888 cflags |= io_put_kbuf(req, sel->val, sel->buf_list);
895 if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished &&
896 io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) {
897 sel->val = IOU_RETRY;
899 /* Known not-empty or unknown state, retry */
900 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) {
901 if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY &&
902 !(sr->flags & IORING_RECV_MSHOT_CAP)) {
906 sr->nr_multishot_loops = 0;
907 sr->flags &= ~IORING_RECV_MSHOT_CAP;
909 sel->val = IOU_REQUEUE;
916 io_req_set_res(req, sel->val, cflags);
917 sel->val = IOU_COMPLETE;
923 struct io_sr_msg *sr, void __user **buf,
929 hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +
930 kmsg->controllen;
932 return -EFAULT;
934 if (kmsg->controllen) {
935 unsigned long control = ubuf + hdr - kmsg->controllen;
937 kmsg->msg.msg_control_user = (void __user *) control;
938 kmsg->msg.msg_controllen = kmsg->controllen;
941 sr->buf = *buf; /* stash for later copy */
943 kmsg->payloadlen = *len = *len - hdr;
960 if (kmsg->namelen)
961 kmsg->msg.msg_name = &hdr.addr;
962 kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT);
963 kmsg->msg.msg_namelen = 0;
965 if (sock->file->f_flags & O_NONBLOCK)
968 err = sock_recvmsg(sock, &kmsg->msg, flags);
974 .controllen = kmsg->controllen - kmsg->msg.msg_controllen,
975 .flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT
979 if (err > kmsg->payloadlen)
980 err = kmsg->payloadlen;
983 if (kmsg->msg.msg_namelen > kmsg->namelen)
984 copy_len += kmsg->namelen;
986 copy_len += kmsg->msg.msg_namelen;
992 hdr.msg.namelen = kmsg->msg.msg_namelen;
997 if (copy_to_user(io->buf, &hdr, copy_len)) {
999 return -EFAULT;
1002 return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen +
1003 kmsg->controllen + err;
1008 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1009 struct io_async_msghdr *kmsg = req->async_data;
1017 sock = sock_from_file(req->file);
1019 return -ENOTSOCK;
1021 if (!(req->flags & REQ_F_POLLED) &&
1022 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1023 return -EAGAIN;
1025 flags = sr->msg_flags;
1032 size_t len = sr->len;
1034 sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
1036 return -ENOBUFS;
1038 if (req->flags & REQ_F_APOLL_MULTISHOT) {
1039 ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len);
1046 iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len);
1049 kmsg->msg.msg_get_inq = 1;
1050 kmsg->msg.msg_inq = -1;
1051 if (req->flags & REQ_F_APOLL_MULTISHOT) {
1052 ret = io_recvmsg_multishot(sock, sr, kmsg, flags,
1056 if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen)
1057 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1059 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg,
1060 kmsg->uaddr, flags);
1064 if (ret == -EAGAIN && force_nonblock) {
1069 sr->done_io += ret;
1072 if (ret == -ERESTARTSYS)
1073 ret = -EINTR;
1075 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
1080 ret += sr->done_io;
1081 else if (sr->done_io)
1082 ret = sr->done_io;
1096 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1105 sr->flags & IORING_RECVSEND_BUNDLE) {
1107 .iovs = &kmsg->fast_iov,
1110 .buf_group = sr->buf_group,
1113 if (kmsg->vec.iovec) {
1114 arg.nr_iovs = kmsg->vec.nr;
1115 arg.iovs = kmsg->vec.iovec;
1119 if (sel->val)
1120 arg.max_len = sel->val;
1121 else if (kmsg->msg.msg_inq > 1)
1122 arg.max_len = min_not_zero(sel->val, (ssize_t) kmsg->msg.msg_inq);
1125 if (sr->flags & IORING_RECV_MSHOT_LIM)
1126 arg.max_len = min_not_zero(arg.max_len, sr->mshot_total_len);
1131 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) {
1132 kmsg->vec.nr = ret;
1133 kmsg->vec.iovec = arg.iovs;
1134 req->flags |= REQ_F_NEED_CLEANUP;
1137 sr->flags |= IORING_RECV_PARTIAL_MAP;
1141 sr->buf = arg.iovs[0].iov_base;
1142 sr->len = arg.iovs[0].iov_len;
1145 iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,
1148 size_t len = sel->val;
1150 *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
1151 if (!sel->addr)
1152 return -ENOBUFS;
1153 sr->buf = sel->addr;
1154 sr->len = len;
1156 ret = import_ubuf(ITER_DEST, sr->buf, sr->len,
1157 &kmsg->msg.msg_iter);
1167 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1168 struct io_async_msghdr *kmsg = req->async_data;
1176 if (!(req->flags & REQ_F_POLLED) &&
1177 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1178 return -EAGAIN;
1180 sock = sock_from_file(req->file);
1182 return -ENOTSOCK;
1184 flags = sr->msg_flags;
1191 sel.val = sr->len;
1194 kmsg->msg.msg_inq = -1;
1197 sr->buf = NULL;
1200 kmsg->msg.msg_flags = 0;
1201 kmsg->msg.msg_inq = -1;
1204 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1206 ret = sock_recvmsg(sock, &kmsg->msg, flags);
1208 if (ret == -EAGAIN && force_nonblock) {
1213 sr->len -= ret;
1214 sr->buf += ret;
1215 sr->done_io += ret;
1218 if (ret == -ERESTARTSYS)
1219 ret = -EINTR;
1221 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) {
1228 ret += sr->done_io;
1229 else if (sr->done_io)
1230 ret = sr->done_io;
1246 if (unlikely(sqe->addr2 || sqe->addr || sqe->addr3))
1247 return -EINVAL;
1249 ifq_idx = READ_ONCE(sqe->zcrx_ifq_idx);
1250 zc->ifq = xa_load(&req->ctx->zcrx_ctxs, ifq_idx);
1251 if (!zc->ifq)
1252 return -EINVAL;
1254 zc->len = READ_ONCE(sqe->len);
1255 zc->flags = READ_ONCE(sqe->ioprio);
1256 zc->msg_flags = READ_ONCE(sqe->msg_flags);
1257 if (zc->msg_flags)
1258 return -EINVAL;
1259 if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT))
1260 return -EINVAL;
1262 if (!(zc->flags & IORING_RECV_MULTISHOT))
1263 return -EINVAL;
1265 req->flags |= REQ_F_APOLL_MULTISHOT;
1277 if (!(req->flags & REQ_F_POLLED) &&
1278 (zc->flags & IORING_RECVSEND_POLL_FIRST))
1279 return -EAGAIN;
1281 sock = sock_from_file(req->file);
1283 return -ENOTSOCK;
1285 len = zc->len;
1286 ret = io_zcrx_recv(req, zc->ifq, sock, zc->msg_flags | MSG_DONTWAIT,
1287 issue_flags, &zc->len);
1288 if (len && zc->len == 0) {
1293 if (unlikely(ret <= 0) && ret != -EAGAIN) {
1294 if (ret == -ERESTARTSYS)
1295 ret = -EINTR;
1309 struct io_async_msghdr *io = req->async_data;
1313 if (zc->notif) {
1314 io_notif_flush(zc->notif);
1315 zc->notif = NULL;
1326 struct io_ring_ctx *ctx = req->ctx;
1331 zc->done_io = 0;
1333 if (unlikely(READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3)))
1334 return -EINVAL;
1336 if (req->flags & REQ_F_CQE_SKIP)
1337 return -EINVAL;
1339 notif = zc->notif = io_alloc_notif(ctx);
1341 return -ENOMEM;
1342 notif->cqe.user_data = req->cqe.user_data;
1343 notif->cqe.res = 0;
1344 notif->cqe.flags = IORING_CQE_F_NOTIF;
1345 req->flags |= REQ_F_NEED_CLEANUP | REQ_F_POLL_NO_LAZY;
1347 zc->flags = READ_ONCE(sqe->ioprio);
1348 if (unlikely(zc->flags & ~IO_ZC_FLAGS_COMMON)) {
1349 if (zc->flags & ~IO_ZC_FLAGS_VALID)
1350 return -EINVAL;
1351 if (zc->flags & IORING_SEND_ZC_REPORT_USAGE) {
1354 nd->zc_report = true;
1355 nd->zc_used = false;
1356 nd->zc_copied = false;
1360 zc->len = READ_ONCE(sqe->len);
1361 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL | MSG_ZEROCOPY;
1362 req->buf_index = READ_ONCE(sqe->buf_index);
1363 if (zc->msg_flags & MSG_DONTWAIT)
1364 req->flags |= REQ_F_NOWAIT;
1366 if (io_is_compat(req->ctx))
1367 zc->msg_flags |= MSG_CMSG_COMPAT;
1371 return -ENOMEM;
1373 if (req->opcode == IORING_OP_SEND_ZC) {
1376 if (unlikely(sqe->addr2 || sqe->file_index))
1377 return -EINVAL;
1383 if (!(zc->flags & IORING_RECVSEND_FIXED_BUF)) {
1384 iomsg->msg.sg_from_iter = io_sg_from_iter_iovec;
1385 return io_notif_account_mem(zc->notif, iomsg->msg.msg_iter.count);
1387 iomsg->msg.sg_from_iter = io_sg_from_iter;
1402 int frag = shinfo->nr_frags;
1409 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS;
1413 bi.bi_size = min(from->count, length);
1414 bi.bi_bvec_done = from->iov_offset;
1418 struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi);
1424 bvec_iter_advance_single(from->bvec, &bi, v.bv_len);
1427 ret = -EMSGSIZE;
1429 shinfo->nr_frags = frag;
1430 from->bvec += bi.bi_idx;
1431 from->nr_segs -= bi.bi_idx;
1432 from->count -= copied;
1433 from->iov_offset = bi.bi_bvec_done;
1435 skb->data_len += copied;
1436 skb->len += copied;
1437 skb->truesize += truesize;
1443 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1444 struct io_async_msghdr *kmsg = req->async_data;
1446 WARN_ON_ONCE(!(sr->flags & IORING_RECVSEND_FIXED_BUF));
1448 sr->notif->buf_index = req->buf_index;
1449 return io_import_reg_buf(sr->notif, &kmsg->msg.msg_iter,
1450 (u64)(uintptr_t)sr->buf, sr->len,
1457 struct io_async_msghdr *kmsg = req->async_data;
1462 sock = sock_from_file(req->file);
1464 return -ENOTSOCK;
1465 if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags))
1466 return -EOPNOTSUPP;
1468 if (!(req->flags & REQ_F_POLLED) &&
1469 (zc->flags & IORING_RECVSEND_POLL_FIRST))
1470 return -EAGAIN;
1472 if (req->flags & REQ_F_IMPORT_BUFFER) {
1473 req->flags &= ~REQ_F_IMPORT_BUFFER;
1479 msg_flags = zc->msg_flags;
1483 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1486 kmsg->msg.msg_flags = msg_flags;
1487 kmsg->msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg;
1488 ret = sock_sendmsg(sock, &kmsg->msg);
1491 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1492 return -EAGAIN;
1494 if (ret > 0 && io_net_retry(sock, kmsg->msg.msg_flags)) {
1495 zc->len -= ret;
1496 zc->buf += ret;
1497 zc->done_io += ret;
1498 return -EAGAIN;
1500 if (ret == -ERESTARTSYS)
1501 ret = -EINTR;
1506 ret += zc->done_io;
1507 else if (zc->done_io)
1508 ret = zc->done_io;
1511 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1515 io_notif_flush(zc->notif);
1516 zc->notif = NULL;
1525 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1526 struct io_async_msghdr *kmsg = req->async_data;
1531 if (req->flags & REQ_F_IMPORT_BUFFER) {
1532 unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs;
1535 ret = io_import_reg_vec(ITER_SOURCE, &kmsg->msg.msg_iter, req,
1536 &kmsg->vec, uvec_segs, issue_flags);
1539 req->flags &= ~REQ_F_IMPORT_BUFFER;
1542 sock = sock_from_file(req->file);
1544 return -ENOTSOCK;
1545 if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags))
1546 return -EOPNOTSUPP;
1548 if (!(req->flags & REQ_F_POLLED) &&
1549 (sr->flags & IORING_RECVSEND_POLL_FIRST))
1550 return -EAGAIN;
1552 flags = sr->msg_flags;
1556 min_ret = iov_iter_count(&kmsg->msg.msg_iter);
1558 kmsg->msg.msg_control_user = sr->msg_control;
1559 kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg;
1560 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags);
1563 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1564 return -EAGAIN;
1567 sr->done_io += ret;
1568 return -EAGAIN;
1570 if (ret == -ERESTARTSYS)
1571 ret = -EINTR;
1576 ret += sr->done_io;
1577 else if (sr->done_io)
1578 ret = sr->done_io;
1581 * If we're in io-wq we can't rely on tw ordering guarantees, defer
1585 io_notif_flush(sr->notif);
1586 sr->notif = NULL;
1595 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg);
1597 if (sr->done_io)
1598 req->cqe.res = sr->done_io;
1600 if ((req->flags & REQ_F_NEED_CLEANUP) &&
1601 (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC))
1602 req->cqe.flags |= IORING_CQE_F_MORE;
1612 if (sqe->len || sqe->buf_index)
1613 return -EINVAL;
1615 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1616 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2));
1617 accept->flags = READ_ONCE(sqe->accept_flags);
1618 accept->nofile = rlimit(RLIMIT_NOFILE);
1619 accept->iou_flags = READ_ONCE(sqe->ioprio);
1620 if (accept->iou_flags & ~ACCEPT_FLAGS)
1621 return -EINVAL;
1623 accept->file_slot = READ_ONCE(sqe->file_index);
1624 if (accept->file_slot) {
1625 if (accept->flags & SOCK_CLOEXEC)
1626 return -EINVAL;
1627 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT &&
1628 accept->file_slot != IORING_FILE_INDEX_ALLOC)
1629 return -EINVAL;
1631 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
1632 return -EINVAL;
1633 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK))
1634 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK;
1635 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT)
1636 req->flags |= REQ_F_APOLL_MULTISHOT;
1637 if (accept->iou_flags & IORING_ACCEPT_DONTWAIT)
1638 req->flags |= REQ_F_NOWAIT;
1646 bool fixed = !!accept->file_slot;
1654 if (!(req->flags & REQ_F_POLLED) &&
1655 accept->iou_flags & IORING_ACCEPT_POLL_FIRST)
1656 return -EAGAIN;
1660 fd = __get_unused_fd_flags(accept->flags, accept->nofile);
1665 arg.is_empty = -1;
1666 file = do_accept(req->file, &arg, accept->addr, accept->addr_len,
1667 accept->flags);
1672 if (ret == -EAGAIN && force_nonblock &&
1673 !(accept->iou_flags & IORING_ACCEPT_DONTWAIT))
1676 if (ret == -ERESTARTSYS)
1677 ret = -EINTR;
1683 accept->file_slot);
1690 if (ret >= 0 && (req->flags & REQ_F_APOLL_MULTISHOT) &&
1692 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || arg.is_empty == -1)
1707 if (sqe->addr || sqe->rw_flags || sqe->buf_index)
1708 return -EINVAL;
1710 sock->domain = READ_ONCE(sqe->fd);
1711 sock->type = READ_ONCE(sqe->off);
1712 sock->protocol = READ_ONCE(sqe->len);
1713 sock->file_slot = READ_ONCE(sqe->file_index);
1714 sock->nofile = rlimit(RLIMIT_NOFILE);
1716 sock->flags = sock->type & ~SOCK_TYPE_MASK;
1717 if (sock->file_slot && (sock->flags & SOCK_CLOEXEC))
1718 return -EINVAL;
1719 if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK))
1720 return -EINVAL;
1727 bool fixed = !!sock->file_slot;
1732 fd = __get_unused_fd_flags(sock->flags, sock->nofile);
1736 file = __sys_socket_file(sock->domain, sock->type, sock->protocol);
1741 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK))
1742 return -EAGAIN;
1743 if (ret == -ERESTARTSYS)
1744 ret = -EINTR;
1751 sock->file_slot);
1762 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
1763 return -EINVAL;
1765 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1766 conn->addr_len = READ_ONCE(sqe->addr2);
1767 conn->in_progress = conn->seen_econnaborted = false;
1771 return -ENOMEM;
1773 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->addr);
1779 struct io_async_msghdr *io = req->async_data;
1784 if (connect->in_progress) {
1787 if (vfs_poll(req->file, &pt) & EPOLLERR)
1793 ret = __sys_connect_file(req->file, &io->addr, connect->addr_len,
1795 if ((ret == -EAGAIN || ret == -EINPROGRESS || ret == -ECONNABORTED)
1797 if (ret == -EINPROGRESS) {
1798 connect->in_progress = true;
1799 } else if (ret == -ECONNABORTED) {
1800 if (connect->seen_econnaborted)
1802 connect->seen_econnaborted = true;
1804 return -EAGAIN;
1806 if (connect->in_progress) {
1808 * At least bluetooth will return -EBADFD on a re-connect
1809 * attempt, and it's (supposedly) also valid to get -EISCONN
1813 if (ret == -EBADFD || ret == -EISCONN) {
1815 ret = sock_error(sock_from_file(req->file)->sk);
1818 if (ret == -ERESTARTSYS)
1819 ret = -EINTR;
1834 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in)
1835 return -EINVAL;
1837 uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr));
1838 bind->addr_len = READ_ONCE(sqe->addr2);
1842 return -ENOMEM;
1843 return move_addr_to_kernel(uaddr, bind->addr_len, &io->addr);
1849 struct io_async_msghdr *io = req->async_data;
1853 sock = sock_from_file(req->file);
1855 return -ENOTSOCK;
1857 ret = __sys_bind_socket(sock, &io->addr, bind->addr_len);
1868 if (sqe->addr || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in || sqe->addr2)
1869 return -EINVAL;
1871 listen->backlog = READ_ONCE(sqe->len);
1881 sock = sock_from_file(req->file);
1883 return -ENOTSOCK;
1885 ret = __sys_listen_socket(sock, listen->backlog);
1896 io_vec_free(&kmsg->vec);