1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/file.h> 5 #include <linux/slab.h> 6 #include <linux/net.h> 7 #include <linux/un.h> 8 #include <linux/compat.h> 9 #include <net/compat.h> 10 #include <net/ipv6.h> 11 #include <linux/io_uring.h> 12 13 #include <uapi/linux/io_uring.h> 14 15 #include "filetable.h" 16 #include "io_uring.h" 17 #include "kbuf.h" 18 #include "alloc_cache.h" 19 #include "net.h" 20 #include "notif.h" 21 #include "rsrc.h" 22 #include "zcrx.h" 23 24 struct io_shutdown { 25 struct file *file; 26 int how; 27 }; 28 29 struct io_accept { 30 struct file *file; 31 struct sockaddr __user *addr; 32 int __user *addr_len; 33 int flags; 34 int iou_flags; 35 u32 file_slot; 36 unsigned long nofile; 37 }; 38 39 struct io_socket { 40 struct file *file; 41 int domain; 42 int type; 43 int protocol; 44 int flags; 45 u32 file_slot; 46 unsigned long nofile; 47 }; 48 49 struct io_connect { 50 struct file *file; 51 struct sockaddr __user *addr; 52 int addr_len; 53 bool in_progress; 54 bool seen_econnaborted; 55 }; 56 57 struct io_bind { 58 struct file *file; 59 int addr_len; 60 }; 61 62 struct io_listen { 63 struct file *file; 64 int backlog; 65 }; 66 67 struct io_sr_msg { 68 struct file *file; 69 union { 70 struct compat_msghdr __user *umsg_compat; 71 struct user_msghdr __user *umsg; 72 void __user *buf; 73 }; 74 int len; 75 unsigned done_io; 76 unsigned msg_flags; 77 unsigned nr_multishot_loops; 78 u16 flags; 79 /* initialised and used only by !msg send variants */ 80 u16 buf_group; 81 /* per-invocation mshot limit */ 82 unsigned mshot_len; 83 /* overall mshot byte limit */ 84 unsigned mshot_total_len; 85 void __user *msg_control; 86 /* used only for send zerocopy */ 87 struct io_kiocb *notif; 88 }; 89 90 /* 91 * The UAPI flags are the lower 8 bits, as that's all sqe->ioprio will hold 92 * anyway. Use the upper 8 bits for internal uses. 93 */ 94 enum sr_retry_flags { 95 IORING_RECV_RETRY = (1U << 15), 96 IORING_RECV_PARTIAL_MAP = (1U << 14), 97 IORING_RECV_MSHOT_CAP = (1U << 13), 98 IORING_RECV_MSHOT_LIM = (1U << 12), 99 IORING_RECV_MSHOT_DONE = (1U << 11), 100 101 IORING_RECV_RETRY_CLEAR = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP, 102 IORING_RECV_NO_RETRY = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP | 103 IORING_RECV_MSHOT_CAP | IORING_RECV_MSHOT_DONE, 104 }; 105 106 /* 107 * Number of times we'll try and do receives if there's more data. If we 108 * exceed this limit, then add us to the back of the queue and retry from 109 * there. This helps fairness between flooding clients. 110 */ 111 #define MULTISHOT_MAX_RETRY 32 112 113 struct io_recvzc { 114 struct file *file; 115 u16 flags; 116 u32 len; 117 struct io_zcrx_ifq *ifq; 118 }; 119 120 static int io_sg_from_iter_iovec(struct sk_buff *skb, 121 struct iov_iter *from, size_t length); 122 static int io_sg_from_iter(struct sk_buff *skb, 123 struct iov_iter *from, size_t length); 124 125 int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 126 { 127 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 128 129 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags || 130 sqe->buf_index || sqe->splice_fd_in)) 131 return -EINVAL; 132 133 shutdown->how = READ_ONCE(sqe->len); 134 req->flags |= REQ_F_FORCE_ASYNC; 135 return 0; 136 } 137 138 int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) 139 { 140 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 141 struct socket *sock; 142 int ret; 143 144 WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); 145 146 sock = sock_from_file(req->file); 147 if (unlikely(!sock)) 148 return -ENOTSOCK; 149 150 ret = __sys_shutdown_sock(sock, shutdown->how); 151 io_req_set_res(req, ret, 0); 152 return IOU_COMPLETE; 153 } 154 155 static bool io_net_retry(struct socket *sock, int flags) 156 { 157 if (!(flags & MSG_WAITALL)) 158 return false; 159 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET; 160 } 161 162 static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg) 163 { 164 if (kmsg->vec.iovec) 165 io_vec_free(&kmsg->vec); 166 } 167 168 static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) 169 { 170 struct io_async_msghdr *hdr = req->async_data; 171 172 /* can't recycle, ensure we free the iovec if we have one */ 173 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) { 174 io_netmsg_iovec_free(hdr); 175 return; 176 } 177 178 /* Let normal cleanup path reap it if we fail adding to the cache */ 179 io_alloc_cache_vec_kasan(&hdr->vec); 180 if (hdr->vec.nr > IO_VEC_CACHE_SOFT_CAP) 181 io_vec_free(&hdr->vec); 182 183 if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr)) 184 io_req_async_data_clear(req, REQ_F_NEED_CLEANUP); 185 } 186 187 static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req) 188 { 189 struct io_ring_ctx *ctx = req->ctx; 190 struct io_async_msghdr *hdr; 191 192 hdr = io_uring_alloc_async_data(&ctx->netmsg_cache, req); 193 if (!hdr) 194 return NULL; 195 196 /* If the async data was cached, we might have an iov cached inside. */ 197 if (hdr->vec.iovec) 198 req->flags |= REQ_F_NEED_CLEANUP; 199 return hdr; 200 } 201 202 static inline void io_mshot_prep_retry(struct io_kiocb *req, 203 struct io_async_msghdr *kmsg) 204 { 205 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 206 207 req->flags &= ~REQ_F_BL_EMPTY; 208 sr->done_io = 0; 209 sr->flags &= ~IORING_RECV_RETRY_CLEAR; 210 sr->len = sr->mshot_len; 211 } 212 213 static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg, 214 const struct iovec __user *uiov, unsigned uvec_seg, 215 int ddir) 216 { 217 struct iovec *iov; 218 int ret, nr_segs; 219 220 if (iomsg->vec.iovec) { 221 nr_segs = iomsg->vec.nr; 222 iov = iomsg->vec.iovec; 223 } else { 224 nr_segs = 1; 225 iov = &iomsg->fast_iov; 226 } 227 228 ret = __import_iovec(ddir, uiov, uvec_seg, nr_segs, &iov, 229 &iomsg->msg.msg_iter, io_is_compat(req->ctx)); 230 if (unlikely(ret < 0)) 231 return ret; 232 233 if (iov) { 234 req->flags |= REQ_F_NEED_CLEANUP; 235 io_vec_reset_iovec(&iomsg->vec, iov, iomsg->msg.msg_iter.nr_segs); 236 } 237 return 0; 238 } 239 240 static int io_copy_msghdr_from_user(struct user_msghdr *msg, 241 struct user_msghdr __user *umsg) 242 { 243 if (!user_access_begin(umsg, sizeof(*umsg))) 244 return -EFAULT; 245 unsafe_get_user(msg->msg_name, &umsg->msg_name, ua_end); 246 unsafe_get_user(msg->msg_namelen, &umsg->msg_namelen, ua_end); 247 unsafe_get_user(msg->msg_iov, &umsg->msg_iov, ua_end); 248 unsafe_get_user(msg->msg_iovlen, &umsg->msg_iovlen, ua_end); 249 unsafe_get_user(msg->msg_control, &umsg->msg_control, ua_end); 250 unsafe_get_user(msg->msg_controllen, &umsg->msg_controllen, ua_end); 251 user_access_end(); 252 return 0; 253 ua_end: 254 user_access_end(); 255 return -EFAULT; 256 } 257 258 static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, 259 struct user_msghdr *msg, 260 struct sockaddr __user **save_addr) 261 { 262 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 263 int ret; 264 265 iomsg->msg.msg_name = &iomsg->addr; 266 iomsg->msg.msg_iter.nr_segs = 0; 267 268 if (io_is_compat(req->ctx)) { 269 struct compat_msghdr cmsg; 270 271 if (copy_from_user(&cmsg, sr->umsg_compat, sizeof(cmsg))) 272 return -EFAULT; 273 274 ret = __get_compat_msghdr(&iomsg->msg, &cmsg, save_addr); 275 if (ret) 276 return ret; 277 278 memset(msg, 0, sizeof(*msg)); 279 msg->msg_namelen = cmsg.msg_namelen; 280 msg->msg_controllen = cmsg.msg_controllen; 281 msg->msg_iov = compat_ptr(cmsg.msg_iov); 282 msg->msg_iovlen = cmsg.msg_iovlen; 283 } else { 284 ret = io_copy_msghdr_from_user(msg, sr->umsg); 285 if (unlikely(ret)) 286 return ret; 287 288 msg->msg_flags = 0; 289 290 ret = __copy_msghdr(&iomsg->msg, msg, save_addr); 291 if (ret) 292 return ret; 293 } 294 295 if (req->flags & REQ_F_BUFFER_SELECT) { 296 if (msg->msg_iovlen == 0) { 297 sr->len = 0; 298 } else if (msg->msg_iovlen > 1) { 299 return -EINVAL; 300 } else { 301 struct iovec fast_iov, *iov; 302 303 iov = iovec_from_user(msg->msg_iov, 1, 1, &fast_iov, 304 io_is_compat(req->ctx)); 305 if (IS_ERR(iov)) 306 return PTR_ERR(iov); 307 sr->len = iov->iov_len; 308 } 309 } 310 return 0; 311 } 312 313 void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) 314 { 315 struct io_async_msghdr *io = req->async_data; 316 317 io_netmsg_iovec_free(io); 318 } 319 320 static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) 321 { 322 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 323 struct io_async_msghdr *kmsg = req->async_data; 324 void __user *addr; 325 u16 addr_len; 326 int ret; 327 328 sr->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); 329 330 if (READ_ONCE(sqe->__pad3[0])) 331 return -EINVAL; 332 333 kmsg->msg.msg_name = NULL; 334 kmsg->msg.msg_namelen = 0; 335 kmsg->msg.msg_control = NULL; 336 kmsg->msg.msg_controllen = 0; 337 kmsg->msg.msg_ubuf = NULL; 338 339 addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 340 addr_len = READ_ONCE(sqe->addr_len); 341 if (addr) { 342 ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr); 343 if (unlikely(ret < 0)) 344 return ret; 345 kmsg->msg.msg_name = &kmsg->addr; 346 kmsg->msg.msg_namelen = addr_len; 347 } 348 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 349 if (!(sr->flags & IORING_SEND_VECTORIZED)) { 350 req->flags |= REQ_F_IMPORT_BUFFER; 351 return 0; 352 } 353 354 kmsg->msg.msg_iter.nr_segs = sr->len; 355 return io_prep_reg_iovec(req, &kmsg->vec, sr->buf, sr->len); 356 } 357 if (req->flags & REQ_F_BUFFER_SELECT) 358 return 0; 359 360 if (sr->flags & IORING_SEND_VECTORIZED) 361 return io_net_import_vec(req, kmsg, sr->buf, sr->len, ITER_SOURCE); 362 363 return import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter); 364 } 365 366 static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) 367 { 368 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 369 struct io_async_msghdr *kmsg = req->async_data; 370 struct user_msghdr msg; 371 int ret; 372 373 sr->flags |= IORING_SEND_VECTORIZED; 374 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 375 ret = io_msg_copy_hdr(req, kmsg, &msg, NULL); 376 if (unlikely(ret)) 377 return ret; 378 /* save msg_control as sys_sendmsg() overwrites it */ 379 sr->msg_control = kmsg->msg.msg_control_user; 380 381 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 382 kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen; 383 return io_prep_reg_iovec(req, &kmsg->vec, msg.msg_iov, 384 msg.msg_iovlen); 385 } 386 if (req->flags & REQ_F_BUFFER_SELECT) 387 return 0; 388 return io_net_import_vec(req, kmsg, msg.msg_iov, msg.msg_iovlen, ITER_SOURCE); 389 } 390 391 #define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE | \ 392 IORING_SEND_VECTORIZED | IORING_RECVSEND_FIXED_BUF) 393 394 int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 395 { 396 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 397 398 sr->done_io = 0; 399 sr->len = READ_ONCE(sqe->len); 400 if (unlikely(sr->len < 0)) 401 return -EINVAL; 402 sr->flags = READ_ONCE(sqe->ioprio); 403 if (sr->flags & ~SENDMSG_FLAGS) 404 return -EINVAL; 405 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 406 /* registered buffer send only supported for plain IORING_OP_SEND */ 407 if (req->opcode != IORING_OP_SEND || 408 (req->flags & REQ_F_BUFFER_SELECT) || 409 (sr->flags & (IORING_RECVSEND_BUNDLE|IORING_SEND_VECTORIZED))) 410 return -EINVAL; 411 req->buf_index = READ_ONCE(sqe->buf_index); 412 } 413 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 414 if (sr->msg_flags & MSG_DONTWAIT) 415 req->flags |= REQ_F_NOWAIT; 416 if (req->flags & REQ_F_BUFFER_SELECT) 417 sr->buf_group = req->buf_index; 418 sr->mshot_total_len = sr->mshot_len = 0; 419 if (sr->flags & IORING_RECVSEND_BUNDLE) { 420 if (req->opcode == IORING_OP_SENDMSG) 421 return -EINVAL; 422 sr->msg_flags |= MSG_WAITALL; 423 req->flags |= REQ_F_MULTISHOT; 424 } 425 426 if (io_is_compat(req->ctx)) 427 sr->msg_flags |= MSG_CMSG_COMPAT; 428 429 if (unlikely(!io_msg_alloc_async(req))) 430 return -ENOMEM; 431 if (req->opcode != IORING_OP_SENDMSG) 432 return io_send_setup(req, sqe); 433 if (unlikely(sqe->addr2 || sqe->file_index)) 434 return -EINVAL; 435 return io_sendmsg_setup(req, sqe); 436 } 437 438 static void io_req_msg_cleanup(struct io_kiocb *req, 439 unsigned int issue_flags) 440 { 441 io_netmsg_recycle(req, issue_flags); 442 } 443 444 /* 445 * For bundle completions, we need to figure out how many segments we consumed. 446 * A bundle could be using a single ITER_UBUF if that's all we mapped, or it 447 * could be using an ITER_IOVEC. If the latter, then if we consumed all of 448 * the segments, then it's a trivial questiont o answer. If we have residual 449 * data in the iter, then loop the segments to figure out how much we 450 * transferred. 451 */ 452 static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret) 453 { 454 struct iovec *iov; 455 int nbufs; 456 457 /* no data is always zero segments, and a ubuf is always 1 segment */ 458 if (ret <= 0) 459 return 0; 460 if (iter_is_ubuf(&kmsg->msg.msg_iter)) 461 return 1; 462 463 iov = kmsg->vec.iovec; 464 if (!iov) 465 iov = &kmsg->fast_iov; 466 467 /* if all data was transferred, it's basic pointer math */ 468 if (!iov_iter_count(&kmsg->msg.msg_iter)) 469 return iter_iov(&kmsg->msg.msg_iter) - iov; 470 471 /* short transfer, count segments */ 472 nbufs = 0; 473 do { 474 int this_len = min_t(int, iov[nbufs].iov_len, ret); 475 476 nbufs++; 477 ret -= this_len; 478 } while (ret); 479 480 return nbufs; 481 } 482 483 /* 484 * A receive that is retried after a partial transfer keeps writing to what it 485 * has mapped. io_net_kbuf_recyle() only commits what was filled so far, which 486 * takes a buffer out of circulation if it's a single one that is consumed as a 487 * whole. The tail of an incrementally consumed buffer, or the yet unused 488 * buffers of a bundle, would be up for grabs. They can't be committed in 489 * advance either, the CQE has no way to tell if they end up not being used. 490 */ 491 static bool io_net_kbuf_can_retry(struct io_kiocb *req, 492 struct io_buffer_list *bl, 493 struct io_async_msghdr *kmsg) 494 { 495 /* not ring provided, or committed at selection time */ 496 if (!(req->flags & REQ_F_BUFFERS_COMMIT)) 497 return true; 498 return !(bl->flags & IOBL_INC) && iter_is_ubuf(&kmsg->msg.msg_iter); 499 } 500 501 static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl, 502 struct io_async_msghdr *kmsg, int len) 503 { 504 req->flags |= REQ_F_BL_NO_RECYCLE; 505 if (req->flags & REQ_F_BUFFERS_COMMIT) 506 io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len)); 507 return IOU_RETRY; 508 } 509 510 static inline bool io_send_finish(struct io_kiocb *req, 511 struct io_async_msghdr *kmsg, 512 struct io_br_sel *sel) 513 { 514 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 515 bool bundle_finished = sel->val <= 0; 516 unsigned int cflags; 517 518 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) { 519 cflags = io_put_kbuf(req, sel->val, sel->buf_list); 520 goto finish; 521 } 522 523 cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val)); 524 525 /* 526 * Don't start new bundles if the buffer list is empty, or if the 527 * current operation needed to go through polling to complete. 528 */ 529 if (bundle_finished || req->flags & (REQ_F_BL_EMPTY | REQ_F_POLLED)) 530 goto finish; 531 532 /* 533 * Fill CQE for this receive and see if we should keep trying to 534 * receive from this socket. 535 */ 536 if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) { 537 io_mshot_prep_retry(req, kmsg); 538 return false; 539 } 540 541 /* Otherwise stop bundle and use the current result. */ 542 finish: 543 io_req_set_res(req, sel->val, cflags); 544 sel->val = IOU_COMPLETE; 545 return true; 546 } 547 548 int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) 549 { 550 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 551 struct io_async_msghdr *kmsg = req->async_data; 552 struct socket *sock; 553 unsigned flags; 554 int min_ret = 0; 555 int ret; 556 557 sock = sock_from_file(req->file); 558 if (unlikely(!sock)) 559 return -ENOTSOCK; 560 561 if (!(req->flags & REQ_F_POLLED) && 562 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 563 return -EAGAIN; 564 565 flags = sr->msg_flags; 566 if (issue_flags & IO_URING_F_NONBLOCK) 567 flags |= MSG_DONTWAIT; 568 if (flags & MSG_WAITALL) 569 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 570 571 kmsg->msg.msg_control_user = sr->msg_control; 572 573 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 574 575 if (ret < min_ret) { 576 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 577 return -EAGAIN; 578 if (ret > 0 && io_net_retry(sock, flags)) { 579 kmsg->msg.msg_controllen = 0; 580 kmsg->msg.msg_control = NULL; 581 sr->done_io += ret; 582 return -EAGAIN; 583 } 584 if (ret == -ERESTARTSYS) 585 ret = -EINTR; 586 req_set_fail(req); 587 } 588 io_req_msg_cleanup(req, issue_flags); 589 if (ret >= 0) 590 ret += sr->done_io; 591 else if (sr->done_io) 592 ret = sr->done_io; 593 io_req_set_res(req, ret, 0); 594 return IOU_COMPLETE; 595 } 596 597 static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags, 598 struct io_br_sel *sel, struct io_async_msghdr *kmsg) 599 { 600 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 601 struct buf_sel_arg arg = { 602 .iovs = &kmsg->fast_iov, 603 .max_len = min_not_zero(sr->len, INT_MAX), 604 .nr_iovs = 1, 605 .buf_group = sr->buf_group, 606 }; 607 int ret; 608 609 if (kmsg->vec.iovec) { 610 arg.nr_iovs = kmsg->vec.nr; 611 arg.iovs = kmsg->vec.iovec; 612 arg.mode = KBUF_MODE_FREE; 613 } 614 615 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) 616 arg.nr_iovs = 1; 617 else 618 arg.mode |= KBUF_MODE_EXPAND; 619 620 ret = io_buffers_select(req, &arg, sel, issue_flags); 621 if (unlikely(ret < 0)) 622 return ret; 623 624 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) { 625 kmsg->vec.nr = ret; 626 kmsg->vec.iovec = arg.iovs; 627 req->flags |= REQ_F_NEED_CLEANUP; 628 } 629 sr->len = arg.out_len; 630 631 if (ret == 1) { 632 sr->buf = arg.iovs[0].iov_base; 633 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, 634 &kmsg->msg.msg_iter); 635 if (unlikely(ret)) 636 return ret; 637 } else { 638 iov_iter_init(&kmsg->msg.msg_iter, ITER_SOURCE, 639 arg.iovs, ret, arg.out_len); 640 } 641 642 return 0; 643 } 644 645 int io_send(struct io_kiocb *req, unsigned int issue_flags) 646 { 647 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 648 struct io_async_msghdr *kmsg = req->async_data; 649 struct io_br_sel sel = { }; 650 struct socket *sock; 651 unsigned flags; 652 int min_ret = 0; 653 int ret; 654 655 sock = sock_from_file(req->file); 656 if (unlikely(!sock)) 657 return -ENOTSOCK; 658 659 if (!(req->flags & REQ_F_POLLED) && 660 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 661 return -EAGAIN; 662 663 if (req->flags & REQ_F_IMPORT_BUFFER) { 664 ret = io_import_reg_buf(req, &kmsg->msg.msg_iter, 665 (u64)(uintptr_t)sr->buf, sr->len, 666 ITER_SOURCE, issue_flags); 667 if (unlikely(ret)) 668 return ret; 669 req->flags &= ~REQ_F_IMPORT_BUFFER; 670 } 671 672 flags = sr->msg_flags; 673 if (issue_flags & IO_URING_F_NONBLOCK) 674 flags |= MSG_DONTWAIT; 675 676 retry_bundle: 677 sel.buf_list = NULL; 678 if (io_do_buffer_select(req)) { 679 ret = io_send_select_buffer(req, issue_flags, &sel, kmsg); 680 if (ret) 681 return ret; 682 } 683 684 /* 685 * If MSG_WAITALL is set, or this is a bundle send, then we need 686 * the full amount. If just bundle is set, if we do a short send 687 * then we complete the bundle sequence rather than continue on. 688 */ 689 if (flags & MSG_WAITALL || sr->flags & IORING_RECVSEND_BUNDLE) 690 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 691 692 flags &= ~MSG_INTERNAL_SENDMSG_FLAGS; 693 kmsg->msg.msg_flags = flags; 694 ret = sock_sendmsg(sock, &kmsg->msg); 695 if (ret < min_ret) { 696 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 697 return -EAGAIN; 698 699 if (ret > 0 && io_net_retry(sock, flags)) { 700 sr->len -= ret; 701 sr->buf += ret; 702 sr->done_io += ret; 703 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); 704 } 705 if (ret == -ERESTARTSYS) 706 ret = -EINTR; 707 req_set_fail(req); 708 } 709 if (ret >= 0) 710 ret += sr->done_io; 711 else if (sr->done_io) 712 ret = sr->done_io; 713 714 sel.val = ret; 715 if (!io_send_finish(req, kmsg, &sel)) 716 goto retry_bundle; 717 718 io_req_msg_cleanup(req, issue_flags); 719 return sel.val; 720 } 721 722 static int io_recvmsg_mshot_prep(struct io_kiocb *req, 723 struct io_async_msghdr *iomsg, 724 int namelen, size_t controllen) 725 { 726 if ((req->flags & (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) == 727 (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) { 728 int hdr; 729 730 if (unlikely(namelen < 0)) 731 return -EOVERFLOW; 732 if (check_add_overflow(sizeof(struct io_uring_recvmsg_out), 733 namelen, &hdr)) 734 return -EOVERFLOW; 735 if (check_add_overflow(hdr, controllen, &hdr)) 736 return -EOVERFLOW; 737 738 iomsg->namelen = namelen; 739 iomsg->controllen = controllen; 740 return 0; 741 } 742 743 return 0; 744 } 745 746 static int io_recvmsg_copy_hdr(struct io_kiocb *req, 747 struct io_async_msghdr *iomsg) 748 { 749 struct user_msghdr msg; 750 int ret; 751 752 ret = io_msg_copy_hdr(req, iomsg, &msg, &iomsg->uaddr); 753 if (unlikely(ret)) 754 return ret; 755 756 if (!(req->flags & REQ_F_BUFFER_SELECT)) { 757 ret = io_net_import_vec(req, iomsg, msg.msg_iov, msg.msg_iovlen, 758 ITER_DEST); 759 if (unlikely(ret)) 760 return ret; 761 } 762 return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen, 763 msg.msg_controllen); 764 } 765 766 static int io_recvmsg_prep_setup(struct io_kiocb *req) 767 { 768 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 769 struct io_async_msghdr *kmsg; 770 771 kmsg = io_msg_alloc_async(req); 772 if (unlikely(!kmsg)) 773 return -ENOMEM; 774 775 if (req->opcode == IORING_OP_RECV) { 776 kmsg->msg.msg_name = NULL; 777 kmsg->msg.msg_namelen = 0; 778 kmsg->msg.msg_inq = 0; 779 kmsg->msg.msg_control = NULL; 780 kmsg->msg.msg_get_inq = 1; 781 kmsg->msg.msg_controllen = 0; 782 kmsg->msg.msg_ubuf = NULL; 783 784 if (req->flags & REQ_F_BUFFER_SELECT) 785 return 0; 786 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 787 req->flags |= REQ_F_IMPORT_BUFFER; 788 return 0; 789 } 790 return import_ubuf(ITER_DEST, sr->buf, sr->len, 791 &kmsg->msg.msg_iter); 792 } 793 794 return io_recvmsg_copy_hdr(req, kmsg); 795 } 796 797 #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT | \ 798 IORING_RECVSEND_BUNDLE | IORING_RECVSEND_FIXED_BUF) 799 800 int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 801 { 802 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 803 804 sr->done_io = 0; 805 806 if (unlikely(sqe->addr2)) 807 return -EINVAL; 808 809 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 810 sr->len = READ_ONCE(sqe->len); 811 if (unlikely(sr->len < 0)) 812 return -EINVAL; 813 sr->flags = READ_ONCE(sqe->ioprio); 814 if (sr->flags & ~RECVMSG_FLAGS) 815 return -EINVAL; 816 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 817 /* registered buffer recv only for plain IORING_OP_RECV */ 818 if (req->opcode != IORING_OP_RECV || 819 (req->flags & REQ_F_BUFFER_SELECT) || 820 (sr->flags & (IORING_RECV_MULTISHOT | IORING_RECVSEND_BUNDLE))) 821 return -EINVAL; 822 req->buf_index = READ_ONCE(sqe->buf_index); 823 } 824 sr->msg_flags = READ_ONCE(sqe->msg_flags); 825 if (sr->msg_flags & MSG_DONTWAIT) 826 req->flags |= REQ_F_NOWAIT; 827 if (sr->msg_flags & MSG_ERRQUEUE) 828 req->flags |= REQ_F_CLEAR_POLLIN; 829 if (req->flags & REQ_F_BUFFER_SELECT) 830 sr->buf_group = req->buf_index; 831 sr->mshot_total_len = sr->mshot_len = 0; 832 if (sr->flags & IORING_RECV_MULTISHOT) { 833 if (!(req->flags & REQ_F_BUFFER_SELECT)) 834 return -EINVAL; 835 if (sr->msg_flags & MSG_WAITALL) 836 return -EINVAL; 837 if (req->opcode == IORING_OP_RECV) { 838 sr->mshot_len = sr->len; 839 sr->mshot_total_len = READ_ONCE(sqe->optlen); 840 if (sr->mshot_total_len) 841 sr->flags |= IORING_RECV_MSHOT_LIM; 842 } else if (sqe->optlen) { 843 return -EINVAL; 844 } 845 req->flags |= REQ_F_APOLL_MULTISHOT; 846 } else if (sqe->optlen) { 847 return -EINVAL; 848 } 849 850 if (sr->flags & IORING_RECVSEND_BUNDLE) { 851 if (req->opcode == IORING_OP_RECVMSG) 852 return -EINVAL; 853 } 854 855 if (io_is_compat(req->ctx)) 856 sr->msg_flags |= MSG_CMSG_COMPAT; 857 858 sr->nr_multishot_loops = 0; 859 return io_recvmsg_prep_setup(req); 860 } 861 862 /* bits to clear in old and inherit in new cflags on bundle retry */ 863 #define CQE_F_MASK (IORING_CQE_F_SOCK_NONEMPTY|IORING_CQE_F_MORE|\ 864 IORING_CQE_F_BUF_MORE) 865 866 /* 867 * Finishes io_recv and io_recvmsg. 868 * 869 * Returns true if it is actually finished, or false if it should run 870 * again (for multishot). 871 */ 872 static inline bool io_recv_finish(struct io_kiocb *req, 873 struct io_async_msghdr *kmsg, 874 struct io_br_sel *sel, bool mshot_finished, 875 unsigned issue_flags, int consumed) 876 { 877 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 878 unsigned int cflags = 0; 879 880 if (kmsg->msg.msg_inq > 0) 881 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 882 883 if (sel->val > 0 && sr->flags & IORING_RECV_MSHOT_LIM) { 884 unsigned int this_len = sel->val - sr->done_io; 885 886 /* 887 * If sr->mshot_total_len hits zero, the limit has been reached. 888 * Mark mshot as finished, and flag MSHOT_DONE as well to 889 * prevent a potential bundle from being retried. 890 */ 891 sr->mshot_total_len -= min_t(unsigned int, this_len, 892 sr->mshot_total_len); 893 if (!sr->mshot_total_len) { 894 sr->flags |= IORING_RECV_MSHOT_DONE; 895 mshot_finished = true; 896 } 897 } 898 899 if (sr->flags & IORING_RECVSEND_BUNDLE) { 900 size_t this_ret = sel->val - sr->done_io; 901 902 cflags |= io_put_kbufs(req, consumed, sel->buf_list, io_bundle_nbufs(kmsg, consumed)); 903 if (sr->flags & IORING_RECV_RETRY) 904 cflags = req->cqe.flags | (cflags & CQE_F_MASK); 905 if (sr->mshot_len && sel->val >= sr->mshot_len) 906 sr->flags |= IORING_RECV_MSHOT_CAP; 907 /* bundle with no more immediate buffers, we're done */ 908 if (req->flags & REQ_F_BL_EMPTY) 909 goto finish; 910 /* 911 * If more is available AND it was a full transfer, retry and 912 * append to this one 913 */ 914 if (!(sr->flags & IORING_RECV_NO_RETRY) && 915 kmsg->msg.msg_inq > 1 && this_ret > 0 && 916 !iov_iter_count(&kmsg->msg.msg_iter)) { 917 req->cqe.flags = cflags & ~CQE_F_MASK; 918 sr->len = kmsg->msg.msg_inq; 919 sr->done_io += this_ret; 920 sr->flags |= IORING_RECV_RETRY; 921 return false; 922 } 923 } else { 924 cflags |= io_put_kbuf(req, consumed, sel->buf_list); 925 } 926 927 /* 928 * Fill CQE for this receive and see if we should keep trying to 929 * receive from this socket. 930 */ 931 if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished && 932 io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) { 933 sel->val = IOU_RETRY; 934 io_mshot_prep_retry(req, kmsg); 935 /* Known not-empty or unknown state, retry */ 936 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) { 937 if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY && 938 !(sr->flags & IORING_RECV_MSHOT_CAP)) { 939 return false; 940 } 941 /* mshot retries exceeded, force a requeue */ 942 sr->nr_multishot_loops = 0; 943 sr->flags &= ~IORING_RECV_MSHOT_CAP; 944 if (issue_flags & IO_URING_F_MULTISHOT) 945 sel->val = IOU_REQUEUE; 946 } 947 return true; 948 } 949 950 /* Finish the request / stop multishot. */ 951 finish: 952 io_req_set_res(req, sel->val, cflags); 953 sel->val = IOU_COMPLETE; 954 io_req_msg_cleanup(req, issue_flags); 955 return true; 956 } 957 958 static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg, 959 struct io_sr_msg *sr, void __user **buf, 960 size_t *len) 961 { 962 unsigned long ubuf = (unsigned long) *buf; 963 unsigned long hdr; 964 965 hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 966 kmsg->controllen; 967 if (*len < hdr) 968 return -EFAULT; 969 970 if (kmsg->controllen) { 971 unsigned long control = ubuf + hdr - kmsg->controllen; 972 973 kmsg->msg.msg_control_user = (void __user *) control; 974 kmsg->msg.msg_controllen = kmsg->controllen; 975 } 976 977 sr->buf = *buf; /* stash for later copy */ 978 *buf = (void __user *) (ubuf + hdr); 979 kmsg->payloadlen = *len = *len - hdr; 980 return 0; 981 } 982 983 struct io_recvmsg_multishot_hdr { 984 struct io_uring_recvmsg_out msg; 985 struct sockaddr_storage addr; 986 }; 987 988 static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io, 989 struct io_async_msghdr *kmsg, 990 unsigned int flags, bool *finished) 991 { 992 int err; 993 int copy_len; 994 struct io_recvmsg_multishot_hdr hdr; 995 996 if (kmsg->namelen) 997 kmsg->msg.msg_name = &hdr.addr; 998 kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT); 999 kmsg->msg.msg_namelen = 0; 1000 1001 if (sock->file->f_flags & O_NONBLOCK) 1002 flags |= MSG_DONTWAIT; 1003 1004 err = sock_recvmsg(sock, &kmsg->msg, flags); 1005 *finished = err <= 0; 1006 if (err < 0) 1007 return err; 1008 1009 hdr.msg = (struct io_uring_recvmsg_out) { 1010 .controllen = kmsg->controllen - kmsg->msg.msg_controllen, 1011 .flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT 1012 }; 1013 1014 hdr.msg.payloadlen = err; 1015 if (err > kmsg->payloadlen) 1016 err = kmsg->payloadlen; 1017 1018 copy_len = sizeof(struct io_uring_recvmsg_out); 1019 if (kmsg->msg.msg_namelen > kmsg->namelen) 1020 copy_len += kmsg->namelen; 1021 else 1022 copy_len += kmsg->msg.msg_namelen; 1023 1024 /* 1025 * "fromlen shall refer to the value before truncation.." 1026 * 1003.1g 1027 */ 1028 hdr.msg.namelen = kmsg->msg.msg_namelen; 1029 1030 /* ensure that there is no gap between hdr and sockaddr_storage */ 1031 BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr, addr) != 1032 sizeof(struct io_uring_recvmsg_out)); 1033 if (copy_to_user(io->buf, &hdr, copy_len)) { 1034 *finished = true; 1035 return -EFAULT; 1036 } 1037 1038 return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 1039 kmsg->controllen + err; 1040 } 1041 1042 int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 1043 { 1044 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1045 struct io_async_msghdr *kmsg = req->async_data; 1046 struct io_br_sel sel = { }; 1047 struct socket *sock; 1048 unsigned flags; 1049 int ret, min_ret = 0; 1050 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1051 bool mshot_finished = true; 1052 int consumed = 0; 1053 size_t len; 1054 1055 sock = sock_from_file(req->file); 1056 if (unlikely(!sock)) 1057 return -ENOTSOCK; 1058 1059 if (!(req->flags & REQ_F_POLLED) && 1060 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1061 return -EAGAIN; 1062 1063 flags = sr->msg_flags; 1064 if (force_nonblock) 1065 flags |= MSG_DONTWAIT; 1066 1067 retry_multishot: 1068 sel.buf_list = NULL; 1069 len = sr->len; 1070 if (io_do_buffer_select(req)) { 1071 sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); 1072 if (!sel.addr) 1073 return -ENOBUFS; 1074 len = min_t(size_t, len, MAX_RW_COUNT); 1075 1076 if (req->flags & REQ_F_APOLL_MULTISHOT) { 1077 ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len); 1078 if (ret) { 1079 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1080 return ret; 1081 } 1082 } 1083 1084 iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len); 1085 } 1086 1087 kmsg->msg.msg_get_inq = 1; 1088 kmsg->msg.msg_inq = -1; 1089 if (req->flags & REQ_F_APOLL_MULTISHOT) { 1090 ret = io_recvmsg_multishot(sock, sr, kmsg, flags, 1091 &mshot_finished); 1092 consumed = ret; 1093 } else { 1094 /* disable partial retry for recvmsg with cmsg attached */ 1095 if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen) 1096 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1097 1098 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, 1099 kmsg->uaddr, flags); 1100 /* 1101 * With MSG_TRUNC, the net layer will return the full size of 1102 * the packet, even if we only filled part of it in the buffers. 1103 * Adjust the returned size to consume only the real part of the 1104 * buffer. 1105 */ 1106 consumed = ret; 1107 if (ret > 0) 1108 consumed = min_t(size_t, ret, len); 1109 } 1110 1111 if (ret < min_ret) { 1112 if (ret == -EAGAIN && force_nonblock) { 1113 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1114 return IOU_RETRY; 1115 } 1116 if (ret > 0 && io_net_retry(sock, flags) && 1117 io_net_kbuf_can_retry(req, sel.buf_list, kmsg)) { 1118 sr->done_io += ret; 1119 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); 1120 } 1121 if (ret == -ERESTARTSYS) 1122 ret = -EINTR; 1123 req_set_fail(req); 1124 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 1125 req_set_fail(req); 1126 } 1127 1128 if (ret > 0) 1129 ret += sr->done_io; 1130 else if (sr->done_io) 1131 ret = sr->done_io; 1132 else 1133 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1134 1135 sel.val = ret; 1136 if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed)) 1137 goto retry_multishot; 1138 1139 return sel.val; 1140 } 1141 1142 static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg, 1143 struct io_br_sel *sel, unsigned int issue_flags) 1144 { 1145 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1146 size_t len; 1147 int ret; 1148 1149 /* 1150 * If the ring isn't locked, then don't use the peek interface 1151 * to grab multiple buffers as we will lock/unlock between 1152 * this selection and posting the buffers. 1153 */ 1154 if (!(issue_flags & IO_URING_F_UNLOCKED) && 1155 sr->flags & IORING_RECVSEND_BUNDLE) { 1156 struct buf_sel_arg arg = { 1157 .iovs = &kmsg->fast_iov, 1158 .nr_iovs = 1, 1159 .mode = KBUF_MODE_EXPAND, 1160 .buf_group = sr->buf_group, 1161 }; 1162 1163 if (kmsg->vec.iovec) { 1164 arg.nr_iovs = kmsg->vec.nr; 1165 arg.iovs = kmsg->vec.iovec; 1166 arg.mode |= KBUF_MODE_FREE; 1167 } 1168 1169 if (sel->val) 1170 arg.max_len = sel->val; 1171 else if (kmsg->msg.msg_inq > 1) 1172 arg.max_len = kmsg->msg.msg_inq; 1173 1174 /* if mshot limited, ensure we don't go over */ 1175 if (sr->flags & IORING_RECV_MSHOT_LIM) 1176 arg.max_len = min_not_zero(arg.max_len, sr->mshot_total_len); 1177 ret = io_buffers_peek(req, &arg, sel); 1178 if (unlikely(ret < 0)) 1179 return ret; 1180 1181 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) { 1182 kmsg->vec.nr = ret; 1183 kmsg->vec.iovec = arg.iovs; 1184 req->flags |= REQ_F_NEED_CLEANUP; 1185 } 1186 if (arg.partial_map) 1187 sr->flags |= IORING_RECV_PARTIAL_MAP; 1188 1189 /* special case 1 vec, can be a fast path */ 1190 if (ret == 1) { 1191 sr->buf = arg.iovs[0].iov_base; 1192 len = sr->len = arg.iovs[0].iov_len; 1193 goto map_ubuf; 1194 } 1195 iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret, 1196 arg.out_len); 1197 len = arg.out_len; 1198 } else { 1199 len = sel->val; 1200 1201 *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); 1202 if (!sel->addr) 1203 return -ENOBUFS; 1204 /* ring provided buffers can be up to 4G, keep len an int */ 1205 len = min_t(size_t, len, MAX_RW_COUNT); 1206 sr->buf = sel->addr; 1207 sr->len = len; 1208 map_ubuf: 1209 ret = import_ubuf(ITER_DEST, sr->buf, sr->len, 1210 &kmsg->msg.msg_iter); 1211 if (unlikely(ret)) 1212 return ret; 1213 } 1214 1215 return len; 1216 } 1217 1218 int io_recv(struct io_kiocb *req, unsigned int issue_flags) 1219 { 1220 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1221 struct io_async_msghdr *kmsg = req->async_data; 1222 struct io_br_sel sel; 1223 struct socket *sock; 1224 unsigned flags; 1225 int ret, min_ret = 0, consumed = 0; 1226 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1227 bool mshot_finished; 1228 size_t len = 0; 1229 1230 sock = sock_from_file(req->file); 1231 if (unlikely(!sock)) 1232 return -ENOTSOCK; 1233 1234 if (!(req->flags & REQ_F_POLLED) && 1235 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1236 return -EAGAIN; 1237 1238 flags = sr->msg_flags; 1239 if (force_nonblock) 1240 flags |= MSG_DONTWAIT; 1241 1242 if (req->flags & REQ_F_IMPORT_BUFFER) { 1243 ret = io_import_reg_buf(req, &kmsg->msg.msg_iter, 1244 (u64)(uintptr_t)sr->buf, sr->len, 1245 ITER_DEST, issue_flags); 1246 if (unlikely(ret)) { 1247 kmsg->msg.msg_inq = -1; 1248 sel.buf_list = NULL; 1249 goto out_free; 1250 } 1251 req->flags &= ~REQ_F_IMPORT_BUFFER; 1252 } 1253 1254 retry_multishot: 1255 sel.buf_list = NULL; 1256 len = sr->len; 1257 if (io_do_buffer_select(req)) { 1258 sel.val = sr->len; 1259 ret = io_recv_buf_select(req, kmsg, &sel, issue_flags); 1260 if (unlikely(ret < 0)) { 1261 kmsg->msg.msg_inq = -1; 1262 goto out_free; 1263 } 1264 len = ret; 1265 sr->buf = NULL; 1266 } 1267 1268 kmsg->msg.msg_flags = 0; 1269 kmsg->msg.msg_inq = -1; 1270 1271 if (flags & MSG_WAITALL) 1272 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1273 1274 ret = sock_recvmsg(sock, &kmsg->msg, flags); 1275 if (ret < min_ret) { 1276 if (ret == -EAGAIN && force_nonblock) { 1277 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1278 return IOU_RETRY; 1279 } 1280 if (ret > 0 && io_net_retry(sock, flags) && 1281 io_net_kbuf_can_retry(req, sel.buf_list, kmsg)) { 1282 sr->len -= ret; 1283 sr->buf += ret; 1284 sr->done_io += ret; 1285 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); 1286 } 1287 if (ret == -ERESTARTSYS) 1288 ret = -EINTR; 1289 req_set_fail(req); 1290 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 1291 out_free: 1292 req_set_fail(req); 1293 } 1294 1295 mshot_finished = ret <= 0; 1296 1297 /* 1298 * With MSG_TRUNC, the net layer will return the full size of 1299 * the packet, even if we only filled part of it in the buffers. 1300 * Adjust the returned size to consume only the real part of the 1301 * buffer. 1302 */ 1303 consumed = ret; 1304 if (ret > 0) 1305 consumed = min_t(size_t, ret, len); 1306 1307 if (ret > 0) 1308 ret += sr->done_io; 1309 else if (sr->done_io) 1310 ret = sr->done_io; 1311 else 1312 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1313 1314 sel.val = ret; 1315 if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed)) 1316 goto retry_multishot; 1317 1318 return sel.val; 1319 } 1320 1321 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1322 { 1323 struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc); 1324 unsigned ifq_idx; 1325 1326 if (unlikely(sqe->addr2 || sqe->addr || sqe->addr3)) 1327 return -EINVAL; 1328 1329 ifq_idx = READ_ONCE(sqe->zcrx_ifq_idx); 1330 zc->ifq = xa_load(&req->ctx->zcrx_ctxs, ifq_idx); 1331 if (!zc->ifq) 1332 return -EINVAL; 1333 1334 zc->len = READ_ONCE(sqe->len); 1335 zc->flags = READ_ONCE(sqe->ioprio); 1336 if (READ_ONCE(sqe->msg_flags)) 1337 return -EINVAL; 1338 if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT)) 1339 return -EINVAL; 1340 /* multishot required */ 1341 if (!(zc->flags & IORING_RECV_MULTISHOT)) 1342 return -EINVAL; 1343 /* All data completions are posted as aux CQEs. */ 1344 req->flags |= REQ_F_APOLL_MULTISHOT; 1345 1346 return 0; 1347 } 1348 1349 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags) 1350 { 1351 struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc); 1352 struct socket *sock; 1353 unsigned int len; 1354 int ret; 1355 1356 sock = sock_from_file(req->file); 1357 if (unlikely(!sock)) 1358 return -ENOTSOCK; 1359 1360 if (!(req->flags & REQ_F_POLLED) && 1361 (zc->flags & IORING_RECVSEND_POLL_FIRST)) 1362 return -EAGAIN; 1363 1364 len = zc->len; 1365 ret = io_zcrx_recv(req, zc->ifq, sock, 0, issue_flags, &zc->len); 1366 if (len && zc->len == 0) { 1367 io_req_set_res(req, 0, 0); 1368 1369 return IOU_COMPLETE; 1370 } 1371 if (unlikely(ret <= 0) && ret != -EAGAIN) { 1372 if (ret == -ERESTARTSYS) 1373 ret = -EINTR; 1374 if (ret == IOU_REQUEUE) 1375 return IOU_REQUEUE; 1376 1377 req_set_fail(req); 1378 io_req_set_res(req, ret, 0); 1379 return IOU_COMPLETE; 1380 } 1381 return IOU_RETRY; 1382 } 1383 1384 void io_send_zc_cleanup(struct io_kiocb *req) 1385 { 1386 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 1387 struct io_async_msghdr *io = req->async_data; 1388 1389 if (req_has_async_data(req)) 1390 io_netmsg_iovec_free(io); 1391 if (zc->notif) { 1392 io_notif_flush(zc->notif); 1393 zc->notif = NULL; 1394 } 1395 } 1396 1397 #define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF) 1398 #define IO_ZC_FLAGS_VALID (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE | \ 1399 IORING_SEND_VECTORIZED) 1400 1401 int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1402 { 1403 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 1404 struct io_ring_ctx *ctx = req->ctx; 1405 struct io_async_msghdr *iomsg; 1406 struct io_kiocb *notif; 1407 u64 user_data; 1408 int ret; 1409 1410 zc->done_io = 0; 1411 1412 if (unlikely(READ_ONCE(sqe->__pad2[0]))) 1413 return -EINVAL; 1414 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */ 1415 if (req->flags & REQ_F_CQE_SKIP) 1416 return -EINVAL; 1417 1418 notif = zc->notif = io_alloc_notif(ctx); 1419 if (!notif) 1420 return -ENOMEM; 1421 user_data = READ_ONCE(sqe->addr3); 1422 if (!user_data) 1423 user_data = req->cqe.user_data; 1424 1425 notif->cqe.user_data = user_data; 1426 notif->cqe.res = 0; 1427 notif->cqe.flags = IORING_CQE_F_NOTIF; 1428 req->flags |= REQ_F_NEED_CLEANUP | REQ_F_POLL_NO_LAZY; 1429 1430 zc->flags = READ_ONCE(sqe->ioprio); 1431 if (unlikely(zc->flags & ~IO_ZC_FLAGS_COMMON)) { 1432 if (zc->flags & ~IO_ZC_FLAGS_VALID) 1433 return -EINVAL; 1434 if (zc->flags & IORING_SEND_ZC_REPORT_USAGE) { 1435 struct io_notif_data *nd = io_notif_to_data(notif); 1436 1437 nd->zc_report = true; 1438 nd->zc_used = false; 1439 nd->zc_copied = false; 1440 } 1441 } 1442 1443 zc->len = READ_ONCE(sqe->len); 1444 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL | MSG_ZEROCOPY; 1445 req->buf_index = READ_ONCE(sqe->buf_index); 1446 if (zc->msg_flags & MSG_DONTWAIT) 1447 req->flags |= REQ_F_NOWAIT; 1448 1449 if (io_is_compat(ctx)) 1450 zc->msg_flags |= MSG_CMSG_COMPAT; 1451 1452 iomsg = io_msg_alloc_async(req); 1453 if (unlikely(!iomsg)) 1454 return -ENOMEM; 1455 1456 if (req->opcode == IORING_OP_SEND_ZC) { 1457 ret = io_send_setup(req, sqe); 1458 } else { 1459 if (unlikely(sqe->addr2 || sqe->file_index)) 1460 return -EINVAL; 1461 ret = io_sendmsg_setup(req, sqe); 1462 } 1463 if (unlikely(ret)) 1464 return ret; 1465 1466 if (!(zc->flags & IORING_RECVSEND_FIXED_BUF)) { 1467 iomsg->msg.sg_from_iter = io_sg_from_iter_iovec; 1468 return io_notif_account_mem(zc->notif, iomsg->msg.msg_iter.count); 1469 } 1470 iomsg->msg.sg_from_iter = io_sg_from_iter; 1471 return 0; 1472 } 1473 1474 static int io_sg_from_iter_iovec(struct sk_buff *skb, 1475 struct iov_iter *from, size_t length) 1476 { 1477 skb_zcopy_downgrade_managed(skb); 1478 return zerocopy_fill_skb_from_iter(skb, from, length); 1479 } 1480 1481 static int io_sg_from_iter(struct sk_buff *skb, 1482 struct iov_iter *from, size_t length) 1483 { 1484 struct skb_shared_info *shinfo = skb_shinfo(skb); 1485 int frag = shinfo->nr_frags; 1486 int ret = 0; 1487 struct bvec_iter bi; 1488 ssize_t copied = 0; 1489 unsigned long truesize = 0; 1490 1491 if (!frag) 1492 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS; 1493 else if (unlikely(!skb_zcopy_managed(skb))) 1494 return zerocopy_fill_skb_from_iter(skb, from, length); 1495 1496 bi.bi_size = min(from->count, length); 1497 bi.bi_offset = from->iov_offset; 1498 bi.bi_idx = 0; 1499 1500 while (bi.bi_size && frag < MAX_SKB_FRAGS) { 1501 struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi); 1502 1503 copied += v.bv_len; 1504 truesize += PAGE_ALIGN(v.bv_len + v.bv_offset); 1505 __skb_fill_page_desc_noacc(shinfo, frag++, v.bv_page, 1506 v.bv_offset, v.bv_len); 1507 bvec_iter_advance_single(from->bvec, &bi, v.bv_len); 1508 } 1509 if (bi.bi_size) 1510 ret = -EMSGSIZE; 1511 1512 shinfo->nr_frags = frag; 1513 from->bvec += bi.bi_idx; 1514 from->nr_segs -= bi.bi_idx; 1515 from->count -= copied; 1516 from->iov_offset = bi.bi_offset; 1517 1518 skb->data_len += copied; 1519 skb->len += copied; 1520 skb->truesize += truesize; 1521 return ret; 1522 } 1523 1524 static int io_send_zc_import(struct io_kiocb *req, 1525 struct io_async_msghdr *kmsg, 1526 unsigned int issue_flags) 1527 { 1528 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1529 struct io_kiocb *notif = sr->notif; 1530 int ret; 1531 1532 WARN_ON_ONCE(!(sr->flags & IORING_RECVSEND_FIXED_BUF)); 1533 1534 notif->buf_index = req->buf_index; 1535 1536 if (!(sr->flags & IORING_SEND_VECTORIZED)) { 1537 ret = io_import_reg_buf(notif, &kmsg->msg.msg_iter, 1538 (u64)(uintptr_t)sr->buf, sr->len, 1539 ITER_SOURCE, issue_flags); 1540 } else { 1541 unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs; 1542 1543 ret = io_import_reg_vec(ITER_SOURCE, &kmsg->msg.msg_iter, 1544 notif, &kmsg->vec, uvec_segs, 1545 issue_flags); 1546 } 1547 1548 if (unlikely(ret)) 1549 return ret; 1550 req->flags &= ~REQ_F_IMPORT_BUFFER; 1551 return 0; 1552 } 1553 1554 int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) 1555 { 1556 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1557 struct io_async_msghdr *kmsg = req->async_data; 1558 struct socket *sock; 1559 unsigned msg_flags; 1560 int ret, min_ret = 0; 1561 1562 sock = sock_from_file(req->file); 1563 if (unlikely(!sock)) 1564 return -ENOTSOCK; 1565 if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags)) 1566 return -EOPNOTSUPP; 1567 if (!(req->flags & REQ_F_POLLED) && 1568 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1569 return -EAGAIN; 1570 1571 if (req->flags & REQ_F_IMPORT_BUFFER) { 1572 ret = io_send_zc_import(req, kmsg, issue_flags); 1573 if (unlikely(ret)) 1574 return ret; 1575 } 1576 1577 msg_flags = sr->msg_flags; 1578 if (issue_flags & IO_URING_F_NONBLOCK) 1579 msg_flags |= MSG_DONTWAIT; 1580 if (msg_flags & MSG_WAITALL) 1581 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1582 1583 kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg; 1584 1585 if (req->opcode == IORING_OP_SEND_ZC) { 1586 msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS; 1587 kmsg->msg.msg_flags = msg_flags; 1588 ret = sock_sendmsg(sock, &kmsg->msg); 1589 } else { 1590 kmsg->msg.msg_control_user = sr->msg_control; 1591 ret = __sys_sendmsg_sock(sock, &kmsg->msg, msg_flags); 1592 } 1593 1594 if (unlikely(ret < min_ret)) { 1595 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1596 return -EAGAIN; 1597 1598 if (ret > 0 && io_net_retry(sock, sr->msg_flags)) { 1599 sr->done_io += ret; 1600 return -EAGAIN; 1601 } 1602 if (ret == -ERESTARTSYS) 1603 ret = -EINTR; 1604 req_set_fail(req); 1605 } 1606 1607 if (ret >= 0) 1608 ret += sr->done_io; 1609 else if (sr->done_io) 1610 ret = sr->done_io; 1611 1612 /* 1613 * If we're in io-wq we can't rely on tw ordering guarantees, defer 1614 * flushing notif to io_send_zc_cleanup() 1615 */ 1616 if (!(issue_flags & IO_URING_F_UNLOCKED)) { 1617 io_notif_flush(sr->notif); 1618 sr->notif = NULL; 1619 io_req_msg_cleanup(req, 0); 1620 } 1621 io_req_set_res(req, ret, IORING_CQE_F_MORE); 1622 return IOU_COMPLETE; 1623 } 1624 1625 void io_sendrecv_fail(struct io_kiocb *req) 1626 { 1627 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1628 1629 if (sr->done_io) 1630 req->cqe.res = sr->done_io; 1631 1632 if ((req->flags & REQ_F_NEED_CLEANUP) && 1633 (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC)) 1634 req->cqe.flags |= IORING_CQE_F_MORE; 1635 } 1636 1637 #define ACCEPT_FLAGS (IORING_ACCEPT_MULTISHOT | IORING_ACCEPT_DONTWAIT | \ 1638 IORING_ACCEPT_POLL_FIRST) 1639 1640 int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1641 { 1642 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1643 1644 if (sqe->len || sqe->buf_index) 1645 return -EINVAL; 1646 1647 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1648 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 1649 accept->flags = READ_ONCE(sqe->accept_flags); 1650 accept->nofile = rlimit(RLIMIT_NOFILE); 1651 accept->iou_flags = READ_ONCE(sqe->ioprio); 1652 if (accept->iou_flags & ~ACCEPT_FLAGS) 1653 return -EINVAL; 1654 1655 accept->file_slot = READ_ONCE(sqe->file_index); 1656 if (accept->file_slot) { 1657 if (accept->flags & SOCK_CLOEXEC) 1658 return -EINVAL; 1659 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT && 1660 accept->file_slot != IORING_FILE_INDEX_ALLOC) 1661 return -EINVAL; 1662 } 1663 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1664 return -EINVAL; 1665 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK)) 1666 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK; 1667 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT) 1668 req->flags |= REQ_F_APOLL_MULTISHOT; 1669 if (accept->iou_flags & IORING_ACCEPT_DONTWAIT) 1670 req->flags |= REQ_F_NOWAIT; 1671 return 0; 1672 } 1673 1674 int io_accept(struct io_kiocb *req, unsigned int issue_flags) 1675 { 1676 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1677 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1678 bool fixed = !!accept->file_slot; 1679 struct proto_accept_arg arg = { 1680 .flags = force_nonblock ? O_NONBLOCK : 0, 1681 }; 1682 struct file *file; 1683 unsigned cflags; 1684 int ret, fd; 1685 1686 if (!(req->flags & REQ_F_POLLED) && 1687 accept->iou_flags & IORING_ACCEPT_POLL_FIRST) 1688 return -EAGAIN; 1689 1690 retry: 1691 if (!fixed) { 1692 fd = __get_unused_fd_flags(accept->flags, accept->nofile); 1693 if (unlikely(fd < 0)) 1694 return fd; 1695 } 1696 arg.err = 0; 1697 arg.is_empty = -1; 1698 file = do_accept(req->file, &arg, accept->addr, accept->addr_len, 1699 accept->flags); 1700 if (IS_ERR(file)) { 1701 if (!fixed) 1702 put_unused_fd(fd); 1703 ret = PTR_ERR(file); 1704 if (ret == -EAGAIN && force_nonblock && 1705 !(accept->iou_flags & IORING_ACCEPT_DONTWAIT)) 1706 return IOU_RETRY; 1707 1708 if (ret == -ERESTARTSYS) 1709 ret = -EINTR; 1710 } else if (!fixed) { 1711 fd_install(fd, file); 1712 ret = fd; 1713 } else { 1714 ret = io_fixed_fd_install(req, issue_flags, file, 1715 accept->file_slot); 1716 } 1717 1718 cflags = 0; 1719 if (!arg.is_empty) 1720 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 1721 1722 if (ret >= 0 && (req->flags & REQ_F_APOLL_MULTISHOT) && 1723 io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) { 1724 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || arg.is_empty == -1) 1725 goto retry; 1726 return IOU_RETRY; 1727 } 1728 1729 io_req_set_res(req, ret, cflags); 1730 if (ret < 0) 1731 req_set_fail(req); 1732 return IOU_COMPLETE; 1733 } 1734 1735 void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req) 1736 { 1737 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1738 1739 bctx->socket.family = sock->domain; 1740 bctx->socket.type = sock->type & SOCK_TYPE_MASK; 1741 bctx->socket.protocol = sock->protocol; 1742 } 1743 1744 void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req) 1745 { 1746 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1747 struct sockaddr_storage *ss = req->async_data; 1748 1749 /* 1750 * move_addr_to_kernel() skips the copy for addr_len == 0, so 1751 * iomsg->addr may hold stale data from a prior CONNECT. Bail 1752 * unless addr_len covers the family discriminator. 1753 */ 1754 if (conn->addr_len < (int)sizeof(sa_family_t)) 1755 return; 1756 1757 bctx->connect.family = ss->ss_family; 1758 switch (ss->ss_family) { 1759 case AF_INET: { 1760 struct sockaddr_in *sin = (struct sockaddr_in *)ss; 1761 1762 if (conn->addr_len < (int)sizeof(*sin)) 1763 break; 1764 bctx->connect.port = sin->sin_port; 1765 bctx->connect.v4_addr = sin->sin_addr.s_addr; 1766 break; 1767 } 1768 case AF_INET6: { 1769 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss; 1770 1771 /* The IPv6 stack accepts SIN6_LEN_RFC2133; port and addr are in it. */ 1772 if (conn->addr_len < SIN6_LEN_RFC2133) 1773 break; 1774 bctx->connect.port = sin6->sin6_port; 1775 memcpy(bctx->connect.v6_addr, &sin6->sin6_addr, 1776 sizeof(bctx->connect.v6_addr)); 1777 break; 1778 } 1779 default: 1780 /* family is set; per-family fields stay zero - family-only filtering */ 1781 break; 1782 } 1783 } 1784 1785 int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1786 { 1787 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1788 1789 if (sqe->addr || sqe->rw_flags || sqe->buf_index) 1790 return -EINVAL; 1791 1792 sock->domain = READ_ONCE(sqe->fd); 1793 sock->type = READ_ONCE(sqe->off); 1794 sock->protocol = READ_ONCE(sqe->len); 1795 sock->file_slot = READ_ONCE(sqe->file_index); 1796 sock->nofile = rlimit(RLIMIT_NOFILE); 1797 1798 sock->flags = sock->type & ~SOCK_TYPE_MASK; 1799 if (sock->file_slot && (sock->flags & SOCK_CLOEXEC)) 1800 return -EINVAL; 1801 if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1802 return -EINVAL; 1803 return 0; 1804 } 1805 1806 int io_socket(struct io_kiocb *req, unsigned int issue_flags) 1807 { 1808 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1809 bool fixed = !!sock->file_slot; 1810 struct file *file; 1811 int ret, fd; 1812 1813 if (!fixed) { 1814 fd = __get_unused_fd_flags(sock->flags, sock->nofile); 1815 if (unlikely(fd < 0)) 1816 return fd; 1817 } 1818 file = __sys_socket_file(sock->domain, sock->type, sock->protocol); 1819 if (IS_ERR(file)) { 1820 if (!fixed) 1821 put_unused_fd(fd); 1822 ret = PTR_ERR(file); 1823 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1824 return -EAGAIN; 1825 if (ret == -ERESTARTSYS) 1826 ret = -EINTR; 1827 req_set_fail(req); 1828 } else if (!fixed) { 1829 fd_install(fd, file); 1830 ret = fd; 1831 } else { 1832 ret = io_fixed_fd_install(req, issue_flags, file, 1833 sock->file_slot); 1834 } 1835 io_req_set_res(req, ret, 0); 1836 return IOU_COMPLETE; 1837 } 1838 1839 int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1840 { 1841 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1842 struct sockaddr_storage *addr; 1843 1844 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 1845 return -EINVAL; 1846 1847 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1848 conn->addr_len = READ_ONCE(sqe->addr2); 1849 conn->in_progress = conn->seen_econnaborted = false; 1850 1851 addr = io_uring_alloc_async_data(NULL, req); 1852 if (unlikely(!addr)) 1853 return -ENOMEM; 1854 1855 return move_addr_to_kernel(conn->addr, conn->addr_len, addr); 1856 } 1857 1858 int io_connect(struct io_kiocb *req, unsigned int issue_flags) 1859 { 1860 struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect); 1861 struct sockaddr_storage *addr = req->async_data; 1862 unsigned file_flags; 1863 int ret; 1864 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1865 1866 if (connect->in_progress) { 1867 struct poll_table_struct pt = { ._key = EPOLLERR }; 1868 1869 if (vfs_poll(req->file, &pt) & EPOLLERR) 1870 goto get_sock_err; 1871 } 1872 1873 file_flags = force_nonblock ? O_NONBLOCK : 0; 1874 1875 ret = __sys_connect_file(req->file, addr, connect->addr_len, file_flags); 1876 if ((ret == -EAGAIN || ret == -EINPROGRESS || ret == -ECONNABORTED) 1877 && force_nonblock) { 1878 if (ret == -EINPROGRESS) { 1879 connect->in_progress = true; 1880 } else if (ret == -ECONNABORTED) { 1881 if (connect->seen_econnaborted) 1882 goto out; 1883 connect->seen_econnaborted = true; 1884 } else if (req->file->f_flags & O_NONBLOCK) { 1885 /* 1886 * __sys_connect_file() adds O_NONBLOCK from the socket 1887 * to every attempt, so a retry from io-wq fails with 1888 * -EAGAIN as well. A poll on the socket doesn't wait 1889 * until a retry can succeed. For example, an AF_UNIX 1890 * client reports EPOLLOUT right away while the 1891 * listener's backlog is full. Complete the request 1892 * with -EAGAIN, like connect(2) does. 1893 */ 1894 goto out; 1895 } 1896 return -EAGAIN; 1897 } 1898 if (connect->in_progress) { 1899 /* 1900 * At least bluetooth will return -EBADFD on a re-connect 1901 * attempt, and it's (supposedly) also valid to get -EISCONN 1902 * which means the previous result is good. For both of these, 1903 * grab the sock_error() and use that for the completion. 1904 */ 1905 if (ret == -EBADFD || ret == -EISCONN) { 1906 get_sock_err: 1907 ret = sock_error(sock_from_file(req->file)->sk); 1908 } 1909 } 1910 if (ret == -ERESTARTSYS) 1911 ret = -EINTR; 1912 out: 1913 if (ret < 0) 1914 req_set_fail(req); 1915 io_req_set_res(req, ret, 0); 1916 return IOU_COMPLETE; 1917 } 1918 1919 /* 1920 * Check if bind request would potentially end up with filename_create(), 1921 * which in turn end up in mnt_want_write() which will grab the fs 1922 * percpu start write sem. This can trigger a lockdep warning. 1923 */ 1924 static int io_bind_file_create(const struct sockaddr_storage *addr, int addr_len) 1925 { 1926 const struct sockaddr_un *sun; 1927 1928 if (addr->ss_family != AF_UNIX) 1929 return 0; 1930 if (addr_len <= offsetof(struct sockaddr_un, sun_path)) 1931 return 0; 1932 sun = (const struct sockaddr_un *) addr; 1933 return sun->sun_path[0] != '\0'; 1934 } 1935 1936 int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1937 { 1938 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind); 1939 struct sockaddr __user *uaddr; 1940 struct sockaddr_storage *addr; 1941 int ret; 1942 1943 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 1944 return -EINVAL; 1945 1946 uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1947 bind->addr_len = READ_ONCE(sqe->addr2); 1948 1949 addr = io_uring_alloc_async_data(NULL, req); 1950 if (unlikely(!addr)) 1951 return -ENOMEM; 1952 ret = move_addr_to_kernel(uaddr, bind->addr_len, addr); 1953 if (unlikely(ret)) 1954 return ret; 1955 if (io_bind_file_create(addr, bind->addr_len)) 1956 req->flags |= REQ_F_FORCE_ASYNC; 1957 return 0; 1958 } 1959 1960 1961 int io_bind(struct io_kiocb *req, unsigned int issue_flags) 1962 { 1963 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind); 1964 struct sockaddr_storage *addr = req->async_data; 1965 struct socket *sock; 1966 int ret; 1967 1968 sock = sock_from_file(req->file); 1969 if (unlikely(!sock)) 1970 return -ENOTSOCK; 1971 1972 ret = __sys_bind_socket(sock, addr, bind->addr_len); 1973 if (ret < 0) 1974 req_set_fail(req); 1975 io_req_set_res(req, ret, 0); 1976 return 0; 1977 } 1978 1979 int io_listen_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1980 { 1981 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen); 1982 1983 if (sqe->addr || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in || sqe->addr2) 1984 return -EINVAL; 1985 1986 listen->backlog = READ_ONCE(sqe->len); 1987 return 0; 1988 } 1989 1990 int io_listen(struct io_kiocb *req, unsigned int issue_flags) 1991 { 1992 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen); 1993 struct socket *sock; 1994 int ret; 1995 1996 sock = sock_from_file(req->file); 1997 if (unlikely(!sock)) 1998 return -ENOTSOCK; 1999 2000 ret = __sys_listen_socket(sock, listen->backlog); 2001 if (ret < 0) 2002 req_set_fail(req); 2003 io_req_set_res(req, ret, 0); 2004 return 0; 2005 } 2006 2007 void io_netmsg_cache_free(const void *entry) 2008 { 2009 struct io_async_msghdr *kmsg = (struct io_async_msghdr *) entry; 2010 2011 io_vec_free(&kmsg->vec); 2012 kfree(kmsg); 2013 } 2014