1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/file.h> 5 #include <linux/slab.h> 6 #include <linux/net.h> 7 #include <linux/un.h> 8 #include <linux/compat.h> 9 #include <net/compat.h> 10 #include <net/ipv6.h> 11 #include <linux/io_uring.h> 12 13 #include <uapi/linux/io_uring.h> 14 15 #include "filetable.h" 16 #include "io_uring.h" 17 #include "kbuf.h" 18 #include "alloc_cache.h" 19 #include "net.h" 20 #include "notif.h" 21 #include "rsrc.h" 22 #include "zcrx.h" 23 24 struct io_shutdown { 25 struct file *file; 26 int how; 27 }; 28 29 struct io_accept { 30 struct file *file; 31 struct sockaddr __user *addr; 32 int __user *addr_len; 33 int flags; 34 int iou_flags; 35 u32 file_slot; 36 unsigned long nofile; 37 }; 38 39 struct io_socket { 40 struct file *file; 41 int domain; 42 int type; 43 int protocol; 44 int flags; 45 u32 file_slot; 46 unsigned long nofile; 47 }; 48 49 struct io_connect { 50 struct file *file; 51 struct sockaddr __user *addr; 52 int addr_len; 53 bool in_progress; 54 bool seen_econnaborted; 55 }; 56 57 struct io_bind { 58 struct file *file; 59 int addr_len; 60 }; 61 62 struct io_listen { 63 struct file *file; 64 int backlog; 65 }; 66 67 struct io_sr_msg { 68 struct file *file; 69 union { 70 struct compat_msghdr __user *umsg_compat; 71 struct user_msghdr __user *umsg; 72 void __user *buf; 73 }; 74 int len; 75 unsigned done_io; 76 unsigned msg_flags; 77 unsigned nr_multishot_loops; 78 u16 flags; 79 /* initialised and used only by !msg send variants */ 80 u16 buf_group; 81 /* per-invocation mshot limit */ 82 unsigned mshot_len; 83 /* overall mshot byte limit */ 84 unsigned mshot_total_len; 85 void __user *msg_control; 86 /* used only for send zerocopy */ 87 struct io_kiocb *notif; 88 }; 89 90 /* 91 * The UAPI flags are the lower 8 bits, as that's all sqe->ioprio will hold 92 * anyway. Use the upper 8 bits for internal uses. 93 */ 94 enum sr_retry_flags { 95 IORING_RECV_RETRY = (1U << 15), 96 IORING_RECV_PARTIAL_MAP = (1U << 14), 97 IORING_RECV_MSHOT_CAP = (1U << 13), 98 IORING_RECV_MSHOT_LIM = (1U << 12), 99 IORING_RECV_MSHOT_DONE = (1U << 11), 100 101 IORING_RECV_RETRY_CLEAR = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP, 102 IORING_RECV_NO_RETRY = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP | 103 IORING_RECV_MSHOT_CAP | IORING_RECV_MSHOT_DONE, 104 }; 105 106 /* 107 * Number of times we'll try and do receives if there's more data. If we 108 * exceed this limit, then add us to the back of the queue and retry from 109 * there. This helps fairness between flooding clients. 110 */ 111 #define MULTISHOT_MAX_RETRY 32 112 113 struct io_recvzc { 114 struct file *file; 115 u16 flags; 116 u32 len; 117 struct io_zcrx_ifq *ifq; 118 }; 119 120 static int io_sg_from_iter_iovec(struct sk_buff *skb, 121 struct iov_iter *from, size_t length); 122 static int io_sg_from_iter(struct sk_buff *skb, 123 struct iov_iter *from, size_t length); 124 125 int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 126 { 127 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 128 129 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags || 130 sqe->buf_index || sqe->splice_fd_in)) 131 return -EINVAL; 132 133 shutdown->how = READ_ONCE(sqe->len); 134 req->flags |= REQ_F_FORCE_ASYNC; 135 return 0; 136 } 137 138 int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) 139 { 140 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 141 struct socket *sock; 142 int ret; 143 144 WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); 145 146 sock = sock_from_file(req->file); 147 if (unlikely(!sock)) 148 return -ENOTSOCK; 149 150 ret = __sys_shutdown_sock(sock, shutdown->how); 151 io_req_set_res(req, ret, 0); 152 return IOU_COMPLETE; 153 } 154 155 static bool io_net_retry(struct socket *sock, int flags) 156 { 157 if (!(flags & MSG_WAITALL)) 158 return false; 159 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET; 160 } 161 162 static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg) 163 { 164 if (kmsg->vec.iovec) 165 io_vec_free(&kmsg->vec); 166 } 167 168 static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) 169 { 170 struct io_async_msghdr *hdr = req->async_data; 171 172 /* can't recycle, ensure we free the iovec if we have one */ 173 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) { 174 io_netmsg_iovec_free(hdr); 175 return; 176 } 177 178 /* Let normal cleanup path reap it if we fail adding to the cache */ 179 io_alloc_cache_vec_kasan(&hdr->vec); 180 if (hdr->vec.nr > IO_VEC_CACHE_SOFT_CAP) 181 io_vec_free(&hdr->vec); 182 183 if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr)) 184 io_req_async_data_clear(req, REQ_F_NEED_CLEANUP); 185 } 186 187 static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req) 188 { 189 struct io_ring_ctx *ctx = req->ctx; 190 struct io_async_msghdr *hdr; 191 192 hdr = io_uring_alloc_async_data(&ctx->netmsg_cache, req); 193 if (!hdr) 194 return NULL; 195 196 /* If the async data was cached, we might have an iov cached inside. */ 197 if (hdr->vec.iovec) 198 req->flags |= REQ_F_NEED_CLEANUP; 199 return hdr; 200 } 201 202 static inline void io_mshot_prep_retry(struct io_kiocb *req, 203 struct io_async_msghdr *kmsg) 204 { 205 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 206 207 req->flags &= ~REQ_F_BL_EMPTY; 208 sr->done_io = 0; 209 sr->flags &= ~IORING_RECV_RETRY_CLEAR; 210 sr->len = sr->mshot_len; 211 } 212 213 static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg, 214 const struct iovec __user *uiov, unsigned uvec_seg, 215 int ddir) 216 { 217 struct iovec *iov; 218 int ret, nr_segs; 219 220 if (iomsg->vec.iovec) { 221 nr_segs = iomsg->vec.nr; 222 iov = iomsg->vec.iovec; 223 } else { 224 nr_segs = 1; 225 iov = &iomsg->fast_iov; 226 } 227 228 ret = __import_iovec(ddir, uiov, uvec_seg, nr_segs, &iov, 229 &iomsg->msg.msg_iter, io_is_compat(req->ctx)); 230 if (unlikely(ret < 0)) 231 return ret; 232 233 if (iov) { 234 req->flags |= REQ_F_NEED_CLEANUP; 235 io_vec_reset_iovec(&iomsg->vec, iov, iomsg->msg.msg_iter.nr_segs); 236 } 237 return 0; 238 } 239 240 static int io_copy_msghdr_from_user(struct user_msghdr *msg, 241 struct user_msghdr __user *umsg) 242 { 243 if (!user_access_begin(umsg, sizeof(*umsg))) 244 return -EFAULT; 245 unsafe_get_user(msg->msg_name, &umsg->msg_name, ua_end); 246 unsafe_get_user(msg->msg_namelen, &umsg->msg_namelen, ua_end); 247 unsafe_get_user(msg->msg_iov, &umsg->msg_iov, ua_end); 248 unsafe_get_user(msg->msg_iovlen, &umsg->msg_iovlen, ua_end); 249 unsafe_get_user(msg->msg_control, &umsg->msg_control, ua_end); 250 unsafe_get_user(msg->msg_controllen, &umsg->msg_controllen, ua_end); 251 user_access_end(); 252 return 0; 253 ua_end: 254 user_access_end(); 255 return -EFAULT; 256 } 257 258 static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, 259 struct user_msghdr *msg, 260 struct sockaddr __user **save_addr) 261 { 262 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 263 int ret; 264 265 iomsg->msg.msg_name = &iomsg->addr; 266 iomsg->msg.msg_iter.nr_segs = 0; 267 268 if (io_is_compat(req->ctx)) { 269 struct compat_msghdr cmsg; 270 271 if (copy_from_user(&cmsg, sr->umsg_compat, sizeof(cmsg))) 272 return -EFAULT; 273 274 ret = __get_compat_msghdr(&iomsg->msg, &cmsg, save_addr); 275 if (ret) 276 return ret; 277 278 memset(msg, 0, sizeof(*msg)); 279 msg->msg_namelen = cmsg.msg_namelen; 280 msg->msg_controllen = cmsg.msg_controllen; 281 msg->msg_iov = compat_ptr(cmsg.msg_iov); 282 msg->msg_iovlen = cmsg.msg_iovlen; 283 } else { 284 ret = io_copy_msghdr_from_user(msg, sr->umsg); 285 if (unlikely(ret)) 286 return ret; 287 288 msg->msg_flags = 0; 289 290 ret = __copy_msghdr(&iomsg->msg, msg, save_addr); 291 if (ret) 292 return ret; 293 } 294 295 if (req->flags & REQ_F_BUFFER_SELECT) { 296 if (msg->msg_iovlen == 0) { 297 sr->len = 0; 298 } else if (msg->msg_iovlen > 1) { 299 return -EINVAL; 300 } else { 301 struct iovec fast_iov, *iov; 302 303 iov = iovec_from_user(msg->msg_iov, 1, 1, &fast_iov, 304 io_is_compat(req->ctx)); 305 if (IS_ERR(iov)) 306 return PTR_ERR(iov); 307 sr->len = iov->iov_len; 308 } 309 } 310 return 0; 311 } 312 313 void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) 314 { 315 struct io_async_msghdr *io = req->async_data; 316 317 io_netmsg_iovec_free(io); 318 } 319 320 static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) 321 { 322 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 323 struct io_async_msghdr *kmsg = req->async_data; 324 void __user *addr; 325 u16 addr_len; 326 int ret; 327 328 sr->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); 329 330 if (READ_ONCE(sqe->__pad3[0])) 331 return -EINVAL; 332 333 kmsg->msg.msg_name = NULL; 334 kmsg->msg.msg_namelen = 0; 335 kmsg->msg.msg_control = NULL; 336 kmsg->msg.msg_controllen = 0; 337 kmsg->msg.msg_ubuf = NULL; 338 339 addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 340 addr_len = READ_ONCE(sqe->addr_len); 341 if (addr) { 342 ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr); 343 if (unlikely(ret < 0)) 344 return ret; 345 kmsg->msg.msg_name = &kmsg->addr; 346 kmsg->msg.msg_namelen = addr_len; 347 } 348 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 349 if (!(sr->flags & IORING_SEND_VECTORIZED)) { 350 req->flags |= REQ_F_IMPORT_BUFFER; 351 return 0; 352 } 353 354 kmsg->msg.msg_iter.nr_segs = sr->len; 355 return io_prep_reg_iovec(req, &kmsg->vec, sr->buf, sr->len); 356 } 357 if (req->flags & REQ_F_BUFFER_SELECT) 358 return 0; 359 360 if (sr->flags & IORING_SEND_VECTORIZED) 361 return io_net_import_vec(req, kmsg, sr->buf, sr->len, ITER_SOURCE); 362 363 return import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter); 364 } 365 366 static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) 367 { 368 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 369 struct io_async_msghdr *kmsg = req->async_data; 370 struct user_msghdr msg; 371 int ret; 372 373 sr->flags |= IORING_SEND_VECTORIZED; 374 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 375 ret = io_msg_copy_hdr(req, kmsg, &msg, NULL); 376 if (unlikely(ret)) 377 return ret; 378 /* save msg_control as sys_sendmsg() overwrites it */ 379 sr->msg_control = kmsg->msg.msg_control_user; 380 381 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 382 kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen; 383 return io_prep_reg_iovec(req, &kmsg->vec, msg.msg_iov, 384 msg.msg_iovlen); 385 } 386 if (req->flags & REQ_F_BUFFER_SELECT) 387 return 0; 388 return io_net_import_vec(req, kmsg, msg.msg_iov, msg.msg_iovlen, ITER_SOURCE); 389 } 390 391 #define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE | \ 392 IORING_SEND_VECTORIZED | IORING_RECVSEND_FIXED_BUF) 393 394 int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 395 { 396 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 397 398 sr->done_io = 0; 399 sr->len = READ_ONCE(sqe->len); 400 if (unlikely(sr->len < 0)) 401 return -EINVAL; 402 sr->flags = READ_ONCE(sqe->ioprio); 403 if (sr->flags & ~SENDMSG_FLAGS) 404 return -EINVAL; 405 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 406 /* registered buffer send only supported for plain IORING_OP_SEND */ 407 if (req->opcode != IORING_OP_SEND || 408 (req->flags & REQ_F_BUFFER_SELECT) || 409 (sr->flags & (IORING_RECVSEND_BUNDLE|IORING_SEND_VECTORIZED))) 410 return -EINVAL; 411 req->buf_index = READ_ONCE(sqe->buf_index); 412 } 413 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 414 if (sr->msg_flags & MSG_DONTWAIT) 415 req->flags |= REQ_F_NOWAIT; 416 if (req->flags & REQ_F_BUFFER_SELECT) 417 sr->buf_group = req->buf_index; 418 sr->mshot_total_len = sr->mshot_len = 0; 419 if (sr->flags & IORING_RECVSEND_BUNDLE) { 420 if (req->opcode == IORING_OP_SENDMSG) 421 return -EINVAL; 422 sr->msg_flags |= MSG_WAITALL; 423 req->flags |= REQ_F_MULTISHOT; 424 } 425 426 if (io_is_compat(req->ctx)) 427 sr->msg_flags |= MSG_CMSG_COMPAT; 428 429 if (unlikely(!io_msg_alloc_async(req))) 430 return -ENOMEM; 431 if (req->opcode != IORING_OP_SENDMSG) 432 return io_send_setup(req, sqe); 433 if (unlikely(sqe->addr2 || sqe->file_index)) 434 return -EINVAL; 435 return io_sendmsg_setup(req, sqe); 436 } 437 438 static void io_req_msg_cleanup(struct io_kiocb *req, 439 unsigned int issue_flags) 440 { 441 io_netmsg_recycle(req, issue_flags); 442 } 443 444 /* 445 * For bundle completions, we need to figure out how many segments we consumed. 446 * A bundle could be using a single ITER_UBUF if that's all we mapped, or it 447 * could be using an ITER_IOVEC. If the latter, then if we consumed all of 448 * the segments, then it's a trivial questiont o answer. If we have residual 449 * data in the iter, then loop the segments to figure out how much we 450 * transferred. 451 */ 452 static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret) 453 { 454 struct iovec *iov; 455 int nbufs; 456 457 /* no data is always zero segments, and a ubuf is always 1 segment */ 458 if (ret <= 0) 459 return 0; 460 if (iter_is_ubuf(&kmsg->msg.msg_iter)) 461 return 1; 462 463 iov = kmsg->vec.iovec; 464 if (!iov) 465 iov = &kmsg->fast_iov; 466 467 /* if all data was transferred, it's basic pointer math */ 468 if (!iov_iter_count(&kmsg->msg.msg_iter)) 469 return iter_iov(&kmsg->msg.msg_iter) - iov; 470 471 /* short transfer, count segments */ 472 nbufs = 0; 473 do { 474 int this_len = min_t(int, iov[nbufs].iov_len, ret); 475 476 nbufs++; 477 ret -= this_len; 478 } while (ret); 479 480 return nbufs; 481 } 482 483 static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl, 484 struct io_async_msghdr *kmsg, int len) 485 { 486 req->flags |= REQ_F_BL_NO_RECYCLE; 487 if (req->flags & REQ_F_BUFFERS_COMMIT) 488 io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len)); 489 return IOU_RETRY; 490 } 491 492 static inline bool io_send_finish(struct io_kiocb *req, 493 struct io_async_msghdr *kmsg, 494 struct io_br_sel *sel) 495 { 496 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 497 bool bundle_finished = sel->val <= 0; 498 unsigned int cflags; 499 500 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) { 501 cflags = io_put_kbuf(req, sel->val, sel->buf_list); 502 goto finish; 503 } 504 505 cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val)); 506 507 /* 508 * Don't start new bundles if the buffer list is empty, or if the 509 * current operation needed to go through polling to complete. 510 */ 511 if (bundle_finished || req->flags & (REQ_F_BL_EMPTY | REQ_F_POLLED)) 512 goto finish; 513 514 /* 515 * Fill CQE for this receive and see if we should keep trying to 516 * receive from this socket. 517 */ 518 if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) { 519 io_mshot_prep_retry(req, kmsg); 520 return false; 521 } 522 523 /* Otherwise stop bundle and use the current result. */ 524 finish: 525 io_req_set_res(req, sel->val, cflags); 526 sel->val = IOU_COMPLETE; 527 return true; 528 } 529 530 int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) 531 { 532 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 533 struct io_async_msghdr *kmsg = req->async_data; 534 struct socket *sock; 535 unsigned flags; 536 int min_ret = 0; 537 int ret; 538 539 sock = sock_from_file(req->file); 540 if (unlikely(!sock)) 541 return -ENOTSOCK; 542 543 if (!(req->flags & REQ_F_POLLED) && 544 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 545 return -EAGAIN; 546 547 flags = sr->msg_flags; 548 if (issue_flags & IO_URING_F_NONBLOCK) 549 flags |= MSG_DONTWAIT; 550 if (flags & MSG_WAITALL) 551 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 552 553 kmsg->msg.msg_control_user = sr->msg_control; 554 555 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 556 557 if (ret < min_ret) { 558 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 559 return -EAGAIN; 560 if (ret > 0 && io_net_retry(sock, flags)) { 561 kmsg->msg.msg_controllen = 0; 562 kmsg->msg.msg_control = NULL; 563 sr->done_io += ret; 564 return -EAGAIN; 565 } 566 if (ret == -ERESTARTSYS) 567 ret = -EINTR; 568 req_set_fail(req); 569 } 570 io_req_msg_cleanup(req, issue_flags); 571 if (ret >= 0) 572 ret += sr->done_io; 573 else if (sr->done_io) 574 ret = sr->done_io; 575 io_req_set_res(req, ret, 0); 576 return IOU_COMPLETE; 577 } 578 579 static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags, 580 struct io_br_sel *sel, struct io_async_msghdr *kmsg) 581 { 582 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 583 struct buf_sel_arg arg = { 584 .iovs = &kmsg->fast_iov, 585 .max_len = min_not_zero(sr->len, INT_MAX), 586 .nr_iovs = 1, 587 .buf_group = sr->buf_group, 588 }; 589 int ret; 590 591 if (kmsg->vec.iovec) { 592 arg.nr_iovs = kmsg->vec.nr; 593 arg.iovs = kmsg->vec.iovec; 594 arg.mode = KBUF_MODE_FREE; 595 } 596 597 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) 598 arg.nr_iovs = 1; 599 else 600 arg.mode |= KBUF_MODE_EXPAND; 601 602 ret = io_buffers_select(req, &arg, sel, issue_flags); 603 if (unlikely(ret < 0)) 604 return ret; 605 606 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) { 607 kmsg->vec.nr = ret; 608 kmsg->vec.iovec = arg.iovs; 609 req->flags |= REQ_F_NEED_CLEANUP; 610 } 611 sr->len = arg.out_len; 612 613 if (ret == 1) { 614 sr->buf = arg.iovs[0].iov_base; 615 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, 616 &kmsg->msg.msg_iter); 617 if (unlikely(ret)) 618 return ret; 619 } else { 620 iov_iter_init(&kmsg->msg.msg_iter, ITER_SOURCE, 621 arg.iovs, ret, arg.out_len); 622 } 623 624 return 0; 625 } 626 627 int io_send(struct io_kiocb *req, unsigned int issue_flags) 628 { 629 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 630 struct io_async_msghdr *kmsg = req->async_data; 631 struct io_br_sel sel = { }; 632 struct socket *sock; 633 unsigned flags; 634 int min_ret = 0; 635 int ret; 636 637 sock = sock_from_file(req->file); 638 if (unlikely(!sock)) 639 return -ENOTSOCK; 640 641 if (!(req->flags & REQ_F_POLLED) && 642 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 643 return -EAGAIN; 644 645 if (req->flags & REQ_F_IMPORT_BUFFER) { 646 ret = io_import_reg_buf(req, &kmsg->msg.msg_iter, 647 (u64)(uintptr_t)sr->buf, sr->len, 648 ITER_SOURCE, issue_flags); 649 if (unlikely(ret)) 650 return ret; 651 req->flags &= ~REQ_F_IMPORT_BUFFER; 652 } 653 654 flags = sr->msg_flags; 655 if (issue_flags & IO_URING_F_NONBLOCK) 656 flags |= MSG_DONTWAIT; 657 658 retry_bundle: 659 sel.buf_list = NULL; 660 if (io_do_buffer_select(req)) { 661 ret = io_send_select_buffer(req, issue_flags, &sel, kmsg); 662 if (ret) 663 return ret; 664 } 665 666 /* 667 * If MSG_WAITALL is set, or this is a bundle send, then we need 668 * the full amount. If just bundle is set, if we do a short send 669 * then we complete the bundle sequence rather than continue on. 670 */ 671 if (flags & MSG_WAITALL || sr->flags & IORING_RECVSEND_BUNDLE) 672 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 673 674 flags &= ~MSG_INTERNAL_SENDMSG_FLAGS; 675 kmsg->msg.msg_flags = flags; 676 ret = sock_sendmsg(sock, &kmsg->msg); 677 if (ret < min_ret) { 678 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 679 return -EAGAIN; 680 681 if (ret > 0 && io_net_retry(sock, flags)) { 682 sr->len -= ret; 683 sr->buf += ret; 684 sr->done_io += ret; 685 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); 686 } 687 if (ret == -ERESTARTSYS) 688 ret = -EINTR; 689 req_set_fail(req); 690 } 691 if (ret >= 0) 692 ret += sr->done_io; 693 else if (sr->done_io) 694 ret = sr->done_io; 695 696 sel.val = ret; 697 if (!io_send_finish(req, kmsg, &sel)) 698 goto retry_bundle; 699 700 io_req_msg_cleanup(req, issue_flags); 701 return sel.val; 702 } 703 704 static int io_recvmsg_mshot_prep(struct io_kiocb *req, 705 struct io_async_msghdr *iomsg, 706 int namelen, size_t controllen) 707 { 708 if ((req->flags & (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) == 709 (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) { 710 int hdr; 711 712 if (unlikely(namelen < 0)) 713 return -EOVERFLOW; 714 if (check_add_overflow(sizeof(struct io_uring_recvmsg_out), 715 namelen, &hdr)) 716 return -EOVERFLOW; 717 if (check_add_overflow(hdr, controllen, &hdr)) 718 return -EOVERFLOW; 719 720 iomsg->namelen = namelen; 721 iomsg->controllen = controllen; 722 return 0; 723 } 724 725 return 0; 726 } 727 728 static int io_recvmsg_copy_hdr(struct io_kiocb *req, 729 struct io_async_msghdr *iomsg) 730 { 731 struct user_msghdr msg; 732 int ret; 733 734 ret = io_msg_copy_hdr(req, iomsg, &msg, &iomsg->uaddr); 735 if (unlikely(ret)) 736 return ret; 737 738 if (!(req->flags & REQ_F_BUFFER_SELECT)) { 739 ret = io_net_import_vec(req, iomsg, msg.msg_iov, msg.msg_iovlen, 740 ITER_DEST); 741 if (unlikely(ret)) 742 return ret; 743 } 744 return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen, 745 msg.msg_controllen); 746 } 747 748 static int io_recvmsg_prep_setup(struct io_kiocb *req) 749 { 750 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 751 struct io_async_msghdr *kmsg; 752 753 kmsg = io_msg_alloc_async(req); 754 if (unlikely(!kmsg)) 755 return -ENOMEM; 756 757 if (req->opcode == IORING_OP_RECV) { 758 kmsg->msg.msg_name = NULL; 759 kmsg->msg.msg_namelen = 0; 760 kmsg->msg.msg_inq = 0; 761 kmsg->msg.msg_control = NULL; 762 kmsg->msg.msg_get_inq = 1; 763 kmsg->msg.msg_controllen = 0; 764 kmsg->msg.msg_ubuf = NULL; 765 766 if (req->flags & REQ_F_BUFFER_SELECT) 767 return 0; 768 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 769 req->flags |= REQ_F_IMPORT_BUFFER; 770 return 0; 771 } 772 return import_ubuf(ITER_DEST, sr->buf, sr->len, 773 &kmsg->msg.msg_iter); 774 } 775 776 return io_recvmsg_copy_hdr(req, kmsg); 777 } 778 779 #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT | \ 780 IORING_RECVSEND_BUNDLE | IORING_RECVSEND_FIXED_BUF) 781 782 int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 783 { 784 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 785 786 sr->done_io = 0; 787 788 if (unlikely(sqe->addr2)) 789 return -EINVAL; 790 791 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 792 sr->len = READ_ONCE(sqe->len); 793 if (unlikely(sr->len < 0)) 794 return -EINVAL; 795 sr->flags = READ_ONCE(sqe->ioprio); 796 if (sr->flags & ~RECVMSG_FLAGS) 797 return -EINVAL; 798 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 799 /* registered buffer recv only for plain IORING_OP_RECV */ 800 if (req->opcode != IORING_OP_RECV || 801 (req->flags & REQ_F_BUFFER_SELECT) || 802 (sr->flags & (IORING_RECV_MULTISHOT | IORING_RECVSEND_BUNDLE))) 803 return -EINVAL; 804 req->buf_index = READ_ONCE(sqe->buf_index); 805 } 806 sr->msg_flags = READ_ONCE(sqe->msg_flags); 807 if (sr->msg_flags & MSG_DONTWAIT) 808 req->flags |= REQ_F_NOWAIT; 809 if (sr->msg_flags & MSG_ERRQUEUE) 810 req->flags |= REQ_F_CLEAR_POLLIN; 811 if (req->flags & REQ_F_BUFFER_SELECT) 812 sr->buf_group = req->buf_index; 813 sr->mshot_total_len = sr->mshot_len = 0; 814 if (sr->flags & IORING_RECV_MULTISHOT) { 815 if (!(req->flags & REQ_F_BUFFER_SELECT)) 816 return -EINVAL; 817 if (sr->msg_flags & MSG_WAITALL) 818 return -EINVAL; 819 if (req->opcode == IORING_OP_RECV) { 820 sr->mshot_len = sr->len; 821 sr->mshot_total_len = READ_ONCE(sqe->optlen); 822 if (sr->mshot_total_len) 823 sr->flags |= IORING_RECV_MSHOT_LIM; 824 } else if (sqe->optlen) { 825 return -EINVAL; 826 } 827 req->flags |= REQ_F_APOLL_MULTISHOT; 828 } else if (sqe->optlen) { 829 return -EINVAL; 830 } 831 832 if (sr->flags & IORING_RECVSEND_BUNDLE) { 833 if (req->opcode == IORING_OP_RECVMSG) 834 return -EINVAL; 835 } 836 837 if (io_is_compat(req->ctx)) 838 sr->msg_flags |= MSG_CMSG_COMPAT; 839 840 sr->nr_multishot_loops = 0; 841 return io_recvmsg_prep_setup(req); 842 } 843 844 /* bits to clear in old and inherit in new cflags on bundle retry */ 845 #define CQE_F_MASK (IORING_CQE_F_SOCK_NONEMPTY|IORING_CQE_F_MORE|\ 846 IORING_CQE_F_BUF_MORE) 847 848 /* 849 * Finishes io_recv and io_recvmsg. 850 * 851 * Returns true if it is actually finished, or false if it should run 852 * again (for multishot). 853 */ 854 static inline bool io_recv_finish(struct io_kiocb *req, 855 struct io_async_msghdr *kmsg, 856 struct io_br_sel *sel, bool mshot_finished, 857 unsigned issue_flags, int consumed) 858 { 859 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 860 unsigned int cflags = 0; 861 862 if (kmsg->msg.msg_inq > 0) 863 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 864 865 if (sel->val > 0 && sr->flags & IORING_RECV_MSHOT_LIM) { 866 /* 867 * If sr->len hits zero, the limit has been reached. Mark 868 * mshot as finished, and flag MSHOT_DONE as well to prevent 869 * a potential bundle from being retried. 870 */ 871 sr->mshot_total_len -= min_t(int, sel->val, sr->mshot_total_len); 872 if (!sr->mshot_total_len) { 873 sr->flags |= IORING_RECV_MSHOT_DONE; 874 mshot_finished = true; 875 } 876 } 877 878 if (sr->flags & IORING_RECVSEND_BUNDLE) { 879 size_t this_ret = sel->val - sr->done_io; 880 881 cflags |= io_put_kbufs(req, consumed, sel->buf_list, io_bundle_nbufs(kmsg, consumed)); 882 if (sr->flags & IORING_RECV_RETRY) 883 cflags = req->cqe.flags | (cflags & CQE_F_MASK); 884 if (sr->mshot_len && sel->val >= sr->mshot_len) 885 sr->flags |= IORING_RECV_MSHOT_CAP; 886 /* bundle with no more immediate buffers, we're done */ 887 if (req->flags & REQ_F_BL_EMPTY) 888 goto finish; 889 /* 890 * If more is available AND it was a full transfer, retry and 891 * append to this one 892 */ 893 if (!(sr->flags & IORING_RECV_NO_RETRY) && 894 kmsg->msg.msg_inq > 1 && this_ret > 0 && 895 !iov_iter_count(&kmsg->msg.msg_iter)) { 896 req->cqe.flags = cflags & ~CQE_F_MASK; 897 sr->len = kmsg->msg.msg_inq; 898 sr->done_io += this_ret; 899 sr->flags |= IORING_RECV_RETRY; 900 return false; 901 } 902 } else { 903 cflags |= io_put_kbuf(req, consumed, sel->buf_list); 904 } 905 906 /* 907 * Fill CQE for this receive and see if we should keep trying to 908 * receive from this socket. 909 */ 910 if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished && 911 io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) { 912 sel->val = IOU_RETRY; 913 io_mshot_prep_retry(req, kmsg); 914 /* Known not-empty or unknown state, retry */ 915 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) { 916 if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY && 917 !(sr->flags & IORING_RECV_MSHOT_CAP)) { 918 return false; 919 } 920 /* mshot retries exceeded, force a requeue */ 921 sr->nr_multishot_loops = 0; 922 sr->flags &= ~IORING_RECV_MSHOT_CAP; 923 if (issue_flags & IO_URING_F_MULTISHOT) 924 sel->val = IOU_REQUEUE; 925 } 926 return true; 927 } 928 929 /* Finish the request / stop multishot. */ 930 finish: 931 io_req_set_res(req, sel->val, cflags); 932 sel->val = IOU_COMPLETE; 933 io_req_msg_cleanup(req, issue_flags); 934 return true; 935 } 936 937 static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg, 938 struct io_sr_msg *sr, void __user **buf, 939 size_t *len) 940 { 941 unsigned long ubuf = (unsigned long) *buf; 942 unsigned long hdr; 943 944 hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 945 kmsg->controllen; 946 if (*len < hdr) 947 return -EFAULT; 948 949 if (kmsg->controllen) { 950 unsigned long control = ubuf + hdr - kmsg->controllen; 951 952 kmsg->msg.msg_control_user = (void __user *) control; 953 kmsg->msg.msg_controllen = kmsg->controllen; 954 } 955 956 sr->buf = *buf; /* stash for later copy */ 957 *buf = (void __user *) (ubuf + hdr); 958 kmsg->payloadlen = *len = *len - hdr; 959 return 0; 960 } 961 962 struct io_recvmsg_multishot_hdr { 963 struct io_uring_recvmsg_out msg; 964 struct sockaddr_storage addr; 965 }; 966 967 static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io, 968 struct io_async_msghdr *kmsg, 969 unsigned int flags, bool *finished) 970 { 971 int err; 972 int copy_len; 973 struct io_recvmsg_multishot_hdr hdr; 974 975 if (kmsg->namelen) 976 kmsg->msg.msg_name = &hdr.addr; 977 kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT); 978 kmsg->msg.msg_namelen = 0; 979 980 if (sock->file->f_flags & O_NONBLOCK) 981 flags |= MSG_DONTWAIT; 982 983 err = sock_recvmsg(sock, &kmsg->msg, flags); 984 *finished = err <= 0; 985 if (err < 0) 986 return err; 987 988 hdr.msg = (struct io_uring_recvmsg_out) { 989 .controllen = kmsg->controllen - kmsg->msg.msg_controllen, 990 .flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT 991 }; 992 993 hdr.msg.payloadlen = err; 994 if (err > kmsg->payloadlen) 995 err = kmsg->payloadlen; 996 997 copy_len = sizeof(struct io_uring_recvmsg_out); 998 if (kmsg->msg.msg_namelen > kmsg->namelen) 999 copy_len += kmsg->namelen; 1000 else 1001 copy_len += kmsg->msg.msg_namelen; 1002 1003 /* 1004 * "fromlen shall refer to the value before truncation.." 1005 * 1003.1g 1006 */ 1007 hdr.msg.namelen = kmsg->msg.msg_namelen; 1008 1009 /* ensure that there is no gap between hdr and sockaddr_storage */ 1010 BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr, addr) != 1011 sizeof(struct io_uring_recvmsg_out)); 1012 if (copy_to_user(io->buf, &hdr, copy_len)) { 1013 *finished = true; 1014 return -EFAULT; 1015 } 1016 1017 return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 1018 kmsg->controllen + err; 1019 } 1020 1021 int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 1022 { 1023 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1024 struct io_async_msghdr *kmsg = req->async_data; 1025 struct io_br_sel sel = { }; 1026 struct socket *sock; 1027 unsigned flags; 1028 int ret, min_ret = 0; 1029 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1030 bool mshot_finished = true; 1031 int consumed = 0; 1032 size_t len; 1033 1034 sock = sock_from_file(req->file); 1035 if (unlikely(!sock)) 1036 return -ENOTSOCK; 1037 1038 if (!(req->flags & REQ_F_POLLED) && 1039 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1040 return -EAGAIN; 1041 1042 flags = sr->msg_flags; 1043 if (force_nonblock) 1044 flags |= MSG_DONTWAIT; 1045 1046 retry_multishot: 1047 sel.buf_list = NULL; 1048 len = sr->len; 1049 if (io_do_buffer_select(req)) { 1050 sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); 1051 if (!sel.addr) 1052 return -ENOBUFS; 1053 1054 if (req->flags & REQ_F_APOLL_MULTISHOT) { 1055 ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len); 1056 if (ret) { 1057 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1058 return ret; 1059 } 1060 } 1061 1062 iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len); 1063 } 1064 1065 kmsg->msg.msg_get_inq = 1; 1066 kmsg->msg.msg_inq = -1; 1067 if (req->flags & REQ_F_APOLL_MULTISHOT) { 1068 ret = io_recvmsg_multishot(sock, sr, kmsg, flags, 1069 &mshot_finished); 1070 consumed = ret; 1071 } else { 1072 /* disable partial retry for recvmsg with cmsg attached */ 1073 if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen) 1074 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1075 1076 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, 1077 kmsg->uaddr, flags); 1078 /* 1079 * With MSG_TRUNC, the net layer will return the full size of 1080 * the packet, even if we only filled part of it in the buffers. 1081 * Adjust the returned size to consume only the real part of the 1082 * buffer. 1083 */ 1084 consumed = ret; 1085 if (ret > 0) 1086 consumed = min_t(size_t, ret, len); 1087 } 1088 1089 if (ret < min_ret) { 1090 if (ret == -EAGAIN && force_nonblock) { 1091 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1092 return IOU_RETRY; 1093 } 1094 if (ret > 0 && io_net_retry(sock, flags)) { 1095 sr->done_io += ret; 1096 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); 1097 } 1098 if (ret == -ERESTARTSYS) 1099 ret = -EINTR; 1100 req_set_fail(req); 1101 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 1102 req_set_fail(req); 1103 } 1104 1105 if (ret > 0) 1106 ret += sr->done_io; 1107 else if (sr->done_io) 1108 ret = sr->done_io; 1109 else 1110 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1111 1112 sel.val = ret; 1113 if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed)) 1114 goto retry_multishot; 1115 1116 return sel.val; 1117 } 1118 1119 static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg, 1120 struct io_br_sel *sel, unsigned int issue_flags) 1121 { 1122 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1123 size_t len; 1124 int ret; 1125 1126 /* 1127 * If the ring isn't locked, then don't use the peek interface 1128 * to grab multiple buffers as we will lock/unlock between 1129 * this selection and posting the buffers. 1130 */ 1131 if (!(issue_flags & IO_URING_F_UNLOCKED) && 1132 sr->flags & IORING_RECVSEND_BUNDLE) { 1133 struct buf_sel_arg arg = { 1134 .iovs = &kmsg->fast_iov, 1135 .nr_iovs = 1, 1136 .mode = KBUF_MODE_EXPAND, 1137 .buf_group = sr->buf_group, 1138 }; 1139 1140 if (kmsg->vec.iovec) { 1141 arg.nr_iovs = kmsg->vec.nr; 1142 arg.iovs = kmsg->vec.iovec; 1143 arg.mode |= KBUF_MODE_FREE; 1144 } 1145 1146 if (sel->val) 1147 arg.max_len = sel->val; 1148 else if (kmsg->msg.msg_inq > 1) 1149 arg.max_len = min_not_zero(sel->val, (ssize_t) kmsg->msg.msg_inq); 1150 1151 /* if mshot limited, ensure we don't go over */ 1152 if (sr->flags & IORING_RECV_MSHOT_LIM) 1153 arg.max_len = min_not_zero(arg.max_len, sr->mshot_total_len); 1154 ret = io_buffers_peek(req, &arg, sel); 1155 if (unlikely(ret < 0)) 1156 return ret; 1157 1158 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) { 1159 kmsg->vec.nr = ret; 1160 kmsg->vec.iovec = arg.iovs; 1161 req->flags |= REQ_F_NEED_CLEANUP; 1162 } 1163 if (arg.partial_map) 1164 sr->flags |= IORING_RECV_PARTIAL_MAP; 1165 1166 /* special case 1 vec, can be a fast path */ 1167 if (ret == 1) { 1168 sr->buf = arg.iovs[0].iov_base; 1169 len = sr->len = arg.iovs[0].iov_len; 1170 goto map_ubuf; 1171 } 1172 iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret, 1173 arg.out_len); 1174 len = arg.out_len; 1175 } else { 1176 len = sel->val; 1177 1178 *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); 1179 if (!sel->addr) 1180 return -ENOBUFS; 1181 sr->buf = sel->addr; 1182 sr->len = len; 1183 map_ubuf: 1184 ret = import_ubuf(ITER_DEST, sr->buf, sr->len, 1185 &kmsg->msg.msg_iter); 1186 if (unlikely(ret)) 1187 return ret; 1188 } 1189 1190 return len; 1191 } 1192 1193 int io_recv(struct io_kiocb *req, unsigned int issue_flags) 1194 { 1195 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1196 struct io_async_msghdr *kmsg = req->async_data; 1197 struct io_br_sel sel; 1198 struct socket *sock; 1199 unsigned flags; 1200 int ret, min_ret = 0, consumed = 0; 1201 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1202 bool mshot_finished; 1203 size_t len = 0; 1204 1205 sock = sock_from_file(req->file); 1206 if (unlikely(!sock)) 1207 return -ENOTSOCK; 1208 1209 if (!(req->flags & REQ_F_POLLED) && 1210 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1211 return -EAGAIN; 1212 1213 flags = sr->msg_flags; 1214 if (force_nonblock) 1215 flags |= MSG_DONTWAIT; 1216 1217 if (req->flags & REQ_F_IMPORT_BUFFER) { 1218 ret = io_import_reg_buf(req, &kmsg->msg.msg_iter, 1219 (u64)(uintptr_t)sr->buf, sr->len, 1220 ITER_DEST, issue_flags); 1221 if (unlikely(ret)) { 1222 kmsg->msg.msg_inq = -1; 1223 sel.buf_list = NULL; 1224 goto out_free; 1225 } 1226 req->flags &= ~REQ_F_IMPORT_BUFFER; 1227 } 1228 1229 retry_multishot: 1230 sel.buf_list = NULL; 1231 len = sr->len; 1232 if (io_do_buffer_select(req)) { 1233 sel.val = sr->len; 1234 ret = io_recv_buf_select(req, kmsg, &sel, issue_flags); 1235 if (unlikely(ret < 0)) { 1236 kmsg->msg.msg_inq = -1; 1237 goto out_free; 1238 } 1239 len = ret; 1240 sr->buf = NULL; 1241 } 1242 1243 kmsg->msg.msg_flags = 0; 1244 kmsg->msg.msg_inq = -1; 1245 1246 if (flags & MSG_WAITALL) 1247 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1248 1249 ret = sock_recvmsg(sock, &kmsg->msg, flags); 1250 if (ret < min_ret) { 1251 if (ret == -EAGAIN && force_nonblock) { 1252 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1253 return IOU_RETRY; 1254 } 1255 if (ret > 0 && io_net_retry(sock, flags)) { 1256 sr->len -= ret; 1257 sr->buf += ret; 1258 sr->done_io += ret; 1259 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); 1260 } 1261 if (ret == -ERESTARTSYS) 1262 ret = -EINTR; 1263 req_set_fail(req); 1264 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 1265 out_free: 1266 req_set_fail(req); 1267 } 1268 1269 mshot_finished = ret <= 0; 1270 1271 /* 1272 * With MSG_TRUNC, the net layer will return the full size of 1273 * the packet, even if we only filled part of it in the buffers. 1274 * Adjust the returned size to consume only the real part of the 1275 * buffer. 1276 */ 1277 consumed = ret; 1278 if (ret > 0) 1279 consumed = min_t(size_t, ret, len); 1280 1281 if (ret > 0) 1282 ret += sr->done_io; 1283 else if (sr->done_io) 1284 ret = sr->done_io; 1285 else 1286 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1287 1288 sel.val = ret; 1289 if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed)) 1290 goto retry_multishot; 1291 1292 return sel.val; 1293 } 1294 1295 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1296 { 1297 struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc); 1298 unsigned ifq_idx; 1299 1300 if (unlikely(sqe->addr2 || sqe->addr || sqe->addr3)) 1301 return -EINVAL; 1302 1303 ifq_idx = READ_ONCE(sqe->zcrx_ifq_idx); 1304 zc->ifq = xa_load(&req->ctx->zcrx_ctxs, ifq_idx); 1305 if (!zc->ifq) 1306 return -EINVAL; 1307 1308 zc->len = READ_ONCE(sqe->len); 1309 zc->flags = READ_ONCE(sqe->ioprio); 1310 if (READ_ONCE(sqe->msg_flags)) 1311 return -EINVAL; 1312 if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT)) 1313 return -EINVAL; 1314 /* multishot required */ 1315 if (!(zc->flags & IORING_RECV_MULTISHOT)) 1316 return -EINVAL; 1317 /* All data completions are posted as aux CQEs. */ 1318 req->flags |= REQ_F_APOLL_MULTISHOT; 1319 1320 return 0; 1321 } 1322 1323 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags) 1324 { 1325 struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc); 1326 struct socket *sock; 1327 unsigned int len; 1328 int ret; 1329 1330 sock = sock_from_file(req->file); 1331 if (unlikely(!sock)) 1332 return -ENOTSOCK; 1333 1334 if (!(req->flags & REQ_F_POLLED) && 1335 (zc->flags & IORING_RECVSEND_POLL_FIRST)) 1336 return -EAGAIN; 1337 1338 len = zc->len; 1339 ret = io_zcrx_recv(req, zc->ifq, sock, 0, issue_flags, &zc->len); 1340 if (len && zc->len == 0) { 1341 io_req_set_res(req, 0, 0); 1342 1343 return IOU_COMPLETE; 1344 } 1345 if (unlikely(ret <= 0) && ret != -EAGAIN) { 1346 if (ret == -ERESTARTSYS) 1347 ret = -EINTR; 1348 if (ret == IOU_REQUEUE) 1349 return IOU_REQUEUE; 1350 1351 req_set_fail(req); 1352 io_req_set_res(req, ret, 0); 1353 return IOU_COMPLETE; 1354 } 1355 return IOU_RETRY; 1356 } 1357 1358 void io_send_zc_cleanup(struct io_kiocb *req) 1359 { 1360 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 1361 struct io_async_msghdr *io = req->async_data; 1362 1363 if (req_has_async_data(req)) 1364 io_netmsg_iovec_free(io); 1365 if (zc->notif) { 1366 io_notif_flush(zc->notif); 1367 zc->notif = NULL; 1368 } 1369 } 1370 1371 #define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF) 1372 #define IO_ZC_FLAGS_VALID (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE | \ 1373 IORING_SEND_VECTORIZED) 1374 1375 int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1376 { 1377 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 1378 struct io_ring_ctx *ctx = req->ctx; 1379 struct io_async_msghdr *iomsg; 1380 struct io_kiocb *notif; 1381 u64 user_data; 1382 int ret; 1383 1384 zc->done_io = 0; 1385 1386 if (unlikely(READ_ONCE(sqe->__pad2[0]))) 1387 return -EINVAL; 1388 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */ 1389 if (req->flags & REQ_F_CQE_SKIP) 1390 return -EINVAL; 1391 1392 notif = zc->notif = io_alloc_notif(ctx); 1393 if (!notif) 1394 return -ENOMEM; 1395 user_data = READ_ONCE(sqe->addr3); 1396 if (!user_data) 1397 user_data = req->cqe.user_data; 1398 1399 notif->cqe.user_data = user_data; 1400 notif->cqe.res = 0; 1401 notif->cqe.flags = IORING_CQE_F_NOTIF; 1402 req->flags |= REQ_F_NEED_CLEANUP | REQ_F_POLL_NO_LAZY; 1403 1404 zc->flags = READ_ONCE(sqe->ioprio); 1405 if (unlikely(zc->flags & ~IO_ZC_FLAGS_COMMON)) { 1406 if (zc->flags & ~IO_ZC_FLAGS_VALID) 1407 return -EINVAL; 1408 if (zc->flags & IORING_SEND_ZC_REPORT_USAGE) { 1409 struct io_notif_data *nd = io_notif_to_data(notif); 1410 1411 nd->zc_report = true; 1412 nd->zc_used = false; 1413 nd->zc_copied = false; 1414 } 1415 } 1416 1417 zc->len = READ_ONCE(sqe->len); 1418 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL | MSG_ZEROCOPY; 1419 req->buf_index = READ_ONCE(sqe->buf_index); 1420 if (zc->msg_flags & MSG_DONTWAIT) 1421 req->flags |= REQ_F_NOWAIT; 1422 1423 if (io_is_compat(ctx)) 1424 zc->msg_flags |= MSG_CMSG_COMPAT; 1425 1426 iomsg = io_msg_alloc_async(req); 1427 if (unlikely(!iomsg)) 1428 return -ENOMEM; 1429 1430 if (req->opcode == IORING_OP_SEND_ZC) { 1431 ret = io_send_setup(req, sqe); 1432 } else { 1433 if (unlikely(sqe->addr2 || sqe->file_index)) 1434 return -EINVAL; 1435 ret = io_sendmsg_setup(req, sqe); 1436 } 1437 if (unlikely(ret)) 1438 return ret; 1439 1440 if (!(zc->flags & IORING_RECVSEND_FIXED_BUF)) { 1441 iomsg->msg.sg_from_iter = io_sg_from_iter_iovec; 1442 return io_notif_account_mem(zc->notif, iomsg->msg.msg_iter.count); 1443 } 1444 iomsg->msg.sg_from_iter = io_sg_from_iter; 1445 return 0; 1446 } 1447 1448 static int io_sg_from_iter_iovec(struct sk_buff *skb, 1449 struct iov_iter *from, size_t length) 1450 { 1451 skb_zcopy_downgrade_managed(skb); 1452 return zerocopy_fill_skb_from_iter(skb, from, length); 1453 } 1454 1455 static int io_sg_from_iter(struct sk_buff *skb, 1456 struct iov_iter *from, size_t length) 1457 { 1458 struct skb_shared_info *shinfo = skb_shinfo(skb); 1459 int frag = shinfo->nr_frags; 1460 int ret = 0; 1461 struct bvec_iter bi; 1462 ssize_t copied = 0; 1463 unsigned long truesize = 0; 1464 1465 if (!frag) 1466 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS; 1467 else if (unlikely(!skb_zcopy_managed(skb))) 1468 return zerocopy_fill_skb_from_iter(skb, from, length); 1469 1470 bi.bi_size = min(from->count, length); 1471 bi.bi_offset = from->iov_offset; 1472 bi.bi_idx = 0; 1473 1474 while (bi.bi_size && frag < MAX_SKB_FRAGS) { 1475 struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi); 1476 1477 copied += v.bv_len; 1478 truesize += PAGE_ALIGN(v.bv_len + v.bv_offset); 1479 __skb_fill_page_desc_noacc(shinfo, frag++, v.bv_page, 1480 v.bv_offset, v.bv_len); 1481 bvec_iter_advance_single(from->bvec, &bi, v.bv_len); 1482 } 1483 if (bi.bi_size) 1484 ret = -EMSGSIZE; 1485 1486 shinfo->nr_frags = frag; 1487 from->bvec += bi.bi_idx; 1488 from->nr_segs -= bi.bi_idx; 1489 from->count -= copied; 1490 from->iov_offset = bi.bi_offset; 1491 1492 skb->data_len += copied; 1493 skb->len += copied; 1494 skb->truesize += truesize; 1495 return ret; 1496 } 1497 1498 static int io_send_zc_import(struct io_kiocb *req, 1499 struct io_async_msghdr *kmsg, 1500 unsigned int issue_flags) 1501 { 1502 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1503 struct io_kiocb *notif = sr->notif; 1504 int ret; 1505 1506 WARN_ON_ONCE(!(sr->flags & IORING_RECVSEND_FIXED_BUF)); 1507 1508 notif->buf_index = req->buf_index; 1509 1510 if (!(sr->flags & IORING_SEND_VECTORIZED)) { 1511 ret = io_import_reg_buf(notif, &kmsg->msg.msg_iter, 1512 (u64)(uintptr_t)sr->buf, sr->len, 1513 ITER_SOURCE, issue_flags); 1514 } else { 1515 unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs; 1516 1517 ret = io_import_reg_vec(ITER_SOURCE, &kmsg->msg.msg_iter, 1518 notif, &kmsg->vec, uvec_segs, 1519 issue_flags); 1520 } 1521 1522 if (unlikely(ret)) 1523 return ret; 1524 req->flags &= ~REQ_F_IMPORT_BUFFER; 1525 return 0; 1526 } 1527 1528 int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) 1529 { 1530 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1531 struct io_async_msghdr *kmsg = req->async_data; 1532 struct socket *sock; 1533 unsigned msg_flags; 1534 int ret, min_ret = 0; 1535 1536 sock = sock_from_file(req->file); 1537 if (unlikely(!sock)) 1538 return -ENOTSOCK; 1539 if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags)) 1540 return -EOPNOTSUPP; 1541 if (!(req->flags & REQ_F_POLLED) && 1542 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1543 return -EAGAIN; 1544 1545 if (req->flags & REQ_F_IMPORT_BUFFER) { 1546 ret = io_send_zc_import(req, kmsg, issue_flags); 1547 if (unlikely(ret)) 1548 return ret; 1549 } 1550 1551 msg_flags = sr->msg_flags; 1552 if (issue_flags & IO_URING_F_NONBLOCK) 1553 msg_flags |= MSG_DONTWAIT; 1554 if (msg_flags & MSG_WAITALL) 1555 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1556 1557 kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg; 1558 1559 if (req->opcode == IORING_OP_SEND_ZC) { 1560 msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS; 1561 kmsg->msg.msg_flags = msg_flags; 1562 ret = sock_sendmsg(sock, &kmsg->msg); 1563 } else { 1564 kmsg->msg.msg_control_user = sr->msg_control; 1565 ret = __sys_sendmsg_sock(sock, &kmsg->msg, msg_flags); 1566 } 1567 1568 if (unlikely(ret < min_ret)) { 1569 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1570 return -EAGAIN; 1571 1572 if (ret > 0 && io_net_retry(sock, sr->msg_flags)) { 1573 sr->done_io += ret; 1574 return -EAGAIN; 1575 } 1576 if (ret == -ERESTARTSYS) 1577 ret = -EINTR; 1578 req_set_fail(req); 1579 } 1580 1581 if (ret >= 0) 1582 ret += sr->done_io; 1583 else if (sr->done_io) 1584 ret = sr->done_io; 1585 1586 /* 1587 * If we're in io-wq we can't rely on tw ordering guarantees, defer 1588 * flushing notif to io_send_zc_cleanup() 1589 */ 1590 if (!(issue_flags & IO_URING_F_UNLOCKED)) { 1591 io_notif_flush(sr->notif); 1592 sr->notif = NULL; 1593 io_req_msg_cleanup(req, 0); 1594 } 1595 io_req_set_res(req, ret, IORING_CQE_F_MORE); 1596 return IOU_COMPLETE; 1597 } 1598 1599 void io_sendrecv_fail(struct io_kiocb *req) 1600 { 1601 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1602 1603 if (sr->done_io) 1604 req->cqe.res = sr->done_io; 1605 1606 if ((req->flags & REQ_F_NEED_CLEANUP) && 1607 (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC)) 1608 req->cqe.flags |= IORING_CQE_F_MORE; 1609 } 1610 1611 #define ACCEPT_FLAGS (IORING_ACCEPT_MULTISHOT | IORING_ACCEPT_DONTWAIT | \ 1612 IORING_ACCEPT_POLL_FIRST) 1613 1614 int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1615 { 1616 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1617 1618 if (sqe->len || sqe->buf_index) 1619 return -EINVAL; 1620 1621 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1622 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 1623 accept->flags = READ_ONCE(sqe->accept_flags); 1624 accept->nofile = rlimit(RLIMIT_NOFILE); 1625 accept->iou_flags = READ_ONCE(sqe->ioprio); 1626 if (accept->iou_flags & ~ACCEPT_FLAGS) 1627 return -EINVAL; 1628 1629 accept->file_slot = READ_ONCE(sqe->file_index); 1630 if (accept->file_slot) { 1631 if (accept->flags & SOCK_CLOEXEC) 1632 return -EINVAL; 1633 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT && 1634 accept->file_slot != IORING_FILE_INDEX_ALLOC) 1635 return -EINVAL; 1636 } 1637 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1638 return -EINVAL; 1639 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK)) 1640 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK; 1641 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT) 1642 req->flags |= REQ_F_APOLL_MULTISHOT; 1643 if (accept->iou_flags & IORING_ACCEPT_DONTWAIT) 1644 req->flags |= REQ_F_NOWAIT; 1645 return 0; 1646 } 1647 1648 int io_accept(struct io_kiocb *req, unsigned int issue_flags) 1649 { 1650 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1651 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1652 bool fixed = !!accept->file_slot; 1653 struct proto_accept_arg arg = { 1654 .flags = force_nonblock ? O_NONBLOCK : 0, 1655 }; 1656 struct file *file; 1657 unsigned cflags; 1658 int ret, fd; 1659 1660 if (!(req->flags & REQ_F_POLLED) && 1661 accept->iou_flags & IORING_ACCEPT_POLL_FIRST) 1662 return -EAGAIN; 1663 1664 retry: 1665 if (!fixed) { 1666 fd = __get_unused_fd_flags(accept->flags, accept->nofile); 1667 if (unlikely(fd < 0)) 1668 return fd; 1669 } 1670 arg.err = 0; 1671 arg.is_empty = -1; 1672 file = do_accept(req->file, &arg, accept->addr, accept->addr_len, 1673 accept->flags); 1674 if (IS_ERR(file)) { 1675 if (!fixed) 1676 put_unused_fd(fd); 1677 ret = PTR_ERR(file); 1678 if (ret == -EAGAIN && force_nonblock && 1679 !(accept->iou_flags & IORING_ACCEPT_DONTWAIT)) 1680 return IOU_RETRY; 1681 1682 if (ret == -ERESTARTSYS) 1683 ret = -EINTR; 1684 } else if (!fixed) { 1685 fd_install(fd, file); 1686 ret = fd; 1687 } else { 1688 ret = io_fixed_fd_install(req, issue_flags, file, 1689 accept->file_slot); 1690 } 1691 1692 cflags = 0; 1693 if (!arg.is_empty) 1694 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 1695 1696 if (ret >= 0 && (req->flags & REQ_F_APOLL_MULTISHOT) && 1697 io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) { 1698 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || arg.is_empty == -1) 1699 goto retry; 1700 return IOU_RETRY; 1701 } 1702 1703 io_req_set_res(req, ret, cflags); 1704 if (ret < 0) 1705 req_set_fail(req); 1706 return IOU_COMPLETE; 1707 } 1708 1709 void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req) 1710 { 1711 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1712 1713 bctx->socket.family = sock->domain; 1714 bctx->socket.type = sock->type; 1715 bctx->socket.protocol = sock->protocol; 1716 } 1717 1718 void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req) 1719 { 1720 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1721 struct sockaddr_storage *ss = req->async_data; 1722 1723 /* 1724 * move_addr_to_kernel() skips the copy for addr_len == 0, so 1725 * iomsg->addr may hold stale data from a prior CONNECT. Bail 1726 * unless addr_len covers the family discriminator. 1727 */ 1728 if (conn->addr_len < (int)sizeof(sa_family_t)) 1729 return; 1730 1731 bctx->connect.family = ss->ss_family; 1732 switch (ss->ss_family) { 1733 case AF_INET: { 1734 struct sockaddr_in *sin = (struct sockaddr_in *)ss; 1735 1736 if (conn->addr_len < (int)sizeof(*sin)) 1737 break; 1738 bctx->connect.port = sin->sin_port; 1739 bctx->connect.v4_addr = sin->sin_addr.s_addr; 1740 break; 1741 } 1742 case AF_INET6: { 1743 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss; 1744 1745 /* The IPv6 stack accepts SIN6_LEN_RFC2133; port and addr are in it. */ 1746 if (conn->addr_len < SIN6_LEN_RFC2133) 1747 break; 1748 bctx->connect.port = sin6->sin6_port; 1749 memcpy(bctx->connect.v6_addr, &sin6->sin6_addr, 1750 sizeof(bctx->connect.v6_addr)); 1751 break; 1752 } 1753 default: 1754 /* family is set; per-family fields stay zero - family-only filtering */ 1755 break; 1756 } 1757 } 1758 1759 int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1760 { 1761 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1762 1763 if (sqe->addr || sqe->rw_flags || sqe->buf_index) 1764 return -EINVAL; 1765 1766 sock->domain = READ_ONCE(sqe->fd); 1767 sock->type = READ_ONCE(sqe->off); 1768 sock->protocol = READ_ONCE(sqe->len); 1769 sock->file_slot = READ_ONCE(sqe->file_index); 1770 sock->nofile = rlimit(RLIMIT_NOFILE); 1771 1772 sock->flags = sock->type & ~SOCK_TYPE_MASK; 1773 if (sock->file_slot && (sock->flags & SOCK_CLOEXEC)) 1774 return -EINVAL; 1775 if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1776 return -EINVAL; 1777 return 0; 1778 } 1779 1780 int io_socket(struct io_kiocb *req, unsigned int issue_flags) 1781 { 1782 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1783 bool fixed = !!sock->file_slot; 1784 struct file *file; 1785 int ret, fd; 1786 1787 if (!fixed) { 1788 fd = __get_unused_fd_flags(sock->flags, sock->nofile); 1789 if (unlikely(fd < 0)) 1790 return fd; 1791 } 1792 file = __sys_socket_file(sock->domain, sock->type, sock->protocol); 1793 if (IS_ERR(file)) { 1794 if (!fixed) 1795 put_unused_fd(fd); 1796 ret = PTR_ERR(file); 1797 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1798 return -EAGAIN; 1799 if (ret == -ERESTARTSYS) 1800 ret = -EINTR; 1801 req_set_fail(req); 1802 } else if (!fixed) { 1803 fd_install(fd, file); 1804 ret = fd; 1805 } else { 1806 ret = io_fixed_fd_install(req, issue_flags, file, 1807 sock->file_slot); 1808 } 1809 io_req_set_res(req, ret, 0); 1810 return IOU_COMPLETE; 1811 } 1812 1813 int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1814 { 1815 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1816 struct sockaddr_storage *addr; 1817 1818 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 1819 return -EINVAL; 1820 1821 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1822 conn->addr_len = READ_ONCE(sqe->addr2); 1823 conn->in_progress = conn->seen_econnaborted = false; 1824 1825 addr = io_uring_alloc_async_data(NULL, req); 1826 if (unlikely(!addr)) 1827 return -ENOMEM; 1828 1829 return move_addr_to_kernel(conn->addr, conn->addr_len, addr); 1830 } 1831 1832 int io_connect(struct io_kiocb *req, unsigned int issue_flags) 1833 { 1834 struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect); 1835 struct sockaddr_storage *addr = req->async_data; 1836 unsigned file_flags; 1837 int ret; 1838 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1839 1840 if (connect->in_progress) { 1841 struct poll_table_struct pt = { ._key = EPOLLERR }; 1842 1843 if (vfs_poll(req->file, &pt) & EPOLLERR) 1844 goto get_sock_err; 1845 } 1846 1847 file_flags = force_nonblock ? O_NONBLOCK : 0; 1848 1849 ret = __sys_connect_file(req->file, addr, connect->addr_len, file_flags); 1850 if ((ret == -EAGAIN || ret == -EINPROGRESS || ret == -ECONNABORTED) 1851 && force_nonblock) { 1852 if (ret == -EINPROGRESS) { 1853 connect->in_progress = true; 1854 } else if (ret == -ECONNABORTED) { 1855 if (connect->seen_econnaborted) 1856 goto out; 1857 connect->seen_econnaborted = true; 1858 } 1859 return -EAGAIN; 1860 } 1861 if (connect->in_progress) { 1862 /* 1863 * At least bluetooth will return -EBADFD on a re-connect 1864 * attempt, and it's (supposedly) also valid to get -EISCONN 1865 * which means the previous result is good. For both of these, 1866 * grab the sock_error() and use that for the completion. 1867 */ 1868 if (ret == -EBADFD || ret == -EISCONN) { 1869 get_sock_err: 1870 ret = sock_error(sock_from_file(req->file)->sk); 1871 } 1872 } 1873 if (ret == -ERESTARTSYS) 1874 ret = -EINTR; 1875 out: 1876 if (ret < 0) 1877 req_set_fail(req); 1878 io_req_set_res(req, ret, 0); 1879 return IOU_COMPLETE; 1880 } 1881 1882 /* 1883 * Check if bind request would potentially end up with filename_create(), 1884 * which in turn end up in mnt_want_write() which will grab the fs 1885 * percpu start write sem. This can trigger a lockdep warning. 1886 */ 1887 static int io_bind_file_create(const struct sockaddr_storage *addr, int addr_len) 1888 { 1889 const struct sockaddr_un *sun; 1890 1891 if (addr->ss_family != AF_UNIX) 1892 return 0; 1893 if (addr_len <= offsetof(struct sockaddr_un, sun_path)) 1894 return 0; 1895 sun = (const struct sockaddr_un *) addr; 1896 return sun->sun_path[0] != '\0'; 1897 } 1898 1899 int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1900 { 1901 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind); 1902 struct sockaddr __user *uaddr; 1903 struct sockaddr_storage *addr; 1904 int ret; 1905 1906 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 1907 return -EINVAL; 1908 1909 uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1910 bind->addr_len = READ_ONCE(sqe->addr2); 1911 1912 addr = io_uring_alloc_async_data(NULL, req); 1913 if (unlikely(!addr)) 1914 return -ENOMEM; 1915 ret = move_addr_to_kernel(uaddr, bind->addr_len, addr); 1916 if (unlikely(ret)) 1917 return ret; 1918 if (io_bind_file_create(addr, bind->addr_len)) 1919 req->flags |= REQ_F_FORCE_ASYNC; 1920 return 0; 1921 } 1922 1923 1924 int io_bind(struct io_kiocb *req, unsigned int issue_flags) 1925 { 1926 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind); 1927 struct sockaddr_storage *addr = req->async_data; 1928 struct socket *sock; 1929 int ret; 1930 1931 sock = sock_from_file(req->file); 1932 if (unlikely(!sock)) 1933 return -ENOTSOCK; 1934 1935 ret = __sys_bind_socket(sock, addr, bind->addr_len); 1936 if (ret < 0) 1937 req_set_fail(req); 1938 io_req_set_res(req, ret, 0); 1939 return 0; 1940 } 1941 1942 int io_listen_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1943 { 1944 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen); 1945 1946 if (sqe->addr || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in || sqe->addr2) 1947 return -EINVAL; 1948 1949 listen->backlog = READ_ONCE(sqe->len); 1950 return 0; 1951 } 1952 1953 int io_listen(struct io_kiocb *req, unsigned int issue_flags) 1954 { 1955 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen); 1956 struct socket *sock; 1957 int ret; 1958 1959 sock = sock_from_file(req->file); 1960 if (unlikely(!sock)) 1961 return -ENOTSOCK; 1962 1963 ret = __sys_listen_socket(sock, listen->backlog); 1964 if (ret < 0) 1965 req_set_fail(req); 1966 io_req_set_res(req, ret, 0); 1967 return 0; 1968 } 1969 1970 void io_netmsg_cache_free(const void *entry) 1971 { 1972 struct io_async_msghdr *kmsg = (struct io_async_msghdr *) entry; 1973 1974 io_vec_free(&kmsg->vec); 1975 kfree(kmsg); 1976 } 1977