1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/file.h> 5 #include <linux/slab.h> 6 #include <linux/net.h> 7 #include <linux/un.h> 8 #include <linux/compat.h> 9 #include <net/compat.h> 10 #include <linux/io_uring.h> 11 12 #include <uapi/linux/io_uring.h> 13 14 #include "filetable.h" 15 #include "io_uring.h" 16 #include "kbuf.h" 17 #include "alloc_cache.h" 18 #include "net.h" 19 #include "notif.h" 20 #include "rsrc.h" 21 #include "zcrx.h" 22 23 struct io_shutdown { 24 struct file *file; 25 int how; 26 }; 27 28 struct io_accept { 29 struct file *file; 30 struct sockaddr __user *addr; 31 int __user *addr_len; 32 int flags; 33 int iou_flags; 34 u32 file_slot; 35 unsigned long nofile; 36 }; 37 38 struct io_socket { 39 struct file *file; 40 int domain; 41 int type; 42 int protocol; 43 int flags; 44 u32 file_slot; 45 unsigned long nofile; 46 }; 47 48 struct io_connect { 49 struct file *file; 50 struct sockaddr __user *addr; 51 int addr_len; 52 bool in_progress; 53 bool seen_econnaborted; 54 }; 55 56 struct io_bind { 57 struct file *file; 58 int addr_len; 59 }; 60 61 struct io_listen { 62 struct file *file; 63 int backlog; 64 }; 65 66 struct io_sr_msg { 67 struct file *file; 68 union { 69 struct compat_msghdr __user *umsg_compat; 70 struct user_msghdr __user *umsg; 71 void __user *buf; 72 }; 73 int len; 74 unsigned done_io; 75 unsigned msg_flags; 76 unsigned nr_multishot_loops; 77 u16 flags; 78 /* initialised and used only by !msg send variants */ 79 u16 buf_group; 80 /* per-invocation mshot limit */ 81 unsigned mshot_len; 82 /* overall mshot byte limit */ 83 unsigned mshot_total_len; 84 void __user *msg_control; 85 /* used only for send zerocopy */ 86 struct io_kiocb *notif; 87 }; 88 89 /* 90 * The UAPI flags are the lower 8 bits, as that's all sqe->ioprio will hold 91 * anyway. Use the upper 8 bits for internal uses. 92 */ 93 enum sr_retry_flags { 94 IORING_RECV_RETRY = (1U << 15), 95 IORING_RECV_PARTIAL_MAP = (1U << 14), 96 IORING_RECV_MSHOT_CAP = (1U << 13), 97 IORING_RECV_MSHOT_LIM = (1U << 12), 98 IORING_RECV_MSHOT_DONE = (1U << 11), 99 100 IORING_RECV_RETRY_CLEAR = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP, 101 IORING_RECV_NO_RETRY = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP | 102 IORING_RECV_MSHOT_CAP | IORING_RECV_MSHOT_DONE, 103 }; 104 105 /* 106 * Number of times we'll try and do receives if there's more data. If we 107 * exceed this limit, then add us to the back of the queue and retry from 108 * there. This helps fairness between flooding clients. 109 */ 110 #define MULTISHOT_MAX_RETRY 32 111 112 struct io_recvzc { 113 struct file *file; 114 u16 flags; 115 u32 len; 116 struct io_zcrx_ifq *ifq; 117 }; 118 119 static int io_sg_from_iter_iovec(struct sk_buff *skb, 120 struct iov_iter *from, size_t length); 121 static int io_sg_from_iter(struct sk_buff *skb, 122 struct iov_iter *from, size_t length); 123 124 int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 125 { 126 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 127 128 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags || 129 sqe->buf_index || sqe->splice_fd_in)) 130 return -EINVAL; 131 132 shutdown->how = READ_ONCE(sqe->len); 133 req->flags |= REQ_F_FORCE_ASYNC; 134 return 0; 135 } 136 137 int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) 138 { 139 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 140 struct socket *sock; 141 int ret; 142 143 WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); 144 145 sock = sock_from_file(req->file); 146 if (unlikely(!sock)) 147 return -ENOTSOCK; 148 149 ret = __sys_shutdown_sock(sock, shutdown->how); 150 io_req_set_res(req, ret, 0); 151 return IOU_COMPLETE; 152 } 153 154 static bool io_net_retry(struct socket *sock, int flags) 155 { 156 if (!(flags & MSG_WAITALL)) 157 return false; 158 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET; 159 } 160 161 static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg) 162 { 163 if (kmsg->vec.iovec) 164 io_vec_free(&kmsg->vec); 165 } 166 167 static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) 168 { 169 struct io_async_msghdr *hdr = req->async_data; 170 171 /* can't recycle, ensure we free the iovec if we have one */ 172 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) { 173 io_netmsg_iovec_free(hdr); 174 return; 175 } 176 177 /* Let normal cleanup path reap it if we fail adding to the cache */ 178 io_alloc_cache_vec_kasan(&hdr->vec); 179 if (hdr->vec.nr > IO_VEC_CACHE_SOFT_CAP) 180 io_vec_free(&hdr->vec); 181 182 if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr)) 183 io_req_async_data_clear(req, REQ_F_NEED_CLEANUP); 184 } 185 186 static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req) 187 { 188 struct io_ring_ctx *ctx = req->ctx; 189 struct io_async_msghdr *hdr; 190 191 hdr = io_uring_alloc_async_data(&ctx->netmsg_cache, req); 192 if (!hdr) 193 return NULL; 194 195 /* If the async data was cached, we might have an iov cached inside. */ 196 if (hdr->vec.iovec) 197 req->flags |= REQ_F_NEED_CLEANUP; 198 return hdr; 199 } 200 201 static inline void io_mshot_prep_retry(struct io_kiocb *req, 202 struct io_async_msghdr *kmsg) 203 { 204 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 205 206 req->flags &= ~REQ_F_BL_EMPTY; 207 sr->done_io = 0; 208 sr->flags &= ~IORING_RECV_RETRY_CLEAR; 209 sr->len = sr->mshot_len; 210 } 211 212 static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg, 213 const struct iovec __user *uiov, unsigned uvec_seg, 214 int ddir) 215 { 216 struct iovec *iov; 217 int ret, nr_segs; 218 219 if (iomsg->vec.iovec) { 220 nr_segs = iomsg->vec.nr; 221 iov = iomsg->vec.iovec; 222 } else { 223 nr_segs = 1; 224 iov = &iomsg->fast_iov; 225 } 226 227 ret = __import_iovec(ddir, uiov, uvec_seg, nr_segs, &iov, 228 &iomsg->msg.msg_iter, io_is_compat(req->ctx)); 229 if (unlikely(ret < 0)) 230 return ret; 231 232 if (iov) { 233 req->flags |= REQ_F_NEED_CLEANUP; 234 io_vec_reset_iovec(&iomsg->vec, iov, iomsg->msg.msg_iter.nr_segs); 235 } 236 return 0; 237 } 238 239 static int io_copy_msghdr_from_user(struct user_msghdr *msg, 240 struct user_msghdr __user *umsg) 241 { 242 if (!user_access_begin(umsg, sizeof(*umsg))) 243 return -EFAULT; 244 unsafe_get_user(msg->msg_name, &umsg->msg_name, ua_end); 245 unsafe_get_user(msg->msg_namelen, &umsg->msg_namelen, ua_end); 246 unsafe_get_user(msg->msg_iov, &umsg->msg_iov, ua_end); 247 unsafe_get_user(msg->msg_iovlen, &umsg->msg_iovlen, ua_end); 248 unsafe_get_user(msg->msg_control, &umsg->msg_control, ua_end); 249 unsafe_get_user(msg->msg_controllen, &umsg->msg_controllen, ua_end); 250 user_access_end(); 251 return 0; 252 ua_end: 253 user_access_end(); 254 return -EFAULT; 255 } 256 257 static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, 258 struct user_msghdr *msg, 259 struct sockaddr __user **save_addr) 260 { 261 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 262 int ret; 263 264 iomsg->msg.msg_name = &iomsg->addr; 265 iomsg->msg.msg_iter.nr_segs = 0; 266 267 if (io_is_compat(req->ctx)) { 268 struct compat_msghdr cmsg; 269 270 if (copy_from_user(&cmsg, sr->umsg_compat, sizeof(cmsg))) 271 return -EFAULT; 272 273 ret = __get_compat_msghdr(&iomsg->msg, &cmsg, save_addr); 274 if (ret) 275 return ret; 276 277 memset(msg, 0, sizeof(*msg)); 278 msg->msg_namelen = cmsg.msg_namelen; 279 msg->msg_controllen = cmsg.msg_controllen; 280 msg->msg_iov = compat_ptr(cmsg.msg_iov); 281 msg->msg_iovlen = cmsg.msg_iovlen; 282 } else { 283 ret = io_copy_msghdr_from_user(msg, sr->umsg); 284 if (unlikely(ret)) 285 return ret; 286 287 msg->msg_flags = 0; 288 289 ret = __copy_msghdr(&iomsg->msg, msg, save_addr); 290 if (ret) 291 return ret; 292 } 293 294 if (req->flags & REQ_F_BUFFER_SELECT) { 295 if (msg->msg_iovlen == 0) { 296 sr->len = 0; 297 } else if (msg->msg_iovlen > 1) { 298 return -EINVAL; 299 } else { 300 struct iovec fast_iov, *iov; 301 302 iov = iovec_from_user(msg->msg_iov, 1, 1, &fast_iov, 303 io_is_compat(req->ctx)); 304 if (IS_ERR(iov)) 305 return PTR_ERR(iov); 306 sr->len = iov->iov_len; 307 } 308 } 309 return 0; 310 } 311 312 void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) 313 { 314 struct io_async_msghdr *io = req->async_data; 315 316 io_netmsg_iovec_free(io); 317 } 318 319 static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) 320 { 321 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 322 struct io_async_msghdr *kmsg = req->async_data; 323 void __user *addr; 324 u16 addr_len; 325 int ret; 326 327 sr->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); 328 329 if (READ_ONCE(sqe->__pad3[0])) 330 return -EINVAL; 331 332 kmsg->msg.msg_name = NULL; 333 kmsg->msg.msg_namelen = 0; 334 kmsg->msg.msg_control = NULL; 335 kmsg->msg.msg_controllen = 0; 336 kmsg->msg.msg_ubuf = NULL; 337 338 addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 339 addr_len = READ_ONCE(sqe->addr_len); 340 if (addr) { 341 ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr); 342 if (unlikely(ret < 0)) 343 return ret; 344 kmsg->msg.msg_name = &kmsg->addr; 345 kmsg->msg.msg_namelen = addr_len; 346 } 347 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 348 if (!(sr->flags & IORING_SEND_VECTORIZED)) { 349 req->flags |= REQ_F_IMPORT_BUFFER; 350 return 0; 351 } 352 353 kmsg->msg.msg_iter.nr_segs = sr->len; 354 return io_prep_reg_iovec(req, &kmsg->vec, sr->buf, sr->len); 355 } 356 if (req->flags & REQ_F_BUFFER_SELECT) 357 return 0; 358 359 if (sr->flags & IORING_SEND_VECTORIZED) 360 return io_net_import_vec(req, kmsg, sr->buf, sr->len, ITER_SOURCE); 361 362 return import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter); 363 } 364 365 static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) 366 { 367 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 368 struct io_async_msghdr *kmsg = req->async_data; 369 struct user_msghdr msg; 370 int ret; 371 372 sr->flags |= IORING_SEND_VECTORIZED; 373 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 374 ret = io_msg_copy_hdr(req, kmsg, &msg, NULL); 375 if (unlikely(ret)) 376 return ret; 377 /* save msg_control as sys_sendmsg() overwrites it */ 378 sr->msg_control = kmsg->msg.msg_control_user; 379 380 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 381 kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen; 382 return io_prep_reg_iovec(req, &kmsg->vec, msg.msg_iov, 383 msg.msg_iovlen); 384 } 385 if (req->flags & REQ_F_BUFFER_SELECT) 386 return 0; 387 return io_net_import_vec(req, kmsg, msg.msg_iov, msg.msg_iovlen, ITER_SOURCE); 388 } 389 390 #define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE | \ 391 IORING_SEND_VECTORIZED | IORING_RECVSEND_FIXED_BUF) 392 393 int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 394 { 395 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 396 397 sr->done_io = 0; 398 sr->len = READ_ONCE(sqe->len); 399 if (unlikely(sr->len < 0)) 400 return -EINVAL; 401 sr->flags = READ_ONCE(sqe->ioprio); 402 if (sr->flags & ~SENDMSG_FLAGS) 403 return -EINVAL; 404 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 405 /* registered buffer send only supported for plain IORING_OP_SEND */ 406 if (req->opcode != IORING_OP_SEND || 407 (req->flags & REQ_F_BUFFER_SELECT) || 408 (sr->flags & (IORING_RECVSEND_BUNDLE|IORING_SEND_VECTORIZED))) 409 return -EINVAL; 410 req->buf_index = READ_ONCE(sqe->buf_index); 411 } 412 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 413 if (sr->msg_flags & MSG_DONTWAIT) 414 req->flags |= REQ_F_NOWAIT; 415 if (req->flags & REQ_F_BUFFER_SELECT) 416 sr->buf_group = req->buf_index; 417 sr->mshot_total_len = sr->mshot_len = 0; 418 if (sr->flags & IORING_RECVSEND_BUNDLE) { 419 if (req->opcode == IORING_OP_SENDMSG) 420 return -EINVAL; 421 sr->msg_flags |= MSG_WAITALL; 422 req->flags |= REQ_F_MULTISHOT; 423 } 424 425 if (io_is_compat(req->ctx)) 426 sr->msg_flags |= MSG_CMSG_COMPAT; 427 428 if (unlikely(!io_msg_alloc_async(req))) 429 return -ENOMEM; 430 if (req->opcode != IORING_OP_SENDMSG) 431 return io_send_setup(req, sqe); 432 if (unlikely(sqe->addr2 || sqe->file_index)) 433 return -EINVAL; 434 return io_sendmsg_setup(req, sqe); 435 } 436 437 static void io_req_msg_cleanup(struct io_kiocb *req, 438 unsigned int issue_flags) 439 { 440 io_netmsg_recycle(req, issue_flags); 441 } 442 443 /* 444 * For bundle completions, we need to figure out how many segments we consumed. 445 * A bundle could be using a single ITER_UBUF if that's all we mapped, or it 446 * could be using an ITER_IOVEC. If the latter, then if we consumed all of 447 * the segments, then it's a trivial questiont o answer. If we have residual 448 * data in the iter, then loop the segments to figure out how much we 449 * transferred. 450 */ 451 static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret) 452 { 453 struct iovec *iov; 454 int nbufs; 455 456 /* no data is always zero segments, and a ubuf is always 1 segment */ 457 if (ret <= 0) 458 return 0; 459 if (iter_is_ubuf(&kmsg->msg.msg_iter)) 460 return 1; 461 462 iov = kmsg->vec.iovec; 463 if (!iov) 464 iov = &kmsg->fast_iov; 465 466 /* if all data was transferred, it's basic pointer math */ 467 if (!iov_iter_count(&kmsg->msg.msg_iter)) 468 return iter_iov(&kmsg->msg.msg_iter) - iov; 469 470 /* short transfer, count segments */ 471 nbufs = 0; 472 do { 473 int this_len = min_t(int, iov[nbufs].iov_len, ret); 474 475 nbufs++; 476 ret -= this_len; 477 } while (ret); 478 479 return nbufs; 480 } 481 482 static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl, 483 struct io_async_msghdr *kmsg, int len) 484 { 485 req->flags |= REQ_F_BL_NO_RECYCLE; 486 if (req->flags & REQ_F_BUFFERS_COMMIT) 487 io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len)); 488 return IOU_RETRY; 489 } 490 491 static inline bool io_send_finish(struct io_kiocb *req, 492 struct io_async_msghdr *kmsg, 493 struct io_br_sel *sel) 494 { 495 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 496 bool bundle_finished = sel->val <= 0; 497 unsigned int cflags; 498 499 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) { 500 cflags = io_put_kbuf(req, sel->val, sel->buf_list); 501 goto finish; 502 } 503 504 cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val)); 505 506 /* 507 * Don't start new bundles if the buffer list is empty, or if the 508 * current operation needed to go through polling to complete. 509 */ 510 if (bundle_finished || req->flags & (REQ_F_BL_EMPTY | REQ_F_POLLED)) 511 goto finish; 512 513 /* 514 * Fill CQE for this receive and see if we should keep trying to 515 * receive from this socket. 516 */ 517 if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) { 518 io_mshot_prep_retry(req, kmsg); 519 return false; 520 } 521 522 /* Otherwise stop bundle and use the current result. */ 523 finish: 524 io_req_set_res(req, sel->val, cflags); 525 sel->val = IOU_COMPLETE; 526 return true; 527 } 528 529 int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) 530 { 531 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 532 struct io_async_msghdr *kmsg = req->async_data; 533 struct socket *sock; 534 unsigned flags; 535 int min_ret = 0; 536 int ret; 537 538 sock = sock_from_file(req->file); 539 if (unlikely(!sock)) 540 return -ENOTSOCK; 541 542 if (!(req->flags & REQ_F_POLLED) && 543 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 544 return -EAGAIN; 545 546 flags = sr->msg_flags; 547 if (issue_flags & IO_URING_F_NONBLOCK) 548 flags |= MSG_DONTWAIT; 549 if (flags & MSG_WAITALL) 550 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 551 552 kmsg->msg.msg_control_user = sr->msg_control; 553 554 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 555 556 if (ret < min_ret) { 557 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 558 return -EAGAIN; 559 if (ret > 0 && io_net_retry(sock, flags)) { 560 kmsg->msg.msg_controllen = 0; 561 kmsg->msg.msg_control = NULL; 562 sr->done_io += ret; 563 return -EAGAIN; 564 } 565 if (ret == -ERESTARTSYS) 566 ret = -EINTR; 567 req_set_fail(req); 568 } 569 io_req_msg_cleanup(req, issue_flags); 570 if (ret >= 0) 571 ret += sr->done_io; 572 else if (sr->done_io) 573 ret = sr->done_io; 574 io_req_set_res(req, ret, 0); 575 return IOU_COMPLETE; 576 } 577 578 static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags, 579 struct io_br_sel *sel, struct io_async_msghdr *kmsg) 580 { 581 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 582 struct buf_sel_arg arg = { 583 .iovs = &kmsg->fast_iov, 584 .max_len = min_not_zero(sr->len, INT_MAX), 585 .nr_iovs = 1, 586 .buf_group = sr->buf_group, 587 }; 588 int ret; 589 590 if (kmsg->vec.iovec) { 591 arg.nr_iovs = kmsg->vec.nr; 592 arg.iovs = kmsg->vec.iovec; 593 arg.mode = KBUF_MODE_FREE; 594 } 595 596 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) 597 arg.nr_iovs = 1; 598 else 599 arg.mode |= KBUF_MODE_EXPAND; 600 601 ret = io_buffers_select(req, &arg, sel, issue_flags); 602 if (unlikely(ret < 0)) 603 return ret; 604 605 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) { 606 kmsg->vec.nr = ret; 607 kmsg->vec.iovec = arg.iovs; 608 req->flags |= REQ_F_NEED_CLEANUP; 609 } 610 sr->len = arg.out_len; 611 612 if (ret == 1) { 613 sr->buf = arg.iovs[0].iov_base; 614 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, 615 &kmsg->msg.msg_iter); 616 if (unlikely(ret)) 617 return ret; 618 } else { 619 iov_iter_init(&kmsg->msg.msg_iter, ITER_SOURCE, 620 arg.iovs, ret, arg.out_len); 621 } 622 623 return 0; 624 } 625 626 int io_send(struct io_kiocb *req, unsigned int issue_flags) 627 { 628 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 629 struct io_async_msghdr *kmsg = req->async_data; 630 struct io_br_sel sel = { }; 631 struct socket *sock; 632 unsigned flags; 633 int min_ret = 0; 634 int ret; 635 636 sock = sock_from_file(req->file); 637 if (unlikely(!sock)) 638 return -ENOTSOCK; 639 640 if (!(req->flags & REQ_F_POLLED) && 641 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 642 return -EAGAIN; 643 644 if (req->flags & REQ_F_IMPORT_BUFFER) { 645 ret = io_import_reg_buf(req, &kmsg->msg.msg_iter, 646 (u64)(uintptr_t)sr->buf, sr->len, 647 ITER_SOURCE, issue_flags); 648 if (unlikely(ret)) 649 return ret; 650 req->flags &= ~REQ_F_IMPORT_BUFFER; 651 } 652 653 flags = sr->msg_flags; 654 if (issue_flags & IO_URING_F_NONBLOCK) 655 flags |= MSG_DONTWAIT; 656 657 retry_bundle: 658 sel.buf_list = NULL; 659 if (io_do_buffer_select(req)) { 660 ret = io_send_select_buffer(req, issue_flags, &sel, kmsg); 661 if (ret) 662 return ret; 663 } 664 665 /* 666 * If MSG_WAITALL is set, or this is a bundle send, then we need 667 * the full amount. If just bundle is set, if we do a short send 668 * then we complete the bundle sequence rather than continue on. 669 */ 670 if (flags & MSG_WAITALL || sr->flags & IORING_RECVSEND_BUNDLE) 671 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 672 673 flags &= ~MSG_INTERNAL_SENDMSG_FLAGS; 674 kmsg->msg.msg_flags = flags; 675 ret = sock_sendmsg(sock, &kmsg->msg); 676 if (ret < min_ret) { 677 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 678 return -EAGAIN; 679 680 if (ret > 0 && io_net_retry(sock, flags)) { 681 sr->len -= ret; 682 sr->buf += ret; 683 sr->done_io += ret; 684 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); 685 } 686 if (ret == -ERESTARTSYS) 687 ret = -EINTR; 688 req_set_fail(req); 689 } 690 if (ret >= 0) 691 ret += sr->done_io; 692 else if (sr->done_io) 693 ret = sr->done_io; 694 695 sel.val = ret; 696 if (!io_send_finish(req, kmsg, &sel)) 697 goto retry_bundle; 698 699 io_req_msg_cleanup(req, issue_flags); 700 return sel.val; 701 } 702 703 static int io_recvmsg_mshot_prep(struct io_kiocb *req, 704 struct io_async_msghdr *iomsg, 705 int namelen, size_t controllen) 706 { 707 if ((req->flags & (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) == 708 (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) { 709 int hdr; 710 711 if (unlikely(namelen < 0)) 712 return -EOVERFLOW; 713 if (check_add_overflow(sizeof(struct io_uring_recvmsg_out), 714 namelen, &hdr)) 715 return -EOVERFLOW; 716 if (check_add_overflow(hdr, controllen, &hdr)) 717 return -EOVERFLOW; 718 719 iomsg->namelen = namelen; 720 iomsg->controllen = controllen; 721 return 0; 722 } 723 724 return 0; 725 } 726 727 static int io_recvmsg_copy_hdr(struct io_kiocb *req, 728 struct io_async_msghdr *iomsg) 729 { 730 struct user_msghdr msg; 731 int ret; 732 733 ret = io_msg_copy_hdr(req, iomsg, &msg, &iomsg->uaddr); 734 if (unlikely(ret)) 735 return ret; 736 737 if (!(req->flags & REQ_F_BUFFER_SELECT)) { 738 ret = io_net_import_vec(req, iomsg, msg.msg_iov, msg.msg_iovlen, 739 ITER_DEST); 740 if (unlikely(ret)) 741 return ret; 742 } 743 return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen, 744 msg.msg_controllen); 745 } 746 747 static int io_recvmsg_prep_setup(struct io_kiocb *req) 748 { 749 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 750 struct io_async_msghdr *kmsg; 751 752 kmsg = io_msg_alloc_async(req); 753 if (unlikely(!kmsg)) 754 return -ENOMEM; 755 756 if (req->opcode == IORING_OP_RECV) { 757 kmsg->msg.msg_name = NULL; 758 kmsg->msg.msg_namelen = 0; 759 kmsg->msg.msg_inq = 0; 760 kmsg->msg.msg_control = NULL; 761 kmsg->msg.msg_get_inq = 1; 762 kmsg->msg.msg_controllen = 0; 763 kmsg->msg.msg_ubuf = NULL; 764 765 if (req->flags & REQ_F_BUFFER_SELECT) 766 return 0; 767 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 768 req->flags |= REQ_F_IMPORT_BUFFER; 769 return 0; 770 } 771 return import_ubuf(ITER_DEST, sr->buf, sr->len, 772 &kmsg->msg.msg_iter); 773 } 774 775 return io_recvmsg_copy_hdr(req, kmsg); 776 } 777 778 #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT | \ 779 IORING_RECVSEND_BUNDLE | IORING_RECVSEND_FIXED_BUF) 780 781 int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 782 { 783 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 784 785 sr->done_io = 0; 786 787 if (unlikely(sqe->addr2)) 788 return -EINVAL; 789 790 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 791 sr->len = READ_ONCE(sqe->len); 792 if (unlikely(sr->len < 0)) 793 return -EINVAL; 794 sr->flags = READ_ONCE(sqe->ioprio); 795 if (sr->flags & ~RECVMSG_FLAGS) 796 return -EINVAL; 797 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 798 /* registered buffer recv only for plain IORING_OP_RECV */ 799 if (req->opcode != IORING_OP_RECV || 800 (req->flags & REQ_F_BUFFER_SELECT) || 801 (sr->flags & (IORING_RECV_MULTISHOT | IORING_RECVSEND_BUNDLE))) 802 return -EINVAL; 803 req->buf_index = READ_ONCE(sqe->buf_index); 804 } 805 sr->msg_flags = READ_ONCE(sqe->msg_flags); 806 if (sr->msg_flags & MSG_DONTWAIT) 807 req->flags |= REQ_F_NOWAIT; 808 if (sr->msg_flags & MSG_ERRQUEUE) 809 req->flags |= REQ_F_CLEAR_POLLIN; 810 if (req->flags & REQ_F_BUFFER_SELECT) 811 sr->buf_group = req->buf_index; 812 sr->mshot_total_len = sr->mshot_len = 0; 813 if (sr->flags & IORING_RECV_MULTISHOT) { 814 if (!(req->flags & REQ_F_BUFFER_SELECT)) 815 return -EINVAL; 816 if (sr->msg_flags & MSG_WAITALL) 817 return -EINVAL; 818 if (req->opcode == IORING_OP_RECV) { 819 sr->mshot_len = sr->len; 820 sr->mshot_total_len = READ_ONCE(sqe->optlen); 821 if (sr->mshot_total_len) 822 sr->flags |= IORING_RECV_MSHOT_LIM; 823 } else if (sqe->optlen) { 824 return -EINVAL; 825 } 826 req->flags |= REQ_F_APOLL_MULTISHOT; 827 } else if (sqe->optlen) { 828 return -EINVAL; 829 } 830 831 if (sr->flags & IORING_RECVSEND_BUNDLE) { 832 if (req->opcode == IORING_OP_RECVMSG) 833 return -EINVAL; 834 } 835 836 if (io_is_compat(req->ctx)) 837 sr->msg_flags |= MSG_CMSG_COMPAT; 838 839 sr->nr_multishot_loops = 0; 840 return io_recvmsg_prep_setup(req); 841 } 842 843 /* bits to clear in old and inherit in new cflags on bundle retry */ 844 #define CQE_F_MASK (IORING_CQE_F_SOCK_NONEMPTY|IORING_CQE_F_MORE|\ 845 IORING_CQE_F_BUF_MORE) 846 847 /* 848 * Finishes io_recv and io_recvmsg. 849 * 850 * Returns true if it is actually finished, or false if it should run 851 * again (for multishot). 852 */ 853 static inline bool io_recv_finish(struct io_kiocb *req, 854 struct io_async_msghdr *kmsg, 855 struct io_br_sel *sel, bool mshot_finished, 856 unsigned issue_flags, int consumed) 857 { 858 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 859 unsigned int cflags = 0; 860 861 if (kmsg->msg.msg_inq > 0) 862 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 863 864 if (sel->val > 0 && sr->flags & IORING_RECV_MSHOT_LIM) { 865 /* 866 * If sr->len hits zero, the limit has been reached. Mark 867 * mshot as finished, and flag MSHOT_DONE as well to prevent 868 * a potential bundle from being retried. 869 */ 870 sr->mshot_total_len -= min_t(int, sel->val, sr->mshot_total_len); 871 if (!sr->mshot_total_len) { 872 sr->flags |= IORING_RECV_MSHOT_DONE; 873 mshot_finished = true; 874 } 875 } 876 877 if (sr->flags & IORING_RECVSEND_BUNDLE) { 878 size_t this_ret = sel->val - sr->done_io; 879 880 cflags |= io_put_kbufs(req, consumed, sel->buf_list, io_bundle_nbufs(kmsg, consumed)); 881 if (sr->flags & IORING_RECV_RETRY) 882 cflags = req->cqe.flags | (cflags & CQE_F_MASK); 883 if (sr->mshot_len && sel->val >= sr->mshot_len) 884 sr->flags |= IORING_RECV_MSHOT_CAP; 885 /* bundle with no more immediate buffers, we're done */ 886 if (req->flags & REQ_F_BL_EMPTY) 887 goto finish; 888 /* 889 * If more is available AND it was a full transfer, retry and 890 * append to this one 891 */ 892 if (!(sr->flags & IORING_RECV_NO_RETRY) && 893 kmsg->msg.msg_inq > 1 && this_ret > 0 && 894 !iov_iter_count(&kmsg->msg.msg_iter)) { 895 req->cqe.flags = cflags & ~CQE_F_MASK; 896 sr->len = kmsg->msg.msg_inq; 897 sr->done_io += this_ret; 898 sr->flags |= IORING_RECV_RETRY; 899 return false; 900 } 901 } else { 902 cflags |= io_put_kbuf(req, consumed, sel->buf_list); 903 } 904 905 /* 906 * Fill CQE for this receive and see if we should keep trying to 907 * receive from this socket. 908 */ 909 if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished && 910 io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) { 911 sel->val = IOU_RETRY; 912 io_mshot_prep_retry(req, kmsg); 913 /* Known not-empty or unknown state, retry */ 914 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) { 915 if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY && 916 !(sr->flags & IORING_RECV_MSHOT_CAP)) { 917 return false; 918 } 919 /* mshot retries exceeded, force a requeue */ 920 sr->nr_multishot_loops = 0; 921 sr->flags &= ~IORING_RECV_MSHOT_CAP; 922 if (issue_flags & IO_URING_F_MULTISHOT) 923 sel->val = IOU_REQUEUE; 924 } 925 return true; 926 } 927 928 /* Finish the request / stop multishot. */ 929 finish: 930 io_req_set_res(req, sel->val, cflags); 931 sel->val = IOU_COMPLETE; 932 io_req_msg_cleanup(req, issue_flags); 933 return true; 934 } 935 936 static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg, 937 struct io_sr_msg *sr, void __user **buf, 938 size_t *len) 939 { 940 unsigned long ubuf = (unsigned long) *buf; 941 unsigned long hdr; 942 943 hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 944 kmsg->controllen; 945 if (*len < hdr) 946 return -EFAULT; 947 948 if (kmsg->controllen) { 949 unsigned long control = ubuf + hdr - kmsg->controllen; 950 951 kmsg->msg.msg_control_user = (void __user *) control; 952 kmsg->msg.msg_controllen = kmsg->controllen; 953 } 954 955 sr->buf = *buf; /* stash for later copy */ 956 *buf = (void __user *) (ubuf + hdr); 957 kmsg->payloadlen = *len = *len - hdr; 958 return 0; 959 } 960 961 struct io_recvmsg_multishot_hdr { 962 struct io_uring_recvmsg_out msg; 963 struct sockaddr_storage addr; 964 }; 965 966 static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io, 967 struct io_async_msghdr *kmsg, 968 unsigned int flags, bool *finished) 969 { 970 int err; 971 int copy_len; 972 struct io_recvmsg_multishot_hdr hdr; 973 974 if (kmsg->namelen) 975 kmsg->msg.msg_name = &hdr.addr; 976 kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT); 977 kmsg->msg.msg_namelen = 0; 978 979 if (sock->file->f_flags & O_NONBLOCK) 980 flags |= MSG_DONTWAIT; 981 982 err = sock_recvmsg(sock, &kmsg->msg, flags); 983 *finished = err <= 0; 984 if (err < 0) 985 return err; 986 987 hdr.msg = (struct io_uring_recvmsg_out) { 988 .controllen = kmsg->controllen - kmsg->msg.msg_controllen, 989 .flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT 990 }; 991 992 hdr.msg.payloadlen = err; 993 if (err > kmsg->payloadlen) 994 err = kmsg->payloadlen; 995 996 copy_len = sizeof(struct io_uring_recvmsg_out); 997 if (kmsg->msg.msg_namelen > kmsg->namelen) 998 copy_len += kmsg->namelen; 999 else 1000 copy_len += kmsg->msg.msg_namelen; 1001 1002 /* 1003 * "fromlen shall refer to the value before truncation.." 1004 * 1003.1g 1005 */ 1006 hdr.msg.namelen = kmsg->msg.msg_namelen; 1007 1008 /* ensure that there is no gap between hdr and sockaddr_storage */ 1009 BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr, addr) != 1010 sizeof(struct io_uring_recvmsg_out)); 1011 if (copy_to_user(io->buf, &hdr, copy_len)) { 1012 *finished = true; 1013 return -EFAULT; 1014 } 1015 1016 return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 1017 kmsg->controllen + err; 1018 } 1019 1020 int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 1021 { 1022 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1023 struct io_async_msghdr *kmsg = req->async_data; 1024 struct io_br_sel sel = { }; 1025 struct socket *sock; 1026 unsigned flags; 1027 int ret, min_ret = 0; 1028 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1029 bool mshot_finished = true; 1030 int consumed = 0; 1031 size_t len; 1032 1033 sock = sock_from_file(req->file); 1034 if (unlikely(!sock)) 1035 return -ENOTSOCK; 1036 1037 if (!(req->flags & REQ_F_POLLED) && 1038 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1039 return -EAGAIN; 1040 1041 flags = sr->msg_flags; 1042 if (force_nonblock) 1043 flags |= MSG_DONTWAIT; 1044 1045 retry_multishot: 1046 sel.buf_list = NULL; 1047 len = sr->len; 1048 if (io_do_buffer_select(req)) { 1049 sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); 1050 if (!sel.addr) 1051 return -ENOBUFS; 1052 1053 if (req->flags & REQ_F_APOLL_MULTISHOT) { 1054 ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len); 1055 if (ret) { 1056 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1057 return ret; 1058 } 1059 } 1060 1061 iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len); 1062 } 1063 1064 kmsg->msg.msg_get_inq = 1; 1065 kmsg->msg.msg_inq = -1; 1066 if (req->flags & REQ_F_APOLL_MULTISHOT) { 1067 ret = io_recvmsg_multishot(sock, sr, kmsg, flags, 1068 &mshot_finished); 1069 consumed = ret; 1070 } else { 1071 /* disable partial retry for recvmsg with cmsg attached */ 1072 if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen) 1073 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1074 1075 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, 1076 kmsg->uaddr, flags); 1077 /* 1078 * With MSG_TRUNC, the net layer will return the full size of 1079 * the packet, even if we only filled part of it in the buffers. 1080 * Adjust the returned size to consume only the real part of the 1081 * buffer. 1082 */ 1083 consumed = ret; 1084 if (ret > 0) 1085 consumed = min_t(size_t, ret, len); 1086 } 1087 1088 if (ret < min_ret) { 1089 if (ret == -EAGAIN && force_nonblock) { 1090 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1091 return IOU_RETRY; 1092 } 1093 if (ret > 0 && io_net_retry(sock, flags)) { 1094 sr->done_io += ret; 1095 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); 1096 } 1097 if (ret == -ERESTARTSYS) 1098 ret = -EINTR; 1099 req_set_fail(req); 1100 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 1101 req_set_fail(req); 1102 } 1103 1104 if (ret > 0) 1105 ret += sr->done_io; 1106 else if (sr->done_io) 1107 ret = sr->done_io; 1108 else 1109 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1110 1111 sel.val = ret; 1112 if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed)) 1113 goto retry_multishot; 1114 1115 return sel.val; 1116 } 1117 1118 static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg, 1119 struct io_br_sel *sel, unsigned int issue_flags) 1120 { 1121 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1122 size_t len; 1123 int ret; 1124 1125 /* 1126 * If the ring isn't locked, then don't use the peek interface 1127 * to grab multiple buffers as we will lock/unlock between 1128 * this selection and posting the buffers. 1129 */ 1130 if (!(issue_flags & IO_URING_F_UNLOCKED) && 1131 sr->flags & IORING_RECVSEND_BUNDLE) { 1132 struct buf_sel_arg arg = { 1133 .iovs = &kmsg->fast_iov, 1134 .nr_iovs = 1, 1135 .mode = KBUF_MODE_EXPAND, 1136 .buf_group = sr->buf_group, 1137 }; 1138 1139 if (kmsg->vec.iovec) { 1140 arg.nr_iovs = kmsg->vec.nr; 1141 arg.iovs = kmsg->vec.iovec; 1142 arg.mode |= KBUF_MODE_FREE; 1143 } 1144 1145 if (sel->val) 1146 arg.max_len = sel->val; 1147 else if (kmsg->msg.msg_inq > 1) 1148 arg.max_len = min_not_zero(sel->val, (ssize_t) kmsg->msg.msg_inq); 1149 1150 /* if mshot limited, ensure we don't go over */ 1151 if (sr->flags & IORING_RECV_MSHOT_LIM) 1152 arg.max_len = min_not_zero(arg.max_len, sr->mshot_total_len); 1153 ret = io_buffers_peek(req, &arg, sel); 1154 if (unlikely(ret < 0)) 1155 return ret; 1156 1157 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) { 1158 kmsg->vec.nr = ret; 1159 kmsg->vec.iovec = arg.iovs; 1160 req->flags |= REQ_F_NEED_CLEANUP; 1161 } 1162 if (arg.partial_map) 1163 sr->flags |= IORING_RECV_PARTIAL_MAP; 1164 1165 /* special case 1 vec, can be a fast path */ 1166 if (ret == 1) { 1167 sr->buf = arg.iovs[0].iov_base; 1168 len = sr->len = arg.iovs[0].iov_len; 1169 goto map_ubuf; 1170 } 1171 iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret, 1172 arg.out_len); 1173 len = arg.out_len; 1174 } else { 1175 len = sel->val; 1176 1177 *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); 1178 if (!sel->addr) 1179 return -ENOBUFS; 1180 sr->buf = sel->addr; 1181 sr->len = len; 1182 map_ubuf: 1183 ret = import_ubuf(ITER_DEST, sr->buf, sr->len, 1184 &kmsg->msg.msg_iter); 1185 if (unlikely(ret)) 1186 return ret; 1187 } 1188 1189 return len; 1190 } 1191 1192 int io_recv(struct io_kiocb *req, unsigned int issue_flags) 1193 { 1194 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1195 struct io_async_msghdr *kmsg = req->async_data; 1196 struct io_br_sel sel; 1197 struct socket *sock; 1198 unsigned flags; 1199 int ret, min_ret = 0, consumed = 0; 1200 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1201 bool mshot_finished; 1202 size_t len = 0; 1203 1204 sock = sock_from_file(req->file); 1205 if (unlikely(!sock)) 1206 return -ENOTSOCK; 1207 1208 if (!(req->flags & REQ_F_POLLED) && 1209 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1210 return -EAGAIN; 1211 1212 flags = sr->msg_flags; 1213 if (force_nonblock) 1214 flags |= MSG_DONTWAIT; 1215 1216 if (req->flags & REQ_F_IMPORT_BUFFER) { 1217 ret = io_import_reg_buf(req, &kmsg->msg.msg_iter, 1218 (u64)(uintptr_t)sr->buf, sr->len, 1219 ITER_DEST, issue_flags); 1220 if (unlikely(ret)) { 1221 kmsg->msg.msg_inq = -1; 1222 sel.buf_list = NULL; 1223 goto out_free; 1224 } 1225 req->flags &= ~REQ_F_IMPORT_BUFFER; 1226 } 1227 1228 retry_multishot: 1229 sel.buf_list = NULL; 1230 len = sr->len; 1231 if (io_do_buffer_select(req)) { 1232 sel.val = sr->len; 1233 ret = io_recv_buf_select(req, kmsg, &sel, issue_flags); 1234 if (unlikely(ret < 0)) { 1235 kmsg->msg.msg_inq = -1; 1236 goto out_free; 1237 } 1238 len = ret; 1239 sr->buf = NULL; 1240 } 1241 1242 kmsg->msg.msg_flags = 0; 1243 kmsg->msg.msg_inq = -1; 1244 1245 if (flags & MSG_WAITALL) 1246 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1247 1248 ret = sock_recvmsg(sock, &kmsg->msg, flags); 1249 if (ret < min_ret) { 1250 if (ret == -EAGAIN && force_nonblock) { 1251 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1252 return IOU_RETRY; 1253 } 1254 if (ret > 0 && io_net_retry(sock, flags)) { 1255 sr->len -= ret; 1256 sr->buf += ret; 1257 sr->done_io += ret; 1258 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); 1259 } 1260 if (ret == -ERESTARTSYS) 1261 ret = -EINTR; 1262 req_set_fail(req); 1263 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 1264 out_free: 1265 req_set_fail(req); 1266 } 1267 1268 mshot_finished = ret <= 0; 1269 1270 /* 1271 * With MSG_TRUNC, the net layer will return the full size of 1272 * the packet, even if we only filled part of it in the buffers. 1273 * Adjust the returned size to consume only the real part of the 1274 * buffer. 1275 */ 1276 consumed = ret; 1277 if (ret > 0) 1278 consumed = min_t(size_t, ret, len); 1279 1280 if (ret > 0) 1281 ret += sr->done_io; 1282 else if (sr->done_io) 1283 ret = sr->done_io; 1284 else 1285 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1286 1287 sel.val = ret; 1288 if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags, consumed)) 1289 goto retry_multishot; 1290 1291 return sel.val; 1292 } 1293 1294 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1295 { 1296 struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc); 1297 unsigned ifq_idx; 1298 1299 if (unlikely(sqe->addr2 || sqe->addr || sqe->addr3)) 1300 return -EINVAL; 1301 1302 ifq_idx = READ_ONCE(sqe->zcrx_ifq_idx); 1303 zc->ifq = xa_load(&req->ctx->zcrx_ctxs, ifq_idx); 1304 if (!zc->ifq) 1305 return -EINVAL; 1306 1307 zc->len = READ_ONCE(sqe->len); 1308 zc->flags = READ_ONCE(sqe->ioprio); 1309 if (READ_ONCE(sqe->msg_flags)) 1310 return -EINVAL; 1311 if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT)) 1312 return -EINVAL; 1313 /* multishot required */ 1314 if (!(zc->flags & IORING_RECV_MULTISHOT)) 1315 return -EINVAL; 1316 /* All data completions are posted as aux CQEs. */ 1317 req->flags |= REQ_F_APOLL_MULTISHOT; 1318 1319 return 0; 1320 } 1321 1322 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags) 1323 { 1324 struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc); 1325 struct socket *sock; 1326 unsigned int len; 1327 int ret; 1328 1329 sock = sock_from_file(req->file); 1330 if (unlikely(!sock)) 1331 return -ENOTSOCK; 1332 1333 if (!(req->flags & REQ_F_POLLED) && 1334 (zc->flags & IORING_RECVSEND_POLL_FIRST)) 1335 return -EAGAIN; 1336 1337 len = zc->len; 1338 ret = io_zcrx_recv(req, zc->ifq, sock, 0, issue_flags, &zc->len); 1339 if (len && zc->len == 0) { 1340 io_req_set_res(req, 0, 0); 1341 1342 return IOU_COMPLETE; 1343 } 1344 if (unlikely(ret <= 0) && ret != -EAGAIN) { 1345 if (ret == -ERESTARTSYS) 1346 ret = -EINTR; 1347 if (ret == IOU_REQUEUE) 1348 return IOU_REQUEUE; 1349 1350 req_set_fail(req); 1351 io_req_set_res(req, ret, 0); 1352 return IOU_COMPLETE; 1353 } 1354 return IOU_RETRY; 1355 } 1356 1357 void io_send_zc_cleanup(struct io_kiocb *req) 1358 { 1359 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 1360 struct io_async_msghdr *io = req->async_data; 1361 1362 if (req_has_async_data(req)) 1363 io_netmsg_iovec_free(io); 1364 if (zc->notif) { 1365 io_notif_flush(zc->notif); 1366 zc->notif = NULL; 1367 } 1368 } 1369 1370 #define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF) 1371 #define IO_ZC_FLAGS_VALID (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE | \ 1372 IORING_SEND_VECTORIZED) 1373 1374 int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1375 { 1376 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 1377 struct io_ring_ctx *ctx = req->ctx; 1378 struct io_async_msghdr *iomsg; 1379 struct io_kiocb *notif; 1380 u64 user_data; 1381 int ret; 1382 1383 zc->done_io = 0; 1384 1385 if (unlikely(READ_ONCE(sqe->__pad2[0]))) 1386 return -EINVAL; 1387 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */ 1388 if (req->flags & REQ_F_CQE_SKIP) 1389 return -EINVAL; 1390 1391 notif = zc->notif = io_alloc_notif(ctx); 1392 if (!notif) 1393 return -ENOMEM; 1394 user_data = READ_ONCE(sqe->addr3); 1395 if (!user_data) 1396 user_data = req->cqe.user_data; 1397 1398 notif->cqe.user_data = user_data; 1399 notif->cqe.res = 0; 1400 notif->cqe.flags = IORING_CQE_F_NOTIF; 1401 req->flags |= REQ_F_NEED_CLEANUP | REQ_F_POLL_NO_LAZY; 1402 1403 zc->flags = READ_ONCE(sqe->ioprio); 1404 if (unlikely(zc->flags & ~IO_ZC_FLAGS_COMMON)) { 1405 if (zc->flags & ~IO_ZC_FLAGS_VALID) 1406 return -EINVAL; 1407 if (zc->flags & IORING_SEND_ZC_REPORT_USAGE) { 1408 struct io_notif_data *nd = io_notif_to_data(notif); 1409 1410 nd->zc_report = true; 1411 nd->zc_used = false; 1412 nd->zc_copied = false; 1413 } 1414 } 1415 1416 zc->len = READ_ONCE(sqe->len); 1417 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL | MSG_ZEROCOPY; 1418 req->buf_index = READ_ONCE(sqe->buf_index); 1419 if (zc->msg_flags & MSG_DONTWAIT) 1420 req->flags |= REQ_F_NOWAIT; 1421 1422 if (io_is_compat(ctx)) 1423 zc->msg_flags |= MSG_CMSG_COMPAT; 1424 1425 iomsg = io_msg_alloc_async(req); 1426 if (unlikely(!iomsg)) 1427 return -ENOMEM; 1428 1429 if (req->opcode == IORING_OP_SEND_ZC) { 1430 ret = io_send_setup(req, sqe); 1431 } else { 1432 if (unlikely(sqe->addr2 || sqe->file_index)) 1433 return -EINVAL; 1434 ret = io_sendmsg_setup(req, sqe); 1435 } 1436 if (unlikely(ret)) 1437 return ret; 1438 1439 if (!(zc->flags & IORING_RECVSEND_FIXED_BUF)) { 1440 iomsg->msg.sg_from_iter = io_sg_from_iter_iovec; 1441 return io_notif_account_mem(zc->notif, iomsg->msg.msg_iter.count); 1442 } 1443 iomsg->msg.sg_from_iter = io_sg_from_iter; 1444 return 0; 1445 } 1446 1447 static int io_sg_from_iter_iovec(struct sk_buff *skb, 1448 struct iov_iter *from, size_t length) 1449 { 1450 skb_zcopy_downgrade_managed(skb); 1451 return zerocopy_fill_skb_from_iter(skb, from, length); 1452 } 1453 1454 static int io_sg_from_iter(struct sk_buff *skb, 1455 struct iov_iter *from, size_t length) 1456 { 1457 struct skb_shared_info *shinfo = skb_shinfo(skb); 1458 int frag = shinfo->nr_frags; 1459 int ret = 0; 1460 struct bvec_iter bi; 1461 ssize_t copied = 0; 1462 unsigned long truesize = 0; 1463 1464 if (!frag) 1465 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS; 1466 else if (unlikely(!skb_zcopy_managed(skb))) 1467 return zerocopy_fill_skb_from_iter(skb, from, length); 1468 1469 bi.bi_size = min(from->count, length); 1470 bi.bi_offset = from->iov_offset; 1471 bi.bi_idx = 0; 1472 1473 while (bi.bi_size && frag < MAX_SKB_FRAGS) { 1474 struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi); 1475 1476 copied += v.bv_len; 1477 truesize += PAGE_ALIGN(v.bv_len + v.bv_offset); 1478 __skb_fill_page_desc_noacc(shinfo, frag++, v.bv_page, 1479 v.bv_offset, v.bv_len); 1480 bvec_iter_advance_single(from->bvec, &bi, v.bv_len); 1481 } 1482 if (bi.bi_size) 1483 ret = -EMSGSIZE; 1484 1485 shinfo->nr_frags = frag; 1486 from->bvec += bi.bi_idx; 1487 from->nr_segs -= bi.bi_idx; 1488 from->count -= copied; 1489 from->iov_offset = bi.bi_offset; 1490 1491 skb->data_len += copied; 1492 skb->len += copied; 1493 skb->truesize += truesize; 1494 return ret; 1495 } 1496 1497 static int io_send_zc_import(struct io_kiocb *req, 1498 struct io_async_msghdr *kmsg, 1499 unsigned int issue_flags) 1500 { 1501 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1502 struct io_kiocb *notif = sr->notif; 1503 int ret; 1504 1505 WARN_ON_ONCE(!(sr->flags & IORING_RECVSEND_FIXED_BUF)); 1506 1507 notif->buf_index = req->buf_index; 1508 1509 if (!(sr->flags & IORING_SEND_VECTORIZED)) { 1510 ret = io_import_reg_buf(notif, &kmsg->msg.msg_iter, 1511 (u64)(uintptr_t)sr->buf, sr->len, 1512 ITER_SOURCE, issue_flags); 1513 } else { 1514 unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs; 1515 1516 ret = io_import_reg_vec(ITER_SOURCE, &kmsg->msg.msg_iter, 1517 notif, &kmsg->vec, uvec_segs, 1518 issue_flags); 1519 } 1520 1521 if (unlikely(ret)) 1522 return ret; 1523 req->flags &= ~REQ_F_IMPORT_BUFFER; 1524 return 0; 1525 } 1526 1527 int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) 1528 { 1529 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1530 struct io_async_msghdr *kmsg = req->async_data; 1531 struct socket *sock; 1532 unsigned msg_flags; 1533 int ret, min_ret = 0; 1534 1535 sock = sock_from_file(req->file); 1536 if (unlikely(!sock)) 1537 return -ENOTSOCK; 1538 if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags)) 1539 return -EOPNOTSUPP; 1540 if (!(req->flags & REQ_F_POLLED) && 1541 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1542 return -EAGAIN; 1543 1544 if (req->flags & REQ_F_IMPORT_BUFFER) { 1545 ret = io_send_zc_import(req, kmsg, issue_flags); 1546 if (unlikely(ret)) 1547 return ret; 1548 } 1549 1550 msg_flags = sr->msg_flags; 1551 if (issue_flags & IO_URING_F_NONBLOCK) 1552 msg_flags |= MSG_DONTWAIT; 1553 if (msg_flags & MSG_WAITALL) 1554 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1555 1556 kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg; 1557 1558 if (req->opcode == IORING_OP_SEND_ZC) { 1559 msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS; 1560 kmsg->msg.msg_flags = msg_flags; 1561 ret = sock_sendmsg(sock, &kmsg->msg); 1562 } else { 1563 kmsg->msg.msg_control_user = sr->msg_control; 1564 ret = __sys_sendmsg_sock(sock, &kmsg->msg, msg_flags); 1565 } 1566 1567 if (unlikely(ret < min_ret)) { 1568 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1569 return -EAGAIN; 1570 1571 if (ret > 0 && io_net_retry(sock, sr->msg_flags)) { 1572 sr->done_io += ret; 1573 return -EAGAIN; 1574 } 1575 if (ret == -ERESTARTSYS) 1576 ret = -EINTR; 1577 req_set_fail(req); 1578 } 1579 1580 if (ret >= 0) 1581 ret += sr->done_io; 1582 else if (sr->done_io) 1583 ret = sr->done_io; 1584 1585 /* 1586 * If we're in io-wq we can't rely on tw ordering guarantees, defer 1587 * flushing notif to io_send_zc_cleanup() 1588 */ 1589 if (!(issue_flags & IO_URING_F_UNLOCKED)) { 1590 io_notif_flush(sr->notif); 1591 sr->notif = NULL; 1592 io_req_msg_cleanup(req, 0); 1593 } 1594 io_req_set_res(req, ret, IORING_CQE_F_MORE); 1595 return IOU_COMPLETE; 1596 } 1597 1598 void io_sendrecv_fail(struct io_kiocb *req) 1599 { 1600 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1601 1602 if (sr->done_io) 1603 req->cqe.res = sr->done_io; 1604 1605 if ((req->flags & REQ_F_NEED_CLEANUP) && 1606 (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC)) 1607 req->cqe.flags |= IORING_CQE_F_MORE; 1608 } 1609 1610 #define ACCEPT_FLAGS (IORING_ACCEPT_MULTISHOT | IORING_ACCEPT_DONTWAIT | \ 1611 IORING_ACCEPT_POLL_FIRST) 1612 1613 int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1614 { 1615 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1616 1617 if (sqe->len || sqe->buf_index) 1618 return -EINVAL; 1619 1620 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1621 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 1622 accept->flags = READ_ONCE(sqe->accept_flags); 1623 accept->nofile = rlimit(RLIMIT_NOFILE); 1624 accept->iou_flags = READ_ONCE(sqe->ioprio); 1625 if (accept->iou_flags & ~ACCEPT_FLAGS) 1626 return -EINVAL; 1627 1628 accept->file_slot = READ_ONCE(sqe->file_index); 1629 if (accept->file_slot) { 1630 if (accept->flags & SOCK_CLOEXEC) 1631 return -EINVAL; 1632 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT && 1633 accept->file_slot != IORING_FILE_INDEX_ALLOC) 1634 return -EINVAL; 1635 } 1636 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1637 return -EINVAL; 1638 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK)) 1639 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK; 1640 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT) 1641 req->flags |= REQ_F_APOLL_MULTISHOT; 1642 if (accept->iou_flags & IORING_ACCEPT_DONTWAIT) 1643 req->flags |= REQ_F_NOWAIT; 1644 return 0; 1645 } 1646 1647 int io_accept(struct io_kiocb *req, unsigned int issue_flags) 1648 { 1649 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1650 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1651 bool fixed = !!accept->file_slot; 1652 struct proto_accept_arg arg = { 1653 .flags = force_nonblock ? O_NONBLOCK : 0, 1654 }; 1655 struct file *file; 1656 unsigned cflags; 1657 int ret, fd; 1658 1659 if (!(req->flags & REQ_F_POLLED) && 1660 accept->iou_flags & IORING_ACCEPT_POLL_FIRST) 1661 return -EAGAIN; 1662 1663 retry: 1664 if (!fixed) { 1665 fd = __get_unused_fd_flags(accept->flags, accept->nofile); 1666 if (unlikely(fd < 0)) 1667 return fd; 1668 } 1669 arg.err = 0; 1670 arg.is_empty = -1; 1671 file = do_accept(req->file, &arg, accept->addr, accept->addr_len, 1672 accept->flags); 1673 if (IS_ERR(file)) { 1674 if (!fixed) 1675 put_unused_fd(fd); 1676 ret = PTR_ERR(file); 1677 if (ret == -EAGAIN && force_nonblock && 1678 !(accept->iou_flags & IORING_ACCEPT_DONTWAIT)) 1679 return IOU_RETRY; 1680 1681 if (ret == -ERESTARTSYS) 1682 ret = -EINTR; 1683 } else if (!fixed) { 1684 fd_install(fd, file); 1685 ret = fd; 1686 } else { 1687 ret = io_fixed_fd_install(req, issue_flags, file, 1688 accept->file_slot); 1689 } 1690 1691 cflags = 0; 1692 if (!arg.is_empty) 1693 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 1694 1695 if (ret >= 0 && (req->flags & REQ_F_APOLL_MULTISHOT) && 1696 io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) { 1697 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || arg.is_empty == -1) 1698 goto retry; 1699 return IOU_RETRY; 1700 } 1701 1702 io_req_set_res(req, ret, cflags); 1703 if (ret < 0) 1704 req_set_fail(req); 1705 return IOU_COMPLETE; 1706 } 1707 1708 void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req) 1709 { 1710 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1711 1712 bctx->socket.family = sock->domain; 1713 bctx->socket.type = sock->type; 1714 bctx->socket.protocol = sock->protocol; 1715 } 1716 1717 void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req) 1718 { 1719 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1720 struct sockaddr_storage *ss = req->async_data; 1721 1722 /* 1723 * move_addr_to_kernel() skips the copy for addr_len == 0, so 1724 * iomsg->addr may hold stale data from a prior CONNECT. Bail 1725 * unless addr_len covers the family discriminator. 1726 */ 1727 if (conn->addr_len < (int)sizeof(sa_family_t)) 1728 return; 1729 1730 bctx->connect.family = ss->ss_family; 1731 switch (ss->ss_family) { 1732 case AF_INET: { 1733 struct sockaddr_in *sin = (struct sockaddr_in *)ss; 1734 1735 if (conn->addr_len < (int)sizeof(*sin)) 1736 break; 1737 bctx->connect.port = sin->sin_port; 1738 bctx->connect.v4_addr = sin->sin_addr.s_addr; 1739 break; 1740 } 1741 case AF_INET6: { 1742 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss; 1743 1744 if (conn->addr_len < (int)sizeof(*sin6)) 1745 break; 1746 bctx->connect.port = sin6->sin6_port; 1747 memcpy(bctx->connect.v6_addr, &sin6->sin6_addr, 1748 sizeof(bctx->connect.v6_addr)); 1749 break; 1750 } 1751 default: 1752 /* family is set; per-family fields stay zero - family-only filtering */ 1753 break; 1754 } 1755 } 1756 1757 int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1758 { 1759 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1760 1761 if (sqe->addr || sqe->rw_flags || sqe->buf_index) 1762 return -EINVAL; 1763 1764 sock->domain = READ_ONCE(sqe->fd); 1765 sock->type = READ_ONCE(sqe->off); 1766 sock->protocol = READ_ONCE(sqe->len); 1767 sock->file_slot = READ_ONCE(sqe->file_index); 1768 sock->nofile = rlimit(RLIMIT_NOFILE); 1769 1770 sock->flags = sock->type & ~SOCK_TYPE_MASK; 1771 if (sock->file_slot && (sock->flags & SOCK_CLOEXEC)) 1772 return -EINVAL; 1773 if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1774 return -EINVAL; 1775 return 0; 1776 } 1777 1778 int io_socket(struct io_kiocb *req, unsigned int issue_flags) 1779 { 1780 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1781 bool fixed = !!sock->file_slot; 1782 struct file *file; 1783 int ret, fd; 1784 1785 if (!fixed) { 1786 fd = __get_unused_fd_flags(sock->flags, sock->nofile); 1787 if (unlikely(fd < 0)) 1788 return fd; 1789 } 1790 file = __sys_socket_file(sock->domain, sock->type, sock->protocol); 1791 if (IS_ERR(file)) { 1792 if (!fixed) 1793 put_unused_fd(fd); 1794 ret = PTR_ERR(file); 1795 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1796 return -EAGAIN; 1797 if (ret == -ERESTARTSYS) 1798 ret = -EINTR; 1799 req_set_fail(req); 1800 } else if (!fixed) { 1801 fd_install(fd, file); 1802 ret = fd; 1803 } else { 1804 ret = io_fixed_fd_install(req, issue_flags, file, 1805 sock->file_slot); 1806 } 1807 io_req_set_res(req, ret, 0); 1808 return IOU_COMPLETE; 1809 } 1810 1811 int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1812 { 1813 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1814 struct sockaddr_storage *addr; 1815 1816 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 1817 return -EINVAL; 1818 1819 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1820 conn->addr_len = READ_ONCE(sqe->addr2); 1821 conn->in_progress = conn->seen_econnaborted = false; 1822 1823 addr = io_uring_alloc_async_data(NULL, req); 1824 if (unlikely(!addr)) 1825 return -ENOMEM; 1826 1827 return move_addr_to_kernel(conn->addr, conn->addr_len, addr); 1828 } 1829 1830 int io_connect(struct io_kiocb *req, unsigned int issue_flags) 1831 { 1832 struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect); 1833 struct sockaddr_storage *addr = req->async_data; 1834 unsigned file_flags; 1835 int ret; 1836 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1837 1838 if (connect->in_progress) { 1839 struct poll_table_struct pt = { ._key = EPOLLERR }; 1840 1841 if (vfs_poll(req->file, &pt) & EPOLLERR) 1842 goto get_sock_err; 1843 } 1844 1845 file_flags = force_nonblock ? O_NONBLOCK : 0; 1846 1847 ret = __sys_connect_file(req->file, addr, connect->addr_len, file_flags); 1848 if ((ret == -EAGAIN || ret == -EINPROGRESS || ret == -ECONNABORTED) 1849 && force_nonblock) { 1850 if (ret == -EINPROGRESS) { 1851 connect->in_progress = true; 1852 } else if (ret == -ECONNABORTED) { 1853 if (connect->seen_econnaborted) 1854 goto out; 1855 connect->seen_econnaborted = true; 1856 } 1857 return -EAGAIN; 1858 } 1859 if (connect->in_progress) { 1860 /* 1861 * At least bluetooth will return -EBADFD on a re-connect 1862 * attempt, and it's (supposedly) also valid to get -EISCONN 1863 * which means the previous result is good. For both of these, 1864 * grab the sock_error() and use that for the completion. 1865 */ 1866 if (ret == -EBADFD || ret == -EISCONN) { 1867 get_sock_err: 1868 ret = sock_error(sock_from_file(req->file)->sk); 1869 } 1870 } 1871 if (ret == -ERESTARTSYS) 1872 ret = -EINTR; 1873 out: 1874 if (ret < 0) 1875 req_set_fail(req); 1876 io_req_set_res(req, ret, 0); 1877 return IOU_COMPLETE; 1878 } 1879 1880 /* 1881 * Check if bind request would potentially end up with filename_create(), 1882 * which in turn end up in mnt_want_write() which will grab the fs 1883 * percpu start write sem. This can trigger a lockdep warning. 1884 */ 1885 static int io_bind_file_create(const struct sockaddr_storage *addr, int addr_len) 1886 { 1887 const struct sockaddr_un *sun; 1888 1889 if (addr->ss_family != AF_UNIX) 1890 return 0; 1891 if (addr_len <= offsetof(struct sockaddr_un, sun_path)) 1892 return 0; 1893 sun = (const struct sockaddr_un *) addr; 1894 return sun->sun_path[0] != '\0'; 1895 } 1896 1897 int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1898 { 1899 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind); 1900 struct sockaddr __user *uaddr; 1901 struct sockaddr_storage *addr; 1902 int ret; 1903 1904 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 1905 return -EINVAL; 1906 1907 uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1908 bind->addr_len = READ_ONCE(sqe->addr2); 1909 1910 addr = io_uring_alloc_async_data(NULL, req); 1911 if (unlikely(!addr)) 1912 return -ENOMEM; 1913 ret = move_addr_to_kernel(uaddr, bind->addr_len, addr); 1914 if (unlikely(ret)) 1915 return ret; 1916 if (io_bind_file_create(addr, bind->addr_len)) 1917 req->flags |= REQ_F_FORCE_ASYNC; 1918 return 0; 1919 } 1920 1921 1922 int io_bind(struct io_kiocb *req, unsigned int issue_flags) 1923 { 1924 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind); 1925 struct sockaddr_storage *addr = req->async_data; 1926 struct socket *sock; 1927 int ret; 1928 1929 sock = sock_from_file(req->file); 1930 if (unlikely(!sock)) 1931 return -ENOTSOCK; 1932 1933 ret = __sys_bind_socket(sock, addr, bind->addr_len); 1934 if (ret < 0) 1935 req_set_fail(req); 1936 io_req_set_res(req, ret, 0); 1937 return 0; 1938 } 1939 1940 int io_listen_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1941 { 1942 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen); 1943 1944 if (sqe->addr || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in || sqe->addr2) 1945 return -EINVAL; 1946 1947 listen->backlog = READ_ONCE(sqe->len); 1948 return 0; 1949 } 1950 1951 int io_listen(struct io_kiocb *req, unsigned int issue_flags) 1952 { 1953 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen); 1954 struct socket *sock; 1955 int ret; 1956 1957 sock = sock_from_file(req->file); 1958 if (unlikely(!sock)) 1959 return -ENOTSOCK; 1960 1961 ret = __sys_listen_socket(sock, listen->backlog); 1962 if (ret < 0) 1963 req_set_fail(req); 1964 io_req_set_res(req, ret, 0); 1965 return 0; 1966 } 1967 1968 void io_netmsg_cache_free(const void *entry) 1969 { 1970 struct io_async_msghdr *kmsg = (struct io_async_msghdr *) entry; 1971 1972 io_vec_free(&kmsg->vec); 1973 kfree(kmsg); 1974 } 1975