1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/file.h> 5 #include <linux/slab.h> 6 #include <linux/net.h> 7 #include <linux/un.h> 8 #include <linux/compat.h> 9 #include <net/compat.h> 10 #include <linux/io_uring.h> 11 12 #include <uapi/linux/io_uring.h> 13 14 #include "filetable.h" 15 #include "io_uring.h" 16 #include "kbuf.h" 17 #include "alloc_cache.h" 18 #include "net.h" 19 #include "notif.h" 20 #include "rsrc.h" 21 #include "zcrx.h" 22 23 struct io_shutdown { 24 struct file *file; 25 int how; 26 }; 27 28 struct io_accept { 29 struct file *file; 30 struct sockaddr __user *addr; 31 int __user *addr_len; 32 int flags; 33 int iou_flags; 34 u32 file_slot; 35 unsigned long nofile; 36 }; 37 38 struct io_socket { 39 struct file *file; 40 int domain; 41 int type; 42 int protocol; 43 int flags; 44 u32 file_slot; 45 unsigned long nofile; 46 }; 47 48 struct io_connect { 49 struct file *file; 50 struct sockaddr __user *addr; 51 int addr_len; 52 bool in_progress; 53 bool seen_econnaborted; 54 }; 55 56 struct io_bind { 57 struct file *file; 58 int addr_len; 59 }; 60 61 struct io_listen { 62 struct file *file; 63 int backlog; 64 }; 65 66 struct io_sr_msg { 67 struct file *file; 68 union { 69 struct compat_msghdr __user *umsg_compat; 70 struct user_msghdr __user *umsg; 71 void __user *buf; 72 }; 73 int len; 74 unsigned done_io; 75 unsigned msg_flags; 76 unsigned nr_multishot_loops; 77 u16 flags; 78 /* initialised and used only by !msg send variants */ 79 u16 buf_group; 80 /* per-invocation mshot limit */ 81 unsigned mshot_len; 82 /* overall mshot byte limit */ 83 unsigned mshot_total_len; 84 void __user *msg_control; 85 /* used only for send zerocopy */ 86 struct io_kiocb *notif; 87 }; 88 89 /* 90 * The UAPI flags are the lower 8 bits, as that's all sqe->ioprio will hold 91 * anyway. Use the upper 8 bits for internal uses. 92 */ 93 enum sr_retry_flags { 94 IORING_RECV_RETRY = (1U << 15), 95 IORING_RECV_PARTIAL_MAP = (1U << 14), 96 IORING_RECV_MSHOT_CAP = (1U << 13), 97 IORING_RECV_MSHOT_LIM = (1U << 12), 98 IORING_RECV_MSHOT_DONE = (1U << 11), 99 100 IORING_RECV_RETRY_CLEAR = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP, 101 IORING_RECV_NO_RETRY = IORING_RECV_RETRY | IORING_RECV_PARTIAL_MAP | 102 IORING_RECV_MSHOT_CAP | IORING_RECV_MSHOT_DONE, 103 }; 104 105 /* 106 * Number of times we'll try and do receives if there's more data. If we 107 * exceed this limit, then add us to the back of the queue and retry from 108 * there. This helps fairness between flooding clients. 109 */ 110 #define MULTISHOT_MAX_RETRY 32 111 112 struct io_recvzc { 113 struct file *file; 114 u16 flags; 115 u32 len; 116 struct io_zcrx_ifq *ifq; 117 }; 118 119 static int io_sg_from_iter_iovec(struct sk_buff *skb, 120 struct iov_iter *from, size_t length); 121 static int io_sg_from_iter(struct sk_buff *skb, 122 struct iov_iter *from, size_t length); 123 124 int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 125 { 126 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 127 128 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags || 129 sqe->buf_index || sqe->splice_fd_in)) 130 return -EINVAL; 131 132 shutdown->how = READ_ONCE(sqe->len); 133 req->flags |= REQ_F_FORCE_ASYNC; 134 return 0; 135 } 136 137 int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) 138 { 139 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 140 struct socket *sock; 141 int ret; 142 143 WARN_ON_ONCE(issue_flags & IO_URING_F_NONBLOCK); 144 145 sock = sock_from_file(req->file); 146 if (unlikely(!sock)) 147 return -ENOTSOCK; 148 149 ret = __sys_shutdown_sock(sock, shutdown->how); 150 io_req_set_res(req, ret, 0); 151 return IOU_COMPLETE; 152 } 153 154 static bool io_net_retry(struct socket *sock, int flags) 155 { 156 if (!(flags & MSG_WAITALL)) 157 return false; 158 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET; 159 } 160 161 static void io_netmsg_iovec_free(struct io_async_msghdr *kmsg) 162 { 163 if (kmsg->vec.iovec) 164 io_vec_free(&kmsg->vec); 165 } 166 167 static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) 168 { 169 struct io_async_msghdr *hdr = req->async_data; 170 171 /* can't recycle, ensure we free the iovec if we have one */ 172 if (unlikely(issue_flags & IO_URING_F_UNLOCKED)) { 173 io_netmsg_iovec_free(hdr); 174 return; 175 } 176 177 /* Let normal cleanup path reap it if we fail adding to the cache */ 178 io_alloc_cache_vec_kasan(&hdr->vec); 179 if (hdr->vec.nr > IO_VEC_CACHE_SOFT_CAP) 180 io_vec_free(&hdr->vec); 181 182 if (io_alloc_cache_put(&req->ctx->netmsg_cache, hdr)) 183 io_req_async_data_clear(req, REQ_F_NEED_CLEANUP); 184 } 185 186 static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req) 187 { 188 struct io_ring_ctx *ctx = req->ctx; 189 struct io_async_msghdr *hdr; 190 191 hdr = io_uring_alloc_async_data(&ctx->netmsg_cache, req); 192 if (!hdr) 193 return NULL; 194 195 /* If the async data was cached, we might have an iov cached inside. */ 196 if (hdr->vec.iovec) 197 req->flags |= REQ_F_NEED_CLEANUP; 198 return hdr; 199 } 200 201 static inline void io_mshot_prep_retry(struct io_kiocb *req, 202 struct io_async_msghdr *kmsg) 203 { 204 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 205 206 req->flags &= ~REQ_F_BL_EMPTY; 207 sr->done_io = 0; 208 sr->flags &= ~IORING_RECV_RETRY_CLEAR; 209 sr->len = sr->mshot_len; 210 } 211 212 static int io_net_import_vec(struct io_kiocb *req, struct io_async_msghdr *iomsg, 213 const struct iovec __user *uiov, unsigned uvec_seg, 214 int ddir) 215 { 216 struct iovec *iov; 217 int ret, nr_segs; 218 219 if (iomsg->vec.iovec) { 220 nr_segs = iomsg->vec.nr; 221 iov = iomsg->vec.iovec; 222 } else { 223 nr_segs = 1; 224 iov = &iomsg->fast_iov; 225 } 226 227 ret = __import_iovec(ddir, uiov, uvec_seg, nr_segs, &iov, 228 &iomsg->msg.msg_iter, io_is_compat(req->ctx)); 229 if (unlikely(ret < 0)) 230 return ret; 231 232 if (iov) { 233 req->flags |= REQ_F_NEED_CLEANUP; 234 io_vec_reset_iovec(&iomsg->vec, iov, iomsg->msg.msg_iter.nr_segs); 235 } 236 return 0; 237 } 238 239 static int io_compat_msg_copy_hdr(struct io_kiocb *req, 240 struct io_async_msghdr *iomsg, 241 struct compat_msghdr *msg, int ddir, 242 struct sockaddr __user **save_addr) 243 { 244 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 245 struct compat_iovec __user *uiov; 246 int ret; 247 248 if (copy_from_user(msg, sr->umsg_compat, sizeof(*msg))) 249 return -EFAULT; 250 251 ret = __get_compat_msghdr(&iomsg->msg, msg, save_addr); 252 if (ret) 253 return ret; 254 255 uiov = compat_ptr(msg->msg_iov); 256 if (req->flags & REQ_F_BUFFER_SELECT) { 257 if (msg->msg_iovlen == 0) { 258 sr->len = 0; 259 } else if (msg->msg_iovlen > 1) { 260 return -EINVAL; 261 } else { 262 struct compat_iovec tmp_iov; 263 264 if (copy_from_user(&tmp_iov, uiov, sizeof(tmp_iov))) 265 return -EFAULT; 266 sr->len = tmp_iov.iov_len; 267 } 268 } 269 return 0; 270 } 271 272 static int io_copy_msghdr_from_user(struct user_msghdr *msg, 273 struct user_msghdr __user *umsg) 274 { 275 if (!user_access_begin(umsg, sizeof(*umsg))) 276 return -EFAULT; 277 unsafe_get_user(msg->msg_name, &umsg->msg_name, ua_end); 278 unsafe_get_user(msg->msg_namelen, &umsg->msg_namelen, ua_end); 279 unsafe_get_user(msg->msg_iov, &umsg->msg_iov, ua_end); 280 unsafe_get_user(msg->msg_iovlen, &umsg->msg_iovlen, ua_end); 281 unsafe_get_user(msg->msg_control, &umsg->msg_control, ua_end); 282 unsafe_get_user(msg->msg_controllen, &umsg->msg_controllen, ua_end); 283 user_access_end(); 284 return 0; 285 ua_end: 286 user_access_end(); 287 return -EFAULT; 288 } 289 290 static int io_msg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg, 291 struct user_msghdr *msg, int ddir, 292 struct sockaddr __user **save_addr) 293 { 294 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 295 struct user_msghdr __user *umsg = sr->umsg; 296 int ret; 297 298 iomsg->msg.msg_name = &iomsg->addr; 299 iomsg->msg.msg_iter.nr_segs = 0; 300 301 if (io_is_compat(req->ctx)) { 302 struct compat_msghdr cmsg; 303 304 ret = io_compat_msg_copy_hdr(req, iomsg, &cmsg, ddir, save_addr); 305 if (ret) 306 return ret; 307 308 memset(msg, 0, sizeof(*msg)); 309 msg->msg_namelen = cmsg.msg_namelen; 310 msg->msg_controllen = cmsg.msg_controllen; 311 msg->msg_iov = compat_ptr(cmsg.msg_iov); 312 msg->msg_iovlen = cmsg.msg_iovlen; 313 return 0; 314 } 315 316 ret = io_copy_msghdr_from_user(msg, umsg); 317 if (unlikely(ret)) 318 return ret; 319 320 msg->msg_flags = 0; 321 322 ret = __copy_msghdr(&iomsg->msg, msg, save_addr); 323 if (ret) 324 return ret; 325 326 if (req->flags & REQ_F_BUFFER_SELECT) { 327 if (msg->msg_iovlen == 0) { 328 sr->len = 0; 329 } else if (msg->msg_iovlen > 1) { 330 return -EINVAL; 331 } else { 332 struct iovec __user *uiov = msg->msg_iov; 333 struct iovec tmp_iov; 334 335 if (copy_from_user(&tmp_iov, uiov, sizeof(tmp_iov))) 336 return -EFAULT; 337 sr->len = tmp_iov.iov_len; 338 } 339 } 340 return 0; 341 } 342 343 void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) 344 { 345 struct io_async_msghdr *io = req->async_data; 346 347 io_netmsg_iovec_free(io); 348 } 349 350 static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) 351 { 352 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 353 struct io_async_msghdr *kmsg = req->async_data; 354 void __user *addr; 355 u16 addr_len; 356 int ret; 357 358 sr->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); 359 360 if (READ_ONCE(sqe->__pad3[0])) 361 return -EINVAL; 362 363 kmsg->msg.msg_name = NULL; 364 kmsg->msg.msg_namelen = 0; 365 kmsg->msg.msg_control = NULL; 366 kmsg->msg.msg_controllen = 0; 367 kmsg->msg.msg_ubuf = NULL; 368 369 addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 370 addr_len = READ_ONCE(sqe->addr_len); 371 if (addr) { 372 ret = move_addr_to_kernel(addr, addr_len, &kmsg->addr); 373 if (unlikely(ret < 0)) 374 return ret; 375 kmsg->msg.msg_name = &kmsg->addr; 376 kmsg->msg.msg_namelen = addr_len; 377 } 378 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 379 if (!(sr->flags & IORING_SEND_VECTORIZED)) { 380 req->flags |= REQ_F_IMPORT_BUFFER; 381 return 0; 382 } 383 384 kmsg->msg.msg_iter.nr_segs = sr->len; 385 return io_prep_reg_iovec(req, &kmsg->vec, sr->buf, sr->len); 386 } 387 if (req->flags & REQ_F_BUFFER_SELECT) 388 return 0; 389 390 if (sr->flags & IORING_SEND_VECTORIZED) 391 return io_net_import_vec(req, kmsg, sr->buf, sr->len, ITER_SOURCE); 392 393 return import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter); 394 } 395 396 static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) 397 { 398 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 399 struct io_async_msghdr *kmsg = req->async_data; 400 struct user_msghdr msg; 401 int ret; 402 403 sr->flags |= IORING_SEND_VECTORIZED; 404 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 405 ret = io_msg_copy_hdr(req, kmsg, &msg, ITER_SOURCE, NULL); 406 if (unlikely(ret)) 407 return ret; 408 /* save msg_control as sys_sendmsg() overwrites it */ 409 sr->msg_control = kmsg->msg.msg_control_user; 410 411 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 412 kmsg->msg.msg_iter.nr_segs = msg.msg_iovlen; 413 return io_prep_reg_iovec(req, &kmsg->vec, msg.msg_iov, 414 msg.msg_iovlen); 415 } 416 if (req->flags & REQ_F_BUFFER_SELECT) 417 return 0; 418 return io_net_import_vec(req, kmsg, msg.msg_iov, msg.msg_iovlen, ITER_SOURCE); 419 } 420 421 #define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE | \ 422 IORING_SEND_VECTORIZED | IORING_RECVSEND_FIXED_BUF) 423 424 int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 425 { 426 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 427 428 sr->done_io = 0; 429 sr->len = READ_ONCE(sqe->len); 430 if (unlikely(sr->len < 0)) 431 return -EINVAL; 432 sr->flags = READ_ONCE(sqe->ioprio); 433 if (sr->flags & ~SENDMSG_FLAGS) 434 return -EINVAL; 435 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 436 /* registered buffer send only supported for plain IORING_OP_SEND */ 437 if (req->opcode != IORING_OP_SEND || 438 (req->flags & REQ_F_BUFFER_SELECT) || 439 (sr->flags & (IORING_RECVSEND_BUNDLE|IORING_SEND_VECTORIZED))) 440 return -EINVAL; 441 req->buf_index = READ_ONCE(sqe->buf_index); 442 } 443 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 444 if (sr->msg_flags & MSG_DONTWAIT) 445 req->flags |= REQ_F_NOWAIT; 446 if (req->flags & REQ_F_BUFFER_SELECT) 447 sr->buf_group = req->buf_index; 448 sr->mshot_total_len = sr->mshot_len = 0; 449 if (sr->flags & IORING_RECVSEND_BUNDLE) { 450 if (req->opcode == IORING_OP_SENDMSG) 451 return -EINVAL; 452 sr->msg_flags |= MSG_WAITALL; 453 req->flags |= REQ_F_MULTISHOT; 454 } 455 456 if (io_is_compat(req->ctx)) 457 sr->msg_flags |= MSG_CMSG_COMPAT; 458 459 if (unlikely(!io_msg_alloc_async(req))) 460 return -ENOMEM; 461 if (req->opcode != IORING_OP_SENDMSG) 462 return io_send_setup(req, sqe); 463 if (unlikely(sqe->addr2 || sqe->file_index)) 464 return -EINVAL; 465 return io_sendmsg_setup(req, sqe); 466 } 467 468 static void io_req_msg_cleanup(struct io_kiocb *req, 469 unsigned int issue_flags) 470 { 471 io_netmsg_recycle(req, issue_flags); 472 } 473 474 /* 475 * For bundle completions, we need to figure out how many segments we consumed. 476 * A bundle could be using a single ITER_UBUF if that's all we mapped, or it 477 * could be using an ITER_IOVEC. If the latter, then if we consumed all of 478 * the segments, then it's a trivial questiont o answer. If we have residual 479 * data in the iter, then loop the segments to figure out how much we 480 * transferred. 481 */ 482 static int io_bundle_nbufs(struct io_async_msghdr *kmsg, int ret) 483 { 484 struct iovec *iov; 485 int nbufs; 486 487 /* no data is always zero segments, and a ubuf is always 1 segment */ 488 if (ret <= 0) 489 return 0; 490 if (iter_is_ubuf(&kmsg->msg.msg_iter)) 491 return 1; 492 493 iov = kmsg->vec.iovec; 494 if (!iov) 495 iov = &kmsg->fast_iov; 496 497 /* if all data was transferred, it's basic pointer math */ 498 if (!iov_iter_count(&kmsg->msg.msg_iter)) 499 return iter_iov(&kmsg->msg.msg_iter) - iov; 500 501 /* short transfer, count segments */ 502 nbufs = 0; 503 do { 504 int this_len = min_t(int, iov[nbufs].iov_len, ret); 505 506 nbufs++; 507 ret -= this_len; 508 } while (ret); 509 510 return nbufs; 511 } 512 513 static int io_net_kbuf_recyle(struct io_kiocb *req, struct io_buffer_list *bl, 514 struct io_async_msghdr *kmsg, int len) 515 { 516 req->flags |= REQ_F_BL_NO_RECYCLE; 517 if (req->flags & REQ_F_BUFFERS_COMMIT) 518 io_kbuf_commit(req, bl, len, io_bundle_nbufs(kmsg, len)); 519 return IOU_RETRY; 520 } 521 522 static inline bool io_send_finish(struct io_kiocb *req, 523 struct io_async_msghdr *kmsg, 524 struct io_br_sel *sel) 525 { 526 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 527 bool bundle_finished = sel->val <= 0; 528 unsigned int cflags; 529 530 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) { 531 cflags = io_put_kbuf(req, sel->val, sel->buf_list); 532 goto finish; 533 } 534 535 cflags = io_put_kbufs(req, sel->val, sel->buf_list, io_bundle_nbufs(kmsg, sel->val)); 536 537 /* 538 * Don't start new bundles if the buffer list is empty, or if the 539 * current operation needed to go through polling to complete. 540 */ 541 if (bundle_finished || req->flags & (REQ_F_BL_EMPTY | REQ_F_POLLED)) 542 goto finish; 543 544 /* 545 * Fill CQE for this receive and see if we should keep trying to 546 * receive from this socket. 547 */ 548 if (io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) { 549 io_mshot_prep_retry(req, kmsg); 550 return false; 551 } 552 553 /* Otherwise stop bundle and use the current result. */ 554 finish: 555 io_req_set_res(req, sel->val, cflags); 556 sel->val = IOU_COMPLETE; 557 return true; 558 } 559 560 int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) 561 { 562 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 563 struct io_async_msghdr *kmsg = req->async_data; 564 struct socket *sock; 565 unsigned flags; 566 int min_ret = 0; 567 int ret; 568 569 sock = sock_from_file(req->file); 570 if (unlikely(!sock)) 571 return -ENOTSOCK; 572 573 if (!(req->flags & REQ_F_POLLED) && 574 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 575 return -EAGAIN; 576 577 flags = sr->msg_flags; 578 if (issue_flags & IO_URING_F_NONBLOCK) 579 flags |= MSG_DONTWAIT; 580 if (flags & MSG_WAITALL) 581 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 582 583 kmsg->msg.msg_control_user = sr->msg_control; 584 585 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 586 587 if (ret < min_ret) { 588 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 589 return -EAGAIN; 590 if (ret > 0 && io_net_retry(sock, flags)) { 591 kmsg->msg.msg_controllen = 0; 592 kmsg->msg.msg_control = NULL; 593 sr->done_io += ret; 594 return -EAGAIN; 595 } 596 if (ret == -ERESTARTSYS) 597 ret = -EINTR; 598 req_set_fail(req); 599 } 600 io_req_msg_cleanup(req, issue_flags); 601 if (ret >= 0) 602 ret += sr->done_io; 603 else if (sr->done_io) 604 ret = sr->done_io; 605 io_req_set_res(req, ret, 0); 606 return IOU_COMPLETE; 607 } 608 609 static int io_send_select_buffer(struct io_kiocb *req, unsigned int issue_flags, 610 struct io_br_sel *sel, struct io_async_msghdr *kmsg) 611 { 612 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 613 struct buf_sel_arg arg = { 614 .iovs = &kmsg->fast_iov, 615 .max_len = min_not_zero(sr->len, INT_MAX), 616 .nr_iovs = 1, 617 .buf_group = sr->buf_group, 618 }; 619 int ret; 620 621 if (kmsg->vec.iovec) { 622 arg.nr_iovs = kmsg->vec.nr; 623 arg.iovs = kmsg->vec.iovec; 624 arg.mode = KBUF_MODE_FREE; 625 } 626 627 if (!(sr->flags & IORING_RECVSEND_BUNDLE)) 628 arg.nr_iovs = 1; 629 else 630 arg.mode |= KBUF_MODE_EXPAND; 631 632 ret = io_buffers_select(req, &arg, sel, issue_flags); 633 if (unlikely(ret < 0)) 634 return ret; 635 636 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) { 637 kmsg->vec.nr = ret; 638 kmsg->vec.iovec = arg.iovs; 639 req->flags |= REQ_F_NEED_CLEANUP; 640 } 641 sr->len = arg.out_len; 642 643 if (ret == 1) { 644 sr->buf = arg.iovs[0].iov_base; 645 ret = import_ubuf(ITER_SOURCE, sr->buf, sr->len, 646 &kmsg->msg.msg_iter); 647 if (unlikely(ret)) 648 return ret; 649 } else { 650 iov_iter_init(&kmsg->msg.msg_iter, ITER_SOURCE, 651 arg.iovs, ret, arg.out_len); 652 } 653 654 return 0; 655 } 656 657 int io_send(struct io_kiocb *req, unsigned int issue_flags) 658 { 659 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 660 struct io_async_msghdr *kmsg = req->async_data; 661 struct io_br_sel sel = { }; 662 struct socket *sock; 663 unsigned flags; 664 int min_ret = 0; 665 int ret; 666 667 sock = sock_from_file(req->file); 668 if (unlikely(!sock)) 669 return -ENOTSOCK; 670 671 if (!(req->flags & REQ_F_POLLED) && 672 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 673 return -EAGAIN; 674 675 if (req->flags & REQ_F_IMPORT_BUFFER) { 676 ret = io_import_reg_buf(req, &kmsg->msg.msg_iter, 677 (u64)(uintptr_t)sr->buf, sr->len, 678 ITER_SOURCE, issue_flags); 679 if (unlikely(ret)) 680 return ret; 681 req->flags &= ~REQ_F_IMPORT_BUFFER; 682 } 683 684 flags = sr->msg_flags; 685 if (issue_flags & IO_URING_F_NONBLOCK) 686 flags |= MSG_DONTWAIT; 687 688 retry_bundle: 689 sel.buf_list = NULL; 690 if (io_do_buffer_select(req)) { 691 ret = io_send_select_buffer(req, issue_flags, &sel, kmsg); 692 if (ret) 693 return ret; 694 } 695 696 /* 697 * If MSG_WAITALL is set, or this is a bundle send, then we need 698 * the full amount. If just bundle is set, if we do a short send 699 * then we complete the bundle sequence rather than continue on. 700 */ 701 if (flags & MSG_WAITALL || sr->flags & IORING_RECVSEND_BUNDLE) 702 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 703 704 flags &= ~MSG_INTERNAL_SENDMSG_FLAGS; 705 kmsg->msg.msg_flags = flags; 706 ret = sock_sendmsg(sock, &kmsg->msg); 707 if (ret < min_ret) { 708 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 709 return -EAGAIN; 710 711 if (ret > 0 && io_net_retry(sock, flags)) { 712 sr->len -= ret; 713 sr->buf += ret; 714 sr->done_io += ret; 715 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); 716 } 717 if (ret == -ERESTARTSYS) 718 ret = -EINTR; 719 req_set_fail(req); 720 } 721 if (ret >= 0) 722 ret += sr->done_io; 723 else if (sr->done_io) 724 ret = sr->done_io; 725 726 sel.val = ret; 727 if (!io_send_finish(req, kmsg, &sel)) 728 goto retry_bundle; 729 730 io_req_msg_cleanup(req, issue_flags); 731 return sel.val; 732 } 733 734 static int io_recvmsg_mshot_prep(struct io_kiocb *req, 735 struct io_async_msghdr *iomsg, 736 int namelen, size_t controllen) 737 { 738 if ((req->flags & (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) == 739 (REQ_F_APOLL_MULTISHOT|REQ_F_BUFFER_SELECT)) { 740 int hdr; 741 742 if (unlikely(namelen < 0)) 743 return -EOVERFLOW; 744 if (check_add_overflow(sizeof(struct io_uring_recvmsg_out), 745 namelen, &hdr)) 746 return -EOVERFLOW; 747 if (check_add_overflow(hdr, controllen, &hdr)) 748 return -EOVERFLOW; 749 750 iomsg->namelen = namelen; 751 iomsg->controllen = controllen; 752 return 0; 753 } 754 755 return 0; 756 } 757 758 static int io_recvmsg_copy_hdr(struct io_kiocb *req, 759 struct io_async_msghdr *iomsg) 760 { 761 struct user_msghdr msg; 762 int ret; 763 764 ret = io_msg_copy_hdr(req, iomsg, &msg, ITER_DEST, &iomsg->uaddr); 765 if (unlikely(ret)) 766 return ret; 767 768 if (!(req->flags & REQ_F_BUFFER_SELECT)) { 769 ret = io_net_import_vec(req, iomsg, msg.msg_iov, msg.msg_iovlen, 770 ITER_DEST); 771 if (unlikely(ret)) 772 return ret; 773 } 774 return io_recvmsg_mshot_prep(req, iomsg, msg.msg_namelen, 775 msg.msg_controllen); 776 } 777 778 static int io_recvmsg_prep_setup(struct io_kiocb *req) 779 { 780 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 781 struct io_async_msghdr *kmsg; 782 783 kmsg = io_msg_alloc_async(req); 784 if (unlikely(!kmsg)) 785 return -ENOMEM; 786 787 if (req->opcode == IORING_OP_RECV) { 788 kmsg->msg.msg_name = NULL; 789 kmsg->msg.msg_namelen = 0; 790 kmsg->msg.msg_inq = 0; 791 kmsg->msg.msg_control = NULL; 792 kmsg->msg.msg_get_inq = 1; 793 kmsg->msg.msg_controllen = 0; 794 kmsg->msg.msg_ubuf = NULL; 795 796 if (req->flags & REQ_F_BUFFER_SELECT) 797 return 0; 798 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 799 req->flags |= REQ_F_IMPORT_BUFFER; 800 return 0; 801 } 802 return import_ubuf(ITER_DEST, sr->buf, sr->len, 803 &kmsg->msg.msg_iter); 804 } 805 806 return io_recvmsg_copy_hdr(req, kmsg); 807 } 808 809 #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT | \ 810 IORING_RECVSEND_BUNDLE | IORING_RECVSEND_FIXED_BUF) 811 812 int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 813 { 814 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 815 816 sr->done_io = 0; 817 818 if (unlikely(sqe->addr2)) 819 return -EINVAL; 820 821 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 822 sr->len = READ_ONCE(sqe->len); 823 if (unlikely(sr->len < 0)) 824 return -EINVAL; 825 sr->flags = READ_ONCE(sqe->ioprio); 826 if (sr->flags & ~RECVMSG_FLAGS) 827 return -EINVAL; 828 if (sr->flags & IORING_RECVSEND_FIXED_BUF) { 829 /* registered buffer recv only for plain IORING_OP_RECV */ 830 if (req->opcode != IORING_OP_RECV || 831 (req->flags & REQ_F_BUFFER_SELECT) || 832 (sr->flags & (IORING_RECV_MULTISHOT | IORING_RECVSEND_BUNDLE))) 833 return -EINVAL; 834 req->buf_index = READ_ONCE(sqe->buf_index); 835 } 836 sr->msg_flags = READ_ONCE(sqe->msg_flags); 837 if (sr->msg_flags & MSG_DONTWAIT) 838 req->flags |= REQ_F_NOWAIT; 839 if (sr->msg_flags & MSG_ERRQUEUE) 840 req->flags |= REQ_F_CLEAR_POLLIN; 841 if (req->flags & REQ_F_BUFFER_SELECT) 842 sr->buf_group = req->buf_index; 843 sr->mshot_total_len = sr->mshot_len = 0; 844 if (sr->flags & IORING_RECV_MULTISHOT) { 845 if (!(req->flags & REQ_F_BUFFER_SELECT)) 846 return -EINVAL; 847 if (sr->msg_flags & MSG_WAITALL) 848 return -EINVAL; 849 if (req->opcode == IORING_OP_RECV) { 850 sr->mshot_len = sr->len; 851 sr->mshot_total_len = READ_ONCE(sqe->optlen); 852 if (sr->mshot_total_len) 853 sr->flags |= IORING_RECV_MSHOT_LIM; 854 } else if (sqe->optlen) { 855 return -EINVAL; 856 } 857 req->flags |= REQ_F_APOLL_MULTISHOT; 858 } else if (sqe->optlen) { 859 return -EINVAL; 860 } 861 862 if (sr->flags & IORING_RECVSEND_BUNDLE) { 863 if (req->opcode == IORING_OP_RECVMSG) 864 return -EINVAL; 865 } 866 867 if (io_is_compat(req->ctx)) 868 sr->msg_flags |= MSG_CMSG_COMPAT; 869 870 sr->nr_multishot_loops = 0; 871 return io_recvmsg_prep_setup(req); 872 } 873 874 /* bits to clear in old and inherit in new cflags on bundle retry */ 875 #define CQE_F_MASK (IORING_CQE_F_SOCK_NONEMPTY|IORING_CQE_F_MORE|\ 876 IORING_CQE_F_BUF_MORE) 877 878 /* 879 * Finishes io_recv and io_recvmsg. 880 * 881 * Returns true if it is actually finished, or false if it should run 882 * again (for multishot). 883 */ 884 static inline bool io_recv_finish(struct io_kiocb *req, 885 struct io_async_msghdr *kmsg, 886 struct io_br_sel *sel, bool mshot_finished, 887 unsigned issue_flags) 888 { 889 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 890 unsigned int cflags = 0; 891 892 if (kmsg->msg.msg_inq > 0) 893 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 894 895 if (sel->val > 0 && sr->flags & IORING_RECV_MSHOT_LIM) { 896 /* 897 * If sr->len hits zero, the limit has been reached. Mark 898 * mshot as finished, and flag MSHOT_DONE as well to prevent 899 * a potential bundle from being retried. 900 */ 901 sr->mshot_total_len -= min_t(int, sel->val, sr->mshot_total_len); 902 if (!sr->mshot_total_len) { 903 sr->flags |= IORING_RECV_MSHOT_DONE; 904 mshot_finished = true; 905 } 906 } 907 908 if (sr->flags & IORING_RECVSEND_BUNDLE) { 909 size_t this_ret = sel->val - sr->done_io; 910 911 cflags |= io_put_kbufs(req, this_ret, sel->buf_list, io_bundle_nbufs(kmsg, this_ret)); 912 if (sr->flags & IORING_RECV_RETRY) 913 cflags = req->cqe.flags | (cflags & CQE_F_MASK); 914 if (sr->mshot_len && sel->val >= sr->mshot_len) 915 sr->flags |= IORING_RECV_MSHOT_CAP; 916 /* bundle with no more immediate buffers, we're done */ 917 if (req->flags & REQ_F_BL_EMPTY) 918 goto finish; 919 /* 920 * If more is available AND it was a full transfer, retry and 921 * append to this one 922 */ 923 if (!(sr->flags & IORING_RECV_NO_RETRY) && 924 kmsg->msg.msg_inq > 1 && this_ret > 0 && 925 !iov_iter_count(&kmsg->msg.msg_iter)) { 926 req->cqe.flags = cflags & ~CQE_F_MASK; 927 sr->len = kmsg->msg.msg_inq; 928 sr->done_io += this_ret; 929 sr->flags |= IORING_RECV_RETRY; 930 return false; 931 } 932 } else { 933 cflags |= io_put_kbuf(req, sel->val, sel->buf_list); 934 } 935 936 /* 937 * Fill CQE for this receive and see if we should keep trying to 938 * receive from this socket. 939 */ 940 if ((req->flags & REQ_F_APOLL_MULTISHOT) && !mshot_finished && 941 io_req_post_cqe(req, sel->val, cflags | IORING_CQE_F_MORE)) { 942 sel->val = IOU_RETRY; 943 io_mshot_prep_retry(req, kmsg); 944 /* Known not-empty or unknown state, retry */ 945 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || kmsg->msg.msg_inq < 0) { 946 if (sr->nr_multishot_loops++ < MULTISHOT_MAX_RETRY && 947 !(sr->flags & IORING_RECV_MSHOT_CAP)) { 948 return false; 949 } 950 /* mshot retries exceeded, force a requeue */ 951 sr->nr_multishot_loops = 0; 952 sr->flags &= ~IORING_RECV_MSHOT_CAP; 953 if (issue_flags & IO_URING_F_MULTISHOT) 954 sel->val = IOU_REQUEUE; 955 } 956 return true; 957 } 958 959 /* Finish the request / stop multishot. */ 960 finish: 961 io_req_set_res(req, sel->val, cflags); 962 sel->val = IOU_COMPLETE; 963 io_req_msg_cleanup(req, issue_flags); 964 return true; 965 } 966 967 static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg, 968 struct io_sr_msg *sr, void __user **buf, 969 size_t *len) 970 { 971 unsigned long ubuf = (unsigned long) *buf; 972 unsigned long hdr; 973 974 hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 975 kmsg->controllen; 976 if (*len < hdr) 977 return -EFAULT; 978 979 if (kmsg->controllen) { 980 unsigned long control = ubuf + hdr - kmsg->controllen; 981 982 kmsg->msg.msg_control_user = (void __user *) control; 983 kmsg->msg.msg_controllen = kmsg->controllen; 984 } 985 986 sr->buf = *buf; /* stash for later copy */ 987 *buf = (void __user *) (ubuf + hdr); 988 kmsg->payloadlen = *len = *len - hdr; 989 return 0; 990 } 991 992 struct io_recvmsg_multishot_hdr { 993 struct io_uring_recvmsg_out msg; 994 struct sockaddr_storage addr; 995 }; 996 997 static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io, 998 struct io_async_msghdr *kmsg, 999 unsigned int flags, bool *finished) 1000 { 1001 int err; 1002 int copy_len; 1003 struct io_recvmsg_multishot_hdr hdr; 1004 1005 if (kmsg->namelen) 1006 kmsg->msg.msg_name = &hdr.addr; 1007 kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT); 1008 kmsg->msg.msg_namelen = 0; 1009 1010 if (sock->file->f_flags & O_NONBLOCK) 1011 flags |= MSG_DONTWAIT; 1012 1013 err = sock_recvmsg(sock, &kmsg->msg, flags); 1014 *finished = err <= 0; 1015 if (err < 0) 1016 return err; 1017 1018 hdr.msg = (struct io_uring_recvmsg_out) { 1019 .controllen = kmsg->controllen - kmsg->msg.msg_controllen, 1020 .flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT 1021 }; 1022 1023 hdr.msg.payloadlen = err; 1024 if (err > kmsg->payloadlen) 1025 err = kmsg->payloadlen; 1026 1027 copy_len = sizeof(struct io_uring_recvmsg_out); 1028 if (kmsg->msg.msg_namelen > kmsg->namelen) 1029 copy_len += kmsg->namelen; 1030 else 1031 copy_len += kmsg->msg.msg_namelen; 1032 1033 /* 1034 * "fromlen shall refer to the value before truncation.." 1035 * 1003.1g 1036 */ 1037 hdr.msg.namelen = kmsg->msg.msg_namelen; 1038 1039 /* ensure that there is no gap between hdr and sockaddr_storage */ 1040 BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr, addr) != 1041 sizeof(struct io_uring_recvmsg_out)); 1042 if (copy_to_user(io->buf, &hdr, copy_len)) { 1043 *finished = true; 1044 return -EFAULT; 1045 } 1046 1047 return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 1048 kmsg->controllen + err; 1049 } 1050 1051 int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 1052 { 1053 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1054 struct io_async_msghdr *kmsg = req->async_data; 1055 struct io_br_sel sel = { }; 1056 struct socket *sock; 1057 unsigned flags; 1058 int ret, min_ret = 0; 1059 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1060 bool mshot_finished = true; 1061 1062 sock = sock_from_file(req->file); 1063 if (unlikely(!sock)) 1064 return -ENOTSOCK; 1065 1066 if (!(req->flags & REQ_F_POLLED) && 1067 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1068 return -EAGAIN; 1069 1070 flags = sr->msg_flags; 1071 if (force_nonblock) 1072 flags |= MSG_DONTWAIT; 1073 1074 retry_multishot: 1075 sel.buf_list = NULL; 1076 if (io_do_buffer_select(req)) { 1077 size_t len = sr->len; 1078 1079 sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); 1080 if (!sel.addr) 1081 return -ENOBUFS; 1082 1083 if (req->flags & REQ_F_APOLL_MULTISHOT) { 1084 ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len); 1085 if (ret) { 1086 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1087 return ret; 1088 } 1089 } 1090 1091 iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len); 1092 } 1093 1094 kmsg->msg.msg_get_inq = 1; 1095 kmsg->msg.msg_inq = -1; 1096 if (req->flags & REQ_F_APOLL_MULTISHOT) { 1097 ret = io_recvmsg_multishot(sock, sr, kmsg, flags, 1098 &mshot_finished); 1099 } else { 1100 /* disable partial retry for recvmsg with cmsg attached */ 1101 if (flags & MSG_WAITALL && !kmsg->msg.msg_controllen) 1102 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1103 1104 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, 1105 kmsg->uaddr, flags); 1106 } 1107 1108 if (ret < min_ret) { 1109 if (ret == -EAGAIN && force_nonblock) { 1110 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1111 return IOU_RETRY; 1112 } 1113 if (ret > 0 && io_net_retry(sock, flags)) { 1114 sr->done_io += ret; 1115 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); 1116 } 1117 if (ret == -ERESTARTSYS) 1118 ret = -EINTR; 1119 req_set_fail(req); 1120 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 1121 req_set_fail(req); 1122 } 1123 1124 if (ret > 0) 1125 ret += sr->done_io; 1126 else if (sr->done_io) 1127 ret = sr->done_io; 1128 else 1129 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1130 1131 sel.val = ret; 1132 if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags)) 1133 goto retry_multishot; 1134 1135 return sel.val; 1136 } 1137 1138 static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg, 1139 struct io_br_sel *sel, unsigned int issue_flags) 1140 { 1141 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1142 int ret; 1143 1144 /* 1145 * If the ring isn't locked, then don't use the peek interface 1146 * to grab multiple buffers as we will lock/unlock between 1147 * this selection and posting the buffers. 1148 */ 1149 if (!(issue_flags & IO_URING_F_UNLOCKED) && 1150 sr->flags & IORING_RECVSEND_BUNDLE) { 1151 struct buf_sel_arg arg = { 1152 .iovs = &kmsg->fast_iov, 1153 .nr_iovs = 1, 1154 .mode = KBUF_MODE_EXPAND, 1155 .buf_group = sr->buf_group, 1156 }; 1157 1158 if (kmsg->vec.iovec) { 1159 arg.nr_iovs = kmsg->vec.nr; 1160 arg.iovs = kmsg->vec.iovec; 1161 arg.mode |= KBUF_MODE_FREE; 1162 } 1163 1164 if (sel->val) 1165 arg.max_len = sel->val; 1166 else if (kmsg->msg.msg_inq > 1) 1167 arg.max_len = min_not_zero(sel->val, (ssize_t) kmsg->msg.msg_inq); 1168 1169 /* if mshot limited, ensure we don't go over */ 1170 if (sr->flags & IORING_RECV_MSHOT_LIM) 1171 arg.max_len = min_not_zero(arg.max_len, sr->mshot_total_len); 1172 ret = io_buffers_peek(req, &arg, sel); 1173 if (unlikely(ret < 0)) 1174 return ret; 1175 1176 if (arg.iovs != &kmsg->fast_iov && arg.iovs != kmsg->vec.iovec) { 1177 kmsg->vec.nr = ret; 1178 kmsg->vec.iovec = arg.iovs; 1179 req->flags |= REQ_F_NEED_CLEANUP; 1180 } 1181 if (arg.partial_map) 1182 sr->flags |= IORING_RECV_PARTIAL_MAP; 1183 1184 /* special case 1 vec, can be a fast path */ 1185 if (ret == 1) { 1186 sr->buf = arg.iovs[0].iov_base; 1187 sr->len = arg.iovs[0].iov_len; 1188 goto map_ubuf; 1189 } 1190 iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret, 1191 arg.out_len); 1192 } else { 1193 size_t len = sel->val; 1194 1195 *sel = io_buffer_select(req, &len, sr->buf_group, issue_flags); 1196 if (!sel->addr) 1197 return -ENOBUFS; 1198 sr->buf = sel->addr; 1199 sr->len = len; 1200 map_ubuf: 1201 ret = import_ubuf(ITER_DEST, sr->buf, sr->len, 1202 &kmsg->msg.msg_iter); 1203 if (unlikely(ret)) 1204 return ret; 1205 } 1206 1207 return 0; 1208 } 1209 1210 int io_recv(struct io_kiocb *req, unsigned int issue_flags) 1211 { 1212 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1213 struct io_async_msghdr *kmsg = req->async_data; 1214 struct io_br_sel sel; 1215 struct socket *sock; 1216 unsigned flags; 1217 int ret, min_ret = 0; 1218 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1219 bool mshot_finished; 1220 1221 sock = sock_from_file(req->file); 1222 if (unlikely(!sock)) 1223 return -ENOTSOCK; 1224 1225 if (!(req->flags & REQ_F_POLLED) && 1226 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1227 return -EAGAIN; 1228 1229 flags = sr->msg_flags; 1230 if (force_nonblock) 1231 flags |= MSG_DONTWAIT; 1232 1233 if (req->flags & REQ_F_IMPORT_BUFFER) { 1234 ret = io_import_reg_buf(req, &kmsg->msg.msg_iter, 1235 (u64)(uintptr_t)sr->buf, sr->len, 1236 ITER_DEST, issue_flags); 1237 if (unlikely(ret)) { 1238 kmsg->msg.msg_inq = -1; 1239 sel.buf_list = NULL; 1240 goto out_free; 1241 } 1242 req->flags &= ~REQ_F_IMPORT_BUFFER; 1243 } 1244 1245 retry_multishot: 1246 sel.buf_list = NULL; 1247 if (io_do_buffer_select(req)) { 1248 sel.val = sr->len; 1249 ret = io_recv_buf_select(req, kmsg, &sel, issue_flags); 1250 if (unlikely(ret < 0)) { 1251 kmsg->msg.msg_inq = -1; 1252 goto out_free; 1253 } 1254 sr->buf = NULL; 1255 } 1256 1257 kmsg->msg.msg_flags = 0; 1258 kmsg->msg.msg_inq = -1; 1259 1260 if (flags & MSG_WAITALL) 1261 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1262 1263 ret = sock_recvmsg(sock, &kmsg->msg, flags); 1264 if (ret < min_ret) { 1265 if (ret == -EAGAIN && force_nonblock) { 1266 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1267 return IOU_RETRY; 1268 } 1269 if (ret > 0 && io_net_retry(sock, flags)) { 1270 sr->len -= ret; 1271 sr->buf += ret; 1272 sr->done_io += ret; 1273 return io_net_kbuf_recyle(req, sel.buf_list, kmsg, ret); 1274 } 1275 if (ret == -ERESTARTSYS) 1276 ret = -EINTR; 1277 req_set_fail(req); 1278 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 1279 out_free: 1280 req_set_fail(req); 1281 } 1282 1283 mshot_finished = ret <= 0; 1284 if (ret > 0) 1285 ret += sr->done_io; 1286 else if (sr->done_io) 1287 ret = sr->done_io; 1288 else 1289 io_kbuf_recycle(req, sel.buf_list, issue_flags); 1290 1291 sel.val = ret; 1292 if (!io_recv_finish(req, kmsg, &sel, mshot_finished, issue_flags)) 1293 goto retry_multishot; 1294 1295 return sel.val; 1296 } 1297 1298 int io_recvzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1299 { 1300 struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc); 1301 unsigned ifq_idx; 1302 1303 if (unlikely(sqe->addr2 || sqe->addr || sqe->addr3)) 1304 return -EINVAL; 1305 1306 ifq_idx = READ_ONCE(sqe->zcrx_ifq_idx); 1307 zc->ifq = xa_load(&req->ctx->zcrx_ctxs, ifq_idx); 1308 if (!zc->ifq) 1309 return -EINVAL; 1310 1311 zc->len = READ_ONCE(sqe->len); 1312 zc->flags = READ_ONCE(sqe->ioprio); 1313 if (READ_ONCE(sqe->msg_flags)) 1314 return -EINVAL; 1315 if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT)) 1316 return -EINVAL; 1317 /* multishot required */ 1318 if (!(zc->flags & IORING_RECV_MULTISHOT)) 1319 return -EINVAL; 1320 /* All data completions are posted as aux CQEs. */ 1321 req->flags |= REQ_F_APOLL_MULTISHOT; 1322 1323 return 0; 1324 } 1325 1326 int io_recvzc(struct io_kiocb *req, unsigned int issue_flags) 1327 { 1328 struct io_recvzc *zc = io_kiocb_to_cmd(req, struct io_recvzc); 1329 struct socket *sock; 1330 unsigned int len; 1331 int ret; 1332 1333 sock = sock_from_file(req->file); 1334 if (unlikely(!sock)) 1335 return -ENOTSOCK; 1336 1337 if (!(req->flags & REQ_F_POLLED) && 1338 (zc->flags & IORING_RECVSEND_POLL_FIRST)) 1339 return -EAGAIN; 1340 1341 len = zc->len; 1342 ret = io_zcrx_recv(req, zc->ifq, sock, 0, issue_flags, &zc->len); 1343 if (len && zc->len == 0) { 1344 io_req_set_res(req, 0, 0); 1345 1346 return IOU_COMPLETE; 1347 } 1348 if (unlikely(ret <= 0) && ret != -EAGAIN) { 1349 if (ret == -ERESTARTSYS) 1350 ret = -EINTR; 1351 if (ret == IOU_REQUEUE) 1352 return IOU_REQUEUE; 1353 1354 req_set_fail(req); 1355 io_req_set_res(req, ret, 0); 1356 return IOU_COMPLETE; 1357 } 1358 return IOU_RETRY; 1359 } 1360 1361 void io_send_zc_cleanup(struct io_kiocb *req) 1362 { 1363 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 1364 struct io_async_msghdr *io = req->async_data; 1365 1366 if (req_has_async_data(req)) 1367 io_netmsg_iovec_free(io); 1368 if (zc->notif) { 1369 io_notif_flush(zc->notif); 1370 zc->notif = NULL; 1371 } 1372 } 1373 1374 #define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF) 1375 #define IO_ZC_FLAGS_VALID (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE | \ 1376 IORING_SEND_VECTORIZED) 1377 1378 int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1379 { 1380 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 1381 struct io_ring_ctx *ctx = req->ctx; 1382 struct io_async_msghdr *iomsg; 1383 struct io_kiocb *notif; 1384 u64 user_data; 1385 int ret; 1386 1387 zc->done_io = 0; 1388 1389 if (unlikely(READ_ONCE(sqe->__pad2[0]))) 1390 return -EINVAL; 1391 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */ 1392 if (req->flags & REQ_F_CQE_SKIP) 1393 return -EINVAL; 1394 1395 notif = zc->notif = io_alloc_notif(ctx); 1396 if (!notif) 1397 return -ENOMEM; 1398 user_data = READ_ONCE(sqe->addr3); 1399 if (!user_data) 1400 user_data = req->cqe.user_data; 1401 1402 notif->cqe.user_data = user_data; 1403 notif->cqe.res = 0; 1404 notif->cqe.flags = IORING_CQE_F_NOTIF; 1405 req->flags |= REQ_F_NEED_CLEANUP | REQ_F_POLL_NO_LAZY; 1406 1407 zc->flags = READ_ONCE(sqe->ioprio); 1408 if (unlikely(zc->flags & ~IO_ZC_FLAGS_COMMON)) { 1409 if (zc->flags & ~IO_ZC_FLAGS_VALID) 1410 return -EINVAL; 1411 if (zc->flags & IORING_SEND_ZC_REPORT_USAGE) { 1412 struct io_notif_data *nd = io_notif_to_data(notif); 1413 1414 nd->zc_report = true; 1415 nd->zc_used = false; 1416 nd->zc_copied = false; 1417 } 1418 } 1419 1420 zc->len = READ_ONCE(sqe->len); 1421 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL | MSG_ZEROCOPY; 1422 req->buf_index = READ_ONCE(sqe->buf_index); 1423 if (zc->msg_flags & MSG_DONTWAIT) 1424 req->flags |= REQ_F_NOWAIT; 1425 1426 if (io_is_compat(ctx)) 1427 zc->msg_flags |= MSG_CMSG_COMPAT; 1428 1429 iomsg = io_msg_alloc_async(req); 1430 if (unlikely(!iomsg)) 1431 return -ENOMEM; 1432 1433 if (req->opcode == IORING_OP_SEND_ZC) { 1434 ret = io_send_setup(req, sqe); 1435 } else { 1436 if (unlikely(sqe->addr2 || sqe->file_index)) 1437 return -EINVAL; 1438 ret = io_sendmsg_setup(req, sqe); 1439 } 1440 if (unlikely(ret)) 1441 return ret; 1442 1443 if (!(zc->flags & IORING_RECVSEND_FIXED_BUF)) { 1444 iomsg->msg.sg_from_iter = io_sg_from_iter_iovec; 1445 return io_notif_account_mem(zc->notif, iomsg->msg.msg_iter.count); 1446 } 1447 iomsg->msg.sg_from_iter = io_sg_from_iter; 1448 return 0; 1449 } 1450 1451 static int io_sg_from_iter_iovec(struct sk_buff *skb, 1452 struct iov_iter *from, size_t length) 1453 { 1454 skb_zcopy_downgrade_managed(skb); 1455 return zerocopy_fill_skb_from_iter(skb, from, length); 1456 } 1457 1458 static int io_sg_from_iter(struct sk_buff *skb, 1459 struct iov_iter *from, size_t length) 1460 { 1461 struct skb_shared_info *shinfo = skb_shinfo(skb); 1462 int frag = shinfo->nr_frags; 1463 int ret = 0; 1464 struct bvec_iter bi; 1465 ssize_t copied = 0; 1466 unsigned long truesize = 0; 1467 1468 if (!frag) 1469 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS; 1470 else if (unlikely(!skb_zcopy_managed(skb))) 1471 return zerocopy_fill_skb_from_iter(skb, from, length); 1472 1473 bi.bi_size = min(from->count, length); 1474 bi.bi_bvec_done = from->iov_offset; 1475 bi.bi_idx = 0; 1476 1477 while (bi.bi_size && frag < MAX_SKB_FRAGS) { 1478 struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi); 1479 1480 copied += v.bv_len; 1481 truesize += PAGE_ALIGN(v.bv_len + v.bv_offset); 1482 __skb_fill_page_desc_noacc(shinfo, frag++, v.bv_page, 1483 v.bv_offset, v.bv_len); 1484 bvec_iter_advance_single(from->bvec, &bi, v.bv_len); 1485 } 1486 if (bi.bi_size) 1487 ret = -EMSGSIZE; 1488 1489 shinfo->nr_frags = frag; 1490 from->bvec += bi.bi_idx; 1491 from->nr_segs -= bi.bi_idx; 1492 from->count -= copied; 1493 from->iov_offset = bi.bi_bvec_done; 1494 1495 skb->data_len += copied; 1496 skb->len += copied; 1497 skb->truesize += truesize; 1498 return ret; 1499 } 1500 1501 static int io_send_zc_import(struct io_kiocb *req, 1502 struct io_async_msghdr *kmsg, 1503 unsigned int issue_flags) 1504 { 1505 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1506 struct io_kiocb *notif = sr->notif; 1507 int ret; 1508 1509 WARN_ON_ONCE(!(sr->flags & IORING_RECVSEND_FIXED_BUF)); 1510 1511 notif->buf_index = req->buf_index; 1512 1513 if (!(sr->flags & IORING_SEND_VECTORIZED)) { 1514 ret = io_import_reg_buf(notif, &kmsg->msg.msg_iter, 1515 (u64)(uintptr_t)sr->buf, sr->len, 1516 ITER_SOURCE, issue_flags); 1517 } else { 1518 unsigned uvec_segs = kmsg->msg.msg_iter.nr_segs; 1519 1520 ret = io_import_reg_vec(ITER_SOURCE, &kmsg->msg.msg_iter, 1521 notif, &kmsg->vec, uvec_segs, 1522 issue_flags); 1523 } 1524 1525 if (unlikely(ret)) 1526 return ret; 1527 req->flags &= ~REQ_F_IMPORT_BUFFER; 1528 return 0; 1529 } 1530 1531 int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) 1532 { 1533 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1534 struct io_async_msghdr *kmsg = req->async_data; 1535 struct socket *sock; 1536 unsigned msg_flags; 1537 int ret, min_ret = 0; 1538 1539 sock = sock_from_file(req->file); 1540 if (unlikely(!sock)) 1541 return -ENOTSOCK; 1542 if (!test_bit(SOCK_SUPPORT_ZC, &sock->flags)) 1543 return -EOPNOTSUPP; 1544 if (!(req->flags & REQ_F_POLLED) && 1545 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1546 return -EAGAIN; 1547 1548 if (req->flags & REQ_F_IMPORT_BUFFER) { 1549 ret = io_send_zc_import(req, kmsg, issue_flags); 1550 if (unlikely(ret)) 1551 return ret; 1552 } 1553 1554 msg_flags = sr->msg_flags; 1555 if (issue_flags & IO_URING_F_NONBLOCK) 1556 msg_flags |= MSG_DONTWAIT; 1557 if (msg_flags & MSG_WAITALL) 1558 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1559 1560 kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg; 1561 1562 if (req->opcode == IORING_OP_SEND_ZC) { 1563 msg_flags &= ~MSG_INTERNAL_SENDMSG_FLAGS; 1564 kmsg->msg.msg_flags = msg_flags; 1565 ret = sock_sendmsg(sock, &kmsg->msg); 1566 } else { 1567 kmsg->msg.msg_control_user = sr->msg_control; 1568 ret = __sys_sendmsg_sock(sock, &kmsg->msg, msg_flags); 1569 } 1570 1571 if (unlikely(ret < min_ret)) { 1572 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1573 return -EAGAIN; 1574 1575 if (ret > 0 && io_net_retry(sock, sr->msg_flags)) { 1576 sr->done_io += ret; 1577 return -EAGAIN; 1578 } 1579 if (ret == -ERESTARTSYS) 1580 ret = -EINTR; 1581 req_set_fail(req); 1582 } 1583 1584 if (ret >= 0) 1585 ret += sr->done_io; 1586 else if (sr->done_io) 1587 ret = sr->done_io; 1588 1589 /* 1590 * If we're in io-wq we can't rely on tw ordering guarantees, defer 1591 * flushing notif to io_send_zc_cleanup() 1592 */ 1593 if (!(issue_flags & IO_URING_F_UNLOCKED)) { 1594 io_notif_flush(sr->notif); 1595 sr->notif = NULL; 1596 io_req_msg_cleanup(req, 0); 1597 } 1598 io_req_set_res(req, ret, IORING_CQE_F_MORE); 1599 return IOU_COMPLETE; 1600 } 1601 1602 void io_sendrecv_fail(struct io_kiocb *req) 1603 { 1604 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1605 1606 if (sr->done_io) 1607 req->cqe.res = sr->done_io; 1608 1609 if ((req->flags & REQ_F_NEED_CLEANUP) && 1610 (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC)) 1611 req->cqe.flags |= IORING_CQE_F_MORE; 1612 } 1613 1614 #define ACCEPT_FLAGS (IORING_ACCEPT_MULTISHOT | IORING_ACCEPT_DONTWAIT | \ 1615 IORING_ACCEPT_POLL_FIRST) 1616 1617 int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1618 { 1619 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1620 1621 if (sqe->len || sqe->buf_index) 1622 return -EINVAL; 1623 1624 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1625 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 1626 accept->flags = READ_ONCE(sqe->accept_flags); 1627 accept->nofile = rlimit(RLIMIT_NOFILE); 1628 accept->iou_flags = READ_ONCE(sqe->ioprio); 1629 if (accept->iou_flags & ~ACCEPT_FLAGS) 1630 return -EINVAL; 1631 1632 accept->file_slot = READ_ONCE(sqe->file_index); 1633 if (accept->file_slot) { 1634 if (accept->flags & SOCK_CLOEXEC) 1635 return -EINVAL; 1636 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT && 1637 accept->file_slot != IORING_FILE_INDEX_ALLOC) 1638 return -EINVAL; 1639 } 1640 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1641 return -EINVAL; 1642 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK)) 1643 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK; 1644 if (accept->iou_flags & IORING_ACCEPT_MULTISHOT) 1645 req->flags |= REQ_F_APOLL_MULTISHOT; 1646 if (accept->iou_flags & IORING_ACCEPT_DONTWAIT) 1647 req->flags |= REQ_F_NOWAIT; 1648 return 0; 1649 } 1650 1651 int io_accept(struct io_kiocb *req, unsigned int issue_flags) 1652 { 1653 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1654 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1655 bool fixed = !!accept->file_slot; 1656 struct proto_accept_arg arg = { 1657 .flags = force_nonblock ? O_NONBLOCK : 0, 1658 }; 1659 struct file *file; 1660 unsigned cflags; 1661 int ret, fd; 1662 1663 if (!(req->flags & REQ_F_POLLED) && 1664 accept->iou_flags & IORING_ACCEPT_POLL_FIRST) 1665 return -EAGAIN; 1666 1667 retry: 1668 if (!fixed) { 1669 fd = __get_unused_fd_flags(accept->flags, accept->nofile); 1670 if (unlikely(fd < 0)) 1671 return fd; 1672 } 1673 arg.err = 0; 1674 arg.is_empty = -1; 1675 file = do_accept(req->file, &arg, accept->addr, accept->addr_len, 1676 accept->flags); 1677 if (IS_ERR(file)) { 1678 if (!fixed) 1679 put_unused_fd(fd); 1680 ret = PTR_ERR(file); 1681 if (ret == -EAGAIN && force_nonblock && 1682 !(accept->iou_flags & IORING_ACCEPT_DONTWAIT)) 1683 return IOU_RETRY; 1684 1685 if (ret == -ERESTARTSYS) 1686 ret = -EINTR; 1687 } else if (!fixed) { 1688 fd_install(fd, file); 1689 ret = fd; 1690 } else { 1691 ret = io_fixed_fd_install(req, issue_flags, file, 1692 accept->file_slot); 1693 } 1694 1695 cflags = 0; 1696 if (!arg.is_empty) 1697 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 1698 1699 if (ret >= 0 && (req->flags & REQ_F_APOLL_MULTISHOT) && 1700 io_req_post_cqe(req, ret, cflags | IORING_CQE_F_MORE)) { 1701 if (cflags & IORING_CQE_F_SOCK_NONEMPTY || arg.is_empty == -1) 1702 goto retry; 1703 return IOU_RETRY; 1704 } 1705 1706 io_req_set_res(req, ret, cflags); 1707 if (ret < 0) 1708 req_set_fail(req); 1709 return IOU_COMPLETE; 1710 } 1711 1712 void io_socket_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req) 1713 { 1714 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1715 1716 bctx->socket.family = sock->domain; 1717 bctx->socket.type = sock->type; 1718 bctx->socket.protocol = sock->protocol; 1719 } 1720 1721 void io_connect_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req) 1722 { 1723 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1724 struct sockaddr_storage *ss = req->async_data; 1725 1726 /* 1727 * move_addr_to_kernel() skips the copy for addr_len == 0, so 1728 * iomsg->addr may hold stale data from a prior CONNECT. Bail 1729 * unless addr_len covers the family discriminator. 1730 */ 1731 if (conn->addr_len < (int)sizeof(sa_family_t)) 1732 return; 1733 1734 bctx->connect.family = ss->ss_family; 1735 switch (ss->ss_family) { 1736 case AF_INET: { 1737 struct sockaddr_in *sin = (struct sockaddr_in *)ss; 1738 1739 if (conn->addr_len < (int)sizeof(*sin)) 1740 break; 1741 bctx->connect.port = sin->sin_port; 1742 bctx->connect.v4_addr = sin->sin_addr.s_addr; 1743 break; 1744 } 1745 case AF_INET6: { 1746 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)ss; 1747 1748 if (conn->addr_len < (int)sizeof(*sin6)) 1749 break; 1750 bctx->connect.port = sin6->sin6_port; 1751 memcpy(bctx->connect.v6_addr, &sin6->sin6_addr, 1752 sizeof(bctx->connect.v6_addr)); 1753 break; 1754 } 1755 default: 1756 /* family is set; per-family fields stay zero - family-only filtering */ 1757 break; 1758 } 1759 } 1760 1761 int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1762 { 1763 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1764 1765 if (sqe->addr || sqe->rw_flags || sqe->buf_index) 1766 return -EINVAL; 1767 1768 sock->domain = READ_ONCE(sqe->fd); 1769 sock->type = READ_ONCE(sqe->off); 1770 sock->protocol = READ_ONCE(sqe->len); 1771 sock->file_slot = READ_ONCE(sqe->file_index); 1772 sock->nofile = rlimit(RLIMIT_NOFILE); 1773 1774 sock->flags = sock->type & ~SOCK_TYPE_MASK; 1775 if (sock->file_slot && (sock->flags & SOCK_CLOEXEC)) 1776 return -EINVAL; 1777 if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1778 return -EINVAL; 1779 return 0; 1780 } 1781 1782 int io_socket(struct io_kiocb *req, unsigned int issue_flags) 1783 { 1784 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1785 bool fixed = !!sock->file_slot; 1786 struct file *file; 1787 int ret, fd; 1788 1789 if (!fixed) { 1790 fd = __get_unused_fd_flags(sock->flags, sock->nofile); 1791 if (unlikely(fd < 0)) 1792 return fd; 1793 } 1794 file = __sys_socket_file(sock->domain, sock->type, sock->protocol); 1795 if (IS_ERR(file)) { 1796 if (!fixed) 1797 put_unused_fd(fd); 1798 ret = PTR_ERR(file); 1799 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1800 return -EAGAIN; 1801 if (ret == -ERESTARTSYS) 1802 ret = -EINTR; 1803 req_set_fail(req); 1804 } else if (!fixed) { 1805 fd_install(fd, file); 1806 ret = fd; 1807 } else { 1808 ret = io_fixed_fd_install(req, issue_flags, file, 1809 sock->file_slot); 1810 } 1811 io_req_set_res(req, ret, 0); 1812 return IOU_COMPLETE; 1813 } 1814 1815 int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1816 { 1817 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1818 struct sockaddr_storage *addr; 1819 1820 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 1821 return -EINVAL; 1822 1823 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1824 conn->addr_len = READ_ONCE(sqe->addr2); 1825 conn->in_progress = conn->seen_econnaborted = false; 1826 1827 addr = io_uring_alloc_async_data(NULL, req); 1828 if (unlikely(!addr)) 1829 return -ENOMEM; 1830 1831 return move_addr_to_kernel(conn->addr, conn->addr_len, addr); 1832 } 1833 1834 int io_connect(struct io_kiocb *req, unsigned int issue_flags) 1835 { 1836 struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect); 1837 struct sockaddr_storage *addr = req->async_data; 1838 unsigned file_flags; 1839 int ret; 1840 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1841 1842 if (connect->in_progress) { 1843 struct poll_table_struct pt = { ._key = EPOLLERR }; 1844 1845 if (vfs_poll(req->file, &pt) & EPOLLERR) 1846 goto get_sock_err; 1847 } 1848 1849 file_flags = force_nonblock ? O_NONBLOCK : 0; 1850 1851 ret = __sys_connect_file(req->file, addr, connect->addr_len, file_flags); 1852 if ((ret == -EAGAIN || ret == -EINPROGRESS || ret == -ECONNABORTED) 1853 && force_nonblock) { 1854 if (ret == -EINPROGRESS) { 1855 connect->in_progress = true; 1856 } else if (ret == -ECONNABORTED) { 1857 if (connect->seen_econnaborted) 1858 goto out; 1859 connect->seen_econnaborted = true; 1860 } 1861 return -EAGAIN; 1862 } 1863 if (connect->in_progress) { 1864 /* 1865 * At least bluetooth will return -EBADFD on a re-connect 1866 * attempt, and it's (supposedly) also valid to get -EISCONN 1867 * which means the previous result is good. For both of these, 1868 * grab the sock_error() and use that for the completion. 1869 */ 1870 if (ret == -EBADFD || ret == -EISCONN) { 1871 get_sock_err: 1872 ret = sock_error(sock_from_file(req->file)->sk); 1873 } 1874 } 1875 if (ret == -ERESTARTSYS) 1876 ret = -EINTR; 1877 out: 1878 if (ret < 0) 1879 req_set_fail(req); 1880 io_req_set_res(req, ret, 0); 1881 return IOU_COMPLETE; 1882 } 1883 1884 /* 1885 * Check if bind request would potentially end up with filename_create(), 1886 * which in turn end up in mnt_want_write() which will grab the fs 1887 * percpu start write sem. This can trigger a lockdep warning. 1888 */ 1889 static int io_bind_file_create(const struct sockaddr_storage *addr, int addr_len) 1890 { 1891 const struct sockaddr_un *sun; 1892 1893 if (addr->ss_family != AF_UNIX) 1894 return 0; 1895 if (addr_len <= offsetof(struct sockaddr_un, sun_path)) 1896 return 0; 1897 sun = (const struct sockaddr_un *) addr; 1898 return sun->sun_path[0] != '\0'; 1899 } 1900 1901 int io_bind_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1902 { 1903 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind); 1904 struct sockaddr __user *uaddr; 1905 struct sockaddr_storage *addr; 1906 int ret; 1907 1908 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 1909 return -EINVAL; 1910 1911 uaddr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1912 bind->addr_len = READ_ONCE(sqe->addr2); 1913 1914 addr = io_uring_alloc_async_data(NULL, req); 1915 if (unlikely(!addr)) 1916 return -ENOMEM; 1917 ret = move_addr_to_kernel(uaddr, bind->addr_len, addr); 1918 if (unlikely(ret)) 1919 return ret; 1920 if (io_bind_file_create(addr, bind->addr_len)) 1921 req->flags |= REQ_F_FORCE_ASYNC; 1922 return 0; 1923 } 1924 1925 1926 int io_bind(struct io_kiocb *req, unsigned int issue_flags) 1927 { 1928 struct io_bind *bind = io_kiocb_to_cmd(req, struct io_bind); 1929 struct sockaddr_storage *addr = req->async_data; 1930 struct socket *sock; 1931 int ret; 1932 1933 sock = sock_from_file(req->file); 1934 if (unlikely(!sock)) 1935 return -ENOTSOCK; 1936 1937 ret = __sys_bind_socket(sock, addr, bind->addr_len); 1938 if (ret < 0) 1939 req_set_fail(req); 1940 io_req_set_res(req, ret, 0); 1941 return 0; 1942 } 1943 1944 int io_listen_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1945 { 1946 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen); 1947 1948 if (sqe->addr || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in || sqe->addr2) 1949 return -EINVAL; 1950 1951 listen->backlog = READ_ONCE(sqe->len); 1952 return 0; 1953 } 1954 1955 int io_listen(struct io_kiocb *req, unsigned int issue_flags) 1956 { 1957 struct io_listen *listen = io_kiocb_to_cmd(req, struct io_listen); 1958 struct socket *sock; 1959 int ret; 1960 1961 sock = sock_from_file(req->file); 1962 if (unlikely(!sock)) 1963 return -ENOTSOCK; 1964 1965 ret = __sys_listen_socket(sock, listen->backlog); 1966 if (ret < 0) 1967 req_set_fail(req); 1968 io_req_set_res(req, ret, 0); 1969 return 0; 1970 } 1971 1972 void io_netmsg_cache_free(const void *entry) 1973 { 1974 struct io_async_msghdr *kmsg = (struct io_async_msghdr *) entry; 1975 1976 io_vec_free(&kmsg->vec); 1977 kfree(kmsg); 1978 } 1979