1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/file.h> 5 #include <linux/slab.h> 6 #include <linux/net.h> 7 #include <linux/compat.h> 8 #include <net/compat.h> 9 #include <linux/io_uring.h> 10 11 #include <uapi/linux/io_uring.h> 12 13 #include "io_uring.h" 14 #include "kbuf.h" 15 #include "alloc_cache.h" 16 #include "net.h" 17 #include "notif.h" 18 #include "rsrc.h" 19 20 #if defined(CONFIG_NET) 21 struct io_shutdown { 22 struct file *file; 23 int how; 24 }; 25 26 struct io_accept { 27 struct file *file; 28 struct sockaddr __user *addr; 29 int __user *addr_len; 30 int flags; 31 u32 file_slot; 32 unsigned long nofile; 33 }; 34 35 struct io_socket { 36 struct file *file; 37 int domain; 38 int type; 39 int protocol; 40 int flags; 41 u32 file_slot; 42 unsigned long nofile; 43 }; 44 45 struct io_connect { 46 struct file *file; 47 struct sockaddr __user *addr; 48 int addr_len; 49 }; 50 51 struct io_sr_msg { 52 struct file *file; 53 union { 54 struct compat_msghdr __user *umsg_compat; 55 struct user_msghdr __user *umsg; 56 void __user *buf; 57 }; 58 unsigned len; 59 unsigned done_io; 60 unsigned msg_flags; 61 u16 flags; 62 /* initialised and used only by !msg send variants */ 63 u16 addr_len; 64 void __user *addr; 65 /* used only for send zerocopy */ 66 struct io_kiocb *notif; 67 }; 68 69 #define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED) 70 71 int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 72 { 73 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 74 75 if (unlikely(sqe->off || sqe->addr || sqe->rw_flags || 76 sqe->buf_index || sqe->splice_fd_in)) 77 return -EINVAL; 78 79 shutdown->how = READ_ONCE(sqe->len); 80 return 0; 81 } 82 83 int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) 84 { 85 struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); 86 struct socket *sock; 87 int ret; 88 89 if (issue_flags & IO_URING_F_NONBLOCK) 90 return -EAGAIN; 91 92 sock = sock_from_file(req->file); 93 if (unlikely(!sock)) 94 return -ENOTSOCK; 95 96 ret = __sys_shutdown_sock(sock, shutdown->how); 97 io_req_set_res(req, ret, 0); 98 return IOU_OK; 99 } 100 101 static bool io_net_retry(struct socket *sock, int flags) 102 { 103 if (!(flags & MSG_WAITALL)) 104 return false; 105 return sock->type == SOCK_STREAM || sock->type == SOCK_SEQPACKET; 106 } 107 108 static void io_netmsg_recycle(struct io_kiocb *req, unsigned int issue_flags) 109 { 110 struct io_async_msghdr *hdr = req->async_data; 111 112 if (!req_has_async_data(req) || issue_flags & IO_URING_F_UNLOCKED) 113 return; 114 115 /* Let normal cleanup path reap it if we fail adding to the cache */ 116 if (io_alloc_cache_put(&req->ctx->netmsg_cache, &hdr->cache)) { 117 req->async_data = NULL; 118 req->flags &= ~REQ_F_ASYNC_DATA; 119 } 120 } 121 122 static struct io_async_msghdr *io_msg_alloc_async(struct io_kiocb *req, 123 unsigned int issue_flags) 124 { 125 struct io_ring_ctx *ctx = req->ctx; 126 struct io_cache_entry *entry; 127 struct io_async_msghdr *hdr; 128 129 if (!(issue_flags & IO_URING_F_UNLOCKED) && 130 (entry = io_alloc_cache_get(&ctx->netmsg_cache)) != NULL) { 131 hdr = container_of(entry, struct io_async_msghdr, cache); 132 hdr->free_iov = NULL; 133 req->flags |= REQ_F_ASYNC_DATA; 134 req->async_data = hdr; 135 return hdr; 136 } 137 138 if (!io_alloc_async_data(req)) { 139 hdr = req->async_data; 140 hdr->free_iov = NULL; 141 return hdr; 142 } 143 return NULL; 144 } 145 146 static inline struct io_async_msghdr *io_msg_alloc_async_prep(struct io_kiocb *req) 147 { 148 /* ->prep_async is always called from the submission context */ 149 return io_msg_alloc_async(req, 0); 150 } 151 152 static int io_setup_async_msg(struct io_kiocb *req, 153 struct io_async_msghdr *kmsg, 154 unsigned int issue_flags) 155 { 156 struct io_async_msghdr *async_msg; 157 158 if (req_has_async_data(req)) 159 return -EAGAIN; 160 async_msg = io_msg_alloc_async(req, issue_flags); 161 if (!async_msg) { 162 kfree(kmsg->free_iov); 163 return -ENOMEM; 164 } 165 req->flags |= REQ_F_NEED_CLEANUP; 166 memcpy(async_msg, kmsg, sizeof(*kmsg)); 167 if (async_msg->msg.msg_name) 168 async_msg->msg.msg_name = &async_msg->addr; 169 /* if were using fast_iov, set it to the new one */ 170 if (!kmsg->free_iov) { 171 size_t fast_idx = kmsg->msg.msg_iter.iov - kmsg->fast_iov; 172 async_msg->msg.msg_iter.iov = &async_msg->fast_iov[fast_idx]; 173 } 174 175 return -EAGAIN; 176 } 177 178 static int io_sendmsg_copy_hdr(struct io_kiocb *req, 179 struct io_async_msghdr *iomsg) 180 { 181 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 182 183 iomsg->msg.msg_name = &iomsg->addr; 184 iomsg->free_iov = iomsg->fast_iov; 185 return sendmsg_copy_msghdr(&iomsg->msg, sr->umsg, sr->msg_flags, 186 &iomsg->free_iov); 187 } 188 189 int io_send_prep_async(struct io_kiocb *req) 190 { 191 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 192 struct io_async_msghdr *io; 193 int ret; 194 195 if (!zc->addr || req_has_async_data(req)) 196 return 0; 197 io = io_msg_alloc_async_prep(req); 198 if (!io) 199 return -ENOMEM; 200 ret = move_addr_to_kernel(zc->addr, zc->addr_len, &io->addr); 201 return ret; 202 } 203 204 static int io_setup_async_addr(struct io_kiocb *req, 205 struct sockaddr_storage *addr_storage, 206 unsigned int issue_flags) 207 { 208 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 209 struct io_async_msghdr *io; 210 211 if (!sr->addr || req_has_async_data(req)) 212 return -EAGAIN; 213 io = io_msg_alloc_async(req, issue_flags); 214 if (!io) 215 return -ENOMEM; 216 memcpy(&io->addr, addr_storage, sizeof(io->addr)); 217 return -EAGAIN; 218 } 219 220 int io_sendmsg_prep_async(struct io_kiocb *req) 221 { 222 int ret; 223 224 if (!io_msg_alloc_async_prep(req)) 225 return -ENOMEM; 226 ret = io_sendmsg_copy_hdr(req, req->async_data); 227 if (!ret) 228 req->flags |= REQ_F_NEED_CLEANUP; 229 return ret; 230 } 231 232 void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) 233 { 234 struct io_async_msghdr *io = req->async_data; 235 236 kfree(io->free_iov); 237 } 238 239 int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 240 { 241 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 242 243 if (req->opcode == IORING_OP_SEND) { 244 if (READ_ONCE(sqe->__pad3[0])) 245 return -EINVAL; 246 sr->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 247 sr->addr_len = READ_ONCE(sqe->addr_len); 248 } else if (sqe->addr2 || sqe->file_index) { 249 return -EINVAL; 250 } 251 252 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 253 sr->len = READ_ONCE(sqe->len); 254 sr->flags = READ_ONCE(sqe->ioprio); 255 if (sr->flags & ~IORING_RECVSEND_POLL_FIRST) 256 return -EINVAL; 257 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 258 if (sr->msg_flags & MSG_DONTWAIT) 259 req->flags |= REQ_F_NOWAIT; 260 261 #ifdef CONFIG_COMPAT 262 if (req->ctx->compat) 263 sr->msg_flags |= MSG_CMSG_COMPAT; 264 #endif 265 sr->done_io = 0; 266 return 0; 267 } 268 269 int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) 270 { 271 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 272 struct io_async_msghdr iomsg, *kmsg; 273 struct socket *sock; 274 unsigned flags; 275 int min_ret = 0; 276 int ret; 277 278 sock = sock_from_file(req->file); 279 if (unlikely(!sock)) 280 return -ENOTSOCK; 281 282 if (req_has_async_data(req)) { 283 kmsg = req->async_data; 284 } else { 285 ret = io_sendmsg_copy_hdr(req, &iomsg); 286 if (ret) 287 return ret; 288 kmsg = &iomsg; 289 } 290 291 if (!(req->flags & REQ_F_POLLED) && 292 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 293 return io_setup_async_msg(req, kmsg, issue_flags); 294 295 flags = sr->msg_flags; 296 if (issue_flags & IO_URING_F_NONBLOCK) 297 flags |= MSG_DONTWAIT; 298 if (flags & MSG_WAITALL) 299 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 300 301 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 302 303 if (ret < min_ret) { 304 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 305 return io_setup_async_msg(req, kmsg, issue_flags); 306 if (ret > 0 && io_net_retry(sock, flags)) { 307 sr->done_io += ret; 308 req->flags |= REQ_F_PARTIAL_IO; 309 return io_setup_async_msg(req, kmsg, issue_flags); 310 } 311 if (ret == -ERESTARTSYS) 312 ret = -EINTR; 313 req_set_fail(req); 314 } 315 /* fast path, check for non-NULL to avoid function call */ 316 if (kmsg->free_iov) 317 kfree(kmsg->free_iov); 318 req->flags &= ~REQ_F_NEED_CLEANUP; 319 io_netmsg_recycle(req, issue_flags); 320 if (ret >= 0) 321 ret += sr->done_io; 322 else if (sr->done_io) 323 ret = sr->done_io; 324 io_req_set_res(req, ret, 0); 325 return IOU_OK; 326 } 327 328 int io_send(struct io_kiocb *req, unsigned int issue_flags) 329 { 330 struct sockaddr_storage __address; 331 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 332 struct msghdr msg; 333 struct iovec iov; 334 struct socket *sock; 335 unsigned flags; 336 int min_ret = 0; 337 int ret; 338 339 msg.msg_name = NULL; 340 msg.msg_control = NULL; 341 msg.msg_controllen = 0; 342 msg.msg_namelen = 0; 343 msg.msg_ubuf = NULL; 344 345 if (sr->addr) { 346 if (req_has_async_data(req)) { 347 struct io_async_msghdr *io = req->async_data; 348 349 msg.msg_name = &io->addr; 350 } else { 351 ret = move_addr_to_kernel(sr->addr, sr->addr_len, &__address); 352 if (unlikely(ret < 0)) 353 return ret; 354 msg.msg_name = (struct sockaddr *)&__address; 355 } 356 msg.msg_namelen = sr->addr_len; 357 } 358 359 if (!(req->flags & REQ_F_POLLED) && 360 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 361 return io_setup_async_addr(req, &__address, issue_flags); 362 363 sock = sock_from_file(req->file); 364 if (unlikely(!sock)) 365 return -ENOTSOCK; 366 367 ret = import_single_range(WRITE, sr->buf, sr->len, &iov, &msg.msg_iter); 368 if (unlikely(ret)) 369 return ret; 370 371 flags = sr->msg_flags; 372 if (issue_flags & IO_URING_F_NONBLOCK) 373 flags |= MSG_DONTWAIT; 374 if (flags & MSG_WAITALL) 375 min_ret = iov_iter_count(&msg.msg_iter); 376 377 msg.msg_flags = flags; 378 ret = sock_sendmsg(sock, &msg); 379 if (ret < min_ret) { 380 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 381 return io_setup_async_addr(req, &__address, issue_flags); 382 383 if (ret > 0 && io_net_retry(sock, flags)) { 384 sr->len -= ret; 385 sr->buf += ret; 386 sr->done_io += ret; 387 req->flags |= REQ_F_PARTIAL_IO; 388 return io_setup_async_addr(req, &__address, issue_flags); 389 } 390 if (ret == -ERESTARTSYS) 391 ret = -EINTR; 392 req_set_fail(req); 393 } 394 if (ret >= 0) 395 ret += sr->done_io; 396 else if (sr->done_io) 397 ret = sr->done_io; 398 io_req_set_res(req, ret, 0); 399 return IOU_OK; 400 } 401 402 static bool io_recvmsg_multishot_overflow(struct io_async_msghdr *iomsg) 403 { 404 int hdr; 405 406 if (iomsg->namelen < 0) 407 return true; 408 if (check_add_overflow((int)sizeof(struct io_uring_recvmsg_out), 409 iomsg->namelen, &hdr)) 410 return true; 411 if (check_add_overflow(hdr, (int)iomsg->controllen, &hdr)) 412 return true; 413 414 return false; 415 } 416 417 static int __io_recvmsg_copy_hdr(struct io_kiocb *req, 418 struct io_async_msghdr *iomsg) 419 { 420 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 421 struct user_msghdr msg; 422 int ret; 423 424 if (copy_from_user(&msg, sr->umsg, sizeof(*sr->umsg))) 425 return -EFAULT; 426 427 ret = __copy_msghdr(&iomsg->msg, &msg, &iomsg->uaddr); 428 if (ret) 429 return ret; 430 431 if (req->flags & REQ_F_BUFFER_SELECT) { 432 if (msg.msg_iovlen == 0) { 433 sr->len = iomsg->fast_iov[0].iov_len = 0; 434 iomsg->fast_iov[0].iov_base = NULL; 435 iomsg->free_iov = NULL; 436 } else if (msg.msg_iovlen > 1) { 437 return -EINVAL; 438 } else { 439 if (copy_from_user(iomsg->fast_iov, msg.msg_iov, sizeof(*msg.msg_iov))) 440 return -EFAULT; 441 sr->len = iomsg->fast_iov[0].iov_len; 442 iomsg->free_iov = NULL; 443 } 444 445 if (req->flags & REQ_F_APOLL_MULTISHOT) { 446 iomsg->namelen = msg.msg_namelen; 447 iomsg->controllen = msg.msg_controllen; 448 if (io_recvmsg_multishot_overflow(iomsg)) 449 return -EOVERFLOW; 450 } 451 } else { 452 iomsg->free_iov = iomsg->fast_iov; 453 ret = __import_iovec(READ, msg.msg_iov, msg.msg_iovlen, UIO_FASTIOV, 454 &iomsg->free_iov, &iomsg->msg.msg_iter, 455 false); 456 if (ret > 0) 457 ret = 0; 458 } 459 460 return ret; 461 } 462 463 #ifdef CONFIG_COMPAT 464 static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, 465 struct io_async_msghdr *iomsg) 466 { 467 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 468 struct compat_msghdr msg; 469 struct compat_iovec __user *uiov; 470 int ret; 471 472 if (copy_from_user(&msg, sr->umsg_compat, sizeof(msg))) 473 return -EFAULT; 474 475 ret = __get_compat_msghdr(&iomsg->msg, &msg, &iomsg->uaddr); 476 if (ret) 477 return ret; 478 479 uiov = compat_ptr(msg.msg_iov); 480 if (req->flags & REQ_F_BUFFER_SELECT) { 481 compat_ssize_t clen; 482 483 if (msg.msg_iovlen == 0) { 484 sr->len = 0; 485 } else if (msg.msg_iovlen > 1) { 486 return -EINVAL; 487 } else { 488 if (!access_ok(uiov, sizeof(*uiov))) 489 return -EFAULT; 490 if (__get_user(clen, &uiov->iov_len)) 491 return -EFAULT; 492 if (clen < 0) 493 return -EINVAL; 494 sr->len = clen; 495 } 496 497 if (req->flags & REQ_F_APOLL_MULTISHOT) { 498 iomsg->namelen = msg.msg_namelen; 499 iomsg->controllen = msg.msg_controllen; 500 if (io_recvmsg_multishot_overflow(iomsg)) 501 return -EOVERFLOW; 502 } 503 } else { 504 iomsg->free_iov = iomsg->fast_iov; 505 ret = __import_iovec(READ, (struct iovec __user *)uiov, msg.msg_iovlen, 506 UIO_FASTIOV, &iomsg->free_iov, 507 &iomsg->msg.msg_iter, true); 508 if (ret < 0) 509 return ret; 510 } 511 512 return 0; 513 } 514 #endif 515 516 static int io_recvmsg_copy_hdr(struct io_kiocb *req, 517 struct io_async_msghdr *iomsg) 518 { 519 iomsg->msg.msg_name = &iomsg->addr; 520 521 #ifdef CONFIG_COMPAT 522 if (req->ctx->compat) 523 return __io_compat_recvmsg_copy_hdr(req, iomsg); 524 #endif 525 526 return __io_recvmsg_copy_hdr(req, iomsg); 527 } 528 529 int io_recvmsg_prep_async(struct io_kiocb *req) 530 { 531 int ret; 532 533 if (!io_msg_alloc_async_prep(req)) 534 return -ENOMEM; 535 ret = io_recvmsg_copy_hdr(req, req->async_data); 536 if (!ret) 537 req->flags |= REQ_F_NEED_CLEANUP; 538 return ret; 539 } 540 541 #define RECVMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECV_MULTISHOT) 542 543 int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 544 { 545 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 546 547 if (unlikely(sqe->file_index || sqe->addr2)) 548 return -EINVAL; 549 550 sr->umsg = u64_to_user_ptr(READ_ONCE(sqe->addr)); 551 sr->len = READ_ONCE(sqe->len); 552 sr->flags = READ_ONCE(sqe->ioprio); 553 if (sr->flags & ~(RECVMSG_FLAGS)) 554 return -EINVAL; 555 sr->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 556 if (sr->msg_flags & MSG_DONTWAIT) 557 req->flags |= REQ_F_NOWAIT; 558 if (sr->msg_flags & MSG_ERRQUEUE) 559 req->flags |= REQ_F_CLEAR_POLLIN; 560 if (sr->flags & IORING_RECV_MULTISHOT) { 561 if (!(req->flags & REQ_F_BUFFER_SELECT)) 562 return -EINVAL; 563 if (sr->msg_flags & MSG_WAITALL) 564 return -EINVAL; 565 if (req->opcode == IORING_OP_RECV && sr->len) 566 return -EINVAL; 567 req->flags |= REQ_F_APOLL_MULTISHOT; 568 } 569 570 #ifdef CONFIG_COMPAT 571 if (req->ctx->compat) 572 sr->msg_flags |= MSG_CMSG_COMPAT; 573 #endif 574 sr->done_io = 0; 575 return 0; 576 } 577 578 static inline void io_recv_prep_retry(struct io_kiocb *req) 579 { 580 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 581 582 sr->done_io = 0; 583 sr->len = 0; /* get from the provided buffer */ 584 } 585 586 /* 587 * Finishes io_recv and io_recvmsg. 588 * 589 * Returns true if it is actually finished, or false if it should run 590 * again (for multishot). 591 */ 592 static inline bool io_recv_finish(struct io_kiocb *req, int *ret, 593 unsigned int cflags, bool mshot_finished) 594 { 595 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 596 io_req_set_res(req, *ret, cflags); 597 *ret = IOU_OK; 598 return true; 599 } 600 601 if (!mshot_finished) { 602 if (io_post_aux_cqe(req->ctx, req->cqe.user_data, *ret, 603 cflags | IORING_CQE_F_MORE, false)) { 604 io_recv_prep_retry(req); 605 return false; 606 } 607 /* 608 * Otherwise stop multishot but use the current result. 609 * Probably will end up going into overflow, but this means 610 * we cannot trust the ordering anymore 611 */ 612 } 613 614 io_req_set_res(req, *ret, cflags); 615 616 if (req->flags & REQ_F_POLLED) 617 *ret = IOU_STOP_MULTISHOT; 618 else 619 *ret = IOU_OK; 620 return true; 621 } 622 623 static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg, 624 struct io_sr_msg *sr, void __user **buf, 625 size_t *len) 626 { 627 unsigned long ubuf = (unsigned long) *buf; 628 unsigned long hdr; 629 630 hdr = sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 631 kmsg->controllen; 632 if (*len < hdr) 633 return -EFAULT; 634 635 if (kmsg->controllen) { 636 unsigned long control = ubuf + hdr - kmsg->controllen; 637 638 kmsg->msg.msg_control_user = (void __user *) control; 639 kmsg->msg.msg_controllen = kmsg->controllen; 640 } 641 642 sr->buf = *buf; /* stash for later copy */ 643 *buf = (void __user *) (ubuf + hdr); 644 kmsg->payloadlen = *len = *len - hdr; 645 return 0; 646 } 647 648 struct io_recvmsg_multishot_hdr { 649 struct io_uring_recvmsg_out msg; 650 struct sockaddr_storage addr; 651 }; 652 653 static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io, 654 struct io_async_msghdr *kmsg, 655 unsigned int flags, bool *finished) 656 { 657 int err; 658 int copy_len; 659 struct io_recvmsg_multishot_hdr hdr; 660 661 if (kmsg->namelen) 662 kmsg->msg.msg_name = &hdr.addr; 663 kmsg->msg.msg_flags = flags & (MSG_CMSG_CLOEXEC|MSG_CMSG_COMPAT); 664 kmsg->msg.msg_namelen = 0; 665 666 if (sock->file->f_flags & O_NONBLOCK) 667 flags |= MSG_DONTWAIT; 668 669 err = sock_recvmsg(sock, &kmsg->msg, flags); 670 *finished = err <= 0; 671 if (err < 0) 672 return err; 673 674 hdr.msg = (struct io_uring_recvmsg_out) { 675 .controllen = kmsg->controllen - kmsg->msg.msg_controllen, 676 .flags = kmsg->msg.msg_flags & ~MSG_CMSG_COMPAT 677 }; 678 679 hdr.msg.payloadlen = err; 680 if (err > kmsg->payloadlen) 681 err = kmsg->payloadlen; 682 683 copy_len = sizeof(struct io_uring_recvmsg_out); 684 if (kmsg->msg.msg_namelen > kmsg->namelen) 685 copy_len += kmsg->namelen; 686 else 687 copy_len += kmsg->msg.msg_namelen; 688 689 /* 690 * "fromlen shall refer to the value before truncation.." 691 * 1003.1g 692 */ 693 hdr.msg.namelen = kmsg->msg.msg_namelen; 694 695 /* ensure that there is no gap between hdr and sockaddr_storage */ 696 BUILD_BUG_ON(offsetof(struct io_recvmsg_multishot_hdr, addr) != 697 sizeof(struct io_uring_recvmsg_out)); 698 if (copy_to_user(io->buf, &hdr, copy_len)) { 699 *finished = true; 700 return -EFAULT; 701 } 702 703 return sizeof(struct io_uring_recvmsg_out) + kmsg->namelen + 704 kmsg->controllen + err; 705 } 706 707 int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) 708 { 709 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 710 struct io_async_msghdr iomsg, *kmsg; 711 struct socket *sock; 712 unsigned int cflags; 713 unsigned flags; 714 int ret, min_ret = 0; 715 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 716 bool mshot_finished = true; 717 718 sock = sock_from_file(req->file); 719 if (unlikely(!sock)) 720 return -ENOTSOCK; 721 722 if (req_has_async_data(req)) { 723 kmsg = req->async_data; 724 } else { 725 ret = io_recvmsg_copy_hdr(req, &iomsg); 726 if (ret) 727 return ret; 728 kmsg = &iomsg; 729 } 730 731 if (!(req->flags & REQ_F_POLLED) && 732 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 733 return io_setup_async_msg(req, kmsg, issue_flags); 734 735 retry_multishot: 736 if (io_do_buffer_select(req)) { 737 void __user *buf; 738 size_t len = sr->len; 739 740 buf = io_buffer_select(req, &len, issue_flags); 741 if (!buf) 742 return -ENOBUFS; 743 744 if (req->flags & REQ_F_APOLL_MULTISHOT) { 745 ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len); 746 if (ret) { 747 io_kbuf_recycle(req, issue_flags); 748 return ret; 749 } 750 } 751 752 kmsg->fast_iov[0].iov_base = buf; 753 kmsg->fast_iov[0].iov_len = len; 754 iov_iter_init(&kmsg->msg.msg_iter, READ, kmsg->fast_iov, 1, 755 len); 756 } 757 758 flags = sr->msg_flags; 759 if (force_nonblock) 760 flags |= MSG_DONTWAIT; 761 if (flags & MSG_WAITALL) 762 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 763 764 kmsg->msg.msg_get_inq = 1; 765 if (req->flags & REQ_F_APOLL_MULTISHOT) 766 ret = io_recvmsg_multishot(sock, sr, kmsg, flags, 767 &mshot_finished); 768 else 769 ret = __sys_recvmsg_sock(sock, &kmsg->msg, sr->umsg, 770 kmsg->uaddr, flags); 771 772 if (ret < min_ret) { 773 if (ret == -EAGAIN && force_nonblock) { 774 ret = io_setup_async_msg(req, kmsg, issue_flags); 775 if (ret == -EAGAIN && (req->flags & IO_APOLL_MULTI_POLLED) == 776 IO_APOLL_MULTI_POLLED) { 777 io_kbuf_recycle(req, issue_flags); 778 return IOU_ISSUE_SKIP_COMPLETE; 779 } 780 return ret; 781 } 782 if (ret > 0 && io_net_retry(sock, flags)) { 783 sr->done_io += ret; 784 req->flags |= REQ_F_PARTIAL_IO; 785 return io_setup_async_msg(req, kmsg, issue_flags); 786 } 787 if (ret == -ERESTARTSYS) 788 ret = -EINTR; 789 req_set_fail(req); 790 } else if ((flags & MSG_WAITALL) && (kmsg->msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 791 req_set_fail(req); 792 } 793 794 if (ret > 0) 795 ret += sr->done_io; 796 else if (sr->done_io) 797 ret = sr->done_io; 798 else 799 io_kbuf_recycle(req, issue_flags); 800 801 cflags = io_put_kbuf(req, issue_flags); 802 if (kmsg->msg.msg_inq) 803 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 804 805 if (!io_recv_finish(req, &ret, cflags, mshot_finished)) 806 goto retry_multishot; 807 808 if (mshot_finished) { 809 io_netmsg_recycle(req, issue_flags); 810 /* fast path, check for non-NULL to avoid function call */ 811 if (kmsg->free_iov) 812 kfree(kmsg->free_iov); 813 req->flags &= ~REQ_F_NEED_CLEANUP; 814 } 815 816 return ret; 817 } 818 819 int io_recv(struct io_kiocb *req, unsigned int issue_flags) 820 { 821 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 822 struct msghdr msg; 823 struct socket *sock; 824 struct iovec iov; 825 unsigned int cflags; 826 unsigned flags; 827 int ret, min_ret = 0; 828 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 829 size_t len = sr->len; 830 831 if (!(req->flags & REQ_F_POLLED) && 832 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 833 return -EAGAIN; 834 835 sock = sock_from_file(req->file); 836 if (unlikely(!sock)) 837 return -ENOTSOCK; 838 839 retry_multishot: 840 if (io_do_buffer_select(req)) { 841 void __user *buf; 842 843 buf = io_buffer_select(req, &len, issue_flags); 844 if (!buf) 845 return -ENOBUFS; 846 sr->buf = buf; 847 } 848 849 ret = import_single_range(READ, sr->buf, len, &iov, &msg.msg_iter); 850 if (unlikely(ret)) 851 goto out_free; 852 853 msg.msg_name = NULL; 854 msg.msg_namelen = 0; 855 msg.msg_control = NULL; 856 msg.msg_get_inq = 1; 857 msg.msg_flags = 0; 858 msg.msg_controllen = 0; 859 msg.msg_iocb = NULL; 860 msg.msg_ubuf = NULL; 861 862 flags = sr->msg_flags; 863 if (force_nonblock) 864 flags |= MSG_DONTWAIT; 865 if (flags & MSG_WAITALL) 866 min_ret = iov_iter_count(&msg.msg_iter); 867 868 ret = sock_recvmsg(sock, &msg, flags); 869 if (ret < min_ret) { 870 if (ret == -EAGAIN && force_nonblock) { 871 if ((req->flags & IO_APOLL_MULTI_POLLED) == IO_APOLL_MULTI_POLLED) { 872 io_kbuf_recycle(req, issue_flags); 873 return IOU_ISSUE_SKIP_COMPLETE; 874 } 875 876 return -EAGAIN; 877 } 878 if (ret > 0 && io_net_retry(sock, flags)) { 879 sr->len -= ret; 880 sr->buf += ret; 881 sr->done_io += ret; 882 req->flags |= REQ_F_PARTIAL_IO; 883 return -EAGAIN; 884 } 885 if (ret == -ERESTARTSYS) 886 ret = -EINTR; 887 req_set_fail(req); 888 } else if ((flags & MSG_WAITALL) && (msg.msg_flags & (MSG_TRUNC | MSG_CTRUNC))) { 889 out_free: 890 req_set_fail(req); 891 } 892 893 if (ret > 0) 894 ret += sr->done_io; 895 else if (sr->done_io) 896 ret = sr->done_io; 897 else 898 io_kbuf_recycle(req, issue_flags); 899 900 cflags = io_put_kbuf(req, issue_flags); 901 if (msg.msg_inq) 902 cflags |= IORING_CQE_F_SOCK_NONEMPTY; 903 904 if (!io_recv_finish(req, &ret, cflags, ret <= 0)) 905 goto retry_multishot; 906 907 return ret; 908 } 909 910 void io_send_zc_cleanup(struct io_kiocb *req) 911 { 912 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 913 struct io_async_msghdr *io; 914 915 if (req_has_async_data(req)) { 916 io = req->async_data; 917 /* might be ->fast_iov if *msg_copy_hdr failed */ 918 if (io->free_iov != io->fast_iov) 919 kfree(io->free_iov); 920 } 921 if (zc->notif) { 922 io_notif_flush(zc->notif); 923 zc->notif = NULL; 924 } 925 } 926 927 int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 928 { 929 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 930 struct io_ring_ctx *ctx = req->ctx; 931 struct io_kiocb *notif; 932 933 if (unlikely(READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3))) 934 return -EINVAL; 935 /* we don't support IOSQE_CQE_SKIP_SUCCESS just yet */ 936 if (req->flags & REQ_F_CQE_SKIP) 937 return -EINVAL; 938 939 zc->flags = READ_ONCE(sqe->ioprio); 940 if (zc->flags & ~(IORING_RECVSEND_POLL_FIRST | 941 IORING_RECVSEND_FIXED_BUF)) 942 return -EINVAL; 943 notif = zc->notif = io_alloc_notif(ctx); 944 if (!notif) 945 return -ENOMEM; 946 notif->cqe.user_data = req->cqe.user_data; 947 notif->cqe.res = 0; 948 notif->cqe.flags = IORING_CQE_F_NOTIF; 949 req->flags |= REQ_F_NEED_CLEANUP; 950 if (zc->flags & IORING_RECVSEND_FIXED_BUF) { 951 unsigned idx = READ_ONCE(sqe->buf_index); 952 953 if (unlikely(idx >= ctx->nr_user_bufs)) 954 return -EFAULT; 955 idx = array_index_nospec(idx, ctx->nr_user_bufs); 956 req->imu = READ_ONCE(ctx->user_bufs[idx]); 957 io_req_set_rsrc_node(notif, ctx, 0); 958 } 959 960 if (req->opcode == IORING_OP_SEND_ZC) { 961 if (READ_ONCE(sqe->__pad3[0])) 962 return -EINVAL; 963 zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 964 zc->addr_len = READ_ONCE(sqe->addr_len); 965 } else { 966 if (unlikely(sqe->addr2 || sqe->file_index)) 967 return -EINVAL; 968 if (unlikely(zc->flags & IORING_RECVSEND_FIXED_BUF)) 969 return -EINVAL; 970 } 971 972 zc->buf = u64_to_user_ptr(READ_ONCE(sqe->addr)); 973 zc->len = READ_ONCE(sqe->len); 974 zc->msg_flags = READ_ONCE(sqe->msg_flags) | MSG_NOSIGNAL; 975 if (zc->msg_flags & MSG_DONTWAIT) 976 req->flags |= REQ_F_NOWAIT; 977 978 zc->done_io = 0; 979 980 #ifdef CONFIG_COMPAT 981 if (req->ctx->compat) 982 zc->msg_flags |= MSG_CMSG_COMPAT; 983 #endif 984 return 0; 985 } 986 987 static int io_sg_from_iter_iovec(struct sock *sk, struct sk_buff *skb, 988 struct iov_iter *from, size_t length) 989 { 990 skb_zcopy_downgrade_managed(skb); 991 return __zerocopy_sg_from_iter(NULL, sk, skb, from, length); 992 } 993 994 static int io_sg_from_iter(struct sock *sk, struct sk_buff *skb, 995 struct iov_iter *from, size_t length) 996 { 997 struct skb_shared_info *shinfo = skb_shinfo(skb); 998 int frag = shinfo->nr_frags; 999 int ret = 0; 1000 struct bvec_iter bi; 1001 ssize_t copied = 0; 1002 unsigned long truesize = 0; 1003 1004 if (!frag) 1005 shinfo->flags |= SKBFL_MANAGED_FRAG_REFS; 1006 else if (unlikely(!skb_zcopy_managed(skb))) 1007 return __zerocopy_sg_from_iter(NULL, sk, skb, from, length); 1008 1009 bi.bi_size = min(from->count, length); 1010 bi.bi_bvec_done = from->iov_offset; 1011 bi.bi_idx = 0; 1012 1013 while (bi.bi_size && frag < MAX_SKB_FRAGS) { 1014 struct bio_vec v = mp_bvec_iter_bvec(from->bvec, bi); 1015 1016 copied += v.bv_len; 1017 truesize += PAGE_ALIGN(v.bv_len + v.bv_offset); 1018 __skb_fill_page_desc_noacc(shinfo, frag++, v.bv_page, 1019 v.bv_offset, v.bv_len); 1020 bvec_iter_advance_single(from->bvec, &bi, v.bv_len); 1021 } 1022 if (bi.bi_size) 1023 ret = -EMSGSIZE; 1024 1025 shinfo->nr_frags = frag; 1026 from->bvec += bi.bi_idx; 1027 from->nr_segs -= bi.bi_idx; 1028 from->count -= copied; 1029 from->iov_offset = bi.bi_bvec_done; 1030 1031 skb->data_len += copied; 1032 skb->len += copied; 1033 skb->truesize += truesize; 1034 1035 if (sk && sk->sk_type == SOCK_STREAM) { 1036 sk_wmem_queued_add(sk, truesize); 1037 if (!skb_zcopy_pure(skb)) 1038 sk_mem_charge(sk, truesize); 1039 } else { 1040 refcount_add(truesize, &skb->sk->sk_wmem_alloc); 1041 } 1042 return ret; 1043 } 1044 1045 int io_send_zc(struct io_kiocb *req, unsigned int issue_flags) 1046 { 1047 struct sockaddr_storage __address; 1048 struct io_sr_msg *zc = io_kiocb_to_cmd(req, struct io_sr_msg); 1049 struct msghdr msg; 1050 struct iovec iov; 1051 struct socket *sock; 1052 unsigned msg_flags; 1053 int ret, min_ret = 0; 1054 1055 sock = sock_from_file(req->file); 1056 if (unlikely(!sock)) 1057 return -ENOTSOCK; 1058 1059 msg.msg_name = NULL; 1060 msg.msg_control = NULL; 1061 msg.msg_controllen = 0; 1062 msg.msg_namelen = 0; 1063 1064 if (zc->addr) { 1065 if (req_has_async_data(req)) { 1066 struct io_async_msghdr *io = req->async_data; 1067 1068 msg.msg_name = &io->addr; 1069 } else { 1070 ret = move_addr_to_kernel(zc->addr, zc->addr_len, &__address); 1071 if (unlikely(ret < 0)) 1072 return ret; 1073 msg.msg_name = (struct sockaddr *)&__address; 1074 } 1075 msg.msg_namelen = zc->addr_len; 1076 } 1077 1078 if (!(req->flags & REQ_F_POLLED) && 1079 (zc->flags & IORING_RECVSEND_POLL_FIRST)) 1080 return io_setup_async_addr(req, &__address, issue_flags); 1081 1082 if (zc->flags & IORING_RECVSEND_FIXED_BUF) { 1083 ret = io_import_fixed(WRITE, &msg.msg_iter, req->imu, 1084 (u64)(uintptr_t)zc->buf, zc->len); 1085 if (unlikely(ret)) 1086 return ret; 1087 msg.sg_from_iter = io_sg_from_iter; 1088 } else { 1089 ret = import_single_range(WRITE, zc->buf, zc->len, &iov, 1090 &msg.msg_iter); 1091 if (unlikely(ret)) 1092 return ret; 1093 ret = io_notif_account_mem(zc->notif, zc->len); 1094 if (unlikely(ret)) 1095 return ret; 1096 msg.sg_from_iter = io_sg_from_iter_iovec; 1097 } 1098 1099 msg_flags = zc->msg_flags | MSG_ZEROCOPY; 1100 if (issue_flags & IO_URING_F_NONBLOCK) 1101 msg_flags |= MSG_DONTWAIT; 1102 if (msg_flags & MSG_WAITALL) 1103 min_ret = iov_iter_count(&msg.msg_iter); 1104 1105 msg.msg_flags = msg_flags; 1106 msg.msg_ubuf = &io_notif_to_data(zc->notif)->uarg; 1107 ret = sock_sendmsg(sock, &msg); 1108 1109 if (unlikely(ret < min_ret)) { 1110 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1111 return io_setup_async_addr(req, &__address, issue_flags); 1112 1113 if (ret > 0 && io_net_retry(sock, msg.msg_flags)) { 1114 zc->len -= ret; 1115 zc->buf += ret; 1116 zc->done_io += ret; 1117 req->flags |= REQ_F_PARTIAL_IO; 1118 return io_setup_async_addr(req, &__address, issue_flags); 1119 } 1120 if (ret == -ERESTARTSYS) 1121 ret = -EINTR; 1122 req_set_fail(req); 1123 } 1124 1125 if (ret >= 0) 1126 ret += zc->done_io; 1127 else if (zc->done_io) 1128 ret = zc->done_io; 1129 1130 /* 1131 * If we're in io-wq we can't rely on tw ordering guarantees, defer 1132 * flushing notif to io_send_zc_cleanup() 1133 */ 1134 if (!(issue_flags & IO_URING_F_UNLOCKED)) { 1135 io_notif_flush(zc->notif); 1136 req->flags &= ~REQ_F_NEED_CLEANUP; 1137 } 1138 io_req_set_res(req, ret, IORING_CQE_F_MORE); 1139 return IOU_OK; 1140 } 1141 1142 int io_sendmsg_zc(struct io_kiocb *req, unsigned int issue_flags) 1143 { 1144 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1145 struct io_async_msghdr iomsg, *kmsg; 1146 struct socket *sock; 1147 unsigned flags; 1148 int ret, min_ret = 0; 1149 1150 sock = sock_from_file(req->file); 1151 if (unlikely(!sock)) 1152 return -ENOTSOCK; 1153 1154 if (req_has_async_data(req)) { 1155 kmsg = req->async_data; 1156 } else { 1157 ret = io_sendmsg_copy_hdr(req, &iomsg); 1158 if (ret) 1159 return ret; 1160 kmsg = &iomsg; 1161 } 1162 1163 if (!(req->flags & REQ_F_POLLED) && 1164 (sr->flags & IORING_RECVSEND_POLL_FIRST)) 1165 return io_setup_async_msg(req, kmsg, issue_flags); 1166 1167 flags = sr->msg_flags | MSG_ZEROCOPY; 1168 if (issue_flags & IO_URING_F_NONBLOCK) 1169 flags |= MSG_DONTWAIT; 1170 if (flags & MSG_WAITALL) 1171 min_ret = iov_iter_count(&kmsg->msg.msg_iter); 1172 1173 kmsg->msg.msg_ubuf = &io_notif_to_data(sr->notif)->uarg; 1174 kmsg->msg.sg_from_iter = io_sg_from_iter_iovec; 1175 ret = __sys_sendmsg_sock(sock, &kmsg->msg, flags); 1176 1177 if (unlikely(ret < min_ret)) { 1178 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1179 return io_setup_async_msg(req, kmsg, issue_flags); 1180 1181 if (ret > 0 && io_net_retry(sock, flags)) { 1182 sr->done_io += ret; 1183 req->flags |= REQ_F_PARTIAL_IO; 1184 return io_setup_async_msg(req, kmsg, issue_flags); 1185 } 1186 if (ret == -ERESTARTSYS) 1187 ret = -EINTR; 1188 req_set_fail(req); 1189 } 1190 /* fast path, check for non-NULL to avoid function call */ 1191 if (kmsg->free_iov) { 1192 kfree(kmsg->free_iov); 1193 kmsg->free_iov = NULL; 1194 } 1195 1196 io_netmsg_recycle(req, issue_flags); 1197 if (ret >= 0) 1198 ret += sr->done_io; 1199 else if (sr->done_io) 1200 ret = sr->done_io; 1201 1202 /* 1203 * If we're in io-wq we can't rely on tw ordering guarantees, defer 1204 * flushing notif to io_send_zc_cleanup() 1205 */ 1206 if (!(issue_flags & IO_URING_F_UNLOCKED)) { 1207 io_notif_flush(sr->notif); 1208 req->flags &= ~REQ_F_NEED_CLEANUP; 1209 } 1210 io_req_set_res(req, ret, IORING_CQE_F_MORE); 1211 return IOU_OK; 1212 } 1213 1214 void io_sendrecv_fail(struct io_kiocb *req) 1215 { 1216 struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); 1217 1218 if (req->flags & REQ_F_PARTIAL_IO) 1219 req->cqe.res = sr->done_io; 1220 1221 if ((req->flags & REQ_F_NEED_CLEANUP) && 1222 (req->opcode == IORING_OP_SEND_ZC || req->opcode == IORING_OP_SENDMSG_ZC)) 1223 req->cqe.flags |= IORING_CQE_F_MORE; 1224 } 1225 1226 int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1227 { 1228 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1229 unsigned flags; 1230 1231 if (sqe->len || sqe->buf_index) 1232 return -EINVAL; 1233 1234 accept->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1235 accept->addr_len = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 1236 accept->flags = READ_ONCE(sqe->accept_flags); 1237 accept->nofile = rlimit(RLIMIT_NOFILE); 1238 flags = READ_ONCE(sqe->ioprio); 1239 if (flags & ~IORING_ACCEPT_MULTISHOT) 1240 return -EINVAL; 1241 1242 accept->file_slot = READ_ONCE(sqe->file_index); 1243 if (accept->file_slot) { 1244 if (accept->flags & SOCK_CLOEXEC) 1245 return -EINVAL; 1246 if (flags & IORING_ACCEPT_MULTISHOT && 1247 accept->file_slot != IORING_FILE_INDEX_ALLOC) 1248 return -EINVAL; 1249 } 1250 if (accept->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1251 return -EINVAL; 1252 if (SOCK_NONBLOCK != O_NONBLOCK && (accept->flags & SOCK_NONBLOCK)) 1253 accept->flags = (accept->flags & ~SOCK_NONBLOCK) | O_NONBLOCK; 1254 if (flags & IORING_ACCEPT_MULTISHOT) 1255 req->flags |= REQ_F_APOLL_MULTISHOT; 1256 return 0; 1257 } 1258 1259 int io_accept(struct io_kiocb *req, unsigned int issue_flags) 1260 { 1261 struct io_ring_ctx *ctx = req->ctx; 1262 struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); 1263 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1264 unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; 1265 bool fixed = !!accept->file_slot; 1266 struct file *file; 1267 int ret, fd; 1268 1269 retry: 1270 if (!fixed) { 1271 fd = __get_unused_fd_flags(accept->flags, accept->nofile); 1272 if (unlikely(fd < 0)) 1273 return fd; 1274 } 1275 file = do_accept(req->file, file_flags, accept->addr, accept->addr_len, 1276 accept->flags); 1277 if (IS_ERR(file)) { 1278 if (!fixed) 1279 put_unused_fd(fd); 1280 ret = PTR_ERR(file); 1281 if (ret == -EAGAIN && force_nonblock) { 1282 /* 1283 * if it's multishot and polled, we don't need to 1284 * return EAGAIN to arm the poll infra since it 1285 * has already been done 1286 */ 1287 if ((req->flags & IO_APOLL_MULTI_POLLED) == 1288 IO_APOLL_MULTI_POLLED) 1289 ret = IOU_ISSUE_SKIP_COMPLETE; 1290 return ret; 1291 } 1292 if (ret == -ERESTARTSYS) 1293 ret = -EINTR; 1294 req_set_fail(req); 1295 } else if (!fixed) { 1296 fd_install(fd, file); 1297 ret = fd; 1298 } else { 1299 ret = io_fixed_fd_install(req, issue_flags, file, 1300 accept->file_slot); 1301 } 1302 1303 if (!(req->flags & REQ_F_APOLL_MULTISHOT)) { 1304 io_req_set_res(req, ret, 0); 1305 return IOU_OK; 1306 } 1307 1308 if (ret >= 0 && 1309 io_post_aux_cqe(ctx, req->cqe.user_data, ret, IORING_CQE_F_MORE, false)) 1310 goto retry; 1311 1312 io_req_set_res(req, ret, 0); 1313 if (req->flags & REQ_F_POLLED) 1314 return IOU_STOP_MULTISHOT; 1315 return IOU_OK; 1316 } 1317 1318 int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1319 { 1320 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1321 1322 if (sqe->addr || sqe->rw_flags || sqe->buf_index) 1323 return -EINVAL; 1324 1325 sock->domain = READ_ONCE(sqe->fd); 1326 sock->type = READ_ONCE(sqe->off); 1327 sock->protocol = READ_ONCE(sqe->len); 1328 sock->file_slot = READ_ONCE(sqe->file_index); 1329 sock->nofile = rlimit(RLIMIT_NOFILE); 1330 1331 sock->flags = sock->type & ~SOCK_TYPE_MASK; 1332 if (sock->file_slot && (sock->flags & SOCK_CLOEXEC)) 1333 return -EINVAL; 1334 if (sock->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 1335 return -EINVAL; 1336 return 0; 1337 } 1338 1339 int io_socket(struct io_kiocb *req, unsigned int issue_flags) 1340 { 1341 struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); 1342 bool fixed = !!sock->file_slot; 1343 struct file *file; 1344 int ret, fd; 1345 1346 if (!fixed) { 1347 fd = __get_unused_fd_flags(sock->flags, sock->nofile); 1348 if (unlikely(fd < 0)) 1349 return fd; 1350 } 1351 file = __sys_socket_file(sock->domain, sock->type, sock->protocol); 1352 if (IS_ERR(file)) { 1353 if (!fixed) 1354 put_unused_fd(fd); 1355 ret = PTR_ERR(file); 1356 if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) 1357 return -EAGAIN; 1358 if (ret == -ERESTARTSYS) 1359 ret = -EINTR; 1360 req_set_fail(req); 1361 } else if (!fixed) { 1362 fd_install(fd, file); 1363 ret = fd; 1364 } else { 1365 ret = io_fixed_fd_install(req, issue_flags, file, 1366 sock->file_slot); 1367 } 1368 io_req_set_res(req, ret, 0); 1369 return IOU_OK; 1370 } 1371 1372 int io_connect_prep_async(struct io_kiocb *req) 1373 { 1374 struct io_async_connect *io = req->async_data; 1375 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1376 1377 return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address); 1378 } 1379 1380 int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 1381 { 1382 struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); 1383 1384 if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) 1385 return -EINVAL; 1386 1387 conn->addr = u64_to_user_ptr(READ_ONCE(sqe->addr)); 1388 conn->addr_len = READ_ONCE(sqe->addr2); 1389 return 0; 1390 } 1391 1392 int io_connect(struct io_kiocb *req, unsigned int issue_flags) 1393 { 1394 struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect); 1395 struct io_async_connect __io, *io; 1396 unsigned file_flags; 1397 int ret; 1398 bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; 1399 1400 if (req_has_async_data(req)) { 1401 io = req->async_data; 1402 } else { 1403 ret = move_addr_to_kernel(connect->addr, 1404 connect->addr_len, 1405 &__io.address); 1406 if (ret) 1407 goto out; 1408 io = &__io; 1409 } 1410 1411 file_flags = force_nonblock ? O_NONBLOCK : 0; 1412 1413 ret = __sys_connect_file(req->file, &io->address, 1414 connect->addr_len, file_flags); 1415 if ((ret == -EAGAIN || ret == -EINPROGRESS) && force_nonblock) { 1416 if (req_has_async_data(req)) 1417 return -EAGAIN; 1418 if (io_alloc_async_data(req)) { 1419 ret = -ENOMEM; 1420 goto out; 1421 } 1422 memcpy(req->async_data, &__io, sizeof(__io)); 1423 return -EAGAIN; 1424 } 1425 if (ret == -ERESTARTSYS) 1426 ret = -EINTR; 1427 out: 1428 if (ret < 0) 1429 req_set_fail(req); 1430 io_req_set_res(req, ret, 0); 1431 return IOU_OK; 1432 } 1433 1434 void io_netmsg_cache_free(struct io_cache_entry *entry) 1435 { 1436 kfree(container_of(entry, struct io_async_msghdr, cache)); 1437 } 1438 #endif 1439