1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * io_uring opcode handling table 4 */ 5 #include <linux/kernel.h> 6 #include <linux/errno.h> 7 #include <linux/fs.h> 8 #include <linux/file.h> 9 #include <linux/io_uring.h> 10 11 #include "io_uring.h" 12 #include "opdef.h" 13 #include "refs.h" 14 #include "tctx.h" 15 #include "sqpoll.h" 16 #include "fdinfo.h" 17 #include "kbuf.h" 18 #include "rsrc.h" 19 20 #include "xattr.h" 21 #include "nop.h" 22 #include "fs.h" 23 #include "splice.h" 24 #include "sync.h" 25 #include "advise.h" 26 #include "openclose.h" 27 #include "uring_cmd.h" 28 #include "epoll.h" 29 #include "statx.h" 30 #include "net.h" 31 #include "msg_ring.h" 32 #include "timeout.h" 33 #include "poll.h" 34 #include "cancel.h" 35 #include "rw.h" 36 #include "waitid.h" 37 #include "futex.h" 38 39 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags) 40 { 41 WARN_ON_ONCE(1); 42 return -ECANCELED; 43 } 44 45 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb, 46 const struct io_uring_sqe *sqe) 47 { 48 return -EOPNOTSUPP; 49 } 50 51 const struct io_issue_def io_issue_defs[] = { 52 [IORING_OP_NOP] = { 53 .audit_skip = 1, 54 .iopoll = 1, 55 .prep = io_nop_prep, 56 .issue = io_nop, 57 }, 58 [IORING_OP_READV] = { 59 .needs_file = 1, 60 .unbound_nonreg_file = 1, 61 .pollin = 1, 62 .buffer_select = 1, 63 .plug = 1, 64 .audit_skip = 1, 65 .ioprio = 1, 66 .iopoll = 1, 67 .iopoll_queue = 1, 68 .vectored = 1, 69 .prep = io_prep_rwv, 70 .issue = io_read, 71 }, 72 [IORING_OP_WRITEV] = { 73 .needs_file = 1, 74 .hash_reg_file = 1, 75 .unbound_nonreg_file = 1, 76 .pollout = 1, 77 .plug = 1, 78 .audit_skip = 1, 79 .ioprio = 1, 80 .iopoll = 1, 81 .iopoll_queue = 1, 82 .vectored = 1, 83 .prep = io_prep_rwv, 84 .issue = io_write, 85 }, 86 [IORING_OP_FSYNC] = { 87 .needs_file = 1, 88 .audit_skip = 1, 89 .prep = io_fsync_prep, 90 .issue = io_fsync, 91 }, 92 [IORING_OP_READ_FIXED] = { 93 .needs_file = 1, 94 .unbound_nonreg_file = 1, 95 .pollin = 1, 96 .plug = 1, 97 .audit_skip = 1, 98 .ioprio = 1, 99 .iopoll = 1, 100 .iopoll_queue = 1, 101 .prep = io_prep_rw_fixed, 102 .issue = io_read, 103 }, 104 [IORING_OP_WRITE_FIXED] = { 105 .needs_file = 1, 106 .hash_reg_file = 1, 107 .unbound_nonreg_file = 1, 108 .pollout = 1, 109 .plug = 1, 110 .audit_skip = 1, 111 .ioprio = 1, 112 .iopoll = 1, 113 .iopoll_queue = 1, 114 .prep = io_prep_rw_fixed, 115 .issue = io_write, 116 }, 117 [IORING_OP_POLL_ADD] = { 118 .needs_file = 1, 119 .unbound_nonreg_file = 1, 120 .audit_skip = 1, 121 .prep = io_poll_add_prep, 122 .issue = io_poll_add, 123 }, 124 [IORING_OP_POLL_REMOVE] = { 125 .audit_skip = 1, 126 .prep = io_poll_remove_prep, 127 .issue = io_poll_remove, 128 }, 129 [IORING_OP_SYNC_FILE_RANGE] = { 130 .needs_file = 1, 131 .audit_skip = 1, 132 .prep = io_sfr_prep, 133 .issue = io_sync_file_range, 134 }, 135 [IORING_OP_SENDMSG] = { 136 .needs_file = 1, 137 .unbound_nonreg_file = 1, 138 .pollout = 1, 139 .ioprio = 1, 140 .manual_alloc = 1, 141 #if defined(CONFIG_NET) 142 .prep = io_sendmsg_prep, 143 .issue = io_sendmsg, 144 #else 145 .prep = io_eopnotsupp_prep, 146 #endif 147 }, 148 [IORING_OP_RECVMSG] = { 149 .needs_file = 1, 150 .unbound_nonreg_file = 1, 151 .pollin = 1, 152 .buffer_select = 1, 153 .ioprio = 1, 154 .manual_alloc = 1, 155 #if defined(CONFIG_NET) 156 .prep = io_recvmsg_prep, 157 .issue = io_recvmsg, 158 #else 159 .prep = io_eopnotsupp_prep, 160 #endif 161 }, 162 [IORING_OP_TIMEOUT] = { 163 .audit_skip = 1, 164 .prep = io_timeout_prep, 165 .issue = io_timeout, 166 }, 167 [IORING_OP_TIMEOUT_REMOVE] = { 168 /* used by timeout updates' prep() */ 169 .audit_skip = 1, 170 .prep = io_timeout_remove_prep, 171 .issue = io_timeout_remove, 172 }, 173 [IORING_OP_ACCEPT] = { 174 .needs_file = 1, 175 .unbound_nonreg_file = 1, 176 .pollin = 1, 177 .poll_exclusive = 1, 178 .ioprio = 1, /* used for flags */ 179 #if defined(CONFIG_NET) 180 .prep = io_accept_prep, 181 .issue = io_accept, 182 #else 183 .prep = io_eopnotsupp_prep, 184 #endif 185 }, 186 [IORING_OP_ASYNC_CANCEL] = { 187 .audit_skip = 1, 188 .prep = io_async_cancel_prep, 189 .issue = io_async_cancel, 190 }, 191 [IORING_OP_LINK_TIMEOUT] = { 192 .audit_skip = 1, 193 .prep = io_link_timeout_prep, 194 .issue = io_no_issue, 195 }, 196 [IORING_OP_CONNECT] = { 197 .needs_file = 1, 198 .unbound_nonreg_file = 1, 199 .pollout = 1, 200 #if defined(CONFIG_NET) 201 .prep = io_connect_prep, 202 .issue = io_connect, 203 #else 204 .prep = io_eopnotsupp_prep, 205 #endif 206 }, 207 [IORING_OP_FALLOCATE] = { 208 .needs_file = 1, 209 .prep = io_fallocate_prep, 210 .issue = io_fallocate, 211 }, 212 [IORING_OP_OPENAT] = { 213 .prep = io_openat_prep, 214 .issue = io_openat, 215 }, 216 [IORING_OP_CLOSE] = { 217 .prep = io_close_prep, 218 .issue = io_close, 219 }, 220 [IORING_OP_FILES_UPDATE] = { 221 .audit_skip = 1, 222 .iopoll = 1, 223 .prep = io_files_update_prep, 224 .issue = io_files_update, 225 }, 226 [IORING_OP_STATX] = { 227 .audit_skip = 1, 228 .prep = io_statx_prep, 229 .issue = io_statx, 230 }, 231 [IORING_OP_READ] = { 232 .needs_file = 1, 233 .unbound_nonreg_file = 1, 234 .pollin = 1, 235 .buffer_select = 1, 236 .plug = 1, 237 .audit_skip = 1, 238 .ioprio = 1, 239 .iopoll = 1, 240 .iopoll_queue = 1, 241 .prep = io_prep_rw, 242 .issue = io_read, 243 }, 244 [IORING_OP_WRITE] = { 245 .needs_file = 1, 246 .hash_reg_file = 1, 247 .unbound_nonreg_file = 1, 248 .pollout = 1, 249 .plug = 1, 250 .audit_skip = 1, 251 .ioprio = 1, 252 .iopoll = 1, 253 .iopoll_queue = 1, 254 .prep = io_prep_rw, 255 .issue = io_write, 256 }, 257 [IORING_OP_FADVISE] = { 258 .needs_file = 1, 259 .audit_skip = 1, 260 .prep = io_fadvise_prep, 261 .issue = io_fadvise, 262 }, 263 [IORING_OP_MADVISE] = { 264 .audit_skip = 1, 265 .prep = io_madvise_prep, 266 .issue = io_madvise, 267 }, 268 [IORING_OP_SEND] = { 269 .needs_file = 1, 270 .unbound_nonreg_file = 1, 271 .pollout = 1, 272 .audit_skip = 1, 273 .ioprio = 1, 274 .manual_alloc = 1, 275 #if defined(CONFIG_NET) 276 .prep = io_sendmsg_prep, 277 .issue = io_send, 278 #else 279 .prep = io_eopnotsupp_prep, 280 #endif 281 }, 282 [IORING_OP_RECV] = { 283 .needs_file = 1, 284 .unbound_nonreg_file = 1, 285 .pollin = 1, 286 .buffer_select = 1, 287 .audit_skip = 1, 288 .ioprio = 1, 289 #if defined(CONFIG_NET) 290 .prep = io_recvmsg_prep, 291 .issue = io_recv, 292 #else 293 .prep = io_eopnotsupp_prep, 294 #endif 295 }, 296 [IORING_OP_OPENAT2] = { 297 .prep = io_openat2_prep, 298 .issue = io_openat2, 299 }, 300 [IORING_OP_EPOLL_CTL] = { 301 .unbound_nonreg_file = 1, 302 .audit_skip = 1, 303 #if defined(CONFIG_EPOLL) 304 .prep = io_epoll_ctl_prep, 305 .issue = io_epoll_ctl, 306 #else 307 .prep = io_eopnotsupp_prep, 308 #endif 309 }, 310 [IORING_OP_SPLICE] = { 311 .needs_file = 1, 312 .hash_reg_file = 1, 313 .unbound_nonreg_file = 1, 314 .audit_skip = 1, 315 .prep = io_splice_prep, 316 .issue = io_splice, 317 }, 318 [IORING_OP_PROVIDE_BUFFERS] = { 319 .audit_skip = 1, 320 .iopoll = 1, 321 .prep = io_provide_buffers_prep, 322 .issue = io_provide_buffers, 323 }, 324 [IORING_OP_REMOVE_BUFFERS] = { 325 .audit_skip = 1, 326 .iopoll = 1, 327 .prep = io_remove_buffers_prep, 328 .issue = io_remove_buffers, 329 }, 330 [IORING_OP_TEE] = { 331 .needs_file = 1, 332 .hash_reg_file = 1, 333 .unbound_nonreg_file = 1, 334 .audit_skip = 1, 335 .prep = io_tee_prep, 336 .issue = io_tee, 337 }, 338 [IORING_OP_SHUTDOWN] = { 339 .needs_file = 1, 340 #if defined(CONFIG_NET) 341 .prep = io_shutdown_prep, 342 .issue = io_shutdown, 343 #else 344 .prep = io_eopnotsupp_prep, 345 #endif 346 }, 347 [IORING_OP_RENAMEAT] = { 348 .prep = io_renameat_prep, 349 .issue = io_renameat, 350 }, 351 [IORING_OP_UNLINKAT] = { 352 .prep = io_unlinkat_prep, 353 .issue = io_unlinkat, 354 }, 355 [IORING_OP_MKDIRAT] = { 356 .prep = io_mkdirat_prep, 357 .issue = io_mkdirat, 358 }, 359 [IORING_OP_SYMLINKAT] = { 360 .prep = io_symlinkat_prep, 361 .issue = io_symlinkat, 362 }, 363 [IORING_OP_LINKAT] = { 364 .prep = io_linkat_prep, 365 .issue = io_linkat, 366 }, 367 [IORING_OP_MSG_RING] = { 368 .needs_file = 1, 369 .iopoll = 1, 370 .prep = io_msg_ring_prep, 371 .issue = io_msg_ring, 372 }, 373 [IORING_OP_FSETXATTR] = { 374 .needs_file = 1, 375 .prep = io_fsetxattr_prep, 376 .issue = io_fsetxattr, 377 }, 378 [IORING_OP_SETXATTR] = { 379 .prep = io_setxattr_prep, 380 .issue = io_setxattr, 381 }, 382 [IORING_OP_FGETXATTR] = { 383 .needs_file = 1, 384 .prep = io_fgetxattr_prep, 385 .issue = io_fgetxattr, 386 }, 387 [IORING_OP_GETXATTR] = { 388 .prep = io_getxattr_prep, 389 .issue = io_getxattr, 390 }, 391 [IORING_OP_SOCKET] = { 392 .audit_skip = 1, 393 #if defined(CONFIG_NET) 394 .prep = io_socket_prep, 395 .issue = io_socket, 396 #else 397 .prep = io_eopnotsupp_prep, 398 #endif 399 }, 400 [IORING_OP_URING_CMD] = { 401 .needs_file = 1, 402 .plug = 1, 403 .iopoll = 1, 404 .iopoll_queue = 1, 405 .prep = io_uring_cmd_prep, 406 .issue = io_uring_cmd, 407 }, 408 [IORING_OP_SEND_ZC] = { 409 .needs_file = 1, 410 .unbound_nonreg_file = 1, 411 .pollout = 1, 412 .audit_skip = 1, 413 .ioprio = 1, 414 .manual_alloc = 1, 415 #if defined(CONFIG_NET) 416 .prep = io_send_zc_prep, 417 .issue = io_send_zc, 418 #else 419 .prep = io_eopnotsupp_prep, 420 #endif 421 }, 422 [IORING_OP_SENDMSG_ZC] = { 423 .needs_file = 1, 424 .unbound_nonreg_file = 1, 425 .pollout = 1, 426 .ioprio = 1, 427 .manual_alloc = 1, 428 #if defined(CONFIG_NET) 429 .prep = io_send_zc_prep, 430 .issue = io_sendmsg_zc, 431 #else 432 .prep = io_eopnotsupp_prep, 433 #endif 434 }, 435 [IORING_OP_READ_MULTISHOT] = { 436 .needs_file = 1, 437 .unbound_nonreg_file = 1, 438 .pollin = 1, 439 .buffer_select = 1, 440 .audit_skip = 1, 441 .prep = io_read_mshot_prep, 442 .issue = io_read_mshot, 443 }, 444 [IORING_OP_WAITID] = { 445 .prep = io_waitid_prep, 446 .issue = io_waitid, 447 }, 448 [IORING_OP_FUTEX_WAIT] = { 449 #if defined(CONFIG_FUTEX) 450 .prep = io_futex_prep, 451 .issue = io_futex_wait, 452 #else 453 .prep = io_eopnotsupp_prep, 454 #endif 455 }, 456 [IORING_OP_FUTEX_WAKE] = { 457 #if defined(CONFIG_FUTEX) 458 .prep = io_futex_prep, 459 .issue = io_futex_wake, 460 #else 461 .prep = io_eopnotsupp_prep, 462 #endif 463 }, 464 [IORING_OP_FUTEX_WAITV] = { 465 #if defined(CONFIG_FUTEX) 466 .prep = io_futexv_prep, 467 .issue = io_futexv_wait, 468 #else 469 .prep = io_eopnotsupp_prep, 470 #endif 471 }, 472 }; 473 474 const struct io_cold_def io_cold_defs[] = { 475 [IORING_OP_NOP] = { 476 .name = "NOP", 477 }, 478 [IORING_OP_READV] = { 479 .async_size = sizeof(struct io_async_rw), 480 .name = "READV", 481 .prep_async = io_readv_prep_async, 482 .cleanup = io_readv_writev_cleanup, 483 .fail = io_rw_fail, 484 }, 485 [IORING_OP_WRITEV] = { 486 .async_size = sizeof(struct io_async_rw), 487 .name = "WRITEV", 488 .prep_async = io_writev_prep_async, 489 .cleanup = io_readv_writev_cleanup, 490 .fail = io_rw_fail, 491 }, 492 [IORING_OP_FSYNC] = { 493 .name = "FSYNC", 494 }, 495 [IORING_OP_READ_FIXED] = { 496 .async_size = sizeof(struct io_async_rw), 497 .name = "READ_FIXED", 498 .fail = io_rw_fail, 499 }, 500 [IORING_OP_WRITE_FIXED] = { 501 .async_size = sizeof(struct io_async_rw), 502 .name = "WRITE_FIXED", 503 .fail = io_rw_fail, 504 }, 505 [IORING_OP_POLL_ADD] = { 506 .name = "POLL_ADD", 507 }, 508 [IORING_OP_POLL_REMOVE] = { 509 .name = "POLL_REMOVE", 510 }, 511 [IORING_OP_SYNC_FILE_RANGE] = { 512 .name = "SYNC_FILE_RANGE", 513 }, 514 [IORING_OP_SENDMSG] = { 515 .name = "SENDMSG", 516 #if defined(CONFIG_NET) 517 .async_size = sizeof(struct io_async_msghdr), 518 .prep_async = io_sendmsg_prep_async, 519 .cleanup = io_sendmsg_recvmsg_cleanup, 520 .fail = io_sendrecv_fail, 521 #endif 522 }, 523 [IORING_OP_RECVMSG] = { 524 .name = "RECVMSG", 525 #if defined(CONFIG_NET) 526 .async_size = sizeof(struct io_async_msghdr), 527 .prep_async = io_recvmsg_prep_async, 528 .cleanup = io_sendmsg_recvmsg_cleanup, 529 .fail = io_sendrecv_fail, 530 #endif 531 }, 532 [IORING_OP_TIMEOUT] = { 533 .async_size = sizeof(struct io_timeout_data), 534 .name = "TIMEOUT", 535 }, 536 [IORING_OP_TIMEOUT_REMOVE] = { 537 .name = "TIMEOUT_REMOVE", 538 }, 539 [IORING_OP_ACCEPT] = { 540 .name = "ACCEPT", 541 }, 542 [IORING_OP_ASYNC_CANCEL] = { 543 .name = "ASYNC_CANCEL", 544 }, 545 [IORING_OP_LINK_TIMEOUT] = { 546 .async_size = sizeof(struct io_timeout_data), 547 .name = "LINK_TIMEOUT", 548 }, 549 [IORING_OP_CONNECT] = { 550 .name = "CONNECT", 551 #if defined(CONFIG_NET) 552 .async_size = sizeof(struct io_async_connect), 553 .prep_async = io_connect_prep_async, 554 #endif 555 }, 556 [IORING_OP_FALLOCATE] = { 557 .name = "FALLOCATE", 558 }, 559 [IORING_OP_OPENAT] = { 560 .name = "OPENAT", 561 .cleanup = io_open_cleanup, 562 }, 563 [IORING_OP_CLOSE] = { 564 .name = "CLOSE", 565 }, 566 [IORING_OP_FILES_UPDATE] = { 567 .name = "FILES_UPDATE", 568 }, 569 [IORING_OP_STATX] = { 570 .name = "STATX", 571 .cleanup = io_statx_cleanup, 572 }, 573 [IORING_OP_READ] = { 574 .async_size = sizeof(struct io_async_rw), 575 .name = "READ", 576 .fail = io_rw_fail, 577 }, 578 [IORING_OP_WRITE] = { 579 .async_size = sizeof(struct io_async_rw), 580 .name = "WRITE", 581 .fail = io_rw_fail, 582 }, 583 [IORING_OP_FADVISE] = { 584 .name = "FADVISE", 585 }, 586 [IORING_OP_MADVISE] = { 587 .name = "MADVISE", 588 }, 589 [IORING_OP_SEND] = { 590 .name = "SEND", 591 #if defined(CONFIG_NET) 592 .async_size = sizeof(struct io_async_msghdr), 593 .fail = io_sendrecv_fail, 594 .prep_async = io_send_prep_async, 595 #endif 596 }, 597 [IORING_OP_RECV] = { 598 .name = "RECV", 599 #if defined(CONFIG_NET) 600 .fail = io_sendrecv_fail, 601 #endif 602 }, 603 [IORING_OP_OPENAT2] = { 604 .name = "OPENAT2", 605 .cleanup = io_open_cleanup, 606 }, 607 [IORING_OP_EPOLL_CTL] = { 608 .name = "EPOLL", 609 }, 610 [IORING_OP_SPLICE] = { 611 .name = "SPLICE", 612 }, 613 [IORING_OP_PROVIDE_BUFFERS] = { 614 .name = "PROVIDE_BUFFERS", 615 }, 616 [IORING_OP_REMOVE_BUFFERS] = { 617 .name = "REMOVE_BUFFERS", 618 }, 619 [IORING_OP_TEE] = { 620 .name = "TEE", 621 }, 622 [IORING_OP_SHUTDOWN] = { 623 .name = "SHUTDOWN", 624 }, 625 [IORING_OP_RENAMEAT] = { 626 .name = "RENAMEAT", 627 .cleanup = io_renameat_cleanup, 628 }, 629 [IORING_OP_UNLINKAT] = { 630 .name = "UNLINKAT", 631 .cleanup = io_unlinkat_cleanup, 632 }, 633 [IORING_OP_MKDIRAT] = { 634 .name = "MKDIRAT", 635 .cleanup = io_mkdirat_cleanup, 636 }, 637 [IORING_OP_SYMLINKAT] = { 638 .name = "SYMLINKAT", 639 .cleanup = io_link_cleanup, 640 }, 641 [IORING_OP_LINKAT] = { 642 .name = "LINKAT", 643 .cleanup = io_link_cleanup, 644 }, 645 [IORING_OP_MSG_RING] = { 646 .name = "MSG_RING", 647 .cleanup = io_msg_ring_cleanup, 648 }, 649 [IORING_OP_FSETXATTR] = { 650 .name = "FSETXATTR", 651 .cleanup = io_xattr_cleanup, 652 }, 653 [IORING_OP_SETXATTR] = { 654 .name = "SETXATTR", 655 .cleanup = io_xattr_cleanup, 656 }, 657 [IORING_OP_FGETXATTR] = { 658 .name = "FGETXATTR", 659 .cleanup = io_xattr_cleanup, 660 }, 661 [IORING_OP_GETXATTR] = { 662 .name = "GETXATTR", 663 .cleanup = io_xattr_cleanup, 664 }, 665 [IORING_OP_SOCKET] = { 666 .name = "SOCKET", 667 }, 668 [IORING_OP_URING_CMD] = { 669 .name = "URING_CMD", 670 .async_size = 2 * sizeof(struct io_uring_sqe), 671 .prep_async = io_uring_cmd_prep_async, 672 }, 673 [IORING_OP_SEND_ZC] = { 674 .name = "SEND_ZC", 675 #if defined(CONFIG_NET) 676 .async_size = sizeof(struct io_async_msghdr), 677 .prep_async = io_send_prep_async, 678 .cleanup = io_send_zc_cleanup, 679 .fail = io_sendrecv_fail, 680 #endif 681 }, 682 [IORING_OP_SENDMSG_ZC] = { 683 .name = "SENDMSG_ZC", 684 #if defined(CONFIG_NET) 685 .async_size = sizeof(struct io_async_msghdr), 686 .prep_async = io_sendmsg_prep_async, 687 .cleanup = io_send_zc_cleanup, 688 .fail = io_sendrecv_fail, 689 #endif 690 }, 691 [IORING_OP_READ_MULTISHOT] = { 692 .name = "READ_MULTISHOT", 693 }, 694 [IORING_OP_WAITID] = { 695 .name = "WAITID", 696 .async_size = sizeof(struct io_waitid_async), 697 }, 698 [IORING_OP_FUTEX_WAIT] = { 699 .name = "FUTEX_WAIT", 700 }, 701 [IORING_OP_FUTEX_WAKE] = { 702 .name = "FUTEX_WAKE", 703 }, 704 [IORING_OP_FUTEX_WAITV] = { 705 .name = "FUTEX_WAITV", 706 }, 707 }; 708 709 const char *io_uring_get_opcode(u8 opcode) 710 { 711 if (opcode < IORING_OP_LAST) 712 return io_cold_defs[opcode].name; 713 return "INVALID"; 714 } 715 716 void __init io_uring_optable_init(void) 717 { 718 int i; 719 720 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST); 721 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST); 722 723 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) { 724 BUG_ON(!io_issue_defs[i].prep); 725 if (io_issue_defs[i].prep != io_eopnotsupp_prep) 726 BUG_ON(!io_issue_defs[i].issue); 727 WARN_ON_ONCE(!io_cold_defs[i].name); 728 } 729 } 730