1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * io_uring opcode handling table 4 */ 5 #include <linux/kernel.h> 6 #include <linux/errno.h> 7 #include <linux/fs.h> 8 #include <linux/file.h> 9 #include <linux/io_uring.h> 10 #include <linux/io_uring/cmd.h> 11 12 #include "io_uring.h" 13 #include "opdef.h" 14 #include "refs.h" 15 #include "tctx.h" 16 #include "sqpoll.h" 17 #include "fdinfo.h" 18 #include "kbuf.h" 19 #include "rsrc.h" 20 21 #include "xattr.h" 22 #include "nop.h" 23 #include "fs.h" 24 #include "splice.h" 25 #include "sync.h" 26 #include "advise.h" 27 #include "openclose.h" 28 #include "uring_cmd.h" 29 #include "epoll.h" 30 #include "statx.h" 31 #include "net.h" 32 #include "msg_ring.h" 33 #include "timeout.h" 34 #include "poll.h" 35 #include "cancel.h" 36 #include "rw.h" 37 #include "waitid.h" 38 #include "futex.h" 39 #include "truncate.h" 40 #include "zcrx.h" 41 42 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags) 43 { 44 WARN_ON_ONCE(1); 45 return -ECANCELED; 46 } 47 48 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb, 49 const struct io_uring_sqe *sqe) 50 { 51 return -EOPNOTSUPP; 52 } 53 54 const struct io_issue_def io_issue_defs[] = { 55 [IORING_OP_NOP] = { 56 .audit_skip = 1, 57 .iopoll = 1, 58 .prep = io_nop_prep, 59 .issue = io_nop, 60 }, 61 [IORING_OP_READV] = { 62 .needs_file = 1, 63 .unbound_nonreg_file = 1, 64 .pollin = 1, 65 .buffer_select = 1, 66 .plug = 1, 67 .audit_skip = 1, 68 .ioprio = 1, 69 .iopoll = 1, 70 .vectored = 1, 71 .async_size = sizeof(struct io_async_rw), 72 .prep = io_prep_readv, 73 .issue = io_read, 74 }, 75 [IORING_OP_WRITEV] = { 76 .needs_file = 1, 77 .hash_reg_file = 1, 78 .unbound_nonreg_file = 1, 79 .pollout = 1, 80 .plug = 1, 81 .audit_skip = 1, 82 .ioprio = 1, 83 .iopoll = 1, 84 .vectored = 1, 85 .async_size = sizeof(struct io_async_rw), 86 .prep = io_prep_writev, 87 .issue = io_write, 88 }, 89 [IORING_OP_FSYNC] = { 90 .needs_file = 1, 91 .audit_skip = 1, 92 .prep = io_fsync_prep, 93 .issue = io_fsync, 94 }, 95 [IORING_OP_READ_FIXED] = { 96 .needs_file = 1, 97 .unbound_nonreg_file = 1, 98 .pollin = 1, 99 .plug = 1, 100 .audit_skip = 1, 101 .ioprio = 1, 102 .iopoll = 1, 103 .async_size = sizeof(struct io_async_rw), 104 .prep = io_prep_read_fixed, 105 .issue = io_read_fixed, 106 }, 107 [IORING_OP_WRITE_FIXED] = { 108 .needs_file = 1, 109 .hash_reg_file = 1, 110 .unbound_nonreg_file = 1, 111 .pollout = 1, 112 .plug = 1, 113 .audit_skip = 1, 114 .ioprio = 1, 115 .iopoll = 1, 116 .async_size = sizeof(struct io_async_rw), 117 .prep = io_prep_write_fixed, 118 .issue = io_write_fixed, 119 }, 120 [IORING_OP_POLL_ADD] = { 121 .needs_file = 1, 122 .unbound_nonreg_file = 1, 123 .audit_skip = 1, 124 .prep = io_poll_add_prep, 125 .issue = io_poll_add, 126 }, 127 [IORING_OP_POLL_REMOVE] = { 128 .audit_skip = 1, 129 .prep = io_poll_remove_prep, 130 .issue = io_poll_remove, 131 }, 132 [IORING_OP_SYNC_FILE_RANGE] = { 133 .needs_file = 1, 134 .audit_skip = 1, 135 .prep = io_sfr_prep, 136 .issue = io_sync_file_range, 137 }, 138 [IORING_OP_SENDMSG] = { 139 .needs_file = 1, 140 .unbound_nonreg_file = 1, 141 .pollout = 1, 142 .ioprio = 1, 143 #if defined(CONFIG_NET) 144 .async_size = sizeof(struct io_async_msghdr), 145 .prep = io_sendmsg_prep, 146 .issue = io_sendmsg, 147 #else 148 .prep = io_eopnotsupp_prep, 149 #endif 150 }, 151 [IORING_OP_RECVMSG] = { 152 .needs_file = 1, 153 .unbound_nonreg_file = 1, 154 .pollin = 1, 155 .buffer_select = 1, 156 .ioprio = 1, 157 #if defined(CONFIG_NET) 158 .async_size = sizeof(struct io_async_msghdr), 159 .prep = io_recvmsg_prep, 160 .issue = io_recvmsg, 161 #else 162 .prep = io_eopnotsupp_prep, 163 #endif 164 }, 165 [IORING_OP_TIMEOUT] = { 166 .audit_skip = 1, 167 .async_size = sizeof(struct io_timeout_data), 168 .prep = io_timeout_prep, 169 .issue = io_timeout, 170 }, 171 [IORING_OP_TIMEOUT_REMOVE] = { 172 /* used by timeout updates' prep() */ 173 .audit_skip = 1, 174 .prep = io_timeout_remove_prep, 175 .issue = io_timeout_remove, 176 }, 177 [IORING_OP_ACCEPT] = { 178 .needs_file = 1, 179 .unbound_nonreg_file = 1, 180 .pollin = 1, 181 .poll_exclusive = 1, 182 .ioprio = 1, /* used for flags */ 183 #if defined(CONFIG_NET) 184 .prep = io_accept_prep, 185 .issue = io_accept, 186 #else 187 .prep = io_eopnotsupp_prep, 188 #endif 189 }, 190 [IORING_OP_ASYNC_CANCEL] = { 191 .audit_skip = 1, 192 .prep = io_async_cancel_prep, 193 .issue = io_async_cancel, 194 }, 195 [IORING_OP_LINK_TIMEOUT] = { 196 .audit_skip = 1, 197 .async_size = sizeof(struct io_timeout_data), 198 .prep = io_link_timeout_prep, 199 .issue = io_no_issue, 200 }, 201 [IORING_OP_CONNECT] = { 202 .needs_file = 1, 203 .unbound_nonreg_file = 1, 204 .pollout = 1, 205 #if defined(CONFIG_NET) 206 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, connect), 207 .async_size = sizeof(struct sockaddr_storage), 208 .prep = io_connect_prep, 209 .issue = io_connect, 210 .filter_populate = io_connect_bpf_populate, 211 #else 212 .prep = io_eopnotsupp_prep, 213 #endif 214 }, 215 [IORING_OP_FALLOCATE] = { 216 .needs_file = 1, 217 .hash_reg_file = 1, 218 .prep = io_fallocate_prep, 219 .issue = io_fallocate, 220 }, 221 [IORING_OP_OPENAT] = { 222 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open), 223 .prep = io_openat_prep, 224 .issue = io_openat, 225 .filter_populate = io_openat_bpf_populate, 226 }, 227 [IORING_OP_CLOSE] = { 228 .prep = io_close_prep, 229 .issue = io_close, 230 }, 231 [IORING_OP_FILES_UPDATE] = { 232 .audit_skip = 1, 233 .iopoll = 1, 234 .prep = io_files_update_prep, 235 .issue = io_files_update, 236 }, 237 [IORING_OP_STATX] = { 238 .audit_skip = 1, 239 .prep = io_statx_prep, 240 .issue = io_statx, 241 }, 242 [IORING_OP_READ] = { 243 .needs_file = 1, 244 .unbound_nonreg_file = 1, 245 .pollin = 1, 246 .buffer_select = 1, 247 .plug = 1, 248 .audit_skip = 1, 249 .ioprio = 1, 250 .iopoll = 1, 251 .async_size = sizeof(struct io_async_rw), 252 .prep = io_prep_read, 253 .issue = io_read, 254 }, 255 [IORING_OP_WRITE] = { 256 .needs_file = 1, 257 .hash_reg_file = 1, 258 .unbound_nonreg_file = 1, 259 .pollout = 1, 260 .plug = 1, 261 .audit_skip = 1, 262 .ioprio = 1, 263 .iopoll = 1, 264 .async_size = sizeof(struct io_async_rw), 265 .prep = io_prep_write, 266 .issue = io_write, 267 }, 268 [IORING_OP_FADVISE] = { 269 .needs_file = 1, 270 .audit_skip = 1, 271 .prep = io_fadvise_prep, 272 .issue = io_fadvise, 273 }, 274 [IORING_OP_MADVISE] = { 275 .audit_skip = 1, 276 .prep = io_madvise_prep, 277 .issue = io_madvise, 278 }, 279 [IORING_OP_SEND] = { 280 .needs_file = 1, 281 .unbound_nonreg_file = 1, 282 .pollout = 1, 283 .audit_skip = 1, 284 .ioprio = 1, 285 .buffer_select = 1, 286 #if defined(CONFIG_NET) 287 .async_size = sizeof(struct io_async_msghdr), 288 .prep = io_sendmsg_prep, 289 .issue = io_send, 290 #else 291 .prep = io_eopnotsupp_prep, 292 #endif 293 }, 294 [IORING_OP_RECV] = { 295 .needs_file = 1, 296 .unbound_nonreg_file = 1, 297 .pollin = 1, 298 .buffer_select = 1, 299 .audit_skip = 1, 300 .ioprio = 1, 301 #if defined(CONFIG_NET) 302 .async_size = sizeof(struct io_async_msghdr), 303 .prep = io_recvmsg_prep, 304 .issue = io_recv, 305 #else 306 .prep = io_eopnotsupp_prep, 307 #endif 308 }, 309 [IORING_OP_OPENAT2] = { 310 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open), 311 .prep = io_openat2_prep, 312 .issue = io_openat2, 313 .filter_populate = io_openat_bpf_populate, 314 }, 315 [IORING_OP_EPOLL_CTL] = { 316 .unbound_nonreg_file = 1, 317 .audit_skip = 1, 318 #if defined(CONFIG_EPOLL) 319 .prep = io_epoll_ctl_prep, 320 .issue = io_epoll_ctl, 321 #else 322 .prep = io_eopnotsupp_prep, 323 #endif 324 }, 325 [IORING_OP_SPLICE] = { 326 .needs_file = 1, 327 .hash_reg_file = 1, 328 .unbound_nonreg_file = 1, 329 .audit_skip = 1, 330 .prep = io_splice_prep, 331 .issue = io_splice, 332 }, 333 [IORING_OP_PROVIDE_BUFFERS] = { 334 .audit_skip = 1, 335 .iopoll = 1, 336 .prep = io_provide_buffers_prep, 337 .issue = io_manage_buffers_legacy, 338 }, 339 [IORING_OP_REMOVE_BUFFERS] = { 340 .audit_skip = 1, 341 .iopoll = 1, 342 .prep = io_remove_buffers_prep, 343 .issue = io_manage_buffers_legacy, 344 }, 345 [IORING_OP_TEE] = { 346 .needs_file = 1, 347 .hash_reg_file = 1, 348 .unbound_nonreg_file = 1, 349 .audit_skip = 1, 350 .prep = io_tee_prep, 351 .issue = io_tee, 352 }, 353 [IORING_OP_SHUTDOWN] = { 354 .needs_file = 1, 355 #if defined(CONFIG_NET) 356 .prep = io_shutdown_prep, 357 .issue = io_shutdown, 358 #else 359 .prep = io_eopnotsupp_prep, 360 #endif 361 }, 362 [IORING_OP_RENAMEAT] = { 363 .prep = io_renameat_prep, 364 .issue = io_renameat, 365 }, 366 [IORING_OP_UNLINKAT] = { 367 .prep = io_unlinkat_prep, 368 .issue = io_unlinkat, 369 }, 370 [IORING_OP_MKDIRAT] = { 371 .prep = io_mkdirat_prep, 372 .issue = io_mkdirat, 373 }, 374 [IORING_OP_SYMLINKAT] = { 375 .prep = io_symlinkat_prep, 376 .issue = io_symlinkat, 377 }, 378 [IORING_OP_LINKAT] = { 379 .prep = io_linkat_prep, 380 .issue = io_linkat, 381 }, 382 [IORING_OP_MSG_RING] = { 383 .needs_file = 1, 384 .iopoll = 1, 385 .prep = io_msg_ring_prep, 386 .issue = io_msg_ring, 387 }, 388 [IORING_OP_FSETXATTR] = { 389 .needs_file = 1, 390 .prep = io_fsetxattr_prep, 391 .issue = io_fsetxattr, 392 }, 393 [IORING_OP_SETXATTR] = { 394 .prep = io_setxattr_prep, 395 .issue = io_setxattr, 396 }, 397 [IORING_OP_FGETXATTR] = { 398 .needs_file = 1, 399 .prep = io_fgetxattr_prep, 400 .issue = io_fgetxattr, 401 }, 402 [IORING_OP_GETXATTR] = { 403 .prep = io_getxattr_prep, 404 .issue = io_getxattr, 405 }, 406 [IORING_OP_SOCKET] = { 407 .audit_skip = 1, 408 #if defined(CONFIG_NET) 409 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, socket), 410 .prep = io_socket_prep, 411 .issue = io_socket, 412 .filter_populate = io_socket_bpf_populate, 413 #else 414 .prep = io_eopnotsupp_prep, 415 #endif 416 }, 417 [IORING_OP_URING_CMD] = { 418 .buffer_select = 1, 419 .needs_file = 1, 420 .plug = 1, 421 .iopoll = 1, 422 .async_size = sizeof(struct io_async_cmd), 423 .prep = io_uring_cmd_prep, 424 .issue = io_uring_cmd, 425 }, 426 [IORING_OP_SEND_ZC] = { 427 .needs_file = 1, 428 .unbound_nonreg_file = 1, 429 .pollout = 1, 430 .audit_skip = 1, 431 .ioprio = 1, 432 #if defined(CONFIG_NET) 433 .async_size = sizeof(struct io_async_msghdr), 434 .prep = io_send_zc_prep, 435 .issue = io_sendmsg_zc, 436 #else 437 .prep = io_eopnotsupp_prep, 438 #endif 439 }, 440 [IORING_OP_SENDMSG_ZC] = { 441 .needs_file = 1, 442 .unbound_nonreg_file = 1, 443 .pollout = 1, 444 .ioprio = 1, 445 #if defined(CONFIG_NET) 446 .async_size = sizeof(struct io_async_msghdr), 447 .prep = io_send_zc_prep, 448 .issue = io_sendmsg_zc, 449 #else 450 .prep = io_eopnotsupp_prep, 451 #endif 452 }, 453 [IORING_OP_READ_MULTISHOT] = { 454 .needs_file = 1, 455 .unbound_nonreg_file = 1, 456 .pollin = 1, 457 .buffer_select = 1, 458 .audit_skip = 1, 459 .async_size = sizeof(struct io_async_rw), 460 .prep = io_read_mshot_prep, 461 .issue = io_read_mshot, 462 }, 463 [IORING_OP_WAITID] = { 464 .async_size = sizeof(struct io_waitid_async), 465 .prep = io_waitid_prep, 466 .issue = io_waitid, 467 }, 468 [IORING_OP_FUTEX_WAIT] = { 469 #if defined(CONFIG_FUTEX) 470 .prep = io_futex_prep, 471 .issue = io_futex_wait, 472 #else 473 .prep = io_eopnotsupp_prep, 474 #endif 475 }, 476 [IORING_OP_FUTEX_WAKE] = { 477 #if defined(CONFIG_FUTEX) 478 .prep = io_futex_prep, 479 .issue = io_futex_wake, 480 #else 481 .prep = io_eopnotsupp_prep, 482 #endif 483 }, 484 [IORING_OP_FUTEX_WAITV] = { 485 #if defined(CONFIG_FUTEX) 486 .prep = io_futexv_prep, 487 .issue = io_futexv_wait, 488 #else 489 .prep = io_eopnotsupp_prep, 490 #endif 491 }, 492 [IORING_OP_FIXED_FD_INSTALL] = { 493 .needs_file = 1, 494 .prep = io_install_fixed_fd_prep, 495 .issue = io_install_fixed_fd, 496 }, 497 [IORING_OP_FTRUNCATE] = { 498 .needs_file = 1, 499 .hash_reg_file = 1, 500 .prep = io_ftruncate_prep, 501 .issue = io_ftruncate, 502 }, 503 [IORING_OP_BIND] = { 504 #if defined(CONFIG_NET) 505 .needs_file = 1, 506 .prep = io_bind_prep, 507 .issue = io_bind, 508 .async_size = sizeof(struct sockaddr_storage), 509 #else 510 .prep = io_eopnotsupp_prep, 511 #endif 512 }, 513 [IORING_OP_LISTEN] = { 514 #if defined(CONFIG_NET) 515 .needs_file = 1, 516 .prep = io_listen_prep, 517 .issue = io_listen, 518 #else 519 .prep = io_eopnotsupp_prep, 520 #endif 521 }, 522 [IORING_OP_RECV_ZC] = { 523 .audit_skip = 1, 524 .needs_file = 1, 525 .unbound_nonreg_file = 1, 526 .pollin = 1, 527 .ioprio = 1, 528 #if defined(CONFIG_NET) 529 .prep = io_recvzc_prep, 530 .issue = io_recvzc, 531 #else 532 .prep = io_eopnotsupp_prep, 533 #endif 534 }, 535 [IORING_OP_EPOLL_WAIT] = { 536 .needs_file = 1, 537 .audit_skip = 1, 538 .pollin = 1, 539 #if defined(CONFIG_EPOLL) 540 .prep = io_epoll_wait_prep, 541 .issue = io_epoll_wait, 542 #else 543 .prep = io_eopnotsupp_prep, 544 #endif 545 }, 546 [IORING_OP_READV_FIXED] = { 547 .needs_file = 1, 548 .unbound_nonreg_file = 1, 549 .pollin = 1, 550 .plug = 1, 551 .audit_skip = 1, 552 .ioprio = 1, 553 .iopoll = 1, 554 .vectored = 1, 555 .async_size = sizeof(struct io_async_rw), 556 .prep = io_prep_readv_fixed, 557 .issue = io_read, 558 }, 559 [IORING_OP_WRITEV_FIXED] = { 560 .needs_file = 1, 561 .hash_reg_file = 1, 562 .unbound_nonreg_file = 1, 563 .pollout = 1, 564 .plug = 1, 565 .audit_skip = 1, 566 .ioprio = 1, 567 .iopoll = 1, 568 .vectored = 1, 569 .async_size = sizeof(struct io_async_rw), 570 .prep = io_prep_writev_fixed, 571 .issue = io_write, 572 }, 573 [IORING_OP_PIPE] = { 574 .prep = io_pipe_prep, 575 .issue = io_pipe, 576 }, 577 [IORING_OP_NOP128] = { 578 .audit_skip = 1, 579 .iopoll = 1, 580 .is_128 = 1, 581 .prep = io_nop_prep, 582 .issue = io_nop, 583 }, 584 [IORING_OP_URING_CMD128] = { 585 .buffer_select = 1, 586 .needs_file = 1, 587 .plug = 1, 588 .iopoll = 1, 589 .is_128 = 1, 590 .async_size = sizeof(struct io_async_cmd), 591 .prep = io_uring_cmd_prep, 592 .issue = io_uring_cmd, 593 }, 594 }; 595 596 const struct io_cold_def io_cold_defs[] = { 597 [IORING_OP_NOP] = { 598 .name = "NOP", 599 }, 600 [IORING_OP_READV] = { 601 .name = "READV", 602 .cleanup = io_readv_writev_cleanup, 603 .fail = io_rw_fail, 604 }, 605 [IORING_OP_WRITEV] = { 606 .name = "WRITEV", 607 .cleanup = io_readv_writev_cleanup, 608 .fail = io_rw_fail, 609 }, 610 [IORING_OP_FSYNC] = { 611 .name = "FSYNC", 612 }, 613 [IORING_OP_READ_FIXED] = { 614 .name = "READ_FIXED", 615 .cleanup = io_readv_writev_cleanup, 616 .fail = io_rw_fail, 617 }, 618 [IORING_OP_WRITE_FIXED] = { 619 .name = "WRITE_FIXED", 620 .cleanup = io_readv_writev_cleanup, 621 .fail = io_rw_fail, 622 }, 623 [IORING_OP_POLL_ADD] = { 624 .name = "POLL_ADD", 625 }, 626 [IORING_OP_POLL_REMOVE] = { 627 .name = "POLL_REMOVE", 628 }, 629 [IORING_OP_SYNC_FILE_RANGE] = { 630 .name = "SYNC_FILE_RANGE", 631 }, 632 [IORING_OP_SENDMSG] = { 633 .name = "SENDMSG", 634 #if defined(CONFIG_NET) 635 .cleanup = io_sendmsg_recvmsg_cleanup, 636 .fail = io_sendrecv_fail, 637 #endif 638 }, 639 [IORING_OP_RECVMSG] = { 640 .name = "RECVMSG", 641 #if defined(CONFIG_NET) 642 .cleanup = io_sendmsg_recvmsg_cleanup, 643 .fail = io_sendrecv_fail, 644 #endif 645 }, 646 [IORING_OP_TIMEOUT] = { 647 .name = "TIMEOUT", 648 }, 649 [IORING_OP_TIMEOUT_REMOVE] = { 650 .name = "TIMEOUT_REMOVE", 651 }, 652 [IORING_OP_ACCEPT] = { 653 .name = "ACCEPT", 654 }, 655 [IORING_OP_ASYNC_CANCEL] = { 656 .name = "ASYNC_CANCEL", 657 }, 658 [IORING_OP_LINK_TIMEOUT] = { 659 .name = "LINK_TIMEOUT", 660 }, 661 [IORING_OP_CONNECT] = { 662 .name = "CONNECT", 663 }, 664 [IORING_OP_FALLOCATE] = { 665 .name = "FALLOCATE", 666 }, 667 [IORING_OP_OPENAT] = { 668 .name = "OPENAT", 669 .cleanup = io_open_cleanup, 670 }, 671 [IORING_OP_CLOSE] = { 672 .name = "CLOSE", 673 }, 674 [IORING_OP_FILES_UPDATE] = { 675 .name = "FILES_UPDATE", 676 }, 677 [IORING_OP_STATX] = { 678 .name = "STATX", 679 .cleanup = io_statx_cleanup, 680 }, 681 [IORING_OP_READ] = { 682 .name = "READ", 683 .cleanup = io_readv_writev_cleanup, 684 .fail = io_rw_fail, 685 }, 686 [IORING_OP_WRITE] = { 687 .name = "WRITE", 688 .cleanup = io_readv_writev_cleanup, 689 .fail = io_rw_fail, 690 }, 691 [IORING_OP_FADVISE] = { 692 .name = "FADVISE", 693 }, 694 [IORING_OP_MADVISE] = { 695 .name = "MADVISE", 696 }, 697 [IORING_OP_SEND] = { 698 .name = "SEND", 699 #if defined(CONFIG_NET) 700 .cleanup = io_sendmsg_recvmsg_cleanup, 701 .fail = io_sendrecv_fail, 702 #endif 703 }, 704 [IORING_OP_RECV] = { 705 .name = "RECV", 706 #if defined(CONFIG_NET) 707 .cleanup = io_sendmsg_recvmsg_cleanup, 708 .fail = io_sendrecv_fail, 709 #endif 710 }, 711 [IORING_OP_OPENAT2] = { 712 .name = "OPENAT2", 713 .cleanup = io_open_cleanup, 714 }, 715 [IORING_OP_EPOLL_CTL] = { 716 .name = "EPOLL", 717 }, 718 [IORING_OP_SPLICE] = { 719 .name = "SPLICE", 720 .cleanup = io_splice_cleanup, 721 }, 722 [IORING_OP_PROVIDE_BUFFERS] = { 723 .name = "PROVIDE_BUFFERS", 724 }, 725 [IORING_OP_REMOVE_BUFFERS] = { 726 .name = "REMOVE_BUFFERS", 727 }, 728 [IORING_OP_TEE] = { 729 .name = "TEE", 730 .cleanup = io_splice_cleanup, 731 }, 732 [IORING_OP_SHUTDOWN] = { 733 .name = "SHUTDOWN", 734 }, 735 [IORING_OP_RENAMEAT] = { 736 .name = "RENAMEAT", 737 .cleanup = io_renameat_cleanup, 738 }, 739 [IORING_OP_UNLINKAT] = { 740 .name = "UNLINKAT", 741 .cleanup = io_unlinkat_cleanup, 742 }, 743 [IORING_OP_MKDIRAT] = { 744 .name = "MKDIRAT", 745 .cleanup = io_mkdirat_cleanup, 746 }, 747 [IORING_OP_SYMLINKAT] = { 748 .name = "SYMLINKAT", 749 .cleanup = io_link_cleanup, 750 }, 751 [IORING_OP_LINKAT] = { 752 .name = "LINKAT", 753 .cleanup = io_link_cleanup, 754 }, 755 [IORING_OP_MSG_RING] = { 756 .name = "MSG_RING", 757 .cleanup = io_msg_ring_cleanup, 758 }, 759 [IORING_OP_FSETXATTR] = { 760 .name = "FSETXATTR", 761 .cleanup = io_xattr_cleanup, 762 }, 763 [IORING_OP_SETXATTR] = { 764 .name = "SETXATTR", 765 .cleanup = io_xattr_cleanup, 766 }, 767 [IORING_OP_FGETXATTR] = { 768 .name = "FGETXATTR", 769 .cleanup = io_xattr_cleanup, 770 }, 771 [IORING_OP_GETXATTR] = { 772 .name = "GETXATTR", 773 .cleanup = io_xattr_cleanup, 774 }, 775 [IORING_OP_SOCKET] = { 776 .name = "SOCKET", 777 }, 778 [IORING_OP_URING_CMD] = { 779 .name = "URING_CMD", 780 .sqe_copy = io_uring_cmd_sqe_copy, 781 .cleanup = io_uring_cmd_cleanup, 782 }, 783 [IORING_OP_SEND_ZC] = { 784 .name = "SEND_ZC", 785 #if defined(CONFIG_NET) 786 .cleanup = io_send_zc_cleanup, 787 .fail = io_sendrecv_fail, 788 #endif 789 }, 790 [IORING_OP_SENDMSG_ZC] = { 791 .name = "SENDMSG_ZC", 792 #if defined(CONFIG_NET) 793 .cleanup = io_send_zc_cleanup, 794 .fail = io_sendrecv_fail, 795 #endif 796 }, 797 [IORING_OP_READ_MULTISHOT] = { 798 .name = "READ_MULTISHOT", 799 .cleanup = io_readv_writev_cleanup, 800 }, 801 [IORING_OP_WAITID] = { 802 .name = "WAITID", 803 }, 804 [IORING_OP_FUTEX_WAIT] = { 805 .name = "FUTEX_WAIT", 806 }, 807 [IORING_OP_FUTEX_WAKE] = { 808 .name = "FUTEX_WAKE", 809 }, 810 [IORING_OP_FUTEX_WAITV] = { 811 .name = "FUTEX_WAITV", 812 }, 813 [IORING_OP_FIXED_FD_INSTALL] = { 814 .name = "FIXED_FD_INSTALL", 815 }, 816 [IORING_OP_FTRUNCATE] = { 817 .name = "FTRUNCATE", 818 }, 819 [IORING_OP_BIND] = { 820 .name = "BIND", 821 }, 822 [IORING_OP_LISTEN] = { 823 .name = "LISTEN", 824 }, 825 [IORING_OP_RECV_ZC] = { 826 .name = "RECV_ZC", 827 }, 828 [IORING_OP_EPOLL_WAIT] = { 829 .name = "EPOLL_WAIT", 830 }, 831 [IORING_OP_READV_FIXED] = { 832 .name = "READV_FIXED", 833 .cleanup = io_readv_writev_cleanup, 834 .fail = io_rw_fail, 835 }, 836 [IORING_OP_WRITEV_FIXED] = { 837 .name = "WRITEV_FIXED", 838 .cleanup = io_readv_writev_cleanup, 839 .fail = io_rw_fail, 840 }, 841 [IORING_OP_PIPE] = { 842 .name = "PIPE", 843 }, 844 [IORING_OP_NOP128] = { 845 .name = "NOP128", 846 }, 847 [IORING_OP_URING_CMD128] = { 848 .name = "URING_CMD128", 849 .sqe_copy = io_uring_cmd_sqe_copy, 850 .cleanup = io_uring_cmd_cleanup, 851 }, 852 }; 853 854 const char *io_uring_get_opcode(u8 opcode) 855 { 856 if (opcode < IORING_OP_LAST) 857 return io_cold_defs[opcode].name; 858 return "INVALID"; 859 } 860 861 bool io_uring_op_supported(u8 opcode) 862 { 863 if (opcode < IORING_OP_LAST && 864 io_issue_defs[opcode].prep != io_eopnotsupp_prep) 865 return true; 866 return false; 867 } 868 869 void __init io_uring_optable_init(void) 870 { 871 int i; 872 873 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST); 874 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST); 875 876 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) { 877 BUG_ON(!io_issue_defs[i].prep); 878 if (io_issue_defs[i].prep != io_eopnotsupp_prep) 879 BUG_ON(!io_issue_defs[i].issue); 880 WARN_ON_ONCE(!io_cold_defs[i].name); 881 } 882 } 883