1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * io_uring opcode handling table
4 */
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
7 #include <linux/fs.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
10 #include <linux/io_uring/cmd.h>
11
12 #include "io_uring.h"
13 #include "opdef.h"
14 #include "refs.h"
15 #include "tctx.h"
16 #include "sqpoll.h"
17 #include "fdinfo.h"
18 #include "kbuf.h"
19 #include "rsrc.h"
20
21 #include "xattr.h"
22 #include "nop.h"
23 #include "fs.h"
24 #include "splice.h"
25 #include "sync.h"
26 #include "advise.h"
27 #include "openclose.h"
28 #include "uring_cmd.h"
29 #include "epoll.h"
30 #include "statx.h"
31 #include "net.h"
32 #include "msg_ring.h"
33 #include "timeout.h"
34 #include "poll.h"
35 #include "cancel.h"
36 #include "rw.h"
37 #include "waitid.h"
38 #include "futex.h"
39 #include "truncate.h"
40 #include "zcrx.h"
41
io_no_issue(struct io_kiocb * req,unsigned int issue_flags)42 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
43 {
44 WARN_ON_ONCE(1);
45 return -ECANCELED;
46 }
47
io_eopnotsupp_prep(struct io_kiocb * kiocb,const struct io_uring_sqe * sqe)48 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
49 const struct io_uring_sqe *sqe)
50 {
51 return -EOPNOTSUPP;
52 }
53
54 const struct io_issue_def io_issue_defs[] = {
55 [IORING_OP_NOP] = {
56 .audit_skip = 1,
57 .iopoll = 1,
58 .prep = io_nop_prep,
59 .issue = io_nop,
60 },
61 [IORING_OP_READV] = {
62 .needs_file = 1,
63 .unbound_nonreg_file = 1,
64 .pollin = 1,
65 .buffer_select = 1,
66 .plug = 1,
67 .audit_skip = 1,
68 .ioprio = 1,
69 .iopoll = 1,
70 .vectored = 1,
71 .async_size = sizeof(struct io_async_rw),
72 .prep = io_prep_readv,
73 .issue = io_read,
74 },
75 [IORING_OP_WRITEV] = {
76 .needs_file = 1,
77 .hash_reg_file = 1,
78 .unbound_nonreg_file = 1,
79 .pollout = 1,
80 .plug = 1,
81 .audit_skip = 1,
82 .ioprio = 1,
83 .iopoll = 1,
84 .vectored = 1,
85 .async_size = sizeof(struct io_async_rw),
86 .prep = io_prep_writev,
87 .issue = io_write,
88 },
89 [IORING_OP_FSYNC] = {
90 .needs_file = 1,
91 .audit_skip = 1,
92 .prep = io_fsync_prep,
93 .issue = io_fsync,
94 },
95 [IORING_OP_READ_FIXED] = {
96 .needs_file = 1,
97 .unbound_nonreg_file = 1,
98 .pollin = 1,
99 .plug = 1,
100 .audit_skip = 1,
101 .ioprio = 1,
102 .iopoll = 1,
103 .async_size = sizeof(struct io_async_rw),
104 .prep = io_prep_read_fixed,
105 .issue = io_read_fixed,
106 },
107 [IORING_OP_WRITE_FIXED] = {
108 .needs_file = 1,
109 .hash_reg_file = 1,
110 .unbound_nonreg_file = 1,
111 .pollout = 1,
112 .plug = 1,
113 .audit_skip = 1,
114 .ioprio = 1,
115 .iopoll = 1,
116 .async_size = sizeof(struct io_async_rw),
117 .prep = io_prep_write_fixed,
118 .issue = io_write_fixed,
119 },
120 [IORING_OP_POLL_ADD] = {
121 .needs_file = 1,
122 .unbound_nonreg_file = 1,
123 .audit_skip = 1,
124 .prep = io_poll_add_prep,
125 .issue = io_poll_add,
126 },
127 [IORING_OP_POLL_REMOVE] = {
128 .audit_skip = 1,
129 .prep = io_poll_remove_prep,
130 .issue = io_poll_remove,
131 },
132 [IORING_OP_SYNC_FILE_RANGE] = {
133 .needs_file = 1,
134 .audit_skip = 1,
135 .prep = io_sfr_prep,
136 .issue = io_sync_file_range,
137 },
138 [IORING_OP_SENDMSG] = {
139 .needs_file = 1,
140 .unbound_nonreg_file = 1,
141 .pollout = 1,
142 .ioprio = 1,
143 #if defined(CONFIG_NET)
144 .async_size = sizeof(struct io_async_msghdr),
145 .prep = io_sendmsg_prep,
146 .issue = io_sendmsg,
147 #else
148 .prep = io_eopnotsupp_prep,
149 #endif
150 },
151 [IORING_OP_RECVMSG] = {
152 .needs_file = 1,
153 .unbound_nonreg_file = 1,
154 .pollin = 1,
155 .buffer_select = 1,
156 .ioprio = 1,
157 #if defined(CONFIG_NET)
158 .async_size = sizeof(struct io_async_msghdr),
159 .prep = io_recvmsg_prep,
160 .issue = io_recvmsg,
161 #else
162 .prep = io_eopnotsupp_prep,
163 #endif
164 },
165 [IORING_OP_TIMEOUT] = {
166 .audit_skip = 1,
167 .async_size = sizeof(struct io_timeout_data),
168 .prep = io_timeout_prep,
169 .issue = io_timeout,
170 },
171 [IORING_OP_TIMEOUT_REMOVE] = {
172 /* used by timeout updates' prep() */
173 .audit_skip = 1,
174 .prep = io_timeout_remove_prep,
175 .issue = io_timeout_remove,
176 },
177 [IORING_OP_ACCEPT] = {
178 .needs_file = 1,
179 .unbound_nonreg_file = 1,
180 .pollin = 1,
181 .poll_exclusive = 1,
182 .ioprio = 1, /* used for flags */
183 #if defined(CONFIG_NET)
184 .prep = io_accept_prep,
185 .issue = io_accept,
186 #else
187 .prep = io_eopnotsupp_prep,
188 #endif
189 },
190 [IORING_OP_ASYNC_CANCEL] = {
191 .audit_skip = 1,
192 .prep = io_async_cancel_prep,
193 .issue = io_async_cancel,
194 },
195 [IORING_OP_LINK_TIMEOUT] = {
196 .audit_skip = 1,
197 .async_size = sizeof(struct io_timeout_data),
198 .prep = io_link_timeout_prep,
199 .issue = io_no_issue,
200 },
201 [IORING_OP_CONNECT] = {
202 .needs_file = 1,
203 .unbound_nonreg_file = 1,
204 .pollout = 1,
205 #if defined(CONFIG_NET)
206 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, connect),
207 .async_size = sizeof(struct sockaddr_storage),
208 .prep = io_connect_prep,
209 .issue = io_connect,
210 .filter_populate = io_connect_bpf_populate,
211 #else
212 .prep = io_eopnotsupp_prep,
213 #endif
214 },
215 [IORING_OP_FALLOCATE] = {
216 .needs_file = 1,
217 .hash_reg_file = 1,
218 .prep = io_fallocate_prep,
219 .issue = io_fallocate,
220 },
221 [IORING_OP_OPENAT] = {
222 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
223 .prep = io_openat_prep,
224 .issue = io_openat,
225 .filter_populate = io_openat_bpf_populate,
226 },
227 [IORING_OP_CLOSE] = {
228 .prep = io_close_prep,
229 .issue = io_close,
230 },
231 [IORING_OP_FILES_UPDATE] = {
232 .audit_skip = 1,
233 .iopoll = 1,
234 .prep = io_files_update_prep,
235 .issue = io_files_update,
236 },
237 [IORING_OP_STATX] = {
238 .audit_skip = 1,
239 .prep = io_statx_prep,
240 .issue = io_statx,
241 },
242 [IORING_OP_READ] = {
243 .needs_file = 1,
244 .unbound_nonreg_file = 1,
245 .pollin = 1,
246 .buffer_select = 1,
247 .plug = 1,
248 .audit_skip = 1,
249 .ioprio = 1,
250 .iopoll = 1,
251 .async_size = sizeof(struct io_async_rw),
252 .prep = io_prep_read,
253 .issue = io_read,
254 },
255 [IORING_OP_WRITE] = {
256 .needs_file = 1,
257 .hash_reg_file = 1,
258 .unbound_nonreg_file = 1,
259 .pollout = 1,
260 .plug = 1,
261 .audit_skip = 1,
262 .ioprio = 1,
263 .iopoll = 1,
264 .async_size = sizeof(struct io_async_rw),
265 .prep = io_prep_write,
266 .issue = io_write,
267 },
268 [IORING_OP_FADVISE] = {
269 .needs_file = 1,
270 .audit_skip = 1,
271 .prep = io_fadvise_prep,
272 .issue = io_fadvise,
273 },
274 [IORING_OP_MADVISE] = {
275 .audit_skip = 1,
276 .prep = io_madvise_prep,
277 .issue = io_madvise,
278 },
279 [IORING_OP_SEND] = {
280 .needs_file = 1,
281 .unbound_nonreg_file = 1,
282 .pollout = 1,
283 .audit_skip = 1,
284 .ioprio = 1,
285 .buffer_select = 1,
286 #if defined(CONFIG_NET)
287 .async_size = sizeof(struct io_async_msghdr),
288 .prep = io_sendmsg_prep,
289 .issue = io_send,
290 #else
291 .prep = io_eopnotsupp_prep,
292 #endif
293 },
294 [IORING_OP_RECV] = {
295 .needs_file = 1,
296 .unbound_nonreg_file = 1,
297 .pollin = 1,
298 .buffer_select = 1,
299 .audit_skip = 1,
300 .ioprio = 1,
301 #if defined(CONFIG_NET)
302 .async_size = sizeof(struct io_async_msghdr),
303 .prep = io_recvmsg_prep,
304 .issue = io_recv,
305 #else
306 .prep = io_eopnotsupp_prep,
307 #endif
308 },
309 [IORING_OP_OPENAT2] = {
310 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, open),
311 .prep = io_openat2_prep,
312 .issue = io_openat2,
313 .filter_populate = io_openat_bpf_populate,
314 },
315 [IORING_OP_EPOLL_CTL] = {
316 .unbound_nonreg_file = 1,
317 .audit_skip = 1,
318 #if defined(CONFIG_EPOLL)
319 .prep = io_epoll_ctl_prep,
320 .issue = io_epoll_ctl,
321 #else
322 .prep = io_eopnotsupp_prep,
323 #endif
324 },
325 [IORING_OP_SPLICE] = {
326 .needs_file = 1,
327 .hash_reg_file = 1,
328 .unbound_nonreg_file = 1,
329 .audit_skip = 1,
330 .prep = io_splice_prep,
331 .issue = io_splice,
332 },
333 [IORING_OP_PROVIDE_BUFFERS] = {
334 .audit_skip = 1,
335 .iopoll = 1,
336 .prep = io_provide_buffers_prep,
337 .issue = io_manage_buffers_legacy,
338 },
339 [IORING_OP_REMOVE_BUFFERS] = {
340 .audit_skip = 1,
341 .iopoll = 1,
342 .prep = io_remove_buffers_prep,
343 .issue = io_manage_buffers_legacy,
344 },
345 [IORING_OP_TEE] = {
346 .needs_file = 1,
347 .hash_reg_file = 1,
348 .unbound_nonreg_file = 1,
349 .audit_skip = 1,
350 .prep = io_tee_prep,
351 .issue = io_tee,
352 },
353 [IORING_OP_SHUTDOWN] = {
354 .needs_file = 1,
355 #if defined(CONFIG_NET)
356 .prep = io_shutdown_prep,
357 .issue = io_shutdown,
358 #else
359 .prep = io_eopnotsupp_prep,
360 #endif
361 },
362 [IORING_OP_RENAMEAT] = {
363 .prep = io_renameat_prep,
364 .issue = io_renameat,
365 },
366 [IORING_OP_UNLINKAT] = {
367 .prep = io_unlinkat_prep,
368 .issue = io_unlinkat,
369 },
370 [IORING_OP_MKDIRAT] = {
371 .prep = io_mkdirat_prep,
372 .issue = io_mkdirat,
373 },
374 [IORING_OP_SYMLINKAT] = {
375 .prep = io_symlinkat_prep,
376 .issue = io_symlinkat,
377 },
378 [IORING_OP_LINKAT] = {
379 .prep = io_linkat_prep,
380 .issue = io_linkat,
381 },
382 [IORING_OP_MSG_RING] = {
383 .needs_file = 1,
384 .iopoll = 1,
385 .prep = io_msg_ring_prep,
386 .issue = io_msg_ring,
387 },
388 [IORING_OP_FSETXATTR] = {
389 .needs_file = 1,
390 .prep = io_fsetxattr_prep,
391 .issue = io_fsetxattr,
392 },
393 [IORING_OP_SETXATTR] = {
394 .prep = io_setxattr_prep,
395 .issue = io_setxattr,
396 },
397 [IORING_OP_FGETXATTR] = {
398 .needs_file = 1,
399 .prep = io_fgetxattr_prep,
400 .issue = io_fgetxattr,
401 },
402 [IORING_OP_GETXATTR] = {
403 .prep = io_getxattr_prep,
404 .issue = io_getxattr,
405 },
406 [IORING_OP_SOCKET] = {
407 .audit_skip = 1,
408 #if defined(CONFIG_NET)
409 .filter_pdu_size = sizeof_field(struct io_uring_bpf_ctx, socket),
410 .prep = io_socket_prep,
411 .issue = io_socket,
412 .filter_populate = io_socket_bpf_populate,
413 #else
414 .prep = io_eopnotsupp_prep,
415 #endif
416 },
417 [IORING_OP_URING_CMD] = {
418 .buffer_select = 1,
419 .needs_file = 1,
420 .plug = 1,
421 .iopoll = 1,
422 .async_size = sizeof(struct io_async_cmd),
423 .prep = io_uring_cmd_prep,
424 .issue = io_uring_cmd,
425 },
426 [IORING_OP_SEND_ZC] = {
427 .needs_file = 1,
428 .unbound_nonreg_file = 1,
429 .pollout = 1,
430 .audit_skip = 1,
431 .ioprio = 1,
432 #if defined(CONFIG_NET)
433 .async_size = sizeof(struct io_async_msghdr),
434 .prep = io_send_zc_prep,
435 .issue = io_sendmsg_zc,
436 #else
437 .prep = io_eopnotsupp_prep,
438 #endif
439 },
440 [IORING_OP_SENDMSG_ZC] = {
441 .needs_file = 1,
442 .unbound_nonreg_file = 1,
443 .pollout = 1,
444 .ioprio = 1,
445 #if defined(CONFIG_NET)
446 .async_size = sizeof(struct io_async_msghdr),
447 .prep = io_send_zc_prep,
448 .issue = io_sendmsg_zc,
449 #else
450 .prep = io_eopnotsupp_prep,
451 #endif
452 },
453 [IORING_OP_READ_MULTISHOT] = {
454 .needs_file = 1,
455 .unbound_nonreg_file = 1,
456 .pollin = 1,
457 .buffer_select = 1,
458 .audit_skip = 1,
459 .async_size = sizeof(struct io_async_rw),
460 .prep = io_read_mshot_prep,
461 .issue = io_read_mshot,
462 },
463 [IORING_OP_WAITID] = {
464 .async_size = sizeof(struct io_waitid_async),
465 .prep = io_waitid_prep,
466 .issue = io_waitid,
467 },
468 [IORING_OP_FUTEX_WAIT] = {
469 #if defined(CONFIG_FUTEX)
470 .prep = io_futex_wait_prep,
471 .issue = io_futex_wait,
472 #else
473 .prep = io_eopnotsupp_prep,
474 #endif
475 },
476 [IORING_OP_FUTEX_WAKE] = {
477 #if defined(CONFIG_FUTEX)
478 .prep = io_futex_prep,
479 .issue = io_futex_wake,
480 #else
481 .prep = io_eopnotsupp_prep,
482 #endif
483 },
484 [IORING_OP_FUTEX_WAITV] = {
485 #if defined(CONFIG_FUTEX)
486 .prep = io_futexv_prep,
487 .issue = io_futexv_wait,
488 #else
489 .prep = io_eopnotsupp_prep,
490 #endif
491 },
492 [IORING_OP_FIXED_FD_INSTALL] = {
493 .needs_file = 1,
494 .prep = io_install_fixed_fd_prep,
495 .issue = io_install_fixed_fd,
496 },
497 [IORING_OP_FTRUNCATE] = {
498 .needs_file = 1,
499 .hash_reg_file = 1,
500 .prep = io_ftruncate_prep,
501 .issue = io_ftruncate,
502 },
503 [IORING_OP_BIND] = {
504 #if defined(CONFIG_NET)
505 .needs_file = 1,
506 .prep = io_bind_prep,
507 .issue = io_bind,
508 .async_size = sizeof(struct sockaddr_storage),
509 #else
510 .prep = io_eopnotsupp_prep,
511 #endif
512 },
513 [IORING_OP_LISTEN] = {
514 #if defined(CONFIG_NET)
515 .needs_file = 1,
516 .prep = io_listen_prep,
517 .issue = io_listen,
518 #else
519 .prep = io_eopnotsupp_prep,
520 #endif
521 },
522 [IORING_OP_RECV_ZC] = {
523 .audit_skip = 1,
524 .needs_file = 1,
525 .unbound_nonreg_file = 1,
526 .pollin = 1,
527 .ioprio = 1,
528 #if defined(CONFIG_NET)
529 .prep = io_recvzc_prep,
530 .issue = io_recvzc,
531 #else
532 .prep = io_eopnotsupp_prep,
533 #endif
534 },
535 [IORING_OP_EPOLL_WAIT] = {
536 .needs_file = 1,
537 .audit_skip = 1,
538 .pollin = 1,
539 #if defined(CONFIG_EPOLL)
540 .prep = io_epoll_wait_prep,
541 .issue = io_epoll_wait,
542 #else
543 .prep = io_eopnotsupp_prep,
544 #endif
545 },
546 [IORING_OP_READV_FIXED] = {
547 .needs_file = 1,
548 .unbound_nonreg_file = 1,
549 .pollin = 1,
550 .plug = 1,
551 .audit_skip = 1,
552 .ioprio = 1,
553 .iopoll = 1,
554 .vectored = 1,
555 .async_size = sizeof(struct io_async_rw),
556 .prep = io_prep_readv_fixed,
557 .issue = io_read,
558 },
559 [IORING_OP_WRITEV_FIXED] = {
560 .needs_file = 1,
561 .hash_reg_file = 1,
562 .unbound_nonreg_file = 1,
563 .pollout = 1,
564 .plug = 1,
565 .audit_skip = 1,
566 .ioprio = 1,
567 .iopoll = 1,
568 .vectored = 1,
569 .async_size = sizeof(struct io_async_rw),
570 .prep = io_prep_writev_fixed,
571 .issue = io_write,
572 },
573 [IORING_OP_PIPE] = {
574 .prep = io_pipe_prep,
575 .issue = io_pipe,
576 },
577 [IORING_OP_NOP128] = {
578 .audit_skip = 1,
579 .iopoll = 1,
580 .is_128 = 1,
581 .prep = io_nop_prep,
582 .issue = io_nop,
583 },
584 [IORING_OP_URING_CMD128] = {
585 .buffer_select = 1,
586 .needs_file = 1,
587 .plug = 1,
588 .iopoll = 1,
589 .is_128 = 1,
590 .async_size = sizeof(struct io_async_cmd),
591 .prep = io_uring_cmd_prep,
592 .issue = io_uring_cmd,
593 },
594 };
595
596 const struct io_cold_def io_cold_defs[] = {
597 [IORING_OP_NOP] = {
598 .name = "NOP",
599 },
600 [IORING_OP_READV] = {
601 .name = "READV",
602 .cleanup = io_readv_writev_cleanup,
603 .fail = io_rw_fail,
604 },
605 [IORING_OP_WRITEV] = {
606 .name = "WRITEV",
607 .cleanup = io_readv_writev_cleanup,
608 .fail = io_rw_fail,
609 },
610 [IORING_OP_FSYNC] = {
611 .name = "FSYNC",
612 },
613 [IORING_OP_READ_FIXED] = {
614 .name = "READ_FIXED",
615 .cleanup = io_readv_writev_cleanup,
616 .fail = io_rw_fail,
617 },
618 [IORING_OP_WRITE_FIXED] = {
619 .name = "WRITE_FIXED",
620 .cleanup = io_readv_writev_cleanup,
621 .fail = io_rw_fail,
622 },
623 [IORING_OP_POLL_ADD] = {
624 .name = "POLL_ADD",
625 },
626 [IORING_OP_POLL_REMOVE] = {
627 .name = "POLL_REMOVE",
628 },
629 [IORING_OP_SYNC_FILE_RANGE] = {
630 .name = "SYNC_FILE_RANGE",
631 },
632 [IORING_OP_SENDMSG] = {
633 .name = "SENDMSG",
634 #if defined(CONFIG_NET)
635 .cleanup = io_sendmsg_recvmsg_cleanup,
636 .fail = io_sendrecv_fail,
637 #endif
638 },
639 [IORING_OP_RECVMSG] = {
640 .name = "RECVMSG",
641 #if defined(CONFIG_NET)
642 .cleanup = io_sendmsg_recvmsg_cleanup,
643 .fail = io_sendrecv_fail,
644 #endif
645 },
646 [IORING_OP_TIMEOUT] = {
647 .name = "TIMEOUT",
648 },
649 [IORING_OP_TIMEOUT_REMOVE] = {
650 .name = "TIMEOUT_REMOVE",
651 },
652 [IORING_OP_ACCEPT] = {
653 .name = "ACCEPT",
654 },
655 [IORING_OP_ASYNC_CANCEL] = {
656 .name = "ASYNC_CANCEL",
657 },
658 [IORING_OP_LINK_TIMEOUT] = {
659 .name = "LINK_TIMEOUT",
660 },
661 [IORING_OP_CONNECT] = {
662 .name = "CONNECT",
663 },
664 [IORING_OP_FALLOCATE] = {
665 .name = "FALLOCATE",
666 },
667 [IORING_OP_OPENAT] = {
668 .name = "OPENAT",
669 .cleanup = io_open_cleanup,
670 },
671 [IORING_OP_CLOSE] = {
672 .name = "CLOSE",
673 },
674 [IORING_OP_FILES_UPDATE] = {
675 .name = "FILES_UPDATE",
676 },
677 [IORING_OP_STATX] = {
678 .name = "STATX",
679 .cleanup = io_statx_cleanup,
680 },
681 [IORING_OP_READ] = {
682 .name = "READ",
683 .cleanup = io_readv_writev_cleanup,
684 .fail = io_rw_fail,
685 },
686 [IORING_OP_WRITE] = {
687 .name = "WRITE",
688 .cleanup = io_readv_writev_cleanup,
689 .fail = io_rw_fail,
690 },
691 [IORING_OP_FADVISE] = {
692 .name = "FADVISE",
693 },
694 [IORING_OP_MADVISE] = {
695 .name = "MADVISE",
696 },
697 [IORING_OP_SEND] = {
698 .name = "SEND",
699 #if defined(CONFIG_NET)
700 .cleanup = io_sendmsg_recvmsg_cleanup,
701 .fail = io_sendrecv_fail,
702 #endif
703 },
704 [IORING_OP_RECV] = {
705 .name = "RECV",
706 #if defined(CONFIG_NET)
707 .cleanup = io_sendmsg_recvmsg_cleanup,
708 .fail = io_sendrecv_fail,
709 #endif
710 },
711 [IORING_OP_OPENAT2] = {
712 .name = "OPENAT2",
713 .cleanup = io_open_cleanup,
714 },
715 [IORING_OP_EPOLL_CTL] = {
716 .name = "EPOLL",
717 },
718 [IORING_OP_SPLICE] = {
719 .name = "SPLICE",
720 .cleanup = io_splice_cleanup,
721 },
722 [IORING_OP_PROVIDE_BUFFERS] = {
723 .name = "PROVIDE_BUFFERS",
724 },
725 [IORING_OP_REMOVE_BUFFERS] = {
726 .name = "REMOVE_BUFFERS",
727 },
728 [IORING_OP_TEE] = {
729 .name = "TEE",
730 .cleanup = io_splice_cleanup,
731 },
732 [IORING_OP_SHUTDOWN] = {
733 .name = "SHUTDOWN",
734 },
735 [IORING_OP_RENAMEAT] = {
736 .name = "RENAMEAT",
737 .cleanup = io_renameat_cleanup,
738 },
739 [IORING_OP_UNLINKAT] = {
740 .name = "UNLINKAT",
741 .cleanup = io_unlinkat_cleanup,
742 },
743 [IORING_OP_MKDIRAT] = {
744 .name = "MKDIRAT",
745 .cleanup = io_mkdirat_cleanup,
746 },
747 [IORING_OP_SYMLINKAT] = {
748 .name = "SYMLINKAT",
749 .cleanup = io_link_cleanup,
750 },
751 [IORING_OP_LINKAT] = {
752 .name = "LINKAT",
753 .cleanup = io_link_cleanup,
754 },
755 [IORING_OP_MSG_RING] = {
756 .name = "MSG_RING",
757 .cleanup = io_msg_ring_cleanup,
758 },
759 [IORING_OP_FSETXATTR] = {
760 .name = "FSETXATTR",
761 .cleanup = io_xattr_cleanup,
762 },
763 [IORING_OP_SETXATTR] = {
764 .name = "SETXATTR",
765 .cleanup = io_xattr_cleanup,
766 },
767 [IORING_OP_FGETXATTR] = {
768 .name = "FGETXATTR",
769 .cleanup = io_xattr_cleanup,
770 },
771 [IORING_OP_GETXATTR] = {
772 .name = "GETXATTR",
773 .cleanup = io_xattr_cleanup,
774 },
775 [IORING_OP_SOCKET] = {
776 .name = "SOCKET",
777 },
778 [IORING_OP_URING_CMD] = {
779 .name = "URING_CMD",
780 .sqe_copy = io_uring_cmd_sqe_copy,
781 .cleanup = io_uring_cmd_cleanup,
782 },
783 [IORING_OP_SEND_ZC] = {
784 .name = "SEND_ZC",
785 #if defined(CONFIG_NET)
786 .cleanup = io_send_zc_cleanup,
787 .fail = io_sendrecv_fail,
788 #endif
789 },
790 [IORING_OP_SENDMSG_ZC] = {
791 .name = "SENDMSG_ZC",
792 #if defined(CONFIG_NET)
793 .cleanup = io_send_zc_cleanup,
794 .fail = io_sendrecv_fail,
795 #endif
796 },
797 [IORING_OP_READ_MULTISHOT] = {
798 .name = "READ_MULTISHOT",
799 .cleanup = io_readv_writev_cleanup,
800 },
801 [IORING_OP_WAITID] = {
802 .name = "WAITID",
803 },
804 [IORING_OP_FUTEX_WAIT] = {
805 .name = "FUTEX_WAIT",
806 },
807 [IORING_OP_FUTEX_WAKE] = {
808 .name = "FUTEX_WAKE",
809 },
810 [IORING_OP_FUTEX_WAITV] = {
811 .name = "FUTEX_WAITV",
812 },
813 [IORING_OP_FIXED_FD_INSTALL] = {
814 .name = "FIXED_FD_INSTALL",
815 },
816 [IORING_OP_FTRUNCATE] = {
817 .name = "FTRUNCATE",
818 },
819 [IORING_OP_BIND] = {
820 .name = "BIND",
821 },
822 [IORING_OP_LISTEN] = {
823 .name = "LISTEN",
824 },
825 [IORING_OP_RECV_ZC] = {
826 .name = "RECV_ZC",
827 },
828 [IORING_OP_EPOLL_WAIT] = {
829 .name = "EPOLL_WAIT",
830 },
831 [IORING_OP_READV_FIXED] = {
832 .name = "READV_FIXED",
833 .cleanup = io_readv_writev_cleanup,
834 .fail = io_rw_fail,
835 },
836 [IORING_OP_WRITEV_FIXED] = {
837 .name = "WRITEV_FIXED",
838 .cleanup = io_readv_writev_cleanup,
839 .fail = io_rw_fail,
840 },
841 [IORING_OP_PIPE] = {
842 .name = "PIPE",
843 },
844 [IORING_OP_NOP128] = {
845 .name = "NOP128",
846 },
847 [IORING_OP_URING_CMD128] = {
848 .name = "URING_CMD128",
849 .sqe_copy = io_uring_cmd_sqe_copy,
850 .cleanup = io_uring_cmd_cleanup,
851 },
852 };
853
io_uring_get_opcode(u8 opcode)854 const char *io_uring_get_opcode(u8 opcode)
855 {
856 if (opcode < IORING_OP_LAST)
857 return io_cold_defs[opcode].name;
858 return "INVALID";
859 }
860
io_uring_op_supported(u8 opcode)861 bool io_uring_op_supported(u8 opcode)
862 {
863 if (opcode < IORING_OP_LAST &&
864 io_issue_defs[opcode].prep != io_eopnotsupp_prep)
865 return true;
866 return false;
867 }
868
io_uring_optable_init(void)869 void __init io_uring_optable_init(void)
870 {
871 int i;
872
873 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
874 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
875
876 for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
877 BUG_ON(!io_issue_defs[i].prep);
878 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
879 BUG_ON(!io_issue_defs[i].issue);
880 WARN_ON_ONCE(!io_cold_defs[i].name);
881 }
882 }
883