1c4320315SJens Axboe // SPDX-License-Identifier: GPL-2.0 2c4320315SJens Axboe /* 3c4320315SJens Axboe * Code related to the io_uring_register() syscall 4c4320315SJens Axboe * 5c4320315SJens Axboe * Copyright (C) 2023 Jens Axboe 6c4320315SJens Axboe */ 7c4320315SJens Axboe #include <linux/kernel.h> 8c4320315SJens Axboe #include <linux/errno.h> 9c4320315SJens Axboe #include <linux/syscalls.h> 10c4320315SJens Axboe #include <linux/refcount.h> 11c4320315SJens Axboe #include <linux/bits.h> 12c4320315SJens Axboe #include <linux/fs.h> 13c4320315SJens Axboe #include <linux/file.h> 14c4320315SJens Axboe #include <linux/slab.h> 15c4320315SJens Axboe #include <linux/uaccess.h> 16c4320315SJens Axboe #include <linux/nospec.h> 17*baf59771SJens Axboe #include <linux/compat.h> 18c4320315SJens Axboe #include <linux/io_uring.h> 19c4320315SJens Axboe #include <linux/io_uring_types.h> 20c4320315SJens Axboe 21c4320315SJens Axboe #include "io_uring.h" 22c4320315SJens Axboe #include "opdef.h" 23c4320315SJens Axboe #include "tctx.h" 24c4320315SJens Axboe #include "rsrc.h" 25c4320315SJens Axboe #include "sqpoll.h" 26c4320315SJens Axboe #include "register.h" 27c4320315SJens Axboe #include "cancel.h" 28c4320315SJens Axboe #include "kbuf.h" 29c4320315SJens Axboe 30c4320315SJens Axboe #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ 31c4320315SJens Axboe IORING_REGISTER_LAST + IORING_OP_LAST) 32c4320315SJens Axboe 33c4320315SJens Axboe static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg, 34c4320315SJens Axboe unsigned int eventfd_async) 35c4320315SJens Axboe { 36c4320315SJens Axboe struct io_ev_fd *ev_fd; 37c4320315SJens Axboe __s32 __user *fds = arg; 38c4320315SJens Axboe int fd; 39c4320315SJens Axboe 40c4320315SJens Axboe ev_fd = rcu_dereference_protected(ctx->io_ev_fd, 41c4320315SJens Axboe lockdep_is_held(&ctx->uring_lock)); 42c4320315SJens Axboe if (ev_fd) 43c4320315SJens Axboe return -EBUSY; 44c4320315SJens Axboe 45c4320315SJens Axboe if (copy_from_user(&fd, fds, sizeof(*fds))) 46c4320315SJens Axboe return -EFAULT; 47c4320315SJens Axboe 48c4320315SJens Axboe ev_fd = kmalloc(sizeof(*ev_fd), GFP_KERNEL); 49c4320315SJens Axboe if (!ev_fd) 50c4320315SJens Axboe return -ENOMEM; 51c4320315SJens Axboe 52c4320315SJens Axboe ev_fd->cq_ev_fd = eventfd_ctx_fdget(fd); 53c4320315SJens Axboe if (IS_ERR(ev_fd->cq_ev_fd)) { 54c4320315SJens Axboe int ret = PTR_ERR(ev_fd->cq_ev_fd); 55c4320315SJens Axboe kfree(ev_fd); 56c4320315SJens Axboe return ret; 57c4320315SJens Axboe } 58c4320315SJens Axboe 59c4320315SJens Axboe spin_lock(&ctx->completion_lock); 60c4320315SJens Axboe ctx->evfd_last_cq_tail = ctx->cached_cq_tail; 61c4320315SJens Axboe spin_unlock(&ctx->completion_lock); 62c4320315SJens Axboe 63c4320315SJens Axboe ev_fd->eventfd_async = eventfd_async; 64c4320315SJens Axboe ctx->has_evfd = true; 65c4320315SJens Axboe rcu_assign_pointer(ctx->io_ev_fd, ev_fd); 66c4320315SJens Axboe atomic_set(&ev_fd->refs, 1); 67c4320315SJens Axboe atomic_set(&ev_fd->ops, 0); 68c4320315SJens Axboe return 0; 69c4320315SJens Axboe } 70c4320315SJens Axboe 71c4320315SJens Axboe int io_eventfd_unregister(struct io_ring_ctx *ctx) 72c4320315SJens Axboe { 73c4320315SJens Axboe struct io_ev_fd *ev_fd; 74c4320315SJens Axboe 75c4320315SJens Axboe ev_fd = rcu_dereference_protected(ctx->io_ev_fd, 76c4320315SJens Axboe lockdep_is_held(&ctx->uring_lock)); 77c4320315SJens Axboe if (ev_fd) { 78c4320315SJens Axboe ctx->has_evfd = false; 79c4320315SJens Axboe rcu_assign_pointer(ctx->io_ev_fd, NULL); 80c4320315SJens Axboe if (!atomic_fetch_or(BIT(IO_EVENTFD_OP_FREE_BIT), &ev_fd->ops)) 81c4320315SJens Axboe call_rcu(&ev_fd->rcu, io_eventfd_ops); 82c4320315SJens Axboe return 0; 83c4320315SJens Axboe } 84c4320315SJens Axboe 85c4320315SJens Axboe return -ENXIO; 86c4320315SJens Axboe } 87c4320315SJens Axboe 88c4320315SJens Axboe static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, 89c4320315SJens Axboe unsigned nr_args) 90c4320315SJens Axboe { 91c4320315SJens Axboe struct io_uring_probe *p; 92c4320315SJens Axboe size_t size; 93c4320315SJens Axboe int i, ret; 94c4320315SJens Axboe 95c4320315SJens Axboe size = struct_size(p, ops, nr_args); 96c4320315SJens Axboe if (size == SIZE_MAX) 97c4320315SJens Axboe return -EOVERFLOW; 98c4320315SJens Axboe p = kzalloc(size, GFP_KERNEL); 99c4320315SJens Axboe if (!p) 100c4320315SJens Axboe return -ENOMEM; 101c4320315SJens Axboe 102c4320315SJens Axboe ret = -EFAULT; 103c4320315SJens Axboe if (copy_from_user(p, arg, size)) 104c4320315SJens Axboe goto out; 105c4320315SJens Axboe ret = -EINVAL; 106c4320315SJens Axboe if (memchr_inv(p, 0, size)) 107c4320315SJens Axboe goto out; 108c4320315SJens Axboe 109c4320315SJens Axboe p->last_op = IORING_OP_LAST - 1; 110c4320315SJens Axboe if (nr_args > IORING_OP_LAST) 111c4320315SJens Axboe nr_args = IORING_OP_LAST; 112c4320315SJens Axboe 113c4320315SJens Axboe for (i = 0; i < nr_args; i++) { 114c4320315SJens Axboe p->ops[i].op = i; 115c4320315SJens Axboe if (!io_issue_defs[i].not_supported) 116c4320315SJens Axboe p->ops[i].flags = IO_URING_OP_SUPPORTED; 117c4320315SJens Axboe } 118c4320315SJens Axboe p->ops_len = i; 119c4320315SJens Axboe 120c4320315SJens Axboe ret = 0; 121c4320315SJens Axboe if (copy_to_user(arg, p, size)) 122c4320315SJens Axboe ret = -EFAULT; 123c4320315SJens Axboe out: 124c4320315SJens Axboe kfree(p); 125c4320315SJens Axboe return ret; 126c4320315SJens Axboe } 127c4320315SJens Axboe 128c4320315SJens Axboe int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) 129c4320315SJens Axboe { 130c4320315SJens Axboe const struct cred *creds; 131c4320315SJens Axboe 132c4320315SJens Axboe creds = xa_erase(&ctx->personalities, id); 133c4320315SJens Axboe if (creds) { 134c4320315SJens Axboe put_cred(creds); 135c4320315SJens Axboe return 0; 136c4320315SJens Axboe } 137c4320315SJens Axboe 138c4320315SJens Axboe return -EINVAL; 139c4320315SJens Axboe } 140c4320315SJens Axboe 141c4320315SJens Axboe 142c4320315SJens Axboe static int io_register_personality(struct io_ring_ctx *ctx) 143c4320315SJens Axboe { 144c4320315SJens Axboe const struct cred *creds; 145c4320315SJens Axboe u32 id; 146c4320315SJens Axboe int ret; 147c4320315SJens Axboe 148c4320315SJens Axboe creds = get_current_cred(); 149c4320315SJens Axboe 150c4320315SJens Axboe ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds, 151c4320315SJens Axboe XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL); 152c4320315SJens Axboe if (ret < 0) { 153c4320315SJens Axboe put_cred(creds); 154c4320315SJens Axboe return ret; 155c4320315SJens Axboe } 156c4320315SJens Axboe return id; 157c4320315SJens Axboe } 158c4320315SJens Axboe 159c4320315SJens Axboe static __cold int io_register_restrictions(struct io_ring_ctx *ctx, 160c4320315SJens Axboe void __user *arg, unsigned int nr_args) 161c4320315SJens Axboe { 162c4320315SJens Axboe struct io_uring_restriction *res; 163c4320315SJens Axboe size_t size; 164c4320315SJens Axboe int i, ret; 165c4320315SJens Axboe 166c4320315SJens Axboe /* Restrictions allowed only if rings started disabled */ 167c4320315SJens Axboe if (!(ctx->flags & IORING_SETUP_R_DISABLED)) 168c4320315SJens Axboe return -EBADFD; 169c4320315SJens Axboe 170c4320315SJens Axboe /* We allow only a single restrictions registration */ 171c4320315SJens Axboe if (ctx->restrictions.registered) 172c4320315SJens Axboe return -EBUSY; 173c4320315SJens Axboe 174c4320315SJens Axboe if (!arg || nr_args > IORING_MAX_RESTRICTIONS) 175c4320315SJens Axboe return -EINVAL; 176c4320315SJens Axboe 177c4320315SJens Axboe size = array_size(nr_args, sizeof(*res)); 178c4320315SJens Axboe if (size == SIZE_MAX) 179c4320315SJens Axboe return -EOVERFLOW; 180c4320315SJens Axboe 181c4320315SJens Axboe res = memdup_user(arg, size); 182c4320315SJens Axboe if (IS_ERR(res)) 183c4320315SJens Axboe return PTR_ERR(res); 184c4320315SJens Axboe 185c4320315SJens Axboe ret = 0; 186c4320315SJens Axboe 187c4320315SJens Axboe for (i = 0; i < nr_args; i++) { 188c4320315SJens Axboe switch (res[i].opcode) { 189c4320315SJens Axboe case IORING_RESTRICTION_REGISTER_OP: 190c4320315SJens Axboe if (res[i].register_op >= IORING_REGISTER_LAST) { 191c4320315SJens Axboe ret = -EINVAL; 192c4320315SJens Axboe goto out; 193c4320315SJens Axboe } 194c4320315SJens Axboe 195c4320315SJens Axboe __set_bit(res[i].register_op, 196c4320315SJens Axboe ctx->restrictions.register_op); 197c4320315SJens Axboe break; 198c4320315SJens Axboe case IORING_RESTRICTION_SQE_OP: 199c4320315SJens Axboe if (res[i].sqe_op >= IORING_OP_LAST) { 200c4320315SJens Axboe ret = -EINVAL; 201c4320315SJens Axboe goto out; 202c4320315SJens Axboe } 203c4320315SJens Axboe 204c4320315SJens Axboe __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op); 205c4320315SJens Axboe break; 206c4320315SJens Axboe case IORING_RESTRICTION_SQE_FLAGS_ALLOWED: 207c4320315SJens Axboe ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags; 208c4320315SJens Axboe break; 209c4320315SJens Axboe case IORING_RESTRICTION_SQE_FLAGS_REQUIRED: 210c4320315SJens Axboe ctx->restrictions.sqe_flags_required = res[i].sqe_flags; 211c4320315SJens Axboe break; 212c4320315SJens Axboe default: 213c4320315SJens Axboe ret = -EINVAL; 214c4320315SJens Axboe goto out; 215c4320315SJens Axboe } 216c4320315SJens Axboe } 217c4320315SJens Axboe 218c4320315SJens Axboe out: 219c4320315SJens Axboe /* Reset all restrictions if an error happened */ 220c4320315SJens Axboe if (ret != 0) 221c4320315SJens Axboe memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); 222c4320315SJens Axboe else 223c4320315SJens Axboe ctx->restrictions.registered = true; 224c4320315SJens Axboe 225c4320315SJens Axboe kfree(res); 226c4320315SJens Axboe return ret; 227c4320315SJens Axboe } 228c4320315SJens Axboe 229c4320315SJens Axboe static int io_register_enable_rings(struct io_ring_ctx *ctx) 230c4320315SJens Axboe { 231c4320315SJens Axboe if (!(ctx->flags & IORING_SETUP_R_DISABLED)) 232c4320315SJens Axboe return -EBADFD; 233c4320315SJens Axboe 234c4320315SJens Axboe if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task) { 235c4320315SJens Axboe WRITE_ONCE(ctx->submitter_task, get_task_struct(current)); 236c4320315SJens Axboe /* 237c4320315SJens Axboe * Lazy activation attempts would fail if it was polled before 238c4320315SJens Axboe * submitter_task is set. 239c4320315SJens Axboe */ 240c4320315SJens Axboe if (wq_has_sleeper(&ctx->poll_wq)) 241c4320315SJens Axboe io_activate_pollwq(ctx); 242c4320315SJens Axboe } 243c4320315SJens Axboe 244c4320315SJens Axboe if (ctx->restrictions.registered) 245c4320315SJens Axboe ctx->restricted = 1; 246c4320315SJens Axboe 247c4320315SJens Axboe ctx->flags &= ~IORING_SETUP_R_DISABLED; 248c4320315SJens Axboe if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) 249c4320315SJens Axboe wake_up(&ctx->sq_data->wait); 250c4320315SJens Axboe return 0; 251c4320315SJens Axboe } 252c4320315SJens Axboe 253c4320315SJens Axboe static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx, 254c4320315SJens Axboe cpumask_var_t new_mask) 255c4320315SJens Axboe { 256c4320315SJens Axboe int ret; 257c4320315SJens Axboe 258c4320315SJens Axboe if (!(ctx->flags & IORING_SETUP_SQPOLL)) { 259c4320315SJens Axboe ret = io_wq_cpu_affinity(current->io_uring, new_mask); 260c4320315SJens Axboe } else { 261c4320315SJens Axboe mutex_unlock(&ctx->uring_lock); 262c4320315SJens Axboe ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask); 263c4320315SJens Axboe mutex_lock(&ctx->uring_lock); 264c4320315SJens Axboe } 265c4320315SJens Axboe 266c4320315SJens Axboe return ret; 267c4320315SJens Axboe } 268c4320315SJens Axboe 269c4320315SJens Axboe static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx, 270c4320315SJens Axboe void __user *arg, unsigned len) 271c4320315SJens Axboe { 272c4320315SJens Axboe cpumask_var_t new_mask; 273c4320315SJens Axboe int ret; 274c4320315SJens Axboe 275c4320315SJens Axboe if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) 276c4320315SJens Axboe return -ENOMEM; 277c4320315SJens Axboe 278c4320315SJens Axboe cpumask_clear(new_mask); 279c4320315SJens Axboe if (len > cpumask_size()) 280c4320315SJens Axboe len = cpumask_size(); 281c4320315SJens Axboe 282*baf59771SJens Axboe #ifdef CONFIG_COMPAT 283*baf59771SJens Axboe if (in_compat_syscall()) 284c4320315SJens Axboe ret = compat_get_bitmap(cpumask_bits(new_mask), 285c4320315SJens Axboe (const compat_ulong_t __user *)arg, 286c4320315SJens Axboe len * 8 /* CHAR_BIT */); 287*baf59771SJens Axboe else 288*baf59771SJens Axboe #endif 289c4320315SJens Axboe ret = copy_from_user(new_mask, arg, len); 290c4320315SJens Axboe 291c4320315SJens Axboe if (ret) { 292c4320315SJens Axboe free_cpumask_var(new_mask); 293c4320315SJens Axboe return -EFAULT; 294c4320315SJens Axboe } 295c4320315SJens Axboe 296c4320315SJens Axboe ret = __io_register_iowq_aff(ctx, new_mask); 297c4320315SJens Axboe free_cpumask_var(new_mask); 298c4320315SJens Axboe return ret; 299c4320315SJens Axboe } 300c4320315SJens Axboe 301c4320315SJens Axboe static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx) 302c4320315SJens Axboe { 303c4320315SJens Axboe return __io_register_iowq_aff(ctx, NULL); 304c4320315SJens Axboe } 305c4320315SJens Axboe 306c4320315SJens Axboe static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx, 307c4320315SJens Axboe void __user *arg) 308c4320315SJens Axboe __must_hold(&ctx->uring_lock) 309c4320315SJens Axboe { 310c4320315SJens Axboe struct io_tctx_node *node; 311c4320315SJens Axboe struct io_uring_task *tctx = NULL; 312c4320315SJens Axboe struct io_sq_data *sqd = NULL; 313c4320315SJens Axboe __u32 new_count[2]; 314c4320315SJens Axboe int i, ret; 315c4320315SJens Axboe 316c4320315SJens Axboe if (copy_from_user(new_count, arg, sizeof(new_count))) 317c4320315SJens Axboe return -EFAULT; 318c4320315SJens Axboe for (i = 0; i < ARRAY_SIZE(new_count); i++) 319c4320315SJens Axboe if (new_count[i] > INT_MAX) 320c4320315SJens Axboe return -EINVAL; 321c4320315SJens Axboe 322c4320315SJens Axboe if (ctx->flags & IORING_SETUP_SQPOLL) { 323c4320315SJens Axboe sqd = ctx->sq_data; 324c4320315SJens Axboe if (sqd) { 325c4320315SJens Axboe /* 326c4320315SJens Axboe * Observe the correct sqd->lock -> ctx->uring_lock 327c4320315SJens Axboe * ordering. Fine to drop uring_lock here, we hold 328c4320315SJens Axboe * a ref to the ctx. 329c4320315SJens Axboe */ 330c4320315SJens Axboe refcount_inc(&sqd->refs); 331c4320315SJens Axboe mutex_unlock(&ctx->uring_lock); 332c4320315SJens Axboe mutex_lock(&sqd->lock); 333c4320315SJens Axboe mutex_lock(&ctx->uring_lock); 334c4320315SJens Axboe if (sqd->thread) 335c4320315SJens Axboe tctx = sqd->thread->io_uring; 336c4320315SJens Axboe } 337c4320315SJens Axboe } else { 338c4320315SJens Axboe tctx = current->io_uring; 339c4320315SJens Axboe } 340c4320315SJens Axboe 341c4320315SJens Axboe BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits)); 342c4320315SJens Axboe 343c4320315SJens Axboe for (i = 0; i < ARRAY_SIZE(new_count); i++) 344c4320315SJens Axboe if (new_count[i]) 345c4320315SJens Axboe ctx->iowq_limits[i] = new_count[i]; 346c4320315SJens Axboe ctx->iowq_limits_set = true; 347c4320315SJens Axboe 348c4320315SJens Axboe if (tctx && tctx->io_wq) { 349c4320315SJens Axboe ret = io_wq_max_workers(tctx->io_wq, new_count); 350c4320315SJens Axboe if (ret) 351c4320315SJens Axboe goto err; 352c4320315SJens Axboe } else { 353c4320315SJens Axboe memset(new_count, 0, sizeof(new_count)); 354c4320315SJens Axboe } 355c4320315SJens Axboe 356c4320315SJens Axboe if (sqd) { 357c4320315SJens Axboe mutex_unlock(&sqd->lock); 358c4320315SJens Axboe io_put_sq_data(sqd); 359c4320315SJens Axboe } 360c4320315SJens Axboe 361c4320315SJens Axboe if (copy_to_user(arg, new_count, sizeof(new_count))) 362c4320315SJens Axboe return -EFAULT; 363c4320315SJens Axboe 364c4320315SJens Axboe /* that's it for SQPOLL, only the SQPOLL task creates requests */ 365c4320315SJens Axboe if (sqd) 366c4320315SJens Axboe return 0; 367c4320315SJens Axboe 368c4320315SJens Axboe /* now propagate the restriction to all registered users */ 369c4320315SJens Axboe list_for_each_entry(node, &ctx->tctx_list, ctx_node) { 370c4320315SJens Axboe struct io_uring_task *tctx = node->task->io_uring; 371c4320315SJens Axboe 372c4320315SJens Axboe if (WARN_ON_ONCE(!tctx->io_wq)) 373c4320315SJens Axboe continue; 374c4320315SJens Axboe 375c4320315SJens Axboe for (i = 0; i < ARRAY_SIZE(new_count); i++) 376c4320315SJens Axboe new_count[i] = ctx->iowq_limits[i]; 377c4320315SJens Axboe /* ignore errors, it always returns zero anyway */ 378c4320315SJens Axboe (void)io_wq_max_workers(tctx->io_wq, new_count); 379c4320315SJens Axboe } 380c4320315SJens Axboe return 0; 381c4320315SJens Axboe err: 382c4320315SJens Axboe if (sqd) { 383c4320315SJens Axboe mutex_unlock(&sqd->lock); 384c4320315SJens Axboe io_put_sq_data(sqd); 385c4320315SJens Axboe } 386c4320315SJens Axboe return ret; 387c4320315SJens Axboe } 388c4320315SJens Axboe 389c4320315SJens Axboe static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, 390c4320315SJens Axboe void __user *arg, unsigned nr_args) 391c4320315SJens Axboe __releases(ctx->uring_lock) 392c4320315SJens Axboe __acquires(ctx->uring_lock) 393c4320315SJens Axboe { 394c4320315SJens Axboe int ret; 395c4320315SJens Axboe 396c4320315SJens Axboe /* 397c4320315SJens Axboe * We don't quiesce the refs for register anymore and so it can't be 398c4320315SJens Axboe * dying as we're holding a file ref here. 399c4320315SJens Axboe */ 400c4320315SJens Axboe if (WARN_ON_ONCE(percpu_ref_is_dying(&ctx->refs))) 401c4320315SJens Axboe return -ENXIO; 402c4320315SJens Axboe 403c4320315SJens Axboe if (ctx->submitter_task && ctx->submitter_task != current) 404c4320315SJens Axboe return -EEXIST; 405c4320315SJens Axboe 406c4320315SJens Axboe if (ctx->restricted) { 407c4320315SJens Axboe opcode = array_index_nospec(opcode, IORING_REGISTER_LAST); 408c4320315SJens Axboe if (!test_bit(opcode, ctx->restrictions.register_op)) 409c4320315SJens Axboe return -EACCES; 410c4320315SJens Axboe } 411c4320315SJens Axboe 412c4320315SJens Axboe switch (opcode) { 413c4320315SJens Axboe case IORING_REGISTER_BUFFERS: 414c4320315SJens Axboe ret = -EFAULT; 415c4320315SJens Axboe if (!arg) 416c4320315SJens Axboe break; 417c4320315SJens Axboe ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL); 418c4320315SJens Axboe break; 419c4320315SJens Axboe case IORING_UNREGISTER_BUFFERS: 420c4320315SJens Axboe ret = -EINVAL; 421c4320315SJens Axboe if (arg || nr_args) 422c4320315SJens Axboe break; 423c4320315SJens Axboe ret = io_sqe_buffers_unregister(ctx); 424c4320315SJens Axboe break; 425c4320315SJens Axboe case IORING_REGISTER_FILES: 426c4320315SJens Axboe ret = -EFAULT; 427c4320315SJens Axboe if (!arg) 428c4320315SJens Axboe break; 429c4320315SJens Axboe ret = io_sqe_files_register(ctx, arg, nr_args, NULL); 430c4320315SJens Axboe break; 431c4320315SJens Axboe case IORING_UNREGISTER_FILES: 432c4320315SJens Axboe ret = -EINVAL; 433c4320315SJens Axboe if (arg || nr_args) 434c4320315SJens Axboe break; 435c4320315SJens Axboe ret = io_sqe_files_unregister(ctx); 436c4320315SJens Axboe break; 437c4320315SJens Axboe case IORING_REGISTER_FILES_UPDATE: 438c4320315SJens Axboe ret = io_register_files_update(ctx, arg, nr_args); 439c4320315SJens Axboe break; 440c4320315SJens Axboe case IORING_REGISTER_EVENTFD: 441c4320315SJens Axboe ret = -EINVAL; 442c4320315SJens Axboe if (nr_args != 1) 443c4320315SJens Axboe break; 444c4320315SJens Axboe ret = io_eventfd_register(ctx, arg, 0); 445c4320315SJens Axboe break; 446c4320315SJens Axboe case IORING_REGISTER_EVENTFD_ASYNC: 447c4320315SJens Axboe ret = -EINVAL; 448c4320315SJens Axboe if (nr_args != 1) 449c4320315SJens Axboe break; 450c4320315SJens Axboe ret = io_eventfd_register(ctx, arg, 1); 451c4320315SJens Axboe break; 452c4320315SJens Axboe case IORING_UNREGISTER_EVENTFD: 453c4320315SJens Axboe ret = -EINVAL; 454c4320315SJens Axboe if (arg || nr_args) 455c4320315SJens Axboe break; 456c4320315SJens Axboe ret = io_eventfd_unregister(ctx); 457c4320315SJens Axboe break; 458c4320315SJens Axboe case IORING_REGISTER_PROBE: 459c4320315SJens Axboe ret = -EINVAL; 460c4320315SJens Axboe if (!arg || nr_args > 256) 461c4320315SJens Axboe break; 462c4320315SJens Axboe ret = io_probe(ctx, arg, nr_args); 463c4320315SJens Axboe break; 464c4320315SJens Axboe case IORING_REGISTER_PERSONALITY: 465c4320315SJens Axboe ret = -EINVAL; 466c4320315SJens Axboe if (arg || nr_args) 467c4320315SJens Axboe break; 468c4320315SJens Axboe ret = io_register_personality(ctx); 469c4320315SJens Axboe break; 470c4320315SJens Axboe case IORING_UNREGISTER_PERSONALITY: 471c4320315SJens Axboe ret = -EINVAL; 472c4320315SJens Axboe if (arg) 473c4320315SJens Axboe break; 474c4320315SJens Axboe ret = io_unregister_personality(ctx, nr_args); 475c4320315SJens Axboe break; 476c4320315SJens Axboe case IORING_REGISTER_ENABLE_RINGS: 477c4320315SJens Axboe ret = -EINVAL; 478c4320315SJens Axboe if (arg || nr_args) 479c4320315SJens Axboe break; 480c4320315SJens Axboe ret = io_register_enable_rings(ctx); 481c4320315SJens Axboe break; 482c4320315SJens Axboe case IORING_REGISTER_RESTRICTIONS: 483c4320315SJens Axboe ret = io_register_restrictions(ctx, arg, nr_args); 484c4320315SJens Axboe break; 485c4320315SJens Axboe case IORING_REGISTER_FILES2: 486c4320315SJens Axboe ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE); 487c4320315SJens Axboe break; 488c4320315SJens Axboe case IORING_REGISTER_FILES_UPDATE2: 489c4320315SJens Axboe ret = io_register_rsrc_update(ctx, arg, nr_args, 490c4320315SJens Axboe IORING_RSRC_FILE); 491c4320315SJens Axboe break; 492c4320315SJens Axboe case IORING_REGISTER_BUFFERS2: 493c4320315SJens Axboe ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER); 494c4320315SJens Axboe break; 495c4320315SJens Axboe case IORING_REGISTER_BUFFERS_UPDATE: 496c4320315SJens Axboe ret = io_register_rsrc_update(ctx, arg, nr_args, 497c4320315SJens Axboe IORING_RSRC_BUFFER); 498c4320315SJens Axboe break; 499c4320315SJens Axboe case IORING_REGISTER_IOWQ_AFF: 500c4320315SJens Axboe ret = -EINVAL; 501c4320315SJens Axboe if (!arg || !nr_args) 502c4320315SJens Axboe break; 503c4320315SJens Axboe ret = io_register_iowq_aff(ctx, arg, nr_args); 504c4320315SJens Axboe break; 505c4320315SJens Axboe case IORING_UNREGISTER_IOWQ_AFF: 506c4320315SJens Axboe ret = -EINVAL; 507c4320315SJens Axboe if (arg || nr_args) 508c4320315SJens Axboe break; 509c4320315SJens Axboe ret = io_unregister_iowq_aff(ctx); 510c4320315SJens Axboe break; 511c4320315SJens Axboe case IORING_REGISTER_IOWQ_MAX_WORKERS: 512c4320315SJens Axboe ret = -EINVAL; 513c4320315SJens Axboe if (!arg || nr_args != 2) 514c4320315SJens Axboe break; 515c4320315SJens Axboe ret = io_register_iowq_max_workers(ctx, arg); 516c4320315SJens Axboe break; 517c4320315SJens Axboe case IORING_REGISTER_RING_FDS: 518c4320315SJens Axboe ret = io_ringfd_register(ctx, arg, nr_args); 519c4320315SJens Axboe break; 520c4320315SJens Axboe case IORING_UNREGISTER_RING_FDS: 521c4320315SJens Axboe ret = io_ringfd_unregister(ctx, arg, nr_args); 522c4320315SJens Axboe break; 523c4320315SJens Axboe case IORING_REGISTER_PBUF_RING: 524c4320315SJens Axboe ret = -EINVAL; 525c4320315SJens Axboe if (!arg || nr_args != 1) 526c4320315SJens Axboe break; 527c4320315SJens Axboe ret = io_register_pbuf_ring(ctx, arg); 528c4320315SJens Axboe break; 529c4320315SJens Axboe case IORING_UNREGISTER_PBUF_RING: 530c4320315SJens Axboe ret = -EINVAL; 531c4320315SJens Axboe if (!arg || nr_args != 1) 532c4320315SJens Axboe break; 533c4320315SJens Axboe ret = io_unregister_pbuf_ring(ctx, arg); 534c4320315SJens Axboe break; 535c4320315SJens Axboe case IORING_REGISTER_SYNC_CANCEL: 536c4320315SJens Axboe ret = -EINVAL; 537c4320315SJens Axboe if (!arg || nr_args != 1) 538c4320315SJens Axboe break; 539c4320315SJens Axboe ret = io_sync_cancel(ctx, arg); 540c4320315SJens Axboe break; 541c4320315SJens Axboe case IORING_REGISTER_FILE_ALLOC_RANGE: 542c4320315SJens Axboe ret = -EINVAL; 543c4320315SJens Axboe if (!arg || nr_args) 544c4320315SJens Axboe break; 545c4320315SJens Axboe ret = io_register_file_alloc_range(ctx, arg); 546c4320315SJens Axboe break; 547d293b1a8SJens Axboe case IORING_REGISTER_PBUF_STATUS: 548d293b1a8SJens Axboe ret = -EINVAL; 549d293b1a8SJens Axboe if (!arg || nr_args != 1) 550d293b1a8SJens Axboe break; 551d293b1a8SJens Axboe ret = io_register_pbuf_status(ctx, arg); 552d293b1a8SJens Axboe break; 553c4320315SJens Axboe default: 554c4320315SJens Axboe ret = -EINVAL; 555c4320315SJens Axboe break; 556c4320315SJens Axboe } 557c4320315SJens Axboe 558c4320315SJens Axboe return ret; 559c4320315SJens Axboe } 560c4320315SJens Axboe 561c4320315SJens Axboe SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, 562c4320315SJens Axboe void __user *, arg, unsigned int, nr_args) 563c4320315SJens Axboe { 564c4320315SJens Axboe struct io_ring_ctx *ctx; 565c4320315SJens Axboe long ret = -EBADF; 566c4320315SJens Axboe struct file *file; 567c4320315SJens Axboe bool use_registered_ring; 568c4320315SJens Axboe 569c4320315SJens Axboe use_registered_ring = !!(opcode & IORING_REGISTER_USE_REGISTERED_RING); 570c4320315SJens Axboe opcode &= ~IORING_REGISTER_USE_REGISTERED_RING; 571c4320315SJens Axboe 572c4320315SJens Axboe if (opcode >= IORING_REGISTER_LAST) 573c4320315SJens Axboe return -EINVAL; 574c4320315SJens Axboe 575c4320315SJens Axboe if (use_registered_ring) { 576c4320315SJens Axboe /* 577c4320315SJens Axboe * Ring fd has been registered via IORING_REGISTER_RING_FDS, we 578c4320315SJens Axboe * need only dereference our task private array to find it. 579c4320315SJens Axboe */ 580c4320315SJens Axboe struct io_uring_task *tctx = current->io_uring; 581c4320315SJens Axboe 582c4320315SJens Axboe if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX)) 583c4320315SJens Axboe return -EINVAL; 584c4320315SJens Axboe fd = array_index_nospec(fd, IO_RINGFD_REG_MAX); 585c4320315SJens Axboe file = tctx->registered_rings[fd]; 586c4320315SJens Axboe if (unlikely(!file)) 587c4320315SJens Axboe return -EBADF; 588c4320315SJens Axboe } else { 589c4320315SJens Axboe file = fget(fd); 590c4320315SJens Axboe if (unlikely(!file)) 591c4320315SJens Axboe return -EBADF; 592c4320315SJens Axboe ret = -EOPNOTSUPP; 593c4320315SJens Axboe if (!io_is_uring_fops(file)) 594c4320315SJens Axboe goto out_fput; 595c4320315SJens Axboe } 596c4320315SJens Axboe 597c4320315SJens Axboe ctx = file->private_data; 598c4320315SJens Axboe 599c4320315SJens Axboe mutex_lock(&ctx->uring_lock); 600c4320315SJens Axboe ret = __io_uring_register(ctx, opcode, arg, nr_args); 601c4320315SJens Axboe mutex_unlock(&ctx->uring_lock); 602c4320315SJens Axboe trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, ret); 603c4320315SJens Axboe out_fput: 604c4320315SJens Axboe if (!use_registered_ring) 605c4320315SJens Axboe fput(file); 606c4320315SJens Axboe return ret; 607c4320315SJens Axboe } 608