1c4320315SJens Axboe // SPDX-License-Identifier: GPL-2.0 2c4320315SJens Axboe /* 3c4320315SJens Axboe * Code related to the io_uring_register() syscall 4c4320315SJens Axboe * 5c4320315SJens Axboe * Copyright (C) 2023 Jens Axboe 6c4320315SJens Axboe */ 7c4320315SJens Axboe #include <linux/kernel.h> 8c4320315SJens Axboe #include <linux/errno.h> 9c4320315SJens Axboe #include <linux/syscalls.h> 10c4320315SJens Axboe #include <linux/refcount.h> 11c4320315SJens Axboe #include <linux/bits.h> 12c4320315SJens Axboe #include <linux/fs.h> 13c4320315SJens Axboe #include <linux/file.h> 14c4320315SJens Axboe #include <linux/slab.h> 15c4320315SJens Axboe #include <linux/uaccess.h> 16c4320315SJens Axboe #include <linux/nospec.h> 17baf59771SJens Axboe #include <linux/compat.h> 18c4320315SJens Axboe #include <linux/io_uring.h> 19c4320315SJens Axboe #include <linux/io_uring_types.h> 20c4320315SJens Axboe 21c4320315SJens Axboe #include "io_uring.h" 22c4320315SJens Axboe #include "opdef.h" 23c4320315SJens Axboe #include "tctx.h" 24c4320315SJens Axboe #include "rsrc.h" 25c4320315SJens Axboe #include "sqpoll.h" 26c4320315SJens Axboe #include "register.h" 27c4320315SJens Axboe #include "cancel.h" 28c4320315SJens Axboe #include "kbuf.h" 29ef1186c1SStefan Roesch #include "napi.h" 30200f3abdSJens Axboe #include "eventfd.h" 31c4320315SJens Axboe 32c4320315SJens Axboe #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ 33c4320315SJens Axboe IORING_REGISTER_LAST + IORING_OP_LAST) 34c4320315SJens Axboe 35c4320315SJens Axboe static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, 36c4320315SJens Axboe unsigned nr_args) 37c4320315SJens Axboe { 38c4320315SJens Axboe struct io_uring_probe *p; 39c4320315SJens Axboe size_t size; 40c4320315SJens Axboe int i, ret; 41c4320315SJens Axboe 426bc9199dSGabriel Krisman Bertazi if (nr_args > IORING_OP_LAST) 436bc9199dSGabriel Krisman Bertazi nr_args = IORING_OP_LAST; 446bc9199dSGabriel Krisman Bertazi 45c4320315SJens Axboe size = struct_size(p, ops, nr_args); 46c4320315SJens Axboe p = kzalloc(size, GFP_KERNEL); 47c4320315SJens Axboe if (!p) 48c4320315SJens Axboe return -ENOMEM; 49c4320315SJens Axboe 50c4320315SJens Axboe ret = -EFAULT; 51c4320315SJens Axboe if (copy_from_user(p, arg, size)) 52c4320315SJens Axboe goto out; 53c4320315SJens Axboe ret = -EINVAL; 54c4320315SJens Axboe if (memchr_inv(p, 0, size)) 55c4320315SJens Axboe goto out; 56c4320315SJens Axboe 57c4320315SJens Axboe p->last_op = IORING_OP_LAST - 1; 58c4320315SJens Axboe 59c4320315SJens Axboe for (i = 0; i < nr_args; i++) { 60c4320315SJens Axboe p->ops[i].op = i; 613e05b222SGabriel Krisman Bertazi if (io_uring_op_supported(i)) 62c4320315SJens Axboe p->ops[i].flags = IO_URING_OP_SUPPORTED; 63c4320315SJens Axboe } 64c4320315SJens Axboe p->ops_len = i; 65c4320315SJens Axboe 66c4320315SJens Axboe ret = 0; 67c4320315SJens Axboe if (copy_to_user(arg, p, size)) 68c4320315SJens Axboe ret = -EFAULT; 69c4320315SJens Axboe out: 70c4320315SJens Axboe kfree(p); 71c4320315SJens Axboe return ret; 72c4320315SJens Axboe } 73c4320315SJens Axboe 74c4320315SJens Axboe int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id) 75c4320315SJens Axboe { 76c4320315SJens Axboe const struct cred *creds; 77c4320315SJens Axboe 78c4320315SJens Axboe creds = xa_erase(&ctx->personalities, id); 79c4320315SJens Axboe if (creds) { 80c4320315SJens Axboe put_cred(creds); 81c4320315SJens Axboe return 0; 82c4320315SJens Axboe } 83c4320315SJens Axboe 84c4320315SJens Axboe return -EINVAL; 85c4320315SJens Axboe } 86c4320315SJens Axboe 87c4320315SJens Axboe 88c4320315SJens Axboe static int io_register_personality(struct io_ring_ctx *ctx) 89c4320315SJens Axboe { 90c4320315SJens Axboe const struct cred *creds; 91c4320315SJens Axboe u32 id; 92c4320315SJens Axboe int ret; 93c4320315SJens Axboe 94c4320315SJens Axboe creds = get_current_cred(); 95c4320315SJens Axboe 96c4320315SJens Axboe ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds, 97c4320315SJens Axboe XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL); 98c4320315SJens Axboe if (ret < 0) { 99c4320315SJens Axboe put_cred(creds); 100c4320315SJens Axboe return ret; 101c4320315SJens Axboe } 102c4320315SJens Axboe return id; 103c4320315SJens Axboe } 104c4320315SJens Axboe 105c4320315SJens Axboe static __cold int io_register_restrictions(struct io_ring_ctx *ctx, 106c4320315SJens Axboe void __user *arg, unsigned int nr_args) 107c4320315SJens Axboe { 108c4320315SJens Axboe struct io_uring_restriction *res; 109c4320315SJens Axboe size_t size; 110c4320315SJens Axboe int i, ret; 111c4320315SJens Axboe 112c4320315SJens Axboe /* Restrictions allowed only if rings started disabled */ 113c4320315SJens Axboe if (!(ctx->flags & IORING_SETUP_R_DISABLED)) 114c4320315SJens Axboe return -EBADFD; 115c4320315SJens Axboe 116c4320315SJens Axboe /* We allow only a single restrictions registration */ 117c4320315SJens Axboe if (ctx->restrictions.registered) 118c4320315SJens Axboe return -EBUSY; 119c4320315SJens Axboe 120c4320315SJens Axboe if (!arg || nr_args > IORING_MAX_RESTRICTIONS) 121c4320315SJens Axboe return -EINVAL; 122c4320315SJens Axboe 123c4320315SJens Axboe size = array_size(nr_args, sizeof(*res)); 124c4320315SJens Axboe if (size == SIZE_MAX) 125c4320315SJens Axboe return -EOVERFLOW; 126c4320315SJens Axboe 127c4320315SJens Axboe res = memdup_user(arg, size); 128c4320315SJens Axboe if (IS_ERR(res)) 129c4320315SJens Axboe return PTR_ERR(res); 130c4320315SJens Axboe 131c4320315SJens Axboe ret = 0; 132c4320315SJens Axboe 133c4320315SJens Axboe for (i = 0; i < nr_args; i++) { 134c4320315SJens Axboe switch (res[i].opcode) { 135c4320315SJens Axboe case IORING_RESTRICTION_REGISTER_OP: 136c4320315SJens Axboe if (res[i].register_op >= IORING_REGISTER_LAST) { 137c4320315SJens Axboe ret = -EINVAL; 138c4320315SJens Axboe goto out; 139c4320315SJens Axboe } 140c4320315SJens Axboe 141c4320315SJens Axboe __set_bit(res[i].register_op, 142c4320315SJens Axboe ctx->restrictions.register_op); 143c4320315SJens Axboe break; 144c4320315SJens Axboe case IORING_RESTRICTION_SQE_OP: 145c4320315SJens Axboe if (res[i].sqe_op >= IORING_OP_LAST) { 146c4320315SJens Axboe ret = -EINVAL; 147c4320315SJens Axboe goto out; 148c4320315SJens Axboe } 149c4320315SJens Axboe 150c4320315SJens Axboe __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op); 151c4320315SJens Axboe break; 152c4320315SJens Axboe case IORING_RESTRICTION_SQE_FLAGS_ALLOWED: 153c4320315SJens Axboe ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags; 154c4320315SJens Axboe break; 155c4320315SJens Axboe case IORING_RESTRICTION_SQE_FLAGS_REQUIRED: 156c4320315SJens Axboe ctx->restrictions.sqe_flags_required = res[i].sqe_flags; 157c4320315SJens Axboe break; 158c4320315SJens Axboe default: 159c4320315SJens Axboe ret = -EINVAL; 160c4320315SJens Axboe goto out; 161c4320315SJens Axboe } 162c4320315SJens Axboe } 163c4320315SJens Axboe 164c4320315SJens Axboe out: 165c4320315SJens Axboe /* Reset all restrictions if an error happened */ 166c4320315SJens Axboe if (ret != 0) 167c4320315SJens Axboe memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); 168c4320315SJens Axboe else 169c4320315SJens Axboe ctx->restrictions.registered = true; 170c4320315SJens Axboe 171c4320315SJens Axboe kfree(res); 172c4320315SJens Axboe return ret; 173c4320315SJens Axboe } 174c4320315SJens Axboe 175c4320315SJens Axboe static int io_register_enable_rings(struct io_ring_ctx *ctx) 176c4320315SJens Axboe { 177c4320315SJens Axboe if (!(ctx->flags & IORING_SETUP_R_DISABLED)) 178c4320315SJens Axboe return -EBADFD; 179c4320315SJens Axboe 180c4320315SJens Axboe if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task) { 181c4320315SJens Axboe WRITE_ONCE(ctx->submitter_task, get_task_struct(current)); 182c4320315SJens Axboe /* 183c4320315SJens Axboe * Lazy activation attempts would fail if it was polled before 184c4320315SJens Axboe * submitter_task is set. 185c4320315SJens Axboe */ 186c4320315SJens Axboe if (wq_has_sleeper(&ctx->poll_wq)) 187c4320315SJens Axboe io_activate_pollwq(ctx); 188c4320315SJens Axboe } 189c4320315SJens Axboe 190c4320315SJens Axboe if (ctx->restrictions.registered) 191c4320315SJens Axboe ctx->restricted = 1; 192c4320315SJens Axboe 193c4320315SJens Axboe ctx->flags &= ~IORING_SETUP_R_DISABLED; 194c4320315SJens Axboe if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) 195c4320315SJens Axboe wake_up(&ctx->sq_data->wait); 196c4320315SJens Axboe return 0; 197c4320315SJens Axboe } 198c4320315SJens Axboe 199c4320315SJens Axboe static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx, 200c4320315SJens Axboe cpumask_var_t new_mask) 201c4320315SJens Axboe { 202c4320315SJens Axboe int ret; 203c4320315SJens Axboe 204c4320315SJens Axboe if (!(ctx->flags & IORING_SETUP_SQPOLL)) { 205c4320315SJens Axboe ret = io_wq_cpu_affinity(current->io_uring, new_mask); 206c4320315SJens Axboe } else { 207c4320315SJens Axboe mutex_unlock(&ctx->uring_lock); 208c4320315SJens Axboe ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask); 209c4320315SJens Axboe mutex_lock(&ctx->uring_lock); 210c4320315SJens Axboe } 211c4320315SJens Axboe 212c4320315SJens Axboe return ret; 213c4320315SJens Axboe } 214c4320315SJens Axboe 215c4320315SJens Axboe static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx, 216c4320315SJens Axboe void __user *arg, unsigned len) 217c4320315SJens Axboe { 218c4320315SJens Axboe cpumask_var_t new_mask; 219c4320315SJens Axboe int ret; 220c4320315SJens Axboe 221c4320315SJens Axboe if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) 222c4320315SJens Axboe return -ENOMEM; 223c4320315SJens Axboe 224c4320315SJens Axboe cpumask_clear(new_mask); 225c4320315SJens Axboe if (len > cpumask_size()) 226c4320315SJens Axboe len = cpumask_size(); 227c4320315SJens Axboe 228baf59771SJens Axboe #ifdef CONFIG_COMPAT 229baf59771SJens Axboe if (in_compat_syscall()) 230c4320315SJens Axboe ret = compat_get_bitmap(cpumask_bits(new_mask), 231c4320315SJens Axboe (const compat_ulong_t __user *)arg, 232c4320315SJens Axboe len * 8 /* CHAR_BIT */); 233baf59771SJens Axboe else 234baf59771SJens Axboe #endif 235c4320315SJens Axboe ret = copy_from_user(new_mask, arg, len); 236c4320315SJens Axboe 237c4320315SJens Axboe if (ret) { 238c4320315SJens Axboe free_cpumask_var(new_mask); 239c4320315SJens Axboe return -EFAULT; 240c4320315SJens Axboe } 241c4320315SJens Axboe 242c4320315SJens Axboe ret = __io_register_iowq_aff(ctx, new_mask); 243c4320315SJens Axboe free_cpumask_var(new_mask); 244c4320315SJens Axboe return ret; 245c4320315SJens Axboe } 246c4320315SJens Axboe 247c4320315SJens Axboe static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx) 248c4320315SJens Axboe { 249c4320315SJens Axboe return __io_register_iowq_aff(ctx, NULL); 250c4320315SJens Axboe } 251c4320315SJens Axboe 252c4320315SJens Axboe static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx, 253c4320315SJens Axboe void __user *arg) 254c4320315SJens Axboe __must_hold(&ctx->uring_lock) 255c4320315SJens Axboe { 256c4320315SJens Axboe struct io_tctx_node *node; 257c4320315SJens Axboe struct io_uring_task *tctx = NULL; 258c4320315SJens Axboe struct io_sq_data *sqd = NULL; 259c4320315SJens Axboe __u32 new_count[2]; 260c4320315SJens Axboe int i, ret; 261c4320315SJens Axboe 262c4320315SJens Axboe if (copy_from_user(new_count, arg, sizeof(new_count))) 263c4320315SJens Axboe return -EFAULT; 264c4320315SJens Axboe for (i = 0; i < ARRAY_SIZE(new_count); i++) 265c4320315SJens Axboe if (new_count[i] > INT_MAX) 266c4320315SJens Axboe return -EINVAL; 267c4320315SJens Axboe 268c4320315SJens Axboe if (ctx->flags & IORING_SETUP_SQPOLL) { 269c4320315SJens Axboe sqd = ctx->sq_data; 270c4320315SJens Axboe if (sqd) { 271c4320315SJens Axboe /* 272c4320315SJens Axboe * Observe the correct sqd->lock -> ctx->uring_lock 273c4320315SJens Axboe * ordering. Fine to drop uring_lock here, we hold 274c4320315SJens Axboe * a ref to the ctx. 275c4320315SJens Axboe */ 276c4320315SJens Axboe refcount_inc(&sqd->refs); 277c4320315SJens Axboe mutex_unlock(&ctx->uring_lock); 278c4320315SJens Axboe mutex_lock(&sqd->lock); 279c4320315SJens Axboe mutex_lock(&ctx->uring_lock); 280c4320315SJens Axboe if (sqd->thread) 281c4320315SJens Axboe tctx = sqd->thread->io_uring; 282c4320315SJens Axboe } 283c4320315SJens Axboe } else { 284c4320315SJens Axboe tctx = current->io_uring; 285c4320315SJens Axboe } 286c4320315SJens Axboe 287c4320315SJens Axboe BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits)); 288c4320315SJens Axboe 289c4320315SJens Axboe for (i = 0; i < ARRAY_SIZE(new_count); i++) 290c4320315SJens Axboe if (new_count[i]) 291c4320315SJens Axboe ctx->iowq_limits[i] = new_count[i]; 292c4320315SJens Axboe ctx->iowq_limits_set = true; 293c4320315SJens Axboe 294c4320315SJens Axboe if (tctx && tctx->io_wq) { 295c4320315SJens Axboe ret = io_wq_max_workers(tctx->io_wq, new_count); 296c4320315SJens Axboe if (ret) 297c4320315SJens Axboe goto err; 298c4320315SJens Axboe } else { 299c4320315SJens Axboe memset(new_count, 0, sizeof(new_count)); 300c4320315SJens Axboe } 301c4320315SJens Axboe 302c4320315SJens Axboe if (sqd) { 30373254a29SHagar Hemdan mutex_unlock(&ctx->uring_lock); 304c4320315SJens Axboe mutex_unlock(&sqd->lock); 305c4320315SJens Axboe io_put_sq_data(sqd); 30673254a29SHagar Hemdan mutex_lock(&ctx->uring_lock); 307c4320315SJens Axboe } 308c4320315SJens Axboe 309c4320315SJens Axboe if (copy_to_user(arg, new_count, sizeof(new_count))) 310c4320315SJens Axboe return -EFAULT; 311c4320315SJens Axboe 312c4320315SJens Axboe /* that's it for SQPOLL, only the SQPOLL task creates requests */ 313c4320315SJens Axboe if (sqd) 314c4320315SJens Axboe return 0; 315c4320315SJens Axboe 316c4320315SJens Axboe /* now propagate the restriction to all registered users */ 317c4320315SJens Axboe list_for_each_entry(node, &ctx->tctx_list, ctx_node) { 3181da2f311SJens Axboe tctx = node->task->io_uring; 319c4320315SJens Axboe if (WARN_ON_ONCE(!tctx->io_wq)) 320c4320315SJens Axboe continue; 321c4320315SJens Axboe 322c4320315SJens Axboe for (i = 0; i < ARRAY_SIZE(new_count); i++) 323c4320315SJens Axboe new_count[i] = ctx->iowq_limits[i]; 324c4320315SJens Axboe /* ignore errors, it always returns zero anyway */ 325c4320315SJens Axboe (void)io_wq_max_workers(tctx->io_wq, new_count); 326c4320315SJens Axboe } 327c4320315SJens Axboe return 0; 328c4320315SJens Axboe err: 329c4320315SJens Axboe if (sqd) { 33073254a29SHagar Hemdan mutex_unlock(&ctx->uring_lock); 331c4320315SJens Axboe mutex_unlock(&sqd->lock); 332c4320315SJens Axboe io_put_sq_data(sqd); 33373254a29SHagar Hemdan mutex_lock(&ctx->uring_lock); 334c4320315SJens Axboe } 335c4320315SJens Axboe return ret; 336c4320315SJens Axboe } 337c4320315SJens Axboe 3382b8e976bSPavel Begunkov static int io_register_clock(struct io_ring_ctx *ctx, 3392b8e976bSPavel Begunkov struct io_uring_clock_register __user *arg) 3402b8e976bSPavel Begunkov { 3412b8e976bSPavel Begunkov struct io_uring_clock_register reg; 3422b8e976bSPavel Begunkov 3432b8e976bSPavel Begunkov if (copy_from_user(®, arg, sizeof(reg))) 3442b8e976bSPavel Begunkov return -EFAULT; 3452b8e976bSPavel Begunkov if (memchr_inv(®.__resv, 0, sizeof(reg.__resv))) 3462b8e976bSPavel Begunkov return -EINVAL; 3472b8e976bSPavel Begunkov 3482b8e976bSPavel Begunkov switch (reg.clockid) { 3492b8e976bSPavel Begunkov case CLOCK_MONOTONIC: 3502b8e976bSPavel Begunkov ctx->clock_offset = 0; 3512b8e976bSPavel Begunkov break; 3522b8e976bSPavel Begunkov case CLOCK_BOOTTIME: 3532b8e976bSPavel Begunkov ctx->clock_offset = TK_OFFS_BOOT; 3542b8e976bSPavel Begunkov break; 3552b8e976bSPavel Begunkov default: 3562b8e976bSPavel Begunkov return -EINVAL; 3572b8e976bSPavel Begunkov } 3582b8e976bSPavel Begunkov 3592b8e976bSPavel Begunkov ctx->clockid = reg.clockid; 3602b8e976bSPavel Begunkov return 0; 3612b8e976bSPavel Begunkov } 3622b8e976bSPavel Begunkov 363c4320315SJens Axboe static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, 364c4320315SJens Axboe void __user *arg, unsigned nr_args) 365c4320315SJens Axboe __releases(ctx->uring_lock) 366c4320315SJens Axboe __acquires(ctx->uring_lock) 367c4320315SJens Axboe { 368c4320315SJens Axboe int ret; 369c4320315SJens Axboe 370c4320315SJens Axboe /* 371c4320315SJens Axboe * We don't quiesce the refs for register anymore and so it can't be 372c4320315SJens Axboe * dying as we're holding a file ref here. 373c4320315SJens Axboe */ 374c4320315SJens Axboe if (WARN_ON_ONCE(percpu_ref_is_dying(&ctx->refs))) 375c4320315SJens Axboe return -ENXIO; 376c4320315SJens Axboe 377c4320315SJens Axboe if (ctx->submitter_task && ctx->submitter_task != current) 378c4320315SJens Axboe return -EEXIST; 379c4320315SJens Axboe 380c4320315SJens Axboe if (ctx->restricted) { 381c4320315SJens Axboe opcode = array_index_nospec(opcode, IORING_REGISTER_LAST); 382c4320315SJens Axboe if (!test_bit(opcode, ctx->restrictions.register_op)) 383c4320315SJens Axboe return -EACCES; 384c4320315SJens Axboe } 385c4320315SJens Axboe 386c4320315SJens Axboe switch (opcode) { 387c4320315SJens Axboe case IORING_REGISTER_BUFFERS: 388c4320315SJens Axboe ret = -EFAULT; 389c4320315SJens Axboe if (!arg) 390c4320315SJens Axboe break; 391c4320315SJens Axboe ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL); 392c4320315SJens Axboe break; 393c4320315SJens Axboe case IORING_UNREGISTER_BUFFERS: 394c4320315SJens Axboe ret = -EINVAL; 395c4320315SJens Axboe if (arg || nr_args) 396c4320315SJens Axboe break; 397c4320315SJens Axboe ret = io_sqe_buffers_unregister(ctx); 398c4320315SJens Axboe break; 399c4320315SJens Axboe case IORING_REGISTER_FILES: 400c4320315SJens Axboe ret = -EFAULT; 401c4320315SJens Axboe if (!arg) 402c4320315SJens Axboe break; 403c4320315SJens Axboe ret = io_sqe_files_register(ctx, arg, nr_args, NULL); 404c4320315SJens Axboe break; 405c4320315SJens Axboe case IORING_UNREGISTER_FILES: 406c4320315SJens Axboe ret = -EINVAL; 407c4320315SJens Axboe if (arg || nr_args) 408c4320315SJens Axboe break; 409c4320315SJens Axboe ret = io_sqe_files_unregister(ctx); 410c4320315SJens Axboe break; 411c4320315SJens Axboe case IORING_REGISTER_FILES_UPDATE: 412c4320315SJens Axboe ret = io_register_files_update(ctx, arg, nr_args); 413c4320315SJens Axboe break; 414c4320315SJens Axboe case IORING_REGISTER_EVENTFD: 415c4320315SJens Axboe ret = -EINVAL; 416c4320315SJens Axboe if (nr_args != 1) 417c4320315SJens Axboe break; 418c4320315SJens Axboe ret = io_eventfd_register(ctx, arg, 0); 419c4320315SJens Axboe break; 420c4320315SJens Axboe case IORING_REGISTER_EVENTFD_ASYNC: 421c4320315SJens Axboe ret = -EINVAL; 422c4320315SJens Axboe if (nr_args != 1) 423c4320315SJens Axboe break; 424c4320315SJens Axboe ret = io_eventfd_register(ctx, arg, 1); 425c4320315SJens Axboe break; 426c4320315SJens Axboe case IORING_UNREGISTER_EVENTFD: 427c4320315SJens Axboe ret = -EINVAL; 428c4320315SJens Axboe if (arg || nr_args) 429c4320315SJens Axboe break; 430c4320315SJens Axboe ret = io_eventfd_unregister(ctx); 431c4320315SJens Axboe break; 432c4320315SJens Axboe case IORING_REGISTER_PROBE: 433c4320315SJens Axboe ret = -EINVAL; 434c4320315SJens Axboe if (!arg || nr_args > 256) 435c4320315SJens Axboe break; 436c4320315SJens Axboe ret = io_probe(ctx, arg, nr_args); 437c4320315SJens Axboe break; 438c4320315SJens Axboe case IORING_REGISTER_PERSONALITY: 439c4320315SJens Axboe ret = -EINVAL; 440c4320315SJens Axboe if (arg || nr_args) 441c4320315SJens Axboe break; 442c4320315SJens Axboe ret = io_register_personality(ctx); 443c4320315SJens Axboe break; 444c4320315SJens Axboe case IORING_UNREGISTER_PERSONALITY: 445c4320315SJens Axboe ret = -EINVAL; 446c4320315SJens Axboe if (arg) 447c4320315SJens Axboe break; 448c4320315SJens Axboe ret = io_unregister_personality(ctx, nr_args); 449c4320315SJens Axboe break; 450c4320315SJens Axboe case IORING_REGISTER_ENABLE_RINGS: 451c4320315SJens Axboe ret = -EINVAL; 452c4320315SJens Axboe if (arg || nr_args) 453c4320315SJens Axboe break; 454c4320315SJens Axboe ret = io_register_enable_rings(ctx); 455c4320315SJens Axboe break; 456c4320315SJens Axboe case IORING_REGISTER_RESTRICTIONS: 457c4320315SJens Axboe ret = io_register_restrictions(ctx, arg, nr_args); 458c4320315SJens Axboe break; 459c4320315SJens Axboe case IORING_REGISTER_FILES2: 460c4320315SJens Axboe ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE); 461c4320315SJens Axboe break; 462c4320315SJens Axboe case IORING_REGISTER_FILES_UPDATE2: 463c4320315SJens Axboe ret = io_register_rsrc_update(ctx, arg, nr_args, 464c4320315SJens Axboe IORING_RSRC_FILE); 465c4320315SJens Axboe break; 466c4320315SJens Axboe case IORING_REGISTER_BUFFERS2: 467c4320315SJens Axboe ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER); 468c4320315SJens Axboe break; 469c4320315SJens Axboe case IORING_REGISTER_BUFFERS_UPDATE: 470c4320315SJens Axboe ret = io_register_rsrc_update(ctx, arg, nr_args, 471c4320315SJens Axboe IORING_RSRC_BUFFER); 472c4320315SJens Axboe break; 473c4320315SJens Axboe case IORING_REGISTER_IOWQ_AFF: 474c4320315SJens Axboe ret = -EINVAL; 475c4320315SJens Axboe if (!arg || !nr_args) 476c4320315SJens Axboe break; 477c4320315SJens Axboe ret = io_register_iowq_aff(ctx, arg, nr_args); 478c4320315SJens Axboe break; 479c4320315SJens Axboe case IORING_UNREGISTER_IOWQ_AFF: 480c4320315SJens Axboe ret = -EINVAL; 481c4320315SJens Axboe if (arg || nr_args) 482c4320315SJens Axboe break; 483c4320315SJens Axboe ret = io_unregister_iowq_aff(ctx); 484c4320315SJens Axboe break; 485c4320315SJens Axboe case IORING_REGISTER_IOWQ_MAX_WORKERS: 486c4320315SJens Axboe ret = -EINVAL; 487c4320315SJens Axboe if (!arg || nr_args != 2) 488c4320315SJens Axboe break; 489c4320315SJens Axboe ret = io_register_iowq_max_workers(ctx, arg); 490c4320315SJens Axboe break; 491c4320315SJens Axboe case IORING_REGISTER_RING_FDS: 492c4320315SJens Axboe ret = io_ringfd_register(ctx, arg, nr_args); 493c4320315SJens Axboe break; 494c4320315SJens Axboe case IORING_UNREGISTER_RING_FDS: 495c4320315SJens Axboe ret = io_ringfd_unregister(ctx, arg, nr_args); 496c4320315SJens Axboe break; 497c4320315SJens Axboe case IORING_REGISTER_PBUF_RING: 498c4320315SJens Axboe ret = -EINVAL; 499c4320315SJens Axboe if (!arg || nr_args != 1) 500c4320315SJens Axboe break; 501c4320315SJens Axboe ret = io_register_pbuf_ring(ctx, arg); 502c4320315SJens Axboe break; 503c4320315SJens Axboe case IORING_UNREGISTER_PBUF_RING: 504c4320315SJens Axboe ret = -EINVAL; 505c4320315SJens Axboe if (!arg || nr_args != 1) 506c4320315SJens Axboe break; 507c4320315SJens Axboe ret = io_unregister_pbuf_ring(ctx, arg); 508c4320315SJens Axboe break; 509c4320315SJens Axboe case IORING_REGISTER_SYNC_CANCEL: 510c4320315SJens Axboe ret = -EINVAL; 511c4320315SJens Axboe if (!arg || nr_args != 1) 512c4320315SJens Axboe break; 513c4320315SJens Axboe ret = io_sync_cancel(ctx, arg); 514c4320315SJens Axboe break; 515c4320315SJens Axboe case IORING_REGISTER_FILE_ALLOC_RANGE: 516c4320315SJens Axboe ret = -EINVAL; 517c4320315SJens Axboe if (!arg || nr_args) 518c4320315SJens Axboe break; 519c4320315SJens Axboe ret = io_register_file_alloc_range(ctx, arg); 520c4320315SJens Axboe break; 521d293b1a8SJens Axboe case IORING_REGISTER_PBUF_STATUS: 522d293b1a8SJens Axboe ret = -EINVAL; 523d293b1a8SJens Axboe if (!arg || nr_args != 1) 524d293b1a8SJens Axboe break; 525d293b1a8SJens Axboe ret = io_register_pbuf_status(ctx, arg); 526d293b1a8SJens Axboe break; 527ef1186c1SStefan Roesch case IORING_REGISTER_NAPI: 528ef1186c1SStefan Roesch ret = -EINVAL; 529ef1186c1SStefan Roesch if (!arg || nr_args != 1) 530ef1186c1SStefan Roesch break; 531ef1186c1SStefan Roesch ret = io_register_napi(ctx, arg); 532ef1186c1SStefan Roesch break; 533ef1186c1SStefan Roesch case IORING_UNREGISTER_NAPI: 534ef1186c1SStefan Roesch ret = -EINVAL; 535ef1186c1SStefan Roesch if (nr_args != 1) 536ef1186c1SStefan Roesch break; 537ef1186c1SStefan Roesch ret = io_unregister_napi(ctx, arg); 538ef1186c1SStefan Roesch break; 5392b8e976bSPavel Begunkov case IORING_REGISTER_CLOCK: 5402b8e976bSPavel Begunkov ret = -EINVAL; 5412b8e976bSPavel Begunkov if (!arg || nr_args) 5422b8e976bSPavel Begunkov break; 5432b8e976bSPavel Begunkov ret = io_register_clock(ctx, arg); 5442b8e976bSPavel Begunkov break; 545*7cc2a6eaSJens Axboe case IORING_REGISTER_COPY_BUFFERS: 546*7cc2a6eaSJens Axboe ret = -EINVAL; 547*7cc2a6eaSJens Axboe if (!arg || nr_args != 1) 548*7cc2a6eaSJens Axboe break; 549*7cc2a6eaSJens Axboe ret = io_register_copy_buffers(ctx, arg); 550*7cc2a6eaSJens Axboe break; 551c4320315SJens Axboe default: 552c4320315SJens Axboe ret = -EINVAL; 553c4320315SJens Axboe break; 554c4320315SJens Axboe } 555c4320315SJens Axboe 556c4320315SJens Axboe return ret; 557c4320315SJens Axboe } 558c4320315SJens Axboe 5590b6d253eSJens Axboe /* 5600b6d253eSJens Axboe * Given an 'fd' value, return the ctx associated with if. If 'registered' is 5610b6d253eSJens Axboe * true, then the registered index is used. Otherwise, the normal fd table. 5620b6d253eSJens Axboe * Caller must call fput() on the returned file, unless it's an ERR_PTR. 5630b6d253eSJens Axboe */ 5640b6d253eSJens Axboe struct file *io_uring_register_get_file(int fd, bool registered) 5650b6d253eSJens Axboe { 5660b6d253eSJens Axboe struct file *file; 5670b6d253eSJens Axboe 5680b6d253eSJens Axboe if (registered) { 5690b6d253eSJens Axboe /* 5700b6d253eSJens Axboe * Ring fd has been registered via IORING_REGISTER_RING_FDS, we 5710b6d253eSJens Axboe * need only dereference our task private array to find it. 5720b6d253eSJens Axboe */ 5730b6d253eSJens Axboe struct io_uring_task *tctx = current->io_uring; 5740b6d253eSJens Axboe 5750b6d253eSJens Axboe if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX)) 5760b6d253eSJens Axboe return ERR_PTR(-EINVAL); 5770b6d253eSJens Axboe fd = array_index_nospec(fd, IO_RINGFD_REG_MAX); 5780b6d253eSJens Axboe file = tctx->registered_rings[fd]; 5790b6d253eSJens Axboe } else { 5800b6d253eSJens Axboe file = fget(fd); 5810b6d253eSJens Axboe } 5820b6d253eSJens Axboe 5830b6d253eSJens Axboe if (unlikely(!file)) 5840b6d253eSJens Axboe return ERR_PTR(-EBADF); 5850b6d253eSJens Axboe if (io_is_uring_fops(file)) 5860b6d253eSJens Axboe return file; 5870b6d253eSJens Axboe fput(file); 5880b6d253eSJens Axboe return ERR_PTR(-EOPNOTSUPP); 5890b6d253eSJens Axboe } 5900b6d253eSJens Axboe 591c4320315SJens Axboe SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, 592c4320315SJens Axboe void __user *, arg, unsigned int, nr_args) 593c4320315SJens Axboe { 594c4320315SJens Axboe struct io_ring_ctx *ctx; 595c4320315SJens Axboe long ret = -EBADF; 596c4320315SJens Axboe struct file *file; 597c4320315SJens Axboe bool use_registered_ring; 598c4320315SJens Axboe 599c4320315SJens Axboe use_registered_ring = !!(opcode & IORING_REGISTER_USE_REGISTERED_RING); 600c4320315SJens Axboe opcode &= ~IORING_REGISTER_USE_REGISTERED_RING; 601c4320315SJens Axboe 602c4320315SJens Axboe if (opcode >= IORING_REGISTER_LAST) 603c4320315SJens Axboe return -EINVAL; 604c4320315SJens Axboe 6050b6d253eSJens Axboe file = io_uring_register_get_file(fd, use_registered_ring); 6060b6d253eSJens Axboe if (IS_ERR(file)) 6070b6d253eSJens Axboe return PTR_ERR(file); 608c4320315SJens Axboe ctx = file->private_data; 609c4320315SJens Axboe 610c4320315SJens Axboe mutex_lock(&ctx->uring_lock); 611c4320315SJens Axboe ret = __io_uring_register(ctx, opcode, arg, nr_args); 612c4320315SJens Axboe mutex_unlock(&ctx->uring_lock); 613c4320315SJens Axboe trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, ret); 614c4320315SJens Axboe if (!use_registered_ring) 615c4320315SJens Axboe fput(file); 616c4320315SJens Axboe return ret; 617c4320315SJens Axboe } 618