1c4320315SJens Axboe // SPDX-License-Identifier: GPL-2.0
2c4320315SJens Axboe /*
3c4320315SJens Axboe * Code related to the io_uring_register() syscall
4c4320315SJens Axboe *
5c4320315SJens Axboe * Copyright (C) 2023 Jens Axboe
6c4320315SJens Axboe */
7c4320315SJens Axboe #include <linux/kernel.h>
8c4320315SJens Axboe #include <linux/errno.h>
9c4320315SJens Axboe #include <linux/syscalls.h>
10c4320315SJens Axboe #include <linux/refcount.h>
11c4320315SJens Axboe #include <linux/bits.h>
12c4320315SJens Axboe #include <linux/fs.h>
13c4320315SJens Axboe #include <linux/file.h>
14c4320315SJens Axboe #include <linux/slab.h>
15c4320315SJens Axboe #include <linux/uaccess.h>
16c4320315SJens Axboe #include <linux/nospec.h>
17baf59771SJens Axboe #include <linux/compat.h>
18c4320315SJens Axboe #include <linux/io_uring.h>
19c4320315SJens Axboe #include <linux/io_uring_types.h>
20c4320315SJens Axboe
21c4320315SJens Axboe #include "io_uring.h"
22c4320315SJens Axboe #include "opdef.h"
23c4320315SJens Axboe #include "tctx.h"
24c4320315SJens Axboe #include "rsrc.h"
25c4320315SJens Axboe #include "sqpoll.h"
26c4320315SJens Axboe #include "register.h"
27c4320315SJens Axboe #include "cancel.h"
28c4320315SJens Axboe #include "kbuf.h"
29ef1186c1SStefan Roesch #include "napi.h"
30200f3abdSJens Axboe #include "eventfd.h"
31c4320315SJens Axboe
32c4320315SJens Axboe #define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \
33c4320315SJens Axboe IORING_REGISTER_LAST + IORING_OP_LAST)
34c4320315SJens Axboe
io_probe(struct io_ring_ctx * ctx,void __user * arg,unsigned nr_args)35c4320315SJens Axboe static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg,
36c4320315SJens Axboe unsigned nr_args)
37c4320315SJens Axboe {
38c4320315SJens Axboe struct io_uring_probe *p;
39c4320315SJens Axboe size_t size;
40c4320315SJens Axboe int i, ret;
41c4320315SJens Axboe
426bc9199dSGabriel Krisman Bertazi if (nr_args > IORING_OP_LAST)
436bc9199dSGabriel Krisman Bertazi nr_args = IORING_OP_LAST;
446bc9199dSGabriel Krisman Bertazi
45c4320315SJens Axboe size = struct_size(p, ops, nr_args);
46c4320315SJens Axboe p = kzalloc(size, GFP_KERNEL);
47c4320315SJens Axboe if (!p)
48c4320315SJens Axboe return -ENOMEM;
49c4320315SJens Axboe
50c4320315SJens Axboe ret = -EFAULT;
51c4320315SJens Axboe if (copy_from_user(p, arg, size))
52c4320315SJens Axboe goto out;
53c4320315SJens Axboe ret = -EINVAL;
54c4320315SJens Axboe if (memchr_inv(p, 0, size))
55c4320315SJens Axboe goto out;
56c4320315SJens Axboe
57c4320315SJens Axboe p->last_op = IORING_OP_LAST - 1;
58c4320315SJens Axboe
59c4320315SJens Axboe for (i = 0; i < nr_args; i++) {
60c4320315SJens Axboe p->ops[i].op = i;
613e05b222SGabriel Krisman Bertazi if (io_uring_op_supported(i))
62c4320315SJens Axboe p->ops[i].flags = IO_URING_OP_SUPPORTED;
63c4320315SJens Axboe }
64c4320315SJens Axboe p->ops_len = i;
65c4320315SJens Axboe
66c4320315SJens Axboe ret = 0;
67c4320315SJens Axboe if (copy_to_user(arg, p, size))
68c4320315SJens Axboe ret = -EFAULT;
69c4320315SJens Axboe out:
70c4320315SJens Axboe kfree(p);
71c4320315SJens Axboe return ret;
72c4320315SJens Axboe }
73c4320315SJens Axboe
io_unregister_personality(struct io_ring_ctx * ctx,unsigned id)74c4320315SJens Axboe int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
75c4320315SJens Axboe {
76c4320315SJens Axboe const struct cred *creds;
77c4320315SJens Axboe
78c4320315SJens Axboe creds = xa_erase(&ctx->personalities, id);
79c4320315SJens Axboe if (creds) {
80c4320315SJens Axboe put_cred(creds);
81c4320315SJens Axboe return 0;
82c4320315SJens Axboe }
83c4320315SJens Axboe
84c4320315SJens Axboe return -EINVAL;
85c4320315SJens Axboe }
86c4320315SJens Axboe
87c4320315SJens Axboe
io_register_personality(struct io_ring_ctx * ctx)88c4320315SJens Axboe static int io_register_personality(struct io_ring_ctx *ctx)
89c4320315SJens Axboe {
90c4320315SJens Axboe const struct cred *creds;
91c4320315SJens Axboe u32 id;
92c4320315SJens Axboe int ret;
93c4320315SJens Axboe
94c4320315SJens Axboe creds = get_current_cred();
95c4320315SJens Axboe
96c4320315SJens Axboe ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds,
97c4320315SJens Axboe XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL);
98c4320315SJens Axboe if (ret < 0) {
99c4320315SJens Axboe put_cred(creds);
100c4320315SJens Axboe return ret;
101c4320315SJens Axboe }
102c4320315SJens Axboe return id;
103c4320315SJens Axboe }
104c4320315SJens Axboe
io_register_restrictions(struct io_ring_ctx * ctx,void __user * arg,unsigned int nr_args)105c4320315SJens Axboe static __cold int io_register_restrictions(struct io_ring_ctx *ctx,
106c4320315SJens Axboe void __user *arg, unsigned int nr_args)
107c4320315SJens Axboe {
108c4320315SJens Axboe struct io_uring_restriction *res;
109c4320315SJens Axboe size_t size;
110c4320315SJens Axboe int i, ret;
111c4320315SJens Axboe
112c4320315SJens Axboe /* Restrictions allowed only if rings started disabled */
113c4320315SJens Axboe if (!(ctx->flags & IORING_SETUP_R_DISABLED))
114c4320315SJens Axboe return -EBADFD;
115c4320315SJens Axboe
116c4320315SJens Axboe /* We allow only a single restrictions registration */
117c4320315SJens Axboe if (ctx->restrictions.registered)
118c4320315SJens Axboe return -EBUSY;
119c4320315SJens Axboe
120c4320315SJens Axboe if (!arg || nr_args > IORING_MAX_RESTRICTIONS)
121c4320315SJens Axboe return -EINVAL;
122c4320315SJens Axboe
123c4320315SJens Axboe size = array_size(nr_args, sizeof(*res));
124c4320315SJens Axboe if (size == SIZE_MAX)
125c4320315SJens Axboe return -EOVERFLOW;
126c4320315SJens Axboe
127c4320315SJens Axboe res = memdup_user(arg, size);
128c4320315SJens Axboe if (IS_ERR(res))
129c4320315SJens Axboe return PTR_ERR(res);
130c4320315SJens Axboe
131c4320315SJens Axboe ret = 0;
132c4320315SJens Axboe
133c4320315SJens Axboe for (i = 0; i < nr_args; i++) {
134c4320315SJens Axboe switch (res[i].opcode) {
135c4320315SJens Axboe case IORING_RESTRICTION_REGISTER_OP:
136c4320315SJens Axboe if (res[i].register_op >= IORING_REGISTER_LAST) {
137c4320315SJens Axboe ret = -EINVAL;
138c4320315SJens Axboe goto out;
139c4320315SJens Axboe }
140c4320315SJens Axboe
141c4320315SJens Axboe __set_bit(res[i].register_op,
142c4320315SJens Axboe ctx->restrictions.register_op);
143c4320315SJens Axboe break;
144c4320315SJens Axboe case IORING_RESTRICTION_SQE_OP:
145c4320315SJens Axboe if (res[i].sqe_op >= IORING_OP_LAST) {
146c4320315SJens Axboe ret = -EINVAL;
147c4320315SJens Axboe goto out;
148c4320315SJens Axboe }
149c4320315SJens Axboe
150c4320315SJens Axboe __set_bit(res[i].sqe_op, ctx->restrictions.sqe_op);
151c4320315SJens Axboe break;
152c4320315SJens Axboe case IORING_RESTRICTION_SQE_FLAGS_ALLOWED:
153c4320315SJens Axboe ctx->restrictions.sqe_flags_allowed = res[i].sqe_flags;
154c4320315SJens Axboe break;
155c4320315SJens Axboe case IORING_RESTRICTION_SQE_FLAGS_REQUIRED:
156c4320315SJens Axboe ctx->restrictions.sqe_flags_required = res[i].sqe_flags;
157c4320315SJens Axboe break;
158c4320315SJens Axboe default:
159c4320315SJens Axboe ret = -EINVAL;
160c4320315SJens Axboe goto out;
161c4320315SJens Axboe }
162c4320315SJens Axboe }
163c4320315SJens Axboe
164c4320315SJens Axboe out:
165c4320315SJens Axboe /* Reset all restrictions if an error happened */
166c4320315SJens Axboe if (ret != 0)
167c4320315SJens Axboe memset(&ctx->restrictions, 0, sizeof(ctx->restrictions));
168c4320315SJens Axboe else
169c4320315SJens Axboe ctx->restrictions.registered = true;
170c4320315SJens Axboe
171c4320315SJens Axboe kfree(res);
172c4320315SJens Axboe return ret;
173c4320315SJens Axboe }
174c4320315SJens Axboe
io_register_enable_rings(struct io_ring_ctx * ctx)175c4320315SJens Axboe static int io_register_enable_rings(struct io_ring_ctx *ctx)
176c4320315SJens Axboe {
177c4320315SJens Axboe if (!(ctx->flags & IORING_SETUP_R_DISABLED))
178c4320315SJens Axboe return -EBADFD;
179c4320315SJens Axboe
180c4320315SJens Axboe if (ctx->flags & IORING_SETUP_SINGLE_ISSUER && !ctx->submitter_task) {
181c4320315SJens Axboe WRITE_ONCE(ctx->submitter_task, get_task_struct(current));
182c4320315SJens Axboe /*
183c4320315SJens Axboe * Lazy activation attempts would fail if it was polled before
184c4320315SJens Axboe * submitter_task is set.
185c4320315SJens Axboe */
186c4320315SJens Axboe if (wq_has_sleeper(&ctx->poll_wq))
187c4320315SJens Axboe io_activate_pollwq(ctx);
188c4320315SJens Axboe }
189c4320315SJens Axboe
190c4320315SJens Axboe if (ctx->restrictions.registered)
191c4320315SJens Axboe ctx->restricted = 1;
192c4320315SJens Axboe
193c4320315SJens Axboe ctx->flags &= ~IORING_SETUP_R_DISABLED;
194c4320315SJens Axboe if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait))
195c4320315SJens Axboe wake_up(&ctx->sq_data->wait);
196c4320315SJens Axboe return 0;
197c4320315SJens Axboe }
198c4320315SJens Axboe
__io_register_iowq_aff(struct io_ring_ctx * ctx,cpumask_var_t new_mask)199c4320315SJens Axboe static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx,
200c4320315SJens Axboe cpumask_var_t new_mask)
201c4320315SJens Axboe {
202c4320315SJens Axboe int ret;
203c4320315SJens Axboe
204c4320315SJens Axboe if (!(ctx->flags & IORING_SETUP_SQPOLL)) {
205c4320315SJens Axboe ret = io_wq_cpu_affinity(current->io_uring, new_mask);
206c4320315SJens Axboe } else {
207c4320315SJens Axboe mutex_unlock(&ctx->uring_lock);
208c4320315SJens Axboe ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask);
209c4320315SJens Axboe mutex_lock(&ctx->uring_lock);
210c4320315SJens Axboe }
211c4320315SJens Axboe
212c4320315SJens Axboe return ret;
213c4320315SJens Axboe }
214c4320315SJens Axboe
io_register_iowq_aff(struct io_ring_ctx * ctx,void __user * arg,unsigned len)215c4320315SJens Axboe static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx,
216c4320315SJens Axboe void __user *arg, unsigned len)
217c4320315SJens Axboe {
218c4320315SJens Axboe cpumask_var_t new_mask;
219c4320315SJens Axboe int ret;
220c4320315SJens Axboe
221c4320315SJens Axboe if (!alloc_cpumask_var(&new_mask, GFP_KERNEL))
222c4320315SJens Axboe return -ENOMEM;
223c4320315SJens Axboe
224c4320315SJens Axboe cpumask_clear(new_mask);
225c4320315SJens Axboe if (len > cpumask_size())
226c4320315SJens Axboe len = cpumask_size();
227c4320315SJens Axboe
228baf59771SJens Axboe #ifdef CONFIG_COMPAT
229baf59771SJens Axboe if (in_compat_syscall())
230c4320315SJens Axboe ret = compat_get_bitmap(cpumask_bits(new_mask),
231c4320315SJens Axboe (const compat_ulong_t __user *)arg,
232c4320315SJens Axboe len * 8 /* CHAR_BIT */);
233baf59771SJens Axboe else
234baf59771SJens Axboe #endif
235c4320315SJens Axboe ret = copy_from_user(new_mask, arg, len);
236c4320315SJens Axboe
237c4320315SJens Axboe if (ret) {
238c4320315SJens Axboe free_cpumask_var(new_mask);
239c4320315SJens Axboe return -EFAULT;
240c4320315SJens Axboe }
241c4320315SJens Axboe
242c4320315SJens Axboe ret = __io_register_iowq_aff(ctx, new_mask);
243c4320315SJens Axboe free_cpumask_var(new_mask);
244c4320315SJens Axboe return ret;
245c4320315SJens Axboe }
246c4320315SJens Axboe
io_unregister_iowq_aff(struct io_ring_ctx * ctx)247c4320315SJens Axboe static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
248c4320315SJens Axboe {
249c4320315SJens Axboe return __io_register_iowq_aff(ctx, NULL);
250c4320315SJens Axboe }
251c4320315SJens Axboe
io_register_iowq_max_workers(struct io_ring_ctx * ctx,void __user * arg)252c4320315SJens Axboe static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
253c4320315SJens Axboe void __user *arg)
254c4320315SJens Axboe __must_hold(&ctx->uring_lock)
255c4320315SJens Axboe {
256c4320315SJens Axboe struct io_tctx_node *node;
257c4320315SJens Axboe struct io_uring_task *tctx = NULL;
258c4320315SJens Axboe struct io_sq_data *sqd = NULL;
259c4320315SJens Axboe __u32 new_count[2];
260c4320315SJens Axboe int i, ret;
261c4320315SJens Axboe
262c4320315SJens Axboe if (copy_from_user(new_count, arg, sizeof(new_count)))
263c4320315SJens Axboe return -EFAULT;
264c4320315SJens Axboe for (i = 0; i < ARRAY_SIZE(new_count); i++)
265c4320315SJens Axboe if (new_count[i] > INT_MAX)
266c4320315SJens Axboe return -EINVAL;
267c4320315SJens Axboe
268c4320315SJens Axboe if (ctx->flags & IORING_SETUP_SQPOLL) {
269c4320315SJens Axboe sqd = ctx->sq_data;
270c4320315SJens Axboe if (sqd) {
271c4320315SJens Axboe /*
272c4320315SJens Axboe * Observe the correct sqd->lock -> ctx->uring_lock
273c4320315SJens Axboe * ordering. Fine to drop uring_lock here, we hold
274c4320315SJens Axboe * a ref to the ctx.
275c4320315SJens Axboe */
276c4320315SJens Axboe refcount_inc(&sqd->refs);
277c4320315SJens Axboe mutex_unlock(&ctx->uring_lock);
278c4320315SJens Axboe mutex_lock(&sqd->lock);
279c4320315SJens Axboe mutex_lock(&ctx->uring_lock);
280c4320315SJens Axboe if (sqd->thread)
281c4320315SJens Axboe tctx = sqd->thread->io_uring;
282c4320315SJens Axboe }
283c4320315SJens Axboe } else {
284c4320315SJens Axboe tctx = current->io_uring;
285c4320315SJens Axboe }
286c4320315SJens Axboe
287c4320315SJens Axboe BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits));
288c4320315SJens Axboe
289c4320315SJens Axboe for (i = 0; i < ARRAY_SIZE(new_count); i++)
290c4320315SJens Axboe if (new_count[i])
291c4320315SJens Axboe ctx->iowq_limits[i] = new_count[i];
292c4320315SJens Axboe ctx->iowq_limits_set = true;
293c4320315SJens Axboe
294c4320315SJens Axboe if (tctx && tctx->io_wq) {
295c4320315SJens Axboe ret = io_wq_max_workers(tctx->io_wq, new_count);
296c4320315SJens Axboe if (ret)
297c4320315SJens Axboe goto err;
298c4320315SJens Axboe } else {
299c4320315SJens Axboe memset(new_count, 0, sizeof(new_count));
300c4320315SJens Axboe }
301c4320315SJens Axboe
302c4320315SJens Axboe if (sqd) {
30373254a29SHagar Hemdan mutex_unlock(&ctx->uring_lock);
304c4320315SJens Axboe mutex_unlock(&sqd->lock);
305c4320315SJens Axboe io_put_sq_data(sqd);
30673254a29SHagar Hemdan mutex_lock(&ctx->uring_lock);
307c4320315SJens Axboe }
308c4320315SJens Axboe
309c4320315SJens Axboe if (copy_to_user(arg, new_count, sizeof(new_count)))
310c4320315SJens Axboe return -EFAULT;
311c4320315SJens Axboe
312c4320315SJens Axboe /* that's it for SQPOLL, only the SQPOLL task creates requests */
313c4320315SJens Axboe if (sqd)
314c4320315SJens Axboe return 0;
315c4320315SJens Axboe
316c4320315SJens Axboe /* now propagate the restriction to all registered users */
317c4320315SJens Axboe list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
3181da2f311SJens Axboe tctx = node->task->io_uring;
319c4320315SJens Axboe if (WARN_ON_ONCE(!tctx->io_wq))
320c4320315SJens Axboe continue;
321c4320315SJens Axboe
322c4320315SJens Axboe for (i = 0; i < ARRAY_SIZE(new_count); i++)
323c4320315SJens Axboe new_count[i] = ctx->iowq_limits[i];
324c4320315SJens Axboe /* ignore errors, it always returns zero anyway */
325c4320315SJens Axboe (void)io_wq_max_workers(tctx->io_wq, new_count);
326c4320315SJens Axboe }
327c4320315SJens Axboe return 0;
328c4320315SJens Axboe err:
329c4320315SJens Axboe if (sqd) {
33073254a29SHagar Hemdan mutex_unlock(&ctx->uring_lock);
331c4320315SJens Axboe mutex_unlock(&sqd->lock);
332c4320315SJens Axboe io_put_sq_data(sqd);
33373254a29SHagar Hemdan mutex_lock(&ctx->uring_lock);
334c4320315SJens Axboe }
335c4320315SJens Axboe return ret;
336c4320315SJens Axboe }
337c4320315SJens Axboe
io_register_clock(struct io_ring_ctx * ctx,struct io_uring_clock_register __user * arg)3382b8e976bSPavel Begunkov static int io_register_clock(struct io_ring_ctx *ctx,
3392b8e976bSPavel Begunkov struct io_uring_clock_register __user *arg)
3402b8e976bSPavel Begunkov {
3412b8e976bSPavel Begunkov struct io_uring_clock_register reg;
3422b8e976bSPavel Begunkov
3432b8e976bSPavel Begunkov if (copy_from_user(®, arg, sizeof(reg)))
3442b8e976bSPavel Begunkov return -EFAULT;
3452b8e976bSPavel Begunkov if (memchr_inv(®.__resv, 0, sizeof(reg.__resv)))
3462b8e976bSPavel Begunkov return -EINVAL;
3472b8e976bSPavel Begunkov
3482b8e976bSPavel Begunkov switch (reg.clockid) {
3492b8e976bSPavel Begunkov case CLOCK_MONOTONIC:
3502b8e976bSPavel Begunkov ctx->clock_offset = 0;
3512b8e976bSPavel Begunkov break;
3522b8e976bSPavel Begunkov case CLOCK_BOOTTIME:
3532b8e976bSPavel Begunkov ctx->clock_offset = TK_OFFS_BOOT;
3542b8e976bSPavel Begunkov break;
3552b8e976bSPavel Begunkov default:
3562b8e976bSPavel Begunkov return -EINVAL;
3572b8e976bSPavel Begunkov }
3582b8e976bSPavel Begunkov
3592b8e976bSPavel Begunkov ctx->clockid = reg.clockid;
3602b8e976bSPavel Begunkov return 0;
3612b8e976bSPavel Begunkov }
3622b8e976bSPavel Begunkov
__io_uring_register(struct io_ring_ctx * ctx,unsigned opcode,void __user * arg,unsigned nr_args)363c4320315SJens Axboe static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
364c4320315SJens Axboe void __user *arg, unsigned nr_args)
365c4320315SJens Axboe __releases(ctx->uring_lock)
366c4320315SJens Axboe __acquires(ctx->uring_lock)
367c4320315SJens Axboe {
368c4320315SJens Axboe int ret;
369c4320315SJens Axboe
370c4320315SJens Axboe /*
371c4320315SJens Axboe * We don't quiesce the refs for register anymore and so it can't be
372c4320315SJens Axboe * dying as we're holding a file ref here.
373c4320315SJens Axboe */
374c4320315SJens Axboe if (WARN_ON_ONCE(percpu_ref_is_dying(&ctx->refs)))
375c4320315SJens Axboe return -ENXIO;
376c4320315SJens Axboe
377c4320315SJens Axboe if (ctx->submitter_task && ctx->submitter_task != current)
378c4320315SJens Axboe return -EEXIST;
379c4320315SJens Axboe
380c4320315SJens Axboe if (ctx->restricted) {
381c4320315SJens Axboe opcode = array_index_nospec(opcode, IORING_REGISTER_LAST);
382c4320315SJens Axboe if (!test_bit(opcode, ctx->restrictions.register_op))
383c4320315SJens Axboe return -EACCES;
384c4320315SJens Axboe }
385c4320315SJens Axboe
386c4320315SJens Axboe switch (opcode) {
387c4320315SJens Axboe case IORING_REGISTER_BUFFERS:
388c4320315SJens Axboe ret = -EFAULT;
389c4320315SJens Axboe if (!arg)
390c4320315SJens Axboe break;
391c4320315SJens Axboe ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL);
392c4320315SJens Axboe break;
393c4320315SJens Axboe case IORING_UNREGISTER_BUFFERS:
394c4320315SJens Axboe ret = -EINVAL;
395c4320315SJens Axboe if (arg || nr_args)
396c4320315SJens Axboe break;
397c4320315SJens Axboe ret = io_sqe_buffers_unregister(ctx);
398c4320315SJens Axboe break;
399c4320315SJens Axboe case IORING_REGISTER_FILES:
400c4320315SJens Axboe ret = -EFAULT;
401c4320315SJens Axboe if (!arg)
402c4320315SJens Axboe break;
403c4320315SJens Axboe ret = io_sqe_files_register(ctx, arg, nr_args, NULL);
404c4320315SJens Axboe break;
405c4320315SJens Axboe case IORING_UNREGISTER_FILES:
406c4320315SJens Axboe ret = -EINVAL;
407c4320315SJens Axboe if (arg || nr_args)
408c4320315SJens Axboe break;
409c4320315SJens Axboe ret = io_sqe_files_unregister(ctx);
410c4320315SJens Axboe break;
411c4320315SJens Axboe case IORING_REGISTER_FILES_UPDATE:
412c4320315SJens Axboe ret = io_register_files_update(ctx, arg, nr_args);
413c4320315SJens Axboe break;
414c4320315SJens Axboe case IORING_REGISTER_EVENTFD:
415c4320315SJens Axboe ret = -EINVAL;
416c4320315SJens Axboe if (nr_args != 1)
417c4320315SJens Axboe break;
418c4320315SJens Axboe ret = io_eventfd_register(ctx, arg, 0);
419c4320315SJens Axboe break;
420c4320315SJens Axboe case IORING_REGISTER_EVENTFD_ASYNC:
421c4320315SJens Axboe ret = -EINVAL;
422c4320315SJens Axboe if (nr_args != 1)
423c4320315SJens Axboe break;
424c4320315SJens Axboe ret = io_eventfd_register(ctx, arg, 1);
425c4320315SJens Axboe break;
426c4320315SJens Axboe case IORING_UNREGISTER_EVENTFD:
427c4320315SJens Axboe ret = -EINVAL;
428c4320315SJens Axboe if (arg || nr_args)
429c4320315SJens Axboe break;
430c4320315SJens Axboe ret = io_eventfd_unregister(ctx);
431c4320315SJens Axboe break;
432c4320315SJens Axboe case IORING_REGISTER_PROBE:
433c4320315SJens Axboe ret = -EINVAL;
434c4320315SJens Axboe if (!arg || nr_args > 256)
435c4320315SJens Axboe break;
436c4320315SJens Axboe ret = io_probe(ctx, arg, nr_args);
437c4320315SJens Axboe break;
438c4320315SJens Axboe case IORING_REGISTER_PERSONALITY:
439c4320315SJens Axboe ret = -EINVAL;
440c4320315SJens Axboe if (arg || nr_args)
441c4320315SJens Axboe break;
442c4320315SJens Axboe ret = io_register_personality(ctx);
443c4320315SJens Axboe break;
444c4320315SJens Axboe case IORING_UNREGISTER_PERSONALITY:
445c4320315SJens Axboe ret = -EINVAL;
446c4320315SJens Axboe if (arg)
447c4320315SJens Axboe break;
448c4320315SJens Axboe ret = io_unregister_personality(ctx, nr_args);
449c4320315SJens Axboe break;
450c4320315SJens Axboe case IORING_REGISTER_ENABLE_RINGS:
451c4320315SJens Axboe ret = -EINVAL;
452c4320315SJens Axboe if (arg || nr_args)
453c4320315SJens Axboe break;
454c4320315SJens Axboe ret = io_register_enable_rings(ctx);
455c4320315SJens Axboe break;
456c4320315SJens Axboe case IORING_REGISTER_RESTRICTIONS:
457c4320315SJens Axboe ret = io_register_restrictions(ctx, arg, nr_args);
458c4320315SJens Axboe break;
459c4320315SJens Axboe case IORING_REGISTER_FILES2:
460c4320315SJens Axboe ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE);
461c4320315SJens Axboe break;
462c4320315SJens Axboe case IORING_REGISTER_FILES_UPDATE2:
463c4320315SJens Axboe ret = io_register_rsrc_update(ctx, arg, nr_args,
464c4320315SJens Axboe IORING_RSRC_FILE);
465c4320315SJens Axboe break;
466c4320315SJens Axboe case IORING_REGISTER_BUFFERS2:
467c4320315SJens Axboe ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER);
468c4320315SJens Axboe break;
469c4320315SJens Axboe case IORING_REGISTER_BUFFERS_UPDATE:
470c4320315SJens Axboe ret = io_register_rsrc_update(ctx, arg, nr_args,
471c4320315SJens Axboe IORING_RSRC_BUFFER);
472c4320315SJens Axboe break;
473c4320315SJens Axboe case IORING_REGISTER_IOWQ_AFF:
474c4320315SJens Axboe ret = -EINVAL;
475c4320315SJens Axboe if (!arg || !nr_args)
476c4320315SJens Axboe break;
477c4320315SJens Axboe ret = io_register_iowq_aff(ctx, arg, nr_args);
478c4320315SJens Axboe break;
479c4320315SJens Axboe case IORING_UNREGISTER_IOWQ_AFF:
480c4320315SJens Axboe ret = -EINVAL;
481c4320315SJens Axboe if (arg || nr_args)
482c4320315SJens Axboe break;
483c4320315SJens Axboe ret = io_unregister_iowq_aff(ctx);
484c4320315SJens Axboe break;
485c4320315SJens Axboe case IORING_REGISTER_IOWQ_MAX_WORKERS:
486c4320315SJens Axboe ret = -EINVAL;
487c4320315SJens Axboe if (!arg || nr_args != 2)
488c4320315SJens Axboe break;
489c4320315SJens Axboe ret = io_register_iowq_max_workers(ctx, arg);
490c4320315SJens Axboe break;
491c4320315SJens Axboe case IORING_REGISTER_RING_FDS:
492c4320315SJens Axboe ret = io_ringfd_register(ctx, arg, nr_args);
493c4320315SJens Axboe break;
494c4320315SJens Axboe case IORING_UNREGISTER_RING_FDS:
495c4320315SJens Axboe ret = io_ringfd_unregister(ctx, arg, nr_args);
496c4320315SJens Axboe break;
497c4320315SJens Axboe case IORING_REGISTER_PBUF_RING:
498c4320315SJens Axboe ret = -EINVAL;
499c4320315SJens Axboe if (!arg || nr_args != 1)
500c4320315SJens Axboe break;
501c4320315SJens Axboe ret = io_register_pbuf_ring(ctx, arg);
502c4320315SJens Axboe break;
503c4320315SJens Axboe case IORING_UNREGISTER_PBUF_RING:
504c4320315SJens Axboe ret = -EINVAL;
505c4320315SJens Axboe if (!arg || nr_args != 1)
506c4320315SJens Axboe break;
507c4320315SJens Axboe ret = io_unregister_pbuf_ring(ctx, arg);
508c4320315SJens Axboe break;
509c4320315SJens Axboe case IORING_REGISTER_SYNC_CANCEL:
510c4320315SJens Axboe ret = -EINVAL;
511c4320315SJens Axboe if (!arg || nr_args != 1)
512c4320315SJens Axboe break;
513c4320315SJens Axboe ret = io_sync_cancel(ctx, arg);
514c4320315SJens Axboe break;
515c4320315SJens Axboe case IORING_REGISTER_FILE_ALLOC_RANGE:
516c4320315SJens Axboe ret = -EINVAL;
517c4320315SJens Axboe if (!arg || nr_args)
518c4320315SJens Axboe break;
519c4320315SJens Axboe ret = io_register_file_alloc_range(ctx, arg);
520c4320315SJens Axboe break;
521d293b1a8SJens Axboe case IORING_REGISTER_PBUF_STATUS:
522d293b1a8SJens Axboe ret = -EINVAL;
523d293b1a8SJens Axboe if (!arg || nr_args != 1)
524d293b1a8SJens Axboe break;
525d293b1a8SJens Axboe ret = io_register_pbuf_status(ctx, arg);
526d293b1a8SJens Axboe break;
527ef1186c1SStefan Roesch case IORING_REGISTER_NAPI:
528ef1186c1SStefan Roesch ret = -EINVAL;
529ef1186c1SStefan Roesch if (!arg || nr_args != 1)
530ef1186c1SStefan Roesch break;
531ef1186c1SStefan Roesch ret = io_register_napi(ctx, arg);
532ef1186c1SStefan Roesch break;
533ef1186c1SStefan Roesch case IORING_UNREGISTER_NAPI:
534ef1186c1SStefan Roesch ret = -EINVAL;
535ef1186c1SStefan Roesch if (nr_args != 1)
536ef1186c1SStefan Roesch break;
537ef1186c1SStefan Roesch ret = io_unregister_napi(ctx, arg);
538ef1186c1SStefan Roesch break;
5392b8e976bSPavel Begunkov case IORING_REGISTER_CLOCK:
5402b8e976bSPavel Begunkov ret = -EINVAL;
5412b8e976bSPavel Begunkov if (!arg || nr_args)
5422b8e976bSPavel Begunkov break;
5432b8e976bSPavel Begunkov ret = io_register_clock(ctx, arg);
5442b8e976bSPavel Begunkov break;
545*7cc2a6eaSJens Axboe case IORING_REGISTER_COPY_BUFFERS:
546*7cc2a6eaSJens Axboe ret = -EINVAL;
547*7cc2a6eaSJens Axboe if (!arg || nr_args != 1)
548*7cc2a6eaSJens Axboe break;
549*7cc2a6eaSJens Axboe ret = io_register_copy_buffers(ctx, arg);
550*7cc2a6eaSJens Axboe break;
551c4320315SJens Axboe default:
552c4320315SJens Axboe ret = -EINVAL;
553c4320315SJens Axboe break;
554c4320315SJens Axboe }
555c4320315SJens Axboe
556c4320315SJens Axboe return ret;
557c4320315SJens Axboe }
558c4320315SJens Axboe
5590b6d253eSJens Axboe /*
5600b6d253eSJens Axboe * Given an 'fd' value, return the ctx associated with if. If 'registered' is
5610b6d253eSJens Axboe * true, then the registered index is used. Otherwise, the normal fd table.
5620b6d253eSJens Axboe * Caller must call fput() on the returned file, unless it's an ERR_PTR.
5630b6d253eSJens Axboe */
io_uring_register_get_file(int fd,bool registered)5640b6d253eSJens Axboe struct file *io_uring_register_get_file(int fd, bool registered)
5650b6d253eSJens Axboe {
5660b6d253eSJens Axboe struct file *file;
5670b6d253eSJens Axboe
5680b6d253eSJens Axboe if (registered) {
5690b6d253eSJens Axboe /*
5700b6d253eSJens Axboe * Ring fd has been registered via IORING_REGISTER_RING_FDS, we
5710b6d253eSJens Axboe * need only dereference our task private array to find it.
5720b6d253eSJens Axboe */
5730b6d253eSJens Axboe struct io_uring_task *tctx = current->io_uring;
5740b6d253eSJens Axboe
5750b6d253eSJens Axboe if (unlikely(!tctx || fd >= IO_RINGFD_REG_MAX))
5760b6d253eSJens Axboe return ERR_PTR(-EINVAL);
5770b6d253eSJens Axboe fd = array_index_nospec(fd, IO_RINGFD_REG_MAX);
5780b6d253eSJens Axboe file = tctx->registered_rings[fd];
5790b6d253eSJens Axboe } else {
5800b6d253eSJens Axboe file = fget(fd);
5810b6d253eSJens Axboe }
5820b6d253eSJens Axboe
5830b6d253eSJens Axboe if (unlikely(!file))
5840b6d253eSJens Axboe return ERR_PTR(-EBADF);
5850b6d253eSJens Axboe if (io_is_uring_fops(file))
5860b6d253eSJens Axboe return file;
5870b6d253eSJens Axboe fput(file);
5880b6d253eSJens Axboe return ERR_PTR(-EOPNOTSUPP);
5890b6d253eSJens Axboe }
5900b6d253eSJens Axboe
SYSCALL_DEFINE4(io_uring_register,unsigned int,fd,unsigned int,opcode,void __user *,arg,unsigned int,nr_args)591c4320315SJens Axboe SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode,
592c4320315SJens Axboe void __user *, arg, unsigned int, nr_args)
593c4320315SJens Axboe {
594c4320315SJens Axboe struct io_ring_ctx *ctx;
595c4320315SJens Axboe long ret = -EBADF;
596c4320315SJens Axboe struct file *file;
597c4320315SJens Axboe bool use_registered_ring;
598c4320315SJens Axboe
599c4320315SJens Axboe use_registered_ring = !!(opcode & IORING_REGISTER_USE_REGISTERED_RING);
600c4320315SJens Axboe opcode &= ~IORING_REGISTER_USE_REGISTERED_RING;
601c4320315SJens Axboe
602c4320315SJens Axboe if (opcode >= IORING_REGISTER_LAST)
603c4320315SJens Axboe return -EINVAL;
604c4320315SJens Axboe
6050b6d253eSJens Axboe file = io_uring_register_get_file(fd, use_registered_ring);
6060b6d253eSJens Axboe if (IS_ERR(file))
6070b6d253eSJens Axboe return PTR_ERR(file);
608c4320315SJens Axboe ctx = file->private_data;
609c4320315SJens Axboe
610c4320315SJens Axboe mutex_lock(&ctx->uring_lock);
611c4320315SJens Axboe ret = __io_uring_register(ctx, opcode, arg, nr_args);
612c4320315SJens Axboe mutex_unlock(&ctx->uring_lock);
613c4320315SJens Axboe trace_io_uring_register(ctx, opcode, ctx->nr_user_files, ctx->nr_user_bufs, ret);
614c4320315SJens Axboe if (!use_registered_ring)
615c4320315SJens Axboe fput(file);
616c4320315SJens Axboe return ret;
617c4320315SJens Axboe }
618