1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/fs.h>
5 #include <linux/file.h>
6 #include <linux/fdtable.h>
7 #include <linux/fsnotify.h>
8 #include <linux/namei.h>
9 #include <linux/pipe_fs_i.h>
10 #include <linux/watch_queue.h>
11 #include <linux/io_uring.h>
12
13 #include <uapi/linux/io_uring.h>
14
15 #include "../fs/internal.h"
16
17 #include "filetable.h"
18 #include "io_uring.h"
19 #include "rsrc.h"
20 #include "openclose.h"
21
22 struct io_open {
23 struct file *file;
24 int dfd;
25 u32 file_slot;
26 struct delayed_filename filename;
27 struct open_how how;
28 unsigned long nofile;
29 };
30
31 struct io_close {
32 struct file *file;
33 int fd;
34 u32 file_slot;
35 };
36
37 struct io_fixed_install {
38 struct file *file;
39 unsigned int o_flags;
40 };
41
io_openat_force_async(struct io_open * open)42 static bool io_openat_force_async(struct io_open *open)
43 {
44 /*
45 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open,
46 * it'll always -EAGAIN. Note that we test for __O_TMPFILE because
47 * O_TMPFILE includes O_DIRECTORY, which isn't a flag we need to force
48 * async for.
49 */
50 return open->how.flags & (O_TRUNC | O_CREAT | __O_TMPFILE);
51 }
52
__io_openat_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)53 static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
54 {
55 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
56 const char __user *fname;
57 int ret;
58
59 if (unlikely(sqe->buf_index))
60 return -EINVAL;
61 if (unlikely(req->flags & REQ_F_FIXED_FILE))
62 return -EBADF;
63
64 /* open.how should be already initialised */
65 if (!(open->how.flags & O_PATH) && force_o_largefile())
66 open->how.flags |= O_LARGEFILE;
67
68 open->dfd = READ_ONCE(sqe->fd);
69 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
70 ret = delayed_getname(&open->filename, fname);
71 if (unlikely(ret))
72 return ret;
73 req->flags |= REQ_F_NEED_CLEANUP;
74
75 open->file_slot = READ_ONCE(sqe->file_index);
76 if (open->file_slot && (open->how.flags & O_CLOEXEC))
77 return -EINVAL;
78
79 open->nofile = rlimit(RLIMIT_NOFILE);
80 if (io_openat_force_async(open))
81 req->flags |= REQ_F_FORCE_ASYNC;
82 return 0;
83 }
84
io_openat_bpf_populate(struct io_uring_bpf_ctx * bctx,struct io_kiocb * req)85 void io_openat_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req)
86 {
87 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
88
89 bctx->open.flags = open->how.flags;
90 bctx->open.mode = open->how.mode;
91 bctx->open.resolve = open->how.resolve;
92 }
93
io_openat_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)94 int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
95 {
96 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
97 u64 mode = READ_ONCE(sqe->len);
98 u64 flags = READ_ONCE(sqe->open_flags);
99
100 open->how = build_open_how(flags, mode);
101 return __io_openat_prep(req, sqe);
102 }
103
io_openat2_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)104 int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
105 {
106 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
107 struct open_how __user *how;
108 size_t len;
109 int ret;
110
111 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
112 len = READ_ONCE(sqe->len);
113 if (len < OPEN_HOW_SIZE_VER0)
114 return -EINVAL;
115
116 ret = copy_struct_from_user(&open->how, sizeof(open->how), how, len);
117 if (ret)
118 return ret;
119
120 return __io_openat_prep(req, sqe);
121 }
122
io_openat2(struct io_kiocb * req,unsigned int issue_flags)123 int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
124 {
125 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
126 struct open_flags op;
127 struct file *file;
128 bool resolve_nonblock, nonblock_set;
129 bool fixed = !!open->file_slot;
130 CLASS(filename_complete_delayed, name)(&open->filename);
131 int ret;
132
133 ret = build_open_flags(&open->how, &op);
134 if (ret)
135 goto err;
136 nonblock_set = op.open_flag & O_NONBLOCK;
137 resolve_nonblock = open->how.resolve & RESOLVE_CACHED;
138 if (issue_flags & IO_URING_F_NONBLOCK) {
139 WARN_ON_ONCE(io_openat_force_async(open));
140 op.lookup_flags |= LOOKUP_CACHED;
141 op.open_flag |= O_NONBLOCK;
142 }
143
144 if (!fixed) {
145 ret = __get_unused_fd_flags(open->how.flags, open->nofile);
146 if (ret < 0)
147 goto err;
148 }
149
150 file = do_file_open(open->dfd, name, &op);
151 if (IS_ERR(file)) {
152 /*
153 * We could hang on to this 'fd' on retrying, but seems like
154 * marginal gain for something that is now known to be a slower
155 * path. So just put it, and we'll get a new one when we retry.
156 */
157 if (!fixed)
158 put_unused_fd(ret);
159
160 ret = PTR_ERR(file);
161 /* only retry if RESOLVE_CACHED wasn't already set by application */
162 if (ret == -EAGAIN && !resolve_nonblock &&
163 (issue_flags & IO_URING_F_NONBLOCK)) {
164 ret = putname_to_delayed(&open->filename,
165 no_free_ptr(name));
166 if (likely(!ret))
167 return -EAGAIN;
168 }
169 goto err;
170 }
171
172 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)
173 file->f_flags &= ~O_NONBLOCK;
174
175 if (!fixed)
176 fd_install(ret, file);
177 else
178 ret = io_fixed_fd_install(req, issue_flags, file,
179 open->file_slot);
180 err:
181 req->flags &= ~REQ_F_NEED_CLEANUP;
182 if (ret < 0)
183 req_set_fail(req);
184 io_req_set_res(req, ret, 0);
185 return IOU_COMPLETE;
186 }
187
io_openat(struct io_kiocb * req,unsigned int issue_flags)188 int io_openat(struct io_kiocb *req, unsigned int issue_flags)
189 {
190 return io_openat2(req, issue_flags);
191 }
192
io_open_cleanup(struct io_kiocb * req)193 void io_open_cleanup(struct io_kiocb *req)
194 {
195 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
196
197 dismiss_delayed_filename(&open->filename);
198 }
199
__io_close_fixed(struct io_ring_ctx * ctx,unsigned int issue_flags,unsigned int offset)200 int __io_close_fixed(struct io_ring_ctx *ctx, unsigned int issue_flags,
201 unsigned int offset)
202 {
203 int ret;
204
205 io_ring_submit_lock(ctx, issue_flags);
206 ret = io_fixed_fd_remove(ctx, offset);
207 io_ring_submit_unlock(ctx, issue_flags);
208
209 return ret;
210 }
211
io_close_fixed(struct io_kiocb * req,unsigned int issue_flags)212 static inline int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags)
213 {
214 struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
215
216 return __io_close_fixed(req->ctx, issue_flags, close->file_slot - 1);
217 }
218
io_close_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)219 int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
220 {
221 struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
222
223 if (sqe->off || sqe->addr || sqe->len || sqe->rw_flags || sqe->buf_index)
224 return -EINVAL;
225 if (req->flags & REQ_F_FIXED_FILE)
226 return -EBADF;
227
228 close->fd = READ_ONCE(sqe->fd);
229 close->file_slot = READ_ONCE(sqe->file_index);
230 if (close->file_slot && close->fd)
231 return -EINVAL;
232
233 return 0;
234 }
235
io_close(struct io_kiocb * req,unsigned int issue_flags)236 int io_close(struct io_kiocb *req, unsigned int issue_flags)
237 {
238 struct files_struct *files = current->files;
239 struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
240 struct file *file;
241 int ret = -EBADF;
242
243 if (close->file_slot) {
244 ret = io_close_fixed(req, issue_flags);
245 goto err;
246 }
247
248 spin_lock(&files->file_lock);
249 file = files_lookup_fd_locked(files, close->fd);
250 if (!file || io_is_uring_fops(file)) {
251 spin_unlock(&files->file_lock);
252 goto err;
253 }
254
255 /* if the file has a flush method, be safe and punt to async */
256 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
257 spin_unlock(&files->file_lock);
258 return -EAGAIN;
259 }
260
261 file = file_close_fd_locked(files, close->fd);
262 spin_unlock(&files->file_lock);
263 if (!file)
264 goto err;
265
266 /* No ->flush() or already async, safely close from here */
267 ret = filp_close(file, current->files);
268 err:
269 if (ret < 0)
270 req_set_fail(req);
271 io_req_set_res(req, ret, 0);
272 return IOU_COMPLETE;
273 }
274
io_install_fixed_fd_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)275 int io_install_fixed_fd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
276 {
277 struct io_fixed_install *ifi;
278 unsigned int flags;
279
280 if (sqe->off || sqe->addr || sqe->len || sqe->buf_index ||
281 sqe->splice_fd_in || sqe->addr3)
282 return -EINVAL;
283
284 /* must be a fixed file */
285 if (!(req->flags & REQ_F_FIXED_FILE))
286 return -EBADF;
287
288 flags = READ_ONCE(sqe->install_fd_flags);
289 if (flags & ~IORING_FIXED_FD_NO_CLOEXEC)
290 return -EINVAL;
291
292 /* ensure the task's creds are used when installing/receiving fds */
293 if (req->flags & REQ_F_CREDS)
294 return -EPERM;
295
296 /* default to O_CLOEXEC, disable if IORING_FIXED_FD_NO_CLOEXEC is set */
297 ifi = io_kiocb_to_cmd(req, struct io_fixed_install);
298 ifi->o_flags = O_CLOEXEC;
299 if (flags & IORING_FIXED_FD_NO_CLOEXEC)
300 ifi->o_flags = 0;
301
302 return 0;
303 }
304
io_install_fixed_fd(struct io_kiocb * req,unsigned int issue_flags)305 int io_install_fixed_fd(struct io_kiocb *req, unsigned int issue_flags)
306 {
307 struct io_fixed_install *ifi;
308 int ret;
309
310 ifi = io_kiocb_to_cmd(req, struct io_fixed_install);
311 ret = receive_fd(req->file, NULL, ifi->o_flags);
312 if (ret < 0)
313 req_set_fail(req);
314 io_req_set_res(req, ret, 0);
315 return IOU_COMPLETE;
316 }
317
318 struct io_pipe {
319 struct file *file;
320 int __user *fds;
321 int flags;
322 int file_slot;
323 unsigned long nofile;
324 };
325
io_pipe_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)326 int io_pipe_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
327 {
328 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
329
330 if (sqe->fd || sqe->off || sqe->addr3)
331 return -EINVAL;
332
333 p->fds = u64_to_user_ptr(READ_ONCE(sqe->addr));
334 p->flags = READ_ONCE(sqe->pipe_flags);
335 if (p->flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE))
336 return -EINVAL;
337
338 p->file_slot = READ_ONCE(sqe->file_index);
339 p->nofile = rlimit(RLIMIT_NOFILE);
340 return 0;
341 }
342
io_pipe_fixed(struct io_kiocb * req,struct file ** files,unsigned int issue_flags)343 static int io_pipe_fixed(struct io_kiocb *req, struct file **files,
344 unsigned int issue_flags)
345 {
346 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
347 struct io_ring_ctx *ctx = req->ctx;
348 bool alloc_slot;
349 int ret, fds[2] = { -1, -1 };
350 int slot = p->file_slot;
351
352 if (p->flags & O_CLOEXEC)
353 return -EINVAL;
354
355 alloc_slot = slot == IORING_FILE_INDEX_ALLOC;
356
357 io_ring_submit_lock(ctx, issue_flags);
358
359 ret = __io_fixed_fd_install(ctx, files[0], slot);
360 if (ret < 0)
361 goto err;
362 fds[0] = alloc_slot ? ret : slot - 1;
363 files[0] = NULL;
364
365 /*
366 * If a specific slot is given, next one will be used for
367 * the write side.
368 */
369 if (!alloc_slot)
370 slot++;
371
372 ret = __io_fixed_fd_install(ctx, files[1], slot);
373 if (ret < 0)
374 goto err;
375 fds[1] = alloc_slot ? ret : slot - 1;
376 files[1] = NULL;
377
378 io_ring_submit_unlock(ctx, issue_flags);
379
380 if (!copy_to_user(p->fds, fds, sizeof(fds)))
381 return 0;
382
383 ret = -EFAULT;
384 io_ring_submit_lock(ctx, issue_flags);
385 err:
386 if (fds[0] != -1)
387 io_fixed_fd_remove(ctx, fds[0]);
388 if (fds[1] != -1)
389 io_fixed_fd_remove(ctx, fds[1]);
390 io_ring_submit_unlock(ctx, issue_flags);
391 return ret;
392 }
393
io_pipe_fd(struct io_kiocb * req,struct file ** files)394 static int io_pipe_fd(struct io_kiocb *req, struct file **files)
395 {
396 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
397 int ret, fds[2] = { -1, -1 };
398
399 ret = __get_unused_fd_flags(p->flags, p->nofile);
400 if (ret < 0)
401 goto err;
402 fds[0] = ret;
403
404 ret = __get_unused_fd_flags(p->flags, p->nofile);
405 if (ret < 0)
406 goto err;
407 fds[1] = ret;
408
409 if (!copy_to_user(p->fds, fds, sizeof(fds))) {
410 fd_install(fds[0], files[0]);
411 fd_install(fds[1], files[1]);
412 return 0;
413 }
414 ret = -EFAULT;
415 err:
416 if (fds[0] != -1)
417 put_unused_fd(fds[0]);
418 if (fds[1] != -1)
419 put_unused_fd(fds[1]);
420 return ret;
421 }
422
io_pipe(struct io_kiocb * req,unsigned int issue_flags)423 int io_pipe(struct io_kiocb *req, unsigned int issue_flags)
424 {
425 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
426 struct file *files[2];
427 int ret;
428
429 ret = create_pipe_files(files, p->flags);
430 if (ret)
431 return ret;
432
433 if (!!p->file_slot)
434 ret = io_pipe_fixed(req, files, issue_flags);
435 else
436 ret = io_pipe_fd(req, files);
437
438 io_req_set_res(req, ret, 0);
439 if (!ret)
440 return IOU_COMPLETE;
441
442 req_set_fail(req);
443 if (files[0])
444 fput(files[0]);
445 if (files[1])
446 fput(files[1]);
447 return ret;
448 }
449