1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/kernel.h>
3 #include <linux/errno.h>
4 #include <linux/fs.h>
5 #include <linux/file.h>
6 #include <linux/fdtable.h>
7 #include <linux/fsnotify.h>
8 #include <linux/namei.h>
9 #include <linux/pipe_fs_i.h>
10 #include <linux/watch_queue.h>
11 #include <linux/io_uring.h>
12
13 #include <uapi/linux/io_uring.h>
14
15 #include "../fs/internal.h"
16
17 #include "io_uring.h"
18 #include "rsrc.h"
19 #include "openclose.h"
20
21 struct io_open {
22 struct file *file;
23 int dfd;
24 u32 file_slot;
25 struct filename *filename;
26 struct open_how how;
27 unsigned long nofile;
28 };
29
30 struct io_close {
31 struct file *file;
32 int fd;
33 u32 file_slot;
34 };
35
36 struct io_fixed_install {
37 struct file *file;
38 unsigned int o_flags;
39 };
40
io_openat_force_async(struct io_open * open)41 static bool io_openat_force_async(struct io_open *open)
42 {
43 /*
44 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open,
45 * it'll always -EAGAIN. Note that we test for __O_TMPFILE because
46 * O_TMPFILE includes O_DIRECTORY, which isn't a flag we need to force
47 * async for.
48 */
49 return open->how.flags & (O_TRUNC | O_CREAT | __O_TMPFILE);
50 }
51
__io_openat_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)52 static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
53 {
54 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
55 const char __user *fname;
56 int ret;
57
58 if (unlikely(sqe->buf_index))
59 return -EINVAL;
60 if (unlikely(req->flags & REQ_F_FIXED_FILE))
61 return -EBADF;
62
63 /* open.how should be already initialised */
64 if (!(open->how.flags & O_PATH) && force_o_largefile())
65 open->how.flags |= O_LARGEFILE;
66
67 open->dfd = READ_ONCE(sqe->fd);
68 fname = u64_to_user_ptr(READ_ONCE(sqe->addr));
69 open->filename = getname(fname);
70 if (IS_ERR(open->filename)) {
71 ret = PTR_ERR(open->filename);
72 open->filename = NULL;
73 return ret;
74 }
75
76 open->file_slot = READ_ONCE(sqe->file_index);
77 if (open->file_slot && (open->how.flags & O_CLOEXEC))
78 return -EINVAL;
79
80 open->nofile = rlimit(RLIMIT_NOFILE);
81 req->flags |= REQ_F_NEED_CLEANUP;
82 if (io_openat_force_async(open))
83 req->flags |= REQ_F_FORCE_ASYNC;
84 return 0;
85 }
86
io_openat_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)87 int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
88 {
89 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
90 u64 mode = READ_ONCE(sqe->len);
91 u64 flags = READ_ONCE(sqe->open_flags);
92
93 open->how = build_open_how(flags, mode);
94 return __io_openat_prep(req, sqe);
95 }
96
io_openat2_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)97 int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
98 {
99 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
100 struct open_how __user *how;
101 size_t len;
102 int ret;
103
104 how = u64_to_user_ptr(READ_ONCE(sqe->addr2));
105 len = READ_ONCE(sqe->len);
106 if (len < OPEN_HOW_SIZE_VER0)
107 return -EINVAL;
108
109 ret = copy_struct_from_user(&open->how, sizeof(open->how), how, len);
110 if (ret)
111 return ret;
112
113 return __io_openat_prep(req, sqe);
114 }
115
io_openat2(struct io_kiocb * req,unsigned int issue_flags)116 int io_openat2(struct io_kiocb *req, unsigned int issue_flags)
117 {
118 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
119 struct open_flags op;
120 struct file *file;
121 bool resolve_nonblock, nonblock_set;
122 bool fixed = !!open->file_slot;
123 int ret;
124
125 ret = build_open_flags(&open->how, &op);
126 if (ret)
127 goto err;
128 nonblock_set = op.open_flag & O_NONBLOCK;
129 resolve_nonblock = open->how.resolve & RESOLVE_CACHED;
130 if (issue_flags & IO_URING_F_NONBLOCK) {
131 WARN_ON_ONCE(io_openat_force_async(open));
132 op.lookup_flags |= LOOKUP_CACHED;
133 op.open_flag |= O_NONBLOCK;
134 }
135
136 if (!fixed) {
137 ret = __get_unused_fd_flags(open->how.flags, open->nofile);
138 if (ret < 0)
139 goto err;
140 }
141
142 file = do_filp_open(open->dfd, open->filename, &op);
143 if (IS_ERR(file)) {
144 /*
145 * We could hang on to this 'fd' on retrying, but seems like
146 * marginal gain for something that is now known to be a slower
147 * path. So just put it, and we'll get a new one when we retry.
148 */
149 if (!fixed)
150 put_unused_fd(ret);
151
152 ret = PTR_ERR(file);
153 /* only retry if RESOLVE_CACHED wasn't already set by application */
154 if (ret == -EAGAIN &&
155 (!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)))
156 return -EAGAIN;
157 goto err;
158 }
159
160 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set)
161 file->f_flags &= ~O_NONBLOCK;
162
163 if (!fixed)
164 fd_install(ret, file);
165 else
166 ret = io_fixed_fd_install(req, issue_flags, file,
167 open->file_slot);
168 err:
169 putname(open->filename);
170 req->flags &= ~REQ_F_NEED_CLEANUP;
171 if (ret < 0)
172 req_set_fail(req);
173 io_req_set_res(req, ret, 0);
174 return IOU_COMPLETE;
175 }
176
io_openat(struct io_kiocb * req,unsigned int issue_flags)177 int io_openat(struct io_kiocb *req, unsigned int issue_flags)
178 {
179 return io_openat2(req, issue_flags);
180 }
181
io_open_cleanup(struct io_kiocb * req)182 void io_open_cleanup(struct io_kiocb *req)
183 {
184 struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
185
186 if (open->filename)
187 putname(open->filename);
188 }
189
__io_close_fixed(struct io_ring_ctx * ctx,unsigned int issue_flags,unsigned int offset)190 int __io_close_fixed(struct io_ring_ctx *ctx, unsigned int issue_flags,
191 unsigned int offset)
192 {
193 int ret;
194
195 io_ring_submit_lock(ctx, issue_flags);
196 ret = io_fixed_fd_remove(ctx, offset);
197 io_ring_submit_unlock(ctx, issue_flags);
198
199 return ret;
200 }
201
io_close_fixed(struct io_kiocb * req,unsigned int issue_flags)202 static inline int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags)
203 {
204 struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
205
206 return __io_close_fixed(req->ctx, issue_flags, close->file_slot - 1);
207 }
208
io_close_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)209 int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
210 {
211 struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
212
213 if (sqe->off || sqe->addr || sqe->len || sqe->rw_flags || sqe->buf_index)
214 return -EINVAL;
215 if (req->flags & REQ_F_FIXED_FILE)
216 return -EBADF;
217
218 close->fd = READ_ONCE(sqe->fd);
219 close->file_slot = READ_ONCE(sqe->file_index);
220 if (close->file_slot && close->fd)
221 return -EINVAL;
222
223 return 0;
224 }
225
io_close(struct io_kiocb * req,unsigned int issue_flags)226 int io_close(struct io_kiocb *req, unsigned int issue_flags)
227 {
228 struct files_struct *files = current->files;
229 struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
230 struct file *file;
231 int ret = -EBADF;
232
233 if (close->file_slot) {
234 ret = io_close_fixed(req, issue_flags);
235 goto err;
236 }
237
238 spin_lock(&files->file_lock);
239 file = files_lookup_fd_locked(files, close->fd);
240 if (!file || io_is_uring_fops(file)) {
241 spin_unlock(&files->file_lock);
242 goto err;
243 }
244
245 /* if the file has a flush method, be safe and punt to async */
246 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) {
247 spin_unlock(&files->file_lock);
248 return -EAGAIN;
249 }
250
251 file = file_close_fd_locked(files, close->fd);
252 spin_unlock(&files->file_lock);
253 if (!file)
254 goto err;
255
256 /* No ->flush() or already async, safely close from here */
257 ret = filp_close(file, current->files);
258 err:
259 if (ret < 0)
260 req_set_fail(req);
261 io_req_set_res(req, ret, 0);
262 return IOU_COMPLETE;
263 }
264
io_install_fixed_fd_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)265 int io_install_fixed_fd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
266 {
267 struct io_fixed_install *ifi;
268 unsigned int flags;
269
270 if (sqe->off || sqe->addr || sqe->len || sqe->buf_index ||
271 sqe->splice_fd_in || sqe->addr3)
272 return -EINVAL;
273
274 /* must be a fixed file */
275 if (!(req->flags & REQ_F_FIXED_FILE))
276 return -EBADF;
277
278 flags = READ_ONCE(sqe->install_fd_flags);
279 if (flags & ~IORING_FIXED_FD_NO_CLOEXEC)
280 return -EINVAL;
281
282 /* ensure the task's creds are used when installing/receiving fds */
283 if (req->flags & REQ_F_CREDS)
284 return -EPERM;
285
286 /* default to O_CLOEXEC, disable if IORING_FIXED_FD_NO_CLOEXEC is set */
287 ifi = io_kiocb_to_cmd(req, struct io_fixed_install);
288 ifi->o_flags = O_CLOEXEC;
289 if (flags & IORING_FIXED_FD_NO_CLOEXEC)
290 ifi->o_flags = 0;
291
292 return 0;
293 }
294
io_install_fixed_fd(struct io_kiocb * req,unsigned int issue_flags)295 int io_install_fixed_fd(struct io_kiocb *req, unsigned int issue_flags)
296 {
297 struct io_fixed_install *ifi;
298 int ret;
299
300 ifi = io_kiocb_to_cmd(req, struct io_fixed_install);
301 ret = receive_fd(req->file, NULL, ifi->o_flags);
302 if (ret < 0)
303 req_set_fail(req);
304 io_req_set_res(req, ret, 0);
305 return IOU_COMPLETE;
306 }
307
308 struct io_pipe {
309 struct file *file;
310 int __user *fds;
311 int flags;
312 int file_slot;
313 unsigned long nofile;
314 };
315
io_pipe_prep(struct io_kiocb * req,const struct io_uring_sqe * sqe)316 int io_pipe_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
317 {
318 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
319
320 if (sqe->fd || sqe->off || sqe->addr3)
321 return -EINVAL;
322
323 p->fds = u64_to_user_ptr(READ_ONCE(sqe->addr));
324 p->flags = READ_ONCE(sqe->pipe_flags);
325 if (p->flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE))
326 return -EINVAL;
327
328 p->file_slot = READ_ONCE(sqe->file_index);
329 p->nofile = rlimit(RLIMIT_NOFILE);
330 return 0;
331 }
332
io_pipe_fixed(struct io_kiocb * req,struct file ** files,unsigned int issue_flags)333 static int io_pipe_fixed(struct io_kiocb *req, struct file **files,
334 unsigned int issue_flags)
335 {
336 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
337 struct io_ring_ctx *ctx = req->ctx;
338 int ret, fds[2] = { -1, -1 };
339 int slot = p->file_slot;
340
341 if (p->flags & O_CLOEXEC)
342 return -EINVAL;
343
344 io_ring_submit_lock(ctx, issue_flags);
345
346 ret = __io_fixed_fd_install(ctx, files[0], slot);
347 if (ret < 0)
348 goto err;
349 fds[0] = ret;
350 files[0] = NULL;
351
352 /*
353 * If a specific slot is given, next one will be used for
354 * the write side.
355 */
356 if (slot != IORING_FILE_INDEX_ALLOC)
357 slot++;
358
359 ret = __io_fixed_fd_install(ctx, files[1], slot);
360 if (ret < 0)
361 goto err;
362 fds[1] = ret;
363 files[1] = NULL;
364
365 io_ring_submit_unlock(ctx, issue_flags);
366
367 if (!copy_to_user(p->fds, fds, sizeof(fds)))
368 return 0;
369
370 ret = -EFAULT;
371 io_ring_submit_lock(ctx, issue_flags);
372 err:
373 if (fds[0] != -1)
374 io_fixed_fd_remove(ctx, fds[0]);
375 if (fds[1] != -1)
376 io_fixed_fd_remove(ctx, fds[1]);
377 io_ring_submit_unlock(ctx, issue_flags);
378 return ret;
379 }
380
io_pipe_fd(struct io_kiocb * req,struct file ** files)381 static int io_pipe_fd(struct io_kiocb *req, struct file **files)
382 {
383 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
384 int ret, fds[2] = { -1, -1 };
385
386 ret = __get_unused_fd_flags(p->flags, p->nofile);
387 if (ret < 0)
388 goto err;
389 fds[0] = ret;
390
391 ret = __get_unused_fd_flags(p->flags, p->nofile);
392 if (ret < 0)
393 goto err;
394 fds[1] = ret;
395
396 if (!copy_to_user(p->fds, fds, sizeof(fds))) {
397 fd_install(fds[0], files[0]);
398 fd_install(fds[1], files[1]);
399 return 0;
400 }
401 ret = -EFAULT;
402 err:
403 if (fds[0] != -1)
404 put_unused_fd(fds[0]);
405 if (fds[1] != -1)
406 put_unused_fd(fds[1]);
407 return ret;
408 }
409
io_pipe(struct io_kiocb * req,unsigned int issue_flags)410 int io_pipe(struct io_kiocb *req, unsigned int issue_flags)
411 {
412 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
413 struct file *files[2];
414 int ret;
415
416 ret = create_pipe_files(files, p->flags);
417 if (ret)
418 return ret;
419
420 if (!!p->file_slot)
421 ret = io_pipe_fixed(req, files, issue_flags);
422 else
423 ret = io_pipe_fd(req, files);
424
425 io_req_set_res(req, ret, 0);
426 if (!ret)
427 return IOU_COMPLETE;
428
429 req_set_fail(req);
430 if (files[0])
431 fput(files[0]);
432 if (files[1])
433 fput(files[1]);
434 return ret;
435 }
436