1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/fs.h> 5 #include <linux/file.h> 6 #include <linux/fdtable.h> 7 #include <linux/fsnotify.h> 8 #include <linux/namei.h> 9 #include <linux/pipe_fs_i.h> 10 #include <linux/watch_queue.h> 11 #include <linux/io_uring.h> 12 13 #include <uapi/linux/io_uring.h> 14 15 #include "../fs/internal.h" 16 17 #include "filetable.h" 18 #include "io_uring.h" 19 #include "rsrc.h" 20 #include "openclose.h" 21 22 struct io_open { 23 struct file *file; 24 int dfd; 25 u32 file_slot; 26 struct delayed_filename filename; 27 struct open_how how; 28 unsigned long nofile; 29 }; 30 31 struct io_close { 32 struct file *file; 33 int fd; 34 u32 file_slot; 35 }; 36 37 struct io_fixed_install { 38 struct file *file; 39 unsigned int o_flags; 40 }; 41 42 static bool io_openat_force_async(struct io_open *open) 43 { 44 /* 45 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open, 46 * it'll always -EAGAIN. Note that we test for __O_TMPFILE because 47 * O_TMPFILE includes O_DIRECTORY, which isn't a flag we need to force 48 * async for. 49 */ 50 return open->how.flags & (O_TRUNC | O_CREAT | __O_TMPFILE); 51 } 52 53 static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 54 { 55 struct io_open *open = io_kiocb_to_cmd(req, struct io_open); 56 const char __user *fname; 57 int ret; 58 59 if (unlikely(sqe->buf_index)) 60 return -EINVAL; 61 if (unlikely(req->flags & REQ_F_FIXED_FILE)) 62 return -EBADF; 63 64 /* open.how should be already initialised */ 65 if (!(open->how.flags & O_PATH) && force_o_largefile()) 66 open->how.flags |= O_LARGEFILE; 67 68 open->dfd = READ_ONCE(sqe->fd); 69 fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); 70 ret = delayed_getname(&open->filename, fname); 71 if (unlikely(ret)) 72 return ret; 73 req->flags |= REQ_F_NEED_CLEANUP; 74 75 open->file_slot = READ_ONCE(sqe->file_index); 76 if (open->file_slot && (open->how.flags & O_CLOEXEC)) 77 return -EINVAL; 78 79 open->nofile = rlimit(RLIMIT_NOFILE); 80 if (io_openat_force_async(open)) 81 req->flags |= REQ_F_FORCE_ASYNC; 82 return 0; 83 } 84 85 void io_openat_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req) 86 { 87 struct io_open *open = io_kiocb_to_cmd(req, struct io_open); 88 89 bctx->open.flags = open->how.flags; 90 bctx->open.mode = open->how.mode; 91 bctx->open.resolve = open->how.resolve; 92 } 93 94 int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 95 { 96 struct io_open *open = io_kiocb_to_cmd(req, struct io_open); 97 u64 mode = READ_ONCE(sqe->len); 98 u64 flags = READ_ONCE(sqe->open_flags); 99 100 open->how = build_open_how(flags, mode); 101 return __io_openat_prep(req, sqe); 102 } 103 104 int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 105 { 106 struct io_open *open = io_kiocb_to_cmd(req, struct io_open); 107 struct open_how __user *how; 108 size_t len; 109 int ret; 110 111 how = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 112 len = READ_ONCE(sqe->len); 113 if (len < OPEN_HOW_SIZE_VER0) 114 return -EINVAL; 115 116 ret = copy_struct_from_user(&open->how, sizeof(open->how), how, len); 117 if (ret) 118 return ret; 119 120 return __io_openat_prep(req, sqe); 121 } 122 123 int io_openat2(struct io_kiocb *req, unsigned int issue_flags) 124 { 125 struct io_open *open = io_kiocb_to_cmd(req, struct io_open); 126 struct open_flags op; 127 struct file *file; 128 bool resolve_nonblock, nonblock_set; 129 bool fixed = !!open->file_slot; 130 CLASS(filename_complete_delayed, name)(&open->filename); 131 int ret; 132 133 ret = build_open_flags(&open->how, &op); 134 if (ret) 135 goto err; 136 nonblock_set = op.open_flag & O_NONBLOCK; 137 resolve_nonblock = open->how.resolve & RESOLVE_CACHED; 138 if (issue_flags & IO_URING_F_NONBLOCK) { 139 WARN_ON_ONCE(io_openat_force_async(open)); 140 op.lookup_flags |= LOOKUP_CACHED; 141 op.open_flag |= O_NONBLOCK; 142 } 143 144 if (!fixed) { 145 ret = __get_unused_fd_flags(open->how.flags, open->nofile); 146 if (ret < 0) 147 goto err; 148 } 149 150 file = do_file_open(open->dfd, name, &op); 151 if (IS_ERR(file)) { 152 /* 153 * We could hang on to this 'fd' on retrying, but seems like 154 * marginal gain for something that is now known to be a slower 155 * path. So just put it, and we'll get a new one when we retry. 156 */ 157 if (!fixed) 158 put_unused_fd(ret); 159 160 ret = PTR_ERR(file); 161 /* only retry if RESOLVE_CACHED wasn't already set by application */ 162 if (ret == -EAGAIN && !resolve_nonblock && 163 (issue_flags & IO_URING_F_NONBLOCK)) { 164 ret = putname_to_delayed(&open->filename, 165 no_free_ptr(name)); 166 if (likely(!ret)) 167 return -EAGAIN; 168 } 169 goto err; 170 } 171 172 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set) 173 file->f_flags &= ~O_NONBLOCK; 174 175 if (!fixed) 176 fd_install(ret, file); 177 else 178 ret = io_fixed_fd_install(req, issue_flags, file, 179 open->file_slot); 180 err: 181 req->flags &= ~REQ_F_NEED_CLEANUP; 182 if (ret < 0) 183 req_set_fail(req); 184 io_req_set_res(req, ret, 0); 185 return IOU_COMPLETE; 186 } 187 188 int io_openat(struct io_kiocb *req, unsigned int issue_flags) 189 { 190 return io_openat2(req, issue_flags); 191 } 192 193 void io_open_cleanup(struct io_kiocb *req) 194 { 195 struct io_open *open = io_kiocb_to_cmd(req, struct io_open); 196 197 dismiss_delayed_filename(&open->filename); 198 } 199 200 int __io_close_fixed(struct io_ring_ctx *ctx, unsigned int issue_flags, 201 unsigned int offset) 202 { 203 int ret; 204 205 io_ring_submit_lock(ctx, issue_flags); 206 ret = io_fixed_fd_remove(ctx, offset); 207 io_ring_submit_unlock(ctx, issue_flags); 208 209 return ret; 210 } 211 212 static inline int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) 213 { 214 struct io_close *close = io_kiocb_to_cmd(req, struct io_close); 215 216 return __io_close_fixed(req->ctx, issue_flags, close->file_slot - 1); 217 } 218 219 int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 220 { 221 struct io_close *close = io_kiocb_to_cmd(req, struct io_close); 222 223 if (sqe->off || sqe->addr || sqe->len || sqe->rw_flags || sqe->buf_index) 224 return -EINVAL; 225 if (req->flags & REQ_F_FIXED_FILE) 226 return -EBADF; 227 228 close->fd = READ_ONCE(sqe->fd); 229 close->file_slot = READ_ONCE(sqe->file_index); 230 if (close->file_slot && close->fd) 231 return -EINVAL; 232 233 return 0; 234 } 235 236 int io_close(struct io_kiocb *req, unsigned int issue_flags) 237 { 238 struct files_struct *files = current->files; 239 struct io_close *close = io_kiocb_to_cmd(req, struct io_close); 240 struct file *file; 241 int ret = -EBADF; 242 243 if (close->file_slot) { 244 ret = io_close_fixed(req, issue_flags); 245 goto err; 246 } 247 248 spin_lock(&files->file_lock); 249 file = files_lookup_fd_locked(files, close->fd); 250 if (!file || io_is_uring_fops(file)) { 251 spin_unlock(&files->file_lock); 252 goto err; 253 } 254 255 /* if the file has a flush method, be safe and punt to async */ 256 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) { 257 spin_unlock(&files->file_lock); 258 return -EAGAIN; 259 } 260 261 file = file_close_fd_locked(files, close->fd); 262 spin_unlock(&files->file_lock); 263 if (!file) 264 goto err; 265 266 /* No ->flush() or already async, safely close from here */ 267 ret = filp_close(file, current->files); 268 err: 269 if (ret < 0) 270 req_set_fail(req); 271 io_req_set_res(req, ret, 0); 272 return IOU_COMPLETE; 273 } 274 275 int io_install_fixed_fd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 276 { 277 struct io_fixed_install *ifi; 278 unsigned int flags; 279 280 if (sqe->off || sqe->addr || sqe->len || sqe->buf_index || 281 sqe->splice_fd_in || sqe->addr3) 282 return -EINVAL; 283 284 /* must be a fixed file */ 285 if (!(req->flags & REQ_F_FIXED_FILE)) 286 return -EBADF; 287 288 flags = READ_ONCE(sqe->install_fd_flags); 289 if (flags & ~IORING_FIXED_FD_NO_CLOEXEC) 290 return -EINVAL; 291 292 /* ensure the task's creds are used when installing/receiving fds */ 293 if (req->flags & REQ_F_CREDS) 294 return -EPERM; 295 296 /* default to O_CLOEXEC, disable if IORING_FIXED_FD_NO_CLOEXEC is set */ 297 ifi = io_kiocb_to_cmd(req, struct io_fixed_install); 298 ifi->o_flags = O_CLOEXEC; 299 if (flags & IORING_FIXED_FD_NO_CLOEXEC) 300 ifi->o_flags = 0; 301 302 return 0; 303 } 304 305 int io_install_fixed_fd(struct io_kiocb *req, unsigned int issue_flags) 306 { 307 struct io_fixed_install *ifi; 308 int ret; 309 310 ifi = io_kiocb_to_cmd(req, struct io_fixed_install); 311 ret = receive_fd(req->file, NULL, ifi->o_flags); 312 if (ret < 0) 313 req_set_fail(req); 314 io_req_set_res(req, ret, 0); 315 return IOU_COMPLETE; 316 } 317 318 struct io_pipe { 319 struct file *file; 320 int __user *fds; 321 int flags; 322 int file_slot; 323 unsigned long nofile; 324 }; 325 326 int io_pipe_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 327 { 328 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); 329 330 if (sqe->fd || sqe->off || sqe->addr3) 331 return -EINVAL; 332 333 p->fds = u64_to_user_ptr(READ_ONCE(sqe->addr)); 334 p->flags = READ_ONCE(sqe->pipe_flags); 335 if (p->flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE)) 336 return -EINVAL; 337 338 p->file_slot = READ_ONCE(sqe->file_index); 339 p->nofile = rlimit(RLIMIT_NOFILE); 340 return 0; 341 } 342 343 static int io_pipe_fixed(struct io_kiocb *req, struct file **files, 344 unsigned int issue_flags) 345 { 346 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); 347 struct io_ring_ctx *ctx = req->ctx; 348 int ret, fds[2] = { -1, -1 }; 349 int slot = p->file_slot; 350 351 if (p->flags & O_CLOEXEC) 352 return -EINVAL; 353 354 io_ring_submit_lock(ctx, issue_flags); 355 356 ret = __io_fixed_fd_install(ctx, files[0], slot); 357 if (ret < 0) 358 goto err; 359 fds[0] = ret; 360 files[0] = NULL; 361 362 /* 363 * If a specific slot is given, next one will be used for 364 * the write side. 365 */ 366 if (slot != IORING_FILE_INDEX_ALLOC) 367 slot++; 368 369 ret = __io_fixed_fd_install(ctx, files[1], slot); 370 if (ret < 0) 371 goto err; 372 fds[1] = ret; 373 files[1] = NULL; 374 375 io_ring_submit_unlock(ctx, issue_flags); 376 377 if (!copy_to_user(p->fds, fds, sizeof(fds))) 378 return 0; 379 380 ret = -EFAULT; 381 io_ring_submit_lock(ctx, issue_flags); 382 err: 383 if (fds[0] != -1) 384 io_fixed_fd_remove(ctx, fds[0]); 385 if (fds[1] != -1) 386 io_fixed_fd_remove(ctx, fds[1]); 387 io_ring_submit_unlock(ctx, issue_flags); 388 return ret; 389 } 390 391 static int io_pipe_fd(struct io_kiocb *req, struct file **files) 392 { 393 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); 394 int ret, fds[2] = { -1, -1 }; 395 396 ret = __get_unused_fd_flags(p->flags, p->nofile); 397 if (ret < 0) 398 goto err; 399 fds[0] = ret; 400 401 ret = __get_unused_fd_flags(p->flags, p->nofile); 402 if (ret < 0) 403 goto err; 404 fds[1] = ret; 405 406 if (!copy_to_user(p->fds, fds, sizeof(fds))) { 407 fd_install(fds[0], files[0]); 408 fd_install(fds[1], files[1]); 409 return 0; 410 } 411 ret = -EFAULT; 412 err: 413 if (fds[0] != -1) 414 put_unused_fd(fds[0]); 415 if (fds[1] != -1) 416 put_unused_fd(fds[1]); 417 return ret; 418 } 419 420 int io_pipe(struct io_kiocb *req, unsigned int issue_flags) 421 { 422 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); 423 struct file *files[2]; 424 int ret; 425 426 ret = create_pipe_files(files, p->flags); 427 if (ret) 428 return ret; 429 430 if (!!p->file_slot) 431 ret = io_pipe_fixed(req, files, issue_flags); 432 else 433 ret = io_pipe_fd(req, files); 434 435 io_req_set_res(req, ret, 0); 436 if (!ret) 437 return IOU_COMPLETE; 438 439 req_set_fail(req); 440 if (files[0]) 441 fput(files[0]); 442 if (files[1]) 443 fput(files[1]); 444 return ret; 445 } 446