1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/kernel.h> 3 #include <linux/errno.h> 4 #include <linux/fs.h> 5 #include <linux/file.h> 6 #include <linux/fdtable.h> 7 #include <linux/fsnotify.h> 8 #include <linux/namei.h> 9 #include <linux/pipe_fs_i.h> 10 #include <linux/watch_queue.h> 11 #include <linux/io_uring.h> 12 13 #include <uapi/linux/io_uring.h> 14 15 #include "../fs/internal.h" 16 17 #include "io_uring.h" 18 #include "rsrc.h" 19 #include "openclose.h" 20 21 struct io_open { 22 struct file *file; 23 int dfd; 24 u32 file_slot; 25 struct filename *filename; 26 struct open_how how; 27 unsigned long nofile; 28 }; 29 30 struct io_close { 31 struct file *file; 32 int fd; 33 u32 file_slot; 34 }; 35 36 struct io_fixed_install { 37 struct file *file; 38 unsigned int o_flags; 39 }; 40 41 static bool io_openat_force_async(struct io_open *open) 42 { 43 /* 44 * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open, 45 * it'll always -EAGAIN. Note that we test for __O_TMPFILE because 46 * O_TMPFILE includes O_DIRECTORY, which isn't a flag we need to force 47 * async for. 48 */ 49 return open->how.flags & (O_TRUNC | O_CREAT | __O_TMPFILE); 50 } 51 52 static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 53 { 54 struct io_open *open = io_kiocb_to_cmd(req, struct io_open); 55 const char __user *fname; 56 int ret; 57 58 if (unlikely(sqe->buf_index)) 59 return -EINVAL; 60 if (unlikely(req->flags & REQ_F_FIXED_FILE)) 61 return -EBADF; 62 63 /* open.how should be already initialised */ 64 if (!(open->how.flags & O_PATH) && force_o_largefile()) 65 open->how.flags |= O_LARGEFILE; 66 67 open->dfd = READ_ONCE(sqe->fd); 68 fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); 69 open->filename = getname(fname); 70 if (IS_ERR(open->filename)) { 71 ret = PTR_ERR(open->filename); 72 open->filename = NULL; 73 return ret; 74 } 75 76 open->file_slot = READ_ONCE(sqe->file_index); 77 if (open->file_slot && (open->how.flags & O_CLOEXEC)) 78 return -EINVAL; 79 80 open->nofile = rlimit(RLIMIT_NOFILE); 81 req->flags |= REQ_F_NEED_CLEANUP; 82 if (io_openat_force_async(open)) 83 req->flags |= REQ_F_FORCE_ASYNC; 84 return 0; 85 } 86 87 int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 88 { 89 struct io_open *open = io_kiocb_to_cmd(req, struct io_open); 90 u64 mode = READ_ONCE(sqe->len); 91 u64 flags = READ_ONCE(sqe->open_flags); 92 93 open->how = build_open_how(flags, mode); 94 return __io_openat_prep(req, sqe); 95 } 96 97 int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 98 { 99 struct io_open *open = io_kiocb_to_cmd(req, struct io_open); 100 struct open_how __user *how; 101 size_t len; 102 int ret; 103 104 how = u64_to_user_ptr(READ_ONCE(sqe->addr2)); 105 len = READ_ONCE(sqe->len); 106 if (len < OPEN_HOW_SIZE_VER0) 107 return -EINVAL; 108 109 ret = copy_struct_from_user(&open->how, sizeof(open->how), how, len); 110 if (ret) 111 return ret; 112 113 return __io_openat_prep(req, sqe); 114 } 115 116 int io_openat2(struct io_kiocb *req, unsigned int issue_flags) 117 { 118 struct io_open *open = io_kiocb_to_cmd(req, struct io_open); 119 struct open_flags op; 120 struct file *file; 121 bool resolve_nonblock, nonblock_set; 122 bool fixed = !!open->file_slot; 123 int ret; 124 125 ret = build_open_flags(&open->how, &op); 126 if (ret) 127 goto err; 128 nonblock_set = op.open_flag & O_NONBLOCK; 129 resolve_nonblock = open->how.resolve & RESOLVE_CACHED; 130 if (issue_flags & IO_URING_F_NONBLOCK) { 131 WARN_ON_ONCE(io_openat_force_async(open)); 132 op.lookup_flags |= LOOKUP_CACHED; 133 op.open_flag |= O_NONBLOCK; 134 } 135 136 if (!fixed) { 137 ret = __get_unused_fd_flags(open->how.flags, open->nofile); 138 if (ret < 0) 139 goto err; 140 } 141 142 file = do_filp_open(open->dfd, open->filename, &op); 143 if (IS_ERR(file)) { 144 /* 145 * We could hang on to this 'fd' on retrying, but seems like 146 * marginal gain for something that is now known to be a slower 147 * path. So just put it, and we'll get a new one when we retry. 148 */ 149 if (!fixed) 150 put_unused_fd(ret); 151 152 ret = PTR_ERR(file); 153 /* only retry if RESOLVE_CACHED wasn't already set by application */ 154 if (ret == -EAGAIN && 155 (!resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK))) 156 return -EAGAIN; 157 goto err; 158 } 159 160 if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set) 161 file->f_flags &= ~O_NONBLOCK; 162 163 if (!fixed) 164 fd_install(ret, file); 165 else 166 ret = io_fixed_fd_install(req, issue_flags, file, 167 open->file_slot); 168 err: 169 putname(open->filename); 170 req->flags &= ~REQ_F_NEED_CLEANUP; 171 if (ret < 0) 172 req_set_fail(req); 173 io_req_set_res(req, ret, 0); 174 return IOU_COMPLETE; 175 } 176 177 int io_openat(struct io_kiocb *req, unsigned int issue_flags) 178 { 179 return io_openat2(req, issue_flags); 180 } 181 182 void io_open_cleanup(struct io_kiocb *req) 183 { 184 struct io_open *open = io_kiocb_to_cmd(req, struct io_open); 185 186 if (open->filename) 187 putname(open->filename); 188 } 189 190 int __io_close_fixed(struct io_ring_ctx *ctx, unsigned int issue_flags, 191 unsigned int offset) 192 { 193 int ret; 194 195 io_ring_submit_lock(ctx, issue_flags); 196 ret = io_fixed_fd_remove(ctx, offset); 197 io_ring_submit_unlock(ctx, issue_flags); 198 199 return ret; 200 } 201 202 static inline int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) 203 { 204 struct io_close *close = io_kiocb_to_cmd(req, struct io_close); 205 206 return __io_close_fixed(req->ctx, issue_flags, close->file_slot - 1); 207 } 208 209 int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 210 { 211 struct io_close *close = io_kiocb_to_cmd(req, struct io_close); 212 213 if (sqe->off || sqe->addr || sqe->len || sqe->rw_flags || sqe->buf_index) 214 return -EINVAL; 215 if (req->flags & REQ_F_FIXED_FILE) 216 return -EBADF; 217 218 close->fd = READ_ONCE(sqe->fd); 219 close->file_slot = READ_ONCE(sqe->file_index); 220 if (close->file_slot && close->fd) 221 return -EINVAL; 222 223 return 0; 224 } 225 226 int io_close(struct io_kiocb *req, unsigned int issue_flags) 227 { 228 struct files_struct *files = current->files; 229 struct io_close *close = io_kiocb_to_cmd(req, struct io_close); 230 struct file *file; 231 int ret = -EBADF; 232 233 if (close->file_slot) { 234 ret = io_close_fixed(req, issue_flags); 235 goto err; 236 } 237 238 spin_lock(&files->file_lock); 239 file = files_lookup_fd_locked(files, close->fd); 240 if (!file || io_is_uring_fops(file)) { 241 spin_unlock(&files->file_lock); 242 goto err; 243 } 244 245 /* if the file has a flush method, be safe and punt to async */ 246 if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) { 247 spin_unlock(&files->file_lock); 248 return -EAGAIN; 249 } 250 251 file = file_close_fd_locked(files, close->fd); 252 spin_unlock(&files->file_lock); 253 if (!file) 254 goto err; 255 256 /* No ->flush() or already async, safely close from here */ 257 ret = filp_close(file, current->files); 258 err: 259 if (ret < 0) 260 req_set_fail(req); 261 io_req_set_res(req, ret, 0); 262 return IOU_COMPLETE; 263 } 264 265 int io_install_fixed_fd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 266 { 267 struct io_fixed_install *ifi; 268 unsigned int flags; 269 270 if (sqe->off || sqe->addr || sqe->len || sqe->buf_index || 271 sqe->splice_fd_in || sqe->addr3) 272 return -EINVAL; 273 274 /* must be a fixed file */ 275 if (!(req->flags & REQ_F_FIXED_FILE)) 276 return -EBADF; 277 278 flags = READ_ONCE(sqe->install_fd_flags); 279 if (flags & ~IORING_FIXED_FD_NO_CLOEXEC) 280 return -EINVAL; 281 282 /* ensure the task's creds are used when installing/receiving fds */ 283 if (req->flags & REQ_F_CREDS) 284 return -EPERM; 285 286 /* default to O_CLOEXEC, disable if IORING_FIXED_FD_NO_CLOEXEC is set */ 287 ifi = io_kiocb_to_cmd(req, struct io_fixed_install); 288 ifi->o_flags = O_CLOEXEC; 289 if (flags & IORING_FIXED_FD_NO_CLOEXEC) 290 ifi->o_flags = 0; 291 292 return 0; 293 } 294 295 int io_install_fixed_fd(struct io_kiocb *req, unsigned int issue_flags) 296 { 297 struct io_fixed_install *ifi; 298 int ret; 299 300 ifi = io_kiocb_to_cmd(req, struct io_fixed_install); 301 ret = receive_fd(req->file, NULL, ifi->o_flags); 302 if (ret < 0) 303 req_set_fail(req); 304 io_req_set_res(req, ret, 0); 305 return IOU_COMPLETE; 306 } 307 308 struct io_pipe { 309 struct file *file; 310 int __user *fds; 311 int flags; 312 int file_slot; 313 unsigned long nofile; 314 }; 315 316 int io_pipe_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) 317 { 318 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); 319 320 if (sqe->fd || sqe->off || sqe->addr3) 321 return -EINVAL; 322 323 p->fds = u64_to_user_ptr(READ_ONCE(sqe->addr)); 324 p->flags = READ_ONCE(sqe->pipe_flags); 325 if (p->flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE)) 326 return -EINVAL; 327 328 p->file_slot = READ_ONCE(sqe->file_index); 329 p->nofile = rlimit(RLIMIT_NOFILE); 330 return 0; 331 } 332 333 static int io_pipe_fixed(struct io_kiocb *req, struct file **files, 334 unsigned int issue_flags) 335 { 336 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); 337 struct io_ring_ctx *ctx = req->ctx; 338 int ret, fds[2] = { -1, -1 }; 339 int slot = p->file_slot; 340 341 if (p->flags & O_CLOEXEC) 342 return -EINVAL; 343 344 io_ring_submit_lock(ctx, issue_flags); 345 346 ret = __io_fixed_fd_install(ctx, files[0], slot); 347 if (ret < 0) 348 goto err; 349 fds[0] = ret; 350 files[0] = NULL; 351 352 /* 353 * If a specific slot is given, next one will be used for 354 * the write side. 355 */ 356 if (slot != IORING_FILE_INDEX_ALLOC) 357 slot++; 358 359 ret = __io_fixed_fd_install(ctx, files[1], slot); 360 if (ret < 0) 361 goto err; 362 fds[1] = ret; 363 files[1] = NULL; 364 365 io_ring_submit_unlock(ctx, issue_flags); 366 367 if (!copy_to_user(p->fds, fds, sizeof(fds))) 368 return 0; 369 370 ret = -EFAULT; 371 io_ring_submit_lock(ctx, issue_flags); 372 err: 373 if (fds[0] != -1) 374 io_fixed_fd_remove(ctx, fds[0]); 375 if (fds[1] != -1) 376 io_fixed_fd_remove(ctx, fds[1]); 377 io_ring_submit_unlock(ctx, issue_flags); 378 return ret; 379 } 380 381 static int io_pipe_fd(struct io_kiocb *req, struct file **files) 382 { 383 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); 384 int ret, fds[2] = { -1, -1 }; 385 386 ret = __get_unused_fd_flags(p->flags, p->nofile); 387 if (ret < 0) 388 goto err; 389 fds[0] = ret; 390 391 ret = __get_unused_fd_flags(p->flags, p->nofile); 392 if (ret < 0) 393 goto err; 394 fds[1] = ret; 395 396 if (!copy_to_user(p->fds, fds, sizeof(fds))) { 397 fd_install(fds[0], files[0]); 398 fd_install(fds[1], files[1]); 399 return 0; 400 } 401 ret = -EFAULT; 402 err: 403 if (fds[0] != -1) 404 put_unused_fd(fds[0]); 405 if (fds[1] != -1) 406 put_unused_fd(fds[1]); 407 return ret; 408 } 409 410 int io_pipe(struct io_kiocb *req, unsigned int issue_flags) 411 { 412 struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); 413 struct file *files[2]; 414 int ret; 415 416 ret = create_pipe_files(files, p->flags); 417 if (ret) 418 return ret; 419 files[0]->f_mode |= FMODE_NOWAIT; 420 files[1]->f_mode |= FMODE_NOWAIT; 421 422 if (!!p->file_slot) 423 ret = io_pipe_fixed(req, files, issue_flags); 424 else 425 ret = io_pipe_fd(req, files); 426 427 io_req_set_res(req, ret, 0); 428 if (!ret) 429 return IOU_COMPLETE; 430 431 req_set_fail(req); 432 if (files[0]) 433 fput(files[0]); 434 if (files[1]) 435 fput(files[1]); 436 return ret; 437 } 438