Lines Matching full:rw
24 #include "rw.h"
53 static int io_iov_compat_buffer_select_prep(struct io_rw *rw) in io_iov_compat_buffer_select_prep() argument
58 uiov = u64_to_user_ptr(rw->addr); in io_iov_compat_buffer_select_prep()
66 rw->len = clen; in io_iov_compat_buffer_select_prep()
75 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_iov_buffer_select_prep() local
77 if (rw->len != 1) in io_iov_buffer_select_prep()
82 return io_iov_compat_buffer_select_prep(rw); in io_iov_buffer_select_prep()
85 uiov = u64_to_user_ptr(rw->addr); in io_iov_buffer_select_prep()
88 rw->len = iov.iov_len; in io_iov_buffer_select_prep()
97 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in __io_import_iovec() local
103 buf = u64_to_user_ptr(rw->addr); in __io_import_iovec()
104 sqe_len = rw->len; in __io_import_iovec()
111 rw->addr = (unsigned long) buf; in __io_import_iovec()
112 rw->len = sqe_len; in __io_import_iovec()
138 static inline int io_import_iovec(int rw, struct io_kiocb *req, in io_import_iovec() argument
144 ret = __io_import_iovec(rw, req, io, issue_flags); in io_import_iovec()
154 struct io_async_rw *rw = req->async_data; in io_rw_recycle() local
159 io_alloc_cache_kasan(&rw->free_iovec, &rw->free_iov_nr); in io_rw_recycle()
160 if (io_alloc_cache_put(&req->ctx->rw_cache, rw)) { in io_rw_recycle()
204 struct io_async_rw *rw; in io_rw_alloc_async() local
206 rw = io_uring_alloc_async_data(&ctx->rw_cache, req); in io_rw_alloc_async()
207 if (!rw) in io_rw_alloc_async()
209 if (rw->free_iovec) in io_rw_alloc_async()
211 rw->bytes_done = 0; in io_rw_alloc_async()
217 struct io_async_rw *rw; in io_prep_rw_setup() local
225 rw = req->async_data; in io_prep_rw_setup()
226 return io_import_iovec(ddir, req, rw, 0); in io_prep_rw_setup()
243 static int io_prep_rw_pi(struct io_kiocb *req, struct io_rw *rw, int ddir, in io_prep_rw_pi() argument
273 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_prep_rw() local
278 rw->kiocb.ki_pos = READ_ONCE(sqe->off); in io_prep_rw()
288 rw->kiocb.ki_ioprio = ioprio; in io_prep_rw()
290 rw->kiocb.ki_ioprio = get_current_ioprio(); in io_prep_rw()
292 rw->kiocb.dio_complete = NULL; in io_prep_rw()
293 rw->kiocb.ki_flags = 0; in io_prep_rw()
296 rw->kiocb.ki_complete = io_complete_rw_iopoll; in io_prep_rw()
298 rw->kiocb.ki_complete = io_complete_rw; in io_prep_rw()
300 rw->addr = READ_ONCE(sqe->addr); in io_prep_rw()
301 rw->len = READ_ONCE(sqe->len); in io_prep_rw()
302 rw->flags = READ_ONCE(sqe->rw_flags); in io_prep_rw()
317 ret = io_prep_rw_pi(req, rw, ddir, attr_ptr, attr_type_mask); in io_prep_rw()
345 * Have to do this validation here, as this is in io_read() rw->len in io_prep_rwv()
364 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_prep_rw_fixed() local
380 ret = io_import_fixed(ddir, &io->iter, node->buf, rw->addr, rw->len); in io_prep_rw_fixed()
401 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_read_mshot_prep() local
412 if (rw->addr || rw->len) in io_read_mshot_prep()
427 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_kiocb_update_pos() local
429 if (rw->kiocb.ki_pos != -1) in io_kiocb_update_pos()
430 return &rw->kiocb.ki_pos; in io_kiocb_update_pos()
434 rw->kiocb.ki_pos = req->file->f_pos; in io_kiocb_update_pos()
435 return &rw->kiocb.ki_pos; in io_kiocb_update_pos()
438 rw->kiocb.ki_pos = 0; in io_kiocb_update_pos()
445 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_rw_should_reissue() local
463 io_meta_restore(io, &rw->kiocb); in io_rw_should_reissue()
474 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_req_end_write() local
476 kiocb_end_write(&rw->kiocb); in io_req_end_write()
486 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_req_io_end() local
488 if (rw->kiocb.ki_flags & IOCB_WRITE) { in io_req_io_end()
524 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_req_rw_complete() local
525 struct kiocb *kiocb = &rw->kiocb; in io_req_rw_complete()
528 long res = kiocb->dio_complete(rw->kiocb.private); in io_req_rw_complete()
544 struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb); in io_complete_rw() local
545 struct io_kiocb *req = cmd_to_io_kiocb(rw); in io_complete_rw()
557 struct io_rw *rw = container_of(kiocb, struct io_rw, kiocb); in io_complete_rw_iopoll() local
558 struct io_kiocb *req = cmd_to_io_kiocb(rw); in io_complete_rw_iopoll()
575 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_rw_done() local
599 io_complete_rw_iopoll(&rw->kiocb, ret); in io_rw_done()
601 io_complete_rw(&rw->kiocb, ret); in io_rw_done()
607 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in kiocb_done() local
611 req->file->f_pos = rw->kiocb.ki_pos; in kiocb_done()
638 static ssize_t loop_rw_iter(int ddir, struct io_rw *rw, struct iov_iter *iter) in loop_rw_iter() argument
640 struct kiocb *kiocb = &rw->kiocb; in loop_rw_iter()
670 addr = u64_to_user_ptr(rw->addr); in loop_rw_iter()
671 len = rw->len; in loop_rw_iter()
688 rw->addr += nr; in loop_rw_iter()
689 rw->len -= nr; in loop_rw_iter()
690 if (!rw->len) in loop_rw_iter()
715 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_async_buf_func() local
723 rw->kiocb.ki_flags &= ~IOCB_WAITQ; in io_async_buf_func()
745 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_rw_should_retry() local
746 struct kiocb *kiocb = &rw->kiocb; in io_rw_should_retry()
777 static inline int io_iter_do_read(struct io_rw *rw, struct iov_iter *iter) in io_iter_do_read() argument
779 struct file *file = rw->kiocb.ki_filp; in io_iter_do_read()
782 return file->f_op->read_iter(&rw->kiocb, iter); in io_iter_do_read()
784 return loop_rw_iter(READ, rw, iter); in io_iter_do_read()
797 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_rw_init_file() local
798 struct kiocb *kiocb = &rw->kiocb; in io_rw_init_file()
810 ret = kiocb_set_rw_flags(kiocb, rw->flags, rw_type); in io_rw_init_file()
859 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in __io_read() local
861 struct kiocb *kiocb = &rw->kiocb; in __io_read()
891 ret = io_iter_do_read(rw, &io->iter); in __io_read()
954 ret = io_iter_do_read(rw, &io->iter); in __io_read()
979 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_read_mshot() local
990 rw->kiocb.ki_complete = NULL; in io_read_mshot()
999 * Reset rw->len to 0 again to avoid clamping future mshot in io_read_mshot()
1003 rw->len = 0; in io_read_mshot()
1021 rw->len = 0; /* similarly to above, reset len to 0 */ in io_read_mshot()
1070 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_write() local
1072 struct kiocb *kiocb = &rw->kiocb; in io_write()
1111 ret2 = loop_rw_iter(WRITE, rw, &io->iter); in io_write()
1176 struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); in io_uring_classic_poll() local
1178 return file->f_op->iopoll(&rw->kiocb, iob, poll_flags); in io_uring_classic_poll()
1311 struct io_async_rw *rw = (struct io_async_rw *) entry; in io_rw_cache_free() local
1313 if (rw->free_iovec) in io_rw_cache_free()
1314 kfree(rw->free_iovec); in io_rw_cache_free()
1315 kfree(rw); in io_rw_cache_free()