1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* Direct I/O support. 3 * 4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/export.h> 9 #include <linux/fs.h> 10 #include <linux/mm.h> 11 #include <linux/pagemap.h> 12 #include <linux/slab.h> 13 #include <linux/uio.h> 14 #include <linux/sched/mm.h> 15 #include <linux/task_io_accounting_ops.h> 16 #include <linux/netfs.h> 17 #include "internal.h" 18 19 static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq) 20 { 21 struct netfs_io_request *rreq = subreq->rreq; 22 size_t rsize; 23 24 rsize = umin(subreq->len, rreq->io_streams[0].sreq_max_len); 25 subreq->len = rsize; 26 27 if (unlikely(rreq->io_streams[0].sreq_max_segs)) { 28 size_t limit = netfs_limit_iter(&rreq->iter, 0, rsize, 29 rreq->io_streams[0].sreq_max_segs); 30 31 if (limit < rsize) { 32 subreq->len = limit; 33 trace_netfs_sreq(subreq, netfs_sreq_trace_limited); 34 } 35 } 36 37 trace_netfs_sreq(subreq, netfs_sreq_trace_prepare); 38 39 subreq->io_iter = rreq->iter; 40 iov_iter_truncate(&subreq->io_iter, subreq->len); 41 iov_iter_advance(&rreq->iter, subreq->len); 42 } 43 44 /* 45 * Perform a read to a buffer from the server, slicing up the region to be read 46 * according to the network rsize. 47 */ 48 static int netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) 49 { 50 unsigned long long start = rreq->start; 51 ssize_t size = rreq->len; 52 int ret = 0; 53 54 atomic_set(&rreq->nr_outstanding, 1); 55 56 do { 57 struct netfs_io_subrequest *subreq; 58 ssize_t slice; 59 60 subreq = netfs_alloc_subrequest(rreq); 61 if (!subreq) { 62 ret = -ENOMEM; 63 break; 64 } 65 66 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 67 subreq->start = start; 68 subreq->len = size; 69 70 atomic_inc(&rreq->nr_outstanding); 71 spin_lock_bh(&rreq->lock); 72 list_add_tail(&subreq->rreq_link, &rreq->subrequests); 73 subreq->prev_donated = rreq->prev_donated; 74 rreq->prev_donated = 0; 75 trace_netfs_sreq(subreq, netfs_sreq_trace_added); 76 spin_unlock_bh(&rreq->lock); 77 78 netfs_stat(&netfs_n_rh_download); 79 if (rreq->netfs_ops->prepare_read) { 80 ret = rreq->netfs_ops->prepare_read(subreq); 81 if (ret < 0) { 82 atomic_dec(&rreq->nr_outstanding); 83 netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_cancel); 84 break; 85 } 86 } 87 88 netfs_prepare_dio_read_iterator(subreq); 89 slice = subreq->len; 90 rreq->netfs_ops->issue_read(subreq); 91 92 size -= slice; 93 start += slice; 94 rreq->submitted += slice; 95 96 if (test_bit(NETFS_RREQ_BLOCKED, &rreq->flags) && 97 test_bit(NETFS_RREQ_NONBLOCK, &rreq->flags)) 98 break; 99 cond_resched(); 100 } while (size > 0); 101 102 if (atomic_dec_and_test(&rreq->nr_outstanding)) 103 netfs_rreq_terminated(rreq, false); 104 return ret; 105 } 106 107 /* 108 * Perform a read to an application buffer, bypassing the pagecache and the 109 * local disk cache. 110 */ 111 static int netfs_unbuffered_read(struct netfs_io_request *rreq, bool sync) 112 { 113 int ret; 114 115 _enter("R=%x %llx-%llx", 116 rreq->debug_id, rreq->start, rreq->start + rreq->len - 1); 117 118 if (rreq->len == 0) { 119 pr_err("Zero-sized read [R=%x]\n", rreq->debug_id); 120 return -EIO; 121 } 122 123 // TODO: Use bounce buffer if requested 124 125 inode_dio_begin(rreq->inode); 126 127 ret = netfs_dispatch_unbuffered_reads(rreq); 128 129 if (!rreq->submitted) { 130 netfs_put_request(rreq, false, netfs_rreq_trace_put_no_submit); 131 inode_dio_end(rreq->inode); 132 ret = 0; 133 goto out; 134 } 135 136 if (sync) { 137 trace_netfs_rreq(rreq, netfs_rreq_trace_wait_ip); 138 wait_on_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS, 139 TASK_UNINTERRUPTIBLE); 140 141 ret = rreq->error; 142 if (ret == 0 && rreq->submitted < rreq->len && 143 rreq->origin != NETFS_DIO_READ) { 144 trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read); 145 ret = -EIO; 146 } 147 } else { 148 ret = -EIOCBQUEUED; 149 } 150 151 out: 152 _leave(" = %d", ret); 153 return ret; 154 } 155 156 /** 157 * netfs_unbuffered_read_iter_locked - Perform an unbuffered or direct I/O read 158 * @iocb: The I/O control descriptor describing the read 159 * @iter: The output buffer (also specifies read length) 160 * 161 * Perform an unbuffered I/O or direct I/O from the file in @iocb to the 162 * output buffer. No use is made of the pagecache. 163 * 164 * The caller must hold any appropriate locks. 165 */ 166 ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *iter) 167 { 168 struct netfs_io_request *rreq; 169 ssize_t ret; 170 size_t orig_count = iov_iter_count(iter); 171 bool sync = is_sync_kiocb(iocb); 172 173 _enter(""); 174 175 if (!orig_count) 176 return 0; /* Don't update atime */ 177 178 ret = kiocb_write_and_wait(iocb, orig_count); 179 if (ret < 0) 180 return ret; 181 file_accessed(iocb->ki_filp); 182 183 rreq = netfs_alloc_request(iocb->ki_filp->f_mapping, iocb->ki_filp, 184 iocb->ki_pos, orig_count, 185 NETFS_DIO_READ); 186 if (IS_ERR(rreq)) 187 return PTR_ERR(rreq); 188 189 netfs_stat(&netfs_n_rh_dio_read); 190 trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_dio_read); 191 192 /* If this is an async op, we have to keep track of the destination 193 * buffer for ourselves as the caller's iterator will be trashed when 194 * we return. 195 * 196 * In such a case, extract an iterator to represent as much of the the 197 * output buffer as we can manage. Note that the extraction might not 198 * be able to allocate a sufficiently large bvec array and may shorten 199 * the request. 200 */ 201 if (user_backed_iter(iter)) { 202 ret = netfs_extract_user_iter(iter, rreq->len, &rreq->iter, 0); 203 if (ret < 0) 204 goto out; 205 rreq->direct_bv = (struct bio_vec *)rreq->iter.bvec; 206 rreq->direct_bv_count = ret; 207 rreq->direct_bv_unpin = iov_iter_extract_will_pin(iter); 208 rreq->len = iov_iter_count(&rreq->iter); 209 } else { 210 rreq->iter = *iter; 211 rreq->len = orig_count; 212 rreq->direct_bv_unpin = false; 213 iov_iter_advance(iter, orig_count); 214 } 215 216 // TODO: Set up bounce buffer if needed 217 218 if (!sync) 219 rreq->iocb = iocb; 220 221 ret = netfs_unbuffered_read(rreq, sync); 222 if (ret < 0) 223 goto out; /* May be -EIOCBQUEUED */ 224 if (sync) { 225 // TODO: Copy from bounce buffer 226 iocb->ki_pos += rreq->transferred; 227 ret = rreq->transferred; 228 } 229 230 out: 231 netfs_put_request(rreq, false, netfs_rreq_trace_put_return); 232 if (ret > 0) 233 orig_count -= ret; 234 return ret; 235 } 236 EXPORT_SYMBOL(netfs_unbuffered_read_iter_locked); 237 238 /** 239 * netfs_unbuffered_read_iter - Perform an unbuffered or direct I/O read 240 * @iocb: The I/O control descriptor describing the read 241 * @iter: The output buffer (also specifies read length) 242 * 243 * Perform an unbuffered I/O or direct I/O from the file in @iocb to the 244 * output buffer. No use is made of the pagecache. 245 */ 246 ssize_t netfs_unbuffered_read_iter(struct kiocb *iocb, struct iov_iter *iter) 247 { 248 struct inode *inode = file_inode(iocb->ki_filp); 249 ssize_t ret; 250 251 if (!iter->count) 252 return 0; /* Don't update atime */ 253 254 ret = netfs_start_io_direct(inode); 255 if (ret == 0) { 256 ret = netfs_unbuffered_read_iter_locked(iocb, iter); 257 netfs_end_io_direct(inode); 258 } 259 return ret; 260 } 261 EXPORT_SYMBOL(netfs_unbuffered_read_iter); 262