1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Direct I/O support.
3 *
4 * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #include <linux/export.h>
9 #include <linux/fs.h>
10 #include <linux/mm.h>
11 #include <linux/pagemap.h>
12 #include <linux/slab.h>
13 #include <linux/uio.h>
14 #include <linux/sched/mm.h>
15 #include <linux/task_io_accounting_ops.h>
16 #include <linux/netfs.h>
17 #include "internal.h"
18
netfs_prepare_dio_read_iterator(struct netfs_io_subrequest * subreq)19 static void netfs_prepare_dio_read_iterator(struct netfs_io_subrequest *subreq)
20 {
21 struct netfs_io_request *rreq = subreq->rreq;
22 size_t rsize;
23
24 rsize = umin(subreq->len, rreq->io_streams[0].sreq_max_len);
25 subreq->len = rsize;
26
27 if (unlikely(rreq->io_streams[0].sreq_max_segs)) {
28 size_t limit = netfs_limit_iter(&rreq->buffer.iter, 0, rsize,
29 rreq->io_streams[0].sreq_max_segs);
30
31 if (limit < rsize) {
32 subreq->len = limit;
33 trace_netfs_sreq(subreq, netfs_sreq_trace_limited);
34 }
35 }
36
37 trace_netfs_sreq(subreq, netfs_sreq_trace_prepare);
38
39 subreq->io_iter = rreq->buffer.iter;
40 iov_iter_truncate(&subreq->io_iter, subreq->len);
41 iov_iter_advance(&rreq->buffer.iter, subreq->len);
42 }
43
44 /*
45 * Perform a read to a buffer from the server, slicing up the region to be read
46 * according to the network rsize.
47 */
netfs_dispatch_unbuffered_reads(struct netfs_io_request * rreq)48 static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq)
49 {
50 unsigned long long start = rreq->start;
51 ssize_t size = rreq->len;
52 int ret;
53
54 do {
55 struct netfs_io_subrequest *subreq;
56 ssize_t slice;
57
58 subreq = netfs_alloc_subrequest(rreq);
59 if (!subreq) {
60 /* Stash the error in the request if there's not
61 * already an error set.
62 */
63 cmpxchg(&rreq->error, 0, -ENOMEM);
64 break;
65 }
66
67 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
68 subreq->start = start;
69 subreq->len = size;
70
71 netfs_queue_read(rreq, subreq);
72
73 netfs_stat(&netfs_n_rh_download);
74 if (rreq->netfs_ops->prepare_read) {
75 ret = rreq->netfs_ops->prepare_read(subreq);
76 if (ret < 0) {
77 netfs_cancel_read(subreq, ret);
78 break;
79 }
80 }
81
82 netfs_prepare_dio_read_iterator(subreq);
83 slice = subreq->len;
84 size -= slice;
85 start += slice;
86 rreq->submitted += slice;
87 if (size <= 0) {
88 smp_wmb(); /* Write lists before ALL_QUEUED. */
89 set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
90 }
91
92 rreq->netfs_ops->issue_read(subreq);
93
94 if (test_bit(NETFS_RREQ_PAUSE, &rreq->flags))
95 netfs_wait_for_paused_read(rreq);
96 if (test_bit(NETFS_RREQ_FAILED, &rreq->flags))
97 break;
98 cond_resched();
99 } while (size > 0);
100
101 if (unlikely(size > 0)) {
102 smp_wmb(); /* Write lists before ALL_QUEUED. */
103 set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags);
104 netfs_wake_collector(rreq);
105 }
106 }
107
108 /*
109 * Perform a read to an application buffer, bypassing the pagecache and the
110 * local disk cache.
111 */
netfs_unbuffered_read(struct netfs_io_request * rreq,bool sync)112 static ssize_t netfs_unbuffered_read(struct netfs_io_request *rreq, bool sync)
113 {
114 ssize_t ret;
115
116 _enter("R=%x %llx-%llx",
117 rreq->debug_id, rreq->start, rreq->start + rreq->len - 1);
118
119 if (rreq->len == 0) {
120 pr_err("Zero-sized read [R=%x]\n", rreq->debug_id);
121 netfs_put_request(rreq, netfs_rreq_trace_put_discard);
122 return -EIO;
123 }
124
125 // TODO: Use bounce buffer if requested
126
127 inode_dio_begin(rreq->inode);
128 netfs_dispatch_unbuffered_reads(rreq);
129
130 /* The collector will get run, even if we don't manage to submit any
131 * subreqs, so we shouldn't call inode_dio_end() here.
132 */
133
134 if (sync)
135 ret = netfs_wait_for_read(rreq);
136 else
137 ret = -EIOCBQUEUED;
138
139 _leave(" = %zd", ret);
140 return ret;
141 }
142
143 /**
144 * netfs_unbuffered_read_iter_locked - Perform an unbuffered or direct I/O read
145 * @iocb: The I/O control descriptor describing the read
146 * @iter: The output buffer (also specifies read length)
147 *
148 * Perform an unbuffered I/O or direct I/O from the file in @iocb to the
149 * output buffer. No use is made of the pagecache.
150 *
151 * The caller must hold any appropriate locks.
152 */
netfs_unbuffered_read_iter_locked(struct kiocb * iocb,struct iov_iter * iter)153 ssize_t netfs_unbuffered_read_iter_locked(struct kiocb *iocb, struct iov_iter *iter)
154 {
155 struct netfs_io_request *rreq;
156 ssize_t ret;
157 size_t orig_count = iov_iter_count(iter);
158 bool sync = is_sync_kiocb(iocb);
159
160 _enter("");
161
162 if (!orig_count)
163 return 0; /* Don't update atime */
164
165 ret = kiocb_write_and_wait(iocb, orig_count);
166 if (ret < 0)
167 return ret;
168 file_accessed(iocb->ki_filp);
169
170 rreq = netfs_alloc_request(iocb->ki_filp->f_mapping, iocb->ki_filp,
171 iocb->ki_pos, orig_count,
172 iocb->ki_flags & IOCB_DIRECT ?
173 NETFS_DIO_READ : NETFS_UNBUFFERED_READ);
174 if (IS_ERR(rreq))
175 return PTR_ERR(rreq);
176
177 netfs_stat(&netfs_n_rh_dio_read);
178 trace_netfs_read(rreq, rreq->start, rreq->len, netfs_read_trace_dio_read);
179
180 /* If this is an async op, we have to keep track of the destination
181 * buffer for ourselves as the caller's iterator will be trashed when
182 * we return.
183 *
184 * In such a case, extract an iterator to represent as much of the the
185 * output buffer as we can manage. Note that the extraction might not
186 * be able to allocate a sufficiently large bvec array and may shorten
187 * the request.
188 */
189 if (user_backed_iter(iter)) {
190 ret = netfs_extract_user_iter(iter, rreq->len, &rreq->buffer.iter, 0);
191 if (ret < 0)
192 goto error_put;
193 rreq->direct_bv = (struct bio_vec *)rreq->buffer.iter.bvec;
194 rreq->direct_bv_count = ret;
195 rreq->direct_bv_unpin = iov_iter_extract_will_pin(iter);
196 rreq->len = iov_iter_count(&rreq->buffer.iter);
197 } else {
198 rreq->buffer.iter = *iter;
199 rreq->len = orig_count;
200 rreq->direct_bv_unpin = false;
201 iov_iter_advance(iter, orig_count);
202 }
203
204 // TODO: Set up bounce buffer if needed
205
206 if (!sync) {
207 rreq->iocb = iocb;
208 __set_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags);
209 }
210
211 ret = netfs_unbuffered_read(rreq, sync);
212 if (ret < 0)
213 goto out; /* May be -EIOCBQUEUED */
214 if (sync) {
215 // TODO: Copy from bounce buffer
216 iocb->ki_pos += rreq->transferred;
217 ret = rreq->transferred;
218 }
219
220 out:
221 netfs_put_request(rreq, netfs_rreq_trace_put_return);
222 if (ret > 0)
223 orig_count -= ret;
224 return ret;
225
226 error_put:
227 netfs_put_failed_request(rreq);
228 return ret;
229 }
230 EXPORT_SYMBOL(netfs_unbuffered_read_iter_locked);
231
232 /**
233 * netfs_unbuffered_read_iter - Perform an unbuffered or direct I/O read
234 * @iocb: The I/O control descriptor describing the read
235 * @iter: The output buffer (also specifies read length)
236 *
237 * Perform an unbuffered I/O or direct I/O from the file in @iocb to the
238 * output buffer. No use is made of the pagecache.
239 */
netfs_unbuffered_read_iter(struct kiocb * iocb,struct iov_iter * iter)240 ssize_t netfs_unbuffered_read_iter(struct kiocb *iocb, struct iov_iter *iter)
241 {
242 struct inode *inode = file_inode(iocb->ki_filp);
243 ssize_t ret;
244
245 if (!iter->count)
246 return 0; /* Don't update atime */
247
248 ret = netfs_start_io_direct(inode);
249 if (ret == 0) {
250 ret = netfs_unbuffered_read_iter_locked(iocb, iter);
251 netfs_end_io_direct(inode);
252 }
253 return ret;
254 }
255 EXPORT_SYMBOL(netfs_unbuffered_read_iter);
256