1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Network filesystem read subrequest retrying.
3 *
4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #include <linux/fs.h>
9 #include <linux/slab.h>
10 #include "internal.h"
11
netfs_reissue_read(struct netfs_io_request * rreq,struct netfs_io_subrequest * subreq)12 static void netfs_reissue_read(struct netfs_io_request *rreq,
13 struct netfs_io_subrequest *subreq)
14 {
15 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
16 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
17 netfs_stat(&netfs_n_rh_retry_read_subreq);
18 subreq->rreq->netfs_ops->issue_read(subreq);
19 }
20
21 /*
22 * Go through the list of failed/short reads, retrying all retryable ones. We
23 * need to switch failed cache reads to network downloads.
24 */
netfs_retry_read_subrequests(struct netfs_io_request * rreq)25 static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
26 {
27 struct netfs_io_subrequest *subreq;
28 struct netfs_io_stream *stream = &rreq->io_streams[0];
29 struct list_head *next;
30
31 _enter("R=%x", rreq->debug_id);
32
33 if (list_empty(&stream->subrequests))
34 return;
35
36 if (rreq->netfs_ops->retry_request)
37 rreq->netfs_ops->retry_request(rreq, NULL);
38
39 /* If there's no renegotiation to do, just resend each retryable subreq
40 * up to the first permanently failed one.
41 */
42 if (!rreq->netfs_ops->prepare_read &&
43 !rreq->cache_resources.ops) {
44 list_for_each_entry(subreq, &stream->subrequests, rreq_link) {
45 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
46 break;
47 if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
48 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
49 subreq->retry_count++;
50 netfs_reset_iter(subreq);
51 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
52 netfs_reissue_read(rreq, subreq);
53 }
54 }
55 return;
56 }
57
58 /* Okay, we need to renegotiate all the download requests and flip any
59 * failed cache reads over to being download requests and negotiate
60 * those also. All fully successful subreqs have been removed from the
61 * list and any spare data from those has been donated.
62 *
63 * What we do is decant the list and rebuild it one subreq at a time so
64 * that we don't end up with donations jumping over a gap we're busy
65 * populating with smaller subrequests. In the event that the subreq
66 * we just launched finishes before we insert the next subreq, it'll
67 * fill in rreq->prev_donated instead.
68 *
69 * Note: Alternatively, we could split the tail subrequest right before
70 * we reissue it and fix up the donations under lock.
71 */
72 next = stream->subrequests.next;
73
74 do {
75 struct netfs_io_subrequest *from, *to, *tmp;
76 struct iov_iter source;
77 unsigned long long start, len;
78 size_t part;
79 bool boundary = false, subreq_superfluous = false;
80
81 /* Go through the subreqs and find the next span of contiguous
82 * buffer that we then rejig (cifs, for example, needs the
83 * rsize renegotiating) and reissue.
84 */
85 from = list_entry(next, struct netfs_io_subrequest, rreq_link);
86 to = from;
87 start = from->start + from->transferred;
88 len = from->len - from->transferred;
89
90 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx",
91 rreq->debug_id, from->debug_index,
92 from->start, from->transferred, from->len);
93
94 if (test_bit(NETFS_SREQ_FAILED, &from->flags) ||
95 !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags))
96 goto abandon;
97
98 list_for_each_continue(next, &stream->subrequests) {
99 subreq = list_entry(next, struct netfs_io_subrequest, rreq_link);
100 if (subreq->start + subreq->transferred != start + len ||
101 test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) ||
102 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
103 break;
104 to = subreq;
105 len += to->len;
106 }
107
108 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len);
109
110 /* Determine the set of buffers we're going to use. Each
111 * subreq gets a subset of a single overall contiguous buffer.
112 */
113 netfs_reset_iter(from);
114 source = from->io_iter;
115 source.count = len;
116
117 /* Work through the sublist. */
118 subreq = from;
119 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) {
120 if (!len) {
121 subreq_superfluous = true;
122 break;
123 }
124 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
125 subreq->start = start - subreq->transferred;
126 subreq->len = len + subreq->transferred;
127 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
128 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
129 subreq->retry_count++;
130
131 trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
132
133 /* Renegotiate max_len (rsize) */
134 stream->sreq_max_len = subreq->len;
135 if (rreq->netfs_ops->prepare_read &&
136 rreq->netfs_ops->prepare_read(subreq) < 0) {
137 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
138 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
139 goto abandon;
140 }
141
142 part = umin(len, stream->sreq_max_len);
143 if (unlikely(stream->sreq_max_segs))
144 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs);
145 subreq->len = subreq->transferred + part;
146 subreq->io_iter = source;
147 iov_iter_truncate(&subreq->io_iter, part);
148 iov_iter_advance(&source, part);
149 len -= part;
150 start += part;
151 if (!len) {
152 if (boundary)
153 __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
154 } else {
155 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
156 }
157
158 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
159 netfs_reissue_read(rreq, subreq);
160 if (subreq == to) {
161 subreq_superfluous = false;
162 break;
163 }
164 }
165
166 /* If we managed to use fewer subreqs, we can discard the
167 * excess; if we used the same number, then we're done.
168 */
169 if (!len) {
170 if (!subreq_superfluous)
171 continue;
172 list_for_each_entry_safe_from(subreq, tmp,
173 &stream->subrequests, rreq_link) {
174 trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous);
175 list_del(&subreq->rreq_link);
176 netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_done);
177 if (subreq == to)
178 break;
179 }
180 continue;
181 }
182
183 /* We ran out of subrequests, so we need to allocate some more
184 * and insert them after.
185 */
186 do {
187 subreq = netfs_alloc_subrequest(rreq);
188 if (!subreq) {
189 subreq = to;
190 goto abandon_after;
191 }
192 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
193 subreq->start = start;
194 subreq->len = len;
195 subreq->stream_nr = stream->stream_nr;
196 subreq->retry_count = 1;
197
198 trace_netfs_sreq_ref(rreq->debug_id, subreq->debug_index,
199 refcount_read(&subreq->ref),
200 netfs_sreq_trace_new);
201
202 list_add(&subreq->rreq_link, &to->rreq_link);
203 to = list_next_entry(to, rreq_link);
204 trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
205
206 stream->sreq_max_len = umin(len, rreq->rsize);
207 stream->sreq_max_segs = 0;
208 if (unlikely(stream->sreq_max_segs))
209 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs);
210
211 netfs_stat(&netfs_n_rh_download);
212 if (rreq->netfs_ops->prepare_read(subreq) < 0) {
213 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
214 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
215 goto abandon;
216 }
217
218 part = umin(len, stream->sreq_max_len);
219 subreq->len = subreq->transferred + part;
220 subreq->io_iter = source;
221 iov_iter_truncate(&subreq->io_iter, part);
222 iov_iter_advance(&source, part);
223
224 len -= part;
225 start += part;
226 if (!len && boundary) {
227 __set_bit(NETFS_SREQ_BOUNDARY, &to->flags);
228 boundary = false;
229 }
230
231 netfs_reissue_read(rreq, subreq);
232 } while (len);
233
234 } while (!list_is_head(next, &stream->subrequests));
235
236 return;
237
238 /* If we hit an error, fail all remaining incomplete subrequests */
239 abandon_after:
240 if (list_is_last(&subreq->rreq_link, &stream->subrequests))
241 return;
242 subreq = list_next_entry(subreq, rreq_link);
243 abandon:
244 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) {
245 if (!subreq->error &&
246 !test_bit(NETFS_SREQ_FAILED, &subreq->flags) &&
247 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
248 continue;
249 subreq->error = -ENOMEM;
250 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
251 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
252 }
253 }
254
255 /*
256 * Retry reads.
257 */
netfs_retry_reads(struct netfs_io_request * rreq)258 void netfs_retry_reads(struct netfs_io_request *rreq)
259 {
260 struct netfs_io_subrequest *subreq;
261 struct netfs_io_stream *stream = &rreq->io_streams[0];
262 DEFINE_WAIT(myself);
263
264 netfs_stat(&netfs_n_rh_retry_read_req);
265
266 set_bit(NETFS_RREQ_RETRYING, &rreq->flags);
267
268 /* Wait for all outstanding I/O to quiesce before performing retries as
269 * we may need to renegotiate the I/O sizes.
270 */
271 list_for_each_entry(subreq, &stream->subrequests, rreq_link) {
272 if (!test_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags))
273 continue;
274
275 trace_netfs_rreq(rreq, netfs_rreq_trace_wait_queue);
276 for (;;) {
277 prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE);
278
279 if (!test_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags))
280 break;
281
282 trace_netfs_sreq(subreq, netfs_sreq_trace_wait_for);
283 schedule();
284 trace_netfs_rreq(rreq, netfs_rreq_trace_woke_queue);
285 }
286
287 finish_wait(&rreq->waitq, &myself);
288 }
289 clear_bit(NETFS_RREQ_RETRYING, &rreq->flags);
290
291 trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit);
292 netfs_retry_read_subrequests(rreq);
293 }
294
295 /*
296 * Unlock any the pages that haven't been unlocked yet due to abandoned
297 * subrequests.
298 */
netfs_unlock_abandoned_read_pages(struct netfs_io_request * rreq)299 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
300 {
301 struct folio_queue *p;
302
303 for (p = rreq->buffer.tail; p; p = p->next) {
304 for (int slot = 0; slot < folioq_count(p); slot++) {
305 struct folio *folio = folioq_folio(p, slot);
306
307 if (folio && !folioq_is_marked2(p, slot)) {
308 trace_netfs_folio(folio, netfs_folio_trace_abandon);
309 folio_unlock(folio);
310 }
311 }
312 }
313 }
314