1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Network filesystem read subrequest retrying.
3 *
4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #include <linux/fs.h>
9 #include <linux/slab.h>
10 #include "internal.h"
11
netfs_reissue_read(struct netfs_io_request * rreq,struct netfs_io_subrequest * subreq)12 static void netfs_reissue_read(struct netfs_io_request *rreq,
13 struct netfs_io_subrequest *subreq)
14 {
15 subreq->error = 0;
16 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
17 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
18 netfs_stat(&netfs_n_rh_retry_read_subreq);
19 subreq->rreq->netfs_ops->issue_read(subreq);
20 }
21
22 /*
23 * Go through the list of failed/short reads, retrying all retryable ones. We
24 * need to switch failed cache reads to network downloads.
25 */
netfs_retry_read_subrequests(struct netfs_io_request * rreq)26 static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
27 {
28 struct netfs_io_subrequest *subreq;
29 struct netfs_io_stream *stream = &rreq->io_streams[0];
30 struct list_head *next;
31
32 _enter("R=%x", rreq->debug_id);
33
34 if (list_empty(&stream->subrequests))
35 return;
36
37 if (rreq->netfs_ops->retry_request)
38 rreq->netfs_ops->retry_request(rreq, NULL);
39
40 /* If there's no renegotiation to do, just resend each retryable subreq
41 * up to the first permanently failed one.
42 */
43 if (!rreq->netfs_ops->prepare_read &&
44 !rreq->cache_resources.ops) {
45 list_for_each_entry(subreq, &stream->subrequests, rreq_link) {
46 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
47 break;
48 if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
49 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
50 subreq->retry_count++;
51 netfs_reset_iter(subreq);
52 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
53 netfs_reissue_read(rreq, subreq);
54 }
55 }
56 return;
57 }
58
59 /* Okay, we need to renegotiate all the download requests and flip any
60 * failed cache reads over to being download requests and negotiate
61 * those also. All fully successful subreqs have been removed from the
62 * list and any spare data from those has been donated.
63 *
64 * What we do is decant the list and rebuild it one subreq at a time so
65 * that we don't end up with donations jumping over a gap we're busy
66 * populating with smaller subrequests. In the event that the subreq
67 * we just launched finishes before we insert the next subreq, it'll
68 * fill in rreq->prev_donated instead.
69 *
70 * Note: Alternatively, we could split the tail subrequest right before
71 * we reissue it and fix up the donations under lock.
72 */
73 next = stream->subrequests.next;
74
75 do {
76 struct netfs_io_subrequest *from, *to, *tmp;
77 struct iov_iter source;
78 unsigned long long start, len;
79 size_t part;
80 bool boundary = false, subreq_superfluous = false;
81
82 /* Go through the subreqs and find the next span of contiguous
83 * buffer that we then rejig (cifs, for example, needs the
84 * rsize renegotiating) and reissue.
85 */
86 from = list_entry(next, struct netfs_io_subrequest, rreq_link);
87 to = from;
88 start = from->start + from->transferred;
89 len = from->len - from->transferred;
90
91 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx",
92 rreq->debug_id, from->debug_index,
93 from->start, from->transferred, from->len);
94
95 if (test_bit(NETFS_SREQ_FAILED, &from->flags) ||
96 !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) {
97 subreq = from;
98 goto abandon;
99 }
100
101 for (;;) {
102 /* Read pointer to subreq before reading subreq state. */
103 next = smp_load_acquire(&next->next);
104 if (next == &stream->subrequests)
105 break;
106
107 subreq = list_entry(next, struct netfs_io_subrequest, rreq_link);
108 if (subreq->start + subreq->transferred != start + len ||
109 test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) ||
110 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
111 break;
112 to = subreq;
113 len += to->len;
114 }
115
116 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len);
117
118 /* Determine the set of buffers we're going to use. Each
119 * subreq gets a subset of a single overall contiguous buffer.
120 */
121 netfs_reset_iter(from);
122 source = from->io_iter;
123 source.count = len;
124
125 /* Work through the sublist. */
126 subreq = from;
127 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) {
128 if (!len) {
129 subreq_superfluous = true;
130 break;
131 }
132 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
133 subreq->start = start - subreq->transferred;
134 subreq->len = len + subreq->transferred;
135 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
136 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
137 subreq->retry_count++;
138
139 trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
140
141 /* Renegotiate max_len (rsize) */
142 stream->sreq_max_len = subreq->len;
143 if (rreq->netfs_ops->prepare_read &&
144 rreq->netfs_ops->prepare_read(subreq) < 0) {
145 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
146 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
147 goto abandon;
148 }
149
150 part = umin(len, stream->sreq_max_len);
151 if (unlikely(stream->sreq_max_segs))
152 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs);
153 subreq->len = subreq->transferred + part;
154 subreq->io_iter = source;
155 iov_iter_truncate(&subreq->io_iter, part);
156 iov_iter_advance(&source, part);
157 len -= part;
158 start += part;
159 if (!len) {
160 if (boundary)
161 __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
162 } else {
163 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
164 }
165
166 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
167 netfs_reissue_read(rreq, subreq);
168 if (subreq == to) {
169 subreq_superfluous = false;
170 break;
171 }
172 }
173
174 /* If we managed to use fewer subreqs, we can discard the
175 * excess; if we used the same number, then we're done.
176 */
177 if (!len) {
178 if (!subreq_superfluous)
179 continue;
180 list_for_each_entry_safe_from(subreq, tmp,
181 &stream->subrequests, rreq_link) {
182 trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous);
183 spin_lock(&rreq->lock);
184 list_del(&subreq->rreq_link);
185 spin_unlock(&rreq->lock);
186 netfs_put_subrequest(subreq, netfs_sreq_trace_put_done);
187 if (subreq == to)
188 break;
189 }
190 subreq = NULL;
191 continue;
192 }
193
194 /* We ran out of subrequests, so we need to allocate some more
195 * and insert them after.
196 */
197 do {
198 subreq = netfs_alloc_subrequest(rreq);
199 if (!subreq) {
200 subreq = to;
201 goto abandon_after;
202 }
203 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
204 subreq->start = start;
205 subreq->len = len;
206 subreq->stream_nr = stream->stream_nr;
207 subreq->retry_count = 1;
208
209 trace_netfs_sreq_ref(rreq->debug_id, subreq->debug_index,
210 refcount_read(&subreq->ref),
211 netfs_sreq_trace_new);
212
213 spin_lock(&rreq->lock);
214 list_add(&subreq->rreq_link, &to->rreq_link);
215 spin_unlock(&rreq->lock);
216 to = subreq;
217 trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
218
219 stream->sreq_max_len = umin(len, rreq->rsize);
220 stream->sreq_max_segs = 0;
221 if (unlikely(stream->sreq_max_segs))
222 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs);
223
224 netfs_stat(&netfs_n_rh_download);
225 if (rreq->netfs_ops->prepare_read(subreq) < 0) {
226 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
227 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
228 goto abandon;
229 }
230
231 part = umin(len, stream->sreq_max_len);
232 subreq->len = subreq->transferred + part;
233 subreq->io_iter = source;
234 iov_iter_truncate(&subreq->io_iter, part);
235 iov_iter_advance(&source, part);
236
237 len -= part;
238 start += part;
239 if (!len && boundary) {
240 __set_bit(NETFS_SREQ_BOUNDARY, &to->flags);
241 boundary = false;
242 }
243
244 netfs_reissue_read(rreq, subreq);
245 } while (len);
246
247 } while (!list_is_head(next, &stream->subrequests));
248
249 return;
250
251 /* If we hit an error, fail all remaining incomplete subrequests */
252 abandon_after:
253 if (list_is_last(&subreq->rreq_link, &stream->subrequests))
254 return;
255 subreq = list_next_entry(subreq, rreq_link);
256 abandon:
257 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) {
258 if (!test_bit(NETFS_SREQ_FAILED, &subreq->flags) &&
259 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
260 continue;
261 subreq->error = -ENOMEM;
262 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
263 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
264 }
265 }
266
267 /*
268 * Retry reads.
269 */
netfs_retry_reads(struct netfs_io_request * rreq)270 void netfs_retry_reads(struct netfs_io_request *rreq)
271 {
272 struct netfs_io_stream *stream = &rreq->io_streams[0];
273
274 netfs_stat(&netfs_n_rh_retry_read_req);
275
276 /* Wait for all outstanding I/O to quiesce before performing retries as
277 * we may need to renegotiate the I/O sizes.
278 */
279 set_bit(NETFS_RREQ_RETRYING, &rreq->flags);
280 netfs_wait_for_in_progress_stream(rreq, stream);
281 clear_bit(NETFS_RREQ_RETRYING, &rreq->flags);
282
283 trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit);
284 netfs_retry_read_subrequests(rreq);
285 }
286
287 /*
288 * Unlock any the pages that haven't been unlocked yet due to abandoned
289 * subrequests.
290 */
netfs_unlock_abandoned_read_pages(struct netfs_io_request * rreq)291 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
292 {
293 struct folio_queue *p;
294
295 for (p = rreq->buffer.tail; p; p = p->next) {
296 for (int slot = 0; slot < folioq_count(p); slot++) {
297 struct folio *folio = folioq_folio(p, slot);
298
299 if (folio && !folioq_is_marked2(p, slot)) {
300 if (folio == rreq->no_unlock_folio &&
301 test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO,
302 &rreq->flags)) {
303 _debug("no unlock");
304 } else {
305 trace_netfs_folio(folio,
306 netfs_folio_trace_abandon);
307 folio_unlock(folio);
308 }
309 }
310 }
311 }
312 }
313