1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Network filesystem read subrequest retrying.
3 *
4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
5 * Written by David Howells (dhowells@redhat.com)
6 */
7
8 #include <linux/fs.h>
9 #include <linux/slab.h>
10 #include "internal.h"
11
netfs_reissue_read(struct netfs_io_request * rreq,struct netfs_io_subrequest * subreq)12 static void netfs_reissue_read(struct netfs_io_request *rreq,
13 struct netfs_io_subrequest *subreq)
14 {
15 struct iov_iter *io_iter = &subreq->io_iter;
16
17 if (iov_iter_is_folioq(io_iter)) {
18 subreq->curr_folioq = (struct folio_queue *)io_iter->folioq;
19 subreq->curr_folioq_slot = io_iter->folioq_slot;
20 subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot];
21 }
22
23 atomic_inc(&rreq->nr_outstanding);
24 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
25 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit);
26 subreq->rreq->netfs_ops->issue_read(subreq);
27 }
28
29 /*
30 * Go through the list of failed/short reads, retrying all retryable ones. We
31 * need to switch failed cache reads to network downloads.
32 */
netfs_retry_read_subrequests(struct netfs_io_request * rreq)33 static void netfs_retry_read_subrequests(struct netfs_io_request *rreq)
34 {
35 struct netfs_io_subrequest *subreq;
36 struct netfs_io_stream *stream0 = &rreq->io_streams[0];
37 LIST_HEAD(sublist);
38 LIST_HEAD(queue);
39
40 _enter("R=%x", rreq->debug_id);
41
42 if (list_empty(&rreq->subrequests))
43 return;
44
45 if (rreq->netfs_ops->retry_request)
46 rreq->netfs_ops->retry_request(rreq, NULL);
47
48 /* If there's no renegotiation to do, just resend each retryable subreq
49 * up to the first permanently failed one.
50 */
51 if (!rreq->netfs_ops->prepare_read &&
52 !rreq->cache_resources.ops) {
53 struct netfs_io_subrequest *subreq;
54
55 list_for_each_entry(subreq, &rreq->subrequests, rreq_link) {
56 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags))
57 break;
58 if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
59 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
60 subreq->retry_count++;
61 netfs_reset_iter(subreq);
62 netfs_reissue_read(rreq, subreq);
63 }
64 }
65 return;
66 }
67
68 /* Okay, we need to renegotiate all the download requests and flip any
69 * failed cache reads over to being download requests and negotiate
70 * those also. All fully successful subreqs have been removed from the
71 * list and any spare data from those has been donated.
72 *
73 * What we do is decant the list and rebuild it one subreq at a time so
74 * that we don't end up with donations jumping over a gap we're busy
75 * populating with smaller subrequests. In the event that the subreq
76 * we just launched finishes before we insert the next subreq, it'll
77 * fill in rreq->prev_donated instead.
78
79 * Note: Alternatively, we could split the tail subrequest right before
80 * we reissue it and fix up the donations under lock.
81 */
82 list_splice_init(&rreq->subrequests, &queue);
83
84 do {
85 struct netfs_io_subrequest *from;
86 struct iov_iter source;
87 unsigned long long start, len;
88 size_t part, deferred_next_donated = 0;
89 bool boundary = false;
90
91 /* Go through the subreqs and find the next span of contiguous
92 * buffer that we then rejig (cifs, for example, needs the
93 * rsize renegotiating) and reissue.
94 */
95 from = list_first_entry(&queue, struct netfs_io_subrequest, rreq_link);
96 list_move_tail(&from->rreq_link, &sublist);
97 start = from->start + from->transferred;
98 len = from->len - from->transferred;
99
100 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx/%zx",
101 rreq->debug_id, from->debug_index,
102 from->start, from->consumed, from->transferred, from->len);
103
104 if (test_bit(NETFS_SREQ_FAILED, &from->flags) ||
105 !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags))
106 goto abandon;
107
108 deferred_next_donated = from->next_donated;
109 while ((subreq = list_first_entry_or_null(
110 &queue, struct netfs_io_subrequest, rreq_link))) {
111 if (subreq->start != start + len ||
112 subreq->transferred > 0 ||
113 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags))
114 break;
115 list_move_tail(&subreq->rreq_link, &sublist);
116 len += subreq->len;
117 deferred_next_donated = subreq->next_donated;
118 if (test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags))
119 break;
120 }
121
122 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len);
123
124 /* Determine the set of buffers we're going to use. Each
125 * subreq gets a subset of a single overall contiguous buffer.
126 */
127 netfs_reset_iter(from);
128 source = from->io_iter;
129 source.count = len;
130
131 /* Work through the sublist. */
132 while ((subreq = list_first_entry_or_null(
133 &sublist, struct netfs_io_subrequest, rreq_link))) {
134 list_del(&subreq->rreq_link);
135
136 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
137 subreq->start = start - subreq->transferred;
138 subreq->len = len + subreq->transferred;
139 stream0->sreq_max_len = subreq->len;
140
141 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
142 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
143 subreq->retry_count++;
144
145 spin_lock_bh(&rreq->lock);
146 list_add_tail(&subreq->rreq_link, &rreq->subrequests);
147 subreq->prev_donated += rreq->prev_donated;
148 rreq->prev_donated = 0;
149 trace_netfs_sreq(subreq, netfs_sreq_trace_retry);
150 spin_unlock_bh(&rreq->lock);
151
152 BUG_ON(!len);
153
154 /* Renegotiate max_len (rsize) */
155 if (rreq->netfs_ops->prepare_read &&
156 rreq->netfs_ops->prepare_read(subreq) < 0) {
157 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed);
158 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
159 }
160
161 part = umin(len, stream0->sreq_max_len);
162 if (unlikely(rreq->io_streams[0].sreq_max_segs))
163 part = netfs_limit_iter(&source, 0, part, stream0->sreq_max_segs);
164 subreq->len = subreq->transferred + part;
165 subreq->io_iter = source;
166 iov_iter_truncate(&subreq->io_iter, part);
167 iov_iter_advance(&source, part);
168 len -= part;
169 start += part;
170 if (!len) {
171 if (boundary)
172 __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
173 subreq->next_donated = deferred_next_donated;
174 } else {
175 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags);
176 subreq->next_donated = 0;
177 }
178
179 netfs_reissue_read(rreq, subreq);
180 if (!len)
181 break;
182
183 /* If we ran out of subrequests, allocate another. */
184 if (list_empty(&sublist)) {
185 subreq = netfs_alloc_subrequest(rreq);
186 if (!subreq)
187 goto abandon;
188 subreq->source = NETFS_DOWNLOAD_FROM_SERVER;
189 subreq->start = start;
190
191 /* We get two refs, but need just one. */
192 netfs_put_subrequest(subreq, false, netfs_sreq_trace_new);
193 trace_netfs_sreq(subreq, netfs_sreq_trace_split);
194 list_add_tail(&subreq->rreq_link, &sublist);
195 }
196 }
197
198 /* If we managed to use fewer subreqs, we can discard the
199 * excess.
200 */
201 while ((subreq = list_first_entry_or_null(
202 &sublist, struct netfs_io_subrequest, rreq_link))) {
203 trace_netfs_sreq(subreq, netfs_sreq_trace_discard);
204 list_del(&subreq->rreq_link);
205 netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_done);
206 }
207
208 } while (!list_empty(&queue));
209
210 return;
211
212 /* If we hit ENOMEM, fail all remaining subrequests */
213 abandon:
214 list_splice_init(&sublist, &queue);
215 list_for_each_entry(subreq, &queue, rreq_link) {
216 if (!subreq->error)
217 subreq->error = -ENOMEM;
218 __clear_bit(NETFS_SREQ_FAILED, &subreq->flags);
219 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
220 }
221 spin_lock_bh(&rreq->lock);
222 list_splice_tail_init(&queue, &rreq->subrequests);
223 spin_unlock_bh(&rreq->lock);
224 }
225
226 /*
227 * Retry reads.
228 */
netfs_retry_reads(struct netfs_io_request * rreq)229 void netfs_retry_reads(struct netfs_io_request *rreq)
230 {
231 trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit);
232
233 atomic_inc(&rreq->nr_outstanding);
234
235 netfs_retry_read_subrequests(rreq);
236
237 if (atomic_dec_and_test(&rreq->nr_outstanding))
238 netfs_rreq_terminated(rreq, false);
239 }
240
241 /*
242 * Unlock any the pages that haven't been unlocked yet due to abandoned
243 * subrequests.
244 */
netfs_unlock_abandoned_read_pages(struct netfs_io_request * rreq)245 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq)
246 {
247 struct folio_queue *p;
248
249 for (p = rreq->buffer; p; p = p->next) {
250 for (int slot = 0; slot < folioq_count(p); slot++) {
251 struct folio *folio = folioq_folio(p, slot);
252
253 if (folio && !folioq_is_marked2(p, slot)) {
254 trace_netfs_folio(folio, netfs_folio_trace_abandon);
255 folio_unlock(folio);
256 }
257 }
258 }
259 }
260