1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Network filesystem read subrequest retrying. 3 * 4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/slab.h> 10 #include "internal.h" 11 12 static void netfs_reissue_read(struct netfs_io_request *rreq, 13 struct netfs_io_subrequest *subreq) 14 { 15 subreq->error = 0; 16 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 17 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); 18 netfs_stat(&netfs_n_rh_retry_read_subreq); 19 subreq->rreq->netfs_ops->issue_read(subreq); 20 } 21 22 /* 23 * Go through the list of failed/short reads, retrying all retryable ones. We 24 * need to switch failed cache reads to network downloads. 25 */ 26 static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) 27 { 28 struct netfs_io_subrequest *subreq; 29 struct netfs_io_stream *stream = &rreq->io_streams[0]; 30 struct list_head *next; 31 32 _enter("R=%x", rreq->debug_id); 33 34 if (list_empty(&stream->subrequests)) 35 return; 36 37 if (rreq->netfs_ops->retry_request) 38 rreq->netfs_ops->retry_request(rreq, NULL); 39 40 /* If there's no renegotiation to do, just resend each retryable subreq 41 * up to the first permanently failed one. 42 */ 43 if (!rreq->netfs_ops->prepare_read && 44 !rreq->cache_resources.ops) { 45 list_for_each_entry(subreq, &stream->subrequests, rreq_link) { 46 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) 47 break; 48 if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { 49 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 50 subreq->retry_count++; 51 netfs_reset_iter(subreq); 52 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 53 netfs_reissue_read(rreq, subreq); 54 } 55 } 56 return; 57 } 58 59 /* Okay, we need to renegotiate all the download requests and flip any 60 * failed cache reads over to being download requests and negotiate 61 * those also. All fully successful subreqs have been removed from the 62 * list and any spare data from those has been donated. 63 * 64 * What we do is decant the list and rebuild it one subreq at a time so 65 * that we don't end up with donations jumping over a gap we're busy 66 * populating with smaller subrequests. In the event that the subreq 67 * we just launched finishes before we insert the next subreq, it'll 68 * fill in rreq->prev_donated instead. 69 * 70 * Note: Alternatively, we could split the tail subrequest right before 71 * we reissue it and fix up the donations under lock. 72 */ 73 next = stream->subrequests.next; 74 75 do { 76 struct netfs_io_subrequest *from, *to, *tmp; 77 struct iov_iter source; 78 unsigned long long start, len; 79 size_t part; 80 bool boundary = false, subreq_superfluous = false; 81 82 /* Go through the subreqs and find the next span of contiguous 83 * buffer that we then rejig (cifs, for example, needs the 84 * rsize renegotiating) and reissue. 85 */ 86 from = list_entry(next, struct netfs_io_subrequest, rreq_link); 87 to = from; 88 start = from->start + from->transferred; 89 len = from->len - from->transferred; 90 91 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx", 92 rreq->debug_id, from->debug_index, 93 from->start, from->transferred, from->len); 94 95 if (test_bit(NETFS_SREQ_FAILED, &from->flags) || 96 !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) 97 goto abandon; 98 99 list_for_each_continue(next, &stream->subrequests) { 100 subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); 101 if (subreq->start + subreq->transferred != start + len || 102 test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || 103 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 104 break; 105 to = subreq; 106 len += to->len; 107 } 108 109 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len); 110 111 /* Determine the set of buffers we're going to use. Each 112 * subreq gets a subset of a single overall contiguous buffer. 113 */ 114 netfs_reset_iter(from); 115 source = from->io_iter; 116 source.count = len; 117 118 /* Work through the sublist. */ 119 subreq = from; 120 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { 121 if (!len) { 122 subreq_superfluous = true; 123 break; 124 } 125 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 126 subreq->start = start - subreq->transferred; 127 subreq->len = len + subreq->transferred; 128 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 129 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 130 subreq->retry_count++; 131 132 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 133 134 /* Renegotiate max_len (rsize) */ 135 stream->sreq_max_len = subreq->len; 136 if (rreq->netfs_ops->prepare_read && 137 rreq->netfs_ops->prepare_read(subreq) < 0) { 138 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 139 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 140 goto abandon; 141 } 142 143 part = umin(len, stream->sreq_max_len); 144 if (unlikely(stream->sreq_max_segs)) 145 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); 146 subreq->len = subreq->transferred + part; 147 subreq->io_iter = source; 148 iov_iter_truncate(&subreq->io_iter, part); 149 iov_iter_advance(&source, part); 150 len -= part; 151 start += part; 152 if (!len) { 153 if (boundary) 154 __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 155 } else { 156 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 157 } 158 159 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 160 netfs_reissue_read(rreq, subreq); 161 if (subreq == to) { 162 subreq_superfluous = false; 163 break; 164 } 165 } 166 167 /* If we managed to use fewer subreqs, we can discard the 168 * excess; if we used the same number, then we're done. 169 */ 170 if (!len) { 171 if (!subreq_superfluous) 172 continue; 173 list_for_each_entry_safe_from(subreq, tmp, 174 &stream->subrequests, rreq_link) { 175 trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous); 176 list_del(&subreq->rreq_link); 177 netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); 178 if (subreq == to) 179 break; 180 } 181 continue; 182 } 183 184 /* We ran out of subrequests, so we need to allocate some more 185 * and insert them after. 186 */ 187 do { 188 subreq = netfs_alloc_subrequest(rreq); 189 if (!subreq) { 190 subreq = to; 191 goto abandon_after; 192 } 193 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 194 subreq->start = start; 195 subreq->len = len; 196 subreq->stream_nr = stream->stream_nr; 197 subreq->retry_count = 1; 198 199 trace_netfs_sreq_ref(rreq->debug_id, subreq->debug_index, 200 refcount_read(&subreq->ref), 201 netfs_sreq_trace_new); 202 203 list_add(&subreq->rreq_link, &to->rreq_link); 204 to = list_next_entry(to, rreq_link); 205 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 206 207 stream->sreq_max_len = umin(len, rreq->rsize); 208 stream->sreq_max_segs = 0; 209 if (unlikely(stream->sreq_max_segs)) 210 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); 211 212 netfs_stat(&netfs_n_rh_download); 213 if (rreq->netfs_ops->prepare_read(subreq) < 0) { 214 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 215 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 216 goto abandon; 217 } 218 219 part = umin(len, stream->sreq_max_len); 220 subreq->len = subreq->transferred + part; 221 subreq->io_iter = source; 222 iov_iter_truncate(&subreq->io_iter, part); 223 iov_iter_advance(&source, part); 224 225 len -= part; 226 start += part; 227 if (!len && boundary) { 228 __set_bit(NETFS_SREQ_BOUNDARY, &to->flags); 229 boundary = false; 230 } 231 232 netfs_reissue_read(rreq, subreq); 233 } while (len); 234 235 } while (!list_is_head(next, &stream->subrequests)); 236 237 return; 238 239 /* If we hit an error, fail all remaining incomplete subrequests */ 240 abandon_after: 241 if (list_is_last(&subreq->rreq_link, &stream->subrequests)) 242 return; 243 subreq = list_next_entry(subreq, rreq_link); 244 abandon: 245 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { 246 if (!test_bit(NETFS_SREQ_FAILED, &subreq->flags) && 247 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 248 continue; 249 subreq->error = -ENOMEM; 250 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 251 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 252 } 253 } 254 255 /* 256 * Retry reads. 257 */ 258 void netfs_retry_reads(struct netfs_io_request *rreq) 259 { 260 struct netfs_io_stream *stream = &rreq->io_streams[0]; 261 262 netfs_stat(&netfs_n_rh_retry_read_req); 263 264 /* Wait for all outstanding I/O to quiesce before performing retries as 265 * we may need to renegotiate the I/O sizes. 266 */ 267 set_bit(NETFS_RREQ_RETRYING, &rreq->flags); 268 netfs_wait_for_in_progress_stream(rreq, stream); 269 clear_bit(NETFS_RREQ_RETRYING, &rreq->flags); 270 271 trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); 272 netfs_retry_read_subrequests(rreq); 273 } 274 275 /* 276 * Unlock any the pages that haven't been unlocked yet due to abandoned 277 * subrequests. 278 */ 279 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) 280 { 281 struct folio_queue *p; 282 283 for (p = rreq->buffer.tail; p; p = p->next) { 284 for (int slot = 0; slot < folioq_count(p); slot++) { 285 struct folio *folio = folioq_folio(p, slot); 286 287 if (folio && !folioq_is_marked2(p, slot)) { 288 trace_netfs_folio(folio, netfs_folio_trace_abandon); 289 folio_unlock(folio); 290 } 291 } 292 } 293 } 294