1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Network filesystem read subrequest retrying. 3 * 4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/slab.h> 10 #include "internal.h" 11 12 static void netfs_reissue_read(struct netfs_io_request *rreq, 13 struct netfs_io_subrequest *subreq) 14 { 15 subreq->error = 0; 16 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 17 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); 18 netfs_stat(&netfs_n_rh_retry_read_subreq); 19 subreq->rreq->netfs_ops->issue_read(subreq); 20 } 21 22 /* 23 * Go through the list of failed/short reads, retrying all retryable ones. We 24 * need to switch failed cache reads to network downloads. 25 */ 26 static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) 27 { 28 struct netfs_io_subrequest *subreq; 29 struct netfs_io_stream *stream = &rreq->io_streams[0]; 30 struct list_head *next; 31 32 _enter("R=%x", rreq->debug_id); 33 34 if (list_empty(&stream->subrequests)) 35 return; 36 37 if (rreq->netfs_ops->retry_request) 38 rreq->netfs_ops->retry_request(rreq, NULL); 39 40 /* If there's no renegotiation to do, just resend each retryable subreq 41 * up to the first permanently failed one. 42 */ 43 if (!rreq->netfs_ops->prepare_read && 44 !rreq->cache_resources.ops) { 45 list_for_each_entry(subreq, &stream->subrequests, rreq_link) { 46 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) 47 break; 48 if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { 49 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 50 subreq->retry_count++; 51 netfs_reset_iter(subreq); 52 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 53 netfs_reissue_read(rreq, subreq); 54 } 55 } 56 return; 57 } 58 59 /* Okay, we need to renegotiate all the download requests and flip any 60 * failed cache reads over to being download requests and negotiate 61 * those also. All fully successful subreqs have been removed from the 62 * list and any spare data from those has been donated. 63 * 64 * What we do is decant the list and rebuild it one subreq at a time so 65 * that we don't end up with donations jumping over a gap we're busy 66 * populating with smaller subrequests. In the event that the subreq 67 * we just launched finishes before we insert the next subreq, it'll 68 * fill in rreq->prev_donated instead. 69 * 70 * Note: Alternatively, we could split the tail subrequest right before 71 * we reissue it and fix up the donations under lock. 72 */ 73 next = stream->subrequests.next; 74 75 do { 76 struct netfs_io_subrequest *from, *to, *tmp; 77 struct iov_iter source; 78 unsigned long long start, len; 79 size_t part; 80 bool boundary = false, subreq_superfluous = false; 81 82 /* Go through the subreqs and find the next span of contiguous 83 * buffer that we then rejig (cifs, for example, needs the 84 * rsize renegotiating) and reissue. 85 */ 86 from = list_entry(next, struct netfs_io_subrequest, rreq_link); 87 to = from; 88 start = from->start + from->transferred; 89 len = from->len - from->transferred; 90 91 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx", 92 rreq->debug_id, from->debug_index, 93 from->start, from->transferred, from->len); 94 95 if (test_bit(NETFS_SREQ_FAILED, &from->flags) || 96 !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) { 97 subreq = from; 98 goto abandon; 99 } 100 101 list_for_each_continue(next, &stream->subrequests) { 102 subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); 103 if (subreq->start + subreq->transferred != start + len || 104 test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || 105 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 106 break; 107 to = subreq; 108 len += to->len; 109 } 110 111 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len); 112 113 /* Determine the set of buffers we're going to use. Each 114 * subreq gets a subset of a single overall contiguous buffer. 115 */ 116 netfs_reset_iter(from); 117 source = from->io_iter; 118 source.count = len; 119 120 /* Work through the sublist. */ 121 subreq = from; 122 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { 123 if (!len) { 124 subreq_superfluous = true; 125 break; 126 } 127 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 128 subreq->start = start - subreq->transferred; 129 subreq->len = len + subreq->transferred; 130 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 131 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 132 subreq->retry_count++; 133 134 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 135 136 /* Renegotiate max_len (rsize) */ 137 stream->sreq_max_len = subreq->len; 138 if (rreq->netfs_ops->prepare_read && 139 rreq->netfs_ops->prepare_read(subreq) < 0) { 140 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 141 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 142 goto abandon; 143 } 144 145 part = umin(len, stream->sreq_max_len); 146 if (unlikely(stream->sreq_max_segs)) 147 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); 148 subreq->len = subreq->transferred + part; 149 subreq->io_iter = source; 150 iov_iter_truncate(&subreq->io_iter, part); 151 iov_iter_advance(&source, part); 152 len -= part; 153 start += part; 154 if (!len) { 155 if (boundary) 156 __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 157 } else { 158 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 159 } 160 161 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 162 netfs_reissue_read(rreq, subreq); 163 if (subreq == to) { 164 subreq_superfluous = false; 165 break; 166 } 167 } 168 169 /* If we managed to use fewer subreqs, we can discard the 170 * excess; if we used the same number, then we're done. 171 */ 172 if (!len) { 173 if (!subreq_superfluous) 174 continue; 175 list_for_each_entry_safe_from(subreq, tmp, 176 &stream->subrequests, rreq_link) { 177 trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous); 178 spin_lock(&rreq->lock); 179 list_del(&subreq->rreq_link); 180 spin_unlock(&rreq->lock); 181 netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); 182 if (subreq == to) 183 break; 184 } 185 subreq = NULL; 186 continue; 187 } 188 189 /* We ran out of subrequests, so we need to allocate some more 190 * and insert them after. 191 */ 192 do { 193 subreq = netfs_alloc_subrequest(rreq); 194 if (!subreq) { 195 subreq = to; 196 goto abandon_after; 197 } 198 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 199 subreq->start = start; 200 subreq->len = len; 201 subreq->stream_nr = stream->stream_nr; 202 subreq->retry_count = 1; 203 204 trace_netfs_sreq_ref(rreq->debug_id, subreq->debug_index, 205 refcount_read(&subreq->ref), 206 netfs_sreq_trace_new); 207 208 spin_lock(&rreq->lock); 209 list_add(&subreq->rreq_link, &to->rreq_link); 210 spin_unlock(&rreq->lock); 211 to = subreq; 212 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 213 214 stream->sreq_max_len = umin(len, rreq->rsize); 215 stream->sreq_max_segs = 0; 216 if (unlikely(stream->sreq_max_segs)) 217 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); 218 219 netfs_stat(&netfs_n_rh_download); 220 if (rreq->netfs_ops->prepare_read(subreq) < 0) { 221 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 222 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 223 goto abandon; 224 } 225 226 part = umin(len, stream->sreq_max_len); 227 subreq->len = subreq->transferred + part; 228 subreq->io_iter = source; 229 iov_iter_truncate(&subreq->io_iter, part); 230 iov_iter_advance(&source, part); 231 232 len -= part; 233 start += part; 234 if (!len && boundary) { 235 __set_bit(NETFS_SREQ_BOUNDARY, &to->flags); 236 boundary = false; 237 } 238 239 netfs_reissue_read(rreq, subreq); 240 } while (len); 241 242 } while (!list_is_head(next, &stream->subrequests)); 243 244 return; 245 246 /* If we hit an error, fail all remaining incomplete subrequests */ 247 abandon_after: 248 if (list_is_last(&subreq->rreq_link, &stream->subrequests)) 249 return; 250 subreq = list_next_entry(subreq, rreq_link); 251 abandon: 252 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { 253 if (!test_bit(NETFS_SREQ_FAILED, &subreq->flags) && 254 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 255 continue; 256 subreq->error = -ENOMEM; 257 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 258 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 259 } 260 } 261 262 /* 263 * Retry reads. 264 */ 265 void netfs_retry_reads(struct netfs_io_request *rreq) 266 { 267 struct netfs_io_stream *stream = &rreq->io_streams[0]; 268 269 netfs_stat(&netfs_n_rh_retry_read_req); 270 271 /* Wait for all outstanding I/O to quiesce before performing retries as 272 * we may need to renegotiate the I/O sizes. 273 */ 274 set_bit(NETFS_RREQ_RETRYING, &rreq->flags); 275 netfs_wait_for_in_progress_stream(rreq, stream); 276 clear_bit(NETFS_RREQ_RETRYING, &rreq->flags); 277 278 trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); 279 netfs_retry_read_subrequests(rreq); 280 } 281 282 /* 283 * Unlock any the pages that haven't been unlocked yet due to abandoned 284 * subrequests. 285 */ 286 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) 287 { 288 struct folio_queue *p; 289 290 for (p = rreq->buffer.tail; p; p = p->next) { 291 for (int slot = 0; slot < folioq_count(p); slot++) { 292 struct folio *folio = folioq_folio(p, slot); 293 294 if (folio && !folioq_is_marked2(p, slot)) { 295 if (folio == rreq->no_unlock_folio && 296 test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, 297 &rreq->flags)) { 298 _debug("no unlock"); 299 } else { 300 trace_netfs_folio(folio, 301 netfs_folio_trace_abandon); 302 folio_unlock(folio); 303 } 304 } 305 } 306 } 307 } 308