1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Network filesystem read subrequest retrying. 3 * 4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/slab.h> 10 #include "internal.h" 11 12 static void netfs_reissue_read(struct netfs_io_request *rreq, 13 struct netfs_io_subrequest *subreq) 14 { 15 subreq->error = 0; 16 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 17 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); 18 netfs_stat(&netfs_n_rh_retry_read_subreq); 19 subreq->rreq->netfs_ops->issue_read(subreq); 20 } 21 22 /* 23 * Go through the list of failed/short reads, retrying all retryable ones. We 24 * need to switch failed cache reads to network downloads. 25 */ 26 static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) 27 { 28 struct netfs_io_subrequest *subreq; 29 struct netfs_io_stream *stream = &rreq->io_streams[0]; 30 struct list_head *next; 31 32 _enter("R=%x", rreq->debug_id); 33 34 if (list_empty(&stream->subrequests)) 35 return; 36 37 if (rreq->netfs_ops->retry_request) 38 rreq->netfs_ops->retry_request(rreq, NULL); 39 40 /* If there's no renegotiation to do, just resend each retryable subreq 41 * up to the first permanently failed one. 42 */ 43 if (!rreq->netfs_ops->prepare_read && 44 !rreq->cache_resources.ops) { 45 list_for_each_entry(subreq, &stream->subrequests, rreq_link) { 46 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) 47 break; 48 if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { 49 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 50 subreq->retry_count++; 51 netfs_reset_iter(subreq); 52 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 53 netfs_reissue_read(rreq, subreq); 54 } 55 } 56 return; 57 } 58 59 /* Okay, we need to renegotiate all the download requests and flip any 60 * failed cache reads over to being download requests and negotiate 61 * those also. All fully successful subreqs have been removed from the 62 * list and any spare data from those has been donated. 63 * 64 * What we do is decant the list and rebuild it one subreq at a time so 65 * that we don't end up with donations jumping over a gap we're busy 66 * populating with smaller subrequests. In the event that the subreq 67 * we just launched finishes before we insert the next subreq, it'll 68 * fill in rreq->prev_donated instead. 69 * 70 * Note: Alternatively, we could split the tail subrequest right before 71 * we reissue it and fix up the donations under lock. 72 */ 73 next = stream->subrequests.next; 74 75 do { 76 struct netfs_io_subrequest *from, *to, *tmp; 77 struct iov_iter source; 78 unsigned long long start, len; 79 size_t part; 80 bool boundary = false, subreq_superfluous = false; 81 82 /* Go through the subreqs and find the next span of contiguous 83 * buffer that we then rejig (cifs, for example, needs the 84 * rsize renegotiating) and reissue. 85 */ 86 from = list_entry(next, struct netfs_io_subrequest, rreq_link); 87 to = from; 88 start = from->start + from->transferred; 89 len = from->len - from->transferred; 90 91 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx", 92 rreq->debug_id, from->debug_index, 93 from->start, from->transferred, from->len); 94 95 if (test_bit(NETFS_SREQ_FAILED, &from->flags) || 96 !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) { 97 subreq = from; 98 goto abandon; 99 } 100 101 for (;;) { 102 /* Read pointer to subreq before reading subreq state. */ 103 next = smp_load_acquire(&next->next); 104 if (next == &stream->subrequests) 105 break; 106 107 subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); 108 if (subreq->start + subreq->transferred != start + len || 109 test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || 110 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 111 break; 112 to = subreq; 113 len += to->len; 114 } 115 116 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len); 117 118 /* Determine the set of buffers we're going to use. Each 119 * subreq gets a subset of a single overall contiguous buffer. 120 */ 121 netfs_reset_iter(from); 122 source = from->io_iter; 123 source.count = len; 124 125 /* Work through the sublist. */ 126 subreq = from; 127 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { 128 if (!len) { 129 subreq_superfluous = true; 130 break; 131 } 132 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 133 subreq->start = start - subreq->transferred; 134 subreq->len = len + subreq->transferred; 135 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 136 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 137 subreq->retry_count++; 138 139 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 140 141 /* Renegotiate max_len (rsize) */ 142 stream->sreq_max_len = subreq->len; 143 if (rreq->netfs_ops->prepare_read && 144 rreq->netfs_ops->prepare_read(subreq) < 0) { 145 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 146 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 147 goto abandon; 148 } 149 150 part = umin(len, stream->sreq_max_len); 151 if (unlikely(stream->sreq_max_segs)) 152 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); 153 subreq->len = subreq->transferred + part; 154 subreq->io_iter = source; 155 iov_iter_truncate(&subreq->io_iter, part); 156 iov_iter_advance(&source, part); 157 len -= part; 158 start += part; 159 if (!len) { 160 if (boundary) 161 __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 162 } else { 163 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 164 } 165 166 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 167 netfs_reissue_read(rreq, subreq); 168 if (subreq == to) { 169 subreq_superfluous = false; 170 break; 171 } 172 } 173 174 /* If we managed to use fewer subreqs, we can discard the 175 * excess; if we used the same number, then we're done. 176 */ 177 if (!len) { 178 if (!subreq_superfluous) 179 continue; 180 list_for_each_entry_safe_from(subreq, tmp, 181 &stream->subrequests, rreq_link) { 182 trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous); 183 spin_lock(&rreq->lock); 184 list_del(&subreq->rreq_link); 185 spin_unlock(&rreq->lock); 186 netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); 187 if (subreq == to) 188 break; 189 } 190 subreq = NULL; 191 continue; 192 } 193 194 /* We ran out of subrequests, so we need to allocate some more 195 * and insert them after. 196 */ 197 do { 198 subreq = netfs_alloc_subrequest(rreq); 199 if (!subreq) { 200 subreq = to; 201 goto abandon_after; 202 } 203 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 204 subreq->start = start; 205 subreq->len = len; 206 subreq->stream_nr = stream->stream_nr; 207 subreq->retry_count = 1; 208 209 trace_netfs_sreq_ref(rreq->debug_id, subreq->debug_index, 210 refcount_read(&subreq->ref), 211 netfs_sreq_trace_new); 212 213 spin_lock(&rreq->lock); 214 list_add(&subreq->rreq_link, &to->rreq_link); 215 spin_unlock(&rreq->lock); 216 to = subreq; 217 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 218 219 stream->sreq_max_len = umin(len, rreq->rsize); 220 stream->sreq_max_segs = 0; 221 if (unlikely(stream->sreq_max_segs)) 222 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); 223 224 netfs_stat(&netfs_n_rh_download); 225 if (rreq->netfs_ops->prepare_read(subreq) < 0) { 226 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 227 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 228 goto abandon; 229 } 230 231 part = umin(len, stream->sreq_max_len); 232 subreq->len = subreq->transferred + part; 233 subreq->io_iter = source; 234 iov_iter_truncate(&subreq->io_iter, part); 235 iov_iter_advance(&source, part); 236 237 len -= part; 238 start += part; 239 if (!len && boundary) { 240 __set_bit(NETFS_SREQ_BOUNDARY, &to->flags); 241 boundary = false; 242 } 243 244 netfs_reissue_read(rreq, subreq); 245 } while (len); 246 247 } while (!list_is_head(next, &stream->subrequests)); 248 249 return; 250 251 /* If we hit an error, fail all remaining incomplete subrequests */ 252 abandon_after: 253 if (list_is_last(&subreq->rreq_link, &stream->subrequests)) 254 return; 255 subreq = list_next_entry(subreq, rreq_link); 256 abandon: 257 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { 258 if (!test_bit(NETFS_SREQ_FAILED, &subreq->flags) && 259 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 260 continue; 261 subreq->error = -ENOMEM; 262 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 263 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 264 } 265 } 266 267 /* 268 * Retry reads. 269 */ 270 void netfs_retry_reads(struct netfs_io_request *rreq) 271 { 272 struct netfs_io_stream *stream = &rreq->io_streams[0]; 273 274 netfs_stat(&netfs_n_rh_retry_read_req); 275 276 /* Wait for all outstanding I/O to quiesce before performing retries as 277 * we may need to renegotiate the I/O sizes. 278 */ 279 set_bit(NETFS_RREQ_RETRYING, &rreq->flags); 280 netfs_wait_for_in_progress_stream(rreq, stream); 281 clear_bit(NETFS_RREQ_RETRYING, &rreq->flags); 282 283 trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); 284 netfs_retry_read_subrequests(rreq); 285 } 286 287 /* 288 * Unlock any the pages that haven't been unlocked yet due to abandoned 289 * subrequests. 290 */ 291 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) 292 { 293 struct folio_queue *p; 294 295 for (p = rreq->buffer.tail; p; p = p->next) { 296 for (int slot = 0; slot < folioq_count(p); slot++) { 297 struct folio *folio = folioq_folio(p, slot); 298 299 if (folio && !folioq_is_marked2(p, slot)) { 300 if (folio == rreq->no_unlock_folio && 301 test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, 302 &rreq->flags)) { 303 _debug("no unlock"); 304 } else { 305 trace_netfs_folio(folio, 306 netfs_folio_trace_abandon); 307 folio_unlock(folio); 308 } 309 } 310 } 311 } 312 } 313