1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Network filesystem read subrequest retrying. 3 * 4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/slab.h> 10 #include "internal.h" 11 12 static void netfs_reissue_read(struct netfs_io_request *rreq, 13 struct netfs_io_subrequest *subreq) 14 { 15 subreq->error = 0; 16 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 17 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); 18 netfs_stat(&netfs_n_rh_retry_read_subreq); 19 subreq->rreq->netfs_ops->issue_read(subreq); 20 } 21 22 /* 23 * Go through the list of failed/short reads, retrying all retryable ones. We 24 * need to switch failed cache reads to network downloads. 25 */ 26 static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) 27 { 28 struct netfs_io_subrequest *subreq; 29 struct netfs_io_stream *stream = &rreq->io_streams[0]; 30 struct list_head *next; 31 32 _enter("R=%x", rreq->debug_id); 33 34 if (list_empty(&stream->subrequests)) 35 return; 36 37 if (rreq->netfs_ops->retry_request) 38 rreq->netfs_ops->retry_request(rreq, NULL); 39 40 /* If there's no renegotiation to do, just resend each retryable subreq 41 * up to the first permanently failed one. 42 */ 43 if (!rreq->netfs_ops->prepare_read && 44 !rreq->cache_resources.ops) { 45 list_for_each_entry(subreq, &stream->subrequests, rreq_link) { 46 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) 47 break; 48 if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { 49 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 50 subreq->retry_count++; 51 netfs_reset_iter(subreq); 52 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 53 netfs_reissue_read(rreq, subreq); 54 } 55 } 56 return; 57 } 58 59 /* Okay, we need to renegotiate all the download requests and flip any 60 * failed cache reads over to being download requests and negotiate 61 * those also. All fully successful subreqs have been removed from the 62 * list and any spare data from those has been donated. 63 * 64 * What we do is decant the list and rebuild it one subreq at a time so 65 * that we don't end up with donations jumping over a gap we're busy 66 * populating with smaller subrequests. In the event that the subreq 67 * we just launched finishes before we insert the next subreq, it'll 68 * fill in rreq->prev_donated instead. 69 * 70 * Note: Alternatively, we could split the tail subrequest right before 71 * we reissue it and fix up the donations under lock. 72 */ 73 next = stream->subrequests.next; 74 75 do { 76 struct netfs_io_subrequest *from, *to, *tmp; 77 struct iov_iter source; 78 uoff_t start, len; 79 size_t part; 80 bool boundary = false, subreq_superfluous = false; 81 82 /* Go through the subreqs and find the next span of contiguous 83 * buffer that we then rejig (cifs, for example, needs the 84 * rsize renegotiating) and reissue. 85 */ 86 from = list_entry(next, struct netfs_io_subrequest, rreq_link); 87 to = from; 88 start = from->start + from->transferred; 89 len = from->len - from->transferred; 90 91 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx", 92 rreq->debug_id, from->debug_index, 93 from->start, from->transferred, from->len); 94 95 if (test_bit(NETFS_SREQ_FAILED, &from->flags) || 96 !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) { 97 subreq = from; 98 goto abandon; 99 } 100 101 for (;;) { 102 /* Read pointer to subreq before reading subreq state. */ 103 next = smp_load_acquire(&next->next); 104 if (next == &stream->subrequests) 105 break; 106 107 subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); 108 if (subreq->start + subreq->transferred != start + len || 109 test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || 110 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 111 break; 112 to = subreq; 113 len += to->len; 114 } 115 116 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len); 117 118 /* Determine the set of buffers we're going to use. Each 119 * subreq gets a subset of a single overall contiguous buffer. 120 */ 121 netfs_reset_iter(from); 122 source = from->io_iter; 123 source.count = len; 124 125 /* Work through the sublist. */ 126 subreq = from; 127 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { 128 if (!len) { 129 subreq_superfluous = true; 130 break; 131 } 132 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 133 subreq->start = start - subreq->transferred; 134 subreq->len = len + subreq->transferred; 135 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 136 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 137 subreq->retry_count++; 138 139 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 140 141 /* Renegotiate max_len (rsize) */ 142 stream->sreq_max_len = subreq->len; 143 if (rreq->netfs_ops->prepare_read && 144 rreq->netfs_ops->prepare_read(subreq) < 0) { 145 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 146 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 147 goto abandon; 148 } 149 150 part = umin(len, stream->sreq_max_len); 151 if (unlikely(stream->sreq_max_segs)) 152 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); 153 subreq->len = subreq->transferred + part; 154 subreq->io_iter = source; 155 iov_iter_truncate(&subreq->io_iter, part); 156 iov_iter_advance(&source, part); 157 len -= part; 158 start += part; 159 if (!len) { 160 if (boundary) 161 __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 162 } else { 163 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 164 } 165 166 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 167 netfs_reissue_read(rreq, subreq); 168 if (subreq == to) { 169 subreq_superfluous = false; 170 break; 171 } 172 } 173 174 /* If we managed to use fewer subreqs, we can discard the 175 * excess; if we used the same number, then we're done. 176 */ 177 if (!len) { 178 if (!subreq_superfluous) 179 continue; 180 list_for_each_entry_safe_from(subreq, tmp, 181 &stream->subrequests, rreq_link) { 182 trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous); 183 spin_lock(&rreq->lock); 184 list_del(&subreq->rreq_link); 185 spin_unlock(&rreq->lock); 186 netfs_put_subrequest(subreq, netfs_sreq_trace_put_done); 187 if (subreq == to) 188 break; 189 } 190 subreq = NULL; 191 continue; 192 } 193 194 /* We ran out of subrequests, so we need to allocate some more 195 * and insert them after. 196 */ 197 do { 198 subreq = netfs_alloc_subrequest(rreq, NETFS_DOWNLOAD_FROM_SERVER); 199 if (!subreq) { 200 subreq = to; 201 goto abandon_after; 202 } 203 subreq->start = start; 204 subreq->len = len; 205 subreq->stream_nr = stream->stream_nr; 206 subreq->retry_count = 1; 207 208 trace_netfs_sreq_ref(rreq->debug_id, subreq->debug_index, 209 refcount_read(&subreq->ref), 210 netfs_sreq_trace_new); 211 212 spin_lock(&rreq->lock); 213 list_add(&subreq->rreq_link, &to->rreq_link); 214 spin_unlock(&rreq->lock); 215 to = subreq; 216 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 217 218 stream->sreq_max_len = umin(len, rreq->rsize); 219 stream->sreq_max_segs = 0; 220 if (unlikely(stream->sreq_max_segs)) 221 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); 222 223 netfs_stat(&netfs_n_rh_download); 224 if (rreq->netfs_ops->prepare_read(subreq) < 0) { 225 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 226 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 227 goto abandon; 228 } 229 230 part = umin(len, stream->sreq_max_len); 231 subreq->len = subreq->transferred + part; 232 subreq->io_iter = source; 233 iov_iter_truncate(&subreq->io_iter, part); 234 iov_iter_advance(&source, part); 235 236 len -= part; 237 start += part; 238 if (!len && boundary) { 239 __set_bit(NETFS_SREQ_BOUNDARY, &to->flags); 240 boundary = false; 241 } 242 243 netfs_reissue_read(rreq, subreq); 244 } while (len); 245 246 } while (!list_is_head(next, &stream->subrequests)); 247 248 return; 249 250 /* If we hit an error, fail all remaining incomplete subrequests */ 251 abandon_after: 252 if (list_is_last(&subreq->rreq_link, &stream->subrequests)) 253 return; 254 subreq = list_next_entry(subreq, rreq_link); 255 abandon: 256 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { 257 if (!test_bit(NETFS_SREQ_FAILED, &subreq->flags) && 258 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 259 continue; 260 subreq->error = -ENOMEM; 261 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 262 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 263 } 264 } 265 266 /* 267 * Retry reads. 268 */ 269 void netfs_retry_reads(struct netfs_io_request *rreq) 270 { 271 struct netfs_io_stream *stream = &rreq->io_streams[0]; 272 273 netfs_stat(&netfs_n_rh_retry_read_req); 274 trace_netfs_rreq(rreq, netfs_rreq_trace_retry_begin); 275 276 /* Wait for all outstanding I/O to quiesce before performing retries as 277 * we may need to renegotiate the I/O sizes. 278 */ 279 set_bit(NETFS_RREQ_RETRYING, &rreq->flags); 280 netfs_wait_for_in_progress_stream(rreq, stream); 281 clear_bit(NETFS_RREQ_RETRYING, &rreq->flags); 282 283 trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); 284 netfs_retry_read_subrequests(rreq); 285 trace_netfs_rreq(rreq, netfs_rreq_trace_retry_end); 286 } 287 288 /* 289 * Unlock any the pages that haven't been unlocked yet due to abandoned 290 * subrequests. 291 */ 292 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) 293 { 294 struct folio_queue *p; 295 296 /* We have to wait for readahead refs to have been released before we 297 * can unlock any folios as the ref-dropper walks i_pages and the only 298 * thing preventing these folios from being removed is the folio lock. 299 */ 300 if (test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags)) 301 netfs_wait_for_put_ra_refs(rreq); 302 303 for (p = rreq->buffer.tail; p; p = p->next) { 304 for (int slot = 0; slot < folioq_count(p); slot++) { 305 struct folio *folio = folioq_folio(p, slot); 306 307 if (!folio) 308 continue; 309 netfs_cancel_copy_to_cache(rreq, folio); 310 311 if (!folioq_is_marked2(p, slot)) { 312 if (folio == rreq->no_unlock_folio && 313 test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, 314 &rreq->flags)) { 315 _debug("no unlock"); 316 } else { 317 trace_netfs_folio(folio, 318 netfs_folio_trace_abandon); 319 folio_unlock(folio); 320 } 321 } 322 } 323 } 324 } 325