1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Network filesystem read subrequest retrying. 3 * 4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/slab.h> 10 #include "internal.h" 11 12 static void netfs_reissue_read(struct netfs_io_request *rreq, 13 struct netfs_io_subrequest *subreq) 14 { 15 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 16 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); 17 netfs_stat(&netfs_n_rh_retry_read_subreq); 18 subreq->rreq->netfs_ops->issue_read(subreq); 19 } 20 21 /* 22 * Go through the list of failed/short reads, retrying all retryable ones. We 23 * need to switch failed cache reads to network downloads. 24 */ 25 static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) 26 { 27 struct netfs_io_subrequest *subreq; 28 struct netfs_io_stream *stream = &rreq->io_streams[0]; 29 struct list_head *next; 30 31 _enter("R=%x", rreq->debug_id); 32 33 if (list_empty(&stream->subrequests)) 34 return; 35 36 if (rreq->netfs_ops->retry_request) 37 rreq->netfs_ops->retry_request(rreq, NULL); 38 39 /* If there's no renegotiation to do, just resend each retryable subreq 40 * up to the first permanently failed one. 41 */ 42 if (!rreq->netfs_ops->prepare_read && 43 !rreq->cache_resources.ops) { 44 list_for_each_entry(subreq, &stream->subrequests, rreq_link) { 45 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) 46 break; 47 if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { 48 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 49 subreq->retry_count++; 50 netfs_reset_iter(subreq); 51 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 52 netfs_reissue_read(rreq, subreq); 53 } 54 } 55 return; 56 } 57 58 /* Okay, we need to renegotiate all the download requests and flip any 59 * failed cache reads over to being download requests and negotiate 60 * those also. All fully successful subreqs have been removed from the 61 * list and any spare data from those has been donated. 62 * 63 * What we do is decant the list and rebuild it one subreq at a time so 64 * that we don't end up with donations jumping over a gap we're busy 65 * populating with smaller subrequests. In the event that the subreq 66 * we just launched finishes before we insert the next subreq, it'll 67 * fill in rreq->prev_donated instead. 68 * 69 * Note: Alternatively, we could split the tail subrequest right before 70 * we reissue it and fix up the donations under lock. 71 */ 72 next = stream->subrequests.next; 73 74 do { 75 struct netfs_io_subrequest *from, *to, *tmp; 76 struct iov_iter source; 77 unsigned long long start, len; 78 size_t part; 79 bool boundary = false, subreq_superfluous = false; 80 81 /* Go through the subreqs and find the next span of contiguous 82 * buffer that we then rejig (cifs, for example, needs the 83 * rsize renegotiating) and reissue. 84 */ 85 from = list_entry(next, struct netfs_io_subrequest, rreq_link); 86 to = from; 87 start = from->start + from->transferred; 88 len = from->len - from->transferred; 89 90 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx", 91 rreq->debug_id, from->debug_index, 92 from->start, from->transferred, from->len); 93 94 if (test_bit(NETFS_SREQ_FAILED, &from->flags) || 95 !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) 96 goto abandon; 97 98 list_for_each_continue(next, &stream->subrequests) { 99 subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); 100 if (subreq->start + subreq->transferred != start + len || 101 test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || 102 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 103 break; 104 to = subreq; 105 len += to->len; 106 } 107 108 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len); 109 110 /* Determine the set of buffers we're going to use. Each 111 * subreq gets a subset of a single overall contiguous buffer. 112 */ 113 netfs_reset_iter(from); 114 source = from->io_iter; 115 source.count = len; 116 117 /* Work through the sublist. */ 118 subreq = from; 119 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { 120 if (!len) { 121 subreq_superfluous = true; 122 break; 123 } 124 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 125 subreq->start = start - subreq->transferred; 126 subreq->len = len + subreq->transferred; 127 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 128 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 129 subreq->retry_count++; 130 131 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 132 133 /* Renegotiate max_len (rsize) */ 134 stream->sreq_max_len = subreq->len; 135 if (rreq->netfs_ops->prepare_read && 136 rreq->netfs_ops->prepare_read(subreq) < 0) { 137 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 138 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 139 goto abandon; 140 } 141 142 part = umin(len, stream->sreq_max_len); 143 if (unlikely(stream->sreq_max_segs)) 144 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); 145 subreq->len = subreq->transferred + part; 146 subreq->io_iter = source; 147 iov_iter_truncate(&subreq->io_iter, part); 148 iov_iter_advance(&source, part); 149 len -= part; 150 start += part; 151 if (!len) { 152 if (boundary) 153 __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 154 } else { 155 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 156 } 157 158 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 159 netfs_reissue_read(rreq, subreq); 160 if (subreq == to) { 161 subreq_superfluous = false; 162 break; 163 } 164 } 165 166 /* If we managed to use fewer subreqs, we can discard the 167 * excess; if we used the same number, then we're done. 168 */ 169 if (!len) { 170 if (!subreq_superfluous) 171 continue; 172 list_for_each_entry_safe_from(subreq, tmp, 173 &stream->subrequests, rreq_link) { 174 trace_netfs_sreq(subreq, netfs_sreq_trace_superfluous); 175 list_del(&subreq->rreq_link); 176 netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_done); 177 if (subreq == to) 178 break; 179 } 180 continue; 181 } 182 183 /* We ran out of subrequests, so we need to allocate some more 184 * and insert them after. 185 */ 186 do { 187 subreq = netfs_alloc_subrequest(rreq); 188 if (!subreq) { 189 subreq = to; 190 goto abandon_after; 191 } 192 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 193 subreq->start = start; 194 subreq->len = len; 195 subreq->stream_nr = stream->stream_nr; 196 subreq->retry_count = 1; 197 198 trace_netfs_sreq_ref(rreq->debug_id, subreq->debug_index, 199 refcount_read(&subreq->ref), 200 netfs_sreq_trace_new); 201 202 list_add(&subreq->rreq_link, &to->rreq_link); 203 to = list_next_entry(to, rreq_link); 204 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 205 206 stream->sreq_max_len = umin(len, rreq->rsize); 207 stream->sreq_max_segs = 0; 208 if (unlikely(stream->sreq_max_segs)) 209 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); 210 211 netfs_stat(&netfs_n_rh_download); 212 if (rreq->netfs_ops->prepare_read(subreq) < 0) { 213 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 214 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 215 goto abandon; 216 } 217 218 part = umin(len, stream->sreq_max_len); 219 subreq->len = subreq->transferred + part; 220 subreq->io_iter = source; 221 iov_iter_truncate(&subreq->io_iter, part); 222 iov_iter_advance(&source, part); 223 224 len -= part; 225 start += part; 226 if (!len && boundary) { 227 __set_bit(NETFS_SREQ_BOUNDARY, &to->flags); 228 boundary = false; 229 } 230 231 netfs_reissue_read(rreq, subreq); 232 } while (len); 233 234 } while (!list_is_head(next, &stream->subrequests)); 235 236 return; 237 238 /* If we hit an error, fail all remaining incomplete subrequests */ 239 abandon_after: 240 if (list_is_last(&subreq->rreq_link, &stream->subrequests)) 241 return; 242 subreq = list_next_entry(subreq, rreq_link); 243 abandon: 244 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { 245 if (!subreq->error && 246 !test_bit(NETFS_SREQ_FAILED, &subreq->flags) && 247 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 248 continue; 249 subreq->error = -ENOMEM; 250 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 251 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 252 } 253 } 254 255 /* 256 * Retry reads. 257 */ 258 void netfs_retry_reads(struct netfs_io_request *rreq) 259 { 260 struct netfs_io_subrequest *subreq; 261 struct netfs_io_stream *stream = &rreq->io_streams[0]; 262 DEFINE_WAIT(myself); 263 264 netfs_stat(&netfs_n_rh_retry_read_req); 265 266 set_bit(NETFS_RREQ_RETRYING, &rreq->flags); 267 268 /* Wait for all outstanding I/O to quiesce before performing retries as 269 * we may need to renegotiate the I/O sizes. 270 */ 271 list_for_each_entry(subreq, &stream->subrequests, rreq_link) { 272 if (!test_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags)) 273 continue; 274 275 trace_netfs_rreq(rreq, netfs_rreq_trace_wait_queue); 276 for (;;) { 277 prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE); 278 279 if (!test_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags)) 280 break; 281 282 trace_netfs_sreq(subreq, netfs_sreq_trace_wait_for); 283 schedule(); 284 trace_netfs_rreq(rreq, netfs_rreq_trace_woke_queue); 285 } 286 287 finish_wait(&rreq->waitq, &myself); 288 } 289 clear_bit(NETFS_RREQ_RETRYING, &rreq->flags); 290 291 trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); 292 netfs_retry_read_subrequests(rreq); 293 } 294 295 /* 296 * Unlock any the pages that haven't been unlocked yet due to abandoned 297 * subrequests. 298 */ 299 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) 300 { 301 struct folio_queue *p; 302 303 for (p = rreq->buffer.tail; p; p = p->next) { 304 for (int slot = 0; slot < folioq_count(p); slot++) { 305 struct folio *folio = folioq_folio(p, slot); 306 307 if (folio && !folioq_is_marked2(p, slot)) { 308 trace_netfs_folio(folio, netfs_folio_trace_abandon); 309 folio_unlock(folio); 310 } 311 } 312 } 313 } 314