1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Network filesystem read subrequest retrying. 3 * 4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/slab.h> 10 #include "internal.h" 11 12 static void netfs_reissue_read(struct netfs_io_request *rreq, 13 struct netfs_io_subrequest *subreq) 14 { 15 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 16 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); 17 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 18 subreq->rreq->netfs_ops->issue_read(subreq); 19 } 20 21 /* 22 * Go through the list of failed/short reads, retrying all retryable ones. We 23 * need to switch failed cache reads to network downloads. 24 */ 25 static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) 26 { 27 struct netfs_io_subrequest *subreq; 28 struct netfs_io_stream *stream = &rreq->io_streams[0]; 29 struct list_head *next; 30 31 _enter("R=%x", rreq->debug_id); 32 33 if (list_empty(&stream->subrequests)) 34 return; 35 36 if (rreq->netfs_ops->retry_request) 37 rreq->netfs_ops->retry_request(rreq, NULL); 38 39 /* If there's no renegotiation to do, just resend each retryable subreq 40 * up to the first permanently failed one. 41 */ 42 if (!rreq->netfs_ops->prepare_read && 43 !rreq->cache_resources.ops) { 44 list_for_each_entry(subreq, &stream->subrequests, rreq_link) { 45 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) 46 break; 47 if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { 48 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 49 subreq->retry_count++; 50 netfs_reset_iter(subreq); 51 netfs_reissue_read(rreq, subreq); 52 } 53 } 54 return; 55 } 56 57 /* Okay, we need to renegotiate all the download requests and flip any 58 * failed cache reads over to being download requests and negotiate 59 * those also. All fully successful subreqs have been removed from the 60 * list and any spare data from those has been donated. 61 * 62 * What we do is decant the list and rebuild it one subreq at a time so 63 * that we don't end up with donations jumping over a gap we're busy 64 * populating with smaller subrequests. In the event that the subreq 65 * we just launched finishes before we insert the next subreq, it'll 66 * fill in rreq->prev_donated instead. 67 * 68 * Note: Alternatively, we could split the tail subrequest right before 69 * we reissue it and fix up the donations under lock. 70 */ 71 next = stream->subrequests.next; 72 73 do { 74 struct netfs_io_subrequest *from, *to, *tmp; 75 struct iov_iter source; 76 unsigned long long start, len; 77 size_t part; 78 bool boundary = false; 79 80 /* Go through the subreqs and find the next span of contiguous 81 * buffer that we then rejig (cifs, for example, needs the 82 * rsize renegotiating) and reissue. 83 */ 84 from = list_entry(next, struct netfs_io_subrequest, rreq_link); 85 to = from; 86 start = from->start + from->transferred; 87 len = from->len - from->transferred; 88 89 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx", 90 rreq->debug_id, from->debug_index, 91 from->start, from->transferred, from->len); 92 93 if (test_bit(NETFS_SREQ_FAILED, &from->flags) || 94 !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) 95 goto abandon; 96 97 list_for_each_continue(next, &stream->subrequests) { 98 subreq = list_entry(next, struct netfs_io_subrequest, rreq_link); 99 if (subreq->start + subreq->transferred != start + len || 100 test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags) || 101 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 102 break; 103 to = subreq; 104 len += to->len; 105 } 106 107 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len); 108 109 /* Determine the set of buffers we're going to use. Each 110 * subreq gets a subset of a single overall contiguous buffer. 111 */ 112 netfs_reset_iter(from); 113 source = from->io_iter; 114 source.count = len; 115 116 /* Work through the sublist. */ 117 subreq = from; 118 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { 119 if (!len) 120 break; 121 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 122 subreq->start = start - subreq->transferred; 123 subreq->len = len + subreq->transferred; 124 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 125 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 126 subreq->retry_count++; 127 128 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 129 130 /* Renegotiate max_len (rsize) */ 131 stream->sreq_max_len = subreq->len; 132 if (rreq->netfs_ops->prepare_read && 133 rreq->netfs_ops->prepare_read(subreq) < 0) { 134 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 135 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 136 goto abandon; 137 } 138 139 part = umin(len, stream->sreq_max_len); 140 if (unlikely(stream->sreq_max_segs)) 141 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); 142 subreq->len = subreq->transferred + part; 143 subreq->io_iter = source; 144 iov_iter_truncate(&subreq->io_iter, part); 145 iov_iter_advance(&source, part); 146 len -= part; 147 start += part; 148 if (!len) { 149 if (boundary) 150 __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 151 } else { 152 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 153 } 154 155 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 156 netfs_reissue_read(rreq, subreq); 157 if (subreq == to) 158 break; 159 } 160 161 /* If we managed to use fewer subreqs, we can discard the 162 * excess; if we used the same number, then we're done. 163 */ 164 if (!len) { 165 if (subreq == to) 166 continue; 167 list_for_each_entry_safe_from(subreq, tmp, 168 &stream->subrequests, rreq_link) { 169 trace_netfs_sreq(subreq, netfs_sreq_trace_discard); 170 list_del(&subreq->rreq_link); 171 netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_done); 172 if (subreq == to) 173 break; 174 } 175 continue; 176 } 177 178 /* We ran out of subrequests, so we need to allocate some more 179 * and insert them after. 180 */ 181 do { 182 subreq = netfs_alloc_subrequest(rreq); 183 if (!subreq) { 184 subreq = to; 185 goto abandon_after; 186 } 187 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 188 subreq->start = start; 189 subreq->len = len; 190 subreq->debug_index = atomic_inc_return(&rreq->subreq_counter); 191 subreq->stream_nr = stream->stream_nr; 192 subreq->retry_count = 1; 193 194 trace_netfs_sreq_ref(rreq->debug_id, subreq->debug_index, 195 refcount_read(&subreq->ref), 196 netfs_sreq_trace_new); 197 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 198 199 list_add(&subreq->rreq_link, &to->rreq_link); 200 to = list_next_entry(to, rreq_link); 201 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 202 203 stream->sreq_max_len = umin(len, rreq->rsize); 204 stream->sreq_max_segs = 0; 205 if (unlikely(stream->sreq_max_segs)) 206 part = netfs_limit_iter(&source, 0, part, stream->sreq_max_segs); 207 208 netfs_stat(&netfs_n_rh_download); 209 if (rreq->netfs_ops->prepare_read(subreq) < 0) { 210 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 211 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 212 goto abandon; 213 } 214 215 part = umin(len, stream->sreq_max_len); 216 subreq->len = subreq->transferred + part; 217 subreq->io_iter = source; 218 iov_iter_truncate(&subreq->io_iter, part); 219 iov_iter_advance(&source, part); 220 221 len -= part; 222 start += part; 223 if (!len && boundary) { 224 __set_bit(NETFS_SREQ_BOUNDARY, &to->flags); 225 boundary = false; 226 } 227 228 netfs_reissue_read(rreq, subreq); 229 } while (len); 230 231 } while (!list_is_head(next, &stream->subrequests)); 232 233 return; 234 235 /* If we hit an error, fail all remaining incomplete subrequests */ 236 abandon_after: 237 if (list_is_last(&subreq->rreq_link, &stream->subrequests)) 238 return; 239 subreq = list_next_entry(subreq, rreq_link); 240 abandon: 241 list_for_each_entry_from(subreq, &stream->subrequests, rreq_link) { 242 if (!subreq->error && 243 !test_bit(NETFS_SREQ_FAILED, &subreq->flags) && 244 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 245 continue; 246 subreq->error = -ENOMEM; 247 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 248 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 249 } 250 } 251 252 /* 253 * Retry reads. 254 */ 255 void netfs_retry_reads(struct netfs_io_request *rreq) 256 { 257 struct netfs_io_subrequest *subreq; 258 struct netfs_io_stream *stream = &rreq->io_streams[0]; 259 260 /* Wait for all outstanding I/O to quiesce before performing retries as 261 * we may need to renegotiate the I/O sizes. 262 */ 263 list_for_each_entry(subreq, &stream->subrequests, rreq_link) { 264 wait_on_bit(&subreq->flags, NETFS_SREQ_IN_PROGRESS, 265 TASK_UNINTERRUPTIBLE); 266 } 267 268 trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); 269 netfs_retry_read_subrequests(rreq); 270 } 271 272 /* 273 * Unlock any the pages that haven't been unlocked yet due to abandoned 274 * subrequests. 275 */ 276 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) 277 { 278 struct folio_queue *p; 279 280 for (p = rreq->buffer.tail; p; p = p->next) { 281 for (int slot = 0; slot < folioq_count(p); slot++) { 282 struct folio *folio = folioq_folio(p, slot); 283 284 if (folio && !folioq_is_marked2(p, slot)) { 285 trace_netfs_folio(folio, netfs_folio_trace_abandon); 286 folio_unlock(folio); 287 } 288 } 289 } 290 } 291