1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Network filesystem read subrequest retrying. 3 * 4 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/slab.h> 10 #include "internal.h" 11 12 static void netfs_reissue_read(struct netfs_io_request *rreq, 13 struct netfs_io_subrequest *subreq) 14 { 15 struct iov_iter *io_iter = &subreq->io_iter; 16 17 if (iov_iter_is_folioq(io_iter)) { 18 subreq->curr_folioq = (struct folio_queue *)io_iter->folioq; 19 subreq->curr_folioq_slot = io_iter->folioq_slot; 20 subreq->curr_folio_order = subreq->curr_folioq->orders[subreq->curr_folioq_slot]; 21 } 22 23 atomic_inc(&rreq->nr_outstanding); 24 __set_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags); 25 netfs_get_subrequest(subreq, netfs_sreq_trace_get_resubmit); 26 subreq->rreq->netfs_ops->issue_read(subreq); 27 } 28 29 /* 30 * Go through the list of failed/short reads, retrying all retryable ones. We 31 * need to switch failed cache reads to network downloads. 32 */ 33 static void netfs_retry_read_subrequests(struct netfs_io_request *rreq) 34 { 35 struct netfs_io_subrequest *subreq; 36 struct netfs_io_stream *stream0 = &rreq->io_streams[0]; 37 LIST_HEAD(sublist); 38 LIST_HEAD(queue); 39 40 _enter("R=%x", rreq->debug_id); 41 42 if (list_empty(&rreq->subrequests)) 43 return; 44 45 if (rreq->netfs_ops->retry_request) 46 rreq->netfs_ops->retry_request(rreq, NULL); 47 48 /* If there's no renegotiation to do, just resend each retryable subreq 49 * up to the first permanently failed one. 50 */ 51 if (!rreq->netfs_ops->prepare_read && 52 !rreq->cache_resources.ops) { 53 struct netfs_io_subrequest *subreq; 54 55 list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { 56 if (test_bit(NETFS_SREQ_FAILED, &subreq->flags)) 57 break; 58 if (__test_and_clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { 59 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 60 subreq->retry_count++; 61 netfs_reset_iter(subreq); 62 netfs_reissue_read(rreq, subreq); 63 } 64 } 65 return; 66 } 67 68 /* Okay, we need to renegotiate all the download requests and flip any 69 * failed cache reads over to being download requests and negotiate 70 * those also. All fully successful subreqs have been removed from the 71 * list and any spare data from those has been donated. 72 * 73 * What we do is decant the list and rebuild it one subreq at a time so 74 * that we don't end up with donations jumping over a gap we're busy 75 * populating with smaller subrequests. In the event that the subreq 76 * we just launched finishes before we insert the next subreq, it'll 77 * fill in rreq->prev_donated instead. 78 79 * Note: Alternatively, we could split the tail subrequest right before 80 * we reissue it and fix up the donations under lock. 81 */ 82 list_splice_init(&rreq->subrequests, &queue); 83 84 do { 85 struct netfs_io_subrequest *from; 86 struct iov_iter source; 87 unsigned long long start, len; 88 size_t part, deferred_next_donated = 0; 89 bool boundary = false; 90 91 /* Go through the subreqs and find the next span of contiguous 92 * buffer that we then rejig (cifs, for example, needs the 93 * rsize renegotiating) and reissue. 94 */ 95 from = list_first_entry(&queue, struct netfs_io_subrequest, rreq_link); 96 list_move_tail(&from->rreq_link, &sublist); 97 start = from->start + from->transferred; 98 len = from->len - from->transferred; 99 100 _debug("from R=%08x[%x] s=%llx ctl=%zx/%zx/%zx", 101 rreq->debug_id, from->debug_index, 102 from->start, from->consumed, from->transferred, from->len); 103 104 if (test_bit(NETFS_SREQ_FAILED, &from->flags) || 105 !test_bit(NETFS_SREQ_NEED_RETRY, &from->flags)) 106 goto abandon; 107 108 deferred_next_donated = from->next_donated; 109 while ((subreq = list_first_entry_or_null( 110 &queue, struct netfs_io_subrequest, rreq_link))) { 111 if (subreq->start != start + len || 112 subreq->transferred > 0 || 113 !test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) 114 break; 115 list_move_tail(&subreq->rreq_link, &sublist); 116 len += subreq->len; 117 deferred_next_donated = subreq->next_donated; 118 if (test_bit(NETFS_SREQ_BOUNDARY, &subreq->flags)) 119 break; 120 } 121 122 _debug(" - range: %llx-%llx %llx", start, start + len - 1, len); 123 124 /* Determine the set of buffers we're going to use. Each 125 * subreq gets a subset of a single overall contiguous buffer. 126 */ 127 netfs_reset_iter(from); 128 source = from->io_iter; 129 source.count = len; 130 131 /* Work through the sublist. */ 132 while ((subreq = list_first_entry_or_null( 133 &sublist, struct netfs_io_subrequest, rreq_link))) { 134 list_del(&subreq->rreq_link); 135 136 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 137 subreq->start = start - subreq->transferred; 138 subreq->len = len + subreq->transferred; 139 stream0->sreq_max_len = subreq->len; 140 141 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 142 __clear_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); 143 subreq->retry_count++; 144 145 spin_lock_bh(&rreq->lock); 146 list_add_tail(&subreq->rreq_link, &rreq->subrequests); 147 subreq->prev_donated += rreq->prev_donated; 148 rreq->prev_donated = 0; 149 trace_netfs_sreq(subreq, netfs_sreq_trace_retry); 150 spin_unlock_bh(&rreq->lock); 151 152 BUG_ON(!len); 153 154 /* Renegotiate max_len (rsize) */ 155 if (rreq->netfs_ops->prepare_read(subreq) < 0) { 156 trace_netfs_sreq(subreq, netfs_sreq_trace_reprep_failed); 157 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 158 } 159 160 part = umin(len, stream0->sreq_max_len); 161 if (unlikely(rreq->io_streams[0].sreq_max_segs)) 162 part = netfs_limit_iter(&source, 0, part, stream0->sreq_max_segs); 163 subreq->len = subreq->transferred + part; 164 subreq->io_iter = source; 165 iov_iter_truncate(&subreq->io_iter, part); 166 iov_iter_advance(&source, part); 167 len -= part; 168 start += part; 169 if (!len) { 170 if (boundary) 171 __set_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 172 subreq->next_donated = deferred_next_donated; 173 } else { 174 __clear_bit(NETFS_SREQ_BOUNDARY, &subreq->flags); 175 subreq->next_donated = 0; 176 } 177 178 netfs_reissue_read(rreq, subreq); 179 if (!len) 180 break; 181 182 /* If we ran out of subrequests, allocate another. */ 183 if (list_empty(&sublist)) { 184 subreq = netfs_alloc_subrequest(rreq); 185 if (!subreq) 186 goto abandon; 187 subreq->source = NETFS_DOWNLOAD_FROM_SERVER; 188 subreq->start = start; 189 190 /* We get two refs, but need just one. */ 191 netfs_put_subrequest(subreq, false, netfs_sreq_trace_new); 192 trace_netfs_sreq(subreq, netfs_sreq_trace_split); 193 list_add_tail(&subreq->rreq_link, &sublist); 194 } 195 } 196 197 /* If we managed to use fewer subreqs, we can discard the 198 * excess. 199 */ 200 while ((subreq = list_first_entry_or_null( 201 &sublist, struct netfs_io_subrequest, rreq_link))) { 202 trace_netfs_sreq(subreq, netfs_sreq_trace_discard); 203 list_del(&subreq->rreq_link); 204 netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_done); 205 } 206 207 } while (!list_empty(&queue)); 208 209 return; 210 211 /* If we hit ENOMEM, fail all remaining subrequests */ 212 abandon: 213 list_splice_init(&sublist, &queue); 214 list_for_each_entry(subreq, &queue, rreq_link) { 215 if (!subreq->error) 216 subreq->error = -ENOMEM; 217 __clear_bit(NETFS_SREQ_FAILED, &subreq->flags); 218 __clear_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 219 } 220 spin_lock_bh(&rreq->lock); 221 list_splice_tail_init(&queue, &rreq->subrequests); 222 spin_unlock_bh(&rreq->lock); 223 } 224 225 /* 226 * Retry reads. 227 */ 228 void netfs_retry_reads(struct netfs_io_request *rreq) 229 { 230 trace_netfs_rreq(rreq, netfs_rreq_trace_resubmit); 231 232 atomic_inc(&rreq->nr_outstanding); 233 234 netfs_retry_read_subrequests(rreq); 235 236 if (atomic_dec_and_test(&rreq->nr_outstanding)) 237 netfs_rreq_terminated(rreq, false); 238 } 239 240 /* 241 * Unlock any the pages that haven't been unlocked yet due to abandoned 242 * subrequests. 243 */ 244 void netfs_unlock_abandoned_read_pages(struct netfs_io_request *rreq) 245 { 246 struct folio_queue *p; 247 248 for (p = rreq->buffer; p; p = p->next) { 249 for (int slot = 0; slot < folioq_count(p); slot++) { 250 struct folio *folio = folioq_folio(p, slot); 251 252 if (folio && !folioq_is_marked2(p, slot)) { 253 trace_netfs_folio(folio, netfs_folio_trace_abandon); 254 folio_unlock(folio); 255 } 256 } 257 } 258 } 259