1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Network filesystem read subrequest result collection, assessment and 3 * retrying. 4 * 5 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved. 6 * Written by David Howells (dhowells@redhat.com) 7 */ 8 9 #include <linux/export.h> 10 #include <linux/fs.h> 11 #include <linux/mm.h> 12 #include <linux/pagemap.h> 13 #include <linux/slab.h> 14 #include <linux/task_io_accounting_ops.h> 15 #include "internal.h" 16 17 /* 18 * Clear the unread part of an I/O request. 19 */ 20 static void netfs_clear_unread(struct netfs_io_subrequest *subreq) 21 { 22 netfs_reset_iter(subreq); 23 WARN_ON_ONCE(subreq->len - subreq->transferred != iov_iter_count(&subreq->io_iter)); 24 iov_iter_zero(iov_iter_count(&subreq->io_iter), &subreq->io_iter); 25 if (subreq->start + subreq->transferred >= subreq->rreq->i_size) 26 __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); 27 } 28 29 /* 30 * Flush, mark and unlock a folio that's now completely read. If we want to 31 * cache the folio, we set the group to NETFS_FOLIO_COPY_TO_CACHE, mark it 32 * dirty and let writeback handle it. 33 */ 34 static void netfs_unlock_read_folio(struct netfs_io_subrequest *subreq, 35 struct netfs_io_request *rreq, 36 struct folio_queue *folioq, 37 int slot) 38 { 39 struct netfs_folio *finfo; 40 struct folio *folio = folioq_folio(folioq, slot); 41 42 flush_dcache_folio(folio); 43 folio_mark_uptodate(folio); 44 45 if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) { 46 finfo = netfs_folio_info(folio); 47 if (finfo) { 48 trace_netfs_folio(folio, netfs_folio_trace_filled_gaps); 49 if (finfo->netfs_group) 50 folio_change_private(folio, finfo->netfs_group); 51 else 52 folio_detach_private(folio); 53 kfree(finfo); 54 } 55 56 if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) { 57 if (!WARN_ON_ONCE(folio_get_private(folio) != NULL)) { 58 trace_netfs_folio(folio, netfs_folio_trace_copy_to_cache); 59 folio_attach_private(folio, NETFS_FOLIO_COPY_TO_CACHE); 60 folio_mark_dirty(folio); 61 } 62 } else { 63 trace_netfs_folio(folio, netfs_folio_trace_read_done); 64 } 65 } else { 66 // TODO: Use of PG_private_2 is deprecated. 67 if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &subreq->flags)) 68 netfs_pgpriv2_mark_copy_to_cache(subreq, rreq, folioq, slot); 69 } 70 71 if (!test_bit(NETFS_RREQ_DONT_UNLOCK_FOLIOS, &rreq->flags)) { 72 if (folio->index == rreq->no_unlock_folio && 73 test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) { 74 _debug("no unlock"); 75 } else { 76 trace_netfs_folio(folio, netfs_folio_trace_read_unlock); 77 folio_unlock(folio); 78 } 79 } 80 81 folioq_clear(folioq, slot); 82 } 83 84 /* 85 * Unlock any folios that are now completely read. Returns true if the 86 * subrequest is removed from the list. 87 */ 88 static bool netfs_consume_read_data(struct netfs_io_subrequest *subreq, bool was_async) 89 { 90 struct netfs_io_subrequest *prev, *next; 91 struct netfs_io_request *rreq = subreq->rreq; 92 struct folio_queue *folioq = subreq->curr_folioq; 93 size_t avail, prev_donated, next_donated, fsize, part, excess; 94 loff_t fpos, start; 95 loff_t fend; 96 int slot = subreq->curr_folioq_slot; 97 98 if (WARN(subreq->transferred > subreq->len, 99 "Subreq overread: R%x[%x] %zu > %zu", 100 rreq->debug_id, subreq->debug_index, 101 subreq->transferred, subreq->len)) 102 subreq->transferred = subreq->len; 103 104 next_folio: 105 fsize = PAGE_SIZE << subreq->curr_folio_order; 106 fpos = round_down(subreq->start + subreq->consumed, fsize); 107 fend = fpos + fsize; 108 109 if (WARN_ON_ONCE(!folioq) || 110 WARN_ON_ONCE(!folioq_folio(folioq, slot)) || 111 WARN_ON_ONCE(folioq_folio(folioq, slot)->index != fpos / PAGE_SIZE)) { 112 pr_err("R=%08x[%x] s=%llx-%llx ctl=%zx/%zx/%zx sl=%u\n", 113 rreq->debug_id, subreq->debug_index, 114 subreq->start, subreq->start + subreq->transferred - 1, 115 subreq->consumed, subreq->transferred, subreq->len, 116 slot); 117 if (folioq) { 118 struct folio *folio = folioq_folio(folioq, slot); 119 120 pr_err("folioq: orders=%02x%02x%02x%02x\n", 121 folioq->orders[0], folioq->orders[1], 122 folioq->orders[2], folioq->orders[3]); 123 if (folio) 124 pr_err("folio: %llx-%llx ix=%llx o=%u qo=%u\n", 125 fpos, fend - 1, folio_pos(folio), folio_order(folio), 126 folioq_folio_order(folioq, slot)); 127 } 128 } 129 130 donation_changed: 131 /* Try to consume the current folio if we've hit or passed the end of 132 * it. There's a possibility that this subreq doesn't start at the 133 * beginning of the folio, in which case we need to donate to/from the 134 * preceding subreq. 135 * 136 * We also need to include any potential donation back from the 137 * following subreq. 138 */ 139 prev_donated = READ_ONCE(subreq->prev_donated); 140 next_donated = READ_ONCE(subreq->next_donated); 141 if (prev_donated || next_donated) { 142 spin_lock_bh(&rreq->lock); 143 prev_donated = subreq->prev_donated; 144 next_donated = subreq->next_donated; 145 subreq->start -= prev_donated; 146 subreq->len += prev_donated; 147 subreq->transferred += prev_donated; 148 prev_donated = subreq->prev_donated = 0; 149 if (subreq->transferred == subreq->len) { 150 subreq->len += next_donated; 151 subreq->transferred += next_donated; 152 next_donated = subreq->next_donated = 0; 153 } 154 trace_netfs_sreq(subreq, netfs_sreq_trace_add_donations); 155 spin_unlock_bh(&rreq->lock); 156 } 157 158 avail = subreq->transferred; 159 if (avail == subreq->len) 160 avail += next_donated; 161 start = subreq->start; 162 if (subreq->consumed == 0) { 163 start -= prev_donated; 164 avail += prev_donated; 165 } else { 166 start += subreq->consumed; 167 avail -= subreq->consumed; 168 } 169 part = umin(avail, fsize); 170 171 trace_netfs_progress(subreq, start, avail, part); 172 173 if (start + avail >= fend) { 174 if (fpos == start) { 175 /* Flush, unlock and mark for caching any folio we've just read. */ 176 subreq->consumed = fend - subreq->start; 177 netfs_unlock_read_folio(subreq, rreq, folioq, slot); 178 folioq_mark2(folioq, slot); 179 if (subreq->consumed >= subreq->len) 180 goto remove_subreq; 181 } else if (fpos < start) { 182 excess = fend - subreq->start; 183 184 spin_lock_bh(&rreq->lock); 185 /* If we complete first on a folio split with the 186 * preceding subreq, donate to that subreq - otherwise 187 * we get the responsibility. 188 */ 189 if (subreq->prev_donated != prev_donated) { 190 spin_unlock_bh(&rreq->lock); 191 goto donation_changed; 192 } 193 194 if (list_is_first(&subreq->rreq_link, &rreq->subrequests)) { 195 spin_unlock_bh(&rreq->lock); 196 pr_err("Can't donate prior to front\n"); 197 goto bad; 198 } 199 200 prev = list_prev_entry(subreq, rreq_link); 201 WRITE_ONCE(prev->next_donated, prev->next_donated + excess); 202 subreq->start += excess; 203 subreq->len -= excess; 204 subreq->transferred -= excess; 205 trace_netfs_donate(rreq, subreq, prev, excess, 206 netfs_trace_donate_tail_to_prev); 207 trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_prev); 208 209 if (subreq->consumed >= subreq->len) 210 goto remove_subreq_locked; 211 spin_unlock_bh(&rreq->lock); 212 } else { 213 pr_err("fpos > start\n"); 214 goto bad; 215 } 216 217 /* Advance the rolling buffer to the next folio. */ 218 slot++; 219 if (slot >= folioq_nr_slots(folioq)) { 220 slot = 0; 221 folioq = folioq->next; 222 subreq->curr_folioq = folioq; 223 } 224 subreq->curr_folioq_slot = slot; 225 if (folioq && folioq_folio(folioq, slot)) 226 subreq->curr_folio_order = folioq->orders[slot]; 227 if (!was_async) 228 cond_resched(); 229 goto next_folio; 230 } 231 232 /* Deal with partial progress. */ 233 if (subreq->transferred < subreq->len) 234 return false; 235 236 /* Donate the remaining downloaded data to one of the neighbouring 237 * subrequests. Note that we may race with them doing the same thing. 238 */ 239 spin_lock_bh(&rreq->lock); 240 241 if (subreq->prev_donated != prev_donated || 242 subreq->next_donated != next_donated) { 243 spin_unlock_bh(&rreq->lock); 244 cond_resched(); 245 goto donation_changed; 246 } 247 248 /* Deal with the trickiest case: that this subreq is in the middle of a 249 * folio, not touching either edge, but finishes first. In such a 250 * case, we donate to the previous subreq, if there is one, so that the 251 * donation is only handled when that completes - and remove this 252 * subreq from the list. 253 * 254 * If the previous subreq finished first, we will have acquired their 255 * donation and should be able to unlock folios and/or donate nextwards. 256 */ 257 if (!subreq->consumed && 258 !prev_donated && 259 !list_is_first(&subreq->rreq_link, &rreq->subrequests)) { 260 prev = list_prev_entry(subreq, rreq_link); 261 WRITE_ONCE(prev->next_donated, prev->next_donated + subreq->len); 262 subreq->start += subreq->len; 263 subreq->len = 0; 264 subreq->transferred = 0; 265 trace_netfs_donate(rreq, subreq, prev, subreq->len, 266 netfs_trace_donate_to_prev); 267 trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_prev); 268 goto remove_subreq_locked; 269 } 270 271 /* If we can't donate down the chain, donate up the chain instead. */ 272 excess = subreq->len - subreq->consumed + next_donated; 273 274 if (!subreq->consumed) 275 excess += prev_donated; 276 277 if (list_is_last(&subreq->rreq_link, &rreq->subrequests)) { 278 rreq->prev_donated = excess; 279 trace_netfs_donate(rreq, subreq, NULL, excess, 280 netfs_trace_donate_to_deferred_next); 281 } else { 282 next = list_next_entry(subreq, rreq_link); 283 WRITE_ONCE(next->prev_donated, excess); 284 trace_netfs_donate(rreq, subreq, next, excess, 285 netfs_trace_donate_to_next); 286 } 287 trace_netfs_sreq(subreq, netfs_sreq_trace_donate_to_next); 288 subreq->len = subreq->consumed; 289 subreq->transferred = subreq->consumed; 290 goto remove_subreq_locked; 291 292 remove_subreq: 293 spin_lock_bh(&rreq->lock); 294 remove_subreq_locked: 295 subreq->consumed = subreq->len; 296 list_del(&subreq->rreq_link); 297 spin_unlock_bh(&rreq->lock); 298 netfs_put_subrequest(subreq, false, netfs_sreq_trace_put_consumed); 299 return true; 300 301 bad: 302 /* Errr... prev and next both donated to us, but insufficient to finish 303 * the folio. 304 */ 305 printk("R=%08x[%x] s=%llx-%llx %zx/%zx/%zx\n", 306 rreq->debug_id, subreq->debug_index, 307 subreq->start, subreq->start + subreq->transferred - 1, 308 subreq->consumed, subreq->transferred, subreq->len); 309 printk("folio: %llx-%llx\n", fpos, fend - 1); 310 printk("donated: prev=%zx next=%zx\n", prev_donated, next_donated); 311 printk("s=%llx av=%zx part=%zx\n", start, avail, part); 312 BUG(); 313 } 314 315 /* 316 * Do page flushing and suchlike after DIO. 317 */ 318 static void netfs_rreq_assess_dio(struct netfs_io_request *rreq) 319 { 320 struct netfs_io_subrequest *subreq; 321 unsigned int i; 322 323 /* Collect unbuffered reads and direct reads, adding up the transfer 324 * sizes until we find the first short or failed subrequest. 325 */ 326 list_for_each_entry(subreq, &rreq->subrequests, rreq_link) { 327 rreq->transferred += subreq->transferred; 328 329 if (subreq->transferred < subreq->len || 330 test_bit(NETFS_SREQ_FAILED, &subreq->flags)) { 331 rreq->error = subreq->error; 332 break; 333 } 334 } 335 336 if (rreq->origin == NETFS_DIO_READ) { 337 for (i = 0; i < rreq->direct_bv_count; i++) { 338 flush_dcache_page(rreq->direct_bv[i].bv_page); 339 // TODO: cifs marks pages in the destination buffer 340 // dirty under some circumstances after a read. Do we 341 // need to do that too? 342 set_page_dirty(rreq->direct_bv[i].bv_page); 343 } 344 } 345 346 if (rreq->iocb) { 347 rreq->iocb->ki_pos += rreq->transferred; 348 if (rreq->iocb->ki_complete) 349 rreq->iocb->ki_complete( 350 rreq->iocb, rreq->error ? rreq->error : rreq->transferred); 351 } 352 if (rreq->netfs_ops->done) 353 rreq->netfs_ops->done(rreq); 354 if (rreq->origin == NETFS_DIO_READ) 355 inode_dio_end(rreq->inode); 356 } 357 358 /* 359 * Assess the state of a read request and decide what to do next. 360 * 361 * Note that we're in normal kernel thread context at this point, possibly 362 * running on a workqueue. 363 */ 364 static void netfs_rreq_assess(struct netfs_io_request *rreq) 365 { 366 trace_netfs_rreq(rreq, netfs_rreq_trace_assess); 367 368 //netfs_rreq_is_still_valid(rreq); 369 370 if (test_and_clear_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags)) { 371 netfs_retry_reads(rreq); 372 return; 373 } 374 375 if (rreq->origin == NETFS_DIO_READ || 376 rreq->origin == NETFS_READ_GAPS) 377 netfs_rreq_assess_dio(rreq); 378 task_io_account_read(rreq->transferred); 379 380 trace_netfs_rreq(rreq, netfs_rreq_trace_wake_ip); 381 clear_bit_unlock(NETFS_RREQ_IN_PROGRESS, &rreq->flags); 382 wake_up_bit(&rreq->flags, NETFS_RREQ_IN_PROGRESS); 383 384 trace_netfs_rreq(rreq, netfs_rreq_trace_done); 385 netfs_clear_subrequests(rreq, false); 386 netfs_unlock_abandoned_read_pages(rreq); 387 if (unlikely(test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags))) 388 netfs_pgpriv2_write_to_the_cache(rreq); 389 } 390 391 void netfs_read_termination_worker(struct work_struct *work) 392 { 393 struct netfs_io_request *rreq = 394 container_of(work, struct netfs_io_request, work); 395 netfs_see_request(rreq, netfs_rreq_trace_see_work); 396 netfs_rreq_assess(rreq); 397 netfs_put_request(rreq, false, netfs_rreq_trace_put_work_complete); 398 } 399 400 /* 401 * Handle the completion of all outstanding I/O operations on a read request. 402 * We inherit a ref from the caller. 403 */ 404 void netfs_rreq_terminated(struct netfs_io_request *rreq, bool was_async) 405 { 406 if (!was_async) 407 return netfs_rreq_assess(rreq); 408 if (!work_pending(&rreq->work)) { 409 netfs_get_request(rreq, netfs_rreq_trace_get_work); 410 if (!queue_work(system_unbound_wq, &rreq->work)) 411 netfs_put_request(rreq, was_async, netfs_rreq_trace_put_work_nq); 412 } 413 } 414 415 /** 416 * netfs_read_subreq_progress - Note progress of a read operation. 417 * @subreq: The read request that has terminated. 418 * @was_async: True if we're in an asynchronous context. 419 * 420 * This tells the read side of netfs lib that a contributory I/O operation has 421 * made some progress and that it may be possible to unlock some folios. 422 * 423 * Before calling, the filesystem should update subreq->transferred to track 424 * the amount of data copied into the output buffer. 425 * 426 * If @was_async is true, the caller might be running in softirq or interrupt 427 * context and we can't sleep. 428 */ 429 void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq, 430 bool was_async) 431 { 432 struct netfs_io_request *rreq = subreq->rreq; 433 434 trace_netfs_sreq(subreq, netfs_sreq_trace_progress); 435 436 if (subreq->transferred > subreq->consumed && 437 (rreq->origin == NETFS_READAHEAD || 438 rreq->origin == NETFS_READPAGE || 439 rreq->origin == NETFS_READ_FOR_WRITE)) { 440 netfs_consume_read_data(subreq, was_async); 441 __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); 442 } 443 } 444 EXPORT_SYMBOL(netfs_read_subreq_progress); 445 446 /** 447 * netfs_read_subreq_terminated - Note the termination of an I/O operation. 448 * @subreq: The I/O request that has terminated. 449 * @error: Error code indicating type of completion. 450 * @was_async: The termination was asynchronous 451 * 452 * This tells the read helper that a contributory I/O operation has terminated, 453 * one way or another, and that it should integrate the results. 454 * 455 * The caller indicates the outcome of the operation through @error, supplying 456 * 0 to indicate a successful or retryable transfer (if NETFS_SREQ_NEED_RETRY 457 * is set) or a negative error code. The helper will look after reissuing I/O 458 * operations as appropriate and writing downloaded data to the cache. 459 * 460 * Before calling, the filesystem should update subreq->transferred to track 461 * the amount of data copied into the output buffer. 462 * 463 * If @was_async is true, the caller might be running in softirq or interrupt 464 * context and we can't sleep. 465 */ 466 void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq, 467 int error, bool was_async) 468 { 469 struct netfs_io_request *rreq = subreq->rreq; 470 471 switch (subreq->source) { 472 case NETFS_READ_FROM_CACHE: 473 netfs_stat(&netfs_n_rh_read_done); 474 break; 475 case NETFS_DOWNLOAD_FROM_SERVER: 476 netfs_stat(&netfs_n_rh_download_done); 477 break; 478 default: 479 break; 480 } 481 482 if (rreq->origin != NETFS_DIO_READ) { 483 /* Collect buffered reads. 484 * 485 * If the read completed validly short, then we can clear the 486 * tail before going on to unlock the folios. 487 */ 488 if (error == 0 && subreq->transferred < subreq->len && 489 (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags) || 490 test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags))) { 491 netfs_clear_unread(subreq); 492 subreq->transferred = subreq->len; 493 trace_netfs_sreq(subreq, netfs_sreq_trace_clear); 494 } 495 if (subreq->transferred > subreq->consumed && 496 (rreq->origin == NETFS_READAHEAD || 497 rreq->origin == NETFS_READPAGE || 498 rreq->origin == NETFS_READ_FOR_WRITE)) { 499 netfs_consume_read_data(subreq, was_async); 500 __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); 501 } 502 rreq->transferred += subreq->transferred; 503 } 504 505 /* Deal with retry requests, short reads and errors. If we retry 506 * but don't make progress, we abandon the attempt. 507 */ 508 if (!error && subreq->transferred < subreq->len) { 509 if (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags)) { 510 trace_netfs_sreq(subreq, netfs_sreq_trace_hit_eof); 511 } else { 512 trace_netfs_sreq(subreq, netfs_sreq_trace_short); 513 if (subreq->transferred > subreq->consumed) { 514 __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 515 __clear_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags); 516 set_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags); 517 } else if (!__test_and_set_bit(NETFS_SREQ_NO_PROGRESS, &subreq->flags)) { 518 __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); 519 set_bit(NETFS_RREQ_NEED_RETRY, &rreq->flags); 520 } else { 521 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); 522 error = -ENODATA; 523 } 524 } 525 } 526 527 subreq->error = error; 528 trace_netfs_sreq(subreq, netfs_sreq_trace_terminated); 529 530 if (unlikely(error < 0)) { 531 trace_netfs_failure(rreq, subreq, error, netfs_fail_read); 532 if (subreq->source == NETFS_READ_FROM_CACHE) { 533 netfs_stat(&netfs_n_rh_read_failed); 534 } else { 535 netfs_stat(&netfs_n_rh_download_failed); 536 set_bit(NETFS_RREQ_FAILED, &rreq->flags); 537 rreq->error = subreq->error; 538 } 539 } 540 541 if (atomic_dec_and_test(&rreq->nr_outstanding)) 542 netfs_rreq_terminated(rreq, was_async); 543 544 netfs_put_subrequest(subreq, was_async, netfs_sreq_trace_put_terminated); 545 } 546 EXPORT_SYMBOL(netfs_read_subreq_terminated); 547