1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Network filesystem read subrequest result collection, assessment and
3 * retrying.
4 *
5 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
6 * Written by David Howells (dhowells@redhat.com)
7 */
8
9 #include <linux/export.h>
10 #include <linux/fs.h>
11 #include <linux/mm.h>
12 #include <linux/pagemap.h>
13 #include <linux/slab.h>
14 #include <linux/task_io_accounting_ops.h>
15 #include "internal.h"
16
17 /* Notes made in the collector */
18 #define HIT_PENDING 0x01 /* A front op was still pending */
19 #define MADE_PROGRESS 0x04 /* Made progress cleaning up a stream or the folio set */
20 #define BUFFERED 0x08 /* The pagecache needs cleaning up */
21 #define NEED_RETRY 0x10 /* A front op requests retrying */
22 #define ABANDON_SREQ 0x80 /* Need to abandon untransferred part of subrequest */
23
24 /*
25 * Clear the unread part of an I/O request.
26 */
netfs_clear_unread(struct netfs_io_subrequest * subreq)27 static void netfs_clear_unread(struct netfs_io_subrequest *subreq)
28 {
29 netfs_reset_iter(subreq);
30 WARN_ON_ONCE(subreq->len - subreq->transferred != iov_iter_count(&subreq->io_iter));
31 iov_iter_zero(iov_iter_count(&subreq->io_iter), &subreq->io_iter);
32 if (subreq->start + subreq->transferred >= subreq->rreq->i_size)
33 __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);
34 }
35
36 /*
37 * Cancel the copy-to-cache mark on a folio.
38 */
netfs_cancel_copy_to_cache(struct netfs_io_request * rreq,struct folio * folio)39 void netfs_cancel_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio)
40 {
41 if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {
42 if (folio_get_private(folio) == NETFS_FOLIO_COPY_TO_CACHE) {
43 folio_detach_private(folio);
44 trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
45 } else if (netfs_folio_group(folio) == NETFS_FOLIO_COPY_TO_CACHE) {
46 struct netfs_folio *finfo = netfs_folio_info(folio);
47
48 finfo->netfs_group = NULL;
49 trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
50 }
51 } else {
52 // TODO: Use of PG_private_2 is deprecated.
53 if (folio_test_private_2(folio)) {
54 folio_end_private_2(folio);
55 trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
56 }
57 }
58 }
59
60 /*
61 * Flush, mark and unlock a folio that's now completely read. If we want to
62 * cache the folio, we set the group to NETFS_FOLIO_COPY_TO_CACHE, mark it
63 * dirty and let writeback handle it.
64 */
netfs_unlock_read_folio(struct netfs_io_request * rreq,struct folio_queue * folioq,int slot)65 static void netfs_unlock_read_folio(struct netfs_io_request *rreq,
66 struct folio_queue *folioq,
67 int slot)
68 {
69 struct netfs_folio *finfo;
70 struct folio *folio = folioq_folio(folioq, slot);
71
72 if (unlikely(folio_pos(folio) < rreq->abandon_to)) {
73 trace_netfs_folio(folio, netfs_folio_trace_abandon);
74 netfs_cancel_copy_to_cache(rreq, folio);
75 goto just_unlock;
76 }
77
78 flush_dcache_folio(folio);
79 folio_mark_uptodate(folio);
80
81 if (unlikely(test_bit(NETFS_RREQ_CANCEL_CACHING, &rreq->flags)))
82 netfs_cancel_copy_to_cache(rreq, folio);
83
84 if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) {
85 if (netfs_folio_group(folio) == NETFS_FOLIO_COPY_TO_CACHE) {
86 trace_netfs_folio(folio, netfs_folio_trace_sched_copy);
87 folio_mark_dirty(folio);
88 } else {
89 finfo = netfs_folio_info(folio);
90 if (finfo) {
91 trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);
92 if (finfo->netfs_group)
93 folio_change_private(folio, finfo->netfs_group);
94 else
95 folio_detach_private(folio);
96 kfree(finfo);
97 }
98 trace_netfs_folio(folio, netfs_folio_trace_read_done);
99 }
100
101 folioq_clear(folioq, slot);
102 } else {
103 // TODO: Use of PG_private_2 is deprecated.
104 if (folio_test_private_2(folio))
105 netfs_pgpriv2_copy_to_cache(rreq, folio);
106 }
107
108 just_unlock:
109 if (folio == rreq->no_unlock_folio &&
110 test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) {
111 _debug("no unlock");
112 } else {
113 trace_netfs_folio(folio, netfs_folio_trace_read_unlock);
114 folio_unlock(folio);
115 }
116
117 folioq_clear(folioq, slot);
118 }
119
120 /*
121 * Determine how much to gather before unlocking more folios.
122 */
netfs_read_set_unlock_at(struct netfs_io_request * rreq)123 void netfs_read_set_unlock_at(struct netfs_io_request *rreq)
124 {
125 struct folio_queue *folioq = rreq->buffer.tail;
126 unsigned int slot = rreq->buffer.first_tail_slot;
127 size_t cleaned_to = rreq->cleaned_to - rreq->start;
128 size_t progress_at = cleaned_to;
129 size_t minimum = 256 * 1024;
130
131 while (progress_at < rreq->len) {
132 if (slot >= folioq_count(folioq)) {
133 folioq = folioq->next;
134 if (!folioq)
135 break;
136 slot = 0;
137 }
138
139 progress_at += folioq_folio_size(folioq, slot);
140 if (progress_at - cleaned_to >= minimum)
141 break;
142 slot++;
143 }
144
145 WRITE_ONCE(rreq->progress_at, progress_at);
146 trace_netfs_read_progress_at(rreq);
147 }
148
149 /*
150 * Unlock any folios we've finished with.
151 */
netfs_read_unlock_folios(struct netfs_io_request * rreq,unsigned int * notes)152 static void netfs_read_unlock_folios(struct netfs_io_request *rreq,
153 unsigned int *notes)
154 {
155 struct folio_queue *folioq = rreq->buffer.tail;
156 unsigned long long collected_to = rreq->collected_to;
157 unsigned int slot = rreq->buffer.first_tail_slot;
158
159 if (rreq->cleaned_to >= rreq->collected_to)
160 return;
161
162 // TODO: Begin decryption
163
164 if (slot >= folioq_nr_slots(folioq)) {
165 folioq = rolling_buffer_delete_spent(&rreq->buffer);
166 if (!folioq) {
167 WRITE_ONCE(rreq->progress_at, rreq->len);
168 return;
169 }
170 slot = 0;
171 }
172
173 /* We have to wait for readahead refs to have been released before we
174 * can unlock any folios as the ref-dropper walks i_pages and the only
175 * thing preventing these folios from being removed is the folio lock.
176 */
177 if (test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags))
178 netfs_wait_for_put_ra_refs(rreq);
179
180 for (;;) {
181 struct folio *folio;
182 unsigned long long fpos, fend;
183 size_t fsize;
184
185 folio = folioq_folio(folioq, slot);
186 if (WARN_ONCE(!folio_test_locked(folio),
187 "R=%08x: folio %lx is not locked\n",
188 rreq->debug_id, folio->index))
189 trace_netfs_folio(folio, netfs_folio_trace_not_locked);
190
191 fsize = folioq_folio_size(folioq, slot);
192 fpos = folio_pos(folio);
193 fend = fpos + fsize;
194
195 trace_netfs_collect_folio(rreq, folio, fend, collected_to);
196
197 /* Unlock any folio we've transferred all of. */
198 if (collected_to < fend)
199 break;
200
201 netfs_unlock_read_folio(rreq, folioq, slot);
202 WRITE_ONCE(rreq->cleaned_to, fpos + fsize);
203 *notes |= MADE_PROGRESS;
204
205 /* Clean up the head folioq. If we clear an entire folioq, then
206 * we can get rid of it provided it's not also the tail folioq
207 * being filled by the issuer.
208 */
209 folioq_clear(folioq, slot);
210 slot++;
211 if (slot >= folioq_nr_slots(folioq)) {
212 folioq = rolling_buffer_delete_spent(&rreq->buffer);
213 if (!folioq)
214 goto done;
215 slot = 0;
216 trace_netfs_folioq(folioq, netfs_trace_folioq_read_progress);
217 }
218
219 if (fpos + fsize >= collected_to)
220 break;
221 }
222
223 rreq->buffer.tail = folioq;
224 done:
225 rreq->buffer.first_tail_slot = slot;
226
227 netfs_read_set_unlock_at(rreq);
228 }
229
230 /*
231 * Collect and assess the results of various read subrequests. We may need to
232 * retry some of the results.
233 *
234 * Note that we have a sequence of subrequests, which may be drawing on
235 * different sources and may or may not be the same size or starting position
236 * and may not even correspond in boundary alignment.
237 */
netfs_collect_read_results(struct netfs_io_request * rreq)238 static void netfs_collect_read_results(struct netfs_io_request *rreq)
239 {
240 struct netfs_io_subrequest *front, *remove;
241 struct netfs_io_stream *stream = &rreq->io_streams[0];
242 unsigned int notes;
243
244 _enter("%llx-%llx", rreq->start, rreq->start + rreq->len);
245 trace_netfs_rreq(rreq, netfs_rreq_trace_collect);
246 trace_netfs_collect(rreq);
247
248 reassess:
249 if (rreq->origin == NETFS_READAHEAD ||
250 rreq->origin == NETFS_READPAGE ||
251 rreq->origin == NETFS_READ_FOR_WRITE)
252 notes = BUFFERED;
253 else
254 notes = 0;
255
256 /* Remove completed subrequests from the front of the stream and
257 * advance the completion point. We stop when we hit something that's
258 * in progress. The issuer thread may be adding stuff to the tail
259 * whilst we're doing this.
260 */
261 front = list_first_entry_or_null_acquire(&stream->subrequests,
262 struct netfs_io_subrequest, rreq_link);
263 /* Read first subreq pointer before IN_PROGRESS flag. */
264
265 while (front) {
266 size_t transferred;
267
268 trace_netfs_collect_sreq(rreq, front);
269 _debug("sreq [%x] %llx %zx/%zx",
270 front->debug_index, front->start, front->transferred, front->len);
271
272 if (stream->collected_to < front->start) {
273 trace_netfs_collect_gap(rreq, stream, front->start, 'F');
274 stream->collected_to = front->start;
275 }
276
277 if (netfs_check_subreq_in_progress(front))
278 notes |= HIT_PENDING;
279 smp_rmb(); /* Read counters after IN_PROGRESS flag. */
280 transferred = READ_ONCE(front->transferred);
281
282 /* If we can now collect the next folio, do so. We don't want
283 * to defer this as we have to decide whether we need to copy
284 * to the cache or not, and that may differ between adjacent
285 * subreqs.
286 */
287 if (notes & BUFFERED) {
288 uoff_t unlock_at = rreq->start + rreq->progress_at;
289
290 /* Clear the tail of a short read. */
291 if (!(notes & HIT_PENDING) &&
292 front->error == 0 &&
293 transferred < front->len &&
294 (test_bit(NETFS_SREQ_HIT_EOF, &front->flags) ||
295 test_bit(NETFS_SREQ_CLEAR_TAIL, &front->flags))) {
296 netfs_clear_unread(front);
297 transferred = front->transferred = front->len;
298 trace_netfs_sreq(front, netfs_sreq_trace_clear);
299 }
300
301 stream->collected_to = front->start + transferred;
302 rreq->collected_to = stream->collected_to;
303
304 if (test_bit(NETFS_SREQ_FAILED, &front->flags)) {
305 rreq->abandon_to = front->start + front->len;
306 front->transferred = front->len;
307 transferred = front->len;
308 trace_netfs_rreq(rreq, netfs_rreq_trace_set_abandon);
309 }
310 if (front->start + transferred >= unlock_at ||
311 test_bit(NETFS_SREQ_HIT_EOF, &front->flags))
312 netfs_read_unlock_folios(rreq, ¬es);
313 } else {
314 stream->collected_to = front->start + transferred;
315 rreq->collected_to = stream->collected_to;
316 }
317
318 /* Stall if the front is still undergoing I/O. */
319 if (notes & HIT_PENDING)
320 break;
321
322 if (test_bit(NETFS_SREQ_FAILED, &front->flags)) {
323 if (!stream->failed) {
324 stream->error = front->error;
325 rreq->error = front->error;
326 set_bit(NETFS_RREQ_FAILED, &rreq->flags);
327 stream->failed = true;
328 }
329 notes |= MADE_PROGRESS | ABANDON_SREQ;
330 } else if (test_bit(NETFS_SREQ_NEED_RETRY, &front->flags)) {
331 stream->need_retry = true;
332 notes |= NEED_RETRY | MADE_PROGRESS;
333 break;
334 } else if (test_bit(NETFS_RREQ_SHORT_TRANSFER, &rreq->flags)) {
335 notes |= MADE_PROGRESS;
336 } else {
337 if (!stream->failed) {
338 stream->transferred += transferred;
339 stream->transferred_valid = true;
340 }
341 if (front->transferred < front->len)
342 set_bit(NETFS_RREQ_SHORT_TRANSFER, &rreq->flags);
343 notes |= MADE_PROGRESS;
344 }
345
346 /* Remove if completely consumed. */
347 stream->source = front->source;
348 spin_lock(&rreq->lock);
349
350 remove = front;
351 trace_netfs_sreq(front,
352 notes & ABANDON_SREQ ?
353 netfs_sreq_trace_abandoned : netfs_sreq_trace_consumed);
354 list_del_init(&front->rreq_link);
355 front = list_first_entry_or_null(&stream->subrequests,
356 struct netfs_io_subrequest, rreq_link);
357 spin_unlock(&rreq->lock);
358 netfs_put_subrequest(remove,
359 notes & ABANDON_SREQ ?
360 netfs_sreq_trace_put_abandon :
361 netfs_sreq_trace_put_done);
362 }
363
364 trace_netfs_collect_stream(rreq, stream);
365 trace_netfs_collect_state(rreq, rreq->collected_to, notes);
366
367 if (!(notes & BUFFERED))
368 rreq->cleaned_to = rreq->collected_to;
369
370 if (notes & NEED_RETRY)
371 goto need_retry;
372 if (notes & MADE_PROGRESS) {
373 netfs_wake_rreq_flag(rreq, NETFS_RREQ_PAUSE, netfs_rreq_trace_unpause);
374 //cond_resched();
375 goto reassess;
376 }
377
378 out:
379 _leave(" = %x", notes);
380 return;
381
382 need_retry:
383 /* Okay... We're going to have to retry parts of the stream. Note
384 * that any partially completed op will have had any wholly transferred
385 * folios removed from it.
386 */
387 _debug("retry");
388 netfs_retry_reads(rreq);
389 goto out;
390 }
391
392 /*
393 * Do page flushing and suchlike after DIO.
394 */
netfs_rreq_assess_dio(struct netfs_io_request * rreq)395 static void netfs_rreq_assess_dio(struct netfs_io_request *rreq)
396 {
397 unsigned int i;
398
399 if (rreq->origin == NETFS_UNBUFFERED_READ ||
400 rreq->origin == NETFS_DIO_READ) {
401 for (i = 0; i < rreq->direct_bv_count; i++) {
402 flush_dcache_page(rreq->direct_bv[i].bv_page);
403 // TODO: cifs marks pages in the destination buffer
404 // dirty under some circumstances after a read. Do we
405 // need to do that too?
406 set_page_dirty(rreq->direct_bv[i].bv_page);
407 }
408 }
409
410 if (rreq->iocb) {
411 rreq->iocb->ki_pos += rreq->transferred;
412 if (rreq->iocb->ki_complete) {
413 trace_netfs_rreq(rreq, netfs_rreq_trace_ki_complete);
414 rreq->iocb->ki_complete(
415 rreq->iocb, rreq->error ? rreq->error : rreq->transferred);
416 }
417 }
418 if (rreq->netfs_ops->done)
419 rreq->netfs_ops->done(rreq);
420 if (rreq->origin == NETFS_UNBUFFERED_READ ||
421 rreq->origin == NETFS_DIO_READ)
422 inode_dio_end(rreq->inode);
423 }
424
425 /*
426 * Do processing after reading a monolithic single object.
427 */
netfs_rreq_assess_single(struct netfs_io_request * rreq)428 static void netfs_rreq_assess_single(struct netfs_io_request *rreq)
429 {
430 struct netfs_io_stream *stream = &rreq->io_streams[0];
431
432 if (!rreq->error && stream->source == NETFS_DOWNLOAD_FROM_SERVER &&
433 fscache_resources_valid(&rreq->cache_resources)) {
434 trace_netfs_rreq(rreq, netfs_rreq_trace_dirty);
435 netfs_single_mark_inode_dirty(rreq->inode);
436 }
437
438 /* To do DIO, the cache has to round the size up, so we need to undo
439 * the rounding.
440 */
441 rreq->transferred = min(rreq->transferred, rreq->i_size);
442
443 if (rreq->iocb) {
444 rreq->iocb->ki_pos += rreq->transferred;
445 if (rreq->iocb->ki_complete) {
446 trace_netfs_rreq(rreq, netfs_rreq_trace_ki_complete);
447 rreq->iocb->ki_complete(
448 rreq->iocb, rreq->error ? rreq->error : rreq->transferred);
449 }
450 }
451 if (rreq->netfs_ops->done)
452 rreq->netfs_ops->done(rreq);
453 }
454
455 /*
456 * Perform the collection of subrequests and folios.
457 *
458 * Note that we're in normal kernel thread context at this point, possibly
459 * running on a workqueue.
460 */
netfs_read_collection(struct netfs_io_request * rreq)461 bool netfs_read_collection(struct netfs_io_request *rreq)
462 {
463 struct netfs_io_stream *stream = &rreq->io_streams[0];
464
465 netfs_collect_read_results(rreq);
466
467 /* We're done when the app thread has finished posting subreqs and the
468 * queue is empty.
469 */
470 if (!test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags))
471 return false;
472 smp_rmb(); /* Read ALL_QUEUED before subreq lists. */
473
474 if (!list_empty(&stream->subrequests))
475 return false;
476
477 /* Okay, declare that all I/O is complete. */
478 rreq->transferred = stream->transferred;
479 trace_netfs_rreq(rreq, netfs_rreq_trace_complete);
480
481 //netfs_rreq_is_still_valid(rreq);
482
483 switch (rreq->origin) {
484 case NETFS_UNBUFFERED_READ:
485 case NETFS_DIO_READ:
486 case NETFS_READ_GAPS:
487 netfs_rreq_assess_dio(rreq);
488 break;
489 case NETFS_READ_SINGLE:
490 netfs_rreq_assess_single(rreq);
491 break;
492 default:
493 break;
494 }
495 task_io_account_read(rreq->transferred);
496
497 netfs_wake_rreq_flag(rreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip);
498 /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */
499
500 trace_netfs_rreq(rreq, netfs_rreq_trace_done);
501 netfs_clear_subrequests(rreq);
502 netfs_unlock_abandoned_read_pages(rreq);
503 if (unlikely(rreq->copy_to_cache))
504 netfs_pgpriv2_end_copy_to_cache(rreq);
505 return true;
506 }
507
netfs_read_collection_worker(struct work_struct * work)508 void netfs_read_collection_worker(struct work_struct *work)
509 {
510 struct netfs_io_request *rreq = container_of(work, struct netfs_io_request, work);
511
512 netfs_see_request(rreq, netfs_rreq_trace_see_work);
513 if (netfs_check_rreq_in_progress(rreq)) {
514 if (netfs_read_collection(rreq))
515 /* Drop the ref from the IN_PROGRESS flag. */
516 netfs_put_request(rreq, netfs_rreq_trace_put_work_ip);
517 else
518 netfs_see_request(rreq, netfs_rreq_trace_see_work_complete);
519 }
520 }
521
522 /**
523 * netfs_read_subreq_progress - Note progress of a read operation.
524 * @subreq: The read request that has terminated.
525 *
526 * This tells the read side of netfs lib that a contributory I/O operation has
527 * made some progress and that it may be possible to unlock some folios.
528 *
529 * Before calling, the filesystem should update subreq->transferred to track
530 * the amount of data copied into the output buffer.
531 */
netfs_read_subreq_progress(struct netfs_io_subrequest * subreq)532 void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq)
533 {
534 struct netfs_io_request *rreq = subreq->rreq;
535 struct netfs_io_stream *stream = &rreq->io_streams[subreq->stream_nr];
536 size_t progress_at = READ_ONCE(rreq->progress_at);
537 uoff_t update_at = rreq->start + progress_at;
538 uoff_t transferred_to = subreq->start + subreq->transferred;
539
540 /* If we are at the head of the queue, wake up the collector,
541 * getting a ref to it if we were the ones to do so.
542 */
543 if (progress_at < rreq->len &&
544 transferred_to >= update_at &&
545 (rreq->origin == NETFS_READAHEAD ||
546 rreq->origin == NETFS_READPAGE ||
547 rreq->origin == NETFS_READ_FOR_WRITE) &&
548 list_is_first(&subreq->rreq_link, &stream->subrequests)
549 ) {
550 trace_netfs_sreq(subreq, netfs_sreq_trace_progress);
551 __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
552 netfs_wake_collector(rreq);
553 }
554 }
555 EXPORT_SYMBOL(netfs_read_subreq_progress);
556
557 /**
558 * netfs_read_subreq_terminated - Note the termination of an I/O operation.
559 * @subreq: The I/O request that has terminated.
560 *
561 * This tells the read helper that a contributory I/O operation has terminated,
562 * one way or another, and that it should integrate the results.
563 *
564 * The caller indicates the outcome of the operation through @subreq->error,
565 * supplying 0 to indicate a successful or retryable transfer (if
566 * NETFS_SREQ_NEED_RETRY is set) or a negative error code. The helper will
567 * look after reissuing I/O operations as appropriate and writing downloaded
568 * data to the cache.
569 *
570 * Before calling, the filesystem should update subreq->transferred to track
571 * the amount of data copied into the output buffer.
572 */
netfs_read_subreq_terminated(struct netfs_io_subrequest * subreq)573 void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq)
574 {
575 struct netfs_io_request *rreq = subreq->rreq;
576
577 switch (subreq->source) {
578 case NETFS_READ_FROM_CACHE:
579 netfs_stat(&netfs_n_rh_read_done);
580 break;
581 case NETFS_DOWNLOAD_FROM_SERVER:
582 netfs_stat(&netfs_n_rh_download_done);
583 break;
584 default:
585 break;
586 }
587
588 /* Deal with retry requests, short reads and errors. If we retry
589 * but don't make progress, we abandon the attempt.
590 */
591 if (!subreq->error && subreq->transferred < subreq->len) {
592 if (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags)) {
593 trace_netfs_sreq(subreq, netfs_sreq_trace_hit_eof);
594 } else if (test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags)) {
595 trace_netfs_sreq(subreq, netfs_sreq_trace_need_clear);
596 } else if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
597 trace_netfs_sreq(subreq, netfs_sreq_trace_need_retry);
598 } else if (test_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags)) {
599 __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
600 trace_netfs_sreq(subreq, netfs_sreq_trace_partial_read);
601 } else {
602 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
603 subreq->error = -ENODATA;
604 trace_netfs_sreq(subreq, netfs_sreq_trace_short);
605 }
606 }
607
608 /* If need retry is set, error should not matter unless we hit too many
609 * retries. Pause the generation of new subreqs
610 */
611 if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
612 trace_netfs_rreq(rreq, netfs_rreq_trace_set_pause);
613 set_bit(NETFS_RREQ_PAUSE, &rreq->flags);
614 goto skip_error_checks;
615 }
616
617 if (unlikely(subreq->error < 0)) {
618 trace_netfs_failure(rreq, subreq, subreq->error, netfs_fail_read);
619 if (subreq->source == NETFS_READ_FROM_CACHE) {
620 netfs_stat(&netfs_n_rh_read_failed);
621 __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
622 } else {
623 netfs_stat(&netfs_n_rh_download_failed);
624 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
625 }
626 trace_netfs_rreq(rreq, netfs_rreq_trace_set_pause);
627 set_bit(NETFS_RREQ_PAUSE, &rreq->flags);
628 }
629
630 skip_error_checks:
631 trace_netfs_sreq(subreq, netfs_sreq_trace_terminated);
632 netfs_subreq_clear_in_progress(subreq);
633 netfs_put_subrequest(subreq, netfs_sreq_trace_put_terminated);
634 }
635 EXPORT_SYMBOL(netfs_read_subreq_terminated);
636
637 /*
638 * Cancel a read subrequest due to preparation failure.
639 */
netfs_cancel_read(struct netfs_io_subrequest * subreq,int error)640 void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error)
641 {
642 trace_netfs_sreq(subreq, netfs_sreq_trace_cancel);
643 subreq->error = error;
644 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
645 netfs_read_subreq_terminated(subreq);
646 }
647
648 /*
649 * Handle termination of a read from the cache.
650 */
netfs_cache_read_terminated(void * priv,ssize_t transferred_or_error)651 void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error)
652 {
653 struct netfs_io_subrequest *subreq = priv;
654
655 if (transferred_or_error > 0) {
656 subreq->error = 0;
657 if (transferred_or_error > 0) {
658 subreq->transferred += transferred_or_error;
659 __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
660 }
661 } else {
662 subreq->error = transferred_or_error;
663 }
664 netfs_read_subreq_terminated(subreq);
665 }
666