1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Network filesystem read subrequest result collection, assessment and
3 * retrying.
4 *
5 * Copyright (C) 2024 Red Hat, Inc. All Rights Reserved.
6 * Written by David Howells (dhowells@redhat.com)
7 */
8
9 #include <linux/export.h>
10 #include <linux/fs.h>
11 #include <linux/mm.h>
12 #include <linux/pagemap.h>
13 #include <linux/slab.h>
14 #include <linux/task_io_accounting_ops.h>
15 #include "internal.h"
16
17 /* Notes made in the collector */
18 #define HIT_PENDING 0x01 /* A front op was still pending */
19 #define MADE_PROGRESS 0x04 /* Made progress cleaning up a stream or the folio set */
20 #define BUFFERED 0x08 /* The pagecache needs cleaning up */
21 #define NEED_RETRY 0x10 /* A front op requests retrying */
22 #define ABANDON_SREQ 0x80 /* Need to abandon untransferred part of subrequest */
23
24 /*
25 * Clear the unread part of an I/O request.
26 */
netfs_clear_unread(struct netfs_io_subrequest * subreq)27 static void netfs_clear_unread(struct netfs_io_subrequest *subreq)
28 {
29 netfs_reset_iter(subreq);
30 WARN_ON_ONCE(subreq->len - subreq->transferred != iov_iter_count(&subreq->io_iter));
31 iov_iter_zero(iov_iter_count(&subreq->io_iter), &subreq->io_iter);
32 if (subreq->start + subreq->transferred >= subreq->rreq->i_size)
33 __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags);
34 }
35
netfs_clear_unread_dio(struct netfs_io_subrequest * subreq)36 static void netfs_clear_unread_dio(struct netfs_io_subrequest *subreq)
37 {
38 uoff_t pos = subreq->start + subreq->transferred;
39 struct netfs_io_request *rreq = subreq->rreq;
40 size_t fill;
41
42 if (pos >= rreq->i_size)
43 return;
44
45 fill = min_t(uoff_t, rreq->i_size - pos,
46 subreq->len - subreq->transferred);
47
48 netfs_reset_iter(subreq);
49 subreq->transferred += iov_iter_zero(fill, &subreq->io_iter);
50 }
51
52 /*
53 * Cancel the copy-to-cache mark on a folio.
54 */
netfs_cancel_copy_to_cache(struct netfs_io_request * rreq,struct folio * folio)55 void netfs_cancel_copy_to_cache(struct netfs_io_request *rreq, struct folio *folio)
56 {
57 if (!netfs_using_pgpriv2(rreq)) {
58 if (folio_get_private(folio) == NETFS_FOLIO_COPY_TO_CACHE) {
59 folio_detach_private(folio);
60 trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
61 } else if (netfs_folio_group(folio) == NETFS_FOLIO_COPY_TO_CACHE) {
62 struct netfs_folio *finfo = netfs_folio_info(folio);
63
64 finfo->netfs_group = NULL;
65 trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
66 }
67 } else {
68 // TODO: Use of PG_private_2 is deprecated.
69 if (folio_test_private_2(folio)) {
70 folio_end_private_2(folio);
71 trace_netfs_folio(folio, netfs_folio_trace_cancel_copy);
72 }
73 }
74 }
75
76 /*
77 * Flush, mark and unlock a folio that's now completely read. If we want to
78 * cache the folio, we set the group to NETFS_FOLIO_COPY_TO_CACHE, mark it
79 * dirty and let writeback handle it.
80 */
netfs_unlock_read_folio(struct netfs_io_request * rreq,struct folio_queue * folioq,int slot)81 static void netfs_unlock_read_folio(struct netfs_io_request *rreq,
82 struct folio_queue *folioq,
83 int slot)
84 {
85 struct netfs_folio *finfo;
86 struct folio *folio = folioq_folio(folioq, slot);
87
88 if (unlikely(folio_pos(folio) < rreq->abandon_to)) {
89 trace_netfs_folio(folio, netfs_folio_trace_abandon);
90 netfs_cancel_copy_to_cache(rreq, folio);
91 goto just_unlock;
92 }
93
94 flush_dcache_folio(folio);
95 folio_mark_uptodate(folio);
96
97 if (unlikely(test_bit(NETFS_RREQ_CANCEL_CACHING, &rreq->flags)))
98 netfs_cancel_copy_to_cache(rreq, folio);
99
100 if (!netfs_using_pgpriv2(rreq)) {
101 if (netfs_folio_group(folio) == NETFS_FOLIO_COPY_TO_CACHE) {
102 trace_netfs_folio(folio, netfs_folio_trace_sched_copy);
103 folio_mark_dirty(folio);
104 } else {
105 finfo = netfs_folio_info(folio);
106 if (finfo) {
107 trace_netfs_folio(folio, netfs_folio_trace_filled_gaps);
108 if (finfo->netfs_group)
109 folio_change_private(folio, finfo->netfs_group);
110 else
111 folio_detach_private(folio);
112 kfree(finfo);
113 }
114 trace_netfs_folio(folio, netfs_folio_trace_read_done);
115 }
116
117 folioq_clear(folioq, slot);
118 } else {
119 // TODO: Use of PG_private_2 is deprecated.
120 if (folio_test_private_2(folio))
121 netfs_pgpriv2_copy_to_cache(rreq, folio);
122 }
123
124 just_unlock:
125 if (folio == rreq->no_unlock_folio &&
126 test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) {
127 _debug("no unlock");
128 } else {
129 trace_netfs_folio(folio, netfs_folio_trace_read_unlock);
130 folio_unlock(folio);
131 }
132
133 folioq_clear(folioq, slot);
134 }
135
136 /*
137 * Determine how much to gather before unlocking more folios.
138 */
netfs_read_set_unlock_at(struct netfs_io_request * rreq)139 void netfs_read_set_unlock_at(struct netfs_io_request *rreq)
140 {
141 struct folio_queue *folioq = rreq->buffer.tail;
142 unsigned int slot = rreq->buffer.first_tail_slot;
143 size_t cleaned_to = rreq->cleaned_to - rreq->start;
144 size_t progress_at = cleaned_to;
145 size_t minimum = 256 * 1024;
146
147 while (progress_at < rreq->len) {
148 if (slot >= folioq_count(folioq)) {
149 folioq = folioq->next;
150 if (!folioq)
151 break;
152 slot = 0;
153 }
154
155 progress_at += folioq_folio_size(folioq, slot);
156 if (progress_at - cleaned_to >= minimum)
157 break;
158 slot++;
159 }
160
161 WRITE_ONCE(rreq->progress_at, progress_at);
162 trace_netfs_read_progress_at(rreq);
163 }
164
165 /*
166 * Unlock any folios we've finished with.
167 */
netfs_read_unlock_folios(struct netfs_io_request * rreq,unsigned int * notes)168 static void netfs_read_unlock_folios(struct netfs_io_request *rreq,
169 unsigned int *notes)
170 {
171 struct folio_queue *folioq = rreq->buffer.tail;
172 unsigned int slot = rreq->buffer.first_tail_slot;
173 uoff_t collected_to = rreq->collected_to;
174
175 if (rreq->cleaned_to >= rreq->collected_to)
176 return;
177
178 // TODO: Begin decryption
179
180 if (slot >= folioq_nr_slots(folioq)) {
181 folioq = rolling_buffer_delete_spent(&rreq->buffer);
182 if (!folioq) {
183 WRITE_ONCE(rreq->progress_at, rreq->len);
184 return;
185 }
186 slot = 0;
187 }
188
189 /* We have to wait for readahead refs to have been released before we
190 * can unlock any folios as the ref-dropper walks i_pages and the only
191 * thing preventing these folios from being removed is the folio lock.
192 */
193 if (test_bit(NETFS_RREQ_NEED_PUT_RA_REFS, &rreq->flags))
194 netfs_wait_for_put_ra_refs(rreq);
195
196 for (;;) {
197 struct folio *folio;
198 uoff_t fpos, fend;
199 size_t fsize;
200
201 folio = folioq_folio(folioq, slot);
202 if (WARN_ONCE(!folio_test_locked(folio),
203 "R=%08x: folio %lx is not locked\n",
204 rreq->debug_id, folio->index))
205 trace_netfs_folio(folio, netfs_folio_trace_not_locked);
206
207 fsize = folioq_folio_size(folioq, slot);
208 fpos = folio_pos(folio);
209 fend = fpos + fsize;
210
211 trace_netfs_collect_folio(rreq, folio);
212
213 /* Unlock any folio we've transferred all of. */
214 if (collected_to < fend)
215 break;
216
217 netfs_unlock_read_folio(rreq, folioq, slot);
218 WRITE_ONCE(rreq->cleaned_to, fpos + fsize);
219 *notes |= MADE_PROGRESS;
220
221 /* Clean up the head folioq. If we clear an entire folioq, then
222 * we can get rid of it provided it's not also the tail folioq
223 * being filled by the issuer.
224 */
225 folioq_clear(folioq, slot);
226 slot++;
227 if (slot >= folioq_nr_slots(folioq)) {
228 folioq = rolling_buffer_delete_spent(&rreq->buffer);
229 if (!folioq)
230 goto done;
231 slot = 0;
232 trace_netfs_folioq(folioq, netfs_trace_folioq_read_progress);
233 }
234
235 if (fpos + fsize >= collected_to)
236 break;
237 }
238
239 rreq->buffer.tail = folioq;
240 done:
241 rreq->buffer.first_tail_slot = slot;
242
243 netfs_read_set_unlock_at(rreq);
244 }
245
246 /*
247 * Collect and assess the results of various read subrequests. We may need to
248 * retry some of the results.
249 *
250 * Note that we have a sequence of subrequests, which may be drawing on
251 * different sources and may or may not be the same size or starting position
252 * and may not even correspond in boundary alignment.
253 */
netfs_collect_read_results(struct netfs_io_request * rreq)254 static void netfs_collect_read_results(struct netfs_io_request *rreq)
255 {
256 struct netfs_io_subrequest *front, *remove;
257 struct netfs_io_stream *stream = &rreq->io_streams[0];
258 unsigned int notes;
259
260 _enter("%llx-%llx", rreq->start, rreq->start + rreq->len);
261 trace_netfs_rreq(rreq, netfs_rreq_trace_collect);
262 trace_netfs_collect(rreq);
263
264 reassess:
265 if (rreq->origin == NETFS_READAHEAD ||
266 rreq->origin == NETFS_READPAGE ||
267 rreq->origin == NETFS_READ_FOR_WRITE)
268 notes = BUFFERED;
269 else
270 notes = 0;
271
272 /* Remove completed subrequests from the front of the stream and
273 * advance the completion point. We stop when we hit something that's
274 * in progress. The issuer thread may be adding stuff to the tail
275 * whilst we're doing this.
276 */
277 front = list_first_entry_or_null_acquire(&stream->subrequests,
278 struct netfs_io_subrequest, rreq_link);
279 /* Read first subreq pointer before IN_PROGRESS flag. */
280
281 while (front) {
282 size_t transferred;
283
284 trace_netfs_collect_sreq(rreq, front);
285 _debug("sreq [%x] %llx %zx/%zx",
286 front->debug_index, front->start, front->transferred, front->len);
287
288 if (stream->collected_to < front->start) {
289 trace_netfs_collect_gap(rreq, stream, front->start, 'F');
290 stream->collected_to = front->start;
291 }
292
293 if (netfs_check_subreq_in_progress(front))
294 notes |= HIT_PENDING;
295 smp_rmb(); /* Read counters after IN_PROGRESS flag. */
296 transferred = READ_ONCE(front->transferred);
297
298 /* If we can now collect the next folio, do so. We don't want
299 * to defer this as we have to decide whether we need to copy
300 * to the cache or not, and that may differ between adjacent
301 * subreqs.
302 */
303 if (notes & BUFFERED) {
304 uoff_t unlock_at = rreq->start + rreq->progress_at;
305
306 /* Clear the tail of a short read. */
307 if (!(notes & HIT_PENDING) &&
308 front->error == 0 &&
309 transferred < front->len &&
310 (test_bit(NETFS_SREQ_HIT_EOF, &front->flags) ||
311 test_bit(NETFS_SREQ_CLEAR_TAIL, &front->flags))) {
312 netfs_clear_unread(front);
313 transferred = front->transferred = front->len;
314 trace_netfs_sreq(front, netfs_sreq_trace_clear);
315 }
316
317 stream->collected_to = front->start + transferred;
318 rreq->collected_to = stream->collected_to;
319
320 if (test_bit(NETFS_SREQ_FAILED, &front->flags)) {
321 rreq->abandon_to = front->start + front->len;
322 front->transferred = front->len;
323 transferred = front->len;
324 trace_netfs_rreq(rreq, netfs_rreq_trace_set_abandon);
325 }
326 if (front->start + transferred >= unlock_at ||
327 test_bit(NETFS_SREQ_HIT_EOF, &front->flags))
328 netfs_read_unlock_folios(rreq, ¬es);
329 } else {
330 if (!(notes & HIT_PENDING) &&
331 front->error == 0 &&
332 transferred < front->len &&
333 test_bit(NETFS_SREQ_CLEAR_TAIL, &front->flags)) {
334 netfs_clear_unread_dio(front);
335 transferred = front->transferred;
336 trace_netfs_sreq(front, netfs_sreq_trace_clear);
337 }
338 stream->collected_to = front->start + transferred;
339 rreq->collected_to = stream->collected_to;
340 }
341
342 /* Stall if the front is still undergoing I/O. */
343 if (notes & HIT_PENDING)
344 break;
345
346 if (test_bit(NETFS_SREQ_FAILED, &front->flags)) {
347 if (!stream->failed) {
348 stream->error = front->error;
349 rreq->error = front->error;
350 set_bit(NETFS_RREQ_FAILED, &rreq->flags);
351 stream->failed = true;
352 }
353 notes |= MADE_PROGRESS | ABANDON_SREQ;
354 } else if (test_bit(NETFS_SREQ_NEED_RETRY, &front->flags)) {
355 stream->need_retry = true;
356 notes |= NEED_RETRY | MADE_PROGRESS;
357 break;
358 } else if (test_bit(NETFS_RREQ_SHORT_TRANSFER, &rreq->flags)) {
359 notes |= MADE_PROGRESS;
360 } else {
361 if (!stream->failed) {
362 stream->transferred += transferred;
363 stream->transferred_valid = true;
364 }
365 if (front->transferred < front->len)
366 set_bit(NETFS_RREQ_SHORT_TRANSFER, &rreq->flags);
367 notes |= MADE_PROGRESS;
368 }
369
370 /* Remove if completely consumed. */
371 stream->source = front->source;
372 spin_lock(&rreq->lock);
373
374 remove = front;
375 trace_netfs_sreq(front,
376 notes & ABANDON_SREQ ?
377 netfs_sreq_trace_abandoned : netfs_sreq_trace_consumed);
378 list_del_init(&front->rreq_link);
379 front = list_first_entry_or_null(&stream->subrequests,
380 struct netfs_io_subrequest, rreq_link);
381 spin_unlock(&rreq->lock);
382 netfs_put_subrequest(remove,
383 notes & ABANDON_SREQ ?
384 netfs_sreq_trace_put_abandon :
385 netfs_sreq_trace_put_done);
386 }
387
388 trace_netfs_collect_stream(rreq, stream);
389 trace_netfs_collect_state(rreq, rreq->collected_to, notes);
390
391 if (!(notes & BUFFERED))
392 rreq->cleaned_to = rreq->collected_to;
393
394 if (notes & NEED_RETRY)
395 goto need_retry;
396 if (notes & MADE_PROGRESS) {
397 netfs_wake_rreq_flag(rreq, NETFS_RREQ_PAUSE, netfs_rreq_trace_unpause);
398 //cond_resched();
399 goto reassess;
400 }
401
402 out:
403 _leave(" = %x", notes);
404 return;
405
406 need_retry:
407 /* Okay... We're going to have to retry parts of the stream. Note
408 * that any partially completed op will have had any wholly transferred
409 * folios removed from it.
410 */
411 _debug("retry");
412 netfs_retry_reads(rreq);
413 goto out;
414 }
415
416 /*
417 * Do page flushing and suchlike after DIO.
418 */
netfs_rreq_assess_dio(struct netfs_io_request * rreq)419 static void netfs_rreq_assess_dio(struct netfs_io_request *rreq)
420 {
421 unsigned int i;
422
423 if (rreq->origin == NETFS_UNBUFFERED_READ ||
424 rreq->origin == NETFS_DIO_READ) {
425 for (i = 0; i < rreq->direct_bv_count; i++) {
426 flush_dcache_page(rreq->direct_bv[i].bv_page);
427 // TODO: cifs marks pages in the destination buffer
428 // dirty under some circumstances after a read. Do we
429 // need to do that too?
430 set_page_dirty(rreq->direct_bv[i].bv_page);
431 }
432 }
433
434 if (rreq->iocb) {
435 rreq->iocb->ki_pos += rreq->transferred;
436 if (rreq->iocb->ki_complete) {
437 trace_netfs_rreq(rreq, netfs_rreq_trace_ki_complete);
438 rreq->iocb->ki_complete(
439 rreq->iocb, rreq->error ? rreq->error : rreq->transferred);
440 }
441 }
442 if (rreq->netfs_ops->done)
443 rreq->netfs_ops->done(rreq);
444 if (rreq->origin == NETFS_UNBUFFERED_READ ||
445 rreq->origin == NETFS_DIO_READ)
446 inode_dio_end(rreq->inode);
447 }
448
449 /*
450 * Do processing after reading a monolithic single object.
451 */
netfs_rreq_assess_single(struct netfs_io_request * rreq)452 static void netfs_rreq_assess_single(struct netfs_io_request *rreq)
453 {
454 struct netfs_io_stream *stream = &rreq->io_streams[0];
455
456 if (!rreq->error && stream->source == NETFS_DOWNLOAD_FROM_SERVER &&
457 fscache_resources_valid(&rreq->cache_resources)) {
458 trace_netfs_rreq(rreq, netfs_rreq_trace_dirty);
459 netfs_single_mark_inode_dirty(rreq->inode);
460 }
461
462 /* To do DIO, the cache has to round the size up, so we need to undo
463 * the rounding.
464 */
465 rreq->transferred = min(rreq->transferred, rreq->i_size);
466
467 if (rreq->iocb) {
468 rreq->iocb->ki_pos += rreq->transferred;
469 if (rreq->iocb->ki_complete) {
470 trace_netfs_rreq(rreq, netfs_rreq_trace_ki_complete);
471 rreq->iocb->ki_complete(
472 rreq->iocb, rreq->error ? rreq->error : rreq->transferred);
473 }
474 }
475 if (rreq->netfs_ops->done)
476 rreq->netfs_ops->done(rreq);
477 }
478
479 /*
480 * Perform the collection of subrequests and folios.
481 *
482 * Note that we're in normal kernel thread context at this point, possibly
483 * running on a workqueue.
484 */
netfs_read_collection(struct netfs_io_request * rreq)485 bool netfs_read_collection(struct netfs_io_request *rreq)
486 {
487 struct netfs_io_stream *stream = &rreq->io_streams[0];
488
489 netfs_collect_read_results(rreq);
490
491 /* We're done when the app thread has finished posting subreqs and the
492 * queue is empty.
493 */
494 if (!netfs_are_all_subreqs_queued(rreq))
495 return false;
496 if (!list_empty(&stream->subrequests))
497 return false;
498
499 /* Okay, declare that all I/O is complete. */
500 rreq->transferred = stream->transferred;
501 trace_netfs_rreq(rreq, netfs_rreq_trace_complete);
502
503 //netfs_rreq_is_still_valid(rreq);
504
505 switch (rreq->origin) {
506 case NETFS_UNBUFFERED_READ:
507 case NETFS_DIO_READ:
508 case NETFS_READ_GAPS:
509 netfs_rreq_assess_dio(rreq);
510 break;
511 case NETFS_READ_SINGLE:
512 netfs_rreq_assess_single(rreq);
513 break;
514 default:
515 break;
516 }
517 task_io_account_read(rreq->transferred);
518
519 netfs_wake_rreq_flag(rreq, NETFS_RREQ_IN_PROGRESS, netfs_rreq_trace_wake_ip);
520 /* As we cleared NETFS_RREQ_IN_PROGRESS, we acquired its ref. */
521
522 trace_netfs_rreq(rreq, netfs_rreq_trace_done);
523 netfs_clear_subrequests(rreq);
524 netfs_unlock_abandoned_read_pages(rreq);
525 if (unlikely(rreq->copy_to_cache))
526 netfs_pgpriv2_end_copy_to_cache(rreq);
527 return true;
528 }
529
netfs_read_collection_worker(struct work_struct * work)530 void netfs_read_collection_worker(struct work_struct *work)
531 {
532 struct netfs_io_request *rreq = container_of(work, struct netfs_io_request, work);
533
534 netfs_see_request(rreq, netfs_rreq_trace_see_work);
535 if (netfs_check_rreq_in_progress(rreq)) {
536 if (netfs_read_collection(rreq))
537 /* Drop the ref from the IN_PROGRESS flag. */
538 netfs_put_request(rreq, netfs_rreq_trace_put_work_ip);
539 else
540 netfs_see_request(rreq, netfs_rreq_trace_see_work_complete);
541 }
542 }
543
544 /**
545 * netfs_read_subreq_progress - Note progress of a read operation.
546 * @subreq: The read request that has terminated.
547 *
548 * This tells the read side of netfs lib that a contributory I/O operation has
549 * made some progress and that it may be possible to unlock some folios.
550 *
551 * Before calling, the filesystem should update subreq->transferred to track
552 * the amount of data copied into the output buffer.
553 */
netfs_read_subreq_progress(struct netfs_io_subrequest * subreq)554 void netfs_read_subreq_progress(struct netfs_io_subrequest *subreq)
555 {
556 struct netfs_io_request *rreq = subreq->rreq;
557 struct netfs_io_stream *stream = &rreq->io_streams[subreq->stream_nr];
558 size_t progress_at = READ_ONCE(rreq->progress_at);
559 uoff_t update_at = rreq->start + progress_at;
560 uoff_t transferred_to = subreq->start + subreq->transferred;
561
562 /* If we are at the head of the queue, wake up the collector,
563 * getting a ref to it if we were the ones to do so.
564 */
565 if (progress_at < rreq->len &&
566 transferred_to >= update_at &&
567 (rreq->origin == NETFS_READAHEAD ||
568 rreq->origin == NETFS_READPAGE ||
569 rreq->origin == NETFS_READ_FOR_WRITE) &&
570 list_is_first(&subreq->rreq_link, &stream->subrequests)
571 ) {
572 trace_netfs_sreq(subreq, netfs_sreq_trace_progress);
573 __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
574 netfs_wake_collector(rreq);
575 }
576 }
577 EXPORT_SYMBOL(netfs_read_subreq_progress);
578
579 /**
580 * netfs_read_subreq_terminated - Note the termination of an I/O operation.
581 * @subreq: The I/O request that has terminated.
582 *
583 * This tells the read helper that a contributory I/O operation has terminated,
584 * one way or another, and that it should integrate the results.
585 *
586 * The caller indicates the outcome of the operation through @subreq->error,
587 * supplying 0 to indicate a successful or retryable transfer (if
588 * NETFS_SREQ_NEED_RETRY is set) or a negative error code. The helper will
589 * look after reissuing I/O operations as appropriate and writing downloaded
590 * data to the cache.
591 *
592 * Before calling, the filesystem should update subreq->transferred to track
593 * the amount of data copied into the output buffer.
594 */
netfs_read_subreq_terminated(struct netfs_io_subrequest * subreq)595 void netfs_read_subreq_terminated(struct netfs_io_subrequest *subreq)
596 {
597 struct netfs_io_request *rreq = subreq->rreq;
598
599 switch (subreq->source) {
600 case NETFS_READ_FROM_CACHE:
601 netfs_stat(&netfs_n_rh_read_done);
602 break;
603 case NETFS_DOWNLOAD_FROM_SERVER:
604 netfs_stat(&netfs_n_rh_download_done);
605 break;
606 default:
607 break;
608 }
609
610 /* Deal with retry requests, short reads and errors. If we retry
611 * but don't make progress, we abandon the attempt.
612 */
613 if (!subreq->error && subreq->transferred < subreq->len) {
614 if (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags)) {
615 trace_netfs_sreq(subreq, netfs_sreq_trace_hit_eof);
616 } else if (test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags)) {
617 trace_netfs_sreq(subreq, netfs_sreq_trace_need_clear);
618 } else if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
619 trace_netfs_sreq(subreq, netfs_sreq_trace_need_retry);
620 } else if (test_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags)) {
621 __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
622 trace_netfs_sreq(subreq, netfs_sreq_trace_partial_read);
623 } else {
624 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
625 subreq->error = -ENODATA;
626 trace_netfs_sreq(subreq, netfs_sreq_trace_short);
627 }
628 }
629
630 /* If need retry is set, error should not matter unless we hit too many
631 * retries. Pause the generation of new subreqs
632 */
633 if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) {
634 trace_netfs_rreq(rreq, netfs_rreq_trace_set_pause);
635 set_bit(NETFS_RREQ_PAUSE, &rreq->flags);
636 goto skip_error_checks;
637 }
638
639 if (unlikely(subreq->error < 0)) {
640 trace_netfs_failure(rreq, subreq, subreq->error, netfs_fail_read);
641 if (subreq->source == NETFS_READ_FROM_CACHE) {
642 netfs_stat(&netfs_n_rh_read_failed);
643 __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags);
644 } else {
645 netfs_stat(&netfs_n_rh_download_failed);
646 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
647 }
648 trace_netfs_rreq(rreq, netfs_rreq_trace_set_pause);
649 set_bit(NETFS_RREQ_PAUSE, &rreq->flags);
650 }
651
652 skip_error_checks:
653 trace_netfs_sreq(subreq, netfs_sreq_trace_terminated);
654 netfs_subreq_clear_in_progress(subreq);
655 netfs_put_subrequest(subreq, netfs_sreq_trace_put_terminated);
656 }
657 EXPORT_SYMBOL(netfs_read_subreq_terminated);
658
659 /*
660 * Cancel a read subrequest due to preparation failure.
661 */
netfs_cancel_read(struct netfs_io_subrequest * subreq,int error)662 void netfs_cancel_read(struct netfs_io_subrequest *subreq, int error)
663 {
664 trace_netfs_sreq(subreq, netfs_sreq_trace_cancel);
665 subreq->error = error;
666 __set_bit(NETFS_SREQ_FAILED, &subreq->flags);
667 netfs_read_subreq_terminated(subreq);
668 }
669
670 /*
671 * Handle termination of a read from the cache.
672 */
netfs_cache_read_terminated(void * priv,ssize_t transferred_or_error)673 void netfs_cache_read_terminated(void *priv, ssize_t transferred_or_error)
674 {
675 struct netfs_io_subrequest *subreq = priv;
676
677 if (transferred_or_error > 0) {
678 subreq->error = 0;
679 if (transferred_or_error > 0) {
680 subreq->transferred += transferred_or_error;
681 __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags);
682 }
683 } else {
684 subreq->error = transferred_or_error;
685 }
686 netfs_read_subreq_terminated(subreq);
687 }
688