Lines Matching +full:x +full:- +full:origin
1 // SPDX-License-Identifier: GPL-2.0-only
31 WARN_ON_ONCE(subreq->len - subreq->transferred != iov_iter_count(&subreq->io_iter)); in netfs_clear_unread()
32 iov_iter_zero(iov_iter_count(&subreq->io_iter), &subreq->io_iter); in netfs_clear_unread()
33 if (subreq->start + subreq->transferred >= subreq->rreq->i_size) in netfs_clear_unread()
34 __set_bit(NETFS_SREQ_HIT_EOF, &subreq->flags); in netfs_clear_unread()
49 if (unlikely(folio_pos(folio) < rreq->abandon_to)) { in netfs_unlock_read_folio()
57 if (!test_bit(NETFS_RREQ_USE_PGPRIV2, &rreq->flags)) { in netfs_unlock_read_folio()
61 if (finfo->netfs_group) in netfs_unlock_read_folio()
62 folio_change_private(folio, finfo->netfs_group); in netfs_unlock_read_folio()
68 if (test_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags)) { in netfs_unlock_read_folio()
81 if (test_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags)) in netfs_unlock_read_folio()
86 if (folio->index == rreq->no_unlock_folio && in netfs_unlock_read_folio()
87 test_bit(NETFS_RREQ_NO_UNLOCK_FOLIO, &rreq->flags)) { in netfs_unlock_read_folio()
103 struct folio_queue *folioq = rreq->buffer.tail; in netfs_read_unlock_folios()
104 unsigned long long collected_to = rreq->collected_to; in netfs_read_unlock_folios()
105 unsigned int slot = rreq->buffer.first_tail_slot; in netfs_read_unlock_folios()
107 if (rreq->cleaned_to >= rreq->collected_to) in netfs_read_unlock_folios()
113 folioq = rolling_buffer_delete_spent(&rreq->buffer); in netfs_read_unlock_folios()
115 rreq->front_folio_order = 0; in netfs_read_unlock_folios()
128 set_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags); in netfs_read_unlock_folios()
132 "R=%08x: folio %lx is not locked\n", in netfs_read_unlock_folios()
133 rreq->debug_id, folio->index)) in netfs_read_unlock_folios()
137 rreq->front_folio_order = order; in netfs_read_unlock_folios()
140 fend = umin(fpos + fsize, rreq->i_size); in netfs_read_unlock_folios()
149 WRITE_ONCE(rreq->cleaned_to, fpos + fsize); in netfs_read_unlock_folios()
152 clear_bit(NETFS_RREQ_FOLIO_COPY_TO_CACHE, &rreq->flags); in netfs_read_unlock_folios()
161 folioq = rolling_buffer_delete_spent(&rreq->buffer); in netfs_read_unlock_folios()
172 rreq->buffer.tail = folioq; in netfs_read_unlock_folios()
174 rreq->buffer.first_tail_slot = slot; in netfs_read_unlock_folios()
188 struct netfs_io_stream *stream = &rreq->io_streams[0]; in netfs_collect_read_results()
191 _enter("%llx-%llx", rreq->start, rreq->start + rreq->len); in netfs_collect_read_results()
196 if (rreq->origin == NETFS_READAHEAD || in netfs_collect_read_results()
197 rreq->origin == NETFS_READPAGE || in netfs_collect_read_results()
198 rreq->origin == NETFS_READ_FOR_WRITE) in netfs_collect_read_results()
208 front = READ_ONCE(stream->front); in netfs_collect_read_results()
213 _debug("sreq [%x] %llx %zx/%zx", in netfs_collect_read_results()
214 front->debug_index, front->start, front->transferred, front->len); in netfs_collect_read_results()
216 if (stream->collected_to < front->start) { in netfs_collect_read_results()
217 trace_netfs_collect_gap(rreq, stream, front->start, 'F'); in netfs_collect_read_results()
218 stream->collected_to = front->start; in netfs_collect_read_results()
224 transferred = READ_ONCE(front->transferred); in netfs_collect_read_results()
232 size_t fsize = PAGE_SIZE << rreq->front_folio_order; in netfs_collect_read_results()
236 front->error == 0 && in netfs_collect_read_results()
237 transferred < front->len && in netfs_collect_read_results()
238 (test_bit(NETFS_SREQ_HIT_EOF, &front->flags) || in netfs_collect_read_results()
239 test_bit(NETFS_SREQ_CLEAR_TAIL, &front->flags))) { in netfs_collect_read_results()
241 transferred = front->transferred = front->len; in netfs_collect_read_results()
245 stream->collected_to = front->start + transferred; in netfs_collect_read_results()
246 rreq->collected_to = stream->collected_to; in netfs_collect_read_results()
248 if (test_bit(NETFS_SREQ_COPY_TO_CACHE, &front->flags)) in netfs_collect_read_results()
251 if (test_bit(NETFS_SREQ_FAILED, &front->flags)) { in netfs_collect_read_results()
252 rreq->abandon_to = front->start + front->len; in netfs_collect_read_results()
253 front->transferred = front->len; in netfs_collect_read_results()
254 transferred = front->len; in netfs_collect_read_results()
257 if (front->start + transferred >= rreq->cleaned_to + fsize || in netfs_collect_read_results()
258 test_bit(NETFS_SREQ_HIT_EOF, &front->flags)) in netfs_collect_read_results()
261 stream->collected_to = front->start + transferred; in netfs_collect_read_results()
262 rreq->collected_to = stream->collected_to; in netfs_collect_read_results()
269 if (test_bit(NETFS_SREQ_FAILED, &front->flags)) { in netfs_collect_read_results()
270 if (!stream->failed) { in netfs_collect_read_results()
271 stream->error = front->error; in netfs_collect_read_results()
272 rreq->error = front->error; in netfs_collect_read_results()
273 set_bit(NETFS_RREQ_FAILED, &rreq->flags); in netfs_collect_read_results()
274 stream->failed = true; in netfs_collect_read_results()
277 } else if (test_bit(NETFS_SREQ_NEED_RETRY, &front->flags)) { in netfs_collect_read_results()
278 stream->need_retry = true; in netfs_collect_read_results()
281 } else if (test_bit(NETFS_RREQ_SHORT_TRANSFER, &rreq->flags)) { in netfs_collect_read_results()
284 if (!stream->failed) { in netfs_collect_read_results()
285 stream->transferred += transferred; in netfs_collect_read_results()
286 stream->transferred_valid = true; in netfs_collect_read_results()
288 if (front->transferred < front->len) in netfs_collect_read_results()
289 set_bit(NETFS_RREQ_SHORT_TRANSFER, &rreq->flags); in netfs_collect_read_results()
294 stream->source = front->source; in netfs_collect_read_results()
295 spin_lock(&rreq->lock); in netfs_collect_read_results()
301 list_del_init(&front->rreq_link); in netfs_collect_read_results()
302 front = list_first_entry_or_null(&stream->subrequests, in netfs_collect_read_results()
304 stream->front = front; in netfs_collect_read_results()
305 spin_unlock(&rreq->lock); in netfs_collect_read_results()
313 trace_netfs_collect_state(rreq, rreq->collected_to, notes); in netfs_collect_read_results()
316 rreq->cleaned_to = rreq->collected_to; in netfs_collect_read_results()
327 _leave(" = %x", notes); in netfs_collect_read_results()
347 if (rreq->origin == NETFS_UNBUFFERED_READ || in netfs_rreq_assess_dio()
348 rreq->origin == NETFS_DIO_READ) { in netfs_rreq_assess_dio()
349 for (i = 0; i < rreq->direct_bv_count; i++) { in netfs_rreq_assess_dio()
350 flush_dcache_page(rreq->direct_bv[i].bv_page); in netfs_rreq_assess_dio()
354 set_page_dirty(rreq->direct_bv[i].bv_page); in netfs_rreq_assess_dio()
358 if (rreq->iocb) { in netfs_rreq_assess_dio()
359 rreq->iocb->ki_pos += rreq->transferred; in netfs_rreq_assess_dio()
360 if (rreq->iocb->ki_complete) { in netfs_rreq_assess_dio()
362 rreq->iocb->ki_complete( in netfs_rreq_assess_dio()
363 rreq->iocb, rreq->error ? rreq->error : rreq->transferred); in netfs_rreq_assess_dio()
366 if (rreq->netfs_ops->done) in netfs_rreq_assess_dio()
367 rreq->netfs_ops->done(rreq); in netfs_rreq_assess_dio()
368 if (rreq->origin == NETFS_UNBUFFERED_READ || in netfs_rreq_assess_dio()
369 rreq->origin == NETFS_DIO_READ) in netfs_rreq_assess_dio()
370 inode_dio_end(rreq->inode); in netfs_rreq_assess_dio()
378 struct netfs_io_stream *stream = &rreq->io_streams[0]; in netfs_rreq_assess_single()
380 if (!rreq->error && stream->source == NETFS_DOWNLOAD_FROM_SERVER && in netfs_rreq_assess_single()
381 fscache_resources_valid(&rreq->cache_resources)) { in netfs_rreq_assess_single()
383 netfs_single_mark_inode_dirty(rreq->inode); in netfs_rreq_assess_single()
386 if (rreq->iocb) { in netfs_rreq_assess_single()
387 rreq->iocb->ki_pos += rreq->transferred; in netfs_rreq_assess_single()
388 if (rreq->iocb->ki_complete) { in netfs_rreq_assess_single()
390 rreq->iocb->ki_complete( in netfs_rreq_assess_single()
391 rreq->iocb, rreq->error ? rreq->error : rreq->transferred); in netfs_rreq_assess_single()
394 if (rreq->netfs_ops->done) in netfs_rreq_assess_single()
395 rreq->netfs_ops->done(rreq); in netfs_rreq_assess_single()
406 struct netfs_io_stream *stream = &rreq->io_streams[0]; in netfs_read_collection()
413 if (!test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags)) in netfs_read_collection()
417 if (!list_empty(&stream->subrequests)) in netfs_read_collection()
421 rreq->transferred = stream->transferred; in netfs_read_collection()
426 switch (rreq->origin) { in netfs_read_collection()
438 task_io_account_read(rreq->transferred); in netfs_read_collection()
446 if (unlikely(rreq->copy_to_cache)) in netfs_read_collection()
466 * netfs_read_subreq_progress - Note progress of a read operation.
472 * Before calling, the filesystem should update subreq->transferred to track
477 struct netfs_io_request *rreq = subreq->rreq; in netfs_read_subreq_progress()
478 struct netfs_io_stream *stream = &rreq->io_streams[0]; in netfs_read_subreq_progress()
479 size_t fsize = PAGE_SIZE << rreq->front_folio_order; in netfs_read_subreq_progress()
486 if (subreq->start + subreq->transferred > rreq->cleaned_to + fsize && in netfs_read_subreq_progress()
487 (rreq->origin == NETFS_READAHEAD || in netfs_read_subreq_progress()
488 rreq->origin == NETFS_READPAGE || in netfs_read_subreq_progress()
489 rreq->origin == NETFS_READ_FOR_WRITE) && in netfs_read_subreq_progress()
490 list_is_first(&subreq->rreq_link, &stream->subrequests) in netfs_read_subreq_progress()
492 __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); in netfs_read_subreq_progress()
499 * netfs_read_subreq_terminated - Note the termination of an I/O operation.
505 * The caller indicates the outcome of the operation through @subreq->error,
511 * Before calling, the filesystem should update subreq->transferred to track
516 struct netfs_io_request *rreq = subreq->rreq; in netfs_read_subreq_terminated()
518 switch (subreq->source) { in netfs_read_subreq_terminated()
532 if (!subreq->error && subreq->transferred < subreq->len) { in netfs_read_subreq_terminated()
533 if (test_bit(NETFS_SREQ_HIT_EOF, &subreq->flags)) { in netfs_read_subreq_terminated()
535 } else if (test_bit(NETFS_SREQ_CLEAR_TAIL, &subreq->flags)) { in netfs_read_subreq_terminated()
537 } else if (test_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags)) { in netfs_read_subreq_terminated()
539 } else if (test_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags)) { in netfs_read_subreq_terminated()
540 __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); in netfs_read_subreq_terminated()
543 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); in netfs_read_subreq_terminated()
544 subreq->error = -ENODATA; in netfs_read_subreq_terminated()
549 if (unlikely(subreq->error < 0)) { in netfs_read_subreq_terminated()
550 trace_netfs_failure(rreq, subreq, subreq->error, netfs_fail_read); in netfs_read_subreq_terminated()
551 if (subreq->source == NETFS_READ_FROM_CACHE) { in netfs_read_subreq_terminated()
553 __set_bit(NETFS_SREQ_NEED_RETRY, &subreq->flags); in netfs_read_subreq_terminated()
556 __set_bit(NETFS_SREQ_FAILED, &subreq->flags); in netfs_read_subreq_terminated()
559 set_bit(NETFS_RREQ_PAUSE, &rreq->flags); in netfs_read_subreq_terminated()
576 subreq->error = 0; in netfs_cache_read_terminated()
578 subreq->transferred += transferred_or_error; in netfs_cache_read_terminated()
579 __set_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags); in netfs_cache_read_terminated()
582 subreq->error = transferred_or_error; in netfs_cache_read_terminated()