1 // SPDX-License-Identifier: GPL-2.0-only
3  * mm/readahead.c - address_space-level file readahead.
17  * folio is present but not up-to-date, readahead will not try to read
18  * it. In that case a simple ->read_folio() will be requested.
29  * contains ->size being the total number of pages, and ->async_size
37  * to be determined: the start of the region to read, the size of the
40  * The start of the region is simply the first page address at or after
46  * this would be less than zero - then zero is used.  NOTE THIS
47  * CALCULATION IS WRONG WHEN THE START OF THE REGION IS NOT THE ACCESSED
57  * page to the start of the new readahead.  In these cases, the size of
65  * larger than the current request, and it is not scaled up, unless it
66  * is at the start of file.
68  * In general readahead is accelerated at the start of the file, as
77  * Readahead requests are sent to the filesystem using the ->readahead()
79  * implementation.  ->readahead() should normally initiate reads on all
81  * error.  The page cache reading code will issue a ->read_folio() request
82  * for any folio which ->readahead() did not read, and only an error
85  * ->readahead() will generally call readahead_folio() repeatedly to get
103  * considered to be important and ->readahead() should not fail them due
113  * ->read_folio() which may be less efficient.
121 #include <linux/backing-dev.h>
128 #include <linux/blk-cgroup.h>
141 	ra->ra_pages = inode_to_bdi(mapping->host)->ra_pages;  in file_ra_state_init()
142 	ra->prev_pos = -1;  in file_ra_state_init()
148 	const struct address_space_operations *aops = rac->mapping->a_ops;  in read_pages()
155 	if (unlikely(rac->_workingset))  in read_pages()
156 		psi_memstall_enter(&rac->_pflags);  in read_pages()
159 	if (aops->readahead) {  in read_pages()
160 		aops->readahead(rac);  in read_pages()
161 		/* Clean up the remaining folios. */  in read_pages()
170 			aops->read_folio(rac->file, folio);  in read_pages()
174 	if (unlikely(rac->_workingset))  in read_pages()
175 		psi_memstall_leave(&rac->_pflags);  in read_pages()
176 	rac->_workingset = false;  in read_pages()
187 	if (folio && ractl->dropbehind)  in ractl_alloc_folio()
194  * page_cache_ra_unbounded - Start unchecked readahead.
197  * @lookahead_size: Where to start the next readahead.
199  * This function is for filesystems to call when they want to start
210 	struct address_space *mapping = ractl->mapping;  in page_cache_ra_unbounded()
222 	 * touch file-backed pages, preventing a deadlock.  Most (all?)  in page_cache_ra_unbounded()
241 					  nr_to_read - lookahead_size,  in page_cache_ra_unbounded()
243 		mark = ra_folio_index - index;  in page_cache_ra_unbounded()
245 	nr_to_read += readahead_index(ractl) - index;  in page_cache_ra_unbounded()
246 	ractl->_index = index;  in page_cache_ra_unbounded()
252 		struct folio *folio = xa_load(&mapping->i_pages, index + i);  in page_cache_ra_unbounded()
265 			ractl->_index += min_nrpages;  in page_cache_ra_unbounded()
266 			i = ractl->_index + ractl->_nr_pages - index;  in page_cache_ra_unbounded()
278 			if (ret == -ENOMEM)  in page_cache_ra_unbounded()
281 			ractl->_index += min_nrpages;  in page_cache_ra_unbounded()
282 			i = ractl->_index + ractl->_nr_pages - index;  in page_cache_ra_unbounded()
287 		ractl->_workingset |= folio_test_workingset(folio);  in page_cache_ra_unbounded()
288 		ractl->_nr_pages += min_nrpages;  in page_cache_ra_unbounded()
293 	 * Now start the IO.  We ignore I/O errors - if the folio is not  in page_cache_ra_unbounded()
312 	struct inode *inode = ractl->mapping->host;  in do_page_cache_ra()
320 	end_index = (isize - 1) >> PAGE_SHIFT;  in do_page_cache_ra()
324 	if (nr_to_read > end_index - index)  in do_page_cache_ra()
325 		nr_to_read = end_index - index + 1;  in do_page_cache_ra()
337 	struct address_space *mapping = ractl->mapping;  in force_page_cache_ra()
338 	struct file_ra_state *ra = ractl->ra;  in force_page_cache_ra()
339 	struct backing_dev_info *bdi = inode_to_bdi(mapping->host);  in force_page_cache_ra()
342 	if (unlikely(!mapping->a_ops->read_folio && !mapping->a_ops->readahead))  in force_page_cache_ra()
347 	 * be up to the optimal hardware IO size  in force_page_cache_ra()
349 	max_pages = max_t(unsigned long, bdi->io_pages, ra->ra_pages);  in force_page_cache_ra()
358 		nr_to_read -= this_chunk;  in force_page_cache_ra()
366  * 1-2 page = 16k, 3-4 page 32k, 5-8 page = 64k, > 8 page = 128k initial
383  *  Get the previous window size, ramp it up, and
389 	unsigned long cur = ra->size;  in get_next_ra_size()
399  * On-demand readahead design.
401  * The fields in struct file_ra_state represent the most-recently-executed
404  *                        |<----- async_size ---------|
405  *     |------------------- size -------------------->|
407  *     ^start             ^page marked with PG_readahead
418  * page at (start+size-async_size) with PG_readahead, and use it as readahead
420  * readahead-for-nothing fuss, saving pointless page cache lookups.
428  * There is a special-case: if the first page which the application tries to
433  * The code ramps up the readahead size aggressively at first, but slow down as
444 		return -ENOMEM;  in ra_alloc_folio()
448 	err = filemap_add_folio(ractl->mapping, folio, index, gfp);  in ra_alloc_folio()
454 	ractl->_nr_pages += 1UL << order;  in ra_alloc_folio()
455 	ractl->_workingset |= folio_test_workingset(folio);  in ra_alloc_folio()
462 	struct address_space *mapping = ractl->mapping;  in page_cache_ra_order()
463 	pgoff_t start = readahead_index(ractl);  in page_cache_ra_order()  local
464 	pgoff_t index = start;  in page_cache_ra_order()
466 	pgoff_t limit = (i_size_read(mapping->host) - 1) >> PAGE_SHIFT;  in page_cache_ra_order()
467 	pgoff_t mark = index + ra->size - ra->async_size;  in page_cache_ra_order()
477 	if (!mapping_large_folio_support(mapping) || ra->size < min_ra_size)  in page_cache_ra_order()
480 	limit = min(limit, index + ra->size - 1);  in page_cache_ra_order()
486 	new_order = min_t(unsigned int, new_order, ilog2(ra->size));  in page_cache_ra_order()
497 	ractl->_index = mapping_align_index(mapping, index);  in page_cache_ra_order()
504 		if (index & ((1UL << order) - 1))  in page_cache_ra_order()
507 		while (order > min_order && index + (1UL << order) - 1 > limit)  in page_cache_ra_order()
508 			order--;  in page_cache_ra_order()
528 	 * ->readahead() may have updated readahead window size so we have to  in page_cache_ra_order()
531 	if (ra->size > index - start)  in page_cache_ra_order()
532 		do_page_cache_ra(ractl, ra->size - (index - start),  in page_cache_ra_order()
533 				 ra->async_size);  in page_cache_ra_order()
539 	struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host);  in ractl_max_pages()
540 	unsigned long max_pages = ractl->ra->ra_pages;  in ractl_max_pages()
544 	 * be up to the optimal hardware IO size  in ractl_max_pages()
546 	if (req_size > max_pages && bdi->io_pages > max_pages)  in ractl_max_pages()
547 		max_pages = min(req_size, bdi->io_pages);  in ractl_max_pages()
555 	bool do_forced_ra = ractl->file && (ractl->file->f_mode & FMODE_RANDOM);  in page_cache_sync_ra()
556 	struct file_ra_state *ra = ractl->ra;  in page_cache_sync_ra()
566 	if (!ra->ra_pages || blk_cgroup_congested()) {  in page_cache_sync_ra()
567 		if (!ractl->file)  in page_cache_sync_ra()
580 	prev_index = (unsigned long long)ra->prev_pos >> PAGE_SHIFT;  in page_cache_sync_ra()
582 	 * A start of file, oversized read, or sequential cache miss:  in page_cache_sync_ra()
583 	 * trivial case: (index - prev_index) == 1  in page_cache_sync_ra()
584 	 * unaligned reads: (index - prev_index) == 0  in page_cache_sync_ra()
586 	if (!index || req_count > max_pages || index - prev_index <= 1UL) {  in page_cache_sync_ra()
587 		ra->start = index;  in page_cache_sync_ra()
588 		ra->size = get_init_ra_size(req_count, max_pages);  in page_cache_sync_ra()
589 		ra->async_size = ra->size > req_count ? ra->size - req_count :  in page_cache_sync_ra()
590 							ra->size >> 1;  in page_cache_sync_ra()
599 	miss = page_cache_prev_miss(ractl->mapping, index - 1, max_pages);  in page_cache_sync_ra()
601 	contig_count = index - miss - 1;  in page_cache_sync_ra()
612 	 * it is a strong indication of long-run stream (or whole-file-read)  in page_cache_sync_ra()
616 	ra->start = index;  in page_cache_sync_ra()
617 	ra->size = min(contig_count + req_count, max_pages);  in page_cache_sync_ra()
618 	ra->async_size = 1;  in page_cache_sync_ra()
620 	ractl->_index = ra->start;  in page_cache_sync_ra()
629 	struct file_ra_state *ra = ractl->ra;  in page_cache_async_ra()
631 	pgoff_t expected, start;  in page_cache_async_ra()  local
635 	if (!ra->ra_pages)  in page_cache_async_ra()
652 	 * Ramp up sizes, and push forward the readahead window.  in page_cache_async_ra()
654 	expected = round_down(ra->start + ra->size - ra->async_size,  in page_cache_async_ra()
657 		ra->start += ra->size;  in page_cache_async_ra()
662 		ra->size = max(ra->size, get_next_ra_size(ra, max_pages));  in page_cache_async_ra()
663 		ra->async_size = ra->size;  in page_cache_async_ra()
671 	 * readahead size. Ramp it up and use it as the new readahead size.  in page_cache_async_ra()
674 	start = page_cache_next_miss(ractl->mapping, index + 1, max_pages);  in page_cache_async_ra()
677 	if (!start || start - index > max_pages)  in page_cache_async_ra()
680 	ra->start = start;  in page_cache_async_ra()
681 	ra->size = start - index;	/* old async_size */  in page_cache_async_ra()
682 	ra->size += req_count;  in page_cache_async_ra()
683 	ra->size = get_next_ra_size(ra, max_pages);  in page_cache_async_ra()
684 	ra->async_size = ra->size;  in page_cache_async_ra()
686 	ractl->_index = ra->start;  in page_cache_async_ra()
695 	if (fd_empty(f) || !(fd_file(f)->f_mode & FMODE_READ))  in ksys_readahead()
696 		return -EBADF;  in ksys_readahead()
701 	 * on this file, then we must return -EINVAL.  in ksys_readahead()
703 	if (!fd_file(f)->f_mapping || !fd_file(f)->f_mapping->a_ops ||  in ksys_readahead()
704 	    (!S_ISREG(file_inode(fd_file(f))->i_mode) &&  in ksys_readahead()
705 	    !S_ISBLK(file_inode(fd_file(f))->i_mode)))  in ksys_readahead()
706 		return -EINVAL;  in ksys_readahead()
724  * readahead_expand - Expand a readahead request
726  * @new_start: The revised start
744 	struct address_space *mapping = ractl->mapping;  in readahead_expand()
745 	struct file_ra_state *ra = ractl->ra;  in readahead_expand()
753 	 * Readahead code should have aligned the ractl->_index to  in readahead_expand()
756 	VM_BUG_ON(!IS_ALIGNED(ractl->_index, min_nrpages));  in readahead_expand()
759 	while (ractl->_index > new_index) {  in readahead_expand()
760 		unsigned long index = ractl->_index - 1;  in readahead_expand()
761 		struct folio *folio = xa_load(&mapping->i_pages, index);  in readahead_expand()
776 				!ractl->_workingset) {  in readahead_expand()
777 			ractl->_workingset = true;  in readahead_expand()
778 			psi_memstall_enter(&ractl->_pflags);  in readahead_expand()
780 		ractl->_nr_pages += min_nrpages;  in readahead_expand()
781 		ractl->_index = folio->index;  in readahead_expand()
784 	new_len += new_start - readahead_pos(ractl);  in readahead_expand()
788 	while (ractl->_nr_pages < new_nr_pages) {  in readahead_expand()
789 		unsigned long index = ractl->_index + ractl->_nr_pages;  in readahead_expand()
790 		struct folio *folio = xa_load(&mapping->i_pages, index);  in readahead_expand()
805 				!ractl->_workingset) {  in readahead_expand()
806 			ractl->_workingset = true;  in readahead_expand()
807 			psi_memstall_enter(&ractl->_pflags);  in readahead_expand()
809 		ractl->_nr_pages += min_nrpages;  in readahead_expand()
811 			ra->size += min_nrpages;  in readahead_expand()
812 			ra->async_size += min_nrpages;  in readahead_expand()