205 void page_cache_ra_unbounded(struct readahead_control *ractl,  in page_cache_ra_unbounded()  argument
208 	struct address_space *mapping = ractl->mapping;  in page_cache_ra_unbounded()
209 	unsigned long ra_folio_index, index = readahead_index(ractl);  in page_cache_ra_unbounded()
235 	ra_folio_index = round_up(readahead_index(ractl) + nr_to_read - lookahead_size,  in page_cache_ra_unbounded()
238 	nr_to_read += readahead_index(ractl) - index;  in page_cache_ra_unbounded()
239 	ractl->_index = index;  in page_cache_ra_unbounded()
257 			read_pages(ractl);  in page_cache_ra_unbounded()
258 			ractl->_index += min_nrpages;  in page_cache_ra_unbounded()
259 			i = ractl->_index + ractl->_nr_pages - index;  in page_cache_ra_unbounded()
273 			read_pages(ractl);  in page_cache_ra_unbounded()
274 			ractl->_index += min_nrpages;  in page_cache_ra_unbounded()
275 			i = ractl->_index + ractl->_nr_pages - index;  in page_cache_ra_unbounded()
280 		ractl->_workingset |= folio_test_workingset(folio);  in page_cache_ra_unbounded()
281 		ractl->_nr_pages += min_nrpages;  in page_cache_ra_unbounded()
290 	read_pages(ractl);  in page_cache_ra_unbounded()
302 static void do_page_cache_ra(struct readahead_control *ractl,  in do_page_cache_ra()  argument
305 	struct inode *inode = ractl->mapping->host;  in do_page_cache_ra()
306 	unsigned long index = readahead_index(ractl);  in do_page_cache_ra()
320 	page_cache_ra_unbounded(ractl, nr_to_read, lookahead_size);  in do_page_cache_ra()
327 void force_page_cache_ra(struct readahead_control *ractl,  in force_page_cache_ra()  argument
330 	struct address_space *mapping = ractl->mapping;  in force_page_cache_ra()
331 	struct file_ra_state *ra = ractl->ra;  in force_page_cache_ra()
349 		do_page_cache_ra(ractl, this_chunk, 0);  in force_page_cache_ra()
430 static inline int ra_alloc_folio(struct readahead_control *ractl, pgoff_t index,  in ra_alloc_folio()  argument
441 	err = filemap_add_folio(ractl->mapping, folio, index, gfp);  in ra_alloc_folio()
447 	ractl->_nr_pages += 1UL << order;  in ra_alloc_folio()
448 	ractl->_workingset |= folio_test_workingset(folio);  in ra_alloc_folio()
452 void page_cache_ra_order(struct readahead_control *ractl,  in page_cache_ra_order()  argument
455 	struct address_space *mapping = ractl->mapping;  in page_cache_ra_order()
456 	pgoff_t start = readahead_index(ractl);  in page_cache_ra_order()
490 	ractl->_index = mapping_align_index(mapping, index);  in page_cache_ra_order()
491 	index = readahead_index(ractl);  in page_cache_ra_order()
502 		err = ra_alloc_folio(ractl, index, mark, order, gfp);  in page_cache_ra_order()
508 	read_pages(ractl);  in page_cache_ra_order()
520 	do_page_cache_ra(ractl, ra->size - (index - start), ra->async_size);  in page_cache_ra_order()
523 static unsigned long ractl_max_pages(struct readahead_control *ractl,  in ractl_max_pages()  argument
526 	struct backing_dev_info *bdi = inode_to_bdi(ractl->mapping->host);  in ractl_max_pages()
527 	unsigned long max_pages = ractl->ra->ra_pages;  in ractl_max_pages()
538 void page_cache_sync_ra(struct readahead_control *ractl,  in page_cache_sync_ra()  argument
541 	pgoff_t index = readahead_index(ractl);  in page_cache_sync_ra()
542 	bool do_forced_ra = ractl->file && (ractl->file->f_mode & FMODE_RANDOM);  in page_cache_sync_ra()
543 	struct file_ra_state *ra = ractl->ra;  in page_cache_sync_ra()
554 		if (!ractl->file)  in page_cache_sync_ra()
562 		force_page_cache_ra(ractl, req_count);  in page_cache_sync_ra()
566 	max_pages = ractl_max_pages(ractl, req_count);  in page_cache_sync_ra()
586 	miss = page_cache_prev_miss(ractl->mapping, index - 1, max_pages);  in page_cache_sync_ra()
594 		do_page_cache_ra(ractl, req_count, 0);  in page_cache_sync_ra()
607 	ractl->_index = ra->start;  in page_cache_sync_ra()
608 	page_cache_ra_order(ractl, ra, 0);  in page_cache_sync_ra()
612 void page_cache_async_ra(struct readahead_control *ractl,  in page_cache_async_ra()  argument
616 	struct file_ra_state *ra = ractl->ra;  in page_cache_async_ra()
617 	pgoff_t index = readahead_index(ractl);  in page_cache_async_ra()
636 	max_pages = ractl_max_pages(ractl, req_count);  in page_cache_async_ra()
657 	start = page_cache_next_miss(ractl->mapping, index + 1, max_pages);  in page_cache_async_ra()
669 	ractl->_index = ra->start;  in page_cache_async_ra()
670 	page_cache_ra_order(ractl, ra, order);  in page_cache_async_ra()
731 void readahead_expand(struct readahead_control *ractl,  in readahead_expand()  argument
734 	struct address_space *mapping = ractl->mapping;  in readahead_expand()
735 	struct file_ra_state *ra = ractl->ra;  in readahead_expand()
746 	VM_BUG_ON(!IS_ALIGNED(ractl->_index, min_nrpages));  in readahead_expand()
749 	while (ractl->_index > new_index) {  in readahead_expand()
750 		unsigned long index = ractl->_index - 1;  in readahead_expand()
766 				!ractl->_workingset) {  in readahead_expand()
767 			ractl->_workingset = true;  in readahead_expand()
768 			psi_memstall_enter(&ractl->_pflags);  in readahead_expand()
770 		ractl->_nr_pages += min_nrpages;  in readahead_expand()
771 		ractl->_index = folio->index;  in readahead_expand()
774 	new_len += new_start - readahead_pos(ractl);  in readahead_expand()
778 	while (ractl->_nr_pages < new_nr_pages) {  in readahead_expand()
779 		unsigned long index = ractl->_index + ractl->_nr_pages;  in readahead_expand()
795 				!ractl->_workingset) {  in readahead_expand()
796 			ractl->_workingset = true;  in readahead_expand()
797 			psi_memstall_enter(&ractl->_pflags);  in readahead_expand()
799 		ractl->_nr_pages += min_nrpages;  in readahead_expand()