2429 static int filemap_update_page(struct kiocb *iocb,  in filemap_update_page()  argument
2435 	if (iocb->ki_flags & IOCB_NOWAIT) {  in filemap_update_page()
2444 		if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_NOIO))  in filemap_update_page()
2446 		if (!(iocb->ki_flags & IOCB_WAITQ)) {  in filemap_update_page()
2455 		error = __folio_lock_async(folio, iocb->ki_waitq);  in filemap_update_page()
2465 	if (filemap_range_uptodate(mapping, iocb->ki_pos, count, folio,  in filemap_update_page()
2470 	if (iocb->ki_flags & (IOCB_NOIO | IOCB_NOWAIT | IOCB_WAITQ))  in filemap_update_page()
2473 	error = filemap_read_folio(iocb->ki_filp, mapping->a_ops->read_folio,  in filemap_update_page()
2485 static int filemap_create_folio(struct kiocb *iocb, struct folio_batch *fbatch)  in filemap_create_folio()  argument
2487 	struct address_space *mapping = iocb->ki_filp->f_mapping;  in filemap_create_folio()
2493 	if (iocb->ki_flags & (IOCB_NOWAIT | IOCB_WAITQ))  in filemap_create_folio()
2499 	if (iocb->ki_flags & IOCB_DONTCACHE)  in filemap_create_folio()
2516 	index = (iocb->ki_pos >> (PAGE_SHIFT + min_order)) << min_order;  in filemap_create_folio()
2524 	error = filemap_read_folio(iocb->ki_filp, mapping->a_ops->read_folio,  in filemap_create_folio()
2538 static int filemap_readahead(struct kiocb *iocb, struct file *file,  in filemap_readahead()  argument
2544 	if (iocb->ki_flags & IOCB_NOIO)  in filemap_readahead()
2546 	if (iocb->ki_flags & IOCB_DONTCACHE)  in filemap_readahead()
2552 static int filemap_get_pages(struct kiocb *iocb, size_t count,  in filemap_get_pages()  argument
2555 	struct file *filp = iocb->ki_filp;  in filemap_get_pages()
2557 	pgoff_t index = iocb->ki_pos >> PAGE_SHIFT;  in filemap_get_pages()
2564 	last_index = DIV_ROUND_UP(iocb->ki_pos + count, PAGE_SIZE);  in filemap_get_pages()
2573 		if (iocb->ki_flags & IOCB_NOIO)  in filemap_get_pages()
2575 		if (iocb->ki_flags & IOCB_NOWAIT)  in filemap_get_pages()
2577 		if (iocb->ki_flags & IOCB_DONTCACHE)  in filemap_get_pages()
2580 		if (iocb->ki_flags & IOCB_NOWAIT)  in filemap_get_pages()
2585 		err = filemap_create_folio(iocb, fbatch);  in filemap_get_pages()
2593 		err = filemap_readahead(iocb, filp, mapping, folio, last_index);  in filemap_get_pages()
2598 		if ((iocb->ki_flags & IOCB_WAITQ) &&  in filemap_get_pages()
2600 			iocb->ki_flags |= IOCB_NOWAIT;  in filemap_get_pages()
2601 		err = filemap_update_page(iocb, mapping, count, folio,  in filemap_get_pages()
2653 ssize_t filemap_read(struct kiocb *iocb, struct iov_iter *iter,  in filemap_read()  argument
2656 	struct file *filp = iocb->ki_filp;  in filemap_read()
2666 	if (unlikely(iocb->ki_pos < 0))  in filemap_read()
2668 	if (unlikely(iocb->ki_pos >= inode->i_sb->s_maxbytes))  in filemap_read()
2673 	iov_iter_truncate(iter, inode->i_sb->s_maxbytes - iocb->ki_pos);  in filemap_read()
2684 		if ((iocb->ki_flags & IOCB_WAITQ) && already_read)  in filemap_read()
2685 			iocb->ki_flags |= IOCB_NOWAIT;  in filemap_read()
2687 		if (unlikely(iocb->ki_pos >= i_size_read(inode)))  in filemap_read()
2690 		error = filemap_get_pages(iocb, iter->count, &fbatch, false);  in filemap_read()
2703 		if (unlikely(iocb->ki_pos >= isize))  in filemap_read()
2705 		end_offset = min_t(loff_t, isize, iocb->ki_pos + iter->count);  in filemap_read()
2717 		if (!pos_same_folio(iocb->ki_pos, last_pos - 1,  in filemap_read()
2724 			size_t offset = iocb->ki_pos & (fsize - 1);  in filemap_read()
2725 			size_t bytes = min_t(loff_t, end_offset - iocb->ki_pos,  in filemap_read()
2744 			iocb->ki_pos += copied;  in filemap_read()
2745 			last_pos = iocb->ki_pos;  in filemap_read()
2760 	} while (iov_iter_count(iter) && iocb->ki_pos < isize && !error);  in filemap_read()
2768 int kiocb_write_and_wait(struct kiocb *iocb, size_t count)  in kiocb_write_and_wait()  argument
2770 	struct address_space *mapping = iocb->ki_filp->f_mapping;  in kiocb_write_and_wait()
2771 	loff_t pos = iocb->ki_pos;  in kiocb_write_and_wait()
2774 	if (iocb->ki_flags & IOCB_NOWAIT) {  in kiocb_write_and_wait()
2809 int kiocb_invalidate_pages(struct kiocb *iocb, size_t count)  in kiocb_invalidate_pages()  argument
2811 	struct address_space *mapping = iocb->ki_filp->f_mapping;  in kiocb_invalidate_pages()
2813 	return filemap_invalidate_pages(mapping, iocb->ki_pos,  in kiocb_invalidate_pages()
2814 					iocb->ki_pos + count - 1,  in kiocb_invalidate_pages()
2815 					iocb->ki_flags & IOCB_NOWAIT);  in kiocb_invalidate_pages()
2841 generic_file_read_iter(struct kiocb *iocb, struct iov_iter *iter)  in generic_file_read_iter()  argument
2849 	if (iocb->ki_flags & IOCB_DIRECT) {  in generic_file_read_iter()
2850 		struct file *file = iocb->ki_filp;  in generic_file_read_iter()
2854 		retval = kiocb_write_and_wait(iocb, count);  in generic_file_read_iter()
2859 		retval = mapping->a_ops->direct_IO(iocb, iter);  in generic_file_read_iter()
2861 			iocb->ki_pos += retval;  in generic_file_read_iter()
2878 		if (iocb->ki_pos >= i_size_read(inode))  in generic_file_read_iter()
2882 	return filemap_read(iocb, iter, retval);  in generic_file_read_iter()
2943 	struct kiocb iocb;  in filemap_splice_read()  local
2952 	init_sync_kiocb(&iocb, in);  in filemap_splice_read()
2953 	iocb.ki_pos = *ppos;  in filemap_splice_read()
2968 		iocb.ki_pos = *ppos;  in filemap_splice_read()
2969 		error = filemap_get_pages(&iocb, len, &fbatch, true);  in filemap_splice_read()
3996 void kiocb_invalidate_post_direct_write(struct kiocb *iocb, size_t count)  in kiocb_invalidate_post_direct_write()  argument
3998 	struct address_space *mapping = iocb->ki_filp->f_mapping;  in kiocb_invalidate_post_direct_write()
4002 			iocb->ki_pos >> PAGE_SHIFT,  in kiocb_invalidate_post_direct_write()
4003 			(iocb->ki_pos + count - 1) >> PAGE_SHIFT))  in kiocb_invalidate_post_direct_write()
4004 		dio_warn_stale_pagecache(iocb->ki_filp);  in kiocb_invalidate_post_direct_write()
4008 generic_file_direct_write(struct kiocb *iocb, struct iov_iter *from)  in generic_file_direct_write()  argument
4010 	struct address_space *mapping = iocb->ki_filp->f_mapping;  in generic_file_direct_write()
4018 	written = kiocb_invalidate_pages(iocb, write_len);  in generic_file_direct_write()
4025 	written = mapping->a_ops->direct_IO(iocb, from);  in generic_file_direct_write()
4046 		loff_t pos = iocb->ki_pos;  in generic_file_direct_write()
4048 		kiocb_invalidate_post_direct_write(iocb, written);  in generic_file_direct_write()
4055 		iocb->ki_pos = pos;  in generic_file_direct_write()
4063 ssize_t generic_perform_write(struct kiocb *iocb, struct iov_iter *i)  in generic_perform_write()  argument
4065 	struct file *file = iocb->ki_filp;  in generic_perform_write()
4066 	loff_t pos = iocb->ki_pos;  in generic_perform_write()
4147 	iocb->ki_pos += written;  in generic_perform_write()
4173 ssize_t __generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)  in __generic_file_write_iter()  argument
4175 	struct file *file = iocb->ki_filp;  in __generic_file_write_iter()
4188 	if (iocb->ki_flags & IOCB_DIRECT) {  in __generic_file_write_iter()
4189 		ret = generic_file_direct_write(iocb, from);  in __generic_file_write_iter()
4199 		return direct_write_fallback(iocb, from, ret,  in __generic_file_write_iter()
4200 				generic_perform_write(iocb, from));  in __generic_file_write_iter()
4203 	return generic_perform_write(iocb, from);  in __generic_file_write_iter()
4220 ssize_t generic_file_write_iter(struct kiocb *iocb, struct iov_iter *from)  in generic_file_write_iter()  argument
4222 	struct file *file = iocb->ki_filp;  in generic_file_write_iter()
4227 	ret = generic_write_checks(iocb, from);  in generic_file_write_iter()
4229 		ret = __generic_file_write_iter(iocb, from);  in generic_file_write_iter()
4233 		ret = generic_write_sync(iocb, ret);  in generic_file_write_iter()