55 static bool ext4_should_use_dio(struct kiocb *iocb, struct iov_iter *iter)  in ext4_should_use_dio()  argument
57 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_should_use_dio()
66 	return IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align);  in ext4_should_use_dio()
69 static ssize_t ext4_dio_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dio_read_iter()  argument
72 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_read_iter()
74 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_read_iter()
81 	if (!ext4_should_use_dio(iocb, to)) {  in ext4_dio_read_iter()
90 		iocb->ki_flags &= ~IOCB_DIRECT;  in ext4_dio_read_iter()
91 		return generic_file_read_iter(iocb, to);  in ext4_dio_read_iter()
94 	ret = iomap_dio_rw(iocb, to, &ext4_iomap_ops, NULL, 0, NULL, 0);  in ext4_dio_read_iter()
97 	file_accessed(iocb->ki_filp);  in ext4_dio_read_iter()
102 static ssize_t ext4_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_dax_read_iter()  argument
104 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_read_iter()
107 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_read_iter()
120 		return generic_file_read_iter(iocb, to);  in ext4_dax_read_iter()
122 	ret = dax_iomap_rw(iocb, to, &ext4_iomap_ops);  in ext4_dax_read_iter()
125 	file_accessed(iocb->ki_filp);  in ext4_dax_read_iter()
130 static ssize_t ext4_file_read_iter(struct kiocb *iocb, struct iov_iter *to)  in ext4_file_read_iter()  argument
132 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_read_iter()
142 		return ext4_dax_read_iter(iocb, to);  in ext4_file_read_iter()
144 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_read_iter()
145 		return ext4_dio_read_iter(iocb, to);  in ext4_file_read_iter()
147 	return generic_file_read_iter(iocb, to);  in ext4_file_read_iter()
243 static ssize_t ext4_generic_write_checks(struct kiocb *iocb,  in ext4_generic_write_checks()  argument
246 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_generic_write_checks()
252 	ret = generic_write_checks(iocb, from);  in ext4_generic_write_checks()
263 		if (iocb->ki_pos >= sbi->s_bitmap_maxbytes)  in ext4_generic_write_checks()
265 		iov_iter_truncate(from, sbi->s_bitmap_maxbytes - iocb->ki_pos);  in ext4_generic_write_checks()
271 static ssize_t ext4_write_checks(struct kiocb *iocb, struct iov_iter *from)  in ext4_write_checks()  argument
275 	count = ext4_generic_write_checks(iocb, from);  in ext4_write_checks()
279 	ret = file_modified(iocb->ki_filp);  in ext4_write_checks()
285 static ssize_t ext4_buffered_write_iter(struct kiocb *iocb,  in ext4_buffered_write_iter()  argument
289 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_buffered_write_iter()
291 	if (iocb->ki_flags & IOCB_NOWAIT)  in ext4_buffered_write_iter()
295 	ret = ext4_write_checks(iocb, from);  in ext4_buffered_write_iter()
299 	ret = generic_perform_write(iocb, from);  in ext4_buffered_write_iter()
305 	return generic_write_sync(iocb, ret);  in ext4_buffered_write_iter()
374 static int ext4_dio_write_end_io(struct kiocb *iocb, ssize_t size,  in ext4_dio_write_end_io()  argument
377 	loff_t pos = iocb->ki_pos;  in ext4_dio_write_end_io()
378 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_end_io()
422 static ssize_t ext4_dio_write_checks(struct kiocb *iocb, struct iov_iter *from,  in ext4_dio_write_checks()  argument
426 	struct file *file = iocb->ki_filp;  in ext4_dio_write_checks()
434 	ret = ext4_generic_write_checks(iocb, from);  in ext4_dio_write_checks()
438 	offset = iocb->ki_pos;  in ext4_dio_write_checks()
459 		if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_checks()
477 		if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_checks()
499 static ssize_t ext4_dio_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dio_write_iter()  argument
503 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dio_write_iter()
504 	loff_t offset = iocb->ki_pos;  in ext4_dio_write_iter()
519 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dio_write_iter()
535 	if (!ext4_should_use_dio(iocb, from)) {  in ext4_dio_write_iter()
540 		return ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
552 	ret = ext4_dio_write_checks(iocb, from, &ilock_shared, &extend,  in ext4_dio_write_iter()
557 	offset = iocb->ki_pos;  in ext4_dio_write_iter()
575 	ret = iomap_dio_rw(iocb, from, iomap_ops, &ext4_dio_write_ops,  in ext4_dio_write_iter()
605 		WARN_ON_ONCE(iocb->ki_flags & IOCB_ATOMIC);  in ext4_dio_write_iter()
607 		offset = iocb->ki_pos;  in ext4_dio_write_iter()
608 		err = ext4_buffered_write_iter(iocb, from);  in ext4_dio_write_iter()
621 		err = filemap_write_and_wait_range(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
624 			invalidate_mapping_pages(iocb->ki_filp->f_mapping,  in ext4_dio_write_iter()
634 ext4_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_dax_write_iter()  argument
641 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_dax_write_iter()
643 	if (iocb->ki_flags & IOCB_NOWAIT) {  in ext4_dax_write_iter()
650 	ret = ext4_write_checks(iocb, from);  in ext4_dax_write_iter()
654 	offset = iocb->ki_pos;  in ext4_dax_write_iter()
674 	ret = dax_iomap_rw(iocb, from, &ext4_iomap_ops);  in ext4_dax_write_iter()
683 		ret = generic_write_sync(iocb, ret);  in ext4_dax_write_iter()
689 ext4_file_write_iter(struct kiocb *iocb, struct iov_iter *from)  in ext4_file_write_iter()  argument
691 	struct inode *inode = file_inode(iocb->ki_filp);  in ext4_file_write_iter()
698 		return ext4_dax_write_iter(iocb, from);  in ext4_file_write_iter()
701 	if (iocb->ki_flags & IOCB_ATOMIC) {  in ext4_file_write_iter()
709 		ret = generic_atomic_write_valid(iocb, from);  in ext4_file_write_iter()
714 	if (iocb->ki_flags & IOCB_DIRECT)  in ext4_file_write_iter()
715 		return ext4_dio_write_iter(iocb, from);  in ext4_file_write_iter()
717 		return ext4_buffered_write_iter(iocb, from);  in ext4_file_write_iter()