63 		struct iomap_dio *dio, unsigned short nr_vecs, blk_opf_t opf)  in iomap_dio_alloc_bio()  argument
65 	if (dio->dops && dio->dops->bio_set)  in iomap_dio_alloc_bio()
67 					GFP_KERNEL, dio->dops->bio_set);  in iomap_dio_alloc_bio()
72 		struct iomap_dio *dio, struct bio *bio, loff_t pos)  in iomap_dio_submit_bio()  argument
74 	struct kiocb *iocb = dio->iocb;  in iomap_dio_submit_bio()
76 	atomic_inc(&dio->ref);  in iomap_dio_submit_bio()
84 	if (dio->dops && dio->dops->submit_io)  in iomap_dio_submit_bio()
85 		dio->dops->submit_io(iter, bio, pos);  in iomap_dio_submit_bio()
90 ssize_t iomap_dio_complete(struct iomap_dio *dio)  in iomap_dio_complete()  argument
92 	const struct iomap_dio_ops *dops = dio->dops;  in iomap_dio_complete()
93 	struct kiocb *iocb = dio->iocb;  in iomap_dio_complete()
95 	ssize_t ret = dio->error;  in iomap_dio_complete()
98 		ret = dops->end_io(iocb, dio->size, ret, dio->flags);  in iomap_dio_complete()
101 		ret = dio->size;  in iomap_dio_complete()
103 		if (offset + ret > dio->i_size &&  in iomap_dio_complete()
104 		    !(dio->flags & IOMAP_DIO_WRITE))  in iomap_dio_complete()
105 			ret = dio->i_size - offset;  in iomap_dio_complete()
120 	if (!dio->error && dio->size && (dio->flags & IOMAP_DIO_WRITE))  in iomap_dio_complete()
121 		kiocb_invalidate_post_direct_write(iocb, dio->size);  in iomap_dio_complete()
132 		if (dio->flags & IOMAP_DIO_NEED_SYNC)  in iomap_dio_complete()
135 			ret += dio->done_before;  in iomap_dio_complete()
137 	trace_iomap_dio_complete(iocb, dio->error, ret);  in iomap_dio_complete()
138 	kfree(dio);  in iomap_dio_complete()
150 	struct iomap_dio *dio = container_of(work, struct iomap_dio, aio.work);  in iomap_dio_complete_work()  local
151 	struct kiocb *iocb = dio->iocb;  in iomap_dio_complete_work()
153 	iocb->ki_complete(iocb, iomap_dio_complete(dio));  in iomap_dio_complete_work()
161 static inline void iomap_dio_set_error(struct iomap_dio *dio, int ret)  in iomap_dio_set_error()  argument
163 	cmpxchg(&dio->error, 0, ret);  in iomap_dio_set_error()
168 	struct iomap_dio *dio = bio->bi_private;  in iomap_dio_bio_end_io()  local
169 	bool should_dirty = (dio->flags & IOMAP_DIO_DIRTY);  in iomap_dio_bio_end_io()
170 	struct kiocb *iocb = dio->iocb;  in iomap_dio_bio_end_io()
173 		iomap_dio_set_error(dio, blk_status_to_errno(bio->bi_status));  in iomap_dio_bio_end_io()
174 	if (!atomic_dec_and_test(&dio->ref))  in iomap_dio_bio_end_io()
181 	if (dio->wait_for_completion) {  in iomap_dio_bio_end_io()
182 		struct task_struct *waiter = dio->submit.waiter;  in iomap_dio_bio_end_io()
184 		WRITE_ONCE(dio->submit.waiter, NULL);  in iomap_dio_bio_end_io()
192 	if (dio->flags & IOMAP_DIO_INLINE_COMP) {  in iomap_dio_bio_end_io()
194 		iomap_dio_complete_work(&dio->aio.work);  in iomap_dio_bio_end_io()
202 	if (dio->flags & IOMAP_DIO_CALLER_COMP) {  in iomap_dio_bio_end_io()
204 		iocb->private = dio;  in iomap_dio_bio_end_io()
229 	INIT_WORK(&dio->aio.work, iomap_dio_complete_work);  in iomap_dio_bio_end_io()
231 			&dio->aio.work);  in iomap_dio_bio_end_io()
242 static int iomap_dio_zero(const struct iomap_iter *iter, struct iomap_dio *dio,  in iomap_dio_zero()  argument
245 	struct inode *inode = file_inode(dio->iocb->ki_filp);  in iomap_dio_zero()
256 	bio = iomap_dio_alloc_bio(iter, dio, 1, REQ_OP_WRITE | REQ_SYNC | REQ_IDLE);  in iomap_dio_zero()
260 	bio->bi_private = dio;  in iomap_dio_zero()
264 	iomap_dio_submit_bio(iter, dio, bio, pos);  in iomap_dio_zero()
273 static inline blk_opf_t iomap_dio_bio_opflags(struct iomap_dio *dio,  in iomap_dio_bio_opflags()  argument
278 	if (!(dio->flags & IOMAP_DIO_WRITE))  in iomap_dio_bio_opflags()
285 		dio->flags &= ~IOMAP_DIO_WRITE_THROUGH;  in iomap_dio_bio_opflags()
293 		struct iomap_dio *dio)  in iomap_dio_bio_iter()  argument
313 	    !bdev_iter_is_aligned(iomap->bdev, dio->submit.iter))  in iomap_dio_bio_iter()
317 		dio->flags |= IOMAP_DIO_UNWRITTEN;  in iomap_dio_bio_iter()
322 		dio->flags |= IOMAP_DIO_COW;  in iomap_dio_bio_iter()
339 		    (dio->flags & IOMAP_DIO_WRITE_THROUGH) &&  in iomap_dio_bio_iter()
342 		else if (dio->flags & IOMAP_DIO_NEED_SYNC)  in iomap_dio_bio_iter()
343 			dio->flags &= ~IOMAP_DIO_CALLER_COMP;  in iomap_dio_bio_iter()
351 	orig_count = iov_iter_count(dio->submit.iter);  in iomap_dio_bio_iter()
352 	iov_iter_truncate(dio->submit.iter, length);  in iomap_dio_bio_iter()
354 	if (!iov_iter_count(dio->submit.iter))  in iomap_dio_bio_iter()
365 	    ((dio->flags & IOMAP_DIO_NEED_SYNC) && !use_fua) ||  in iomap_dio_bio_iter()
366 	    ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode)))  in iomap_dio_bio_iter()
367 		dio->flags &= ~IOMAP_DIO_CALLER_COMP;  in iomap_dio_bio_iter()
374 	if (!(dio->flags & (IOMAP_DIO_INLINE_COMP|IOMAP_DIO_CALLER_COMP)))  in iomap_dio_bio_iter()
375 		dio->iocb->ki_flags &= ~IOCB_HIPRI;  in iomap_dio_bio_iter()
381 		ret = iomap_dio_zero(iter, dio, pos - pad, pad);  in iomap_dio_bio_iter()
386 	bio_opf = iomap_dio_bio_opflags(dio, iomap, use_fua, atomic);  in iomap_dio_bio_iter()
388 	nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter, BIO_MAX_VECS);  in iomap_dio_bio_iter()
391 		if (dio->error) {  in iomap_dio_bio_iter()
392 			iov_iter_revert(dio->submit.iter, copied);  in iomap_dio_bio_iter()
397 		bio = iomap_dio_alloc_bio(iter, dio, nr_pages, bio_opf);  in iomap_dio_bio_iter()
402 		bio->bi_ioprio = dio->iocb->ki_ioprio;  in iomap_dio_bio_iter()
403 		bio->bi_private = dio;  in iomap_dio_bio_iter()
406 		ret = bio_iov_iter_get_pages(bio, dio->submit.iter);  in iomap_dio_bio_iter()
430 		if (dio->flags & IOMAP_DIO_WRITE)  in iomap_dio_bio_iter()
432 		else if (dio->flags & IOMAP_DIO_DIRTY)  in iomap_dio_bio_iter()
435 		dio->size += n;  in iomap_dio_bio_iter()
438 		nr_pages = bio_iov_vecs_to_alloc(dio->submit.iter,  in iomap_dio_bio_iter()
444 			dio->iocb->ki_flags &= ~IOCB_HIPRI;  in iomap_dio_bio_iter()
445 		iomap_dio_submit_bio(iter, dio, bio, pos);  in iomap_dio_bio_iter()
457 	    ((dio->flags & IOMAP_DIO_WRITE) && pos >= i_size_read(inode))) {  in iomap_dio_bio_iter()
461 			ret = iomap_dio_zero(iter, dio, pos,  in iomap_dio_bio_iter()
466 	iov_iter_reexpand(dio->submit.iter, orig_count - copied);  in iomap_dio_bio_iter()
473 		struct iomap_dio *dio)  in iomap_dio_hole_iter()  argument
475 	loff_t length = iov_iter_zero(iomap_length(iter), dio->submit.iter);  in iomap_dio_hole_iter()
477 	dio->size += length;  in iomap_dio_hole_iter()
484 		struct iomap_dio *dio)  in iomap_dio_inline_iter()  argument
487 	struct iov_iter *iter = dio->submit.iter;  in iomap_dio_inline_iter()
496 	if (dio->flags & IOMAP_DIO_WRITE) {  in iomap_dio_inline_iter()
510 	dio->size += copied;  in iomap_dio_inline_iter()
517 		struct iomap_dio *dio)  in iomap_dio_iter()  argument
521 		if (WARN_ON_ONCE(dio->flags & IOMAP_DIO_WRITE))  in iomap_dio_iter()
523 		return iomap_dio_hole_iter(iter, dio);  in iomap_dio_iter()
525 		if (!(dio->flags & IOMAP_DIO_WRITE))  in iomap_dio_iter()
526 			return iomap_dio_hole_iter(iter, dio);  in iomap_dio_iter()
527 		return iomap_dio_bio_iter(iter, dio);  in iomap_dio_iter()
529 		return iomap_dio_bio_iter(iter, dio);  in iomap_dio_iter()
531 		return iomap_dio_inline_iter(iter, dio);  in iomap_dio_iter()
540 				    dio->iocb->ki_filp, current->comm);  in iomap_dio_iter()
584 	struct iomap_dio *dio;  in __iomap_dio_rw()  local
592 	dio = kmalloc(sizeof(*dio), GFP_KERNEL);  in __iomap_dio_rw()
593 	if (!dio)  in __iomap_dio_rw()
596 	dio->iocb = iocb;  in __iomap_dio_rw()
597 	atomic_set(&dio->ref, 1);  in __iomap_dio_rw()
598 	dio->size = 0;  in __iomap_dio_rw()
599 	dio->i_size = i_size_read(inode);  in __iomap_dio_rw()
600 	dio->dops = dops;  in __iomap_dio_rw()
601 	dio->error = 0;  in __iomap_dio_rw()
602 	dio->flags = 0;  in __iomap_dio_rw()
603 	dio->done_before = done_before;  in __iomap_dio_rw()
605 	dio->submit.iter = iter;  in __iomap_dio_rw()
606 	dio->submit.waiter = current;  in __iomap_dio_rw()
616 		dio->flags |= IOMAP_DIO_INLINE_COMP;  in __iomap_dio_rw()
618 		if (iomi.pos >= dio->i_size)  in __iomap_dio_rw()
622 			dio->flags |= IOMAP_DIO_DIRTY;  in __iomap_dio_rw()
629 		dio->flags |= IOMAP_DIO_WRITE;  in __iomap_dio_rw()
638 			dio->flags |= IOMAP_DIO_CALLER_COMP;  in __iomap_dio_rw()
642 			if (iomi.pos >= dio->i_size ||  in __iomap_dio_rw()
643 			    iomi.pos + iomi.len > dio->i_size)  in __iomap_dio_rw()
650 			dio->flags |= IOMAP_DIO_NEED_SYNC;  in __iomap_dio_rw()
662 				dio->flags |= IOMAP_DIO_WRITE_THROUGH;  in __iomap_dio_rw()
701 		iomi.processed = iomap_dio_iter(&iomi, dio);  in __iomap_dio_rw()
716 	if (iov_iter_rw(iter) == READ && iomi.pos >= dio->i_size)  in __iomap_dio_rw()
717 		iov_iter_revert(iter, iomi.pos - dio->i_size);  in __iomap_dio_rw()
719 	if (ret == -EFAULT && dio->size && (dio_flags & IOMAP_DIO_PARTIAL)) {  in __iomap_dio_rw()
731 		iomap_dio_set_error(dio, ret);  in __iomap_dio_rw()
738 	if (dio->flags & IOMAP_DIO_WRITE_THROUGH)  in __iomap_dio_rw()
739 		dio->flags &= ~IOMAP_DIO_NEED_SYNC;  in __iomap_dio_rw()
756 	dio->wait_for_completion = wait_for_completion;  in __iomap_dio_rw()
757 	if (!atomic_dec_and_test(&dio->ref)) {  in __iomap_dio_rw()
765 			if (!READ_ONCE(dio->submit.waiter))  in __iomap_dio_rw()
773 	return dio;  in __iomap_dio_rw()
776 	kfree(dio);  in __iomap_dio_rw()
788 	struct iomap_dio *dio;  in iomap_dio_rw()  local
790 	dio = __iomap_dio_rw(iocb, iter, ops, dops, dio_flags, private,  in iomap_dio_rw()
792 	if (IS_ERR_OR_NULL(dio))  in iomap_dio_rw()
793 		return PTR_ERR_OR_ZERO(dio);  in iomap_dio_rw()
794 	return iomap_dio_complete(dio);  in iomap_dio_rw()