134 	struct blkdev_dio *dio = bio->bi_private;  in blkdev_bio_end_io()  local
135 	bool should_dirty = dio->flags & DIO_SHOULD_DIRTY;  in blkdev_bio_end_io()
136 	bool is_sync = dio->flags & DIO_IS_SYNC;  in blkdev_bio_end_io()
138 	if (bio->bi_status && !dio->bio.bi_status)  in blkdev_bio_end_io()
139 		dio->bio.bi_status = bio->bi_status;  in blkdev_bio_end_io()
144 	if (atomic_dec_and_test(&dio->ref)) {  in blkdev_bio_end_io()
146 			struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io()
151 			if (likely(!dio->bio.bi_status)) {  in blkdev_bio_end_io()
152 				ret = dio->size;  in blkdev_bio_end_io()
155 				ret = blk_status_to_errno(dio->bio.bi_status);  in blkdev_bio_end_io()
158 			dio->iocb->ki_complete(iocb, ret);  in blkdev_bio_end_io()
159 			bio_put(&dio->bio);  in blkdev_bio_end_io()
161 			struct task_struct *waiter = dio->waiter;  in blkdev_bio_end_io()
163 			WRITE_ONCE(dio->waiter, NULL);  in blkdev_bio_end_io()
180 	struct blkdev_dio *dio;  in __blkdev_direct_IO()  local
189 	dio = container_of(bio, struct blkdev_dio, bio);  in __blkdev_direct_IO()
190 	atomic_set(&dio->ref, 1);  in __blkdev_direct_IO()
199 		dio->flags = DIO_IS_SYNC;  in __blkdev_direct_IO()
200 		dio->waiter = current;  in __blkdev_direct_IO()
202 		dio->flags = 0;  in __blkdev_direct_IO()
203 		dio->iocb = iocb;  in __blkdev_direct_IO()
206 	dio->size = 0;  in __blkdev_direct_IO()
208 		dio->flags |= DIO_SHOULD_DIRTY;  in __blkdev_direct_IO()
216 		bio->bi_private = dio;  in __blkdev_direct_IO()
248 			if (dio->flags & DIO_SHOULD_DIRTY)  in __blkdev_direct_IO()
253 		dio->size += bio->bi_iter.bi_size;  in __blkdev_direct_IO()
261 		atomic_inc(&dio->ref);  in __blkdev_direct_IO()
273 		if (!READ_ONCE(dio->waiter))  in __blkdev_direct_IO()
280 		ret = blk_status_to_errno(dio->bio.bi_status);  in __blkdev_direct_IO()
282 		ret = dio->size;  in __blkdev_direct_IO()
284 	bio_put(&dio->bio);  in __blkdev_direct_IO()
296 	struct blkdev_dio *dio = container_of(bio, struct blkdev_dio, bio);  in blkdev_bio_end_io_async()  local
297 	struct kiocb *iocb = dio->iocb;  in blkdev_bio_end_io_async()
303 		ret = dio->size;  in blkdev_bio_end_io_async()
314 	if (dio->flags & DIO_SHOULD_DIRTY) {  in blkdev_bio_end_io_async()
329 	struct blkdev_dio *dio;  in __blkdev_direct_IO_async()  local
336 	dio = container_of(bio, struct blkdev_dio, bio);  in __blkdev_direct_IO_async()
337 	dio->flags = 0;  in __blkdev_direct_IO_async()
338 	dio->iocb = iocb;  in __blkdev_direct_IO_async()
358 	dio->size = bio->bi_iter.bi_size;  in __blkdev_direct_IO_async()
362 			dio->flags |= DIO_SHOULD_DIRTY;  in __blkdev_direct_IO_async()