56 struct ext4_io_end_vec *ext4_alloc_io_end_vec(ext4_io_end_t *io_end)  in ext4_alloc_io_end_vec()  argument
64 	list_add_tail(&io_end_vec->list, &io_end->list_vec);  in ext4_alloc_io_end_vec()
68 static void ext4_free_io_end_vec(ext4_io_end_t *io_end)  in ext4_free_io_end_vec()  argument
72 	if (list_empty(&io_end->list_vec))  in ext4_free_io_end_vec()
74 	list_for_each_entry_safe(io_end_vec, tmp, &io_end->list_vec, list) {  in ext4_free_io_end_vec()
80 struct ext4_io_end_vec *ext4_last_io_end_vec(ext4_io_end_t *io_end)  in ext4_last_io_end_vec()  argument
82 	BUG_ON(list_empty(&io_end->list_vec));  in ext4_last_io_end_vec()
83 	return list_last_entry(&io_end->list_vec, struct ext4_io_end_vec, list);  in ext4_last_io_end_vec()
149 static void ext4_release_io_end(ext4_io_end_t *io_end)  in ext4_release_io_end()  argument
153 	BUG_ON(!list_empty(&io_end->list));  in ext4_release_io_end()
154 	BUG_ON(io_end->flag & EXT4_IO_END_UNWRITTEN);  in ext4_release_io_end()
155 	WARN_ON(io_end->handle);  in ext4_release_io_end()
157 	for (bio = io_end->bio; bio; bio = next_bio) {  in ext4_release_io_end()
162 	ext4_free_io_end_vec(io_end);  in ext4_release_io_end()
163 	kmem_cache_free(io_end_cachep, io_end);  in ext4_release_io_end()
175 static int ext4_end_io_end(ext4_io_end_t *io_end)  in ext4_end_io_end()  argument
177 	struct inode *inode = io_end->inode;  in ext4_end_io_end()
178 	handle_t *handle = io_end->handle;  in ext4_end_io_end()
184 		   io_end, inode->i_ino, io_end->list.next, io_end->list.prev);  in ext4_end_io_end()
190 	io_end->handle = NULL;  /* Following call will use up the handle */  in ext4_end_io_end()
191 	if (unlikely(io_end->flag & EXT4_IO_END_FAILED)) {  in ext4_end_io_end()
199 		ret = ext4_convert_unwritten_io_end_vec(handle, io_end);  in ext4_end_io_end()
202 	    io_end->flag & EXT4_IO_END_UNWRITTEN) {  in ext4_end_io_end()
209 	ext4_clear_io_unwritten_flag(io_end);  in ext4_end_io_end()
210 	ext4_release_io_end(io_end);  in ext4_end_io_end()
218 	ext4_io_end_t *io_end, *io_end0, *io_end1;  in dump_completed_IO()  local
224 	list_for_each_entry(io_end, head, list) {  in dump_completed_IO()
225 		cur = &io_end->list;  in dump_completed_IO()
232 			    io_end, inode->i_ino, io_end0, io_end1);  in dump_completed_IO()
237 static bool ext4_io_end_defer_completion(ext4_io_end_t *io_end)  in ext4_io_end_defer_completion()  argument
239 	if (io_end->flag & EXT4_IO_END_UNWRITTEN &&  in ext4_io_end_defer_completion()
240 	    !list_empty(&io_end->list_vec))  in ext4_io_end_defer_completion()
242 	if (test_opt(io_end->inode->i_sb, DATA_ERR_ABORT) &&  in ext4_io_end_defer_completion()
243 	    io_end->flag & EXT4_IO_END_FAILED &&  in ext4_io_end_defer_completion()
244 	    !ext4_emergency_state(io_end->inode->i_sb))  in ext4_io_end_defer_completion()
250 static void ext4_add_complete_io(ext4_io_end_t *io_end)  in ext4_add_complete_io()  argument
252 	struct ext4_inode_info *ei = EXT4_I(io_end->inode);  in ext4_add_complete_io()
253 	struct ext4_sb_info *sbi = EXT4_SB(io_end->inode->i_sb);  in ext4_add_complete_io()
258 	WARN_ON(!(io_end->flag & EXT4_IO_END_DEFER_COMPLETION));  in ext4_add_complete_io()
259 	WARN_ON(io_end->flag & EXT4_IO_END_UNWRITTEN &&  in ext4_add_complete_io()
260 		!io_end->handle && sbi->s_journal);  in ext4_add_complete_io()
261 	WARN_ON(!io_end->bio);  in ext4_add_complete_io()
267 	list_add_tail(&io_end->list, &ei->i_rsv_conversion_list);  in ext4_add_complete_io()
274 	ext4_io_end_t *io_end;  in ext4_do_flush_completed_IO()  local
286 		io_end = list_entry(unwritten.next, ext4_io_end_t, list);  in ext4_do_flush_completed_IO()
287 		BUG_ON(!(io_end->flag & EXT4_IO_END_DEFER_COMPLETION));  in ext4_do_flush_completed_IO()
288 		list_del_init(&io_end->list);  in ext4_do_flush_completed_IO()
290 		err = ext4_end_io_end(io_end);  in ext4_do_flush_completed_IO()
310 	ext4_io_end_t *io_end = kmem_cache_zalloc(io_end_cachep, flags);  in ext4_init_io_end()  local
312 	if (io_end) {  in ext4_init_io_end()
313 		io_end->inode = inode;  in ext4_init_io_end()
314 		INIT_LIST_HEAD(&io_end->list);  in ext4_init_io_end()
315 		INIT_LIST_HEAD(&io_end->list_vec);  in ext4_init_io_end()
316 		refcount_set(&io_end->count, 1);  in ext4_init_io_end()
318 	return io_end;  in ext4_init_io_end()
321 void ext4_put_io_end_defer(ext4_io_end_t *io_end)  in ext4_put_io_end_defer()  argument
323 	if (refcount_dec_and_test(&io_end->count)) {  in ext4_put_io_end_defer()
324 		if (ext4_io_end_defer_completion(io_end))  in ext4_put_io_end_defer()
325 			return ext4_add_complete_io(io_end);  in ext4_put_io_end_defer()
327 		ext4_release_io_end(io_end);  in ext4_put_io_end_defer()
331 int ext4_put_io_end(ext4_io_end_t *io_end)  in ext4_put_io_end()  argument
333 	if (refcount_dec_and_test(&io_end->count)) {  in ext4_put_io_end()
334 		if (ext4_io_end_defer_completion(io_end))  in ext4_put_io_end()
335 			return ext4_end_io_end(io_end);  in ext4_put_io_end()
337 		ext4_release_io_end(io_end);  in ext4_put_io_end()
342 ext4_io_end_t *ext4_get_io_end(ext4_io_end_t *io_end)  in ext4_get_io_end()  argument
344 	refcount_inc(&io_end->count);  in ext4_get_io_end()
345 	return io_end;  in ext4_get_io_end()
351 	ext4_io_end_t *io_end = bio->bi_private;  in ext4_end_bio()  local
354 	if (WARN_ONCE(!io_end, "io_end is NULL: %pg: sector %Lu len %u err %d\n",  in ext4_end_bio()
366 		struct inode *inode = io_end->inode;  in ext4_end_bio()
373 		io_end->flag |= EXT4_IO_END_FAILED;  in ext4_end_bio()
378 	if (ext4_io_end_defer_completion(io_end)) {  in ext4_end_bio()
384 		bio->bi_private = xchg(&io_end->bio, bio);  in ext4_end_bio()
385 		ext4_put_io_end_defer(io_end);  in ext4_end_bio()
391 		ext4_put_io_end_defer(io_end);  in ext4_end_bio()
414 	io->io_end = NULL;  in ext4_io_submit_init()
430 	bio->bi_private = ext4_get_io_end(io->io_end);  in io_submit_init_bio()