Lines Matching +full:wait +full:- +full:retry +full:- +full:us
1 // SPDX-License-Identifier: GPL-2.0
6 #include "delalloc-space.h"
7 #include "direct-io.h"
8 #include "extent-tree.h"
39 struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree; in lock_extent_direct()
46 return -EAGAIN; in lock_extent_direct()
55 ret = -EAGAIN; in lock_extent_direct()
67 lockend - lockstart + 1); in lock_extent_direct()
77 (!writing || !filemap_range_has_page(inode->i_mapping, in lock_extent_direct()
86 ret = -EAGAIN; in lock_extent_direct()
91 * found is for a buffered write, we can not wait for it in lock_extent_direct()
92 * to complete and retry, because if we do so we can in lock_extent_direct()
105 test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags)) in lock_extent_direct()
108 ret = nowait ? -EAGAIN : -ENOTBLK; in lock_extent_direct()
112 * We could trigger writeback for this range (and wait in lock_extent_direct()
115 * but that can lead us to a deadlock with a concurrent in lock_extent_direct()
120 * complete), which makes readahead wait for that in lock_extent_direct()
124 ret = nowait ? -EAGAIN : -ENOTBLK; in lock_extent_direct()
160 start + file_extent->num_bytes - 1, false); in btrfs_create_dio_extent()
164 ASSERT(!dio_data->ordered); in btrfs_create_dio_extent()
165 dio_data->ordered = ordered; in btrfs_create_dio_extent()
176 struct btrfs_root *root = inode->root; in btrfs_new_extent_direct()
177 struct btrfs_fs_info *fs_info = root->fs_info; in btrfs_new_extent_direct()
186 ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize, in btrfs_new_extent_direct()
188 if (ret == -EAGAIN) { in btrfs_new_extent_direct()
190 wait_on_bit_io(&inode->root->fs_info->flags, BTRFS_FS_NEED_ZONE_FINISH, in btrfs_new_extent_direct()
241 if ((em->flags & EXTENT_FLAG_PREALLOC) || in btrfs_get_blocks_direct_write()
242 ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) && in btrfs_get_blocks_direct_write()
243 em->disk_bytenr != EXTENT_MAP_HOLE)) { in btrfs_get_blocks_direct_write()
244 if (em->flags & EXTENT_FLAG_PREALLOC) in btrfs_get_blocks_direct_write()
248 len = min(len, em->len - (start - em->start)); in btrfs_get_blocks_direct_write()
249 block_start = extent_map_block_start(em) + (start - em->start); in btrfs_get_blocks_direct_write()
267 /* Our caller expects us to free the input extent map. */ in btrfs_get_blocks_direct_write()
271 if (nowait && (ret == -ENOSPC || ret == -EDQUOT)) in btrfs_get_blocks_direct_write()
272 ret = -EAGAIN; in btrfs_get_blocks_direct_write()
291 dio_data->nocow_done = true; in btrfs_get_blocks_direct_write()
293 /* Our caller expects us to free the input extent map. */ in btrfs_get_blocks_direct_write()
298 ret = -EAGAIN; in btrfs_get_blocks_direct_write()
306 if (!dio_data->data_space_reserved) { in btrfs_get_blocks_direct_write()
307 ret = -ENOSPC; in btrfs_get_blocks_direct_write()
327 len = min(len, em->len - (start - em->start)); in btrfs_get_blocks_direct_write()
330 prev_len - len, true); in btrfs_get_blocks_direct_write()
362 struct btrfs_dio_data *dio_data = iter->private; in btrfs_dio_iomap_begin()
379 * -EAGAIN at this point so that the normal path is used. in btrfs_dio_iomap_begin()
382 return -EAGAIN; in btrfs_dio_iomap_begin()
389 len = min_t(u64, len, fs_info->sectorsize * BTRFS_MAX_BIO_SECTORS); in btrfs_dio_iomap_begin()
392 lockend = start + len - 1; in btrfs_dio_iomap_begin()
398 * outstanding dirty pages are on disk - the first flush only starts in btrfs_dio_iomap_begin()
406 * dirty or under writeback (same as for the non-compression case). in btrfs_dio_iomap_begin()
413 &BTRFS_I(inode)->runtime_flags)) { in btrfs_dio_iomap_begin()
415 if (filemap_range_needs_writeback(inode->i_mapping, in btrfs_dio_iomap_begin()
417 return -EAGAIN; in btrfs_dio_iomap_begin()
419 ret = filemap_fdatawrite_range(inode->i_mapping, start, in btrfs_dio_iomap_begin()
420 start + length - 1); in btrfs_dio_iomap_begin()
439 &dio_data->data_reserved, in btrfs_dio_iomap_begin()
442 dio_data->data_space_reserved = true; in btrfs_dio_iomap_begin()
443 else if (ret && !(BTRFS_I(inode)->flags & in btrfs_dio_iomap_begin()
473 * We return -ENOTBLK because that's what makes DIO go ahead and go back in btrfs_dio_iomap_begin()
477 if (extent_map_is_compressed(em) || em->disk_bytenr == EXTENT_MAP_INLINE) { in btrfs_dio_iomap_begin()
480 * If we are in a NOWAIT context, return -EAGAIN in order to in btrfs_dio_iomap_begin()
484 * space - this happens if we were able to read some data from in btrfs_dio_iomap_begin()
485 * previous non-compressed extents and then when we fallback to in btrfs_dio_iomap_begin()
489 * of bytes previously read is > 0, so it does not return -EFAULT). in btrfs_dio_iomap_begin()
491 ret = (flags & IOMAP_NOWAIT) ? -EAGAIN : -ENOTBLK; in btrfs_dio_iomap_begin()
495 len = min(len, em->len - (start - em->start)); in btrfs_dio_iomap_begin()
499 * (or a mix of extents and holes), then we return -EAGAIN to make the in btrfs_dio_iomap_begin()
513 * which case iomap_dio_rw() may return us EIOCBQUEUED if not all of in btrfs_dio_iomap_begin()
515 * which we return back to our caller - we should only return EIOCBQUEUED in btrfs_dio_iomap_begin()
520 ret = -EAGAIN; in btrfs_dio_iomap_begin()
530 len = min(len, em->len - (start - em->start)); in btrfs_dio_iomap_begin()
531 if (dio_data->data_space_reserved) { in btrfs_dio_iomap_begin()
535 if (dio_data->nocow_done) { in btrfs_dio_iomap_begin()
540 release_len = data_alloc_len - len; in btrfs_dio_iomap_begin()
545 dio_data->data_reserved, in btrfs_dio_iomap_begin()
556 if ((em->disk_bytenr == EXTENT_MAP_HOLE) || in btrfs_dio_iomap_begin()
557 ((em->flags & EXTENT_FLAG_PREALLOC) && !write)) { in btrfs_dio_iomap_begin()
558 iomap->addr = IOMAP_NULL_ADDR; in btrfs_dio_iomap_begin()
559 iomap->type = IOMAP_HOLE; in btrfs_dio_iomap_begin()
561 iomap->addr = extent_map_block_start(em) + (start - em->start); in btrfs_dio_iomap_begin()
562 iomap->type = IOMAP_MAPPED; in btrfs_dio_iomap_begin()
564 iomap->offset = start; in btrfs_dio_iomap_begin()
565 iomap->bdev = fs_info->fs_devices->latest_dev->bdev; in btrfs_dio_iomap_begin()
566 iomap->length = len; in btrfs_dio_iomap_begin()
578 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend, in btrfs_dio_iomap_begin()
583 unlock_dio_extent(&BTRFS_I(inode)->io_tree, start + len, in btrfs_dio_iomap_begin()
594 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend, in btrfs_dio_iomap_begin()
597 if (dio_data->data_space_reserved) { in btrfs_dio_iomap_begin()
599 dio_data->data_reserved, in btrfs_dio_iomap_begin()
601 extent_changeset_free(dio_data->data_reserved); in btrfs_dio_iomap_begin()
611 struct btrfs_dio_data *dio_data = iter->private; in btrfs_dio_iomap_end()
612 size_t submitted = dio_data->submitted; in btrfs_dio_iomap_end()
616 if (!write && (iomap->type == IOMAP_HOLE)) { in btrfs_dio_iomap_end()
618 unlock_dio_extent(&BTRFS_I(inode)->io_tree, pos, in btrfs_dio_iomap_end()
619 pos + length - 1, NULL); in btrfs_dio_iomap_end()
625 length -= submitted; in btrfs_dio_iomap_end()
627 btrfs_finish_ordered_extent(dio_data->ordered, NULL, in btrfs_dio_iomap_end()
630 unlock_dio_extent(&BTRFS_I(inode)->io_tree, pos, in btrfs_dio_iomap_end()
631 pos + length - 1, NULL); in btrfs_dio_iomap_end()
632 ret = -ENOTBLK; in btrfs_dio_iomap_end()
635 btrfs_put_ordered_extent(dio_data->ordered); in btrfs_dio_iomap_end()
636 dio_data->ordered = NULL; in btrfs_dio_iomap_end()
640 extent_changeset_free(dio_data->data_reserved); in btrfs_dio_iomap_end()
648 struct btrfs_inode *inode = bbio->inode; in btrfs_dio_end_io()
649 struct bio *bio = &bbio->bio; in btrfs_dio_end_io()
651 if (bio->bi_status) { in btrfs_dio_end_io()
652 btrfs_warn(inode->root->fs_info, in btrfs_dio_end_io()
654 btrfs_ino(inode), bio->bi_opf, in btrfs_dio_end_io()
655 dip->file_offset, dip->bytes, bio->bi_status); in btrfs_dio_end_io()
659 btrfs_finish_ordered_extent(bbio->ordered, NULL, in btrfs_dio_end_io()
660 dip->file_offset, dip->bytes, in btrfs_dio_end_io()
661 !bio->bi_status); in btrfs_dio_end_io()
663 unlock_dio_extent(&inode->io_tree, dip->file_offset, in btrfs_dio_end_io()
664 dip->file_offset + dip->bytes - 1, NULL); in btrfs_dio_end_io()
667 bbio->bio.bi_private = bbio->private; in btrfs_dio_end_io()
674 u64 start = (u64)bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; in btrfs_extract_ordered_extent()
675 u64 len = bbio->bio.bi_iter.bi_size; in btrfs_extract_ordered_extent()
680 if (WARN_ON_ONCE(start != ordered->disk_bytenr)) in btrfs_extract_ordered_extent()
681 return -EINVAL; in btrfs_extract_ordered_extent()
684 if (ordered->disk_num_bytes == len) { in btrfs_extract_ordered_extent()
685 refcount_inc(&ordered->refs); in btrfs_extract_ordered_extent()
686 bbio->ordered = ordered; in btrfs_extract_ordered_extent()
692 * a pre-existing one. in btrfs_extract_ordered_extent()
694 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags)) { in btrfs_extract_ordered_extent()
695 ret = split_extent_map(bbio->inode, bbio->file_offset, in btrfs_extract_ordered_extent()
696 ordered->num_bytes, len, in btrfs_extract_ordered_extent()
697 ordered->disk_bytenr); in btrfs_extract_ordered_extent()
705 bbio->ordered = new; in btrfs_extract_ordered_extent()
715 struct btrfs_dio_data *dio_data = iter->private; in btrfs_dio_submit_io()
717 btrfs_bio_init(bbio, BTRFS_I(iter->inode)->root->fs_info, in btrfs_dio_submit_io()
718 btrfs_dio_end_io, bio->bi_private); in btrfs_dio_submit_io()
719 bbio->inode = BTRFS_I(iter->inode); in btrfs_dio_submit_io()
720 bbio->file_offset = file_offset; in btrfs_dio_submit_io()
722 dip->file_offset = file_offset; in btrfs_dio_submit_io()
723 dip->bytes = bio->bi_iter.bi_size; in btrfs_dio_submit_io()
725 dio_data->submitted += bio->bi_iter.bi_size; in btrfs_dio_submit_io()
734 if (iter->flags & IOMAP_WRITE) { in btrfs_dio_submit_io()
737 ret = btrfs_extract_ordered_extent(bbio, dio_data->ordered); in btrfs_dio_submit_io()
739 btrfs_finish_ordered_extent(dio_data->ordered, NULL, in btrfs_dio_submit_io()
740 file_offset, dip->bytes, in btrfs_dio_submit_io()
742 bio->bi_status = errno_to_blk_status(ret); in btrfs_dio_submit_io()
782 const u32 blocksize_mask = fs_info->sectorsize - 1; in check_direct_IO()
785 return -EINVAL; in check_direct_IO()
788 return -EINVAL; in check_direct_IO()
795 struct file *file = iocb->ki_filp; in btrfs_direct_write()
807 if (iocb->ki_flags & IOCB_NOWAIT) in btrfs_direct_write()
816 if (iocb->ki_pos + iov_iter_count(from) <= i_size_read(inode) && IS_NOSEC(inode)) in btrfs_direct_write()
843 pos = iocb->ki_pos; in btrfs_direct_write()
845 * Re-check since file size may have changed just before taking the in btrfs_direct_write()
867 * find that ordered extent and then wait for it to complete (at in btrfs_direct_write()
875 * So here we disable page faults in the iov_iter and then retry if we in btrfs_direct_write()
876 * got -EFAULT, faulting in the pages before the retry. in btrfs_direct_write()
879 from->nofault = true; in btrfs_direct_write()
881 from->nofault = false; in btrfs_direct_write()
889 * deadlock on the inode lock - we are already holding it and we in btrfs_direct_write()
894 ASSERT(current->journal_info == NULL); in btrfs_direct_write()
895 current->journal_info = BTRFS_TRANS_DIO_WRITE_STUB; in btrfs_direct_write()
897 current->journal_info = NULL; in btrfs_direct_write()
904 if (iov_iter_count(from) > 0 && (ret == -EFAULT || ret > 0)) { in btrfs_direct_write()
908 * possible of the remainder pages and retry. We do this without in btrfs_direct_write()
914 * infinite loop if we retry after faulting the pages in, since in btrfs_direct_write()
921 ret = -ENOTBLK; in btrfs_direct_write()
932 * If 'ret' is -ENOTBLK or we have not written all data, then it means in btrfs_direct_write()
935 if ((ret < 0 && ret != -ENOTBLK) || !iov_iter_count(from)) in btrfs_direct_write()
940 * If we are in a NOWAIT context, then return -EAGAIN to signal the caller in btrfs_direct_write()
941 * it must retry the operation in a context where blocking is acceptable, in btrfs_direct_write()
945 if (iocb->ki_flags & IOCB_NOWAIT) { in btrfs_direct_write()
946 ret = -EAGAIN; in btrfs_direct_write()
950 pos = iocb->ki_pos; in btrfs_direct_write()
960 endbyte = pos + written_buffered - 1; in btrfs_direct_write()
964 ret = filemap_fdatawait_range(inode->i_mapping, pos, endbyte); in btrfs_direct_write()
968 iocb->ki_pos = pos + written_buffered; in btrfs_direct_write()
969 invalidate_mapping_pages(file->f_mapping, pos >> PAGE_SHIFT, in btrfs_direct_write()
988 for (seg = 0; seg < iter->nr_segs; seg++) { in check_direct_read()
989 for (i = seg + 1; i < iter->nr_segs; i++) { in check_direct_read()
993 if (iov1->iov_base == iov2->iov_base) in check_direct_read()
994 return -EINVAL; in check_direct_read()
1002 struct inode *inode = file_inode(iocb->ki_filp); in btrfs_direct_read()
1010 if (check_direct_read(inode_to_fs_info(inode), to, iocb->ki_pos)) in btrfs_direct_read()
1020 * which can still trigger page fault ins despite having set ->nofault in btrfs_direct_read()
1031 to->nofault = true; in btrfs_direct_read()
1033 to->nofault = false; in btrfs_direct_read()
1040 if (iov_iter_count(to) > 0 && (ret == -EFAULT || ret > 0)) { in btrfs_direct_read()
1053 * We made some progress since the last retry or this is in btrfs_direct_read()
1055 * as possible and retry. in btrfs_direct_read()
1071 return -ENOMEM; in btrfs_init_dio()