Lines Matching refs:fio
417 static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio) in f2fs_io_flags() argument
423 if (fio->op != REQ_OP_WRITE) in f2fs_io_flags()
425 if (fio->type == DATA) in f2fs_io_flags()
426 io_flag = fio->sbi->data_io_flag; in f2fs_io_flags()
427 else if (fio->type == NODE) in f2fs_io_flags()
428 io_flag = fio->sbi->node_io_flag; in f2fs_io_flags()
441 if (BIT(fio->temp) & meta_flag) in f2fs_io_flags()
443 if (BIT(fio->temp) & fua_flag) in f2fs_io_flags()
446 if (fio->type == DATA && in f2fs_io_flags()
447 F2FS_I(fio->folio->mapping->host)->ioprio_hint == F2FS_IOPRIO_WRITE) in f2fs_io_flags()
453 static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) in __bio_alloc() argument
455 struct f2fs_sb_info *sbi = fio->sbi; in __bio_alloc()
460 bdev = f2fs_target_device(sbi, fio->new_blkaddr, §or); in __bio_alloc()
462 fio->op | fio->op_flags | f2fs_io_flags(fio), in __bio_alloc()
465 if (is_read_io(fio->op)) { in __bio_alloc()
472 fio->type, fio->temp); in __bio_alloc()
476 if (fio->io_wbc) in __bio_alloc()
477 wbc_init_bio(fio->io_wbc, bio); in __bio_alloc()
484 const struct f2fs_io_info *fio, in f2fs_set_bio_crypt_ctx() argument
491 if (!fio || !fio->encrypted_page) in f2fs_set_bio_crypt_ctx()
497 const struct f2fs_io_info *fio) in f2fs_crypt_mergeable_bio() argument
503 if (fio && fio->encrypted_page) in f2fs_crypt_mergeable_bio()
530 struct f2fs_io_info *fio = &io->fio; in __submit_merged_bio() local
535 if (is_read_io(fio->op)) { in __submit_merged_bio()
536 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); in __submit_merged_bio()
537 f2fs_submit_read_bio(io->sbi, io->bio, fio->type); in __submit_merged_bio()
539 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); in __submit_merged_bio()
540 f2fs_submit_write_bio(io->sbi, io->bio, fio->type); in __submit_merged_bio()
630 io->fio.type = META_FLUSH; in __f2fs_submit_merged_write()
688 int f2fs_submit_page_bio(struct f2fs_io_info *fio) in f2fs_submit_page_bio() argument
691 struct folio *fio_folio = fio->folio; in f2fs_submit_page_bio()
692 struct folio *data_folio = fio->encrypted_page ? in f2fs_submit_page_bio()
693 page_folio(fio->encrypted_page) : fio_folio; in f2fs_submit_page_bio()
695 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, in f2fs_submit_page_bio()
696 fio->is_por ? META_POR : (__is_meta_io(fio) ? in f2fs_submit_page_bio()
700 trace_f2fs_submit_folio_bio(data_folio, fio); in f2fs_submit_page_bio()
703 bio = __bio_alloc(fio, 1); in f2fs_submit_page_bio()
706 fio_folio->index, fio, GFP_NOIO); in f2fs_submit_page_bio()
709 if (fio->io_wbc && !is_read_io(fio->op)) in f2fs_submit_page_bio()
710 wbc_account_cgroup_owner(fio->io_wbc, fio_folio, PAGE_SIZE); in f2fs_submit_page_bio()
712 inc_page_count(fio->sbi, is_read_io(fio->op) ? in f2fs_submit_page_bio()
713 __read_io_type(data_folio) : WB_DATA_TYPE(fio->folio, false)); in f2fs_submit_page_bio()
716 f2fs_submit_read_bio(fio->sbi, bio, fio->type); in f2fs_submit_page_bio()
718 f2fs_submit_write_bio(fio->sbi, bio, fio->type); in f2fs_submit_page_bio()
734 struct f2fs_io_info *fio) in io_type_is_mergeable() argument
738 if (io->fio.op != fio->op) in io_type_is_mergeable()
740 return (io->fio.op_flags & mask) == (fio->op_flags & mask); in io_type_is_mergeable()
745 struct f2fs_io_info *fio, in io_is_mergeable() argument
751 return io_type_is_mergeable(io, fio); in io_is_mergeable()
777 static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio, in add_ipu_page() argument
780 struct folio *fio_folio = fio->folio; in add_ipu_page()
781 struct f2fs_sb_info *sbi = fio->sbi; in add_ipu_page()
799 *fio->last_block, in add_ipu_page()
800 fio->new_blkaddr)); in add_ipu_page()
803 fio_folio->index, fio) && in add_ipu_page()
883 int f2fs_merge_page_bio(struct f2fs_io_info *fio) in f2fs_merge_page_bio() argument
885 struct bio *bio = *fio->bio; in f2fs_merge_page_bio()
886 struct folio *data_folio = fio->encrypted_page ? in f2fs_merge_page_bio()
887 page_folio(fio->encrypted_page) : fio->folio; in f2fs_merge_page_bio()
888 struct folio *folio = fio->folio; in f2fs_merge_page_bio()
890 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, in f2fs_merge_page_bio()
891 __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC)) in f2fs_merge_page_bio()
894 trace_f2fs_submit_folio_bio(data_folio, fio); in f2fs_merge_page_bio()
896 if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block, in f2fs_merge_page_bio()
897 fio->new_blkaddr)) in f2fs_merge_page_bio()
898 f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL); in f2fs_merge_page_bio()
901 bio = __bio_alloc(fio, BIO_MAX_VECS); in f2fs_merge_page_bio()
903 folio->index, fio, GFP_NOIO); in f2fs_merge_page_bio()
905 add_bio_entry(fio->sbi, bio, data_folio, fio->temp); in f2fs_merge_page_bio()
907 if (add_ipu_page(fio, &bio, data_folio)) in f2fs_merge_page_bio()
911 if (fio->io_wbc) in f2fs_merge_page_bio()
912 wbc_account_cgroup_owner(fio->io_wbc, folio, folio_size(folio)); in f2fs_merge_page_bio()
914 inc_page_count(fio->sbi, WB_DATA_TYPE(folio, false)); in f2fs_merge_page_bio()
916 *fio->last_block = fio->new_blkaddr; in f2fs_merge_page_bio()
917 *fio->bio = bio; in f2fs_merge_page_bio()
944 void f2fs_submit_page_write(struct f2fs_io_info *fio) in f2fs_submit_page_write() argument
946 struct f2fs_sb_info *sbi = fio->sbi; in f2fs_submit_page_write()
947 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); in f2fs_submit_page_write()
948 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; in f2fs_submit_page_write()
952 f2fs_bug_on(sbi, is_read_io(fio->op)); in f2fs_submit_page_write()
965 if (fio->in_list) { in f2fs_submit_page_write()
971 fio = list_first_entry(&io->io_list, in f2fs_submit_page_write()
973 list_del(&fio->list); in f2fs_submit_page_write()
977 verify_fio_blkaddr(fio); in f2fs_submit_page_write()
979 if (fio->encrypted_page) in f2fs_submit_page_write()
980 bio_folio = page_folio(fio->encrypted_page); in f2fs_submit_page_write()
981 else if (fio->compressed_page) in f2fs_submit_page_write()
982 bio_folio = page_folio(fio->compressed_page); in f2fs_submit_page_write()
984 bio_folio = fio->folio; in f2fs_submit_page_write()
987 fio->submitted = 1; in f2fs_submit_page_write()
989 type = WB_DATA_TYPE(bio_folio, fio->compressed_page); in f2fs_submit_page_write()
993 (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio, in f2fs_submit_page_write()
994 fio->new_blkaddr) || in f2fs_submit_page_write()
995 !f2fs_crypt_mergeable_bio(io->bio, fio_inode(fio), in f2fs_submit_page_write()
996 bio_folio->index, fio))) in f2fs_submit_page_write()
1000 io->bio = __bio_alloc(fio, BIO_MAX_VECS); in f2fs_submit_page_write()
1001 f2fs_set_bio_crypt_ctx(io->bio, fio_inode(fio), in f2fs_submit_page_write()
1002 bio_folio->index, fio, GFP_NOIO); in f2fs_submit_page_write()
1003 io->fio = *fio; in f2fs_submit_page_write()
1011 if (fio->io_wbc) in f2fs_submit_page_write()
1012 wbc_account_cgroup_owner(fio->io_wbc, fio->folio, in f2fs_submit_page_write()
1013 folio_size(fio->folio)); in f2fs_submit_page_write()
1015 io->last_block_in_bio = fio->new_blkaddr; in f2fs_submit_page_write()
1017 trace_f2fs_submit_folio_write(fio->folio, fio); in f2fs_submit_page_write()
1020 is_end_zone_blkaddr(sbi, fio->new_blkaddr)) { in f2fs_submit_page_write()
1030 if (fio->in_list) in f2fs_submit_page_write()
2494 int f2fs_encrypt_one_page(struct f2fs_io_info *fio) in f2fs_encrypt_one_page() argument
2496 struct inode *inode = fio_inode(fio); in f2fs_encrypt_one_page()
2504 page = fio->compressed_page ? fio->compressed_page : fio->page; in f2fs_encrypt_one_page()
2510 fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page_folio(page), in f2fs_encrypt_one_page()
2512 if (IS_ERR(fio->encrypted_page)) { in f2fs_encrypt_one_page()
2514 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) { in f2fs_encrypt_one_page()
2515 f2fs_flush_merged_writes(fio->sbi); in f2fs_encrypt_one_page()
2520 return PTR_ERR(fio->encrypted_page); in f2fs_encrypt_one_page()
2523 mfolio = filemap_lock_folio(META_MAPPING(fio->sbi), fio->old_blkaddr); in f2fs_encrypt_one_page()
2527 page_address(fio->encrypted_page), PAGE_SIZE); in f2fs_encrypt_one_page()
2534 struct f2fs_io_info *fio) in check_inplace_update_policy() argument
2554 if (IS_F2FS_IPU_ASYNC(sbi) && fio && fio->op == REQ_OP_WRITE && in check_inplace_update_policy()
2555 !(fio->op_flags & REQ_SYNC) && !IS_ENCRYPTED(inode)) in check_inplace_update_policy()
2562 if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) && in check_inplace_update_policy()
2563 !f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) in check_inplace_update_policy()
2569 bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio) in f2fs_should_update_inplace() argument
2582 return check_inplace_update_policy(inode, fio); in f2fs_should_update_inplace()
2585 bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) in f2fs_should_update_outplace() argument
2592 if (fio && is_sbi_flag_set(sbi, SBI_NEED_FSCK)) in f2fs_should_update_outplace()
2615 if (fio) { in f2fs_should_update_outplace()
2616 if (page_private_gcing(fio->page)) in f2fs_should_update_outplace()
2619 f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) in f2fs_should_update_outplace()
2625 static inline bool need_inplace_update(struct f2fs_io_info *fio) in need_inplace_update() argument
2627 struct inode *inode = fio_inode(fio); in need_inplace_update()
2629 if (f2fs_should_update_outplace(inode, fio)) in need_inplace_update()
2632 return f2fs_should_update_inplace(inode, fio); in need_inplace_update()
2635 int f2fs_do_write_data_page(struct f2fs_io_info *fio) in f2fs_do_write_data_page() argument
2637 struct folio *folio = fio->folio; in f2fs_do_write_data_page()
2653 if (need_inplace_update(fio) && in f2fs_do_write_data_page()
2655 &fio->old_blkaddr)) { in f2fs_do_write_data_page()
2656 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, in f2fs_do_write_data_page()
2661 fio->need_lock = LOCK_DONE; in f2fs_do_write_data_page()
2666 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi)) in f2fs_do_write_data_page()
2673 fio->old_blkaddr = dn.data_blkaddr; in f2fs_do_write_data_page()
2676 if (fio->old_blkaddr == NULL_ADDR) { in f2fs_do_write_data_page()
2682 if (__is_valid_data_blkaddr(fio->old_blkaddr) && in f2fs_do_write_data_page()
2683 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, in f2fs_do_write_data_page()
2690 if (fio->meta_gc) in f2fs_do_write_data_page()
2691 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr); in f2fs_do_write_data_page()
2698 (__is_valid_data_blkaddr(fio->old_blkaddr) && in f2fs_do_write_data_page()
2699 need_inplace_update(fio))) { in f2fs_do_write_data_page()
2700 err = f2fs_encrypt_one_page(fio); in f2fs_do_write_data_page()
2706 if (fio->need_lock == LOCK_REQ) in f2fs_do_write_data_page()
2707 f2fs_unlock_op(fio->sbi); in f2fs_do_write_data_page()
2708 err = f2fs_inplace_write_data(fio); in f2fs_do_write_data_page()
2711 fscrypt_finalize_bounce_page(&fio->encrypted_page); in f2fs_do_write_data_page()
2720 if (fio->need_lock == LOCK_RETRY) { in f2fs_do_write_data_page()
2721 if (!f2fs_trylock_op(fio->sbi)) { in f2fs_do_write_data_page()
2725 fio->need_lock = LOCK_REQ; in f2fs_do_write_data_page()
2728 err = f2fs_get_node_info(fio->sbi, dn.nid, &ni, false); in f2fs_do_write_data_page()
2732 fio->version = ni.version; in f2fs_do_write_data_page()
2734 err = f2fs_encrypt_one_page(fio); in f2fs_do_write_data_page()
2740 if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR) in f2fs_do_write_data_page()
2741 f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false); in f2fs_do_write_data_page()
2744 f2fs_outplace_write_data(&dn, fio); in f2fs_do_write_data_page()
2752 if (fio->need_lock == LOCK_REQ) in f2fs_do_write_data_page()
2753 f2fs_unlock_op(fio->sbi); in f2fs_do_write_data_page()
2775 struct f2fs_io_info fio = { in f2fs_write_single_data_page() local
2841 fio.need_lock = LOCK_DONE; in f2fs_write_single_data_page()
2842 err = f2fs_do_write_data_page(&fio); in f2fs_write_single_data_page()
2859 err = f2fs_do_write_data_page(&fio); in f2fs_write_single_data_page()
2862 fio.need_lock = LOCK_REQ; in f2fs_write_single_data_page()
2863 err = f2fs_do_write_data_page(&fio); in f2fs_write_single_data_page()
2899 *submitted = fio.submitted; in f2fs_write_single_data_page()