1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2024, Alibaba Cloud 4 * Copyright 2026 Docker, Inc. 5 */ 6 #include "internal.h" 7 #include <trace/events/erofs.h> 8 9 struct erofs_fileio_rq { 10 struct bio_vec bvecs[16]; 11 struct bio bio; 12 struct kiocb iocb; 13 struct super_block *sb; 14 refcount_t ref; 15 }; 16 17 struct erofs_fileio { 18 struct erofs_map_blocks map; 19 struct erofs_map_dev dev; 20 struct erofs_fileio_rq *rq; 21 }; 22 23 static void erofs_fileio_ki_complete(struct kiocb *iocb, long ret) 24 { 25 struct erofs_fileio_rq *rq = 26 container_of(iocb, struct erofs_fileio_rq, iocb); 27 struct folio_iter fi; 28 29 if (ret >= 0 && ret != rq->bio.bi_iter.bi_size) 30 ret = -EIO; 31 if (!rq->bio.bi_end_io) { 32 bio_for_each_folio_all(fi, &rq->bio) { 33 DBG_BUGON(folio_test_uptodate(fi.folio)); 34 iomap_finish_folio_read(fi.folio, fi.offset, fi.length, 35 ret < 0 ? ret : 0); 36 } 37 } else if (ret < 0 && !rq->bio.bi_status) { 38 rq->bio.bi_status = errno_to_blk_status(ret); 39 } 40 bio_endio(&rq->bio); 41 bio_uninit(&rq->bio); 42 if (refcount_dec_and_test(&rq->ref)) 43 kfree(rq); 44 } 45 46 static void erofs_fileio_rq_submit(struct erofs_fileio_rq *rq) 47 { 48 struct iov_iter iter; 49 ssize_t ret; 50 51 if (!rq) 52 return; 53 rq->iocb.ki_pos = rq->bio.bi_iter.bi_sector << SECTOR_SHIFT; 54 rq->iocb.ki_ioprio = get_current_ioprio(); 55 rq->iocb.ki_complete = erofs_fileio_ki_complete; 56 if (test_opt(&EROFS_SB(rq->sb)->opt, DIRECT_IO) && 57 rq->iocb.ki_filp->f_mode & FMODE_CAN_ODIRECT) 58 rq->iocb.ki_flags = IOCB_DIRECT; 59 iov_iter_bvec(&iter, ITER_DEST, rq->bvecs, rq->bio.bi_vcnt, 60 rq->bio.bi_iter.bi_size); 61 scoped_with_creds(rq->iocb.ki_filp->f_cred) 62 ret = vfs_iocb_iter_read(rq->iocb.ki_filp, &rq->iocb, &iter); 63 if (ret != -EIOCBQUEUED) 64 erofs_fileio_ki_complete(&rq->iocb, ret); 65 if (refcount_dec_and_test(&rq->ref)) 66 kfree(rq); 67 } 68 69 static struct erofs_fileio_rq *erofs_fileio_rq_alloc(struct super_block *sb, 70 struct file *file) 71 { 72 struct erofs_fileio_rq *rq = kzalloc_obj(*rq, GFP_KERNEL | __GFP_NOFAIL); 73 74 bio_init(&rq->bio, NULL, rq->bvecs, ARRAY_SIZE(rq->bvecs), REQ_OP_READ); 75 rq->iocb.ki_filp = file; 76 rq->sb = sb; 77 refcount_set(&rq->ref, 2); 78 return rq; 79 } 80 81 struct bio *erofs_fileio_bio_alloc(struct erofs_map_dev *mdev) 82 { 83 return &erofs_fileio_rq_alloc(mdev->m_sb, mdev->m_dif->file)->bio; 84 } 85 86 void erofs_fileio_submit_bio(struct bio *bio) 87 { 88 erofs_fileio_rq_submit(container_of(bio, struct erofs_fileio_rq, bio)); 89 } 90 91 static void erofs_fileio_iomap_submit_read(const struct iomap_iter *iter, 92 struct iomap_read_folio_ctx *ctx) 93 { 94 erofs_fileio_rq_submit((struct erofs_fileio_rq *)ctx->read_ctx); 95 ctx->read_ctx = NULL; 96 } 97 98 static int erofs_fileio_iomap_read_folio_range(const struct iomap_iter *iter, 99 struct iomap_read_folio_ctx *ctx, size_t plen) 100 { 101 struct erofs_iomap_iter_ctx *iter_ctx = iter->private; 102 struct folio *folio = ctx->cur_folio; 103 struct erofs_fileio_rq *rq = ctx->read_ctx; 104 sector_t sector = iomap_sector(&iter->iomap, iter->pos); 105 106 if (!rq || bio_end_sector(&rq->bio) != sector || 107 !bio_add_folio(&rq->bio, folio, plen, 108 offset_in_folio(folio, iter->pos))) { 109 struct super_block *sb = iter_ctx->realinode->i_sb; 110 struct erofs_device_info *dif = iter->iomap.private; 111 112 /* submit the existing range if there was one. */ 113 if (ctx->read_ctx) 114 erofs_fileio_iomap_submit_read(iter, ctx); 115 rq = erofs_fileio_rq_alloc(sb, dif->file); 116 rq->bio.bi_iter.bi_sector = sector; 117 bio_add_folio_nofail(&rq->bio, folio, plen, 118 offset_in_folio(folio, iter->pos)); 119 ctx->read_ctx = rq; 120 ctx->read_ctx_file_offset = iter->pos; 121 } 122 return 0; 123 } 124 125 const struct iomap_read_ops erofs_fileio_iomap_read_ops = { 126 .read_folio_range = erofs_fileio_iomap_read_folio_range, 127 .submit_read = erofs_fileio_iomap_submit_read, 128 }; 129 130 static void erofs_fileio_meta_endio(struct bio *bio) 131 { 132 bool uptodate = !bio->bi_status; 133 struct folio_iter fi; 134 135 bio_for_each_folio_all(fi, bio) 136 folio_end_read(fi.folio, uptodate); 137 } 138 139 int erofs_read_meta_folio(struct file *file, struct folio *folio) 140 { 141 struct erofs_fileio io = { 142 .dev = { .m_pa = folio_pos(folio), }, 143 }; 144 struct inode *inode = folio_inode(folio); 145 int err; 146 147 err = erofs_map_dev(inode->i_sb, &io.dev); 148 if (err) 149 return err; 150 151 io.rq = erofs_fileio_rq_alloc(inode->i_sb, io.dev.m_dif->file); 152 io.rq->bio.bi_end_io = erofs_fileio_meta_endio; 153 io.rq->bio.bi_iter.bi_sector = 154 (io.dev.m_dif->fsoff + io.dev.m_pa) >> 9; 155 bio_add_folio_nofail(&io.rq->bio, folio, folio_size(folio), 0); 156 erofs_fileio_rq_submit(io.rq); 157 return 0; 158 } 159