1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2010 Red Hat, Inc. 4 * Copyright (C) 2016-2023 Christoph Hellwig. 5 */ 6 #include <linux/bio-integrity.h> 7 #include <linux/iomap.h> 8 #include <linux/pagemap.h> 9 #include "internal.h" 10 #include "trace.h" 11 12 static DEFINE_SPINLOCK(failed_read_lock); 13 static struct bio_list failed_read_list = BIO_EMPTY_LIST; 14 15 static u32 __iomap_read_end_io(struct bio *bio, int error) 16 { 17 struct folio_iter fi; 18 u32 folio_count = 0; 19 20 bio_for_each_folio_all(fi, bio) { 21 iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error); 22 folio_count++; 23 } 24 if (bio_integrity(bio)) 25 fs_bio_integrity_free(bio); 26 bio_put(bio); 27 return folio_count; 28 } 29 30 static void 31 iomap_fail_reads( 32 struct work_struct *work) 33 { 34 struct bio *bio; 35 struct bio_list tmp = BIO_EMPTY_LIST; 36 unsigned long flags; 37 38 spin_lock_irqsave(&failed_read_lock, flags); 39 bio_list_merge_init(&tmp, &failed_read_list); 40 spin_unlock_irqrestore(&failed_read_lock, flags); 41 42 while ((bio = bio_list_pop(&tmp)) != NULL) { 43 __iomap_read_end_io(bio, blk_status_to_errno(bio->bi_status)); 44 cond_resched(); 45 } 46 } 47 48 static DECLARE_WORK(failed_read_work, iomap_fail_reads); 49 50 static void iomap_fail_buffered_read(struct bio *bio) 51 { 52 unsigned long flags; 53 54 /* 55 * Bounce I/O errors to a workqueue to avoid nested i_lock acquisitions 56 * in the fserror code. The caller no longer owns the bio reference 57 * after the spinlock drops. 58 */ 59 spin_lock_irqsave(&failed_read_lock, flags); 60 if (bio_list_empty(&failed_read_list)) 61 WARN_ON_ONCE(!schedule_work(&failed_read_work)); 62 bio_list_add(&failed_read_list, bio); 63 spin_unlock_irqrestore(&failed_read_lock, flags); 64 } 65 66 static void iomap_read_end_io(struct bio *bio) 67 { 68 if (bio->bi_status) { 69 iomap_fail_buffered_read(bio); 70 return; 71 } 72 73 __iomap_read_end_io(bio, 0); 74 } 75 76 u32 iomap_finish_ioend_buffered_read(struct iomap_ioend *ioend) 77 { 78 return __iomap_read_end_io(&ioend->io_bio, ioend->io_error); 79 } 80 81 void iomap_bio_submit_read_endio(const struct iomap_iter *iter, 82 struct iomap_read_folio_ctx *ctx, bio_end_io_t end_io) 83 { 84 struct bio *bio = ctx->read_ctx; 85 86 bio->bi_end_io = end_io; 87 if (iter->iomap.flags & IOMAP_F_INTEGRITY) 88 fs_bio_integrity_alloc(bio); 89 submit_bio(bio); 90 91 ctx->read_ctx = NULL; 92 } 93 EXPORT_SYMBOL_GPL(iomap_bio_submit_read_endio); 94 95 static void iomap_bio_submit_read(const struct iomap_iter *iter, 96 struct iomap_read_folio_ctx *ctx) 97 { 98 return iomap_bio_submit_read_endio(iter, ctx, iomap_read_end_io); 99 } 100 101 static struct bio_set *iomap_read_bio_set(struct iomap_read_folio_ctx *ctx) 102 { 103 if (ctx->ops && ctx->ops->bio_set) 104 return ctx->ops->bio_set; 105 return &fs_bio_set; 106 } 107 108 static void iomap_read_alloc_bio(const struct iomap_iter *iter, 109 struct iomap_read_folio_ctx *ctx, size_t plen) 110 { 111 const struct iomap *iomap = &iter->iomap; 112 unsigned int nr_vecs = DIV_ROUND_UP(iomap_length(iter), PAGE_SIZE); 113 struct bio_set *bio_set = iomap_read_bio_set(ctx); 114 struct folio *folio = ctx->cur_folio; 115 gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL); 116 gfp_t orig_gfp = gfp; 117 struct bio *bio; 118 119 /* Submit the existing range if there was one. */ 120 if (ctx->read_ctx) 121 ctx->ops->submit_read(iter, ctx); 122 123 /* Same as readahead_gfp_mask: */ 124 if (ctx->rac) 125 gfp |= __GFP_NORETRY | __GFP_NOWARN; 126 127 /* 128 * If the bio_alloc fails, try it again for a single page to avoid 129 * having to deal with partial page reads. This emulates what 130 * do_mpage_read_folio does. 131 */ 132 bio = bio_alloc_bioset(iomap->bdev, bio_max_segs(nr_vecs), REQ_OP_READ, 133 gfp, bio_set); 134 if (!bio) 135 bio = bio_alloc_bioset(iomap->bdev, 1, REQ_OP_READ, orig_gfp, 136 bio_set); 137 if (ctx->rac) 138 bio->bi_opf |= REQ_RAHEAD; 139 bio->bi_iter.bi_sector = iomap_sector(iomap, iter->pos); 140 bio_add_folio_nofail(bio, folio, plen, 141 offset_in_folio(folio, iter->pos)); 142 ctx->read_ctx = bio; 143 ctx->read_ctx_file_offset = iter->pos; 144 } 145 146 int iomap_bio_read_folio_range(const struct iomap_iter *iter, 147 struct iomap_read_folio_ctx *ctx, size_t plen) 148 { 149 struct folio *folio = ctx->cur_folio; 150 struct bio *bio = ctx->read_ctx; 151 152 if (!bio || 153 bio_end_sector(bio) != iomap_sector(&iter->iomap, iter->pos) || 154 bio->bi_iter.bi_size > iomap_max_bio_size(&iter->iomap) - plen || 155 !bio_add_folio(bio, folio, plen, offset_in_folio(folio, iter->pos))) 156 iomap_read_alloc_bio(iter, ctx, plen); 157 return 0; 158 } 159 EXPORT_SYMBOL_GPL(iomap_bio_read_folio_range); 160 161 const struct iomap_read_ops iomap_bio_read_ops = { 162 .read_folio_range = iomap_bio_read_folio_range, 163 .submit_read = iomap_bio_submit_read, 164 }; 165 EXPORT_SYMBOL_GPL(iomap_bio_read_ops); 166 167 int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter, 168 struct folio *folio, loff_t pos, size_t len) 169 { 170 const struct iomap *srcmap = iomap_iter_srcmap(iter); 171 sector_t sector = iomap_sector(srcmap, pos); 172 struct bio_vec bvec; 173 struct bio bio; 174 int error; 175 176 bio_init(&bio, srcmap->bdev, &bvec, 1, REQ_OP_READ); 177 bio.bi_iter.bi_sector = sector; 178 bio_add_folio_nofail(&bio, folio, len, offset_in_folio(folio, pos)); 179 if (srcmap->flags & IOMAP_F_INTEGRITY) 180 fs_bio_integrity_alloc(&bio); 181 error = submit_bio_wait(&bio); 182 if (bio_integrity(&bio)) { 183 if (!error) 184 error = fs_bio_integrity_verify(&bio, sector, len); 185 fs_bio_integrity_free(&bio); 186 } 187 bio_uninit(&bio); 188 return error; 189 } 190