1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2010 Red Hat, Inc.
4 * Copyright (C) 2016-2023 Christoph Hellwig.
5 */
6 #include <linux/bio-integrity.h>
7 #include <linux/iomap.h>
8 #include <linux/pagemap.h>
9 #include "internal.h"
10 #include "trace.h"
11
12 static DEFINE_SPINLOCK(failed_read_lock);
13 static struct bio_list failed_read_list = BIO_EMPTY_LIST;
14
__iomap_read_end_io(struct bio * bio,int error)15 static u32 __iomap_read_end_io(struct bio *bio, int error)
16 {
17 struct folio_iter fi;
18 u32 folio_count = 0;
19
20 bio_for_each_folio_all(fi, bio) {
21 iomap_finish_folio_read(fi.folio, fi.offset, fi.length, error);
22 folio_count++;
23 }
24 if (bio_integrity(bio))
25 fs_bio_integrity_free(bio);
26 bio_put(bio);
27 return folio_count;
28 }
29
30 static void
iomap_fail_reads(struct work_struct * work)31 iomap_fail_reads(
32 struct work_struct *work)
33 {
34 struct bio *bio;
35 struct bio_list tmp = BIO_EMPTY_LIST;
36 unsigned long flags;
37
38 spin_lock_irqsave(&failed_read_lock, flags);
39 bio_list_merge_init(&tmp, &failed_read_list);
40 spin_unlock_irqrestore(&failed_read_lock, flags);
41
42 while ((bio = bio_list_pop(&tmp)) != NULL) {
43 __iomap_read_end_io(bio, blk_status_to_errno(bio->bi_status));
44 cond_resched();
45 }
46 }
47
48 static DECLARE_WORK(failed_read_work, iomap_fail_reads);
49
iomap_fail_buffered_read(struct bio * bio)50 static void iomap_fail_buffered_read(struct bio *bio)
51 {
52 unsigned long flags;
53
54 /*
55 * Bounce I/O errors to a workqueue to avoid nested i_lock acquisitions
56 * in the fserror code. The caller no longer owns the bio reference
57 * after the spinlock drops.
58 */
59 spin_lock_irqsave(&failed_read_lock, flags);
60 if (bio_list_empty(&failed_read_list))
61 WARN_ON_ONCE(!schedule_work(&failed_read_work));
62 bio_list_add(&failed_read_list, bio);
63 spin_unlock_irqrestore(&failed_read_lock, flags);
64 }
65
iomap_read_end_io(struct bio * bio)66 static void iomap_read_end_io(struct bio *bio)
67 {
68 if (bio->bi_status) {
69 iomap_fail_buffered_read(bio);
70 return;
71 }
72
73 __iomap_read_end_io(bio, 0);
74 }
75
iomap_finish_ioend_buffered_read(struct iomap_ioend * ioend)76 u32 iomap_finish_ioend_buffered_read(struct iomap_ioend *ioend)
77 {
78 return __iomap_read_end_io(&ioend->io_bio, ioend->io_error);
79 }
80
iomap_bio_submit_read_endio(const struct iomap_iter * iter,struct iomap_read_folio_ctx * ctx,bio_end_io_t end_io)81 void iomap_bio_submit_read_endio(const struct iomap_iter *iter,
82 struct iomap_read_folio_ctx *ctx, bio_end_io_t end_io)
83 {
84 struct bio *bio = ctx->read_ctx;
85
86 bio->bi_end_io = end_io;
87 if (iter->iomap.flags & IOMAP_F_INTEGRITY)
88 fs_bio_integrity_alloc(bio);
89 submit_bio(bio);
90
91 ctx->read_ctx = NULL;
92 }
93 EXPORT_SYMBOL_GPL(iomap_bio_submit_read_endio);
94
iomap_bio_submit_read(const struct iomap_iter * iter,struct iomap_read_folio_ctx * ctx)95 static void iomap_bio_submit_read(const struct iomap_iter *iter,
96 struct iomap_read_folio_ctx *ctx)
97 {
98 return iomap_bio_submit_read_endio(iter, ctx, iomap_read_end_io);
99 }
100
iomap_read_bio_set(struct iomap_read_folio_ctx * ctx)101 static struct bio_set *iomap_read_bio_set(struct iomap_read_folio_ctx *ctx)
102 {
103 if (ctx->ops && ctx->ops->bio_set)
104 return ctx->ops->bio_set;
105 return &fs_bio_set;
106 }
107
iomap_read_alloc_bio(const struct iomap_iter * iter,struct iomap_read_folio_ctx * ctx,size_t plen)108 static void iomap_read_alloc_bio(const struct iomap_iter *iter,
109 struct iomap_read_folio_ctx *ctx, size_t plen)
110 {
111 const struct iomap *iomap = &iter->iomap;
112 unsigned int nr_vecs = DIV_ROUND_UP(iomap_length(iter), PAGE_SIZE);
113 struct bio_set *bio_set = iomap_read_bio_set(ctx);
114 struct folio *folio = ctx->cur_folio;
115 gfp_t gfp = mapping_gfp_constraint(folio->mapping, GFP_KERNEL);
116 gfp_t orig_gfp = gfp;
117 struct bio *bio;
118
119 /* Submit the existing range if there was one. */
120 if (ctx->read_ctx)
121 ctx->ops->submit_read(iter, ctx);
122
123 /* Same as readahead_gfp_mask: */
124 if (ctx->rac)
125 gfp |= __GFP_NORETRY | __GFP_NOWARN;
126
127 /*
128 * If the bio_alloc fails, try it again for a single page to avoid
129 * having to deal with partial page reads. This emulates what
130 * do_mpage_read_folio does.
131 */
132 bio = bio_alloc_bioset(iomap->bdev, bio_max_segs(nr_vecs), REQ_OP_READ,
133 gfp, bio_set);
134 if (!bio)
135 bio = bio_alloc_bioset(iomap->bdev, 1, REQ_OP_READ, orig_gfp,
136 bio_set);
137 if (ctx->rac)
138 bio->bi_opf |= REQ_RAHEAD;
139 bio->bi_iter.bi_sector = iomap_sector(iomap, iter->pos);
140 bio_add_folio_nofail(bio, folio, plen,
141 offset_in_folio(folio, iter->pos));
142 ctx->read_ctx = bio;
143 ctx->read_ctx_file_offset = iter->pos;
144 }
145
iomap_bio_read_folio_range(const struct iomap_iter * iter,struct iomap_read_folio_ctx * ctx,size_t plen)146 int iomap_bio_read_folio_range(const struct iomap_iter *iter,
147 struct iomap_read_folio_ctx *ctx, size_t plen)
148 {
149 struct folio *folio = ctx->cur_folio;
150 struct bio *bio = ctx->read_ctx;
151
152 if (!bio ||
153 bio_end_sector(bio) != iomap_sector(&iter->iomap, iter->pos) ||
154 bio->bi_iter.bi_size > iomap_max_bio_size(&iter->iomap) - plen ||
155 !bio_add_folio(bio, folio, plen, offset_in_folio(folio, iter->pos)))
156 iomap_read_alloc_bio(iter, ctx, plen);
157 return 0;
158 }
159 EXPORT_SYMBOL_GPL(iomap_bio_read_folio_range);
160
161 const struct iomap_read_ops iomap_bio_read_ops = {
162 .read_folio_range = iomap_bio_read_folio_range,
163 .submit_read = iomap_bio_submit_read,
164 };
165 EXPORT_SYMBOL_GPL(iomap_bio_read_ops);
166
iomap_bio_read_folio_range_sync(const struct iomap_iter * iter,struct folio * folio,loff_t pos,size_t len)167 int iomap_bio_read_folio_range_sync(const struct iomap_iter *iter,
168 struct folio *folio, loff_t pos, size_t len)
169 {
170 const struct iomap *srcmap = iomap_iter_srcmap(iter);
171 sector_t sector = iomap_sector(srcmap, pos);
172 struct bio_vec bvec;
173 struct bio bio;
174 int error;
175
176 bio_init(&bio, srcmap->bdev, &bvec, 1, REQ_OP_READ);
177 bio.bi_iter.bi_sector = sector;
178 bio_add_folio_nofail(&bio, folio, len, offset_in_folio(folio, pos));
179 if (srcmap->flags & IOMAP_F_INTEGRITY)
180 fs_bio_integrity_alloc(&bio);
181 error = submit_bio_wait(&bio);
182 if (bio_integrity(&bio)) {
183 if (!error)
184 error = fs_bio_integrity_verify(&bio, sector, len);
185 fs_bio_integrity_free(&bio);
186 }
187 bio_uninit(&bio);
188 return error;
189 }
190