1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2016-2025 Christoph Hellwig.
4 */
5 #include <linux/bio-integrity.h>
6 #include <linux/iomap.h>
7 #include <linux/list_sort.h>
8 #include <linux/pagemap.h>
9 #include <linux/writeback.h>
10 #include <linux/fserror.h>
11 #include "internal.h"
12 #include "trace.h"
13
14 struct bio_set iomap_ioend_bioset;
15 EXPORT_SYMBOL_GPL(iomap_ioend_bioset);
16 static struct bio_set iomap_ioend_split_bioset;
17
iomap_init_ioend(struct inode * inode,struct bio * bio,loff_t file_offset,u16 ioend_flags)18 struct iomap_ioend *iomap_init_ioend(struct inode *inode,
19 struct bio *bio, loff_t file_offset, u16 ioend_flags)
20 {
21 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
22
23 atomic_set(&ioend->io_remaining, 1);
24 ioend->io_error = 0;
25 ioend->io_parent = NULL;
26 INIT_LIST_HEAD(&ioend->io_list);
27 ioend->io_flags = ioend_flags;
28 ioend->io_bvec_offset = bio->bi_iter.bi_offset;
29 ioend->io_inode = inode;
30 ioend->io_offset = file_offset;
31 ioend->io_size = bio->bi_iter.bi_size;
32 ioend->io_sector = bio->bi_iter.bi_sector;
33 ioend->io_vi = NULL;
34 ioend->io_private = NULL;
35 return ioend;
36 }
37 EXPORT_SYMBOL_GPL(iomap_init_ioend);
38
39 /*
40 * We're now finished for good with this ioend structure. Update the folio
41 * state, release holds on bios, and finally free up memory. Do not use the
42 * ioend after this.
43 */
iomap_finish_ioend_buffered_write(struct iomap_ioend * ioend)44 static u32 iomap_finish_ioend_buffered_write(struct iomap_ioend *ioend)
45 {
46 struct inode *inode = ioend->io_inode;
47 struct bio *bio = &ioend->io_bio;
48 struct folio_iter fi;
49 u32 folio_count = 0;
50
51 if (ioend->io_error) {
52 mapping_set_error(inode->i_mapping, ioend->io_error);
53 if (!bio_flagged(bio, BIO_QUIET)) {
54 pr_err_ratelimited(
55 "%s: writeback error on inode %llu, offset %lld, sector %llu",
56 inode->i_sb->s_id, inode->i_ino,
57 ioend->io_offset, ioend->io_sector);
58 }
59 }
60
61 /* walk all folios in bio, ending page IO on them */
62 bio_for_each_folio_all(fi, bio) {
63 if (ioend->io_error)
64 fserror_report_io(inode, FSERR_BUFFERED_WRITE,
65 folio_pos(fi.folio) + fi.offset,
66 fi.length, ioend->io_error,
67 GFP_ATOMIC);
68 iomap_finish_folio_write(inode, fi.folio, fi.length);
69 folio_count++;
70 }
71
72 if (bio_integrity(bio))
73 fs_bio_integrity_free(bio);
74 bio_put(bio); /* frees the ioend */
75 return folio_count;
76 }
77
ioend_writeback_end_bio(struct bio * bio)78 static void ioend_writeback_end_bio(struct bio *bio)
79 {
80 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
81
82 if (bio->bi_status && bio_complete_in_task(bio))
83 return;
84 ioend->io_error = blk_status_to_errno(bio->bi_status);
85 iomap_finish_ioend_buffered_write(ioend);
86 }
87
88 /*
89 * We cannot cancel the ioend directly in case of an error, so call the bio end
90 * I/O handler with the error status here to run the normal I/O completion
91 * handler.
92 */
iomap_ioend_writeback_submit(struct iomap_writepage_ctx * wpc,int error)93 int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error)
94 {
95 struct iomap_ioend *ioend = wpc->wb_ctx;
96
97 if (!ioend->io_bio.bi_end_io)
98 ioend->io_bio.bi_end_io = ioend_writeback_end_bio;
99
100 if (WARN_ON_ONCE(wpc->iomap.flags & IOMAP_F_ANON_WRITE))
101 error = -EIO;
102
103 if (error) {
104 ioend->io_bio.bi_status = errno_to_blk_status(error);
105 bio_endio(&ioend->io_bio);
106 return error;
107 }
108
109 if (ioend->io_flags & IOMAP_IOEND_INTEGRITY)
110 fs_bio_integrity_generate(&ioend->io_bio);
111 submit_bio(&ioend->io_bio);
112 return 0;
113 }
114 EXPORT_SYMBOL_GPL(iomap_ioend_writeback_submit);
115
iomap_alloc_ioend(struct iomap_writepage_ctx * wpc,loff_t pos,u16 ioend_flags)116 static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc,
117 loff_t pos, u16 ioend_flags)
118 {
119 struct bio *bio;
120
121 bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS,
122 REQ_OP_WRITE | wbc_to_write_flags(wpc->wbc),
123 GFP_NOFS, &iomap_ioend_bioset);
124 bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos);
125 bio->bi_write_hint = wpc->inode->i_write_hint;
126 wbc_init_bio(wpc->wbc, bio);
127 wpc->nr_folios = 0;
128 return iomap_init_ioend(wpc->inode, bio, pos, ioend_flags);
129 }
130
iomap_can_add_to_ioend(struct iomap_writepage_ctx * wpc,loff_t pos,unsigned int map_len,u16 ioend_flags)131 static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos,
132 unsigned int map_len, u16 ioend_flags)
133 {
134 struct iomap_ioend *ioend = wpc->wb_ctx;
135
136 if (ioend->io_bio.bi_iter.bi_size >
137 iomap_max_bio_size(&wpc->iomap) - map_len)
138 return false;
139 if (ioend_flags & IOMAP_IOEND_BOUNDARY)
140 return false;
141 if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
142 (ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
143 return false;
144 if (pos != ioend->io_offset + ioend->io_size)
145 return false;
146 if (!(wpc->iomap.flags & IOMAP_F_ANON_WRITE) &&
147 iomap_sector(&wpc->iomap, pos) != bio_end_sector(&ioend->io_bio))
148 return false;
149 /*
150 * Limit ioend bio chain lengths to minimise IO completion latency. This
151 * also prevents long tight loops ending page writeback on all the
152 * folios in the ioend.
153 */
154 if (wpc->nr_folios >= IOEND_BATCH_SIZE)
155 return false;
156 return true;
157 }
158
159 /*
160 * Test to see if we have an existing ioend structure that we could append to
161 * first; otherwise finish off the current ioend and start another.
162 *
163 * If a new ioend is created and cached, the old ioend is submitted to the block
164 * layer instantly. Batching optimisations are provided by higher level block
165 * plugging.
166 *
167 * At the end of a writeback pass, there will be a cached ioend remaining on the
168 * writepage context that the caller will need to submit.
169 */
iomap_add_to_ioend(struct iomap_writepage_ctx * wpc,struct folio * folio,loff_t pos,loff_t end_pos,unsigned int dirty_len)170 ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio,
171 loff_t pos, loff_t end_pos, unsigned int dirty_len)
172 {
173 struct iomap_ioend *ioend = wpc->wb_ctx;
174 size_t poff = offset_in_folio(folio, pos);
175 unsigned int ioend_flags = iomap_ioend_flags(&wpc->iomap);
176 unsigned int map_len = min_t(u64, dirty_len,
177 wpc->iomap.offset + wpc->iomap.length - pos);
178 int error;
179
180 trace_iomap_add_to_ioend(wpc->inode, pos, dirty_len, &wpc->iomap);
181
182 WARN_ON_ONCE(!folio->private && map_len < dirty_len);
183
184 switch (wpc->iomap.type) {
185 case IOMAP_HOLE:
186 return map_len;
187 case IOMAP_UNWRITTEN:
188 case IOMAP_MAPPED:
189 break;
190 default:
191 WARN_ON_ONCE(1);
192 return -EIO;
193 }
194
195 if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY))
196 ioend_flags |= IOMAP_IOEND_BOUNDARY;
197
198 if (!ioend || !iomap_can_add_to_ioend(wpc, pos, map_len, ioend_flags)) {
199 new_ioend:
200 if (ioend) {
201 error = wpc->ops->writeback_submit(wpc, 0);
202 if (error) {
203 /*
204 * ->writeback_submit() completed the ioend
205 * with an error, so drop the stale context;
206 * iomap_writepages() would otherwise submit
207 * it a second time.
208 */
209 wpc->wb_ctx = NULL;
210 return error;
211 }
212 }
213 wpc->wb_ctx = ioend = iomap_alloc_ioend(wpc, pos, ioend_flags);
214 }
215
216 if (!bio_add_folio(&ioend->io_bio, folio, map_len, poff))
217 goto new_ioend;
218
219 if (folio_test_dropbehind(folio))
220 bio_set_flag(&ioend->io_bio, BIO_COMPLETE_IN_TASK);
221
222 /*
223 * Clamp io_offset and io_size to the incore EOF so that ondisk
224 * file size updates in the ioend completion are byte-accurate.
225 * This avoids recovering files with zeroed tail regions when
226 * writeback races with appending writes:
227 *
228 * Thread 1: Thread 2:
229 * ------------ -----------
230 * write [A, A+B]
231 * update inode size to A+B
232 * submit I/O [A, A+BS]
233 * write [A+B, A+B+C]
234 * update inode size to A+B+C
235 * <I/O completes, updates disk size to min(A+B+C, A+BS)>
236 * <power failure>
237 *
238 * After reboot:
239 * 1) with A+B+C < A+BS, the file has zero padding in range
240 * [A+B, A+B+C]
241 *
242 * |< Block Size (BS) >|
243 * |DDDDDDDDDDDD0000000000000|
244 * ^ ^ ^
245 * A A+B A+B+C
246 * (EOF)
247 *
248 * 2) with A+B+C > A+BS, the file has zero padding in range
249 * [A+B, A+BS]
250 *
251 * |< Block Size (BS) >|< Block Size (BS) >|
252 * |DDDDDDDDDDDD0000000000000|00000000000000000000000000|
253 * ^ ^ ^ ^
254 * A A+B A+BS A+B+C
255 * (EOF)
256 *
257 * D = Valid Data
258 * 0 = Zero Padding
259 *
260 * Note that this defeats the ability to chain the ioends of
261 * appending writes.
262 */
263 ioend->io_size += map_len;
264 if (ioend->io_offset + ioend->io_size > end_pos) {
265 if (ioend->io_offset >= end_pos)
266 ioend->io_size = 0;
267 else
268 ioend->io_size = end_pos - ioend->io_offset;
269 }
270
271 wbc_account_cgroup_owner(wpc->wbc, folio, map_len);
272 return map_len;
273 }
274 EXPORT_SYMBOL_GPL(iomap_add_to_ioend);
275
276 #ifdef CONFIG_BLK_DEV_INTEGRITY
iomap_ioend_integrity_verify(struct iomap_ioend * ioend)277 int iomap_ioend_integrity_verify(struct iomap_ioend *ioend)
278 {
279 struct bvec_iter data_iter = BVEC_ITER_IOEND(ioend);
280
281 return fs_bio_integrity_verify(&ioend->io_bio, &data_iter);
282 }
283 EXPORT_SYMBOL_GPL(iomap_ioend_integrity_verify);
284 #endif /* CONFIG_BLK_DEV_INTEGRITY */
285
iomap_finish_ioend(struct iomap_ioend * ioend,int error)286 static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error)
287 {
288 if (ioend->io_parent) {
289 struct bio *bio = &ioend->io_bio;
290
291 ioend = ioend->io_parent;
292 bio_put(bio);
293 }
294
295 if (error)
296 cmpxchg(&ioend->io_error, 0, error);
297
298 if (!atomic_dec_and_test(&ioend->io_remaining))
299 return 0;
300
301 if (ioend->io_flags & IOMAP_IOEND_DIRECT)
302 return iomap_finish_ioend_direct(ioend);
303 if (bio_op(&ioend->io_bio) == REQ_OP_READ)
304 return iomap_finish_ioend_buffered_read(ioend);
305 return iomap_finish_ioend_buffered_write(ioend);
306 }
307
308 /*
309 * Ioend completion routine for merged bios. This can only be called from task
310 * contexts as merged ioends can be of unbound length. Hence we have to break up
311 * the writeback completions into manageable chunks to avoid long scheduler
312 * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get
313 * good batch processing throughput without creating adverse scheduler latency
314 * conditions.
315 */
iomap_finish_ioends(struct iomap_ioend * ioend,int error)316 void iomap_finish_ioends(struct iomap_ioend *ioend, int error)
317 {
318 struct list_head tmp;
319 u32 completions;
320
321 might_sleep();
322
323 list_replace_init(&ioend->io_list, &tmp);
324 completions = iomap_finish_ioend(ioend, error);
325
326 while (!list_empty(&tmp)) {
327 if (completions > IOEND_BATCH_SIZE * 8) {
328 cond_resched();
329 completions = 0;
330 }
331 ioend = list_first_entry(&tmp, struct iomap_ioend, io_list);
332 list_del_init(&ioend->io_list);
333 completions += iomap_finish_ioend(ioend, error);
334 }
335 }
336 EXPORT_SYMBOL_GPL(iomap_finish_ioends);
337
338 /*
339 * We can merge two adjacent ioends if they have the same set of work to do.
340 */
iomap_ioend_can_merge(struct iomap_ioend * ioend,struct iomap_ioend * next)341 static bool iomap_ioend_can_merge(struct iomap_ioend *ioend,
342 struct iomap_ioend *next)
343 {
344 /*
345 * There is no point in merging reads as there is no completion
346 * processing that can be easily batched up for them.
347 */
348 if (bio_op(&ioend->io_bio) == REQ_OP_READ ||
349 bio_op(&next->io_bio) == REQ_OP_READ)
350 return false;
351
352 if (ioend->io_bio.bi_status != next->io_bio.bi_status)
353 return false;
354 if (ioend->io_private != next->io_private)
355 return false;
356 if (next->io_flags & IOMAP_IOEND_BOUNDARY)
357 return false;
358 if ((ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
359 (next->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
360 return false;
361 if (ioend->io_offset + ioend->io_size != next->io_offset)
362 return false;
363 /*
364 * Do not merge physically discontiguous ioends. The filesystem
365 * completion functions will have to iterate the physical
366 * discontiguities even if we merge the ioends at a logical level, so
367 * we don't gain anything by merging physical discontiguities here.
368 *
369 * We cannot use bio->bi_iter.bi_sector here as it is modified during
370 * submission so does not point to the start sector of the bio at
371 * completion.
372 */
373 if (ioend->io_sector + (ioend->io_size >> SECTOR_SHIFT) !=
374 next->io_sector)
375 return false;
376 return true;
377 }
378
iomap_ioend_try_merge(struct iomap_ioend * ioend,struct list_head * more_ioends)379 void iomap_ioend_try_merge(struct iomap_ioend *ioend,
380 struct list_head *more_ioends)
381 {
382 struct iomap_ioend *next;
383
384 INIT_LIST_HEAD(&ioend->io_list);
385
386 while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend,
387 io_list))) {
388 if (!iomap_ioend_can_merge(ioend, next))
389 break;
390 list_move_tail(&next->io_list, &ioend->io_list);
391 ioend->io_size += next->io_size;
392 }
393 }
394 EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
395
iomap_ioend_compare(void * priv,const struct list_head * a,const struct list_head * b)396 static int iomap_ioend_compare(void *priv, const struct list_head *a,
397 const struct list_head *b)
398 {
399 struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list);
400 struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list);
401
402 if (ia->io_offset < ib->io_offset)
403 return -1;
404 if (ia->io_offset > ib->io_offset)
405 return 1;
406 return 0;
407 }
408
iomap_sort_ioends(struct list_head * ioend_list)409 void iomap_sort_ioends(struct list_head *ioend_list)
410 {
411 list_sort(NULL, ioend_list, iomap_ioend_compare);
412 }
413 EXPORT_SYMBOL_GPL(iomap_sort_ioends);
414
415 /*
416 * Split up to the first @max_len bytes from @ioend if the ioend covers more
417 * than @max_len bytes.
418 *
419 * If @is_append is set, the split will be based on the hardware limits for
420 * REQ_OP_ZONE_APPEND commands and can be less than @max_len if the hardware
421 * limits don't allow the entire @max_len length.
422 *
423 * The bio embedded into @ioend must be a REQ_OP_WRITE because the block layer
424 * does not allow splitting REQ_OP_ZONE_APPEND bios. The file systems has to
425 * switch the operation after this call, but before submitting the bio.
426 */
iomap_split_ioend(struct iomap_ioend * ioend,unsigned int max_len,bool is_append)427 struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend,
428 unsigned int max_len, bool is_append)
429 {
430 struct bio *bio = &ioend->io_bio;
431 struct iomap_ioend *split_ioend;
432 unsigned int nr_segs;
433 int sector_offset;
434 struct bio *split;
435
436 if (is_append) {
437 struct queue_limits *lim = bdev_limits(bio->bi_bdev);
438
439 max_len = min(max_len,
440 lim->max_zone_append_sectors << SECTOR_SHIFT);
441
442 sector_offset = bio_split_rw_at(bio, lim, &nr_segs, max_len);
443 if (unlikely(sector_offset < 0))
444 return ERR_PTR(sector_offset);
445 if (!sector_offset)
446 return NULL;
447 } else {
448 if (bio->bi_iter.bi_size <= max_len)
449 return NULL;
450 sector_offset = max_len >> SECTOR_SHIFT;
451 }
452
453 /* ensure the split ioend is still block size aligned */
454 sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT,
455 i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT;
456
457 split = bio_split(bio, sector_offset, GFP_NOFS,
458 &iomap_ioend_split_bioset);
459 if (IS_ERR(split))
460 return ERR_CAST(split);
461 split->bi_private = bio->bi_private;
462 split->bi_end_io = bio->bi_end_io;
463
464 split_ioend = iomap_init_ioend(ioend->io_inode, split, ioend->io_offset,
465 ioend->io_flags);
466 split_ioend->io_parent = ioend;
467
468 atomic_inc(&ioend->io_remaining);
469 ioend->io_offset += split_ioend->io_size;
470 ioend->io_size -= split_ioend->io_size;
471
472 split_ioend->io_sector = ioend->io_sector;
473 if (!is_append)
474 ioend->io_sector += (split_ioend->io_size >> SECTOR_SHIFT);
475 return split_ioend;
476 }
477 EXPORT_SYMBOL_GPL(iomap_split_ioend);
478
iomap_bounce_read(struct iomap_ioend * orig_ioend,unsigned int minsize,void (* submit_ioend)(struct iomap_ioend * ioend))479 void iomap_bounce_read(struct iomap_ioend *orig_ioend, unsigned int minsize,
480 void (*submit_ioend)(struct iomap_ioend *ioend))
481 {
482 struct inode *inode = orig_ioend->io_inode;
483 struct bio *orig_bio = &orig_ioend->io_bio;
484 loff_t file_offset = orig_ioend->io_offset;
485 sector_t sector = orig_ioend->io_sector;
486 size_t total_len = round_up(orig_ioend->io_size, minsize);
487
488 WARN_ON_ONCE(!(orig_ioend->io_flags & IOMAP_IOEND_DIRECT));
489
490 /* We can't poll a bio that is not passed on to hardware */
491 orig_bio->bi_opf &= ~REQ_POLLED;
492
493 do {
494 struct iomap_ioend *ioend;
495 struct bio *bio;
496 int error;
497
498 bio = bio_alloc_bioset(orig_bio->bi_bdev,
499 min(total_len / minsize, BIO_MAX_VECS),
500 orig_bio->bi_opf, GFP_KERNEL,
501 &iomap_ioend_split_bioset);
502 error = bio_alloc_bounce_folios(bio, total_len, minsize);
503 if (error) {
504 bio_put(bio);
505 orig_bio->bi_status = errno_to_blk_status(error);
506 break;
507 }
508 bio->bi_ioprio = orig_bio->bi_ioprio;
509 bio->bi_write_hint = orig_bio->bi_write_hint;
510 bio->bi_write_stream = orig_bio->bi_write_stream;
511 bio->bi_iter.bi_sector = sector;
512
513 ioend = iomap_init_ioend(inode, bio, file_offset,
514 orig_ioend->io_flags);
515
516 total_len -= bio->bi_iter.bi_size;
517 file_offset += bio->bi_iter.bi_size;
518 sector += (bio->bi_iter.bi_size >> SECTOR_SHIFT);
519
520 bio->bi_private = orig_bio;
521 bio_inc_remaining(orig_bio);
522 submit_ioend(ioend);
523 } while (total_len > 0);
524
525 bio_endio(&orig_ioend->io_bio);
526 }
527 EXPORT_SYMBOL_GPL(iomap_bounce_read);
528
iomap_ioend_unbounce(struct iomap_ioend * orig_ioend,struct iomap_ioend * ioend)529 static void iomap_ioend_unbounce(struct iomap_ioend *orig_ioend,
530 struct iomap_ioend *ioend)
531 {
532 struct bio *orig_bio = &orig_ioend->io_bio;
533 struct iov_iter to;
534 struct bio_vec *bv;
535 int i;
536
537 iov_iter_bvec(&to, ITER_DEST, orig_bio->bi_io_vec, orig_bio->bi_vcnt,
538 orig_ioend->io_size);
539 to.iov_offset = orig_ioend->io_bvec_offset;
540
541 if (ioend->io_offset != orig_ioend->io_offset) {
542 WARN_ON_ONCE(ioend->io_offset < orig_ioend->io_offset);
543 iov_iter_advance(&to, ioend->io_offset - orig_ioend->io_offset);
544 }
545
546 /* copying to pinned pages should always work */
547 bio_for_each_bvec_all(bv, &ioend->io_bio, i)
548 WARN_ON_ONCE(copy_to_iter(bvec_virt(bv), bv->bv_len, &to) !=
549 bv->bv_len);
550 }
551
iomap_bounce_read_end_io(struct iomap_ioend * ioend,struct bio * orig_bio,int error)552 void iomap_bounce_read_end_io(struct iomap_ioend *ioend, struct bio *orig_bio,
553 int error)
554 {
555 if (error)
556 orig_bio->bi_status = errno_to_blk_status(error);
557 else
558 iomap_ioend_unbounce(iomap_ioend_from_bio(orig_bio), ioend);
559
560 bio_free_folios(&ioend->io_bio);
561 if (bio_integrity(&ioend->io_bio))
562 fs_bio_integrity_free(&ioend->io_bio);
563 bio_put(&ioend->io_bio);
564
565 bio_endio(orig_bio);
566 }
567 EXPORT_SYMBOL_GPL(iomap_bounce_read_end_io);
568
iomap_ioend_init(void)569 static int __init iomap_ioend_init(void)
570 {
571 const unsigned int nr_mempool_entries = 4 * (PAGE_SIZE / SECTOR_SIZE);
572 int error;
573
574 error = bioset_init(&iomap_ioend_bioset, nr_mempool_entries,
575 offsetof(struct iomap_ioend, io_bio),
576 BIOSET_NEED_BVECS);
577 if (error)
578 return error;
579 error = bioset_init(&iomap_ioend_split_bioset, nr_mempool_entries,
580 offsetof(struct iomap_ioend, io_bio),
581 BIOSET_NEED_BVECS);
582 if (error)
583 goto out_exit_ioend_bioset;
584 return 0;
585
586 out_exit_ioend_bioset:
587 bioset_exit(&iomap_ioend_bioset);
588 return error;
589 }
590 fs_initcall(iomap_ioend_init);
591