1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2016-2025 Christoph Hellwig.
4 */
5 #include <linux/bio-integrity.h>
6 #include <linux/iomap.h>
7 #include <linux/list_sort.h>
8 #include <linux/pagemap.h>
9 #include <linux/writeback.h>
10 #include <linux/fserror.h>
11 #include "internal.h"
12 #include "trace.h"
13
14 struct bio_set iomap_ioend_bioset;
15 EXPORT_SYMBOL_GPL(iomap_ioend_bioset);
16 static struct bio_set iomap_ioend_split_bioset;
17
iomap_init_ioend(struct inode * inode,struct bio * bio,loff_t file_offset,u16 ioend_flags)18 struct iomap_ioend *iomap_init_ioend(struct inode *inode,
19 struct bio *bio, loff_t file_offset, u16 ioend_flags)
20 {
21 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
22
23 atomic_set(&ioend->io_remaining, 1);
24 ioend->io_error = 0;
25 ioend->io_parent = NULL;
26 INIT_LIST_HEAD(&ioend->io_list);
27 ioend->io_flags = ioend_flags;
28 ioend->io_inode = inode;
29 ioend->io_offset = file_offset;
30 ioend->io_size = bio->bi_iter.bi_size;
31 ioend->io_sector = bio->bi_iter.bi_sector;
32 ioend->io_vi = NULL;
33 ioend->io_private = NULL;
34 return ioend;
35 }
36 EXPORT_SYMBOL_GPL(iomap_init_ioend);
37
38 /*
39 * We're now finished for good with this ioend structure. Update the folio
40 * state, release holds on bios, and finally free up memory. Do not use the
41 * ioend after this.
42 */
iomap_finish_ioend_buffered_write(struct iomap_ioend * ioend)43 static u32 iomap_finish_ioend_buffered_write(struct iomap_ioend *ioend)
44 {
45 struct inode *inode = ioend->io_inode;
46 struct bio *bio = &ioend->io_bio;
47 struct folio_iter fi;
48 u32 folio_count = 0;
49
50 if (ioend->io_error) {
51 mapping_set_error(inode->i_mapping, ioend->io_error);
52 if (!bio_flagged(bio, BIO_QUIET)) {
53 pr_err_ratelimited(
54 "%s: writeback error on inode %llu, offset %lld, sector %llu",
55 inode->i_sb->s_id, inode->i_ino,
56 ioend->io_offset, ioend->io_sector);
57 }
58 }
59
60 /* walk all folios in bio, ending page IO on them */
61 bio_for_each_folio_all(fi, bio) {
62 if (ioend->io_error)
63 fserror_report_io(inode, FSERR_BUFFERED_WRITE,
64 folio_pos(fi.folio) + fi.offset,
65 fi.length, ioend->io_error,
66 GFP_ATOMIC);
67 iomap_finish_folio_write(inode, fi.folio, fi.length);
68 folio_count++;
69 }
70
71 if (bio_integrity(bio))
72 fs_bio_integrity_free(bio);
73 bio_put(bio); /* frees the ioend */
74 return folio_count;
75 }
76
77 static DEFINE_SPINLOCK(failed_ioend_lock);
78 static LIST_HEAD(failed_ioend_list);
79
80 static void
iomap_fail_ioends(struct work_struct * work)81 iomap_fail_ioends(
82 struct work_struct *work)
83 {
84 struct iomap_ioend *ioend;
85 struct list_head tmp;
86 unsigned long flags;
87
88 spin_lock_irqsave(&failed_ioend_lock, flags);
89 list_replace_init(&failed_ioend_list, &tmp);
90 spin_unlock_irqrestore(&failed_ioend_lock, flags);
91
92 while ((ioend = list_first_entry_or_null(&tmp, struct iomap_ioend,
93 io_list))) {
94 list_del_init(&ioend->io_list);
95 iomap_finish_ioend_buffered_write(ioend);
96 cond_resched();
97 }
98 }
99
100 static DECLARE_WORK(failed_ioend_work, iomap_fail_ioends);
101
iomap_fail_ioend_buffered(struct iomap_ioend * ioend)102 static void iomap_fail_ioend_buffered(struct iomap_ioend *ioend)
103 {
104 unsigned long flags;
105
106 /*
107 * Bounce I/O errors to a workqueue to avoid nested i_lock acquisitions
108 * in the fserror code. The caller no longer owns the ioend reference
109 * after the spinlock drops.
110 */
111 spin_lock_irqsave(&failed_ioend_lock, flags);
112 if (list_empty(&failed_ioend_list))
113 WARN_ON_ONCE(!schedule_work(&failed_ioend_work));
114 list_add_tail(&ioend->io_list, &failed_ioend_list);
115 spin_unlock_irqrestore(&failed_ioend_lock, flags);
116 }
117
ioend_writeback_end_bio(struct bio * bio)118 static void ioend_writeback_end_bio(struct bio *bio)
119 {
120 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
121
122 ioend->io_error = blk_status_to_errno(bio->bi_status);
123 if (ioend->io_error) {
124 iomap_fail_ioend_buffered(ioend);
125 return;
126 }
127
128 iomap_finish_ioend_buffered_write(ioend);
129 }
130
131 /*
132 * We cannot cancel the ioend directly in case of an error, so call the bio end
133 * I/O handler with the error status here to run the normal I/O completion
134 * handler.
135 */
iomap_ioend_writeback_submit(struct iomap_writepage_ctx * wpc,int error)136 int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error)
137 {
138 struct iomap_ioend *ioend = wpc->wb_ctx;
139
140 if (!ioend->io_bio.bi_end_io)
141 ioend->io_bio.bi_end_io = ioend_writeback_end_bio;
142
143 if (WARN_ON_ONCE(wpc->iomap.flags & IOMAP_F_ANON_WRITE))
144 error = -EIO;
145
146 if (error) {
147 ioend->io_bio.bi_status = errno_to_blk_status(error);
148 bio_endio(&ioend->io_bio);
149 return error;
150 }
151
152 if (wpc->iomap.flags & IOMAP_F_INTEGRITY)
153 fs_bio_integrity_generate(&ioend->io_bio);
154 submit_bio(&ioend->io_bio);
155 return 0;
156 }
157 EXPORT_SYMBOL_GPL(iomap_ioend_writeback_submit);
158
iomap_alloc_ioend(struct iomap_writepage_ctx * wpc,loff_t pos,u16 ioend_flags)159 static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc,
160 loff_t pos, u16 ioend_flags)
161 {
162 struct bio *bio;
163
164 bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS,
165 REQ_OP_WRITE | wbc_to_write_flags(wpc->wbc),
166 GFP_NOFS, &iomap_ioend_bioset);
167 bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos);
168 bio->bi_write_hint = wpc->inode->i_write_hint;
169 wbc_init_bio(wpc->wbc, bio);
170 wpc->nr_folios = 0;
171 return iomap_init_ioend(wpc->inode, bio, pos, ioend_flags);
172 }
173
iomap_can_add_to_ioend(struct iomap_writepage_ctx * wpc,loff_t pos,unsigned int map_len,u16 ioend_flags)174 static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos,
175 unsigned int map_len, u16 ioend_flags)
176 {
177 struct iomap_ioend *ioend = wpc->wb_ctx;
178
179 if (ioend->io_bio.bi_iter.bi_size >
180 iomap_max_bio_size(&wpc->iomap) - map_len)
181 return false;
182 if (ioend_flags & IOMAP_IOEND_BOUNDARY)
183 return false;
184 if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
185 (ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
186 return false;
187 if (pos != ioend->io_offset + ioend->io_size)
188 return false;
189 if (!(wpc->iomap.flags & IOMAP_F_ANON_WRITE) &&
190 iomap_sector(&wpc->iomap, pos) != bio_end_sector(&ioend->io_bio))
191 return false;
192 /*
193 * Limit ioend bio chain lengths to minimise IO completion latency. This
194 * also prevents long tight loops ending page writeback on all the
195 * folios in the ioend.
196 */
197 if (wpc->nr_folios >= IOEND_BATCH_SIZE)
198 return false;
199 return true;
200 }
201
202 /*
203 * Test to see if we have an existing ioend structure that we could append to
204 * first; otherwise finish off the current ioend and start another.
205 *
206 * If a new ioend is created and cached, the old ioend is submitted to the block
207 * layer instantly. Batching optimisations are provided by higher level block
208 * plugging.
209 *
210 * At the end of a writeback pass, there will be a cached ioend remaining on the
211 * writepage context that the caller will need to submit.
212 */
iomap_add_to_ioend(struct iomap_writepage_ctx * wpc,struct folio * folio,loff_t pos,loff_t end_pos,unsigned int dirty_len)213 ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio,
214 loff_t pos, loff_t end_pos, unsigned int dirty_len)
215 {
216 struct iomap_ioend *ioend = wpc->wb_ctx;
217 size_t poff = offset_in_folio(folio, pos);
218 unsigned int ioend_flags = 0;
219 unsigned int map_len = min_t(u64, dirty_len,
220 wpc->iomap.offset + wpc->iomap.length - pos);
221 int error;
222
223 trace_iomap_add_to_ioend(wpc->inode, pos, dirty_len, &wpc->iomap);
224
225 WARN_ON_ONCE(!folio->private && map_len < dirty_len);
226
227 switch (wpc->iomap.type) {
228 case IOMAP_UNWRITTEN:
229 ioend_flags |= IOMAP_IOEND_UNWRITTEN;
230 break;
231 case IOMAP_MAPPED:
232 break;
233 case IOMAP_HOLE:
234 return map_len;
235 default:
236 WARN_ON_ONCE(1);
237 return -EIO;
238 }
239
240 if (wpc->iomap.flags & IOMAP_F_SHARED)
241 ioend_flags |= IOMAP_IOEND_SHARED;
242 if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY))
243 ioend_flags |= IOMAP_IOEND_BOUNDARY;
244
245 if (!ioend || !iomap_can_add_to_ioend(wpc, pos, map_len, ioend_flags)) {
246 new_ioend:
247 if (ioend) {
248 error = wpc->ops->writeback_submit(wpc, 0);
249 if (error)
250 return error;
251 }
252 wpc->wb_ctx = ioend = iomap_alloc_ioend(wpc, pos, ioend_flags);
253 }
254
255 if (!bio_add_folio(&ioend->io_bio, folio, map_len, poff))
256 goto new_ioend;
257
258 if (folio_test_dropbehind(folio))
259 bio_set_flag(&ioend->io_bio, BIO_COMPLETE_IN_TASK);
260
261 /*
262 * Clamp io_offset and io_size to the incore EOF so that ondisk
263 * file size updates in the ioend completion are byte-accurate.
264 * This avoids recovering files with zeroed tail regions when
265 * writeback races with appending writes:
266 *
267 * Thread 1: Thread 2:
268 * ------------ -----------
269 * write [A, A+B]
270 * update inode size to A+B
271 * submit I/O [A, A+BS]
272 * write [A+B, A+B+C]
273 * update inode size to A+B+C
274 * <I/O completes, updates disk size to min(A+B+C, A+BS)>
275 * <power failure>
276 *
277 * After reboot:
278 * 1) with A+B+C < A+BS, the file has zero padding in range
279 * [A+B, A+B+C]
280 *
281 * |< Block Size (BS) >|
282 * |DDDDDDDDDDDD0000000000000|
283 * ^ ^ ^
284 * A A+B A+B+C
285 * (EOF)
286 *
287 * 2) with A+B+C > A+BS, the file has zero padding in range
288 * [A+B, A+BS]
289 *
290 * |< Block Size (BS) >|< Block Size (BS) >|
291 * |DDDDDDDDDDDD0000000000000|00000000000000000000000000|
292 * ^ ^ ^ ^
293 * A A+B A+BS A+B+C
294 * (EOF)
295 *
296 * D = Valid Data
297 * 0 = Zero Padding
298 *
299 * Note that this defeats the ability to chain the ioends of
300 * appending writes.
301 */
302 ioend->io_size += map_len;
303 if (ioend->io_offset + ioend->io_size > end_pos) {
304 if (ioend->io_offset >= end_pos)
305 ioend->io_size = 0;
306 else
307 ioend->io_size = end_pos - ioend->io_offset;
308 }
309
310 wbc_account_cgroup_owner(wpc->wbc, folio, map_len);
311 return map_len;
312 }
313 EXPORT_SYMBOL_GPL(iomap_add_to_ioend);
314
iomap_finish_ioend(struct iomap_ioend * ioend,int error)315 static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error)
316 {
317 if (ioend->io_parent) {
318 struct bio *bio = &ioend->io_bio;
319
320 ioend = ioend->io_parent;
321 bio_put(bio);
322 }
323
324 if (error)
325 cmpxchg(&ioend->io_error, 0, error);
326
327 if (!atomic_dec_and_test(&ioend->io_remaining))
328 return 0;
329
330 if (!ioend->io_error &&
331 bio_integrity(&ioend->io_bio) &&
332 bio_op(&ioend->io_bio) == REQ_OP_READ) {
333 ioend->io_error = fs_bio_integrity_verify(&ioend->io_bio,
334 ioend->io_sector, ioend->io_size);
335 }
336
337 if (ioend->io_flags & IOMAP_IOEND_DIRECT)
338 return iomap_finish_ioend_direct(ioend);
339 if (bio_op(&ioend->io_bio) == REQ_OP_READ)
340 return iomap_finish_ioend_buffered_read(ioend);
341 return iomap_finish_ioend_buffered_write(ioend);
342 }
343
344 /*
345 * Ioend completion routine for merged bios. This can only be called from task
346 * contexts as merged ioends can be of unbound length. Hence we have to break up
347 * the writeback completions into manageable chunks to avoid long scheduler
348 * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get
349 * good batch processing throughput without creating adverse scheduler latency
350 * conditions.
351 */
iomap_finish_ioends(struct iomap_ioend * ioend,int error)352 void iomap_finish_ioends(struct iomap_ioend *ioend, int error)
353 {
354 struct list_head tmp;
355 u32 completions;
356
357 might_sleep();
358
359 list_replace_init(&ioend->io_list, &tmp);
360 completions = iomap_finish_ioend(ioend, error);
361
362 while (!list_empty(&tmp)) {
363 if (completions > IOEND_BATCH_SIZE * 8) {
364 cond_resched();
365 completions = 0;
366 }
367 ioend = list_first_entry(&tmp, struct iomap_ioend, io_list);
368 list_del_init(&ioend->io_list);
369 completions += iomap_finish_ioend(ioend, error);
370 }
371 }
372 EXPORT_SYMBOL_GPL(iomap_finish_ioends);
373
374 /*
375 * We can merge two adjacent ioends if they have the same set of work to do.
376 */
iomap_ioend_can_merge(struct iomap_ioend * ioend,struct iomap_ioend * next)377 static bool iomap_ioend_can_merge(struct iomap_ioend *ioend,
378 struct iomap_ioend *next)
379 {
380 /*
381 * There is no point in merging reads as there is no completion
382 * processing that can be easily batched up for them.
383 */
384 if (bio_op(&ioend->io_bio) == REQ_OP_READ ||
385 bio_op(&next->io_bio) == REQ_OP_READ)
386 return false;
387
388 if (ioend->io_bio.bi_status != next->io_bio.bi_status)
389 return false;
390 if (ioend->io_private != next->io_private)
391 return false;
392 if (next->io_flags & IOMAP_IOEND_BOUNDARY)
393 return false;
394 if ((ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
395 (next->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
396 return false;
397 if (ioend->io_offset + ioend->io_size != next->io_offset)
398 return false;
399 /*
400 * Do not merge physically discontiguous ioends. The filesystem
401 * completion functions will have to iterate the physical
402 * discontiguities even if we merge the ioends at a logical level, so
403 * we don't gain anything by merging physical discontiguities here.
404 *
405 * We cannot use bio->bi_iter.bi_sector here as it is modified during
406 * submission so does not point to the start sector of the bio at
407 * completion.
408 */
409 if (ioend->io_sector + (ioend->io_size >> SECTOR_SHIFT) !=
410 next->io_sector)
411 return false;
412 return true;
413 }
414
iomap_ioend_try_merge(struct iomap_ioend * ioend,struct list_head * more_ioends)415 void iomap_ioend_try_merge(struct iomap_ioend *ioend,
416 struct list_head *more_ioends)
417 {
418 struct iomap_ioend *next;
419
420 INIT_LIST_HEAD(&ioend->io_list);
421
422 while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend,
423 io_list))) {
424 if (!iomap_ioend_can_merge(ioend, next))
425 break;
426 list_move_tail(&next->io_list, &ioend->io_list);
427 ioend->io_size += next->io_size;
428 }
429 }
430 EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
431
iomap_ioend_compare(void * priv,const struct list_head * a,const struct list_head * b)432 static int iomap_ioend_compare(void *priv, const struct list_head *a,
433 const struct list_head *b)
434 {
435 struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list);
436 struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list);
437
438 if (ia->io_offset < ib->io_offset)
439 return -1;
440 if (ia->io_offset > ib->io_offset)
441 return 1;
442 return 0;
443 }
444
iomap_sort_ioends(struct list_head * ioend_list)445 void iomap_sort_ioends(struct list_head *ioend_list)
446 {
447 list_sort(NULL, ioend_list, iomap_ioend_compare);
448 }
449 EXPORT_SYMBOL_GPL(iomap_sort_ioends);
450
451 /*
452 * Split up to the first @max_len bytes from @ioend if the ioend covers more
453 * than @max_len bytes.
454 *
455 * If @is_append is set, the split will be based on the hardware limits for
456 * REQ_OP_ZONE_APPEND commands and can be less than @max_len if the hardware
457 * limits don't allow the entire @max_len length.
458 *
459 * The bio embedded into @ioend must be a REQ_OP_WRITE because the block layer
460 * does not allow splitting REQ_OP_ZONE_APPEND bios. The file systems has to
461 * switch the operation after this call, but before submitting the bio.
462 */
iomap_split_ioend(struct iomap_ioend * ioend,unsigned int max_len,bool is_append)463 struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend,
464 unsigned int max_len, bool is_append)
465 {
466 struct bio *bio = &ioend->io_bio;
467 struct iomap_ioend *split_ioend;
468 unsigned int nr_segs;
469 int sector_offset;
470 struct bio *split;
471
472 if (is_append) {
473 struct queue_limits *lim = bdev_limits(bio->bi_bdev);
474
475 max_len = min(max_len,
476 lim->max_zone_append_sectors << SECTOR_SHIFT);
477
478 sector_offset = bio_split_rw_at(bio, lim, &nr_segs, max_len);
479 if (unlikely(sector_offset < 0))
480 return ERR_PTR(sector_offset);
481 if (!sector_offset)
482 return NULL;
483 } else {
484 if (bio->bi_iter.bi_size <= max_len)
485 return NULL;
486 sector_offset = max_len >> SECTOR_SHIFT;
487 }
488
489 /* ensure the split ioend is still block size aligned */
490 sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT,
491 i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT;
492
493 split = bio_split(bio, sector_offset, GFP_NOFS,
494 &iomap_ioend_split_bioset);
495 if (IS_ERR(split))
496 return ERR_CAST(split);
497 split->bi_private = bio->bi_private;
498 split->bi_end_io = bio->bi_end_io;
499
500 split_ioend = iomap_init_ioend(ioend->io_inode, split, ioend->io_offset,
501 ioend->io_flags);
502 split_ioend->io_parent = ioend;
503
504 atomic_inc(&ioend->io_remaining);
505 ioend->io_offset += split_ioend->io_size;
506 ioend->io_size -= split_ioend->io_size;
507
508 split_ioend->io_sector = ioend->io_sector;
509 if (!is_append)
510 ioend->io_sector += (split_ioend->io_size >> SECTOR_SHIFT);
511 return split_ioend;
512 }
513 EXPORT_SYMBOL_GPL(iomap_split_ioend);
514
iomap_ioend_init(void)515 static int __init iomap_ioend_init(void)
516 {
517 const unsigned int nr_mempool_entries = 4 * (PAGE_SIZE / SECTOR_SIZE);
518 int error;
519
520 error = bioset_init(&iomap_ioend_bioset, nr_mempool_entries,
521 offsetof(struct iomap_ioend, io_bio),
522 BIOSET_NEED_BVECS);
523 if (error)
524 return error;
525 error = bioset_init(&iomap_ioend_split_bioset, nr_mempool_entries,
526 offsetof(struct iomap_ioend, io_bio),
527 BIOSET_NEED_BVECS);
528 if (error)
529 goto out_exit_ioend_bioset;
530 return 0;
531
532 out_exit_ioend_bioset:
533 bioset_exit(&iomap_ioend_bioset);
534 return error;
535 }
536 fs_initcall(iomap_ioend_init);
537