1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2016-2025 Christoph Hellwig.
4 */
5 #include <linux/iomap.h>
6 #include <linux/list_sort.h>
7 #include <linux/pagemap.h>
8 #include <linux/writeback.h>
9 #include "internal.h"
10 #include "trace.h"
11
12 struct bio_set iomap_ioend_bioset;
13 EXPORT_SYMBOL_GPL(iomap_ioend_bioset);
14
iomap_init_ioend(struct inode * inode,struct bio * bio,loff_t file_offset,u16 ioend_flags)15 struct iomap_ioend *iomap_init_ioend(struct inode *inode,
16 struct bio *bio, loff_t file_offset, u16 ioend_flags)
17 {
18 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
19
20 atomic_set(&ioend->io_remaining, 1);
21 ioend->io_error = 0;
22 ioend->io_parent = NULL;
23 INIT_LIST_HEAD(&ioend->io_list);
24 ioend->io_flags = ioend_flags;
25 ioend->io_inode = inode;
26 ioend->io_offset = file_offset;
27 ioend->io_size = bio->bi_iter.bi_size;
28 ioend->io_sector = bio->bi_iter.bi_sector;
29 ioend->io_private = NULL;
30 return ioend;
31 }
32 EXPORT_SYMBOL_GPL(iomap_init_ioend);
33
34 /*
35 * We're now finished for good with this ioend structure. Update the folio
36 * state, release holds on bios, and finally free up memory. Do not use the
37 * ioend after this.
38 */
iomap_finish_ioend_buffered(struct iomap_ioend * ioend)39 static u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend)
40 {
41 struct inode *inode = ioend->io_inode;
42 struct bio *bio = &ioend->io_bio;
43 struct folio_iter fi;
44 u32 folio_count = 0;
45
46 if (ioend->io_error) {
47 mapping_set_error(inode->i_mapping, ioend->io_error);
48 if (!bio_flagged(bio, BIO_QUIET)) {
49 pr_err_ratelimited(
50 "%s: writeback error on inode %lu, offset %lld, sector %llu",
51 inode->i_sb->s_id, inode->i_ino,
52 ioend->io_offset, ioend->io_sector);
53 }
54 }
55
56 /* walk all folios in bio, ending page IO on them */
57 bio_for_each_folio_all(fi, bio) {
58 iomap_finish_folio_write(inode, fi.folio, fi.length);
59 folio_count++;
60 }
61
62 bio_put(bio); /* frees the ioend */
63 return folio_count;
64 }
65
ioend_writeback_end_bio(struct bio * bio)66 static void ioend_writeback_end_bio(struct bio *bio)
67 {
68 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio);
69
70 ioend->io_error = blk_status_to_errno(bio->bi_status);
71 iomap_finish_ioend_buffered(ioend);
72 }
73
74 /*
75 * We cannot cancel the ioend directly in case of an error, so call the bio end
76 * I/O handler with the error status here to run the normal I/O completion
77 * handler.
78 */
iomap_ioend_writeback_submit(struct iomap_writepage_ctx * wpc,int error)79 int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error)
80 {
81 struct iomap_ioend *ioend = wpc->wb_ctx;
82
83 if (!ioend->io_bio.bi_end_io)
84 ioend->io_bio.bi_end_io = ioend_writeback_end_bio;
85
86 if (WARN_ON_ONCE(wpc->iomap.flags & IOMAP_F_ANON_WRITE))
87 error = -EIO;
88
89 if (error) {
90 ioend->io_bio.bi_status = errno_to_blk_status(error);
91 bio_endio(&ioend->io_bio);
92 return error;
93 }
94
95 submit_bio(&ioend->io_bio);
96 return 0;
97 }
98 EXPORT_SYMBOL_GPL(iomap_ioend_writeback_submit);
99
iomap_alloc_ioend(struct iomap_writepage_ctx * wpc,loff_t pos,u16 ioend_flags)100 static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc,
101 loff_t pos, u16 ioend_flags)
102 {
103 struct bio *bio;
104
105 bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS,
106 REQ_OP_WRITE | wbc_to_write_flags(wpc->wbc),
107 GFP_NOFS, &iomap_ioend_bioset);
108 bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos);
109 bio->bi_write_hint = wpc->inode->i_write_hint;
110 wbc_init_bio(wpc->wbc, bio);
111 wpc->nr_folios = 0;
112 return iomap_init_ioend(wpc->inode, bio, pos, ioend_flags);
113 }
114
iomap_can_add_to_ioend(struct iomap_writepage_ctx * wpc,loff_t pos,u16 ioend_flags)115 static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos,
116 u16 ioend_flags)
117 {
118 struct iomap_ioend *ioend = wpc->wb_ctx;
119
120 if (ioend_flags & IOMAP_IOEND_BOUNDARY)
121 return false;
122 if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
123 (ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
124 return false;
125 if (pos != ioend->io_offset + ioend->io_size)
126 return false;
127 if (!(wpc->iomap.flags & IOMAP_F_ANON_WRITE) &&
128 iomap_sector(&wpc->iomap, pos) != bio_end_sector(&ioend->io_bio))
129 return false;
130 /*
131 * Limit ioend bio chain lengths to minimise IO completion latency. This
132 * also prevents long tight loops ending page writeback on all the
133 * folios in the ioend.
134 */
135 if (wpc->nr_folios >= IOEND_BATCH_SIZE)
136 return false;
137 return true;
138 }
139
140 /*
141 * Test to see if we have an existing ioend structure that we could append to
142 * first; otherwise finish off the current ioend and start another.
143 *
144 * If a new ioend is created and cached, the old ioend is submitted to the block
145 * layer instantly. Batching optimisations are provided by higher level block
146 * plugging.
147 *
148 * At the end of a writeback pass, there will be a cached ioend remaining on the
149 * writepage context that the caller will need to submit.
150 */
iomap_add_to_ioend(struct iomap_writepage_ctx * wpc,struct folio * folio,loff_t pos,loff_t end_pos,unsigned int dirty_len)151 ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio,
152 loff_t pos, loff_t end_pos, unsigned int dirty_len)
153 {
154 struct iomap_ioend *ioend = wpc->wb_ctx;
155 size_t poff = offset_in_folio(folio, pos);
156 unsigned int ioend_flags = 0;
157 unsigned int map_len = min_t(u64, dirty_len,
158 wpc->iomap.offset + wpc->iomap.length - pos);
159 int error;
160
161 trace_iomap_add_to_ioend(wpc->inode, pos, dirty_len, &wpc->iomap);
162
163 WARN_ON_ONCE(!folio->private && map_len < dirty_len);
164
165 switch (wpc->iomap.type) {
166 case IOMAP_INLINE:
167 WARN_ON_ONCE(1);
168 return -EIO;
169 case IOMAP_HOLE:
170 return map_len;
171 default:
172 break;
173 }
174
175 if (wpc->iomap.type == IOMAP_UNWRITTEN)
176 ioend_flags |= IOMAP_IOEND_UNWRITTEN;
177 if (wpc->iomap.flags & IOMAP_F_SHARED)
178 ioend_flags |= IOMAP_IOEND_SHARED;
179 if (folio_test_dropbehind(folio))
180 ioend_flags |= IOMAP_IOEND_DONTCACHE;
181 if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY))
182 ioend_flags |= IOMAP_IOEND_BOUNDARY;
183
184 if (!ioend || !iomap_can_add_to_ioend(wpc, pos, ioend_flags)) {
185 new_ioend:
186 if (ioend) {
187 error = wpc->ops->writeback_submit(wpc, 0);
188 if (error)
189 return error;
190 }
191 wpc->wb_ctx = ioend = iomap_alloc_ioend(wpc, pos, ioend_flags);
192 }
193
194 if (!bio_add_folio(&ioend->io_bio, folio, map_len, poff))
195 goto new_ioend;
196
197 /*
198 * Clamp io_offset and io_size to the incore EOF so that ondisk
199 * file size updates in the ioend completion are byte-accurate.
200 * This avoids recovering files with zeroed tail regions when
201 * writeback races with appending writes:
202 *
203 * Thread 1: Thread 2:
204 * ------------ -----------
205 * write [A, A+B]
206 * update inode size to A+B
207 * submit I/O [A, A+BS]
208 * write [A+B, A+B+C]
209 * update inode size to A+B+C
210 * <I/O completes, updates disk size to min(A+B+C, A+BS)>
211 * <power failure>
212 *
213 * After reboot:
214 * 1) with A+B+C < A+BS, the file has zero padding in range
215 * [A+B, A+B+C]
216 *
217 * |< Block Size (BS) >|
218 * |DDDDDDDDDDDD0000000000000|
219 * ^ ^ ^
220 * A A+B A+B+C
221 * (EOF)
222 *
223 * 2) with A+B+C > A+BS, the file has zero padding in range
224 * [A+B, A+BS]
225 *
226 * |< Block Size (BS) >|< Block Size (BS) >|
227 * |DDDDDDDDDDDD0000000000000|00000000000000000000000000|
228 * ^ ^ ^ ^
229 * A A+B A+BS A+B+C
230 * (EOF)
231 *
232 * D = Valid Data
233 * 0 = Zero Padding
234 *
235 * Note that this defeats the ability to chain the ioends of
236 * appending writes.
237 */
238 ioend->io_size += map_len;
239 if (ioend->io_offset + ioend->io_size > end_pos)
240 ioend->io_size = end_pos - ioend->io_offset;
241
242 wbc_account_cgroup_owner(wpc->wbc, folio, map_len);
243 return map_len;
244 }
245 EXPORT_SYMBOL_GPL(iomap_add_to_ioend);
246
iomap_finish_ioend(struct iomap_ioend * ioend,int error)247 static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error)
248 {
249 if (ioend->io_parent) {
250 struct bio *bio = &ioend->io_bio;
251
252 ioend = ioend->io_parent;
253 bio_put(bio);
254 }
255
256 if (error)
257 cmpxchg(&ioend->io_error, 0, error);
258
259 if (!atomic_dec_and_test(&ioend->io_remaining))
260 return 0;
261 if (ioend->io_flags & IOMAP_IOEND_DIRECT)
262 return iomap_finish_ioend_direct(ioend);
263 return iomap_finish_ioend_buffered(ioend);
264 }
265
266 /*
267 * Ioend completion routine for merged bios. This can only be called from task
268 * contexts as merged ioends can be of unbound length. Hence we have to break up
269 * the writeback completions into manageable chunks to avoid long scheduler
270 * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get
271 * good batch processing throughput without creating adverse scheduler latency
272 * conditions.
273 */
iomap_finish_ioends(struct iomap_ioend * ioend,int error)274 void iomap_finish_ioends(struct iomap_ioend *ioend, int error)
275 {
276 struct list_head tmp;
277 u32 completions;
278
279 might_sleep();
280
281 list_replace_init(&ioend->io_list, &tmp);
282 completions = iomap_finish_ioend(ioend, error);
283
284 while (!list_empty(&tmp)) {
285 if (completions > IOEND_BATCH_SIZE * 8) {
286 cond_resched();
287 completions = 0;
288 }
289 ioend = list_first_entry(&tmp, struct iomap_ioend, io_list);
290 list_del_init(&ioend->io_list);
291 completions += iomap_finish_ioend(ioend, error);
292 }
293 }
294 EXPORT_SYMBOL_GPL(iomap_finish_ioends);
295
296 /*
297 * We can merge two adjacent ioends if they have the same set of work to do.
298 */
iomap_ioend_can_merge(struct iomap_ioend * ioend,struct iomap_ioend * next)299 static bool iomap_ioend_can_merge(struct iomap_ioend *ioend,
300 struct iomap_ioend *next)
301 {
302 if (ioend->io_bio.bi_status != next->io_bio.bi_status)
303 return false;
304 if (next->io_flags & IOMAP_IOEND_BOUNDARY)
305 return false;
306 if ((ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS) !=
307 (next->io_flags & IOMAP_IOEND_NOMERGE_FLAGS))
308 return false;
309 if (ioend->io_offset + ioend->io_size != next->io_offset)
310 return false;
311 /*
312 * Do not merge physically discontiguous ioends. The filesystem
313 * completion functions will have to iterate the physical
314 * discontiguities even if we merge the ioends at a logical level, so
315 * we don't gain anything by merging physical discontiguities here.
316 *
317 * We cannot use bio->bi_iter.bi_sector here as it is modified during
318 * submission so does not point to the start sector of the bio at
319 * completion.
320 */
321 if (ioend->io_sector + (ioend->io_size >> SECTOR_SHIFT) !=
322 next->io_sector)
323 return false;
324 return true;
325 }
326
iomap_ioend_try_merge(struct iomap_ioend * ioend,struct list_head * more_ioends)327 void iomap_ioend_try_merge(struct iomap_ioend *ioend,
328 struct list_head *more_ioends)
329 {
330 struct iomap_ioend *next;
331
332 INIT_LIST_HEAD(&ioend->io_list);
333
334 while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend,
335 io_list))) {
336 if (!iomap_ioend_can_merge(ioend, next))
337 break;
338 list_move_tail(&next->io_list, &ioend->io_list);
339 ioend->io_size += next->io_size;
340 }
341 }
342 EXPORT_SYMBOL_GPL(iomap_ioend_try_merge);
343
iomap_ioend_compare(void * priv,const struct list_head * a,const struct list_head * b)344 static int iomap_ioend_compare(void *priv, const struct list_head *a,
345 const struct list_head *b)
346 {
347 struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list);
348 struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list);
349
350 if (ia->io_offset < ib->io_offset)
351 return -1;
352 if (ia->io_offset > ib->io_offset)
353 return 1;
354 return 0;
355 }
356
iomap_sort_ioends(struct list_head * ioend_list)357 void iomap_sort_ioends(struct list_head *ioend_list)
358 {
359 list_sort(NULL, ioend_list, iomap_ioend_compare);
360 }
361 EXPORT_SYMBOL_GPL(iomap_sort_ioends);
362
363 /*
364 * Split up to the first @max_len bytes from @ioend if the ioend covers more
365 * than @max_len bytes.
366 *
367 * If @is_append is set, the split will be based on the hardware limits for
368 * REQ_OP_ZONE_APPEND commands and can be less than @max_len if the hardware
369 * limits don't allow the entire @max_len length.
370 *
371 * The bio embedded into @ioend must be a REQ_OP_WRITE because the block layer
372 * does not allow splitting REQ_OP_ZONE_APPEND bios. The file systems has to
373 * switch the operation after this call, but before submitting the bio.
374 */
iomap_split_ioend(struct iomap_ioend * ioend,unsigned int max_len,bool is_append)375 struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend,
376 unsigned int max_len, bool is_append)
377 {
378 struct bio *bio = &ioend->io_bio;
379 struct iomap_ioend *split_ioend;
380 unsigned int nr_segs;
381 int sector_offset;
382 struct bio *split;
383
384 if (is_append) {
385 struct queue_limits *lim = bdev_limits(bio->bi_bdev);
386
387 max_len = min(max_len,
388 lim->max_zone_append_sectors << SECTOR_SHIFT);
389
390 sector_offset = bio_split_rw_at(bio, lim, &nr_segs, max_len);
391 if (unlikely(sector_offset < 0))
392 return ERR_PTR(sector_offset);
393 if (!sector_offset)
394 return NULL;
395 } else {
396 if (bio->bi_iter.bi_size <= max_len)
397 return NULL;
398 sector_offset = max_len >> SECTOR_SHIFT;
399 }
400
401 /* ensure the split ioend is still block size aligned */
402 sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT,
403 i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT;
404
405 split = bio_split(bio, sector_offset, GFP_NOFS, &iomap_ioend_bioset);
406 if (IS_ERR(split))
407 return ERR_CAST(split);
408 split->bi_private = bio->bi_private;
409 split->bi_end_io = bio->bi_end_io;
410
411 split_ioend = iomap_init_ioend(ioend->io_inode, split, ioend->io_offset,
412 ioend->io_flags);
413 split_ioend->io_parent = ioend;
414
415 atomic_inc(&ioend->io_remaining);
416 ioend->io_offset += split_ioend->io_size;
417 ioend->io_size -= split_ioend->io_size;
418
419 split_ioend->io_sector = ioend->io_sector;
420 if (!is_append)
421 ioend->io_sector += (split_ioend->io_size >> SECTOR_SHIFT);
422 return split_ioend;
423 }
424 EXPORT_SYMBOL_GPL(iomap_split_ioend);
425
iomap_ioend_init(void)426 static int __init iomap_ioend_init(void)
427 {
428 return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE),
429 offsetof(struct iomap_ioend, io_bio),
430 BIOSET_NEED_BVECS);
431 }
432 fs_initcall(iomap_ioend_init);
433