1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-2025 Christoph Hellwig. 4 */ 5 #include <linux/iomap.h> 6 #include <linux/list_sort.h> 7 #include <linux/pagemap.h> 8 #include <linux/writeback.h> 9 #include "internal.h" 10 #include "trace.h" 11 12 struct bio_set iomap_ioend_bioset; 13 EXPORT_SYMBOL_GPL(iomap_ioend_bioset); 14 15 struct iomap_ioend *iomap_init_ioend(struct inode *inode, 16 struct bio *bio, loff_t file_offset, u16 ioend_flags) 17 { 18 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); 19 20 atomic_set(&ioend->io_remaining, 1); 21 ioend->io_error = 0; 22 ioend->io_parent = NULL; 23 INIT_LIST_HEAD(&ioend->io_list); 24 ioend->io_flags = ioend_flags; 25 ioend->io_inode = inode; 26 ioend->io_offset = file_offset; 27 ioend->io_size = bio->bi_iter.bi_size; 28 ioend->io_sector = bio->bi_iter.bi_sector; 29 ioend->io_private = NULL; 30 return ioend; 31 } 32 EXPORT_SYMBOL_GPL(iomap_init_ioend); 33 34 /* 35 * We're now finished for good with this ioend structure. Update the folio 36 * state, release holds on bios, and finally free up memory. Do not use the 37 * ioend after this. 38 */ 39 static u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend) 40 { 41 struct inode *inode = ioend->io_inode; 42 struct bio *bio = &ioend->io_bio; 43 struct folio_iter fi; 44 u32 folio_count = 0; 45 46 if (ioend->io_error) { 47 mapping_set_error(inode->i_mapping, ioend->io_error); 48 if (!bio_flagged(bio, BIO_QUIET)) { 49 pr_err_ratelimited( 50 "%s: writeback error on inode %lu, offset %lld, sector %llu", 51 inode->i_sb->s_id, inode->i_ino, 52 ioend->io_offset, ioend->io_sector); 53 } 54 } 55 56 /* walk all folios in bio, ending page IO on them */ 57 bio_for_each_folio_all(fi, bio) { 58 iomap_finish_folio_write(inode, fi.folio, fi.length); 59 folio_count++; 60 } 61 62 bio_put(bio); /* frees the ioend */ 63 return folio_count; 64 } 65 66 static void ioend_writeback_end_bio(struct bio *bio) 67 { 68 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); 69 70 ioend->io_error = blk_status_to_errno(bio->bi_status); 71 iomap_finish_ioend_buffered(ioend); 72 } 73 74 /* 75 * We cannot cancel the ioend directly in case of an error, so call the bio end 76 * I/O handler with the error status here to run the normal I/O completion 77 * handler. 78 */ 79 int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error) 80 { 81 struct iomap_ioend *ioend = wpc->wb_ctx; 82 83 if (!ioend->io_bio.bi_end_io) 84 ioend->io_bio.bi_end_io = ioend_writeback_end_bio; 85 86 if (WARN_ON_ONCE(wpc->iomap.flags & IOMAP_F_ANON_WRITE)) 87 error = -EIO; 88 89 if (error) { 90 ioend->io_bio.bi_status = errno_to_blk_status(error); 91 bio_endio(&ioend->io_bio); 92 return error; 93 } 94 95 submit_bio(&ioend->io_bio); 96 return 0; 97 } 98 EXPORT_SYMBOL_GPL(iomap_ioend_writeback_submit); 99 100 static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc, 101 loff_t pos, u16 ioend_flags) 102 { 103 struct bio *bio; 104 105 bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS, 106 REQ_OP_WRITE | wbc_to_write_flags(wpc->wbc), 107 GFP_NOFS, &iomap_ioend_bioset); 108 bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos); 109 bio->bi_write_hint = wpc->inode->i_write_hint; 110 wbc_init_bio(wpc->wbc, bio); 111 wpc->nr_folios = 0; 112 return iomap_init_ioend(wpc->inode, bio, pos, ioend_flags); 113 } 114 115 static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos, 116 u16 ioend_flags) 117 { 118 struct iomap_ioend *ioend = wpc->wb_ctx; 119 120 if (ioend_flags & IOMAP_IOEND_BOUNDARY) 121 return false; 122 if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) != 123 (ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS)) 124 return false; 125 if (pos != ioend->io_offset + ioend->io_size) 126 return false; 127 if (!(wpc->iomap.flags & IOMAP_F_ANON_WRITE) && 128 iomap_sector(&wpc->iomap, pos) != bio_end_sector(&ioend->io_bio)) 129 return false; 130 /* 131 * Limit ioend bio chain lengths to minimise IO completion latency. This 132 * also prevents long tight loops ending page writeback on all the 133 * folios in the ioend. 134 */ 135 if (wpc->nr_folios >= IOEND_BATCH_SIZE) 136 return false; 137 return true; 138 } 139 140 /* 141 * Test to see if we have an existing ioend structure that we could append to 142 * first; otherwise finish off the current ioend and start another. 143 * 144 * If a new ioend is created and cached, the old ioend is submitted to the block 145 * layer instantly. Batching optimisations are provided by higher level block 146 * plugging. 147 * 148 * At the end of a writeback pass, there will be a cached ioend remaining on the 149 * writepage context that the caller will need to submit. 150 */ 151 ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio, 152 loff_t pos, loff_t end_pos, unsigned int dirty_len) 153 { 154 struct iomap_ioend *ioend = wpc->wb_ctx; 155 size_t poff = offset_in_folio(folio, pos); 156 unsigned int ioend_flags = 0; 157 unsigned int map_len = min_t(u64, dirty_len, 158 wpc->iomap.offset + wpc->iomap.length - pos); 159 int error; 160 161 trace_iomap_add_to_ioend(wpc->inode, pos, dirty_len, &wpc->iomap); 162 163 WARN_ON_ONCE(!folio->private && map_len < dirty_len); 164 165 switch (wpc->iomap.type) { 166 case IOMAP_INLINE: 167 WARN_ON_ONCE(1); 168 return -EIO; 169 case IOMAP_HOLE: 170 return map_len; 171 default: 172 break; 173 } 174 175 if (wpc->iomap.type == IOMAP_UNWRITTEN) 176 ioend_flags |= IOMAP_IOEND_UNWRITTEN; 177 if (wpc->iomap.flags & IOMAP_F_SHARED) 178 ioend_flags |= IOMAP_IOEND_SHARED; 179 if (folio_test_dropbehind(folio)) 180 ioend_flags |= IOMAP_IOEND_DONTCACHE; 181 if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY)) 182 ioend_flags |= IOMAP_IOEND_BOUNDARY; 183 184 if (!ioend || !iomap_can_add_to_ioend(wpc, pos, ioend_flags)) { 185 new_ioend: 186 if (ioend) { 187 error = wpc->ops->writeback_submit(wpc, 0); 188 if (error) 189 return error; 190 } 191 wpc->wb_ctx = ioend = iomap_alloc_ioend(wpc, pos, ioend_flags); 192 } 193 194 if (!bio_add_folio(&ioend->io_bio, folio, map_len, poff)) 195 goto new_ioend; 196 197 /* 198 * Clamp io_offset and io_size to the incore EOF so that ondisk 199 * file size updates in the ioend completion are byte-accurate. 200 * This avoids recovering files with zeroed tail regions when 201 * writeback races with appending writes: 202 * 203 * Thread 1: Thread 2: 204 * ------------ ----------- 205 * write [A, A+B] 206 * update inode size to A+B 207 * submit I/O [A, A+BS] 208 * write [A+B, A+B+C] 209 * update inode size to A+B+C 210 * <I/O completes, updates disk size to min(A+B+C, A+BS)> 211 * <power failure> 212 * 213 * After reboot: 214 * 1) with A+B+C < A+BS, the file has zero padding in range 215 * [A+B, A+B+C] 216 * 217 * |< Block Size (BS) >| 218 * |DDDDDDDDDDDD0000000000000| 219 * ^ ^ ^ 220 * A A+B A+B+C 221 * (EOF) 222 * 223 * 2) with A+B+C > A+BS, the file has zero padding in range 224 * [A+B, A+BS] 225 * 226 * |< Block Size (BS) >|< Block Size (BS) >| 227 * |DDDDDDDDDDDD0000000000000|00000000000000000000000000| 228 * ^ ^ ^ ^ 229 * A A+B A+BS A+B+C 230 * (EOF) 231 * 232 * D = Valid Data 233 * 0 = Zero Padding 234 * 235 * Note that this defeats the ability to chain the ioends of 236 * appending writes. 237 */ 238 ioend->io_size += map_len; 239 if (ioend->io_offset + ioend->io_size > end_pos) 240 ioend->io_size = end_pos - ioend->io_offset; 241 242 wbc_account_cgroup_owner(wpc->wbc, folio, map_len); 243 return map_len; 244 } 245 EXPORT_SYMBOL_GPL(iomap_add_to_ioend); 246 247 static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error) 248 { 249 if (ioend->io_parent) { 250 struct bio *bio = &ioend->io_bio; 251 252 ioend = ioend->io_parent; 253 bio_put(bio); 254 } 255 256 if (error) 257 cmpxchg(&ioend->io_error, 0, error); 258 259 if (!atomic_dec_and_test(&ioend->io_remaining)) 260 return 0; 261 if (ioend->io_flags & IOMAP_IOEND_DIRECT) 262 return iomap_finish_ioend_direct(ioend); 263 return iomap_finish_ioend_buffered(ioend); 264 } 265 266 /* 267 * Ioend completion routine for merged bios. This can only be called from task 268 * contexts as merged ioends can be of unbound length. Hence we have to break up 269 * the writeback completions into manageable chunks to avoid long scheduler 270 * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get 271 * good batch processing throughput without creating adverse scheduler latency 272 * conditions. 273 */ 274 void iomap_finish_ioends(struct iomap_ioend *ioend, int error) 275 { 276 struct list_head tmp; 277 u32 completions; 278 279 might_sleep(); 280 281 list_replace_init(&ioend->io_list, &tmp); 282 completions = iomap_finish_ioend(ioend, error); 283 284 while (!list_empty(&tmp)) { 285 if (completions > IOEND_BATCH_SIZE * 8) { 286 cond_resched(); 287 completions = 0; 288 } 289 ioend = list_first_entry(&tmp, struct iomap_ioend, io_list); 290 list_del_init(&ioend->io_list); 291 completions += iomap_finish_ioend(ioend, error); 292 } 293 } 294 EXPORT_SYMBOL_GPL(iomap_finish_ioends); 295 296 /* 297 * We can merge two adjacent ioends if they have the same set of work to do. 298 */ 299 static bool iomap_ioend_can_merge(struct iomap_ioend *ioend, 300 struct iomap_ioend *next) 301 { 302 if (ioend->io_bio.bi_status != next->io_bio.bi_status) 303 return false; 304 if (next->io_flags & IOMAP_IOEND_BOUNDARY) 305 return false; 306 if ((ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS) != 307 (next->io_flags & IOMAP_IOEND_NOMERGE_FLAGS)) 308 return false; 309 if (ioend->io_offset + ioend->io_size != next->io_offset) 310 return false; 311 /* 312 * Do not merge physically discontiguous ioends. The filesystem 313 * completion functions will have to iterate the physical 314 * discontiguities even if we merge the ioends at a logical level, so 315 * we don't gain anything by merging physical discontiguities here. 316 * 317 * We cannot use bio->bi_iter.bi_sector here as it is modified during 318 * submission so does not point to the start sector of the bio at 319 * completion. 320 */ 321 if (ioend->io_sector + (ioend->io_size >> SECTOR_SHIFT) != 322 next->io_sector) 323 return false; 324 return true; 325 } 326 327 void iomap_ioend_try_merge(struct iomap_ioend *ioend, 328 struct list_head *more_ioends) 329 { 330 struct iomap_ioend *next; 331 332 INIT_LIST_HEAD(&ioend->io_list); 333 334 while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend, 335 io_list))) { 336 if (!iomap_ioend_can_merge(ioend, next)) 337 break; 338 list_move_tail(&next->io_list, &ioend->io_list); 339 ioend->io_size += next->io_size; 340 } 341 } 342 EXPORT_SYMBOL_GPL(iomap_ioend_try_merge); 343 344 static int iomap_ioend_compare(void *priv, const struct list_head *a, 345 const struct list_head *b) 346 { 347 struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list); 348 struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list); 349 350 if (ia->io_offset < ib->io_offset) 351 return -1; 352 if (ia->io_offset > ib->io_offset) 353 return 1; 354 return 0; 355 } 356 357 void iomap_sort_ioends(struct list_head *ioend_list) 358 { 359 list_sort(NULL, ioend_list, iomap_ioend_compare); 360 } 361 EXPORT_SYMBOL_GPL(iomap_sort_ioends); 362 363 /* 364 * Split up to the first @max_len bytes from @ioend if the ioend covers more 365 * than @max_len bytes. 366 * 367 * If @is_append is set, the split will be based on the hardware limits for 368 * REQ_OP_ZONE_APPEND commands and can be less than @max_len if the hardware 369 * limits don't allow the entire @max_len length. 370 * 371 * The bio embedded into @ioend must be a REQ_OP_WRITE because the block layer 372 * does not allow splitting REQ_OP_ZONE_APPEND bios. The file systems has to 373 * switch the operation after this call, but before submitting the bio. 374 */ 375 struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend, 376 unsigned int max_len, bool is_append) 377 { 378 struct bio *bio = &ioend->io_bio; 379 struct iomap_ioend *split_ioend; 380 unsigned int nr_segs; 381 int sector_offset; 382 struct bio *split; 383 384 if (is_append) { 385 struct queue_limits *lim = bdev_limits(bio->bi_bdev); 386 387 max_len = min(max_len, 388 lim->max_zone_append_sectors << SECTOR_SHIFT); 389 390 sector_offset = bio_split_rw_at(bio, lim, &nr_segs, max_len); 391 if (unlikely(sector_offset < 0)) 392 return ERR_PTR(sector_offset); 393 if (!sector_offset) 394 return NULL; 395 } else { 396 if (bio->bi_iter.bi_size <= max_len) 397 return NULL; 398 sector_offset = max_len >> SECTOR_SHIFT; 399 } 400 401 /* ensure the split ioend is still block size aligned */ 402 sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT, 403 i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT; 404 405 split = bio_split(bio, sector_offset, GFP_NOFS, &iomap_ioend_bioset); 406 if (IS_ERR(split)) 407 return ERR_CAST(split); 408 split->bi_private = bio->bi_private; 409 split->bi_end_io = bio->bi_end_io; 410 411 split_ioend = iomap_init_ioend(ioend->io_inode, split, ioend->io_offset, 412 ioend->io_flags); 413 split_ioend->io_parent = ioend; 414 415 atomic_inc(&ioend->io_remaining); 416 ioend->io_offset += split_ioend->io_size; 417 ioend->io_size -= split_ioend->io_size; 418 419 split_ioend->io_sector = ioend->io_sector; 420 if (!is_append) 421 ioend->io_sector += (split_ioend->io_size >> SECTOR_SHIFT); 422 return split_ioend; 423 } 424 EXPORT_SYMBOL_GPL(iomap_split_ioend); 425 426 static int __init iomap_ioend_init(void) 427 { 428 return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE), 429 offsetof(struct iomap_ioend, io_bio), 430 BIOSET_NEED_BVECS); 431 } 432 fs_initcall(iomap_ioend_init); 433