1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-2025 Christoph Hellwig. 4 */ 5 #include <linux/iomap.h> 6 #include <linux/list_sort.h> 7 #include <linux/pagemap.h> 8 #include <linux/writeback.h> 9 #include "internal.h" 10 #include "trace.h" 11 12 struct bio_set iomap_ioend_bioset; 13 EXPORT_SYMBOL_GPL(iomap_ioend_bioset); 14 15 struct iomap_ioend *iomap_init_ioend(struct inode *inode, 16 struct bio *bio, loff_t file_offset, u16 ioend_flags) 17 { 18 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); 19 20 atomic_set(&ioend->io_remaining, 1); 21 ioend->io_error = 0; 22 ioend->io_parent = NULL; 23 INIT_LIST_HEAD(&ioend->io_list); 24 ioend->io_flags = ioend_flags; 25 ioend->io_inode = inode; 26 ioend->io_offset = file_offset; 27 ioend->io_size = bio->bi_iter.bi_size; 28 ioend->io_sector = bio->bi_iter.bi_sector; 29 ioend->io_private = NULL; 30 return ioend; 31 } 32 EXPORT_SYMBOL_GPL(iomap_init_ioend); 33 34 /* 35 * We're now finished for good with this ioend structure. Update the folio 36 * state, release holds on bios, and finally free up memory. Do not use the 37 * ioend after this. 38 */ 39 static u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend) 40 { 41 struct inode *inode = ioend->io_inode; 42 struct bio *bio = &ioend->io_bio; 43 struct folio_iter fi; 44 u32 folio_count = 0; 45 46 if (ioend->io_error) { 47 mapping_set_error(inode->i_mapping, ioend->io_error); 48 if (!bio_flagged(bio, BIO_QUIET)) { 49 pr_err_ratelimited( 50 "%s: writeback error on inode %lu, offset %lld, sector %llu", 51 inode->i_sb->s_id, inode->i_ino, 52 ioend->io_offset, ioend->io_sector); 53 } 54 } 55 56 /* walk all folios in bio, ending page IO on them */ 57 bio_for_each_folio_all(fi, bio) { 58 iomap_finish_folio_write(inode, fi.folio, fi.length); 59 folio_count++; 60 } 61 62 bio_put(bio); /* frees the ioend */ 63 return folio_count; 64 } 65 66 static void ioend_writeback_end_bio(struct bio *bio) 67 { 68 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); 69 70 ioend->io_error = blk_status_to_errno(bio->bi_status); 71 iomap_finish_ioend_buffered(ioend); 72 } 73 74 /* 75 * We cannot cancel the ioend directly in case of an error, so call the bio end 76 * I/O handler with the error status here to run the normal I/O completion 77 * handler. 78 */ 79 int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error) 80 { 81 struct iomap_ioend *ioend = wpc->wb_ctx; 82 83 if (!ioend->io_bio.bi_end_io) 84 ioend->io_bio.bi_end_io = ioend_writeback_end_bio; 85 86 if (WARN_ON_ONCE(wpc->iomap.flags & IOMAP_F_ANON_WRITE)) 87 error = -EIO; 88 89 if (error) { 90 ioend->io_bio.bi_status = errno_to_blk_status(error); 91 bio_endio(&ioend->io_bio); 92 return error; 93 } 94 95 submit_bio(&ioend->io_bio); 96 return 0; 97 } 98 EXPORT_SYMBOL_GPL(iomap_ioend_writeback_submit); 99 100 static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc, 101 loff_t pos, u16 ioend_flags) 102 { 103 struct bio *bio; 104 105 bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS, 106 REQ_OP_WRITE | wbc_to_write_flags(wpc->wbc), 107 GFP_NOFS, &iomap_ioend_bioset); 108 bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos); 109 bio->bi_write_hint = wpc->inode->i_write_hint; 110 wbc_init_bio(wpc->wbc, bio); 111 wpc->nr_folios = 0; 112 return iomap_init_ioend(wpc->inode, bio, pos, ioend_flags); 113 } 114 115 static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos, 116 u16 ioend_flags) 117 { 118 struct iomap_ioend *ioend = wpc->wb_ctx; 119 120 if (ioend_flags & IOMAP_IOEND_BOUNDARY) 121 return false; 122 if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) != 123 (ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS)) 124 return false; 125 if (pos != ioend->io_offset + ioend->io_size) 126 return false; 127 if (!(wpc->iomap.flags & IOMAP_F_ANON_WRITE) && 128 iomap_sector(&wpc->iomap, pos) != bio_end_sector(&ioend->io_bio)) 129 return false; 130 /* 131 * Limit ioend bio chain lengths to minimise IO completion latency. This 132 * also prevents long tight loops ending page writeback on all the 133 * folios in the ioend. 134 */ 135 if (wpc->nr_folios >= IOEND_BATCH_SIZE) 136 return false; 137 return true; 138 } 139 140 /* 141 * Test to see if we have an existing ioend structure that we could append to 142 * first; otherwise finish off the current ioend and start another. 143 * 144 * If a new ioend is created and cached, the old ioend is submitted to the block 145 * layer instantly. Batching optimisations are provided by higher level block 146 * plugging. 147 * 148 * At the end of a writeback pass, there will be a cached ioend remaining on the 149 * writepage context that the caller will need to submit. 150 */ 151 ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio, 152 loff_t pos, loff_t end_pos, unsigned int dirty_len) 153 { 154 struct iomap_ioend *ioend = wpc->wb_ctx; 155 size_t poff = offset_in_folio(folio, pos); 156 unsigned int ioend_flags = 0; 157 unsigned int map_len = min_t(u64, dirty_len, 158 wpc->iomap.offset + wpc->iomap.length - pos); 159 int error; 160 161 trace_iomap_add_to_ioend(wpc->inode, pos, dirty_len, &wpc->iomap); 162 163 WARN_ON_ONCE(!folio->private && map_len < dirty_len); 164 165 switch (wpc->iomap.type) { 166 case IOMAP_INLINE: 167 WARN_ON_ONCE(1); 168 return -EIO; 169 case IOMAP_HOLE: 170 return map_len; 171 default: 172 break; 173 } 174 175 if (wpc->iomap.type == IOMAP_UNWRITTEN) 176 ioend_flags |= IOMAP_IOEND_UNWRITTEN; 177 if (wpc->iomap.flags & IOMAP_F_SHARED) 178 ioend_flags |= IOMAP_IOEND_SHARED; 179 if (folio_test_dropbehind(folio)) 180 ioend_flags |= IOMAP_IOEND_DONTCACHE; 181 if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY)) 182 ioend_flags |= IOMAP_IOEND_BOUNDARY; 183 184 if (!ioend || !iomap_can_add_to_ioend(wpc, pos, ioend_flags)) { 185 new_ioend: 186 if (ioend) { 187 error = wpc->ops->writeback_submit(wpc, 0); 188 if (error) 189 return error; 190 } 191 wpc->wb_ctx = ioend = iomap_alloc_ioend(wpc, pos, ioend_flags); 192 } 193 194 if (!bio_add_folio(&ioend->io_bio, folio, map_len, poff)) 195 goto new_ioend; 196 197 iomap_start_folio_write(wpc->inode, folio, map_len); 198 199 /* 200 * Clamp io_offset and io_size to the incore EOF so that ondisk 201 * file size updates in the ioend completion are byte-accurate. 202 * This avoids recovering files with zeroed tail regions when 203 * writeback races with appending writes: 204 * 205 * Thread 1: Thread 2: 206 * ------------ ----------- 207 * write [A, A+B] 208 * update inode size to A+B 209 * submit I/O [A, A+BS] 210 * write [A+B, A+B+C] 211 * update inode size to A+B+C 212 * <I/O completes, updates disk size to min(A+B+C, A+BS)> 213 * <power failure> 214 * 215 * After reboot: 216 * 1) with A+B+C < A+BS, the file has zero padding in range 217 * [A+B, A+B+C] 218 * 219 * |< Block Size (BS) >| 220 * |DDDDDDDDDDDD0000000000000| 221 * ^ ^ ^ 222 * A A+B A+B+C 223 * (EOF) 224 * 225 * 2) with A+B+C > A+BS, the file has zero padding in range 226 * [A+B, A+BS] 227 * 228 * |< Block Size (BS) >|< Block Size (BS) >| 229 * |DDDDDDDDDDDD0000000000000|00000000000000000000000000| 230 * ^ ^ ^ ^ 231 * A A+B A+BS A+B+C 232 * (EOF) 233 * 234 * D = Valid Data 235 * 0 = Zero Padding 236 * 237 * Note that this defeats the ability to chain the ioends of 238 * appending writes. 239 */ 240 ioend->io_size += map_len; 241 if (ioend->io_offset + ioend->io_size > end_pos) 242 ioend->io_size = end_pos - ioend->io_offset; 243 244 wbc_account_cgroup_owner(wpc->wbc, folio, map_len); 245 return map_len; 246 } 247 EXPORT_SYMBOL_GPL(iomap_add_to_ioend); 248 249 static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error) 250 { 251 if (ioend->io_parent) { 252 struct bio *bio = &ioend->io_bio; 253 254 ioend = ioend->io_parent; 255 bio_put(bio); 256 } 257 258 if (error) 259 cmpxchg(&ioend->io_error, 0, error); 260 261 if (!atomic_dec_and_test(&ioend->io_remaining)) 262 return 0; 263 if (ioend->io_flags & IOMAP_IOEND_DIRECT) 264 return iomap_finish_ioend_direct(ioend); 265 return iomap_finish_ioend_buffered(ioend); 266 } 267 268 /* 269 * Ioend completion routine for merged bios. This can only be called from task 270 * contexts as merged ioends can be of unbound length. Hence we have to break up 271 * the writeback completions into manageable chunks to avoid long scheduler 272 * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get 273 * good batch processing throughput without creating adverse scheduler latency 274 * conditions. 275 */ 276 void iomap_finish_ioends(struct iomap_ioend *ioend, int error) 277 { 278 struct list_head tmp; 279 u32 completions; 280 281 might_sleep(); 282 283 list_replace_init(&ioend->io_list, &tmp); 284 completions = iomap_finish_ioend(ioend, error); 285 286 while (!list_empty(&tmp)) { 287 if (completions > IOEND_BATCH_SIZE * 8) { 288 cond_resched(); 289 completions = 0; 290 } 291 ioend = list_first_entry(&tmp, struct iomap_ioend, io_list); 292 list_del_init(&ioend->io_list); 293 completions += iomap_finish_ioend(ioend, error); 294 } 295 } 296 EXPORT_SYMBOL_GPL(iomap_finish_ioends); 297 298 /* 299 * We can merge two adjacent ioends if they have the same set of work to do. 300 */ 301 static bool iomap_ioend_can_merge(struct iomap_ioend *ioend, 302 struct iomap_ioend *next) 303 { 304 if (ioend->io_bio.bi_status != next->io_bio.bi_status) 305 return false; 306 if (next->io_flags & IOMAP_IOEND_BOUNDARY) 307 return false; 308 if ((ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS) != 309 (next->io_flags & IOMAP_IOEND_NOMERGE_FLAGS)) 310 return false; 311 if (ioend->io_offset + ioend->io_size != next->io_offset) 312 return false; 313 /* 314 * Do not merge physically discontiguous ioends. The filesystem 315 * completion functions will have to iterate the physical 316 * discontiguities even if we merge the ioends at a logical level, so 317 * we don't gain anything by merging physical discontiguities here. 318 * 319 * We cannot use bio->bi_iter.bi_sector here as it is modified during 320 * submission so does not point to the start sector of the bio at 321 * completion. 322 */ 323 if (ioend->io_sector + (ioend->io_size >> SECTOR_SHIFT) != 324 next->io_sector) 325 return false; 326 return true; 327 } 328 329 void iomap_ioend_try_merge(struct iomap_ioend *ioend, 330 struct list_head *more_ioends) 331 { 332 struct iomap_ioend *next; 333 334 INIT_LIST_HEAD(&ioend->io_list); 335 336 while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend, 337 io_list))) { 338 if (!iomap_ioend_can_merge(ioend, next)) 339 break; 340 list_move_tail(&next->io_list, &ioend->io_list); 341 ioend->io_size += next->io_size; 342 } 343 } 344 EXPORT_SYMBOL_GPL(iomap_ioend_try_merge); 345 346 static int iomap_ioend_compare(void *priv, const struct list_head *a, 347 const struct list_head *b) 348 { 349 struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list); 350 struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list); 351 352 if (ia->io_offset < ib->io_offset) 353 return -1; 354 if (ia->io_offset > ib->io_offset) 355 return 1; 356 return 0; 357 } 358 359 void iomap_sort_ioends(struct list_head *ioend_list) 360 { 361 list_sort(NULL, ioend_list, iomap_ioend_compare); 362 } 363 EXPORT_SYMBOL_GPL(iomap_sort_ioends); 364 365 /* 366 * Split up to the first @max_len bytes from @ioend if the ioend covers more 367 * than @max_len bytes. 368 * 369 * If @is_append is set, the split will be based on the hardware limits for 370 * REQ_OP_ZONE_APPEND commands and can be less than @max_len if the hardware 371 * limits don't allow the entire @max_len length. 372 * 373 * The bio embedded into @ioend must be a REQ_OP_WRITE because the block layer 374 * does not allow splitting REQ_OP_ZONE_APPEND bios. The file systems has to 375 * switch the operation after this call, but before submitting the bio. 376 */ 377 struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend, 378 unsigned int max_len, bool is_append) 379 { 380 struct bio *bio = &ioend->io_bio; 381 struct iomap_ioend *split_ioend; 382 unsigned int nr_segs; 383 int sector_offset; 384 struct bio *split; 385 386 if (is_append) { 387 struct queue_limits *lim = bdev_limits(bio->bi_bdev); 388 389 max_len = min(max_len, 390 lim->max_zone_append_sectors << SECTOR_SHIFT); 391 392 sector_offset = bio_split_rw_at(bio, lim, &nr_segs, max_len); 393 if (unlikely(sector_offset < 0)) 394 return ERR_PTR(sector_offset); 395 if (!sector_offset) 396 return NULL; 397 } else { 398 if (bio->bi_iter.bi_size <= max_len) 399 return NULL; 400 sector_offset = max_len >> SECTOR_SHIFT; 401 } 402 403 /* ensure the split ioend is still block size aligned */ 404 sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT, 405 i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT; 406 407 split = bio_split(bio, sector_offset, GFP_NOFS, &iomap_ioend_bioset); 408 if (IS_ERR(split)) 409 return ERR_CAST(split); 410 split->bi_private = bio->bi_private; 411 split->bi_end_io = bio->bi_end_io; 412 413 split_ioend = iomap_init_ioend(ioend->io_inode, split, ioend->io_offset, 414 ioend->io_flags); 415 split_ioend->io_parent = ioend; 416 417 atomic_inc(&ioend->io_remaining); 418 ioend->io_offset += split_ioend->io_size; 419 ioend->io_size -= split_ioend->io_size; 420 421 split_ioend->io_sector = ioend->io_sector; 422 if (!is_append) 423 ioend->io_sector += (split_ioend->io_size >> SECTOR_SHIFT); 424 return split_ioend; 425 } 426 EXPORT_SYMBOL_GPL(iomap_split_ioend); 427 428 static int __init iomap_ioend_init(void) 429 { 430 return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE), 431 offsetof(struct iomap_ioend, io_bio), 432 BIOSET_NEED_BVECS); 433 } 434 fs_initcall(iomap_ioend_init); 435