1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-2025 Christoph Hellwig. 4 */ 5 #include <linux/iomap.h> 6 #include <linux/list_sort.h> 7 #include <linux/pagemap.h> 8 #include <linux/writeback.h> 9 #include <linux/fserror.h> 10 #include "internal.h" 11 #include "trace.h" 12 13 struct bio_set iomap_ioend_bioset; 14 EXPORT_SYMBOL_GPL(iomap_ioend_bioset); 15 16 struct iomap_ioend *iomap_init_ioend(struct inode *inode, 17 struct bio *bio, loff_t file_offset, u16 ioend_flags) 18 { 19 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); 20 21 atomic_set(&ioend->io_remaining, 1); 22 ioend->io_error = 0; 23 ioend->io_parent = NULL; 24 INIT_LIST_HEAD(&ioend->io_list); 25 ioend->io_flags = ioend_flags; 26 ioend->io_inode = inode; 27 ioend->io_offset = file_offset; 28 ioend->io_size = bio->bi_iter.bi_size; 29 ioend->io_sector = bio->bi_iter.bi_sector; 30 ioend->io_private = NULL; 31 return ioend; 32 } 33 EXPORT_SYMBOL_GPL(iomap_init_ioend); 34 35 /* 36 * We're now finished for good with this ioend structure. Update the folio 37 * state, release holds on bios, and finally free up memory. Do not use the 38 * ioend after this. 39 */ 40 static u32 iomap_finish_ioend_buffered(struct iomap_ioend *ioend) 41 { 42 struct inode *inode = ioend->io_inode; 43 struct bio *bio = &ioend->io_bio; 44 struct folio_iter fi; 45 u32 folio_count = 0; 46 47 if (ioend->io_error) { 48 mapping_set_error(inode->i_mapping, ioend->io_error); 49 if (!bio_flagged(bio, BIO_QUIET)) { 50 pr_err_ratelimited( 51 "%s: writeback error on inode %lu, offset %lld, sector %llu", 52 inode->i_sb->s_id, inode->i_ino, 53 ioend->io_offset, ioend->io_sector); 54 } 55 } 56 57 /* walk all folios in bio, ending page IO on them */ 58 bio_for_each_folio_all(fi, bio) { 59 if (ioend->io_error) 60 fserror_report_io(inode, FSERR_BUFFERED_WRITE, 61 folio_pos(fi.folio) + fi.offset, 62 fi.length, ioend->io_error, 63 GFP_ATOMIC); 64 iomap_finish_folio_write(inode, fi.folio, fi.length); 65 folio_count++; 66 } 67 68 bio_put(bio); /* frees the ioend */ 69 return folio_count; 70 } 71 72 static void ioend_writeback_end_bio(struct bio *bio) 73 { 74 struct iomap_ioend *ioend = iomap_ioend_from_bio(bio); 75 76 ioend->io_error = blk_status_to_errno(bio->bi_status); 77 iomap_finish_ioend_buffered(ioend); 78 } 79 80 /* 81 * We cannot cancel the ioend directly in case of an error, so call the bio end 82 * I/O handler with the error status here to run the normal I/O completion 83 * handler. 84 */ 85 int iomap_ioend_writeback_submit(struct iomap_writepage_ctx *wpc, int error) 86 { 87 struct iomap_ioend *ioend = wpc->wb_ctx; 88 89 if (!ioend->io_bio.bi_end_io) 90 ioend->io_bio.bi_end_io = ioend_writeback_end_bio; 91 92 if (WARN_ON_ONCE(wpc->iomap.flags & IOMAP_F_ANON_WRITE)) 93 error = -EIO; 94 95 if (error) { 96 ioend->io_bio.bi_status = errno_to_blk_status(error); 97 bio_endio(&ioend->io_bio); 98 return error; 99 } 100 101 submit_bio(&ioend->io_bio); 102 return 0; 103 } 104 EXPORT_SYMBOL_GPL(iomap_ioend_writeback_submit); 105 106 static struct iomap_ioend *iomap_alloc_ioend(struct iomap_writepage_ctx *wpc, 107 loff_t pos, u16 ioend_flags) 108 { 109 struct bio *bio; 110 111 bio = bio_alloc_bioset(wpc->iomap.bdev, BIO_MAX_VECS, 112 REQ_OP_WRITE | wbc_to_write_flags(wpc->wbc), 113 GFP_NOFS, &iomap_ioend_bioset); 114 bio->bi_iter.bi_sector = iomap_sector(&wpc->iomap, pos); 115 bio->bi_write_hint = wpc->inode->i_write_hint; 116 wbc_init_bio(wpc->wbc, bio); 117 wpc->nr_folios = 0; 118 return iomap_init_ioend(wpc->inode, bio, pos, ioend_flags); 119 } 120 121 static bool iomap_can_add_to_ioend(struct iomap_writepage_ctx *wpc, loff_t pos, 122 u16 ioend_flags) 123 { 124 struct iomap_ioend *ioend = wpc->wb_ctx; 125 126 if (ioend_flags & IOMAP_IOEND_BOUNDARY) 127 return false; 128 if ((ioend_flags & IOMAP_IOEND_NOMERGE_FLAGS) != 129 (ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS)) 130 return false; 131 if (pos != ioend->io_offset + ioend->io_size) 132 return false; 133 if (!(wpc->iomap.flags & IOMAP_F_ANON_WRITE) && 134 iomap_sector(&wpc->iomap, pos) != bio_end_sector(&ioend->io_bio)) 135 return false; 136 /* 137 * Limit ioend bio chain lengths to minimise IO completion latency. This 138 * also prevents long tight loops ending page writeback on all the 139 * folios in the ioend. 140 */ 141 if (wpc->nr_folios >= IOEND_BATCH_SIZE) 142 return false; 143 return true; 144 } 145 146 /* 147 * Test to see if we have an existing ioend structure that we could append to 148 * first; otherwise finish off the current ioend and start another. 149 * 150 * If a new ioend is created and cached, the old ioend is submitted to the block 151 * layer instantly. Batching optimisations are provided by higher level block 152 * plugging. 153 * 154 * At the end of a writeback pass, there will be a cached ioend remaining on the 155 * writepage context that the caller will need to submit. 156 */ 157 ssize_t iomap_add_to_ioend(struct iomap_writepage_ctx *wpc, struct folio *folio, 158 loff_t pos, loff_t end_pos, unsigned int dirty_len) 159 { 160 struct iomap_ioend *ioend = wpc->wb_ctx; 161 size_t poff = offset_in_folio(folio, pos); 162 unsigned int ioend_flags = 0; 163 unsigned int map_len = min_t(u64, dirty_len, 164 wpc->iomap.offset + wpc->iomap.length - pos); 165 int error; 166 167 trace_iomap_add_to_ioend(wpc->inode, pos, dirty_len, &wpc->iomap); 168 169 WARN_ON_ONCE(!folio->private && map_len < dirty_len); 170 171 switch (wpc->iomap.type) { 172 case IOMAP_INLINE: 173 WARN_ON_ONCE(1); 174 return -EIO; 175 case IOMAP_HOLE: 176 return map_len; 177 default: 178 break; 179 } 180 181 if (wpc->iomap.type == IOMAP_UNWRITTEN) 182 ioend_flags |= IOMAP_IOEND_UNWRITTEN; 183 if (wpc->iomap.flags & IOMAP_F_SHARED) 184 ioend_flags |= IOMAP_IOEND_SHARED; 185 if (folio_test_dropbehind(folio)) 186 ioend_flags |= IOMAP_IOEND_DONTCACHE; 187 if (pos == wpc->iomap.offset && (wpc->iomap.flags & IOMAP_F_BOUNDARY)) 188 ioend_flags |= IOMAP_IOEND_BOUNDARY; 189 190 if (!ioend || !iomap_can_add_to_ioend(wpc, pos, ioend_flags)) { 191 new_ioend: 192 if (ioend) { 193 error = wpc->ops->writeback_submit(wpc, 0); 194 if (error) 195 return error; 196 } 197 wpc->wb_ctx = ioend = iomap_alloc_ioend(wpc, pos, ioend_flags); 198 } 199 200 if (!bio_add_folio(&ioend->io_bio, folio, map_len, poff)) 201 goto new_ioend; 202 203 /* 204 * Clamp io_offset and io_size to the incore EOF so that ondisk 205 * file size updates in the ioend completion are byte-accurate. 206 * This avoids recovering files with zeroed tail regions when 207 * writeback races with appending writes: 208 * 209 * Thread 1: Thread 2: 210 * ------------ ----------- 211 * write [A, A+B] 212 * update inode size to A+B 213 * submit I/O [A, A+BS] 214 * write [A+B, A+B+C] 215 * update inode size to A+B+C 216 * <I/O completes, updates disk size to min(A+B+C, A+BS)> 217 * <power failure> 218 * 219 * After reboot: 220 * 1) with A+B+C < A+BS, the file has zero padding in range 221 * [A+B, A+B+C] 222 * 223 * |< Block Size (BS) >| 224 * |DDDDDDDDDDDD0000000000000| 225 * ^ ^ ^ 226 * A A+B A+B+C 227 * (EOF) 228 * 229 * 2) with A+B+C > A+BS, the file has zero padding in range 230 * [A+B, A+BS] 231 * 232 * |< Block Size (BS) >|< Block Size (BS) >| 233 * |DDDDDDDDDDDD0000000000000|00000000000000000000000000| 234 * ^ ^ ^ ^ 235 * A A+B A+BS A+B+C 236 * (EOF) 237 * 238 * D = Valid Data 239 * 0 = Zero Padding 240 * 241 * Note that this defeats the ability to chain the ioends of 242 * appending writes. 243 */ 244 ioend->io_size += map_len; 245 if (ioend->io_offset + ioend->io_size > end_pos) 246 ioend->io_size = end_pos - ioend->io_offset; 247 248 wbc_account_cgroup_owner(wpc->wbc, folio, map_len); 249 return map_len; 250 } 251 EXPORT_SYMBOL_GPL(iomap_add_to_ioend); 252 253 static u32 iomap_finish_ioend(struct iomap_ioend *ioend, int error) 254 { 255 if (ioend->io_parent) { 256 struct bio *bio = &ioend->io_bio; 257 258 ioend = ioend->io_parent; 259 bio_put(bio); 260 } 261 262 if (error) 263 cmpxchg(&ioend->io_error, 0, error); 264 265 if (!atomic_dec_and_test(&ioend->io_remaining)) 266 return 0; 267 if (ioend->io_flags & IOMAP_IOEND_DIRECT) 268 return iomap_finish_ioend_direct(ioend); 269 return iomap_finish_ioend_buffered(ioend); 270 } 271 272 /* 273 * Ioend completion routine for merged bios. This can only be called from task 274 * contexts as merged ioends can be of unbound length. Hence we have to break up 275 * the writeback completions into manageable chunks to avoid long scheduler 276 * holdoffs. We aim to keep scheduler holdoffs down below 10ms so that we get 277 * good batch processing throughput without creating adverse scheduler latency 278 * conditions. 279 */ 280 void iomap_finish_ioends(struct iomap_ioend *ioend, int error) 281 { 282 struct list_head tmp; 283 u32 completions; 284 285 might_sleep(); 286 287 list_replace_init(&ioend->io_list, &tmp); 288 completions = iomap_finish_ioend(ioend, error); 289 290 while (!list_empty(&tmp)) { 291 if (completions > IOEND_BATCH_SIZE * 8) { 292 cond_resched(); 293 completions = 0; 294 } 295 ioend = list_first_entry(&tmp, struct iomap_ioend, io_list); 296 list_del_init(&ioend->io_list); 297 completions += iomap_finish_ioend(ioend, error); 298 } 299 } 300 EXPORT_SYMBOL_GPL(iomap_finish_ioends); 301 302 /* 303 * We can merge two adjacent ioends if they have the same set of work to do. 304 */ 305 static bool iomap_ioend_can_merge(struct iomap_ioend *ioend, 306 struct iomap_ioend *next) 307 { 308 /* 309 * There is no point in merging reads as there is no completion 310 * processing that can be easily batched up for them. 311 */ 312 if (bio_op(&ioend->io_bio) == REQ_OP_READ || 313 bio_op(&next->io_bio) == REQ_OP_READ) 314 return false; 315 316 if (ioend->io_bio.bi_status != next->io_bio.bi_status) 317 return false; 318 if (next->io_flags & IOMAP_IOEND_BOUNDARY) 319 return false; 320 if ((ioend->io_flags & IOMAP_IOEND_NOMERGE_FLAGS) != 321 (next->io_flags & IOMAP_IOEND_NOMERGE_FLAGS)) 322 return false; 323 if (ioend->io_offset + ioend->io_size != next->io_offset) 324 return false; 325 /* 326 * Do not merge physically discontiguous ioends. The filesystem 327 * completion functions will have to iterate the physical 328 * discontiguities even if we merge the ioends at a logical level, so 329 * we don't gain anything by merging physical discontiguities here. 330 * 331 * We cannot use bio->bi_iter.bi_sector here as it is modified during 332 * submission so does not point to the start sector of the bio at 333 * completion. 334 */ 335 if (ioend->io_sector + (ioend->io_size >> SECTOR_SHIFT) != 336 next->io_sector) 337 return false; 338 return true; 339 } 340 341 void iomap_ioend_try_merge(struct iomap_ioend *ioend, 342 struct list_head *more_ioends) 343 { 344 struct iomap_ioend *next; 345 346 INIT_LIST_HEAD(&ioend->io_list); 347 348 while ((next = list_first_entry_or_null(more_ioends, struct iomap_ioend, 349 io_list))) { 350 if (!iomap_ioend_can_merge(ioend, next)) 351 break; 352 list_move_tail(&next->io_list, &ioend->io_list); 353 ioend->io_size += next->io_size; 354 } 355 } 356 EXPORT_SYMBOL_GPL(iomap_ioend_try_merge); 357 358 static int iomap_ioend_compare(void *priv, const struct list_head *a, 359 const struct list_head *b) 360 { 361 struct iomap_ioend *ia = container_of(a, struct iomap_ioend, io_list); 362 struct iomap_ioend *ib = container_of(b, struct iomap_ioend, io_list); 363 364 if (ia->io_offset < ib->io_offset) 365 return -1; 366 if (ia->io_offset > ib->io_offset) 367 return 1; 368 return 0; 369 } 370 371 void iomap_sort_ioends(struct list_head *ioend_list) 372 { 373 list_sort(NULL, ioend_list, iomap_ioend_compare); 374 } 375 EXPORT_SYMBOL_GPL(iomap_sort_ioends); 376 377 /* 378 * Split up to the first @max_len bytes from @ioend if the ioend covers more 379 * than @max_len bytes. 380 * 381 * If @is_append is set, the split will be based on the hardware limits for 382 * REQ_OP_ZONE_APPEND commands and can be less than @max_len if the hardware 383 * limits don't allow the entire @max_len length. 384 * 385 * The bio embedded into @ioend must be a REQ_OP_WRITE because the block layer 386 * does not allow splitting REQ_OP_ZONE_APPEND bios. The file systems has to 387 * switch the operation after this call, but before submitting the bio. 388 */ 389 struct iomap_ioend *iomap_split_ioend(struct iomap_ioend *ioend, 390 unsigned int max_len, bool is_append) 391 { 392 struct bio *bio = &ioend->io_bio; 393 struct iomap_ioend *split_ioend; 394 unsigned int nr_segs; 395 int sector_offset; 396 struct bio *split; 397 398 if (is_append) { 399 struct queue_limits *lim = bdev_limits(bio->bi_bdev); 400 401 max_len = min(max_len, 402 lim->max_zone_append_sectors << SECTOR_SHIFT); 403 404 sector_offset = bio_split_rw_at(bio, lim, &nr_segs, max_len); 405 if (unlikely(sector_offset < 0)) 406 return ERR_PTR(sector_offset); 407 if (!sector_offset) 408 return NULL; 409 } else { 410 if (bio->bi_iter.bi_size <= max_len) 411 return NULL; 412 sector_offset = max_len >> SECTOR_SHIFT; 413 } 414 415 /* ensure the split ioend is still block size aligned */ 416 sector_offset = ALIGN_DOWN(sector_offset << SECTOR_SHIFT, 417 i_blocksize(ioend->io_inode)) >> SECTOR_SHIFT; 418 419 split = bio_split(bio, sector_offset, GFP_NOFS, &iomap_ioend_bioset); 420 if (IS_ERR(split)) 421 return ERR_CAST(split); 422 split->bi_private = bio->bi_private; 423 split->bi_end_io = bio->bi_end_io; 424 425 split_ioend = iomap_init_ioend(ioend->io_inode, split, ioend->io_offset, 426 ioend->io_flags); 427 split_ioend->io_parent = ioend; 428 429 atomic_inc(&ioend->io_remaining); 430 ioend->io_offset += split_ioend->io_size; 431 ioend->io_size -= split_ioend->io_size; 432 433 split_ioend->io_sector = ioend->io_sector; 434 if (!is_append) 435 ioend->io_sector += (split_ioend->io_size >> SECTOR_SHIFT); 436 return split_ioend; 437 } 438 EXPORT_SYMBOL_GPL(iomap_split_ioend); 439 440 static int __init iomap_ioend_init(void) 441 { 442 return bioset_init(&iomap_ioend_bioset, 4 * (PAGE_SIZE / SECTOR_SIZE), 443 offsetof(struct iomap_ioend, io_bio), 444 BIOSET_NEED_BVECS); 445 } 446 fs_initcall(iomap_ioend_init); 447