1 // SPDX-License-Identifier: GPL-2.0 2 #ifndef NO_BCACHEFS_FS 3 4 #include "bcachefs.h" 5 #include "alloc_foreground.h" 6 #include "bkey_buf.h" 7 #include "fs-io.h" 8 #include "fs-io-buffered.h" 9 #include "fs-io-direct.h" 10 #include "fs-io-pagecache.h" 11 #include "io_read.h" 12 #include "io_write.h" 13 14 #include <linux/backing-dev.h> 15 #include <linux/pagemap.h> 16 #include <linux/writeback.h> 17 18 static inline bool bio_full(struct bio *bio, unsigned len) 19 { 20 if (bio->bi_vcnt >= bio->bi_max_vecs) 21 return true; 22 if (bio->bi_iter.bi_size > UINT_MAX - len) 23 return true; 24 return false; 25 } 26 27 /* readpage(s): */ 28 29 static void bch2_readpages_end_io(struct bio *bio) 30 { 31 struct folio_iter fi; 32 33 bio_for_each_folio_all(fi, bio) 34 folio_end_read(fi.folio, bio->bi_status == BLK_STS_OK); 35 36 bio_put(bio); 37 } 38 39 struct readpages_iter { 40 struct address_space *mapping; 41 unsigned idx; 42 folios folios; 43 }; 44 45 static int readpages_iter_init(struct readpages_iter *iter, 46 struct readahead_control *ractl) 47 { 48 struct folio *folio; 49 50 *iter = (struct readpages_iter) { ractl->mapping }; 51 52 while ((folio = __readahead_folio(ractl))) { 53 if (!bch2_folio_create(folio, GFP_KERNEL) || 54 darray_push(&iter->folios, folio)) { 55 bch2_folio_release(folio); 56 ractl->_nr_pages += folio_nr_pages(folio); 57 ractl->_index -= folio_nr_pages(folio); 58 return iter->folios.nr ? 0 : -ENOMEM; 59 } 60 61 folio_put(folio); 62 } 63 64 return 0; 65 } 66 67 static inline struct folio *readpage_iter_peek(struct readpages_iter *iter) 68 { 69 if (iter->idx >= iter->folios.nr) 70 return NULL; 71 return iter->folios.data[iter->idx]; 72 } 73 74 static inline void readpage_iter_advance(struct readpages_iter *iter) 75 { 76 iter->idx++; 77 } 78 79 static bool extent_partial_reads_expensive(struct bkey_s_c k) 80 { 81 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); 82 struct bch_extent_crc_unpacked crc; 83 const union bch_extent_entry *i; 84 85 bkey_for_each_crc(k.k, ptrs, crc, i) 86 if (crc.csum_type || crc.compression_type) 87 return true; 88 return false; 89 } 90 91 static int readpage_bio_extend(struct btree_trans *trans, 92 struct readpages_iter *iter, 93 struct bio *bio, 94 unsigned sectors_this_extent, 95 bool get_more) 96 { 97 /* Don't hold btree locks while allocating memory: */ 98 bch2_trans_unlock(trans); 99 100 while (bio_sectors(bio) < sectors_this_extent && 101 bio->bi_vcnt < bio->bi_max_vecs) { 102 struct folio *folio = readpage_iter_peek(iter); 103 int ret; 104 105 if (folio) { 106 readpage_iter_advance(iter); 107 } else { 108 pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT; 109 110 if (!get_more) 111 break; 112 113 folio = xa_load(&iter->mapping->i_pages, folio_offset); 114 if (folio && !xa_is_value(folio)) 115 break; 116 117 folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0); 118 if (!folio) 119 break; 120 121 if (!__bch2_folio_create(folio, GFP_KERNEL)) { 122 folio_put(folio); 123 break; 124 } 125 126 ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_KERNEL); 127 if (ret) { 128 __bch2_folio_release(folio); 129 folio_put(folio); 130 break; 131 } 132 133 folio_put(folio); 134 } 135 136 BUG_ON(folio_sector(folio) != bio_end_sector(bio)); 137 138 BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0)); 139 } 140 141 return bch2_trans_relock(trans); 142 } 143 144 static void bchfs_read(struct btree_trans *trans, 145 struct bch_read_bio *rbio, 146 subvol_inum inum, 147 struct readpages_iter *readpages_iter) 148 { 149 struct bch_fs *c = trans->c; 150 struct btree_iter iter; 151 struct bkey_buf sk; 152 int flags = BCH_READ_RETRY_IF_STALE| 153 BCH_READ_MAY_PROMOTE; 154 int ret = 0; 155 156 rbio->c = c; 157 rbio->start_time = local_clock(); 158 rbio->subvol = inum.subvol; 159 160 bch2_bkey_buf_init(&sk); 161 bch2_trans_begin(trans); 162 bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, 163 POS(inum.inum, rbio->bio.bi_iter.bi_sector), 164 BTREE_ITER_slots); 165 while (1) { 166 struct bkey_s_c k; 167 unsigned bytes, sectors, offset_into_extent; 168 enum btree_id data_btree = BTREE_ID_extents; 169 170 bch2_trans_begin(trans); 171 172 u32 snapshot; 173 ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot); 174 if (ret) 175 goto err; 176 177 bch2_btree_iter_set_snapshot(&iter, snapshot); 178 179 bch2_btree_iter_set_pos(&iter, 180 POS(inum.inum, rbio->bio.bi_iter.bi_sector)); 181 182 k = bch2_btree_iter_peek_slot(&iter); 183 ret = bkey_err(k); 184 if (ret) 185 goto err; 186 187 offset_into_extent = iter.pos.offset - 188 bkey_start_offset(k.k); 189 sectors = k.k->size - offset_into_extent; 190 191 bch2_bkey_buf_reassemble(&sk, c, k); 192 193 ret = bch2_read_indirect_extent(trans, &data_btree, 194 &offset_into_extent, &sk); 195 if (ret) 196 goto err; 197 198 k = bkey_i_to_s_c(sk.k); 199 200 sectors = min(sectors, k.k->size - offset_into_extent); 201 202 if (readpages_iter) { 203 ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors, 204 extent_partial_reads_expensive(k)); 205 if (ret) 206 goto err; 207 } 208 209 bytes = min(sectors, bio_sectors(&rbio->bio)) << 9; 210 swap(rbio->bio.bi_iter.bi_size, bytes); 211 212 if (rbio->bio.bi_iter.bi_size == bytes) 213 flags |= BCH_READ_LAST_FRAGMENT; 214 215 bch2_bio_page_state_set(&rbio->bio, k); 216 217 bch2_read_extent(trans, rbio, iter.pos, 218 data_btree, k, offset_into_extent, flags); 219 220 if (flags & BCH_READ_LAST_FRAGMENT) 221 break; 222 223 swap(rbio->bio.bi_iter.bi_size, bytes); 224 bio_advance(&rbio->bio, bytes); 225 err: 226 if (ret && 227 !bch2_err_matches(ret, BCH_ERR_transaction_restart)) 228 break; 229 } 230 bch2_trans_iter_exit(trans, &iter); 231 232 if (ret) { 233 bch_err_inum_offset_ratelimited(c, 234 iter.pos.inode, 235 iter.pos.offset << 9, 236 "read error %i from btree lookup", ret); 237 rbio->bio.bi_status = BLK_STS_IOERR; 238 bio_endio(&rbio->bio); 239 } 240 241 bch2_bkey_buf_exit(&sk, c); 242 } 243 244 void bch2_readahead(struct readahead_control *ractl) 245 { 246 struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host); 247 struct bch_fs *c = inode->v.i_sb->s_fs_info; 248 struct bch_io_opts opts; 249 struct folio *folio; 250 struct readpages_iter readpages_iter; 251 252 bch2_inode_opts_get(&opts, c, &inode->ei_inode); 253 254 int ret = readpages_iter_init(&readpages_iter, ractl); 255 if (ret) 256 return; 257 258 bch2_pagecache_add_get(inode); 259 260 struct btree_trans *trans = bch2_trans_get(c); 261 while ((folio = readpage_iter_peek(&readpages_iter))) { 262 unsigned n = min_t(unsigned, 263 readpages_iter.folios.nr - 264 readpages_iter.idx, 265 BIO_MAX_VECS); 266 struct bch_read_bio *rbio = 267 rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ, 268 GFP_KERNEL, &c->bio_read), 269 opts); 270 271 readpage_iter_advance(&readpages_iter); 272 273 rbio->bio.bi_iter.bi_sector = folio_sector(folio); 274 rbio->bio.bi_end_io = bch2_readpages_end_io; 275 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0)); 276 277 bchfs_read(trans, rbio, inode_inum(inode), 278 &readpages_iter); 279 bch2_trans_unlock(trans); 280 } 281 bch2_trans_put(trans); 282 283 bch2_pagecache_add_put(inode); 284 285 darray_exit(&readpages_iter.folios); 286 } 287 288 static void bch2_read_single_folio_end_io(struct bio *bio) 289 { 290 complete(bio->bi_private); 291 } 292 293 int bch2_read_single_folio(struct folio *folio, struct address_space *mapping) 294 { 295 struct bch_inode_info *inode = to_bch_ei(mapping->host); 296 struct bch_fs *c = inode->v.i_sb->s_fs_info; 297 struct bch_read_bio *rbio; 298 struct bch_io_opts opts; 299 int ret; 300 DECLARE_COMPLETION_ONSTACK(done); 301 302 if (!bch2_folio_create(folio, GFP_KERNEL)) 303 return -ENOMEM; 304 305 bch2_inode_opts_get(&opts, c, &inode->ei_inode); 306 307 rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_KERNEL, &c->bio_read), 308 opts); 309 rbio->bio.bi_private = &done; 310 rbio->bio.bi_end_io = bch2_read_single_folio_end_io; 311 312 rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC; 313 rbio->bio.bi_iter.bi_sector = folio_sector(folio); 314 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0)); 315 316 bch2_trans_run(c, (bchfs_read(trans, rbio, inode_inum(inode), NULL), 0)); 317 wait_for_completion(&done); 318 319 ret = blk_status_to_errno(rbio->bio.bi_status); 320 bio_put(&rbio->bio); 321 322 if (ret < 0) 323 return ret; 324 325 folio_mark_uptodate(folio); 326 return 0; 327 } 328 329 int bch2_read_folio(struct file *file, struct folio *folio) 330 { 331 int ret; 332 333 ret = bch2_read_single_folio(folio, folio->mapping); 334 folio_unlock(folio); 335 return bch2_err_class(ret); 336 } 337 338 /* writepages: */ 339 340 struct bch_writepage_io { 341 struct bch_inode_info *inode; 342 343 /* must be last: */ 344 struct bch_write_op op; 345 }; 346 347 struct bch_writepage_state { 348 struct bch_writepage_io *io; 349 struct bch_io_opts opts; 350 struct bch_folio_sector *tmp; 351 unsigned tmp_sectors; 352 }; 353 354 static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c, 355 struct bch_inode_info *inode) 356 { 357 struct bch_writepage_state ret = { 0 }; 358 359 bch2_inode_opts_get(&ret.opts, c, &inode->ei_inode); 360 return ret; 361 } 362 363 /* 364 * Determine when a writepage io is full. We have to limit writepage bios to a 365 * single page per bvec (i.e. 1MB with 4k pages) because that is the limit to 366 * what the bounce path in bch2_write_extent() can handle. In theory we could 367 * loosen this restriction for non-bounce I/O, but we don't have that context 368 * here. Ideally, we can up this limit and make it configurable in the future 369 * when the bounce path can be enhanced to accommodate larger source bios. 370 */ 371 static inline bool bch_io_full(struct bch_writepage_io *io, unsigned len) 372 { 373 struct bio *bio = &io->op.wbio.bio; 374 return bio_full(bio, len) || 375 (bio->bi_iter.bi_size + len > BIO_MAX_VECS * PAGE_SIZE); 376 } 377 378 static void bch2_writepage_io_done(struct bch_write_op *op) 379 { 380 struct bch_writepage_io *io = 381 container_of(op, struct bch_writepage_io, op); 382 struct bch_fs *c = io->op.c; 383 struct bio *bio = &io->op.wbio.bio; 384 struct folio_iter fi; 385 unsigned i; 386 387 if (io->op.error) { 388 set_bit(EI_INODE_ERROR, &io->inode->ei_flags); 389 390 bio_for_each_folio_all(fi, bio) { 391 struct bch_folio *s; 392 393 mapping_set_error(fi.folio->mapping, -EIO); 394 395 s = __bch2_folio(fi.folio); 396 spin_lock(&s->lock); 397 for (i = 0; i < folio_sectors(fi.folio); i++) 398 s->s[i].nr_replicas = 0; 399 spin_unlock(&s->lock); 400 } 401 } 402 403 if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) { 404 bio_for_each_folio_all(fi, bio) { 405 struct bch_folio *s; 406 407 s = __bch2_folio(fi.folio); 408 spin_lock(&s->lock); 409 for (i = 0; i < folio_sectors(fi.folio); i++) 410 s->s[i].nr_replicas = 0; 411 spin_unlock(&s->lock); 412 } 413 } 414 415 /* 416 * racing with fallocate can cause us to add fewer sectors than 417 * expected - but we shouldn't add more sectors than expected: 418 */ 419 WARN_ON_ONCE(io->op.i_sectors_delta > 0); 420 421 /* 422 * (error (due to going RO) halfway through a page can screw that up 423 * slightly) 424 * XXX wtf? 425 BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS); 426 */ 427 428 /* 429 * The writeback flag is effectively our ref on the inode - 430 * fixup i_blocks before calling folio_end_writeback: 431 */ 432 bch2_i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta); 433 434 bio_for_each_folio_all(fi, bio) { 435 struct bch_folio *s = __bch2_folio(fi.folio); 436 437 if (atomic_dec_and_test(&s->write_count)) 438 folio_end_writeback(fi.folio); 439 } 440 441 bio_put(&io->op.wbio.bio); 442 } 443 444 static void bch2_writepage_do_io(struct bch_writepage_state *w) 445 { 446 struct bch_writepage_io *io = w->io; 447 448 w->io = NULL; 449 closure_call(&io->op.cl, bch2_write, NULL, NULL); 450 } 451 452 /* 453 * Get a bch_writepage_io and add @page to it - appending to an existing one if 454 * possible, else allocating a new one: 455 */ 456 static void bch2_writepage_io_alloc(struct bch_fs *c, 457 struct writeback_control *wbc, 458 struct bch_writepage_state *w, 459 struct bch_inode_info *inode, 460 u64 sector, 461 unsigned nr_replicas) 462 { 463 struct bch_write_op *op; 464 465 w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS, 466 REQ_OP_WRITE, 467 GFP_KERNEL, 468 &c->writepage_bioset), 469 struct bch_writepage_io, op.wbio.bio); 470 471 w->io->inode = inode; 472 op = &w->io->op; 473 bch2_write_op_init(op, c, w->opts); 474 op->target = w->opts.foreground_target; 475 op->nr_replicas = nr_replicas; 476 op->res.nr_replicas = nr_replicas; 477 op->write_point = writepoint_hashed(inode->ei_last_dirtied); 478 op->subvol = inode->ei_inum.subvol; 479 op->pos = POS(inode->v.i_ino, sector); 480 op->end_io = bch2_writepage_io_done; 481 op->devs_need_flush = &inode->ei_devs_need_flush; 482 op->wbio.bio.bi_iter.bi_sector = sector; 483 op->wbio.bio.bi_opf = wbc_to_write_flags(wbc); 484 } 485 486 static int __bch2_writepage(struct folio *folio, 487 struct writeback_control *wbc, 488 void *data) 489 { 490 struct bch_inode_info *inode = to_bch_ei(folio->mapping->host); 491 struct bch_fs *c = inode->v.i_sb->s_fs_info; 492 struct bch_writepage_state *w = data; 493 struct bch_folio *s; 494 unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX; 495 loff_t i_size = i_size_read(&inode->v); 496 int ret; 497 498 EBUG_ON(!folio_test_uptodate(folio)); 499 500 /* Is the folio fully inside i_size? */ 501 if (folio_end_pos(folio) <= i_size) 502 goto do_io; 503 504 /* Is the folio fully outside i_size? (truncate in progress) */ 505 if (folio_pos(folio) >= i_size) { 506 folio_unlock(folio); 507 return 0; 508 } 509 510 /* 511 * The folio straddles i_size. It must be zeroed out on each and every 512 * writepage invocation because it may be mmapped. "A file is mapped 513 * in multiples of the folio size. For a file that is not a multiple of 514 * the folio size, the remaining memory is zeroed when mapped, and 515 * writes to that region are not written out to the file." 516 */ 517 folio_zero_segment(folio, 518 i_size - folio_pos(folio), 519 folio_size(folio)); 520 do_io: 521 f_sectors = folio_sectors(folio); 522 s = bch2_folio(folio); 523 524 if (f_sectors > w->tmp_sectors) { 525 kfree(w->tmp); 526 w->tmp = kcalloc(f_sectors, sizeof(struct bch_folio_sector), GFP_NOFS|__GFP_NOFAIL); 527 w->tmp_sectors = f_sectors; 528 } 529 530 /* 531 * Things get really hairy with errors during writeback: 532 */ 533 ret = bch2_get_folio_disk_reservation(c, inode, folio, false); 534 BUG_ON(ret); 535 536 /* Before unlocking the page, get copy of reservations: */ 537 spin_lock(&s->lock); 538 memcpy(w->tmp, s->s, sizeof(struct bch_folio_sector) * f_sectors); 539 540 for (i = 0; i < f_sectors; i++) { 541 if (s->s[i].state < SECTOR_dirty) 542 continue; 543 544 nr_replicas_this_write = 545 min_t(unsigned, nr_replicas_this_write, 546 s->s[i].nr_replicas + 547 s->s[i].replicas_reserved); 548 } 549 550 for (i = 0; i < f_sectors; i++) { 551 if (s->s[i].state < SECTOR_dirty) 552 continue; 553 554 s->s[i].nr_replicas = w->opts.compression 555 ? 0 : nr_replicas_this_write; 556 557 s->s[i].replicas_reserved = 0; 558 bch2_folio_sector_set(folio, s, i, SECTOR_allocated); 559 } 560 spin_unlock(&s->lock); 561 562 BUG_ON(atomic_read(&s->write_count)); 563 atomic_set(&s->write_count, 1); 564 565 BUG_ON(folio_test_writeback(folio)); 566 folio_start_writeback(folio); 567 568 folio_unlock(folio); 569 570 offset = 0; 571 while (1) { 572 unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0; 573 u64 sector; 574 575 while (offset < f_sectors && 576 w->tmp[offset].state < SECTOR_dirty) 577 offset++; 578 579 if (offset == f_sectors) 580 break; 581 582 while (offset + sectors < f_sectors && 583 w->tmp[offset + sectors].state >= SECTOR_dirty) { 584 reserved_sectors += w->tmp[offset + sectors].replicas_reserved; 585 dirty_sectors += w->tmp[offset + sectors].state == SECTOR_dirty; 586 sectors++; 587 } 588 BUG_ON(!sectors); 589 590 sector = folio_sector(folio) + offset; 591 592 if (w->io && 593 (w->io->op.res.nr_replicas != nr_replicas_this_write || 594 bch_io_full(w->io, sectors << 9) || 595 bio_end_sector(&w->io->op.wbio.bio) != sector)) 596 bch2_writepage_do_io(w); 597 598 if (!w->io) 599 bch2_writepage_io_alloc(c, wbc, w, inode, sector, 600 nr_replicas_this_write); 601 602 atomic_inc(&s->write_count); 603 604 BUG_ON(inode != w->io->inode); 605 BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio, 606 sectors << 9, offset << 9)); 607 608 /* Check for writing past i_size: */ 609 WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) > 610 round_up(i_size, block_bytes(c)) && 611 !test_bit(BCH_FS_emergency_ro, &c->flags), 612 "writing past i_size: %llu > %llu (unrounded %llu)\n", 613 bio_end_sector(&w->io->op.wbio.bio) << 9, 614 round_up(i_size, block_bytes(c)), 615 i_size); 616 617 w->io->op.res.sectors += reserved_sectors; 618 w->io->op.i_sectors_delta -= dirty_sectors; 619 w->io->op.new_i_size = i_size; 620 621 offset += sectors; 622 } 623 624 if (atomic_dec_and_test(&s->write_count)) 625 folio_end_writeback(folio); 626 627 return 0; 628 } 629 630 int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc) 631 { 632 struct bch_fs *c = mapping->host->i_sb->s_fs_info; 633 struct bch_writepage_state w = 634 bch_writepage_state_init(c, to_bch_ei(mapping->host)); 635 struct blk_plug plug; 636 int ret; 637 638 blk_start_plug(&plug); 639 ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w); 640 if (w.io) 641 bch2_writepage_do_io(&w); 642 blk_finish_plug(&plug); 643 kfree(w.tmp); 644 return bch2_err_class(ret); 645 } 646 647 /* buffered writes: */ 648 649 int bch2_write_begin(struct file *file, struct address_space *mapping, 650 loff_t pos, unsigned len, 651 struct folio **foliop, void **fsdata) 652 { 653 struct bch_inode_info *inode = to_bch_ei(mapping->host); 654 struct bch_fs *c = inode->v.i_sb->s_fs_info; 655 struct bch2_folio_reservation *res; 656 struct folio *folio; 657 unsigned offset; 658 int ret = -ENOMEM; 659 660 res = kmalloc(sizeof(*res), GFP_KERNEL); 661 if (!res) 662 return -ENOMEM; 663 664 bch2_folio_reservation_init(c, inode, res); 665 *fsdata = res; 666 667 bch2_pagecache_add_get(inode); 668 669 folio = __filemap_get_folio(mapping, pos >> PAGE_SHIFT, 670 FGP_WRITEBEGIN | fgf_set_order(len), 671 mapping_gfp_mask(mapping)); 672 if (IS_ERR_OR_NULL(folio)) 673 goto err_unlock; 674 675 offset = pos - folio_pos(folio); 676 len = min_t(size_t, len, folio_end_pos(folio) - pos); 677 678 if (folio_test_uptodate(folio)) 679 goto out; 680 681 /* If we're writing entire folio, don't need to read it in first: */ 682 if (!offset && len == folio_size(folio)) 683 goto out; 684 685 if (!offset && pos + len >= inode->v.i_size) { 686 folio_zero_segment(folio, len, folio_size(folio)); 687 flush_dcache_folio(folio); 688 goto out; 689 } 690 691 if (folio_pos(folio) >= inode->v.i_size) { 692 folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio)); 693 flush_dcache_folio(folio); 694 goto out; 695 } 696 readpage: 697 ret = bch2_read_single_folio(folio, mapping); 698 if (ret) 699 goto err; 700 out: 701 ret = bch2_folio_set(c, inode_inum(inode), &folio, 1); 702 if (ret) 703 goto err; 704 705 ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len); 706 if (ret) { 707 if (!folio_test_uptodate(folio)) { 708 /* 709 * If the folio hasn't been read in, we won't know if we 710 * actually need a reservation - we don't actually need 711 * to read here, we just need to check if the folio is 712 * fully backed by uncompressed data: 713 */ 714 goto readpage; 715 } 716 717 goto err; 718 } 719 720 *foliop = folio; 721 return 0; 722 err: 723 folio_unlock(folio); 724 folio_put(folio); 725 err_unlock: 726 bch2_pagecache_add_put(inode); 727 kfree(res); 728 *fsdata = NULL; 729 return bch2_err_class(ret); 730 } 731 732 int bch2_write_end(struct file *file, struct address_space *mapping, 733 loff_t pos, unsigned len, unsigned copied, 734 struct folio *folio, void *fsdata) 735 { 736 struct bch_inode_info *inode = to_bch_ei(mapping->host); 737 struct bch_fs *c = inode->v.i_sb->s_fs_info; 738 struct bch2_folio_reservation *res = fsdata; 739 unsigned offset = pos - folio_pos(folio); 740 741 lockdep_assert_held(&inode->v.i_rwsem); 742 BUG_ON(offset + copied > folio_size(folio)); 743 744 if (unlikely(copied < len && !folio_test_uptodate(folio))) { 745 /* 746 * The folio needs to be read in, but that would destroy 747 * our partial write - simplest thing is to just force 748 * userspace to redo the write: 749 */ 750 folio_zero_range(folio, 0, folio_size(folio)); 751 flush_dcache_folio(folio); 752 copied = 0; 753 } 754 755 spin_lock(&inode->v.i_lock); 756 if (pos + copied > inode->v.i_size) 757 i_size_write(&inode->v, pos + copied); 758 spin_unlock(&inode->v.i_lock); 759 760 if (copied) { 761 if (!folio_test_uptodate(folio)) 762 folio_mark_uptodate(folio); 763 764 bch2_set_folio_dirty(c, inode, folio, res, offset, copied); 765 766 inode->ei_last_dirtied = (unsigned long) current; 767 } 768 769 folio_unlock(folio); 770 folio_put(folio); 771 bch2_pagecache_add_put(inode); 772 773 bch2_folio_reservation_put(c, inode, res); 774 kfree(res); 775 776 return copied; 777 } 778 779 static noinline void folios_trunc(folios *fs, struct folio **fi) 780 { 781 while (fs->data + fs->nr > fi) { 782 struct folio *f = darray_pop(fs); 783 784 folio_unlock(f); 785 folio_put(f); 786 } 787 } 788 789 static int __bch2_buffered_write(struct bch_inode_info *inode, 790 struct address_space *mapping, 791 struct iov_iter *iter, 792 loff_t pos, unsigned len) 793 { 794 struct bch_fs *c = inode->v.i_sb->s_fs_info; 795 struct bch2_folio_reservation res; 796 folios fs; 797 struct folio *f; 798 unsigned copied = 0, f_offset, f_copied; 799 u64 end = pos + len, f_pos, f_len; 800 loff_t last_folio_pos = inode->v.i_size; 801 int ret = 0; 802 803 BUG_ON(!len); 804 805 bch2_folio_reservation_init(c, inode, &res); 806 darray_init(&fs); 807 808 ret = bch2_filemap_get_contig_folios_d(mapping, pos, end, 809 FGP_WRITEBEGIN | fgf_set_order(len), 810 mapping_gfp_mask(mapping), &fs); 811 if (ret) 812 goto out; 813 814 BUG_ON(!fs.nr); 815 816 f = darray_first(fs); 817 if (pos != folio_pos(f) && !folio_test_uptodate(f)) { 818 ret = bch2_read_single_folio(f, mapping); 819 if (ret) 820 goto out; 821 } 822 823 f = darray_last(fs); 824 end = min(end, folio_end_pos(f)); 825 last_folio_pos = folio_pos(f); 826 if (end != folio_end_pos(f) && !folio_test_uptodate(f)) { 827 if (end >= inode->v.i_size) { 828 folio_zero_range(f, 0, folio_size(f)); 829 } else { 830 ret = bch2_read_single_folio(f, mapping); 831 if (ret) 832 goto out; 833 } 834 } 835 836 ret = bch2_folio_set(c, inode_inum(inode), fs.data, fs.nr); 837 if (ret) 838 goto out; 839 840 f_pos = pos; 841 f_offset = pos - folio_pos(darray_first(fs)); 842 darray_for_each(fs, fi) { 843 ssize_t f_reserved; 844 845 f = *fi; 846 f_len = min(end, folio_end_pos(f)) - f_pos; 847 f_reserved = bch2_folio_reservation_get_partial(c, inode, f, &res, f_offset, f_len); 848 849 if (unlikely(f_reserved != f_len)) { 850 if (f_reserved < 0) { 851 if (f == darray_first(fs)) { 852 ret = f_reserved; 853 goto out; 854 } 855 856 folios_trunc(&fs, fi); 857 end = min(end, folio_end_pos(darray_last(fs))); 858 } else { 859 if (!folio_test_uptodate(f)) { 860 ret = bch2_read_single_folio(f, mapping); 861 if (ret) 862 goto out; 863 } 864 865 folios_trunc(&fs, fi + 1); 866 end = f_pos + f_reserved; 867 } 868 869 break; 870 } 871 872 f_pos = folio_end_pos(f); 873 f_offset = 0; 874 } 875 876 if (mapping_writably_mapped(mapping)) 877 darray_for_each(fs, fi) 878 flush_dcache_folio(*fi); 879 880 f_pos = pos; 881 f_offset = pos - folio_pos(darray_first(fs)); 882 darray_for_each(fs, fi) { 883 f = *fi; 884 f_len = min(end, folio_end_pos(f)) - f_pos; 885 f_copied = copy_folio_from_iter_atomic(f, f_offset, f_len, iter); 886 if (!f_copied) { 887 folios_trunc(&fs, fi); 888 break; 889 } 890 891 if (!folio_test_uptodate(f) && 892 f_copied != folio_size(f) && 893 pos + copied + f_copied < inode->v.i_size) { 894 iov_iter_revert(iter, f_copied); 895 folio_zero_range(f, 0, folio_size(f)); 896 folios_trunc(&fs, fi); 897 break; 898 } 899 900 flush_dcache_folio(f); 901 copied += f_copied; 902 903 if (f_copied != f_len) { 904 folios_trunc(&fs, fi + 1); 905 break; 906 } 907 908 f_pos = folio_end_pos(f); 909 f_offset = 0; 910 } 911 912 if (!copied) 913 goto out; 914 915 end = pos + copied; 916 917 spin_lock(&inode->v.i_lock); 918 if (end > inode->v.i_size) 919 i_size_write(&inode->v, end); 920 spin_unlock(&inode->v.i_lock); 921 922 f_pos = pos; 923 f_offset = pos - folio_pos(darray_first(fs)); 924 darray_for_each(fs, fi) { 925 f = *fi; 926 f_len = min(end, folio_end_pos(f)) - f_pos; 927 928 if (!folio_test_uptodate(f)) 929 folio_mark_uptodate(f); 930 931 bch2_set_folio_dirty(c, inode, f, &res, f_offset, f_len); 932 933 f_pos = folio_end_pos(f); 934 f_offset = 0; 935 } 936 937 inode->ei_last_dirtied = (unsigned long) current; 938 out: 939 darray_for_each(fs, fi) { 940 folio_unlock(*fi); 941 folio_put(*fi); 942 } 943 944 /* 945 * If the last folio added to the mapping starts beyond current EOF, we 946 * performed a short write but left around at least one post-EOF folio. 947 * Clean up the mapping before we return. 948 */ 949 if (last_folio_pos >= inode->v.i_size) 950 truncate_pagecache(&inode->v, inode->v.i_size); 951 952 darray_exit(&fs); 953 bch2_folio_reservation_put(c, inode, &res); 954 955 return copied ?: ret; 956 } 957 958 static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter) 959 { 960 struct file *file = iocb->ki_filp; 961 struct address_space *mapping = file->f_mapping; 962 struct bch_inode_info *inode = file_bch_inode(file); 963 loff_t pos = iocb->ki_pos; 964 ssize_t written = 0; 965 int ret = 0; 966 967 bch2_pagecache_add_get(inode); 968 969 do { 970 unsigned offset = pos & (PAGE_SIZE - 1); 971 unsigned bytes = iov_iter_count(iter); 972 again: 973 /* 974 * Bring in the user page that we will copy from _first_. 975 * Otherwise there's a nasty deadlock on copying from the 976 * same page as we're writing to, without it being marked 977 * up-to-date. 978 * 979 * Not only is this an optimisation, but it is also required 980 * to check that the address is actually valid, when atomic 981 * usercopies are used, below. 982 */ 983 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) { 984 bytes = min_t(unsigned long, iov_iter_count(iter), 985 PAGE_SIZE - offset); 986 987 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) { 988 ret = -EFAULT; 989 break; 990 } 991 } 992 993 if (unlikely(fatal_signal_pending(current))) { 994 ret = -EINTR; 995 break; 996 } 997 998 ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes); 999 if (unlikely(ret < 0)) 1000 break; 1001 1002 cond_resched(); 1003 1004 if (unlikely(ret == 0)) { 1005 /* 1006 * If we were unable to copy any data at all, we must 1007 * fall back to a single segment length write. 1008 * 1009 * If we didn't fallback here, we could livelock 1010 * because not all segments in the iov can be copied at 1011 * once without a pagefault. 1012 */ 1013 bytes = min_t(unsigned long, PAGE_SIZE - offset, 1014 iov_iter_single_seg_count(iter)); 1015 goto again; 1016 } 1017 pos += ret; 1018 written += ret; 1019 ret = 0; 1020 1021 balance_dirty_pages_ratelimited(mapping); 1022 } while (iov_iter_count(iter)); 1023 1024 bch2_pagecache_add_put(inode); 1025 1026 return written ? written : ret; 1027 } 1028 1029 ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from) 1030 { 1031 struct file *file = iocb->ki_filp; 1032 struct bch_inode_info *inode = file_bch_inode(file); 1033 ssize_t ret; 1034 1035 if (iocb->ki_flags & IOCB_DIRECT) { 1036 ret = bch2_direct_write(iocb, from); 1037 goto out; 1038 } 1039 1040 inode_lock(&inode->v); 1041 1042 ret = generic_write_checks(iocb, from); 1043 if (ret <= 0) 1044 goto unlock; 1045 1046 ret = file_remove_privs(file); 1047 if (ret) 1048 goto unlock; 1049 1050 ret = file_update_time(file); 1051 if (ret) 1052 goto unlock; 1053 1054 ret = bch2_buffered_write(iocb, from); 1055 if (likely(ret > 0)) 1056 iocb->ki_pos += ret; 1057 unlock: 1058 inode_unlock(&inode->v); 1059 1060 if (ret > 0) 1061 ret = generic_write_sync(iocb, ret); 1062 out: 1063 return bch2_err_class(ret); 1064 } 1065 1066 void bch2_fs_fs_io_buffered_exit(struct bch_fs *c) 1067 { 1068 bioset_exit(&c->writepage_bioset); 1069 } 1070 1071 int bch2_fs_fs_io_buffered_init(struct bch_fs *c) 1072 { 1073 if (bioset_init(&c->writepage_bioset, 1074 4, offsetof(struct bch_writepage_io, op.wbio.bio), 1075 BIOSET_NEED_BVECS)) 1076 return -BCH_ERR_ENOMEM_writepage_bioset_init; 1077 1078 return 0; 1079 } 1080 1081 #endif /* NO_BCACHEFS_FS */ 1082