1 // SPDX-License-Identifier: GPL-2.0 2 #ifndef NO_BCACHEFS_FS 3 4 #include "bcachefs.h" 5 #include "alloc_foreground.h" 6 #include "bkey_buf.h" 7 #include "fs-io.h" 8 #include "fs-io-buffered.h" 9 #include "fs-io-direct.h" 10 #include "fs-io-pagecache.h" 11 #include "io_read.h" 12 #include "io_write.h" 13 14 #include <linux/backing-dev.h> 15 #include <linux/pagemap.h> 16 #include <linux/writeback.h> 17 18 static inline bool bio_full(struct bio *bio, unsigned len) 19 { 20 if (bio->bi_vcnt >= bio->bi_max_vecs) 21 return true; 22 if (bio->bi_iter.bi_size > UINT_MAX - len) 23 return true; 24 return false; 25 } 26 27 /* readpage(s): */ 28 29 static void bch2_readpages_end_io(struct bio *bio) 30 { 31 struct folio_iter fi; 32 33 bio_for_each_folio_all(fi, bio) 34 folio_end_read(fi.folio, bio->bi_status == BLK_STS_OK); 35 36 bio_put(bio); 37 } 38 39 struct readpages_iter { 40 struct address_space *mapping; 41 unsigned idx; 42 folios folios; 43 }; 44 45 static int readpages_iter_init(struct readpages_iter *iter, 46 struct readahead_control *ractl) 47 { 48 struct folio *folio; 49 50 *iter = (struct readpages_iter) { ractl->mapping }; 51 52 while ((folio = __readahead_folio(ractl))) { 53 if (!bch2_folio_create(folio, GFP_KERNEL) || 54 darray_push(&iter->folios, folio)) { 55 bch2_folio_release(folio); 56 ractl->_nr_pages += folio_nr_pages(folio); 57 ractl->_index -= folio_nr_pages(folio); 58 return iter->folios.nr ? 0 : -ENOMEM; 59 } 60 61 folio_put(folio); 62 } 63 64 return 0; 65 } 66 67 static inline struct folio *readpage_iter_peek(struct readpages_iter *iter) 68 { 69 if (iter->idx >= iter->folios.nr) 70 return NULL; 71 return iter->folios.data[iter->idx]; 72 } 73 74 static inline void readpage_iter_advance(struct readpages_iter *iter) 75 { 76 iter->idx++; 77 } 78 79 static bool extent_partial_reads_expensive(struct bkey_s_c k) 80 { 81 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); 82 struct bch_extent_crc_unpacked crc; 83 const union bch_extent_entry *i; 84 85 bkey_for_each_crc(k.k, ptrs, crc, i) 86 if (crc.csum_type || crc.compression_type) 87 return true; 88 return false; 89 } 90 91 static int readpage_bio_extend(struct btree_trans *trans, 92 struct readpages_iter *iter, 93 struct bio *bio, 94 unsigned sectors_this_extent, 95 bool get_more) 96 { 97 /* Don't hold btree locks while allocating memory: */ 98 bch2_trans_unlock(trans); 99 100 while (bio_sectors(bio) < sectors_this_extent && 101 bio->bi_vcnt < bio->bi_max_vecs) { 102 struct folio *folio = readpage_iter_peek(iter); 103 int ret; 104 105 if (folio) { 106 readpage_iter_advance(iter); 107 } else { 108 pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT; 109 110 if (!get_more) 111 break; 112 113 unsigned sectors_remaining = sectors_this_extent - bio_sectors(bio); 114 115 if (sectors_remaining < PAGE_SECTORS << mapping_min_folio_order(iter->mapping)) 116 break; 117 118 unsigned order = ilog2(rounddown_pow_of_two(sectors_remaining) / PAGE_SECTORS); 119 120 /* ensure proper alignment */ 121 order = min(order, __ffs(folio_offset|BIT(31))); 122 123 folio = xa_load(&iter->mapping->i_pages, folio_offset); 124 if (folio && !xa_is_value(folio)) 125 break; 126 127 folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), order); 128 if (!folio) 129 break; 130 131 if (!__bch2_folio_create(folio, GFP_KERNEL)) { 132 folio_put(folio); 133 break; 134 } 135 136 ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_KERNEL); 137 if (ret) { 138 __bch2_folio_release(folio); 139 folio_put(folio); 140 break; 141 } 142 143 folio_put(folio); 144 } 145 146 BUG_ON(folio_sector(folio) != bio_end_sector(bio)); 147 148 BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0)); 149 } 150 151 return bch2_trans_relock(trans); 152 } 153 154 static void bchfs_read(struct btree_trans *trans, 155 struct bch_read_bio *rbio, 156 subvol_inum inum, 157 struct readpages_iter *readpages_iter) 158 { 159 struct bch_fs *c = trans->c; 160 struct btree_iter iter; 161 struct bkey_buf sk; 162 int flags = BCH_READ_retry_if_stale| 163 BCH_READ_may_promote; 164 int ret = 0; 165 166 rbio->subvol = inum.subvol; 167 168 bch2_bkey_buf_init(&sk); 169 bch2_trans_begin(trans); 170 bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, 171 POS(inum.inum, rbio->bio.bi_iter.bi_sector), 172 BTREE_ITER_slots); 173 while (1) { 174 struct bkey_s_c k; 175 unsigned bytes, sectors; 176 s64 offset_into_extent; 177 enum btree_id data_btree = BTREE_ID_extents; 178 179 bch2_trans_begin(trans); 180 181 u32 snapshot; 182 ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot); 183 if (ret) 184 goto err; 185 186 bch2_btree_iter_set_snapshot(trans, &iter, snapshot); 187 188 bch2_btree_iter_set_pos(trans, &iter, 189 POS(inum.inum, rbio->bio.bi_iter.bi_sector)); 190 191 k = bch2_btree_iter_peek_slot(trans, &iter); 192 ret = bkey_err(k); 193 if (ret) 194 goto err; 195 196 offset_into_extent = iter.pos.offset - 197 bkey_start_offset(k.k); 198 sectors = k.k->size - offset_into_extent; 199 200 bch2_bkey_buf_reassemble(&sk, c, k); 201 202 ret = bch2_read_indirect_extent(trans, &data_btree, 203 &offset_into_extent, &sk); 204 if (ret) 205 goto err; 206 207 k = bkey_i_to_s_c(sk.k); 208 209 sectors = min_t(unsigned, sectors, k.k->size - offset_into_extent); 210 211 if (readpages_iter) { 212 ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors, 213 extent_partial_reads_expensive(k)); 214 if (ret) 215 goto err; 216 } 217 218 bytes = min(sectors, bio_sectors(&rbio->bio)) << 9; 219 swap(rbio->bio.bi_iter.bi_size, bytes); 220 221 if (rbio->bio.bi_iter.bi_size == bytes) 222 flags |= BCH_READ_last_fragment; 223 224 bch2_bio_page_state_set(&rbio->bio, k); 225 226 bch2_read_extent(trans, rbio, iter.pos, 227 data_btree, k, offset_into_extent, flags); 228 swap(rbio->bio.bi_iter.bi_size, bytes); 229 230 if (flags & BCH_READ_last_fragment) 231 break; 232 233 bio_advance(&rbio->bio, bytes); 234 err: 235 if (ret && 236 !bch2_err_matches(ret, BCH_ERR_transaction_restart)) 237 break; 238 } 239 bch2_trans_iter_exit(trans, &iter); 240 241 if (ret) { 242 struct printbuf buf = PRINTBUF; 243 lockrestart_do(trans, 244 bch2_inum_offset_err_msg_trans(trans, &buf, inum, iter.pos.offset << 9)); 245 prt_printf(&buf, "read error %i from btree lookup", ret); 246 bch_err_ratelimited(c, "%s", buf.buf); 247 printbuf_exit(&buf); 248 249 rbio->bio.bi_status = BLK_STS_IOERR; 250 bio_endio(&rbio->bio); 251 } 252 253 bch2_bkey_buf_exit(&sk, c); 254 } 255 256 void bch2_readahead(struct readahead_control *ractl) 257 { 258 struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host); 259 struct bch_fs *c = inode->v.i_sb->s_fs_info; 260 struct bch_io_opts opts; 261 struct folio *folio; 262 struct readpages_iter readpages_iter; 263 struct blk_plug plug; 264 265 bch2_inode_opts_get(&opts, c, &inode->ei_inode); 266 267 int ret = readpages_iter_init(&readpages_iter, ractl); 268 if (ret) 269 return; 270 271 /* 272 * Besides being a general performance optimization, plugging helps with 273 * avoiding btree transaction srcu warnings - submitting a bio can 274 * block, and we don't want todo that with the transaction locked. 275 * 276 * However, plugged bios are submitted when we schedule; we ideally 277 * would have our own scheduler hook to call unlock_long() before 278 * scheduling. 279 */ 280 blk_start_plug(&plug); 281 bch2_pagecache_add_get(inode); 282 283 struct btree_trans *trans = bch2_trans_get(c); 284 while ((folio = readpage_iter_peek(&readpages_iter))) { 285 unsigned n = min_t(unsigned, 286 readpages_iter.folios.nr - 287 readpages_iter.idx, 288 BIO_MAX_VECS); 289 struct bch_read_bio *rbio = 290 rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ, 291 GFP_KERNEL, &c->bio_read), 292 c, 293 opts, 294 bch2_readpages_end_io); 295 296 readpage_iter_advance(&readpages_iter); 297 298 rbio->bio.bi_iter.bi_sector = folio_sector(folio); 299 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0)); 300 301 bchfs_read(trans, rbio, inode_inum(inode), 302 &readpages_iter); 303 bch2_trans_unlock(trans); 304 } 305 bch2_trans_put(trans); 306 307 bch2_pagecache_add_put(inode); 308 blk_finish_plug(&plug); 309 darray_exit(&readpages_iter.folios); 310 } 311 312 static void bch2_read_single_folio_end_io(struct bio *bio) 313 { 314 complete(bio->bi_private); 315 } 316 317 int bch2_read_single_folio(struct folio *folio, struct address_space *mapping) 318 { 319 struct bch_inode_info *inode = to_bch_ei(mapping->host); 320 struct bch_fs *c = inode->v.i_sb->s_fs_info; 321 struct bch_read_bio *rbio; 322 struct bch_io_opts opts; 323 struct blk_plug plug; 324 int ret; 325 DECLARE_COMPLETION_ONSTACK(done); 326 327 BUG_ON(folio_test_uptodate(folio)); 328 BUG_ON(folio_test_dirty(folio)); 329 330 if (!bch2_folio_create(folio, GFP_KERNEL)) 331 return -ENOMEM; 332 333 bch2_inode_opts_get(&opts, c, &inode->ei_inode); 334 335 rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_KERNEL, &c->bio_read), 336 c, 337 opts, 338 bch2_read_single_folio_end_io); 339 rbio->bio.bi_private = &done; 340 rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC; 341 rbio->bio.bi_iter.bi_sector = folio_sector(folio); 342 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0)); 343 344 blk_start_plug(&plug); 345 bch2_trans_run(c, (bchfs_read(trans, rbio, inode_inum(inode), NULL), 0)); 346 blk_finish_plug(&plug); 347 wait_for_completion(&done); 348 349 ret = blk_status_to_errno(rbio->bio.bi_status); 350 bio_put(&rbio->bio); 351 352 if (ret < 0) 353 return ret; 354 355 folio_mark_uptodate(folio); 356 return 0; 357 } 358 359 int bch2_read_folio(struct file *file, struct folio *folio) 360 { 361 int ret; 362 363 ret = bch2_read_single_folio(folio, folio->mapping); 364 folio_unlock(folio); 365 return bch2_err_class(ret); 366 } 367 368 /* writepages: */ 369 370 struct bch_writepage_io { 371 struct bch_inode_info *inode; 372 373 /* must be last: */ 374 struct bch_write_op op; 375 }; 376 377 struct bch_writepage_state { 378 struct bch_writepage_io *io; 379 struct bch_io_opts opts; 380 struct bch_folio_sector *tmp; 381 unsigned tmp_sectors; 382 }; 383 384 static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c, 385 struct bch_inode_info *inode) 386 { 387 struct bch_writepage_state ret = { 0 }; 388 389 bch2_inode_opts_get(&ret.opts, c, &inode->ei_inode); 390 return ret; 391 } 392 393 /* 394 * Determine when a writepage io is full. We have to limit writepage bios to a 395 * single page per bvec (i.e. 1MB with 4k pages) because that is the limit to 396 * what the bounce path in bch2_write_extent() can handle. In theory we could 397 * loosen this restriction for non-bounce I/O, but we don't have that context 398 * here. Ideally, we can up this limit and make it configurable in the future 399 * when the bounce path can be enhanced to accommodate larger source bios. 400 */ 401 static inline bool bch_io_full(struct bch_writepage_io *io, unsigned len) 402 { 403 struct bio *bio = &io->op.wbio.bio; 404 return bio_full(bio, len) || 405 (bio->bi_iter.bi_size + len > BIO_MAX_VECS * PAGE_SIZE); 406 } 407 408 static void bch2_writepage_io_done(struct bch_write_op *op) 409 { 410 struct bch_writepage_io *io = 411 container_of(op, struct bch_writepage_io, op); 412 struct bch_fs *c = io->op.c; 413 struct bio *bio = &io->op.wbio.bio; 414 struct folio_iter fi; 415 unsigned i; 416 417 if (io->op.error) { 418 set_bit(EI_INODE_ERROR, &io->inode->ei_flags); 419 420 bio_for_each_folio_all(fi, bio) { 421 struct bch_folio *s; 422 423 mapping_set_error(fi.folio->mapping, -EIO); 424 425 s = __bch2_folio(fi.folio); 426 spin_lock(&s->lock); 427 for (i = 0; i < folio_sectors(fi.folio); i++) 428 s->s[i].nr_replicas = 0; 429 spin_unlock(&s->lock); 430 } 431 } 432 433 if (io->op.flags & BCH_WRITE_wrote_data_inline) { 434 bio_for_each_folio_all(fi, bio) { 435 struct bch_folio *s; 436 437 s = __bch2_folio(fi.folio); 438 spin_lock(&s->lock); 439 for (i = 0; i < folio_sectors(fi.folio); i++) 440 s->s[i].nr_replicas = 0; 441 spin_unlock(&s->lock); 442 } 443 } 444 445 /* 446 * racing with fallocate can cause us to add fewer sectors than 447 * expected - but we shouldn't add more sectors than expected: 448 */ 449 WARN_ON_ONCE(io->op.i_sectors_delta > 0); 450 451 /* 452 * (error (due to going RO) halfway through a page can screw that up 453 * slightly) 454 * XXX wtf? 455 BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS); 456 */ 457 458 /* 459 * The writeback flag is effectively our ref on the inode - 460 * fixup i_blocks before calling folio_end_writeback: 461 */ 462 bch2_i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta); 463 464 bio_for_each_folio_all(fi, bio) { 465 struct bch_folio *s = __bch2_folio(fi.folio); 466 467 if (atomic_dec_and_test(&s->write_count)) 468 folio_end_writeback(fi.folio); 469 } 470 471 bio_put(&io->op.wbio.bio); 472 } 473 474 static void bch2_writepage_do_io(struct bch_writepage_state *w) 475 { 476 struct bch_writepage_io *io = w->io; 477 478 w->io = NULL; 479 closure_call(&io->op.cl, bch2_write, NULL, NULL); 480 } 481 482 /* 483 * Get a bch_writepage_io and add @page to it - appending to an existing one if 484 * possible, else allocating a new one: 485 */ 486 static void bch2_writepage_io_alloc(struct bch_fs *c, 487 struct writeback_control *wbc, 488 struct bch_writepage_state *w, 489 struct bch_inode_info *inode, 490 u64 sector, 491 unsigned nr_replicas) 492 { 493 struct bch_write_op *op; 494 495 w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS, 496 REQ_OP_WRITE, 497 GFP_KERNEL, 498 &c->writepage_bioset), 499 struct bch_writepage_io, op.wbio.bio); 500 501 w->io->inode = inode; 502 op = &w->io->op; 503 bch2_write_op_init(op, c, w->opts); 504 op->target = w->opts.foreground_target; 505 op->nr_replicas = nr_replicas; 506 op->res.nr_replicas = nr_replicas; 507 op->write_point = writepoint_hashed(inode->ei_last_dirtied); 508 op->subvol = inode->ei_inum.subvol; 509 op->pos = POS(inode->v.i_ino, sector); 510 op->end_io = bch2_writepage_io_done; 511 op->devs_need_flush = &inode->ei_devs_need_flush; 512 op->wbio.bio.bi_iter.bi_sector = sector; 513 op->wbio.bio.bi_opf = wbc_to_write_flags(wbc); 514 } 515 516 static int __bch2_writepage(struct folio *folio, 517 struct writeback_control *wbc, 518 void *data) 519 { 520 struct bch_inode_info *inode = to_bch_ei(folio->mapping->host); 521 struct bch_fs *c = inode->v.i_sb->s_fs_info; 522 struct bch_writepage_state *w = data; 523 struct bch_folio *s; 524 unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX; 525 loff_t i_size = i_size_read(&inode->v); 526 int ret; 527 528 EBUG_ON(!folio_test_uptodate(folio)); 529 530 /* Is the folio fully inside i_size? */ 531 if (folio_end_pos(folio) <= i_size) 532 goto do_io; 533 534 /* Is the folio fully outside i_size? (truncate in progress) */ 535 if (folio_pos(folio) >= i_size) { 536 folio_unlock(folio); 537 return 0; 538 } 539 540 /* 541 * The folio straddles i_size. It must be zeroed out on each and every 542 * writepage invocation because it may be mmapped. "A file is mapped 543 * in multiples of the folio size. For a file that is not a multiple of 544 * the folio size, the remaining memory is zeroed when mapped, and 545 * writes to that region are not written out to the file." 546 */ 547 folio_zero_segment(folio, 548 i_size - folio_pos(folio), 549 folio_size(folio)); 550 do_io: 551 f_sectors = folio_sectors(folio); 552 s = bch2_folio(folio); 553 554 if (f_sectors > w->tmp_sectors) { 555 kfree(w->tmp); 556 w->tmp = kcalloc(f_sectors, sizeof(struct bch_folio_sector), GFP_NOFS|__GFP_NOFAIL); 557 w->tmp_sectors = f_sectors; 558 } 559 560 /* 561 * Things get really hairy with errors during writeback: 562 */ 563 ret = bch2_get_folio_disk_reservation(c, inode, folio, false); 564 BUG_ON(ret); 565 566 /* Before unlocking the page, get copy of reservations: */ 567 spin_lock(&s->lock); 568 memcpy(w->tmp, s->s, sizeof(struct bch_folio_sector) * f_sectors); 569 570 for (i = 0; i < f_sectors; i++) { 571 if (s->s[i].state < SECTOR_dirty) 572 continue; 573 574 nr_replicas_this_write = 575 min_t(unsigned, nr_replicas_this_write, 576 s->s[i].nr_replicas + 577 s->s[i].replicas_reserved); 578 } 579 580 for (i = 0; i < f_sectors; i++) { 581 if (s->s[i].state < SECTOR_dirty) 582 continue; 583 584 s->s[i].nr_replicas = w->opts.compression 585 ? 0 : nr_replicas_this_write; 586 587 s->s[i].replicas_reserved = 0; 588 bch2_folio_sector_set(folio, s, i, SECTOR_allocated); 589 } 590 spin_unlock(&s->lock); 591 592 BUG_ON(atomic_read(&s->write_count)); 593 atomic_set(&s->write_count, 1); 594 595 BUG_ON(folio_test_writeback(folio)); 596 folio_start_writeback(folio); 597 598 folio_unlock(folio); 599 600 offset = 0; 601 while (1) { 602 unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0; 603 u64 sector; 604 605 while (offset < f_sectors && 606 w->tmp[offset].state < SECTOR_dirty) 607 offset++; 608 609 if (offset == f_sectors) 610 break; 611 612 while (offset + sectors < f_sectors && 613 w->tmp[offset + sectors].state >= SECTOR_dirty) { 614 reserved_sectors += w->tmp[offset + sectors].replicas_reserved; 615 dirty_sectors += w->tmp[offset + sectors].state == SECTOR_dirty; 616 sectors++; 617 } 618 BUG_ON(!sectors); 619 620 sector = folio_sector(folio) + offset; 621 622 if (w->io && 623 (w->io->op.res.nr_replicas != nr_replicas_this_write || 624 bch_io_full(w->io, sectors << 9) || 625 bio_end_sector(&w->io->op.wbio.bio) != sector)) 626 bch2_writepage_do_io(w); 627 628 if (!w->io) 629 bch2_writepage_io_alloc(c, wbc, w, inode, sector, 630 nr_replicas_this_write); 631 632 atomic_inc(&s->write_count); 633 634 BUG_ON(inode != w->io->inode); 635 BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio, 636 sectors << 9, offset << 9)); 637 638 w->io->op.res.sectors += reserved_sectors; 639 w->io->op.i_sectors_delta -= dirty_sectors; 640 w->io->op.new_i_size = i_size; 641 642 offset += sectors; 643 } 644 645 if (atomic_dec_and_test(&s->write_count)) 646 folio_end_writeback(folio); 647 648 return 0; 649 } 650 651 int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc) 652 { 653 struct bch_fs *c = mapping->host->i_sb->s_fs_info; 654 struct bch_writepage_state w = 655 bch_writepage_state_init(c, to_bch_ei(mapping->host)); 656 struct blk_plug plug; 657 int ret; 658 659 blk_start_plug(&plug); 660 ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w); 661 if (w.io) 662 bch2_writepage_do_io(&w); 663 blk_finish_plug(&plug); 664 kfree(w.tmp); 665 return bch2_err_class(ret); 666 } 667 668 /* buffered writes: */ 669 670 int bch2_write_begin(struct file *file, struct address_space *mapping, 671 loff_t pos, unsigned len, 672 struct folio **foliop, void **fsdata) 673 { 674 struct bch_inode_info *inode = to_bch_ei(mapping->host); 675 struct bch_fs *c = inode->v.i_sb->s_fs_info; 676 struct bch2_folio_reservation *res; 677 struct folio *folio; 678 unsigned offset; 679 int ret = -ENOMEM; 680 681 res = kmalloc(sizeof(*res), GFP_KERNEL); 682 if (!res) 683 return -ENOMEM; 684 685 bch2_folio_reservation_init(c, inode, res); 686 *fsdata = res; 687 688 bch2_pagecache_add_get(inode); 689 690 folio = __filemap_get_folio(mapping, pos >> PAGE_SHIFT, 691 FGP_WRITEBEGIN | fgf_set_order(len), 692 mapping_gfp_mask(mapping)); 693 if (IS_ERR(folio)) 694 goto err_unlock; 695 696 offset = pos - folio_pos(folio); 697 len = min_t(size_t, len, folio_end_pos(folio) - pos); 698 699 if (folio_test_uptodate(folio)) 700 goto out; 701 702 /* If we're writing entire folio, don't need to read it in first: */ 703 if (!offset && len == folio_size(folio)) 704 goto out; 705 706 if (!offset && pos + len >= inode->v.i_size) { 707 folio_zero_segment(folio, len, folio_size(folio)); 708 flush_dcache_folio(folio); 709 goto out; 710 } 711 712 if (folio_pos(folio) >= inode->v.i_size) { 713 folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio)); 714 flush_dcache_folio(folio); 715 goto out; 716 } 717 readpage: 718 ret = bch2_read_single_folio(folio, mapping); 719 if (ret) 720 goto err; 721 out: 722 ret = bch2_folio_set(c, inode_inum(inode), &folio, 1); 723 if (ret) 724 goto err; 725 726 ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len); 727 if (ret) { 728 if (!folio_test_uptodate(folio)) { 729 /* 730 * If the folio hasn't been read in, we won't know if we 731 * actually need a reservation - we don't actually need 732 * to read here, we just need to check if the folio is 733 * fully backed by uncompressed data: 734 */ 735 goto readpage; 736 } 737 738 goto err; 739 } 740 741 *foliop = folio; 742 return 0; 743 err: 744 folio_unlock(folio); 745 folio_put(folio); 746 err_unlock: 747 bch2_pagecache_add_put(inode); 748 kfree(res); 749 *fsdata = NULL; 750 return bch2_err_class(ret); 751 } 752 753 int bch2_write_end(struct file *file, struct address_space *mapping, 754 loff_t pos, unsigned len, unsigned copied, 755 struct folio *folio, void *fsdata) 756 { 757 struct bch_inode_info *inode = to_bch_ei(mapping->host); 758 struct bch_fs *c = inode->v.i_sb->s_fs_info; 759 struct bch2_folio_reservation *res = fsdata; 760 unsigned offset = pos - folio_pos(folio); 761 762 lockdep_assert_held(&inode->v.i_rwsem); 763 BUG_ON(offset + copied > folio_size(folio)); 764 765 if (unlikely(copied < len && !folio_test_uptodate(folio))) { 766 /* 767 * The folio needs to be read in, but that would destroy 768 * our partial write - simplest thing is to just force 769 * userspace to redo the write: 770 */ 771 folio_zero_range(folio, 0, folio_size(folio)); 772 flush_dcache_folio(folio); 773 copied = 0; 774 } 775 776 spin_lock(&inode->v.i_lock); 777 if (pos + copied > inode->v.i_size) 778 i_size_write(&inode->v, pos + copied); 779 spin_unlock(&inode->v.i_lock); 780 781 if (copied) { 782 if (!folio_test_uptodate(folio)) 783 folio_mark_uptodate(folio); 784 785 bch2_set_folio_dirty(c, inode, folio, res, offset, copied); 786 787 inode->ei_last_dirtied = (unsigned long) current; 788 } 789 790 folio_unlock(folio); 791 folio_put(folio); 792 bch2_pagecache_add_put(inode); 793 794 bch2_folio_reservation_put(c, inode, res); 795 kfree(res); 796 797 return copied; 798 } 799 800 static noinline void folios_trunc(folios *fs, struct folio **fi) 801 { 802 while (fs->data + fs->nr > fi) { 803 struct folio *f = darray_pop(fs); 804 805 folio_unlock(f); 806 folio_put(f); 807 } 808 } 809 810 static int __bch2_buffered_write(struct bch_inode_info *inode, 811 struct address_space *mapping, 812 struct iov_iter *iter, 813 loff_t pos, unsigned len) 814 { 815 struct bch_fs *c = inode->v.i_sb->s_fs_info; 816 struct bch2_folio_reservation res; 817 folios fs; 818 struct folio *f; 819 unsigned copied = 0, f_offset, f_copied; 820 u64 end = pos + len, f_pos, f_len; 821 loff_t last_folio_pos = inode->v.i_size; 822 int ret = 0; 823 824 BUG_ON(!len); 825 826 bch2_folio_reservation_init(c, inode, &res); 827 darray_init(&fs); 828 829 ret = bch2_filemap_get_contig_folios_d(mapping, pos, end, 830 FGP_WRITEBEGIN | fgf_set_order(len), 831 mapping_gfp_mask(mapping), &fs); 832 if (ret) 833 goto out; 834 835 BUG_ON(!fs.nr); 836 837 f = darray_first(fs); 838 if (pos != folio_pos(f) && !folio_test_uptodate(f)) { 839 ret = bch2_read_single_folio(f, mapping); 840 if (ret) 841 goto out; 842 } 843 844 f = darray_last(fs); 845 end = min(end, folio_end_pos(f)); 846 last_folio_pos = folio_pos(f); 847 if (end != folio_end_pos(f) && !folio_test_uptodate(f)) { 848 if (end >= inode->v.i_size) { 849 folio_zero_range(f, 0, folio_size(f)); 850 } else { 851 ret = bch2_read_single_folio(f, mapping); 852 if (ret) 853 goto out; 854 } 855 } 856 857 ret = bch2_folio_set(c, inode_inum(inode), fs.data, fs.nr); 858 if (ret) 859 goto out; 860 861 f_pos = pos; 862 f_offset = pos - folio_pos(darray_first(fs)); 863 darray_for_each(fs, fi) { 864 ssize_t f_reserved; 865 866 f = *fi; 867 f_len = min(end, folio_end_pos(f)) - f_pos; 868 f_reserved = bch2_folio_reservation_get_partial(c, inode, f, &res, f_offset, f_len); 869 870 if (unlikely(f_reserved != f_len)) { 871 if (f_reserved < 0) { 872 if (f == darray_first(fs)) { 873 ret = f_reserved; 874 goto out; 875 } 876 877 folios_trunc(&fs, fi); 878 end = min(end, folio_end_pos(darray_last(fs))); 879 } else { 880 if (!folio_test_uptodate(f)) { 881 ret = bch2_read_single_folio(f, mapping); 882 if (ret) 883 goto out; 884 } 885 886 folios_trunc(&fs, fi + 1); 887 end = f_pos + f_reserved; 888 } 889 890 break; 891 } 892 893 f_pos = folio_end_pos(f); 894 f_offset = 0; 895 } 896 897 if (mapping_writably_mapped(mapping)) 898 darray_for_each(fs, fi) 899 flush_dcache_folio(*fi); 900 901 f_pos = pos; 902 f_offset = pos - folio_pos(darray_first(fs)); 903 darray_for_each(fs, fi) { 904 f = *fi; 905 f_len = min(end, folio_end_pos(f)) - f_pos; 906 f_copied = copy_folio_from_iter_atomic(f, f_offset, f_len, iter); 907 if (!f_copied) { 908 folios_trunc(&fs, fi); 909 break; 910 } 911 912 if (!folio_test_uptodate(f) && 913 f_copied != folio_size(f) && 914 pos + copied + f_copied < inode->v.i_size) { 915 iov_iter_revert(iter, f_copied); 916 folio_zero_range(f, 0, folio_size(f)); 917 folios_trunc(&fs, fi); 918 break; 919 } 920 921 flush_dcache_folio(f); 922 copied += f_copied; 923 924 if (f_copied != f_len) { 925 folios_trunc(&fs, fi + 1); 926 break; 927 } 928 929 f_pos = folio_end_pos(f); 930 f_offset = 0; 931 } 932 933 if (!copied) 934 goto out; 935 936 end = pos + copied; 937 938 spin_lock(&inode->v.i_lock); 939 if (end > inode->v.i_size) 940 i_size_write(&inode->v, end); 941 spin_unlock(&inode->v.i_lock); 942 943 f_pos = pos; 944 f_offset = pos - folio_pos(darray_first(fs)); 945 darray_for_each(fs, fi) { 946 f = *fi; 947 f_len = min(end, folio_end_pos(f)) - f_pos; 948 949 if (!folio_test_uptodate(f)) 950 folio_mark_uptodate(f); 951 952 bch2_set_folio_dirty(c, inode, f, &res, f_offset, f_len); 953 954 f_pos = folio_end_pos(f); 955 f_offset = 0; 956 } 957 958 inode->ei_last_dirtied = (unsigned long) current; 959 out: 960 darray_for_each(fs, fi) { 961 folio_unlock(*fi); 962 folio_put(*fi); 963 } 964 965 /* 966 * If the last folio added to the mapping starts beyond current EOF, we 967 * performed a short write but left around at least one post-EOF folio. 968 * Clean up the mapping before we return. 969 */ 970 if (last_folio_pos >= inode->v.i_size) 971 truncate_pagecache(&inode->v, inode->v.i_size); 972 973 darray_exit(&fs); 974 bch2_folio_reservation_put(c, inode, &res); 975 976 return copied ?: ret; 977 } 978 979 static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter) 980 { 981 struct file *file = iocb->ki_filp; 982 struct address_space *mapping = file->f_mapping; 983 struct bch_inode_info *inode = file_bch_inode(file); 984 loff_t pos = iocb->ki_pos; 985 ssize_t written = 0; 986 int ret = 0; 987 988 bch2_pagecache_add_get(inode); 989 990 do { 991 unsigned offset = pos & (PAGE_SIZE - 1); 992 unsigned bytes = iov_iter_count(iter); 993 again: 994 /* 995 * Bring in the user page that we will copy from _first_. 996 * Otherwise there's a nasty deadlock on copying from the 997 * same page as we're writing to, without it being marked 998 * up-to-date. 999 * 1000 * Not only is this an optimisation, but it is also required 1001 * to check that the address is actually valid, when atomic 1002 * usercopies are used, below. 1003 */ 1004 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) { 1005 bytes = min_t(unsigned long, iov_iter_count(iter), 1006 PAGE_SIZE - offset); 1007 1008 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) { 1009 ret = -EFAULT; 1010 break; 1011 } 1012 } 1013 1014 if (unlikely(fatal_signal_pending(current))) { 1015 ret = -EINTR; 1016 break; 1017 } 1018 1019 ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes); 1020 if (unlikely(ret < 0)) 1021 break; 1022 1023 cond_resched(); 1024 1025 if (unlikely(ret == 0)) { 1026 /* 1027 * If we were unable to copy any data at all, we must 1028 * fall back to a single segment length write. 1029 * 1030 * If we didn't fallback here, we could livelock 1031 * because not all segments in the iov can be copied at 1032 * once without a pagefault. 1033 */ 1034 bytes = min_t(unsigned long, PAGE_SIZE - offset, 1035 iov_iter_single_seg_count(iter)); 1036 goto again; 1037 } 1038 pos += ret; 1039 written += ret; 1040 ret = 0; 1041 1042 balance_dirty_pages_ratelimited(mapping); 1043 } while (iov_iter_count(iter)); 1044 1045 bch2_pagecache_add_put(inode); 1046 1047 return written ? written : ret; 1048 } 1049 1050 ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from) 1051 { 1052 struct file *file = iocb->ki_filp; 1053 struct bch_inode_info *inode = file_bch_inode(file); 1054 ssize_t ret; 1055 1056 if (iocb->ki_flags & IOCB_DIRECT) { 1057 ret = bch2_direct_write(iocb, from); 1058 goto out; 1059 } 1060 1061 inode_lock(&inode->v); 1062 1063 ret = generic_write_checks(iocb, from); 1064 if (ret <= 0) 1065 goto unlock; 1066 1067 ret = file_remove_privs(file); 1068 if (ret) 1069 goto unlock; 1070 1071 ret = file_update_time(file); 1072 if (ret) 1073 goto unlock; 1074 1075 ret = bch2_buffered_write(iocb, from); 1076 if (likely(ret > 0)) 1077 iocb->ki_pos += ret; 1078 unlock: 1079 inode_unlock(&inode->v); 1080 1081 if (ret > 0) 1082 ret = generic_write_sync(iocb, ret); 1083 out: 1084 return bch2_err_class(ret); 1085 } 1086 1087 void bch2_fs_fs_io_buffered_exit(struct bch_fs *c) 1088 { 1089 bioset_exit(&c->writepage_bioset); 1090 } 1091 1092 int bch2_fs_fs_io_buffered_init(struct bch_fs *c) 1093 { 1094 if (bioset_init(&c->writepage_bioset, 1095 4, offsetof(struct bch_writepage_io, op.wbio.bio), 1096 BIOSET_NEED_BVECS)) 1097 return -BCH_ERR_ENOMEM_writepage_bioset_init; 1098 1099 return 0; 1100 } 1101 1102 #endif /* NO_BCACHEFS_FS */ 1103