1 // SPDX-License-Identifier: GPL-2.0 2 #ifndef NO_BCACHEFS_FS 3 4 #include "bcachefs.h" 5 #include "alloc_foreground.h" 6 #include "bkey_buf.h" 7 #include "fs-io.h" 8 #include "fs-io-buffered.h" 9 #include "fs-io-direct.h" 10 #include "fs-io-pagecache.h" 11 #include "io_read.h" 12 #include "io_write.h" 13 14 #include <linux/backing-dev.h> 15 #include <linux/pagemap.h> 16 #include <linux/writeback.h> 17 18 static inline bool bio_full(struct bio *bio, unsigned len) 19 { 20 if (bio->bi_vcnt >= bio->bi_max_vecs) 21 return true; 22 if (bio->bi_iter.bi_size > UINT_MAX - len) 23 return true; 24 return false; 25 } 26 27 /* readpage(s): */ 28 29 static void bch2_readpages_end_io(struct bio *bio) 30 { 31 struct folio_iter fi; 32 33 bio_for_each_folio_all(fi, bio) { 34 if (!bio->bi_status) { 35 folio_mark_uptodate(fi.folio); 36 } else { 37 folio_clear_uptodate(fi.folio); 38 folio_set_error(fi.folio); 39 } 40 folio_unlock(fi.folio); 41 } 42 43 bio_put(bio); 44 } 45 46 struct readpages_iter { 47 struct address_space *mapping; 48 unsigned idx; 49 folios folios; 50 }; 51 52 static int readpages_iter_init(struct readpages_iter *iter, 53 struct readahead_control *ractl) 54 { 55 struct folio **fi; 56 int ret; 57 58 memset(iter, 0, sizeof(*iter)); 59 60 iter->mapping = ractl->mapping; 61 62 ret = bch2_filemap_get_contig_folios_d(iter->mapping, 63 ractl->_index << PAGE_SHIFT, 64 (ractl->_index + ractl->_nr_pages) << PAGE_SHIFT, 65 0, mapping_gfp_mask(iter->mapping), 66 &iter->folios); 67 if (ret) 68 return ret; 69 70 darray_for_each(iter->folios, fi) { 71 ractl->_nr_pages -= 1U << folio_order(*fi); 72 __bch2_folio_create(*fi, __GFP_NOFAIL|GFP_KERNEL); 73 folio_put(*fi); 74 folio_put(*fi); 75 } 76 77 return 0; 78 } 79 80 static inline struct folio *readpage_iter_peek(struct readpages_iter *iter) 81 { 82 if (iter->idx >= iter->folios.nr) 83 return NULL; 84 return iter->folios.data[iter->idx]; 85 } 86 87 static inline void readpage_iter_advance(struct readpages_iter *iter) 88 { 89 iter->idx++; 90 } 91 92 static bool extent_partial_reads_expensive(struct bkey_s_c k) 93 { 94 struct bkey_ptrs_c ptrs = bch2_bkey_ptrs_c(k); 95 struct bch_extent_crc_unpacked crc; 96 const union bch_extent_entry *i; 97 98 bkey_for_each_crc(k.k, ptrs, crc, i) 99 if (crc.csum_type || crc.compression_type) 100 return true; 101 return false; 102 } 103 104 static int readpage_bio_extend(struct btree_trans *trans, 105 struct readpages_iter *iter, 106 struct bio *bio, 107 unsigned sectors_this_extent, 108 bool get_more) 109 { 110 /* Don't hold btree locks while allocating memory: */ 111 bch2_trans_unlock(trans); 112 113 while (bio_sectors(bio) < sectors_this_extent && 114 bio->bi_vcnt < bio->bi_max_vecs) { 115 struct folio *folio = readpage_iter_peek(iter); 116 int ret; 117 118 if (folio) { 119 readpage_iter_advance(iter); 120 } else { 121 pgoff_t folio_offset = bio_end_sector(bio) >> PAGE_SECTORS_SHIFT; 122 123 if (!get_more) 124 break; 125 126 folio = xa_load(&iter->mapping->i_pages, folio_offset); 127 if (folio && !xa_is_value(folio)) 128 break; 129 130 folio = filemap_alloc_folio(readahead_gfp_mask(iter->mapping), 0); 131 if (!folio) 132 break; 133 134 if (!__bch2_folio_create(folio, GFP_KERNEL)) { 135 folio_put(folio); 136 break; 137 } 138 139 ret = filemap_add_folio(iter->mapping, folio, folio_offset, GFP_KERNEL); 140 if (ret) { 141 __bch2_folio_release(folio); 142 folio_put(folio); 143 break; 144 } 145 146 folio_put(folio); 147 } 148 149 BUG_ON(folio_sector(folio) != bio_end_sector(bio)); 150 151 BUG_ON(!bio_add_folio(bio, folio, folio_size(folio), 0)); 152 } 153 154 return bch2_trans_relock(trans); 155 } 156 157 static void bchfs_read(struct btree_trans *trans, 158 struct bch_read_bio *rbio, 159 subvol_inum inum, 160 struct readpages_iter *readpages_iter) 161 { 162 struct bch_fs *c = trans->c; 163 struct btree_iter iter; 164 struct bkey_buf sk; 165 int flags = BCH_READ_RETRY_IF_STALE| 166 BCH_READ_MAY_PROMOTE; 167 u32 snapshot; 168 int ret = 0; 169 170 rbio->c = c; 171 rbio->start_time = local_clock(); 172 rbio->subvol = inum.subvol; 173 174 bch2_bkey_buf_init(&sk); 175 retry: 176 bch2_trans_begin(trans); 177 iter = (struct btree_iter) { NULL }; 178 179 ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot); 180 if (ret) 181 goto err; 182 183 bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, 184 SPOS(inum.inum, rbio->bio.bi_iter.bi_sector, snapshot), 185 BTREE_ITER_SLOTS); 186 while (1) { 187 struct bkey_s_c k; 188 unsigned bytes, sectors, offset_into_extent; 189 enum btree_id data_btree = BTREE_ID_extents; 190 191 /* 192 * read_extent -> io_time_reset may cause a transaction restart 193 * without returning an error, we need to check for that here: 194 */ 195 ret = bch2_trans_relock(trans); 196 if (ret) 197 break; 198 199 bch2_btree_iter_set_pos(&iter, 200 POS(inum.inum, rbio->bio.bi_iter.bi_sector)); 201 202 k = bch2_btree_iter_peek_slot(&iter); 203 ret = bkey_err(k); 204 if (ret) 205 break; 206 207 offset_into_extent = iter.pos.offset - 208 bkey_start_offset(k.k); 209 sectors = k.k->size - offset_into_extent; 210 211 bch2_bkey_buf_reassemble(&sk, c, k); 212 213 ret = bch2_read_indirect_extent(trans, &data_btree, 214 &offset_into_extent, &sk); 215 if (ret) 216 break; 217 218 k = bkey_i_to_s_c(sk.k); 219 220 sectors = min(sectors, k.k->size - offset_into_extent); 221 222 if (readpages_iter) { 223 ret = readpage_bio_extend(trans, readpages_iter, &rbio->bio, sectors, 224 extent_partial_reads_expensive(k)); 225 if (ret) 226 break; 227 } 228 229 bytes = min(sectors, bio_sectors(&rbio->bio)) << 9; 230 swap(rbio->bio.bi_iter.bi_size, bytes); 231 232 if (rbio->bio.bi_iter.bi_size == bytes) 233 flags |= BCH_READ_LAST_FRAGMENT; 234 235 bch2_bio_page_state_set(&rbio->bio, k); 236 237 bch2_read_extent(trans, rbio, iter.pos, 238 data_btree, k, offset_into_extent, flags); 239 240 if (flags & BCH_READ_LAST_FRAGMENT) 241 break; 242 243 swap(rbio->bio.bi_iter.bi_size, bytes); 244 bio_advance(&rbio->bio, bytes); 245 246 ret = btree_trans_too_many_iters(trans); 247 if (ret) 248 break; 249 } 250 err: 251 bch2_trans_iter_exit(trans, &iter); 252 253 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) 254 goto retry; 255 256 if (ret) { 257 bch_err_inum_offset_ratelimited(c, 258 iter.pos.inode, 259 iter.pos.offset << 9, 260 "read error %i from btree lookup", ret); 261 rbio->bio.bi_status = BLK_STS_IOERR; 262 bio_endio(&rbio->bio); 263 } 264 265 bch2_bkey_buf_exit(&sk, c); 266 } 267 268 void bch2_readahead(struct readahead_control *ractl) 269 { 270 struct bch_inode_info *inode = to_bch_ei(ractl->mapping->host); 271 struct bch_fs *c = inode->v.i_sb->s_fs_info; 272 struct bch_io_opts opts; 273 struct btree_trans *trans = bch2_trans_get(c); 274 struct folio *folio; 275 struct readpages_iter readpages_iter; 276 int ret; 277 278 bch2_inode_opts_get(&opts, c, &inode->ei_inode); 279 280 ret = readpages_iter_init(&readpages_iter, ractl); 281 BUG_ON(ret); 282 283 bch2_pagecache_add_get(inode); 284 285 while ((folio = readpage_iter_peek(&readpages_iter))) { 286 unsigned n = min_t(unsigned, 287 readpages_iter.folios.nr - 288 readpages_iter.idx, 289 BIO_MAX_VECS); 290 struct bch_read_bio *rbio = 291 rbio_init(bio_alloc_bioset(NULL, n, REQ_OP_READ, 292 GFP_KERNEL, &c->bio_read), 293 opts); 294 295 readpage_iter_advance(&readpages_iter); 296 297 rbio->bio.bi_iter.bi_sector = folio_sector(folio); 298 rbio->bio.bi_end_io = bch2_readpages_end_io; 299 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0)); 300 301 bchfs_read(trans, rbio, inode_inum(inode), 302 &readpages_iter); 303 bch2_trans_unlock(trans); 304 } 305 306 bch2_pagecache_add_put(inode); 307 308 bch2_trans_put(trans); 309 darray_exit(&readpages_iter.folios); 310 } 311 312 static void __bchfs_readfolio(struct bch_fs *c, struct bch_read_bio *rbio, 313 subvol_inum inum, struct folio *folio) 314 { 315 bch2_folio_create(folio, __GFP_NOFAIL); 316 317 rbio->bio.bi_opf = REQ_OP_READ|REQ_SYNC; 318 rbio->bio.bi_iter.bi_sector = folio_sector(folio); 319 BUG_ON(!bio_add_folio(&rbio->bio, folio, folio_size(folio), 0)); 320 321 bch2_trans_run(c, (bchfs_read(trans, rbio, inum, NULL), 0)); 322 } 323 324 static void bch2_read_single_folio_end_io(struct bio *bio) 325 { 326 complete(bio->bi_private); 327 } 328 329 int bch2_read_single_folio(struct folio *folio, struct address_space *mapping) 330 { 331 struct bch_inode_info *inode = to_bch_ei(mapping->host); 332 struct bch_fs *c = inode->v.i_sb->s_fs_info; 333 struct bch_read_bio *rbio; 334 struct bch_io_opts opts; 335 int ret; 336 DECLARE_COMPLETION_ONSTACK(done); 337 338 bch2_inode_opts_get(&opts, c, &inode->ei_inode); 339 340 rbio = rbio_init(bio_alloc_bioset(NULL, 1, REQ_OP_READ, GFP_KERNEL, &c->bio_read), 341 opts); 342 rbio->bio.bi_private = &done; 343 rbio->bio.bi_end_io = bch2_read_single_folio_end_io; 344 345 __bchfs_readfolio(c, rbio, inode_inum(inode), folio); 346 wait_for_completion(&done); 347 348 ret = blk_status_to_errno(rbio->bio.bi_status); 349 bio_put(&rbio->bio); 350 351 if (ret < 0) 352 return ret; 353 354 folio_mark_uptodate(folio); 355 return 0; 356 } 357 358 int bch2_read_folio(struct file *file, struct folio *folio) 359 { 360 int ret; 361 362 ret = bch2_read_single_folio(folio, folio->mapping); 363 folio_unlock(folio); 364 return bch2_err_class(ret); 365 } 366 367 /* writepages: */ 368 369 struct bch_writepage_io { 370 struct bch_inode_info *inode; 371 372 /* must be last: */ 373 struct bch_write_op op; 374 }; 375 376 struct bch_writepage_state { 377 struct bch_writepage_io *io; 378 struct bch_io_opts opts; 379 struct bch_folio_sector *tmp; 380 unsigned tmp_sectors; 381 }; 382 383 static inline struct bch_writepage_state bch_writepage_state_init(struct bch_fs *c, 384 struct bch_inode_info *inode) 385 { 386 struct bch_writepage_state ret = { 0 }; 387 388 bch2_inode_opts_get(&ret.opts, c, &inode->ei_inode); 389 return ret; 390 } 391 392 /* 393 * Determine when a writepage io is full. We have to limit writepage bios to a 394 * single page per bvec (i.e. 1MB with 4k pages) because that is the limit to 395 * what the bounce path in bch2_write_extent() can handle. In theory we could 396 * loosen this restriction for non-bounce I/O, but we don't have that context 397 * here. Ideally, we can up this limit and make it configurable in the future 398 * when the bounce path can be enhanced to accommodate larger source bios. 399 */ 400 static inline bool bch_io_full(struct bch_writepage_io *io, unsigned len) 401 { 402 struct bio *bio = &io->op.wbio.bio; 403 return bio_full(bio, len) || 404 (bio->bi_iter.bi_size + len > BIO_MAX_VECS * PAGE_SIZE); 405 } 406 407 static void bch2_writepage_io_done(struct bch_write_op *op) 408 { 409 struct bch_writepage_io *io = 410 container_of(op, struct bch_writepage_io, op); 411 struct bch_fs *c = io->op.c; 412 struct bio *bio = &io->op.wbio.bio; 413 struct folio_iter fi; 414 unsigned i; 415 416 if (io->op.error) { 417 set_bit(EI_INODE_ERROR, &io->inode->ei_flags); 418 419 bio_for_each_folio_all(fi, bio) { 420 struct bch_folio *s; 421 422 folio_set_error(fi.folio); 423 mapping_set_error(fi.folio->mapping, -EIO); 424 425 s = __bch2_folio(fi.folio); 426 spin_lock(&s->lock); 427 for (i = 0; i < folio_sectors(fi.folio); i++) 428 s->s[i].nr_replicas = 0; 429 spin_unlock(&s->lock); 430 } 431 } 432 433 if (io->op.flags & BCH_WRITE_WROTE_DATA_INLINE) { 434 bio_for_each_folio_all(fi, bio) { 435 struct bch_folio *s; 436 437 s = __bch2_folio(fi.folio); 438 spin_lock(&s->lock); 439 for (i = 0; i < folio_sectors(fi.folio); i++) 440 s->s[i].nr_replicas = 0; 441 spin_unlock(&s->lock); 442 } 443 } 444 445 /* 446 * racing with fallocate can cause us to add fewer sectors than 447 * expected - but we shouldn't add more sectors than expected: 448 */ 449 WARN_ON_ONCE(io->op.i_sectors_delta > 0); 450 451 /* 452 * (error (due to going RO) halfway through a page can screw that up 453 * slightly) 454 * XXX wtf? 455 BUG_ON(io->op.op.i_sectors_delta >= PAGE_SECTORS); 456 */ 457 458 /* 459 * PageWriteback is effectively our ref on the inode - fixup i_blocks 460 * before calling end_page_writeback: 461 */ 462 bch2_i_sectors_acct(c, io->inode, NULL, io->op.i_sectors_delta); 463 464 bio_for_each_folio_all(fi, bio) { 465 struct bch_folio *s = __bch2_folio(fi.folio); 466 467 if (atomic_dec_and_test(&s->write_count)) 468 folio_end_writeback(fi.folio); 469 } 470 471 bio_put(&io->op.wbio.bio); 472 } 473 474 static void bch2_writepage_do_io(struct bch_writepage_state *w) 475 { 476 struct bch_writepage_io *io = w->io; 477 478 w->io = NULL; 479 closure_call(&io->op.cl, bch2_write, NULL, NULL); 480 } 481 482 /* 483 * Get a bch_writepage_io and add @page to it - appending to an existing one if 484 * possible, else allocating a new one: 485 */ 486 static void bch2_writepage_io_alloc(struct bch_fs *c, 487 struct writeback_control *wbc, 488 struct bch_writepage_state *w, 489 struct bch_inode_info *inode, 490 u64 sector, 491 unsigned nr_replicas) 492 { 493 struct bch_write_op *op; 494 495 w->io = container_of(bio_alloc_bioset(NULL, BIO_MAX_VECS, 496 REQ_OP_WRITE, 497 GFP_KERNEL, 498 &c->writepage_bioset), 499 struct bch_writepage_io, op.wbio.bio); 500 501 w->io->inode = inode; 502 op = &w->io->op; 503 bch2_write_op_init(op, c, w->opts); 504 op->target = w->opts.foreground_target; 505 op->nr_replicas = nr_replicas; 506 op->res.nr_replicas = nr_replicas; 507 op->write_point = writepoint_hashed(inode->ei_last_dirtied); 508 op->subvol = inode->ei_subvol; 509 op->pos = POS(inode->v.i_ino, sector); 510 op->end_io = bch2_writepage_io_done; 511 op->devs_need_flush = &inode->ei_devs_need_flush; 512 op->wbio.bio.bi_iter.bi_sector = sector; 513 op->wbio.bio.bi_opf = wbc_to_write_flags(wbc); 514 } 515 516 static int __bch2_writepage(struct folio *folio, 517 struct writeback_control *wbc, 518 void *data) 519 { 520 struct bch_inode_info *inode = to_bch_ei(folio->mapping->host); 521 struct bch_fs *c = inode->v.i_sb->s_fs_info; 522 struct bch_writepage_state *w = data; 523 struct bch_folio *s; 524 unsigned i, offset, f_sectors, nr_replicas_this_write = U32_MAX; 525 loff_t i_size = i_size_read(&inode->v); 526 int ret; 527 528 EBUG_ON(!folio_test_uptodate(folio)); 529 530 /* Is the folio fully inside i_size? */ 531 if (folio_end_pos(folio) <= i_size) 532 goto do_io; 533 534 /* Is the folio fully outside i_size? (truncate in progress) */ 535 if (folio_pos(folio) >= i_size) { 536 folio_unlock(folio); 537 return 0; 538 } 539 540 /* 541 * The folio straddles i_size. It must be zeroed out on each and every 542 * writepage invocation because it may be mmapped. "A file is mapped 543 * in multiples of the folio size. For a file that is not a multiple of 544 * the folio size, the remaining memory is zeroed when mapped, and 545 * writes to that region are not written out to the file." 546 */ 547 folio_zero_segment(folio, 548 i_size - folio_pos(folio), 549 folio_size(folio)); 550 do_io: 551 f_sectors = folio_sectors(folio); 552 s = bch2_folio(folio); 553 554 if (f_sectors > w->tmp_sectors) { 555 kfree(w->tmp); 556 w->tmp = kcalloc(f_sectors, sizeof(struct bch_folio_sector), __GFP_NOFAIL); 557 w->tmp_sectors = f_sectors; 558 } 559 560 /* 561 * Things get really hairy with errors during writeback: 562 */ 563 ret = bch2_get_folio_disk_reservation(c, inode, folio, false); 564 BUG_ON(ret); 565 566 /* Before unlocking the page, get copy of reservations: */ 567 spin_lock(&s->lock); 568 memcpy(w->tmp, s->s, sizeof(struct bch_folio_sector) * f_sectors); 569 570 for (i = 0; i < f_sectors; i++) { 571 if (s->s[i].state < SECTOR_dirty) 572 continue; 573 574 nr_replicas_this_write = 575 min_t(unsigned, nr_replicas_this_write, 576 s->s[i].nr_replicas + 577 s->s[i].replicas_reserved); 578 } 579 580 for (i = 0; i < f_sectors; i++) { 581 if (s->s[i].state < SECTOR_dirty) 582 continue; 583 584 s->s[i].nr_replicas = w->opts.compression 585 ? 0 : nr_replicas_this_write; 586 587 s->s[i].replicas_reserved = 0; 588 bch2_folio_sector_set(folio, s, i, SECTOR_allocated); 589 } 590 spin_unlock(&s->lock); 591 592 BUG_ON(atomic_read(&s->write_count)); 593 atomic_set(&s->write_count, 1); 594 595 BUG_ON(folio_test_writeback(folio)); 596 folio_start_writeback(folio); 597 598 folio_unlock(folio); 599 600 offset = 0; 601 while (1) { 602 unsigned sectors = 0, dirty_sectors = 0, reserved_sectors = 0; 603 u64 sector; 604 605 while (offset < f_sectors && 606 w->tmp[offset].state < SECTOR_dirty) 607 offset++; 608 609 if (offset == f_sectors) 610 break; 611 612 while (offset + sectors < f_sectors && 613 w->tmp[offset + sectors].state >= SECTOR_dirty) { 614 reserved_sectors += w->tmp[offset + sectors].replicas_reserved; 615 dirty_sectors += w->tmp[offset + sectors].state == SECTOR_dirty; 616 sectors++; 617 } 618 BUG_ON(!sectors); 619 620 sector = folio_sector(folio) + offset; 621 622 if (w->io && 623 (w->io->op.res.nr_replicas != nr_replicas_this_write || 624 bch_io_full(w->io, sectors << 9) || 625 bio_end_sector(&w->io->op.wbio.bio) != sector)) 626 bch2_writepage_do_io(w); 627 628 if (!w->io) 629 bch2_writepage_io_alloc(c, wbc, w, inode, sector, 630 nr_replicas_this_write); 631 632 atomic_inc(&s->write_count); 633 634 BUG_ON(inode != w->io->inode); 635 BUG_ON(!bio_add_folio(&w->io->op.wbio.bio, folio, 636 sectors << 9, offset << 9)); 637 638 /* Check for writing past i_size: */ 639 WARN_ONCE((bio_end_sector(&w->io->op.wbio.bio) << 9) > 640 round_up(i_size, block_bytes(c)) && 641 !test_bit(BCH_FS_EMERGENCY_RO, &c->flags), 642 "writing past i_size: %llu > %llu (unrounded %llu)\n", 643 bio_end_sector(&w->io->op.wbio.bio) << 9, 644 round_up(i_size, block_bytes(c)), 645 i_size); 646 647 w->io->op.res.sectors += reserved_sectors; 648 w->io->op.i_sectors_delta -= dirty_sectors; 649 w->io->op.new_i_size = i_size; 650 651 offset += sectors; 652 } 653 654 if (atomic_dec_and_test(&s->write_count)) 655 folio_end_writeback(folio); 656 657 return 0; 658 } 659 660 int bch2_writepages(struct address_space *mapping, struct writeback_control *wbc) 661 { 662 struct bch_fs *c = mapping->host->i_sb->s_fs_info; 663 struct bch_writepage_state w = 664 bch_writepage_state_init(c, to_bch_ei(mapping->host)); 665 struct blk_plug plug; 666 int ret; 667 668 blk_start_plug(&plug); 669 ret = write_cache_pages(mapping, wbc, __bch2_writepage, &w); 670 if (w.io) 671 bch2_writepage_do_io(&w); 672 blk_finish_plug(&plug); 673 kfree(w.tmp); 674 return bch2_err_class(ret); 675 } 676 677 /* buffered writes: */ 678 679 int bch2_write_begin(struct file *file, struct address_space *mapping, 680 loff_t pos, unsigned len, 681 struct page **pagep, void **fsdata) 682 { 683 struct bch_inode_info *inode = to_bch_ei(mapping->host); 684 struct bch_fs *c = inode->v.i_sb->s_fs_info; 685 struct bch2_folio_reservation *res; 686 struct folio *folio; 687 unsigned offset; 688 int ret = -ENOMEM; 689 690 res = kmalloc(sizeof(*res), GFP_KERNEL); 691 if (!res) 692 return -ENOMEM; 693 694 bch2_folio_reservation_init(c, inode, res); 695 *fsdata = res; 696 697 bch2_pagecache_add_get(inode); 698 699 folio = __filemap_get_folio(mapping, pos >> PAGE_SHIFT, 700 FGP_LOCK|FGP_WRITE|FGP_CREAT|FGP_STABLE, 701 mapping_gfp_mask(mapping)); 702 if (IS_ERR_OR_NULL(folio)) 703 goto err_unlock; 704 705 offset = pos - folio_pos(folio); 706 len = min_t(size_t, len, folio_end_pos(folio) - pos); 707 708 if (folio_test_uptodate(folio)) 709 goto out; 710 711 /* If we're writing entire folio, don't need to read it in first: */ 712 if (!offset && len == folio_size(folio)) 713 goto out; 714 715 if (!offset && pos + len >= inode->v.i_size) { 716 folio_zero_segment(folio, len, folio_size(folio)); 717 flush_dcache_folio(folio); 718 goto out; 719 } 720 721 if (folio_pos(folio) >= inode->v.i_size) { 722 folio_zero_segments(folio, 0, offset, offset + len, folio_size(folio)); 723 flush_dcache_folio(folio); 724 goto out; 725 } 726 readpage: 727 ret = bch2_read_single_folio(folio, mapping); 728 if (ret) 729 goto err; 730 out: 731 ret = bch2_folio_set(c, inode_inum(inode), &folio, 1); 732 if (ret) 733 goto err; 734 735 ret = bch2_folio_reservation_get(c, inode, folio, res, offset, len); 736 if (ret) { 737 if (!folio_test_uptodate(folio)) { 738 /* 739 * If the folio hasn't been read in, we won't know if we 740 * actually need a reservation - we don't actually need 741 * to read here, we just need to check if the folio is 742 * fully backed by uncompressed data: 743 */ 744 goto readpage; 745 } 746 747 goto err; 748 } 749 750 *pagep = &folio->page; 751 return 0; 752 err: 753 folio_unlock(folio); 754 folio_put(folio); 755 *pagep = NULL; 756 err_unlock: 757 bch2_pagecache_add_put(inode); 758 kfree(res); 759 *fsdata = NULL; 760 return bch2_err_class(ret); 761 } 762 763 int bch2_write_end(struct file *file, struct address_space *mapping, 764 loff_t pos, unsigned len, unsigned copied, 765 struct page *page, void *fsdata) 766 { 767 struct bch_inode_info *inode = to_bch_ei(mapping->host); 768 struct bch_fs *c = inode->v.i_sb->s_fs_info; 769 struct bch2_folio_reservation *res = fsdata; 770 struct folio *folio = page_folio(page); 771 unsigned offset = pos - folio_pos(folio); 772 773 lockdep_assert_held(&inode->v.i_rwsem); 774 BUG_ON(offset + copied > folio_size(folio)); 775 776 if (unlikely(copied < len && !folio_test_uptodate(folio))) { 777 /* 778 * The folio needs to be read in, but that would destroy 779 * our partial write - simplest thing is to just force 780 * userspace to redo the write: 781 */ 782 folio_zero_range(folio, 0, folio_size(folio)); 783 flush_dcache_folio(folio); 784 copied = 0; 785 } 786 787 spin_lock(&inode->v.i_lock); 788 if (pos + copied > inode->v.i_size) 789 i_size_write(&inode->v, pos + copied); 790 spin_unlock(&inode->v.i_lock); 791 792 if (copied) { 793 if (!folio_test_uptodate(folio)) 794 folio_mark_uptodate(folio); 795 796 bch2_set_folio_dirty(c, inode, folio, res, offset, copied); 797 798 inode->ei_last_dirtied = (unsigned long) current; 799 } 800 801 folio_unlock(folio); 802 folio_put(folio); 803 bch2_pagecache_add_put(inode); 804 805 bch2_folio_reservation_put(c, inode, res); 806 kfree(res); 807 808 return copied; 809 } 810 811 static noinline void folios_trunc(folios *fs, struct folio **fi) 812 { 813 while (fs->data + fs->nr > fi) { 814 struct folio *f = darray_pop(fs); 815 816 folio_unlock(f); 817 folio_put(f); 818 } 819 } 820 821 static int __bch2_buffered_write(struct bch_inode_info *inode, 822 struct address_space *mapping, 823 struct iov_iter *iter, 824 loff_t pos, unsigned len) 825 { 826 struct bch_fs *c = inode->v.i_sb->s_fs_info; 827 struct bch2_folio_reservation res; 828 folios fs; 829 struct folio **fi, *f; 830 unsigned copied = 0, f_offset, f_copied; 831 u64 end = pos + len, f_pos, f_len; 832 loff_t last_folio_pos = inode->v.i_size; 833 int ret = 0; 834 835 BUG_ON(!len); 836 837 bch2_folio_reservation_init(c, inode, &res); 838 darray_init(&fs); 839 840 ret = bch2_filemap_get_contig_folios_d(mapping, pos, end, 841 FGP_LOCK|FGP_WRITE|FGP_STABLE|FGP_CREAT, 842 mapping_gfp_mask(mapping), 843 &fs); 844 if (ret) 845 goto out; 846 847 BUG_ON(!fs.nr); 848 849 f = darray_first(fs); 850 if (pos != folio_pos(f) && !folio_test_uptodate(f)) { 851 ret = bch2_read_single_folio(f, mapping); 852 if (ret) 853 goto out; 854 } 855 856 f = darray_last(fs); 857 end = min(end, folio_end_pos(f)); 858 last_folio_pos = folio_pos(f); 859 if (end != folio_end_pos(f) && !folio_test_uptodate(f)) { 860 if (end >= inode->v.i_size) { 861 folio_zero_range(f, 0, folio_size(f)); 862 } else { 863 ret = bch2_read_single_folio(f, mapping); 864 if (ret) 865 goto out; 866 } 867 } 868 869 ret = bch2_folio_set(c, inode_inum(inode), fs.data, fs.nr); 870 if (ret) 871 goto out; 872 873 f_pos = pos; 874 f_offset = pos - folio_pos(darray_first(fs)); 875 darray_for_each(fs, fi) { 876 f = *fi; 877 f_len = min(end, folio_end_pos(f)) - f_pos; 878 879 /* 880 * XXX: per POSIX and fstests generic/275, on -ENOSPC we're 881 * supposed to write as much as we have disk space for. 882 * 883 * On failure here we should still write out a partial page if 884 * we aren't completely out of disk space - we don't do that 885 * yet: 886 */ 887 ret = bch2_folio_reservation_get(c, inode, f, &res, f_offset, f_len); 888 if (unlikely(ret)) { 889 folios_trunc(&fs, fi); 890 if (!fs.nr) 891 goto out; 892 893 end = min(end, folio_end_pos(darray_last(fs))); 894 break; 895 } 896 897 f_pos = folio_end_pos(f); 898 f_offset = 0; 899 } 900 901 if (mapping_writably_mapped(mapping)) 902 darray_for_each(fs, fi) 903 flush_dcache_folio(*fi); 904 905 f_pos = pos; 906 f_offset = pos - folio_pos(darray_first(fs)); 907 darray_for_each(fs, fi) { 908 f = *fi; 909 f_len = min(end, folio_end_pos(f)) - f_pos; 910 f_copied = copy_page_from_iter_atomic(&f->page, f_offset, f_len, iter); 911 if (!f_copied) { 912 folios_trunc(&fs, fi); 913 break; 914 } 915 916 if (!folio_test_uptodate(f) && 917 f_copied != folio_size(f) && 918 pos + copied + f_copied < inode->v.i_size) { 919 iov_iter_revert(iter, f_copied); 920 folio_zero_range(f, 0, folio_size(f)); 921 folios_trunc(&fs, fi); 922 break; 923 } 924 925 flush_dcache_folio(f); 926 copied += f_copied; 927 928 if (f_copied != f_len) { 929 folios_trunc(&fs, fi + 1); 930 break; 931 } 932 933 f_pos = folio_end_pos(f); 934 f_offset = 0; 935 } 936 937 if (!copied) 938 goto out; 939 940 end = pos + copied; 941 942 spin_lock(&inode->v.i_lock); 943 if (end > inode->v.i_size) 944 i_size_write(&inode->v, end); 945 spin_unlock(&inode->v.i_lock); 946 947 f_pos = pos; 948 f_offset = pos - folio_pos(darray_first(fs)); 949 darray_for_each(fs, fi) { 950 f = *fi; 951 f_len = min(end, folio_end_pos(f)) - f_pos; 952 953 if (!folio_test_uptodate(f)) 954 folio_mark_uptodate(f); 955 956 bch2_set_folio_dirty(c, inode, f, &res, f_offset, f_len); 957 958 f_pos = folio_end_pos(f); 959 f_offset = 0; 960 } 961 962 inode->ei_last_dirtied = (unsigned long) current; 963 out: 964 darray_for_each(fs, fi) { 965 folio_unlock(*fi); 966 folio_put(*fi); 967 } 968 969 /* 970 * If the last folio added to the mapping starts beyond current EOF, we 971 * performed a short write but left around at least one post-EOF folio. 972 * Clean up the mapping before we return. 973 */ 974 if (last_folio_pos >= inode->v.i_size) 975 truncate_pagecache(&inode->v, inode->v.i_size); 976 977 darray_exit(&fs); 978 bch2_folio_reservation_put(c, inode, &res); 979 980 return copied ?: ret; 981 } 982 983 static ssize_t bch2_buffered_write(struct kiocb *iocb, struct iov_iter *iter) 984 { 985 struct file *file = iocb->ki_filp; 986 struct address_space *mapping = file->f_mapping; 987 struct bch_inode_info *inode = file_bch_inode(file); 988 loff_t pos = iocb->ki_pos; 989 ssize_t written = 0; 990 int ret = 0; 991 992 bch2_pagecache_add_get(inode); 993 994 do { 995 unsigned offset = pos & (PAGE_SIZE - 1); 996 unsigned bytes = iov_iter_count(iter); 997 again: 998 /* 999 * Bring in the user page that we will copy from _first_. 1000 * Otherwise there's a nasty deadlock on copying from the 1001 * same page as we're writing to, without it being marked 1002 * up-to-date. 1003 * 1004 * Not only is this an optimisation, but it is also required 1005 * to check that the address is actually valid, when atomic 1006 * usercopies are used, below. 1007 */ 1008 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) { 1009 bytes = min_t(unsigned long, iov_iter_count(iter), 1010 PAGE_SIZE - offset); 1011 1012 if (unlikely(fault_in_iov_iter_readable(iter, bytes))) { 1013 ret = -EFAULT; 1014 break; 1015 } 1016 } 1017 1018 if (unlikely(fatal_signal_pending(current))) { 1019 ret = -EINTR; 1020 break; 1021 } 1022 1023 ret = __bch2_buffered_write(inode, mapping, iter, pos, bytes); 1024 if (unlikely(ret < 0)) 1025 break; 1026 1027 cond_resched(); 1028 1029 if (unlikely(ret == 0)) { 1030 /* 1031 * If we were unable to copy any data at all, we must 1032 * fall back to a single segment length write. 1033 * 1034 * If we didn't fallback here, we could livelock 1035 * because not all segments in the iov can be copied at 1036 * once without a pagefault. 1037 */ 1038 bytes = min_t(unsigned long, PAGE_SIZE - offset, 1039 iov_iter_single_seg_count(iter)); 1040 goto again; 1041 } 1042 pos += ret; 1043 written += ret; 1044 ret = 0; 1045 1046 balance_dirty_pages_ratelimited(mapping); 1047 } while (iov_iter_count(iter)); 1048 1049 bch2_pagecache_add_put(inode); 1050 1051 return written ? written : ret; 1052 } 1053 1054 ssize_t bch2_write_iter(struct kiocb *iocb, struct iov_iter *from) 1055 { 1056 struct file *file = iocb->ki_filp; 1057 struct bch_inode_info *inode = file_bch_inode(file); 1058 ssize_t ret; 1059 1060 if (iocb->ki_flags & IOCB_DIRECT) { 1061 ret = bch2_direct_write(iocb, from); 1062 goto out; 1063 } 1064 1065 inode_lock(&inode->v); 1066 1067 ret = generic_write_checks(iocb, from); 1068 if (ret <= 0) 1069 goto unlock; 1070 1071 ret = file_remove_privs(file); 1072 if (ret) 1073 goto unlock; 1074 1075 ret = file_update_time(file); 1076 if (ret) 1077 goto unlock; 1078 1079 ret = bch2_buffered_write(iocb, from); 1080 if (likely(ret > 0)) 1081 iocb->ki_pos += ret; 1082 unlock: 1083 inode_unlock(&inode->v); 1084 1085 if (ret > 0) 1086 ret = generic_write_sync(iocb, ret); 1087 out: 1088 return bch2_err_class(ret); 1089 } 1090 1091 void bch2_fs_fs_io_buffered_exit(struct bch_fs *c) 1092 { 1093 bioset_exit(&c->writepage_bioset); 1094 } 1095 1096 int bch2_fs_fs_io_buffered_init(struct bch_fs *c) 1097 { 1098 if (bioset_init(&c->writepage_bioset, 1099 4, offsetof(struct bch_writepage_io, op.wbio.bio), 1100 BIOSET_NEED_BVECS)) 1101 return -BCH_ERR_ENOMEM_writepage_bioset_init; 1102 1103 return 0; 1104 } 1105 1106 #endif /* NO_BCACHEFS_FS */ 1107