1 // SPDX-License-Identifier: GPL-2.0 2 #ifndef NO_BCACHEFS_FS 3 4 #include "bcachefs.h" 5 #include "alloc_foreground.h" 6 #include "bkey_buf.h" 7 #include "btree_update.h" 8 #include "buckets.h" 9 #include "clock.h" 10 #include "error.h" 11 #include "extents.h" 12 #include "extent_update.h" 13 #include "fs.h" 14 #include "fs-io.h" 15 #include "fs-io-buffered.h" 16 #include "fs-io-pagecache.h" 17 #include "fsck.h" 18 #include "inode.h" 19 #include "journal.h" 20 #include "io_misc.h" 21 #include "keylist.h" 22 #include "quota.h" 23 #include "reflink.h" 24 #include "trace.h" 25 26 #include <linux/aio.h> 27 #include <linux/backing-dev.h> 28 #include <linux/falloc.h> 29 #include <linux/migrate.h> 30 #include <linux/mmu_context.h> 31 #include <linux/pagevec.h> 32 #include <linux/rmap.h> 33 #include <linux/sched/signal.h> 34 #include <linux/task_io_accounting_ops.h> 35 #include <linux/uio.h> 36 37 #include <trace/events/writeback.h> 38 39 struct nocow_flush { 40 struct closure *cl; 41 struct bch_dev *ca; 42 struct bio bio; 43 }; 44 45 static void nocow_flush_endio(struct bio *_bio) 46 { 47 48 struct nocow_flush *bio = container_of(_bio, struct nocow_flush, bio); 49 50 closure_put(bio->cl); 51 percpu_ref_put(&bio->ca->io_ref); 52 bio_put(&bio->bio); 53 } 54 55 void bch2_inode_flush_nocow_writes_async(struct bch_fs *c, 56 struct bch_inode_info *inode, 57 struct closure *cl) 58 { 59 struct nocow_flush *bio; 60 struct bch_dev *ca; 61 struct bch_devs_mask devs; 62 unsigned dev; 63 64 dev = find_first_bit(inode->ei_devs_need_flush.d, BCH_SB_MEMBERS_MAX); 65 if (dev == BCH_SB_MEMBERS_MAX) 66 return; 67 68 devs = inode->ei_devs_need_flush; 69 memset(&inode->ei_devs_need_flush, 0, sizeof(inode->ei_devs_need_flush)); 70 71 for_each_set_bit(dev, devs.d, BCH_SB_MEMBERS_MAX) { 72 rcu_read_lock(); 73 ca = rcu_dereference(c->devs[dev]); 74 if (ca && !percpu_ref_tryget(&ca->io_ref)) 75 ca = NULL; 76 rcu_read_unlock(); 77 78 if (!ca) 79 continue; 80 81 bio = container_of(bio_alloc_bioset(ca->disk_sb.bdev, 0, 82 REQ_OP_WRITE|REQ_PREFLUSH, 83 GFP_KERNEL, 84 &c->nocow_flush_bioset), 85 struct nocow_flush, bio); 86 bio->cl = cl; 87 bio->ca = ca; 88 bio->bio.bi_end_io = nocow_flush_endio; 89 closure_bio_submit(&bio->bio, cl); 90 } 91 } 92 93 static int bch2_inode_flush_nocow_writes(struct bch_fs *c, 94 struct bch_inode_info *inode) 95 { 96 struct closure cl; 97 98 closure_init_stack(&cl); 99 bch2_inode_flush_nocow_writes_async(c, inode, &cl); 100 closure_sync(&cl); 101 102 return 0; 103 } 104 105 /* i_size updates: */ 106 107 struct inode_new_size { 108 loff_t new_size; 109 u64 now; 110 unsigned fields; 111 }; 112 113 static int inode_set_size(struct btree_trans *trans, 114 struct bch_inode_info *inode, 115 struct bch_inode_unpacked *bi, 116 void *p) 117 { 118 struct inode_new_size *s = p; 119 120 bi->bi_size = s->new_size; 121 if (s->fields & ATTR_ATIME) 122 bi->bi_atime = s->now; 123 if (s->fields & ATTR_MTIME) 124 bi->bi_mtime = s->now; 125 if (s->fields & ATTR_CTIME) 126 bi->bi_ctime = s->now; 127 128 return 0; 129 } 130 131 int __must_check bch2_write_inode_size(struct bch_fs *c, 132 struct bch_inode_info *inode, 133 loff_t new_size, unsigned fields) 134 { 135 struct inode_new_size s = { 136 .new_size = new_size, 137 .now = bch2_current_time(c), 138 .fields = fields, 139 }; 140 141 return bch2_write_inode(c, inode, inode_set_size, &s, fields); 142 } 143 144 void __bch2_i_sectors_acct(struct bch_fs *c, struct bch_inode_info *inode, 145 struct quota_res *quota_res, s64 sectors) 146 { 147 bch2_fs_inconsistent_on((s64) inode->v.i_blocks + sectors < 0, c, 148 "inode %lu i_blocks underflow: %llu + %lli < 0 (ondisk %lli)", 149 inode->v.i_ino, (u64) inode->v.i_blocks, sectors, 150 inode->ei_inode.bi_sectors); 151 inode->v.i_blocks += sectors; 152 153 #ifdef CONFIG_BCACHEFS_QUOTA 154 if (quota_res && 155 !test_bit(EI_INODE_SNAPSHOT, &inode->ei_flags) && 156 sectors > 0) { 157 BUG_ON(sectors > quota_res->sectors); 158 BUG_ON(sectors > inode->ei_quota_reserved); 159 160 quota_res->sectors -= sectors; 161 inode->ei_quota_reserved -= sectors; 162 } else { 163 bch2_quota_acct(c, inode->ei_qid, Q_SPC, sectors, KEY_TYPE_QUOTA_WARN); 164 } 165 #endif 166 } 167 168 /* fsync: */ 169 170 /* 171 * inode->ei_inode.bi_journal_seq won't be up to date since it's set in an 172 * insert trigger: look up the btree inode instead 173 */ 174 static int bch2_flush_inode(struct bch_fs *c, 175 struct bch_inode_info *inode) 176 { 177 if (c->opts.journal_flush_disabled) 178 return 0; 179 180 if (!bch2_write_ref_tryget(c, BCH_WRITE_REF_fsync)) 181 return -EROFS; 182 183 struct bch_inode_unpacked u; 184 int ret = bch2_inode_find_by_inum(c, inode_inum(inode), &u) ?: 185 bch2_journal_flush_seq(&c->journal, u.bi_journal_seq) ?: 186 bch2_inode_flush_nocow_writes(c, inode); 187 bch2_write_ref_put(c, BCH_WRITE_REF_fsync); 188 return ret; 189 } 190 191 int bch2_fsync(struct file *file, loff_t start, loff_t end, int datasync) 192 { 193 struct bch_inode_info *inode = file_bch_inode(file); 194 struct bch_fs *c = inode->v.i_sb->s_fs_info; 195 int ret, err; 196 197 trace_bch2_fsync(file, datasync); 198 199 ret = file_write_and_wait_range(file, start, end); 200 if (ret) 201 goto out; 202 ret = sync_inode_metadata(&inode->v, 1); 203 if (ret) 204 goto out; 205 ret = bch2_flush_inode(c, inode); 206 out: 207 ret = bch2_err_class(ret); 208 if (ret == -EROFS) 209 ret = -EIO; 210 211 err = file_check_and_advance_wb_err(file); 212 if (!ret) 213 ret = err; 214 215 return ret; 216 } 217 218 /* truncate: */ 219 220 static inline int range_has_data(struct bch_fs *c, u32 subvol, 221 struct bpos start, 222 struct bpos end) 223 { 224 return bch2_trans_run(c, 225 for_each_btree_key_in_subvolume_upto(trans, iter, BTREE_ID_extents, start, end, 226 subvol, 0, k, ({ 227 bkey_extent_is_data(k.k) && !bkey_extent_is_unwritten(k); 228 }))); 229 } 230 231 static int __bch2_truncate_folio(struct bch_inode_info *inode, 232 pgoff_t index, loff_t start, loff_t end) 233 { 234 struct bch_fs *c = inode->v.i_sb->s_fs_info; 235 struct address_space *mapping = inode->v.i_mapping; 236 struct bch_folio *s; 237 unsigned start_offset; 238 unsigned end_offset; 239 unsigned i; 240 struct folio *folio; 241 s64 i_sectors_delta = 0; 242 int ret = 0; 243 u64 end_pos; 244 245 folio = filemap_lock_folio(mapping, index); 246 if (IS_ERR_OR_NULL(folio)) { 247 /* 248 * XXX: we're doing two index lookups when we end up reading the 249 * folio 250 */ 251 ret = range_has_data(c, inode->ei_inum.subvol, 252 POS(inode->v.i_ino, (index << PAGE_SECTORS_SHIFT)), 253 POS(inode->v.i_ino, (index << PAGE_SECTORS_SHIFT) + PAGE_SECTORS)); 254 if (ret <= 0) 255 return ret; 256 257 folio = __filemap_get_folio(mapping, index, 258 FGP_LOCK|FGP_CREAT, GFP_KERNEL); 259 if (IS_ERR_OR_NULL(folio)) { 260 ret = -ENOMEM; 261 goto out; 262 } 263 } 264 265 BUG_ON(start >= folio_end_pos(folio)); 266 BUG_ON(end <= folio_pos(folio)); 267 268 start_offset = max(start, folio_pos(folio)) - folio_pos(folio); 269 end_offset = min_t(u64, end, folio_end_pos(folio)) - folio_pos(folio); 270 271 /* Folio boundary? Nothing to do */ 272 if (start_offset == 0 && 273 end_offset == folio_size(folio)) { 274 ret = 0; 275 goto unlock; 276 } 277 278 s = bch2_folio_create(folio, 0); 279 if (!s) { 280 ret = -ENOMEM; 281 goto unlock; 282 } 283 284 if (!folio_test_uptodate(folio)) { 285 ret = bch2_read_single_folio(folio, mapping); 286 if (ret) 287 goto unlock; 288 } 289 290 ret = bch2_folio_set(c, inode_inum(inode), &folio, 1); 291 if (ret) 292 goto unlock; 293 294 for (i = round_up(start_offset, block_bytes(c)) >> 9; 295 i < round_down(end_offset, block_bytes(c)) >> 9; 296 i++) { 297 s->s[i].nr_replicas = 0; 298 299 i_sectors_delta -= s->s[i].state == SECTOR_dirty; 300 bch2_folio_sector_set(folio, s, i, SECTOR_unallocated); 301 } 302 303 bch2_i_sectors_acct(c, inode, NULL, i_sectors_delta); 304 305 /* 306 * Caller needs to know whether this folio will be written out by 307 * writeback - doing an i_size update if necessary - or whether it will 308 * be responsible for the i_size update. 309 * 310 * Note that we shouldn't ever see a folio beyond EOF, but check and 311 * warn if so. This has been observed by failure to clean up folios 312 * after a short write and there's still a chance reclaim will fix 313 * things up. 314 */ 315 WARN_ON_ONCE(folio_pos(folio) >= inode->v.i_size); 316 end_pos = folio_end_pos(folio); 317 if (inode->v.i_size > folio_pos(folio)) 318 end_pos = min_t(u64, inode->v.i_size, end_pos); 319 ret = s->s[folio_pos_to_s(folio, end_pos - 1)].state >= SECTOR_dirty; 320 321 folio_zero_segment(folio, start_offset, end_offset); 322 323 /* 324 * Bit of a hack - we don't want truncate to fail due to -ENOSPC. 325 * 326 * XXX: because we aren't currently tracking whether the folio has actual 327 * data in it (vs. just 0s, or only partially written) this wrong. ick. 328 */ 329 BUG_ON(bch2_get_folio_disk_reservation(c, inode, folio, false)); 330 331 /* 332 * This removes any writeable userspace mappings; we need to force 333 * .page_mkwrite to be called again before any mmapped writes, to 334 * redirty the full page: 335 */ 336 folio_mkclean(folio); 337 filemap_dirty_folio(mapping, folio); 338 unlock: 339 folio_unlock(folio); 340 folio_put(folio); 341 out: 342 return ret; 343 } 344 345 static int bch2_truncate_folio(struct bch_inode_info *inode, loff_t from) 346 { 347 return __bch2_truncate_folio(inode, from >> PAGE_SHIFT, 348 from, ANYSINT_MAX(loff_t)); 349 } 350 351 static int bch2_truncate_folios(struct bch_inode_info *inode, 352 loff_t start, loff_t end) 353 { 354 int ret = __bch2_truncate_folio(inode, start >> PAGE_SHIFT, 355 start, end); 356 357 if (ret >= 0 && 358 start >> PAGE_SHIFT != end >> PAGE_SHIFT) 359 ret = __bch2_truncate_folio(inode, 360 (end - 1) >> PAGE_SHIFT, 361 start, end); 362 return ret; 363 } 364 365 static int bch2_extend(struct mnt_idmap *idmap, 366 struct bch_inode_info *inode, 367 struct bch_inode_unpacked *inode_u, 368 struct iattr *iattr) 369 { 370 struct address_space *mapping = inode->v.i_mapping; 371 int ret; 372 373 /* 374 * sync appends: 375 * 376 * this has to be done _before_ extending i_size: 377 */ 378 ret = filemap_write_and_wait_range(mapping, inode_u->bi_size, S64_MAX); 379 if (ret) 380 return ret; 381 382 truncate_setsize(&inode->v, iattr->ia_size); 383 384 return bch2_setattr_nonsize(idmap, inode, iattr); 385 } 386 387 int bchfs_truncate(struct mnt_idmap *idmap, 388 struct bch_inode_info *inode, struct iattr *iattr) 389 { 390 struct bch_fs *c = inode->v.i_sb->s_fs_info; 391 struct address_space *mapping = inode->v.i_mapping; 392 struct bch_inode_unpacked inode_u; 393 s64 i_sectors_delta = 0; 394 int ret = 0; 395 396 /* 397 * If the truncate call with change the size of the file, the 398 * cmtimes should be updated. If the size will not change, we 399 * do not need to update the cmtimes. 400 */ 401 if (iattr->ia_size != inode->v.i_size) { 402 if (!(iattr->ia_valid & ATTR_MTIME)) 403 ktime_get_coarse_real_ts64(&iattr->ia_mtime); 404 if (!(iattr->ia_valid & ATTR_CTIME)) 405 ktime_get_coarse_real_ts64(&iattr->ia_ctime); 406 iattr->ia_valid |= ATTR_MTIME|ATTR_CTIME; 407 } 408 409 inode_dio_wait(&inode->v); 410 bch2_pagecache_block_get(inode); 411 412 ret = bch2_inode_find_by_inum(c, inode_inum(inode), &inode_u); 413 if (ret) 414 goto err; 415 416 /* 417 * check this before next assertion; on filesystem error our normal 418 * invariants are a bit broken (truncate has to truncate the page cache 419 * before the inode). 420 */ 421 ret = bch2_journal_error(&c->journal); 422 if (ret) 423 goto err; 424 425 WARN_ONCE(!test_bit(EI_INODE_ERROR, &inode->ei_flags) && 426 inode->v.i_size < inode_u.bi_size, 427 "truncate spotted in mem i_size < btree i_size: %llu < %llu\n", 428 (u64) inode->v.i_size, inode_u.bi_size); 429 430 if (iattr->ia_size > inode->v.i_size) { 431 ret = bch2_extend(idmap, inode, &inode_u, iattr); 432 goto err; 433 } 434 435 iattr->ia_valid &= ~ATTR_SIZE; 436 437 ret = bch2_truncate_folio(inode, iattr->ia_size); 438 if (unlikely(ret < 0)) 439 goto err; 440 441 truncate_setsize(&inode->v, iattr->ia_size); 442 443 /* 444 * When extending, we're going to write the new i_size to disk 445 * immediately so we need to flush anything above the current on disk 446 * i_size first: 447 * 448 * Also, when extending we need to flush the page that i_size currently 449 * straddles - if it's mapped to userspace, we need to ensure that 450 * userspace has to redirty it and call .mkwrite -> set_page_dirty 451 * again to allocate the part of the page that was extended. 452 */ 453 if (iattr->ia_size > inode_u.bi_size) 454 ret = filemap_write_and_wait_range(mapping, 455 inode_u.bi_size, 456 iattr->ia_size - 1); 457 else if (iattr->ia_size & (PAGE_SIZE - 1)) 458 ret = filemap_write_and_wait_range(mapping, 459 round_down(iattr->ia_size, PAGE_SIZE), 460 iattr->ia_size - 1); 461 if (ret) 462 goto err; 463 464 ret = bch2_truncate(c, inode_inum(inode), iattr->ia_size, &i_sectors_delta); 465 bch2_i_sectors_acct(c, inode, NULL, i_sectors_delta); 466 467 if (unlikely(ret)) { 468 /* 469 * If we error here, VFS caches are now inconsistent with btree 470 */ 471 set_bit(EI_INODE_ERROR, &inode->ei_flags); 472 goto err; 473 } 474 475 bch2_fs_inconsistent_on(!inode->v.i_size && inode->v.i_blocks && 476 !bch2_journal_error(&c->journal), c, 477 "inode %lu truncated to 0 but i_blocks %llu (ondisk %lli)", 478 inode->v.i_ino, (u64) inode->v.i_blocks, 479 inode->ei_inode.bi_sectors); 480 481 ret = bch2_setattr_nonsize(idmap, inode, iattr); 482 err: 483 bch2_pagecache_block_put(inode); 484 return bch2_err_class(ret); 485 } 486 487 /* fallocate: */ 488 489 static int inode_update_times_fn(struct btree_trans *trans, 490 struct bch_inode_info *inode, 491 struct bch_inode_unpacked *bi, void *p) 492 { 493 struct bch_fs *c = inode->v.i_sb->s_fs_info; 494 495 bi->bi_mtime = bi->bi_ctime = bch2_current_time(c); 496 return 0; 497 } 498 499 static noinline long bchfs_fpunch(struct bch_inode_info *inode, loff_t offset, loff_t len) 500 { 501 struct bch_fs *c = inode->v.i_sb->s_fs_info; 502 u64 end = offset + len; 503 u64 block_start = round_up(offset, block_bytes(c)); 504 u64 block_end = round_down(end, block_bytes(c)); 505 bool truncated_last_page; 506 int ret = 0; 507 508 ret = bch2_truncate_folios(inode, offset, end); 509 if (unlikely(ret < 0)) 510 goto err; 511 512 truncated_last_page = ret; 513 514 truncate_pagecache_range(&inode->v, offset, end - 1); 515 516 if (block_start < block_end) { 517 s64 i_sectors_delta = 0; 518 519 ret = bch2_fpunch(c, inode_inum(inode), 520 block_start >> 9, block_end >> 9, 521 &i_sectors_delta); 522 bch2_i_sectors_acct(c, inode, NULL, i_sectors_delta); 523 } 524 525 mutex_lock(&inode->ei_update_lock); 526 if (end >= inode->v.i_size && !truncated_last_page) { 527 ret = bch2_write_inode_size(c, inode, inode->v.i_size, 528 ATTR_MTIME|ATTR_CTIME); 529 } else { 530 ret = bch2_write_inode(c, inode, inode_update_times_fn, NULL, 531 ATTR_MTIME|ATTR_CTIME); 532 } 533 mutex_unlock(&inode->ei_update_lock); 534 err: 535 return ret; 536 } 537 538 static noinline long bchfs_fcollapse_finsert(struct bch_inode_info *inode, 539 loff_t offset, loff_t len, 540 bool insert) 541 { 542 struct bch_fs *c = inode->v.i_sb->s_fs_info; 543 struct address_space *mapping = inode->v.i_mapping; 544 s64 i_sectors_delta = 0; 545 int ret = 0; 546 547 if ((offset | len) & (block_bytes(c) - 1)) 548 return -EINVAL; 549 550 if (insert) { 551 if (offset >= inode->v.i_size) 552 return -EINVAL; 553 } else { 554 if (offset + len >= inode->v.i_size) 555 return -EINVAL; 556 } 557 558 ret = bch2_write_invalidate_inode_pages_range(mapping, offset, LLONG_MAX); 559 if (ret) 560 return ret; 561 562 if (insert) 563 i_size_write(&inode->v, inode->v.i_size + len); 564 565 ret = bch2_fcollapse_finsert(c, inode_inum(inode), offset >> 9, len >> 9, 566 insert, &i_sectors_delta); 567 if (!ret && !insert) 568 i_size_write(&inode->v, inode->v.i_size - len); 569 bch2_i_sectors_acct(c, inode, NULL, i_sectors_delta); 570 571 return ret; 572 } 573 574 static noinline int __bchfs_fallocate(struct bch_inode_info *inode, int mode, 575 u64 start_sector, u64 end_sector) 576 { 577 struct bch_fs *c = inode->v.i_sb->s_fs_info; 578 struct btree_trans *trans = bch2_trans_get(c); 579 struct btree_iter iter; 580 struct bpos end_pos = POS(inode->v.i_ino, end_sector); 581 struct bch_io_opts opts; 582 int ret = 0; 583 584 bch2_inode_opts_get(&opts, c, &inode->ei_inode); 585 586 bch2_trans_iter_init(trans, &iter, BTREE_ID_extents, 587 POS(inode->v.i_ino, start_sector), 588 BTREE_ITER_slots|BTREE_ITER_intent); 589 590 while (!ret && bkey_lt(iter.pos, end_pos)) { 591 s64 i_sectors_delta = 0; 592 struct quota_res quota_res = { 0 }; 593 struct bkey_s_c k; 594 unsigned sectors; 595 bool is_allocation; 596 u64 hole_start, hole_end; 597 u32 snapshot; 598 599 bch2_trans_begin(trans); 600 601 ret = bch2_subvolume_get_snapshot(trans, 602 inode->ei_inum.subvol, &snapshot); 603 if (ret) 604 goto bkey_err; 605 606 bch2_btree_iter_set_snapshot(&iter, snapshot); 607 608 k = bch2_btree_iter_peek_slot(&iter); 609 if ((ret = bkey_err(k))) 610 goto bkey_err; 611 612 hole_start = iter.pos.offset; 613 hole_end = bpos_min(k.k->p, end_pos).offset; 614 is_allocation = bkey_extent_is_allocation(k.k); 615 616 /* already reserved */ 617 if (bkey_extent_is_reservation(k) && 618 bch2_bkey_nr_ptrs_fully_allocated(k) >= opts.data_replicas) { 619 bch2_btree_iter_advance(&iter); 620 continue; 621 } 622 623 if (bkey_extent_is_data(k.k) && 624 !(mode & FALLOC_FL_ZERO_RANGE)) { 625 bch2_btree_iter_advance(&iter); 626 continue; 627 } 628 629 if (!(mode & FALLOC_FL_ZERO_RANGE)) { 630 /* 631 * Lock ordering - can't be holding btree locks while 632 * blocking on a folio lock: 633 */ 634 if (bch2_clamp_data_hole(&inode->v, 635 &hole_start, 636 &hole_end, 637 opts.data_replicas, true)) 638 ret = drop_locks_do(trans, 639 (bch2_clamp_data_hole(&inode->v, 640 &hole_start, 641 &hole_end, 642 opts.data_replicas, false), 0)); 643 bch2_btree_iter_set_pos(&iter, POS(iter.pos.inode, hole_start)); 644 645 if (ret) 646 goto bkey_err; 647 648 if (hole_start == hole_end) 649 continue; 650 } 651 652 sectors = hole_end - hole_start; 653 654 if (!is_allocation) { 655 ret = bch2_quota_reservation_add(c, inode, 656 "a_res, sectors, true); 657 if (unlikely(ret)) 658 goto bkey_err; 659 } 660 661 ret = bch2_extent_fallocate(trans, inode_inum(inode), &iter, 662 sectors, opts, &i_sectors_delta, 663 writepoint_hashed((unsigned long) current)); 664 if (ret) 665 goto bkey_err; 666 667 bch2_i_sectors_acct(c, inode, "a_res, i_sectors_delta); 668 669 if (bch2_mark_pagecache_reserved(inode, &hole_start, 670 iter.pos.offset, true)) 671 drop_locks_do(trans, 672 bch2_mark_pagecache_reserved(inode, &hole_start, 673 iter.pos.offset, false)); 674 bkey_err: 675 bch2_quota_reservation_put(c, inode, "a_res); 676 if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) 677 ret = 0; 678 } 679 680 if (bch2_err_matches(ret, ENOSPC) && (mode & FALLOC_FL_ZERO_RANGE)) { 681 struct quota_res quota_res = { 0 }; 682 s64 i_sectors_delta = 0; 683 684 bch2_fpunch_at(trans, &iter, inode_inum(inode), 685 end_sector, &i_sectors_delta); 686 bch2_i_sectors_acct(c, inode, "a_res, i_sectors_delta); 687 bch2_quota_reservation_put(c, inode, "a_res); 688 } 689 690 bch2_trans_iter_exit(trans, &iter); 691 bch2_trans_put(trans); 692 return ret; 693 } 694 695 static noinline long bchfs_fallocate(struct bch_inode_info *inode, int mode, 696 loff_t offset, loff_t len) 697 { 698 struct bch_fs *c = inode->v.i_sb->s_fs_info; 699 u64 end = offset + len; 700 u64 block_start = round_down(offset, block_bytes(c)); 701 u64 block_end = round_up(end, block_bytes(c)); 702 bool truncated_last_page = false; 703 int ret, ret2 = 0; 704 705 if (!(mode & FALLOC_FL_KEEP_SIZE) && end > inode->v.i_size) { 706 ret = inode_newsize_ok(&inode->v, end); 707 if (ret) 708 return ret; 709 } 710 711 if (mode & FALLOC_FL_ZERO_RANGE) { 712 ret = bch2_truncate_folios(inode, offset, end); 713 if (unlikely(ret < 0)) 714 return ret; 715 716 truncated_last_page = ret; 717 718 truncate_pagecache_range(&inode->v, offset, end - 1); 719 720 block_start = round_up(offset, block_bytes(c)); 721 block_end = round_down(end, block_bytes(c)); 722 } 723 724 ret = __bchfs_fallocate(inode, mode, block_start >> 9, block_end >> 9); 725 726 /* 727 * On -ENOSPC in ZERO_RANGE mode, we still want to do the inode update, 728 * so that the VFS cache i_size is consistent with the btree i_size: 729 */ 730 if (ret && 731 !(bch2_err_matches(ret, ENOSPC) && (mode & FALLOC_FL_ZERO_RANGE))) 732 return ret; 733 734 if (mode & FALLOC_FL_KEEP_SIZE && end > inode->v.i_size) 735 end = inode->v.i_size; 736 737 if (end >= inode->v.i_size && 738 (((mode & FALLOC_FL_ZERO_RANGE) && !truncated_last_page) || 739 !(mode & FALLOC_FL_KEEP_SIZE))) { 740 spin_lock(&inode->v.i_lock); 741 i_size_write(&inode->v, end); 742 spin_unlock(&inode->v.i_lock); 743 744 mutex_lock(&inode->ei_update_lock); 745 ret2 = bch2_write_inode_size(c, inode, end, 0); 746 mutex_unlock(&inode->ei_update_lock); 747 } 748 749 return ret ?: ret2; 750 } 751 752 long bch2_fallocate_dispatch(struct file *file, int mode, 753 loff_t offset, loff_t len) 754 { 755 struct bch_inode_info *inode = file_bch_inode(file); 756 struct bch_fs *c = inode->v.i_sb->s_fs_info; 757 long ret; 758 759 if (!bch2_write_ref_tryget(c, BCH_WRITE_REF_fallocate)) 760 return -EROFS; 761 762 inode_lock(&inode->v); 763 inode_dio_wait(&inode->v); 764 bch2_pagecache_block_get(inode); 765 766 ret = file_modified(file); 767 if (ret) 768 goto err; 769 770 if (!(mode & ~(FALLOC_FL_KEEP_SIZE|FALLOC_FL_ZERO_RANGE))) 771 ret = bchfs_fallocate(inode, mode, offset, len); 772 else if (mode == (FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE)) 773 ret = bchfs_fpunch(inode, offset, len); 774 else if (mode == FALLOC_FL_INSERT_RANGE) 775 ret = bchfs_fcollapse_finsert(inode, offset, len, true); 776 else if (mode == FALLOC_FL_COLLAPSE_RANGE) 777 ret = bchfs_fcollapse_finsert(inode, offset, len, false); 778 else 779 ret = -EOPNOTSUPP; 780 err: 781 bch2_pagecache_block_put(inode); 782 inode_unlock(&inode->v); 783 bch2_write_ref_put(c, BCH_WRITE_REF_fallocate); 784 785 return bch2_err_class(ret); 786 } 787 788 /* 789 * Take a quota reservation for unallocated blocks in a given file range 790 * Does not check pagecache 791 */ 792 static int quota_reserve_range(struct bch_inode_info *inode, 793 struct quota_res *res, 794 u64 start, u64 end) 795 { 796 struct bch_fs *c = inode->v.i_sb->s_fs_info; 797 u64 sectors = end - start; 798 799 int ret = bch2_trans_run(c, 800 for_each_btree_key_in_subvolume_upto(trans, iter, 801 BTREE_ID_extents, 802 POS(inode->v.i_ino, start), 803 POS(inode->v.i_ino, end - 1), 804 inode->ei_inum.subvol, 0, k, ({ 805 if (bkey_extent_is_allocation(k.k)) { 806 u64 s = min(end, k.k->p.offset) - 807 max(start, bkey_start_offset(k.k)); 808 BUG_ON(s > sectors); 809 sectors -= s; 810 } 811 812 0; 813 }))); 814 815 return ret ?: bch2_quota_reservation_add(c, inode, res, sectors, true); 816 } 817 818 loff_t bch2_remap_file_range(struct file *file_src, loff_t pos_src, 819 struct file *file_dst, loff_t pos_dst, 820 loff_t len, unsigned remap_flags) 821 { 822 struct bch_inode_info *src = file_bch_inode(file_src); 823 struct bch_inode_info *dst = file_bch_inode(file_dst); 824 struct bch_fs *c = src->v.i_sb->s_fs_info; 825 struct quota_res quota_res = { 0 }; 826 s64 i_sectors_delta = 0; 827 u64 aligned_len; 828 loff_t ret = 0; 829 830 if (remap_flags & ~(REMAP_FILE_DEDUP|REMAP_FILE_ADVISORY)) 831 return -EINVAL; 832 833 if ((pos_src & (block_bytes(c) - 1)) || 834 (pos_dst & (block_bytes(c) - 1))) 835 return -EINVAL; 836 837 if (src == dst && 838 abs(pos_src - pos_dst) < len) 839 return -EINVAL; 840 841 lock_two_nondirectories(&src->v, &dst->v); 842 bch2_lock_inodes(INODE_PAGECACHE_BLOCK, src, dst); 843 844 inode_dio_wait(&src->v); 845 inode_dio_wait(&dst->v); 846 847 ret = generic_remap_file_range_prep(file_src, pos_src, 848 file_dst, pos_dst, 849 &len, remap_flags); 850 if (ret < 0 || len == 0) 851 goto err; 852 853 aligned_len = round_up((u64) len, block_bytes(c)); 854 855 ret = bch2_write_invalidate_inode_pages_range(dst->v.i_mapping, 856 pos_dst, pos_dst + len - 1); 857 if (ret) 858 goto err; 859 860 ret = quota_reserve_range(dst, "a_res, pos_dst >> 9, 861 (pos_dst + aligned_len) >> 9); 862 if (ret) 863 goto err; 864 865 if (!(remap_flags & REMAP_FILE_DEDUP)) 866 file_update_time(file_dst); 867 868 bch2_mark_pagecache_unallocated(src, pos_src >> 9, 869 (pos_src + aligned_len) >> 9); 870 871 ret = bch2_remap_range(c, 872 inode_inum(dst), pos_dst >> 9, 873 inode_inum(src), pos_src >> 9, 874 aligned_len >> 9, 875 pos_dst + len, &i_sectors_delta); 876 if (ret < 0) 877 goto err; 878 879 /* 880 * due to alignment, we might have remapped slightly more than requsted 881 */ 882 ret = min((u64) ret << 9, (u64) len); 883 884 bch2_i_sectors_acct(c, dst, "a_res, i_sectors_delta); 885 886 spin_lock(&dst->v.i_lock); 887 if (pos_dst + ret > dst->v.i_size) 888 i_size_write(&dst->v, pos_dst + ret); 889 spin_unlock(&dst->v.i_lock); 890 891 if ((file_dst->f_flags & (__O_SYNC | O_DSYNC)) || 892 IS_SYNC(file_inode(file_dst))) 893 ret = bch2_flush_inode(c, dst); 894 err: 895 bch2_quota_reservation_put(c, dst, "a_res); 896 bch2_unlock_inodes(INODE_PAGECACHE_BLOCK, src, dst); 897 unlock_two_nondirectories(&src->v, &dst->v); 898 899 return bch2_err_class(ret); 900 } 901 902 /* fseek: */ 903 904 static loff_t bch2_seek_data(struct file *file, u64 offset) 905 { 906 struct bch_inode_info *inode = file_bch_inode(file); 907 struct bch_fs *c = inode->v.i_sb->s_fs_info; 908 subvol_inum inum = inode_inum(inode); 909 u64 isize, next_data = MAX_LFS_FILESIZE; 910 911 isize = i_size_read(&inode->v); 912 if (offset >= isize) 913 return -ENXIO; 914 915 int ret = bch2_trans_run(c, 916 for_each_btree_key_in_subvolume_upto(trans, iter, BTREE_ID_extents, 917 POS(inode->v.i_ino, offset >> 9), 918 POS(inode->v.i_ino, U64_MAX), 919 inum.subvol, 0, k, ({ 920 if (bkey_extent_is_data(k.k)) { 921 next_data = max(offset, bkey_start_offset(k.k) << 9); 922 break; 923 } else if (k.k->p.offset >> 9 > isize) 924 break; 925 0; 926 }))); 927 if (ret) 928 return ret; 929 930 if (next_data > offset) 931 next_data = bch2_seek_pagecache_data(&inode->v, 932 offset, next_data, 0, false); 933 934 if (next_data >= isize) 935 return -ENXIO; 936 937 return vfs_setpos(file, next_data, MAX_LFS_FILESIZE); 938 } 939 940 static loff_t bch2_seek_hole(struct file *file, u64 offset) 941 { 942 struct bch_inode_info *inode = file_bch_inode(file); 943 struct bch_fs *c = inode->v.i_sb->s_fs_info; 944 subvol_inum inum = inode_inum(inode); 945 u64 isize, next_hole = MAX_LFS_FILESIZE; 946 947 isize = i_size_read(&inode->v); 948 if (offset >= isize) 949 return -ENXIO; 950 951 int ret = bch2_trans_run(c, 952 for_each_btree_key_in_subvolume_upto(trans, iter, BTREE_ID_extents, 953 POS(inode->v.i_ino, offset >> 9), 954 POS(inode->v.i_ino, U64_MAX), 955 inum.subvol, BTREE_ITER_slots, k, ({ 956 if (k.k->p.inode != inode->v.i_ino) { 957 next_hole = bch2_seek_pagecache_hole(&inode->v, 958 offset, MAX_LFS_FILESIZE, 0, false); 959 break; 960 } else if (!bkey_extent_is_data(k.k)) { 961 next_hole = bch2_seek_pagecache_hole(&inode->v, 962 max(offset, bkey_start_offset(k.k) << 9), 963 k.k->p.offset << 9, 0, false); 964 965 if (next_hole < k.k->p.offset << 9) 966 break; 967 } else { 968 offset = max(offset, bkey_start_offset(k.k) << 9); 969 } 970 0; 971 }))); 972 if (ret) 973 return ret; 974 975 if (next_hole > isize) 976 next_hole = isize; 977 978 return vfs_setpos(file, next_hole, MAX_LFS_FILESIZE); 979 } 980 981 loff_t bch2_llseek(struct file *file, loff_t offset, int whence) 982 { 983 loff_t ret; 984 985 switch (whence) { 986 case SEEK_SET: 987 case SEEK_CUR: 988 case SEEK_END: 989 ret = generic_file_llseek(file, offset, whence); 990 break; 991 case SEEK_DATA: 992 ret = bch2_seek_data(file, offset); 993 break; 994 case SEEK_HOLE: 995 ret = bch2_seek_hole(file, offset); 996 break; 997 default: 998 ret = -EINVAL; 999 break; 1000 } 1001 1002 return bch2_err_class(ret); 1003 } 1004 1005 void bch2_fs_fsio_exit(struct bch_fs *c) 1006 { 1007 bioset_exit(&c->nocow_flush_bioset); 1008 } 1009 1010 int bch2_fs_fsio_init(struct bch_fs *c) 1011 { 1012 if (bioset_init(&c->nocow_flush_bioset, 1013 1, offsetof(struct nocow_flush, bio), 0)) 1014 return -BCH_ERR_ENOMEM_nocow_flush_bioset_init; 1015 1016 return 0; 1017 } 1018 1019 #endif /* NO_BCACHEFS_FS */ 1020