1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fs/f2fs/data.c 4 * 5 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 6 * http://www.samsung.com/ 7 */ 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/buffer_head.h> 11 #include <linux/sched/mm.h> 12 #include <linux/mpage.h> 13 #include <linux/writeback.h> 14 #include <linux/pagevec.h> 15 #include <linux/blkdev.h> 16 #include <linux/bio.h> 17 #include <linux/blk-crypto.h> 18 #include <linux/swap.h> 19 #include <linux/prefetch.h> 20 #include <linux/uio.h> 21 #include <linux/sched/signal.h> 22 #include <linux/fiemap.h> 23 #include <linux/iomap.h> 24 25 #include "f2fs.h" 26 #include "node.h" 27 #include "segment.h" 28 #include "iostat.h" 29 #include <trace/events/f2fs.h> 30 31 #define NUM_PREALLOC_POST_READ_CTXS 128 32 33 static struct kmem_cache *bio_post_read_ctx_cache; 34 static struct kmem_cache *bio_entry_slab; 35 static mempool_t *bio_post_read_ctx_pool; 36 static struct bio_set f2fs_bioset; 37 38 #define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE 39 40 int __init f2fs_init_bioset(void) 41 { 42 return bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE, 43 0, BIOSET_NEED_BVECS); 44 } 45 46 void f2fs_destroy_bioset(void) 47 { 48 bioset_exit(&f2fs_bioset); 49 } 50 51 bool f2fs_is_cp_guaranteed(struct page *page) 52 { 53 struct address_space *mapping = page->mapping; 54 struct inode *inode; 55 struct f2fs_sb_info *sbi; 56 57 if (!mapping) 58 return false; 59 60 inode = mapping->host; 61 sbi = F2FS_I_SB(inode); 62 63 if (inode->i_ino == F2FS_META_INO(sbi) || 64 inode->i_ino == F2FS_NODE_INO(sbi) || 65 S_ISDIR(inode->i_mode)) 66 return true; 67 68 if ((S_ISREG(inode->i_mode) && IS_NOQUOTA(inode)) || 69 page_private_gcing(page)) 70 return true; 71 return false; 72 } 73 74 static enum count_type __read_io_type(struct page *page) 75 { 76 struct address_space *mapping = page_file_mapping(page); 77 78 if (mapping) { 79 struct inode *inode = mapping->host; 80 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 81 82 if (inode->i_ino == F2FS_META_INO(sbi)) 83 return F2FS_RD_META; 84 85 if (inode->i_ino == F2FS_NODE_INO(sbi)) 86 return F2FS_RD_NODE; 87 } 88 return F2FS_RD_DATA; 89 } 90 91 /* postprocessing steps for read bios */ 92 enum bio_post_read_step { 93 #ifdef CONFIG_FS_ENCRYPTION 94 STEP_DECRYPT = BIT(0), 95 #else 96 STEP_DECRYPT = 0, /* compile out the decryption-related code */ 97 #endif 98 #ifdef CONFIG_F2FS_FS_COMPRESSION 99 STEP_DECOMPRESS = BIT(1), 100 #else 101 STEP_DECOMPRESS = 0, /* compile out the decompression-related code */ 102 #endif 103 #ifdef CONFIG_FS_VERITY 104 STEP_VERITY = BIT(2), 105 #else 106 STEP_VERITY = 0, /* compile out the verity-related code */ 107 #endif 108 }; 109 110 struct bio_post_read_ctx { 111 struct bio *bio; 112 struct f2fs_sb_info *sbi; 113 struct work_struct work; 114 unsigned int enabled_steps; 115 /* 116 * decompression_attempted keeps track of whether 117 * f2fs_end_read_compressed_page() has been called on the pages in the 118 * bio that belong to a compressed cluster yet. 119 */ 120 bool decompression_attempted; 121 block_t fs_blkaddr; 122 }; 123 124 /* 125 * Update and unlock a bio's pages, and free the bio. 126 * 127 * This marks pages up-to-date only if there was no error in the bio (I/O error, 128 * decryption error, or verity error), as indicated by bio->bi_status. 129 * 130 * "Compressed pages" (pagecache pages backed by a compressed cluster on-disk) 131 * aren't marked up-to-date here, as decompression is done on a per-compression- 132 * cluster basis rather than a per-bio basis. Instead, we only must do two 133 * things for each compressed page here: call f2fs_end_read_compressed_page() 134 * with failed=true if an error occurred before it would have normally gotten 135 * called (i.e., I/O error or decryption error, but *not* verity error), and 136 * release the bio's reference to the decompress_io_ctx of the page's cluster. 137 */ 138 static void f2fs_finish_read_bio(struct bio *bio, bool in_task) 139 { 140 struct bio_vec *bv; 141 struct bvec_iter_all iter_all; 142 struct bio_post_read_ctx *ctx = bio->bi_private; 143 144 bio_for_each_segment_all(bv, bio, iter_all) { 145 struct page *page = bv->bv_page; 146 147 if (f2fs_is_compressed_page(page)) { 148 if (ctx && !ctx->decompression_attempted) 149 f2fs_end_read_compressed_page(page, true, 0, 150 in_task); 151 f2fs_put_page_dic(page, in_task); 152 continue; 153 } 154 155 if (bio->bi_status) 156 ClearPageUptodate(page); 157 else 158 SetPageUptodate(page); 159 dec_page_count(F2FS_P_SB(page), __read_io_type(page)); 160 unlock_page(page); 161 } 162 163 if (ctx) 164 mempool_free(ctx, bio_post_read_ctx_pool); 165 bio_put(bio); 166 } 167 168 static void f2fs_verify_bio(struct work_struct *work) 169 { 170 struct bio_post_read_ctx *ctx = 171 container_of(work, struct bio_post_read_ctx, work); 172 struct bio *bio = ctx->bio; 173 bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS); 174 175 /* 176 * fsverity_verify_bio() may call readahead() again, and while verity 177 * will be disabled for this, decryption and/or decompression may still 178 * be needed, resulting in another bio_post_read_ctx being allocated. 179 * So to prevent deadlocks we need to release the current ctx to the 180 * mempool first. This assumes that verity is the last post-read step. 181 */ 182 mempool_free(ctx, bio_post_read_ctx_pool); 183 bio->bi_private = NULL; 184 185 /* 186 * Verify the bio's pages with fs-verity. Exclude compressed pages, 187 * as those were handled separately by f2fs_end_read_compressed_page(). 188 */ 189 if (may_have_compressed_pages) { 190 struct bio_vec *bv; 191 struct bvec_iter_all iter_all; 192 193 bio_for_each_segment_all(bv, bio, iter_all) { 194 struct page *page = bv->bv_page; 195 196 if (!f2fs_is_compressed_page(page) && 197 !fsverity_verify_page(page)) { 198 bio->bi_status = BLK_STS_IOERR; 199 break; 200 } 201 } 202 } else { 203 fsverity_verify_bio(bio); 204 } 205 206 f2fs_finish_read_bio(bio, true); 207 } 208 209 /* 210 * If the bio's data needs to be verified with fs-verity, then enqueue the 211 * verity work for the bio. Otherwise finish the bio now. 212 * 213 * Note that to avoid deadlocks, the verity work can't be done on the 214 * decryption/decompression workqueue. This is because verifying the data pages 215 * can involve reading verity metadata pages from the file, and these verity 216 * metadata pages may be encrypted and/or compressed. 217 */ 218 static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task) 219 { 220 struct bio_post_read_ctx *ctx = bio->bi_private; 221 222 if (ctx && (ctx->enabled_steps & STEP_VERITY)) { 223 INIT_WORK(&ctx->work, f2fs_verify_bio); 224 fsverity_enqueue_verify_work(&ctx->work); 225 } else { 226 f2fs_finish_read_bio(bio, in_task); 227 } 228 } 229 230 /* 231 * Handle STEP_DECOMPRESS by decompressing any compressed clusters whose last 232 * remaining page was read by @ctx->bio. 233 * 234 * Note that a bio may span clusters (even a mix of compressed and uncompressed 235 * clusters) or be for just part of a cluster. STEP_DECOMPRESS just indicates 236 * that the bio includes at least one compressed page. The actual decompression 237 * is done on a per-cluster basis, not a per-bio basis. 238 */ 239 static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx, 240 bool in_task) 241 { 242 struct bio_vec *bv; 243 struct bvec_iter_all iter_all; 244 bool all_compressed = true; 245 block_t blkaddr = ctx->fs_blkaddr; 246 247 bio_for_each_segment_all(bv, ctx->bio, iter_all) { 248 struct page *page = bv->bv_page; 249 250 if (f2fs_is_compressed_page(page)) 251 f2fs_end_read_compressed_page(page, false, blkaddr, 252 in_task); 253 else 254 all_compressed = false; 255 256 blkaddr++; 257 } 258 259 ctx->decompression_attempted = true; 260 261 /* 262 * Optimization: if all the bio's pages are compressed, then scheduling 263 * the per-bio verity work is unnecessary, as verity will be fully 264 * handled at the compression cluster level. 265 */ 266 if (all_compressed) 267 ctx->enabled_steps &= ~STEP_VERITY; 268 } 269 270 static void f2fs_post_read_work(struct work_struct *work) 271 { 272 struct bio_post_read_ctx *ctx = 273 container_of(work, struct bio_post_read_ctx, work); 274 struct bio *bio = ctx->bio; 275 276 if ((ctx->enabled_steps & STEP_DECRYPT) && !fscrypt_decrypt_bio(bio)) { 277 f2fs_finish_read_bio(bio, true); 278 return; 279 } 280 281 if (ctx->enabled_steps & STEP_DECOMPRESS) 282 f2fs_handle_step_decompress(ctx, true); 283 284 f2fs_verify_and_finish_bio(bio, true); 285 } 286 287 static void f2fs_read_end_io(struct bio *bio) 288 { 289 struct f2fs_sb_info *sbi = F2FS_P_SB(bio_first_page_all(bio)); 290 struct bio_post_read_ctx *ctx; 291 bool intask = in_task(); 292 293 iostat_update_and_unbind_ctx(bio); 294 ctx = bio->bi_private; 295 296 if (time_to_inject(sbi, FAULT_READ_IO)) 297 bio->bi_status = BLK_STS_IOERR; 298 299 if (bio->bi_status) { 300 f2fs_finish_read_bio(bio, intask); 301 return; 302 } 303 304 if (ctx) { 305 unsigned int enabled_steps = ctx->enabled_steps & 306 (STEP_DECRYPT | STEP_DECOMPRESS); 307 308 /* 309 * If we have only decompression step between decompression and 310 * decrypt, we don't need post processing for this. 311 */ 312 if (enabled_steps == STEP_DECOMPRESS && 313 !f2fs_low_mem_mode(sbi)) { 314 f2fs_handle_step_decompress(ctx, intask); 315 } else if (enabled_steps) { 316 INIT_WORK(&ctx->work, f2fs_post_read_work); 317 queue_work(ctx->sbi->post_read_wq, &ctx->work); 318 return; 319 } 320 } 321 322 f2fs_verify_and_finish_bio(bio, intask); 323 } 324 325 static void f2fs_write_end_io(struct bio *bio) 326 { 327 struct f2fs_sb_info *sbi; 328 struct bio_vec *bvec; 329 struct bvec_iter_all iter_all; 330 331 iostat_update_and_unbind_ctx(bio); 332 sbi = bio->bi_private; 333 334 if (time_to_inject(sbi, FAULT_WRITE_IO)) 335 bio->bi_status = BLK_STS_IOERR; 336 337 bio_for_each_segment_all(bvec, bio, iter_all) { 338 struct page *page = bvec->bv_page; 339 enum count_type type = WB_DATA_TYPE(page, false); 340 341 fscrypt_finalize_bounce_page(&page); 342 343 #ifdef CONFIG_F2FS_FS_COMPRESSION 344 if (f2fs_is_compressed_page(page)) { 345 f2fs_compress_write_end_io(bio, page); 346 continue; 347 } 348 #endif 349 350 if (unlikely(bio->bi_status)) { 351 mapping_set_error(page->mapping, -EIO); 352 if (type == F2FS_WB_CP_DATA) 353 f2fs_stop_checkpoint(sbi, true, 354 STOP_CP_REASON_WRITE_FAIL); 355 } 356 357 f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) && 358 page->index != nid_of_node(page)); 359 360 dec_page_count(sbi, type); 361 if (f2fs_in_warm_node_list(sbi, page)) 362 f2fs_del_fsync_node_entry(sbi, page); 363 clear_page_private_gcing(page); 364 end_page_writeback(page); 365 } 366 if (!get_pages(sbi, F2FS_WB_CP_DATA) && 367 wq_has_sleeper(&sbi->cp_wait)) 368 wake_up(&sbi->cp_wait); 369 370 bio_put(bio); 371 } 372 373 #ifdef CONFIG_BLK_DEV_ZONED 374 static void f2fs_zone_write_end_io(struct bio *bio) 375 { 376 struct f2fs_bio_info *io = (struct f2fs_bio_info *)bio->bi_private; 377 378 bio->bi_private = io->bi_private; 379 complete(&io->zone_wait); 380 f2fs_write_end_io(bio); 381 } 382 #endif 383 384 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, 385 block_t blk_addr, sector_t *sector) 386 { 387 struct block_device *bdev = sbi->sb->s_bdev; 388 int i; 389 390 if (f2fs_is_multi_device(sbi)) { 391 for (i = 0; i < sbi->s_ndevs; i++) { 392 if (FDEV(i).start_blk <= blk_addr && 393 FDEV(i).end_blk >= blk_addr) { 394 blk_addr -= FDEV(i).start_blk; 395 bdev = FDEV(i).bdev; 396 break; 397 } 398 } 399 } 400 401 if (sector) 402 *sector = SECTOR_FROM_BLOCK(blk_addr); 403 return bdev; 404 } 405 406 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) 407 { 408 int i; 409 410 if (!f2fs_is_multi_device(sbi)) 411 return 0; 412 413 for (i = 0; i < sbi->s_ndevs; i++) 414 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) 415 return i; 416 return 0; 417 } 418 419 static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio) 420 { 421 unsigned int temp_mask = GENMASK(NR_TEMP_TYPE - 1, 0); 422 unsigned int fua_flag, meta_flag, io_flag; 423 blk_opf_t op_flags = 0; 424 425 if (fio->op != REQ_OP_WRITE) 426 return 0; 427 if (fio->type == DATA) 428 io_flag = fio->sbi->data_io_flag; 429 else if (fio->type == NODE) 430 io_flag = fio->sbi->node_io_flag; 431 else 432 return 0; 433 434 fua_flag = io_flag & temp_mask; 435 meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask; 436 437 /* 438 * data/node io flag bits per temp: 439 * REQ_META | REQ_FUA | 440 * 5 | 4 | 3 | 2 | 1 | 0 | 441 * Cold | Warm | Hot | Cold | Warm | Hot | 442 */ 443 if (BIT(fio->temp) & meta_flag) 444 op_flags |= REQ_META; 445 if (BIT(fio->temp) & fua_flag) 446 op_flags |= REQ_FUA; 447 return op_flags; 448 } 449 450 static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) 451 { 452 struct f2fs_sb_info *sbi = fio->sbi; 453 struct block_device *bdev; 454 sector_t sector; 455 struct bio *bio; 456 457 bdev = f2fs_target_device(sbi, fio->new_blkaddr, §or); 458 bio = bio_alloc_bioset(bdev, npages, 459 fio->op | fio->op_flags | f2fs_io_flags(fio), 460 GFP_NOIO, &f2fs_bioset); 461 bio->bi_iter.bi_sector = sector; 462 if (is_read_io(fio->op)) { 463 bio->bi_end_io = f2fs_read_end_io; 464 bio->bi_private = NULL; 465 } else { 466 bio->bi_end_io = f2fs_write_end_io; 467 bio->bi_private = sbi; 468 bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, 469 fio->type, fio->temp); 470 } 471 iostat_alloc_and_bind_ctx(sbi, bio, NULL); 472 473 if (fio->io_wbc) 474 wbc_init_bio(fio->io_wbc, bio); 475 476 return bio; 477 } 478 479 static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, 480 pgoff_t first_idx, 481 const struct f2fs_io_info *fio, 482 gfp_t gfp_mask) 483 { 484 /* 485 * The f2fs garbage collector sets ->encrypted_page when it wants to 486 * read/write raw data without encryption. 487 */ 488 if (!fio || !fio->encrypted_page) 489 fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask); 490 } 491 492 static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode, 493 pgoff_t next_idx, 494 const struct f2fs_io_info *fio) 495 { 496 /* 497 * The f2fs garbage collector sets ->encrypted_page when it wants to 498 * read/write raw data without encryption. 499 */ 500 if (fio && fio->encrypted_page) 501 return !bio_has_crypt_ctx(bio); 502 503 return fscrypt_mergeable_bio(bio, inode, next_idx); 504 } 505 506 void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio, 507 enum page_type type) 508 { 509 WARN_ON_ONCE(!is_read_io(bio_op(bio))); 510 trace_f2fs_submit_read_bio(sbi->sb, type, bio); 511 512 iostat_update_submit_ctx(bio, type); 513 submit_bio(bio); 514 } 515 516 static void f2fs_submit_write_bio(struct f2fs_sb_info *sbi, struct bio *bio, 517 enum page_type type) 518 { 519 WARN_ON_ONCE(is_read_io(bio_op(bio))); 520 521 if (f2fs_lfs_mode(sbi) && current->plug && PAGE_TYPE_ON_MAIN(type)) 522 blk_finish_plug(current->plug); 523 524 trace_f2fs_submit_write_bio(sbi->sb, type, bio); 525 iostat_update_submit_ctx(bio, type); 526 submit_bio(bio); 527 } 528 529 static void __submit_merged_bio(struct f2fs_bio_info *io) 530 { 531 struct f2fs_io_info *fio = &io->fio; 532 533 if (!io->bio) 534 return; 535 536 if (is_read_io(fio->op)) { 537 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); 538 f2fs_submit_read_bio(io->sbi, io->bio, fio->type); 539 } else { 540 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); 541 f2fs_submit_write_bio(io->sbi, io->bio, fio->type); 542 } 543 io->bio = NULL; 544 } 545 546 static bool __has_merged_page(struct bio *bio, struct inode *inode, 547 struct page *page, nid_t ino) 548 { 549 struct bio_vec *bvec; 550 struct bvec_iter_all iter_all; 551 552 if (!bio) 553 return false; 554 555 if (!inode && !page && !ino) 556 return true; 557 558 bio_for_each_segment_all(bvec, bio, iter_all) { 559 struct page *target = bvec->bv_page; 560 561 if (fscrypt_is_bounce_page(target)) { 562 target = fscrypt_pagecache_page(target); 563 if (IS_ERR(target)) 564 continue; 565 } 566 if (f2fs_is_compressed_page(target)) { 567 target = f2fs_compress_control_page(target); 568 if (IS_ERR(target)) 569 continue; 570 } 571 572 if (inode && inode == target->mapping->host) 573 return true; 574 if (page && page == target) 575 return true; 576 if (ino && ino == ino_of_node(target)) 577 return true; 578 } 579 580 return false; 581 } 582 583 int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi) 584 { 585 int i; 586 587 for (i = 0; i < NR_PAGE_TYPE; i++) { 588 int n = (i == META) ? 1 : NR_TEMP_TYPE; 589 int j; 590 591 sbi->write_io[i] = f2fs_kmalloc(sbi, 592 array_size(n, sizeof(struct f2fs_bio_info)), 593 GFP_KERNEL); 594 if (!sbi->write_io[i]) 595 return -ENOMEM; 596 597 for (j = HOT; j < n; j++) { 598 struct f2fs_bio_info *io = &sbi->write_io[i][j]; 599 600 init_f2fs_rwsem(&io->io_rwsem); 601 io->sbi = sbi; 602 io->bio = NULL; 603 io->last_block_in_bio = 0; 604 spin_lock_init(&io->io_lock); 605 INIT_LIST_HEAD(&io->io_list); 606 INIT_LIST_HEAD(&io->bio_list); 607 init_f2fs_rwsem(&io->bio_list_lock); 608 #ifdef CONFIG_BLK_DEV_ZONED 609 init_completion(&io->zone_wait); 610 io->zone_pending_bio = NULL; 611 io->bi_private = NULL; 612 #endif 613 } 614 } 615 616 return 0; 617 } 618 619 static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, 620 enum page_type type, enum temp_type temp) 621 { 622 enum page_type btype = PAGE_TYPE_OF_BIO(type); 623 struct f2fs_bio_info *io = sbi->write_io[btype] + temp; 624 625 f2fs_down_write(&io->io_rwsem); 626 627 if (!io->bio) 628 goto unlock_out; 629 630 /* change META to META_FLUSH in the checkpoint procedure */ 631 if (type >= META_FLUSH) { 632 io->fio.type = META_FLUSH; 633 io->bio->bi_opf |= REQ_META | REQ_PRIO | REQ_SYNC; 634 if (!test_opt(sbi, NOBARRIER)) 635 io->bio->bi_opf |= REQ_PREFLUSH | REQ_FUA; 636 } 637 __submit_merged_bio(io); 638 unlock_out: 639 f2fs_up_write(&io->io_rwsem); 640 } 641 642 static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, 643 struct inode *inode, struct page *page, 644 nid_t ino, enum page_type type, bool force) 645 { 646 enum temp_type temp; 647 bool ret = true; 648 649 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { 650 if (!force) { 651 enum page_type btype = PAGE_TYPE_OF_BIO(type); 652 struct f2fs_bio_info *io = sbi->write_io[btype] + temp; 653 654 f2fs_down_read(&io->io_rwsem); 655 ret = __has_merged_page(io->bio, inode, page, ino); 656 f2fs_up_read(&io->io_rwsem); 657 } 658 if (ret) 659 __f2fs_submit_merged_write(sbi, type, temp); 660 661 /* TODO: use HOT temp only for meta pages now. */ 662 if (type >= META) 663 break; 664 } 665 } 666 667 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type) 668 { 669 __submit_merged_write_cond(sbi, NULL, NULL, 0, type, true); 670 } 671 672 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, 673 struct inode *inode, struct page *page, 674 nid_t ino, enum page_type type) 675 { 676 __submit_merged_write_cond(sbi, inode, page, ino, type, false); 677 } 678 679 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi) 680 { 681 f2fs_submit_merged_write(sbi, DATA); 682 f2fs_submit_merged_write(sbi, NODE); 683 f2fs_submit_merged_write(sbi, META); 684 } 685 686 /* 687 * Fill the locked page with data located in the block address. 688 * A caller needs to unlock the page on failure. 689 */ 690 int f2fs_submit_page_bio(struct f2fs_io_info *fio) 691 { 692 struct bio *bio; 693 struct page *page = fio->encrypted_page ? 694 fio->encrypted_page : fio->page; 695 696 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, 697 fio->is_por ? META_POR : (__is_meta_io(fio) ? 698 META_GENERIC : DATA_GENERIC_ENHANCE))) 699 return -EFSCORRUPTED; 700 701 trace_f2fs_submit_page_bio(page, fio); 702 703 /* Allocate a new bio */ 704 bio = __bio_alloc(fio, 1); 705 706 f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, 707 fio->page->index, fio, GFP_NOIO); 708 709 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { 710 bio_put(bio); 711 return -EFAULT; 712 } 713 714 if (fio->io_wbc && !is_read_io(fio->op)) 715 wbc_account_cgroup_owner(fio->io_wbc, fio->page, PAGE_SIZE); 716 717 inc_page_count(fio->sbi, is_read_io(fio->op) ? 718 __read_io_type(page) : WB_DATA_TYPE(fio->page, false)); 719 720 if (is_read_io(bio_op(bio))) 721 f2fs_submit_read_bio(fio->sbi, bio, fio->type); 722 else 723 f2fs_submit_write_bio(fio->sbi, bio, fio->type); 724 return 0; 725 } 726 727 static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, 728 block_t last_blkaddr, block_t cur_blkaddr) 729 { 730 if (unlikely(sbi->max_io_bytes && 731 bio->bi_iter.bi_size >= sbi->max_io_bytes)) 732 return false; 733 if (last_blkaddr + 1 != cur_blkaddr) 734 return false; 735 return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL); 736 } 737 738 static bool io_type_is_mergeable(struct f2fs_bio_info *io, 739 struct f2fs_io_info *fio) 740 { 741 if (io->fio.op != fio->op) 742 return false; 743 return io->fio.op_flags == fio->op_flags; 744 } 745 746 static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, 747 struct f2fs_bio_info *io, 748 struct f2fs_io_info *fio, 749 block_t last_blkaddr, 750 block_t cur_blkaddr) 751 { 752 if (!page_is_mergeable(sbi, bio, last_blkaddr, cur_blkaddr)) 753 return false; 754 return io_type_is_mergeable(io, fio); 755 } 756 757 static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio, 758 struct page *page, enum temp_type temp) 759 { 760 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; 761 struct bio_entry *be; 762 763 be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS, true, NULL); 764 be->bio = bio; 765 bio_get(bio); 766 767 if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE) 768 f2fs_bug_on(sbi, 1); 769 770 f2fs_down_write(&io->bio_list_lock); 771 list_add_tail(&be->list, &io->bio_list); 772 f2fs_up_write(&io->bio_list_lock); 773 } 774 775 static void del_bio_entry(struct bio_entry *be) 776 { 777 list_del(&be->list); 778 kmem_cache_free(bio_entry_slab, be); 779 } 780 781 static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio, 782 struct page *page) 783 { 784 struct f2fs_sb_info *sbi = fio->sbi; 785 enum temp_type temp; 786 bool found = false; 787 int ret = -EAGAIN; 788 789 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) { 790 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; 791 struct list_head *head = &io->bio_list; 792 struct bio_entry *be; 793 794 f2fs_down_write(&io->bio_list_lock); 795 list_for_each_entry(be, head, list) { 796 if (be->bio != *bio) 797 continue; 798 799 found = true; 800 801 f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio, 802 *fio->last_block, 803 fio->new_blkaddr)); 804 if (f2fs_crypt_mergeable_bio(*bio, 805 fio->page->mapping->host, 806 fio->page->index, fio) && 807 bio_add_page(*bio, page, PAGE_SIZE, 0) == 808 PAGE_SIZE) { 809 ret = 0; 810 break; 811 } 812 813 /* page can't be merged into bio; submit the bio */ 814 del_bio_entry(be); 815 f2fs_submit_write_bio(sbi, *bio, DATA); 816 break; 817 } 818 f2fs_up_write(&io->bio_list_lock); 819 } 820 821 if (ret) { 822 bio_put(*bio); 823 *bio = NULL; 824 } 825 826 return ret; 827 } 828 829 void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, 830 struct bio **bio, struct page *page) 831 { 832 enum temp_type temp; 833 bool found = false; 834 struct bio *target = bio ? *bio : NULL; 835 836 f2fs_bug_on(sbi, !target && !page); 837 838 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) { 839 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; 840 struct list_head *head = &io->bio_list; 841 struct bio_entry *be; 842 843 if (list_empty(head)) 844 continue; 845 846 f2fs_down_read(&io->bio_list_lock); 847 list_for_each_entry(be, head, list) { 848 if (target) 849 found = (target == be->bio); 850 else 851 found = __has_merged_page(be->bio, NULL, 852 page, 0); 853 if (found) 854 break; 855 } 856 f2fs_up_read(&io->bio_list_lock); 857 858 if (!found) 859 continue; 860 861 found = false; 862 863 f2fs_down_write(&io->bio_list_lock); 864 list_for_each_entry(be, head, list) { 865 if (target) 866 found = (target == be->bio); 867 else 868 found = __has_merged_page(be->bio, NULL, 869 page, 0); 870 if (found) { 871 target = be->bio; 872 del_bio_entry(be); 873 break; 874 } 875 } 876 f2fs_up_write(&io->bio_list_lock); 877 } 878 879 if (found) 880 f2fs_submit_write_bio(sbi, target, DATA); 881 if (bio && *bio) { 882 bio_put(*bio); 883 *bio = NULL; 884 } 885 } 886 887 int f2fs_merge_page_bio(struct f2fs_io_info *fio) 888 { 889 struct bio *bio = *fio->bio; 890 struct page *page = fio->encrypted_page ? 891 fio->encrypted_page : fio->page; 892 893 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, 894 __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC)) 895 return -EFSCORRUPTED; 896 897 trace_f2fs_submit_page_bio(page, fio); 898 899 if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block, 900 fio->new_blkaddr)) 901 f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL); 902 alloc_new: 903 if (!bio) { 904 bio = __bio_alloc(fio, BIO_MAX_VECS); 905 f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, 906 fio->page->index, fio, GFP_NOIO); 907 908 add_bio_entry(fio->sbi, bio, page, fio->temp); 909 } else { 910 if (add_ipu_page(fio, &bio, page)) 911 goto alloc_new; 912 } 913 914 if (fio->io_wbc) 915 wbc_account_cgroup_owner(fio->io_wbc, fio->page, PAGE_SIZE); 916 917 inc_page_count(fio->sbi, WB_DATA_TYPE(page, false)); 918 919 *fio->last_block = fio->new_blkaddr; 920 *fio->bio = bio; 921 922 return 0; 923 } 924 925 #ifdef CONFIG_BLK_DEV_ZONED 926 static bool is_end_zone_blkaddr(struct f2fs_sb_info *sbi, block_t blkaddr) 927 { 928 struct block_device *bdev = sbi->sb->s_bdev; 929 int devi = 0; 930 931 if (f2fs_is_multi_device(sbi)) { 932 devi = f2fs_target_device_index(sbi, blkaddr); 933 if (blkaddr < FDEV(devi).start_blk || 934 blkaddr > FDEV(devi).end_blk) { 935 f2fs_err(sbi, "Invalid block %x", blkaddr); 936 return false; 937 } 938 blkaddr -= FDEV(devi).start_blk; 939 bdev = FDEV(devi).bdev; 940 } 941 return bdev_is_zoned(bdev) && 942 f2fs_blkz_is_seq(sbi, devi, blkaddr) && 943 (blkaddr % sbi->blocks_per_blkz == sbi->blocks_per_blkz - 1); 944 } 945 #endif 946 947 void f2fs_submit_page_write(struct f2fs_io_info *fio) 948 { 949 struct f2fs_sb_info *sbi = fio->sbi; 950 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); 951 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; 952 struct page *bio_page; 953 enum count_type type; 954 955 f2fs_bug_on(sbi, is_read_io(fio->op)); 956 957 f2fs_down_write(&io->io_rwsem); 958 next: 959 #ifdef CONFIG_BLK_DEV_ZONED 960 if (f2fs_sb_has_blkzoned(sbi) && btype < META && io->zone_pending_bio) { 961 wait_for_completion_io(&io->zone_wait); 962 bio_put(io->zone_pending_bio); 963 io->zone_pending_bio = NULL; 964 io->bi_private = NULL; 965 } 966 #endif 967 968 if (fio->in_list) { 969 spin_lock(&io->io_lock); 970 if (list_empty(&io->io_list)) { 971 spin_unlock(&io->io_lock); 972 goto out; 973 } 974 fio = list_first_entry(&io->io_list, 975 struct f2fs_io_info, list); 976 list_del(&fio->list); 977 spin_unlock(&io->io_lock); 978 } 979 980 verify_fio_blkaddr(fio); 981 982 if (fio->encrypted_page) 983 bio_page = fio->encrypted_page; 984 else if (fio->compressed_page) 985 bio_page = fio->compressed_page; 986 else 987 bio_page = fio->page; 988 989 /* set submitted = true as a return value */ 990 fio->submitted = 1; 991 992 type = WB_DATA_TYPE(bio_page, fio->compressed_page); 993 inc_page_count(sbi, type); 994 995 if (io->bio && 996 (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio, 997 fio->new_blkaddr) || 998 !f2fs_crypt_mergeable_bio(io->bio, fio->page->mapping->host, 999 bio_page->index, fio))) 1000 __submit_merged_bio(io); 1001 alloc_new: 1002 if (io->bio == NULL) { 1003 io->bio = __bio_alloc(fio, BIO_MAX_VECS); 1004 f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host, 1005 bio_page->index, fio, GFP_NOIO); 1006 io->fio = *fio; 1007 } 1008 1009 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) { 1010 __submit_merged_bio(io); 1011 goto alloc_new; 1012 } 1013 1014 if (fio->io_wbc) 1015 wbc_account_cgroup_owner(fio->io_wbc, fio->page, PAGE_SIZE); 1016 1017 io->last_block_in_bio = fio->new_blkaddr; 1018 1019 trace_f2fs_submit_page_write(fio->page, fio); 1020 #ifdef CONFIG_BLK_DEV_ZONED 1021 if (f2fs_sb_has_blkzoned(sbi) && btype < META && 1022 is_end_zone_blkaddr(sbi, fio->new_blkaddr)) { 1023 bio_get(io->bio); 1024 reinit_completion(&io->zone_wait); 1025 io->bi_private = io->bio->bi_private; 1026 io->bio->bi_private = io; 1027 io->bio->bi_end_io = f2fs_zone_write_end_io; 1028 io->zone_pending_bio = io->bio; 1029 __submit_merged_bio(io); 1030 } 1031 #endif 1032 if (fio->in_list) 1033 goto next; 1034 out: 1035 if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || 1036 !f2fs_is_checkpoint_ready(sbi)) 1037 __submit_merged_bio(io); 1038 f2fs_up_write(&io->io_rwsem); 1039 } 1040 1041 static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, 1042 unsigned nr_pages, blk_opf_t op_flag, 1043 pgoff_t first_idx, bool for_write) 1044 { 1045 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1046 struct bio *bio; 1047 struct bio_post_read_ctx *ctx = NULL; 1048 unsigned int post_read_steps = 0; 1049 sector_t sector; 1050 struct block_device *bdev = f2fs_target_device(sbi, blkaddr, §or); 1051 1052 bio = bio_alloc_bioset(bdev, bio_max_segs(nr_pages), 1053 REQ_OP_READ | op_flag, 1054 for_write ? GFP_NOIO : GFP_KERNEL, &f2fs_bioset); 1055 if (!bio) 1056 return ERR_PTR(-ENOMEM); 1057 bio->bi_iter.bi_sector = sector; 1058 f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS); 1059 bio->bi_end_io = f2fs_read_end_io; 1060 1061 if (fscrypt_inode_uses_fs_layer_crypto(inode)) 1062 post_read_steps |= STEP_DECRYPT; 1063 1064 if (f2fs_need_verity(inode, first_idx)) 1065 post_read_steps |= STEP_VERITY; 1066 1067 /* 1068 * STEP_DECOMPRESS is handled specially, since a compressed file might 1069 * contain both compressed and uncompressed clusters. We'll allocate a 1070 * bio_post_read_ctx if the file is compressed, but the caller is 1071 * responsible for enabling STEP_DECOMPRESS if it's actually needed. 1072 */ 1073 1074 if (post_read_steps || f2fs_compressed_file(inode)) { 1075 /* Due to the mempool, this never fails. */ 1076 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS); 1077 ctx->bio = bio; 1078 ctx->sbi = sbi; 1079 ctx->enabled_steps = post_read_steps; 1080 ctx->fs_blkaddr = blkaddr; 1081 ctx->decompression_attempted = false; 1082 bio->bi_private = ctx; 1083 } 1084 iostat_alloc_and_bind_ctx(sbi, bio, ctx); 1085 1086 return bio; 1087 } 1088 1089 /* This can handle encryption stuffs */ 1090 static int f2fs_submit_page_read(struct inode *inode, struct page *page, 1091 block_t blkaddr, blk_opf_t op_flags, 1092 bool for_write) 1093 { 1094 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1095 struct bio *bio; 1096 1097 bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags, 1098 page->index, for_write); 1099 if (IS_ERR(bio)) 1100 return PTR_ERR(bio); 1101 1102 /* wait for GCed page writeback via META_MAPPING */ 1103 f2fs_wait_on_block_writeback(inode, blkaddr); 1104 1105 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { 1106 iostat_update_and_unbind_ctx(bio); 1107 if (bio->bi_private) 1108 mempool_free(bio->bi_private, bio_post_read_ctx_pool); 1109 bio_put(bio); 1110 return -EFAULT; 1111 } 1112 inc_page_count(sbi, F2FS_RD_DATA); 1113 f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE); 1114 f2fs_submit_read_bio(sbi, bio, DATA); 1115 return 0; 1116 } 1117 1118 static void __set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) 1119 { 1120 __le32 *addr = get_dnode_addr(dn->inode, dn->node_page); 1121 1122 dn->data_blkaddr = blkaddr; 1123 addr[dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr); 1124 } 1125 1126 /* 1127 * Lock ordering for the change of data block address: 1128 * ->data_page 1129 * ->node_page 1130 * update block addresses in the node page 1131 */ 1132 void f2fs_set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) 1133 { 1134 f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true); 1135 __set_data_blkaddr(dn, blkaddr); 1136 if (set_page_dirty(dn->node_page)) 1137 dn->node_changed = true; 1138 } 1139 1140 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) 1141 { 1142 f2fs_set_data_blkaddr(dn, blkaddr); 1143 f2fs_update_read_extent_cache(dn); 1144 } 1145 1146 /* dn->ofs_in_node will be returned with up-to-date last block pointer */ 1147 int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count) 1148 { 1149 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1150 int err; 1151 1152 if (!count) 1153 return 0; 1154 1155 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) 1156 return -EPERM; 1157 err = inc_valid_block_count(sbi, dn->inode, &count, true); 1158 if (unlikely(err)) 1159 return err; 1160 1161 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid, 1162 dn->ofs_in_node, count); 1163 1164 f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true); 1165 1166 for (; count > 0; dn->ofs_in_node++) { 1167 block_t blkaddr = f2fs_data_blkaddr(dn); 1168 1169 if (blkaddr == NULL_ADDR) { 1170 __set_data_blkaddr(dn, NEW_ADDR); 1171 count--; 1172 } 1173 } 1174 1175 if (set_page_dirty(dn->node_page)) 1176 dn->node_changed = true; 1177 return 0; 1178 } 1179 1180 /* Should keep dn->ofs_in_node unchanged */ 1181 int f2fs_reserve_new_block(struct dnode_of_data *dn) 1182 { 1183 unsigned int ofs_in_node = dn->ofs_in_node; 1184 int ret; 1185 1186 ret = f2fs_reserve_new_blocks(dn, 1); 1187 dn->ofs_in_node = ofs_in_node; 1188 return ret; 1189 } 1190 1191 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) 1192 { 1193 bool need_put = dn->inode_page ? false : true; 1194 int err; 1195 1196 err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE); 1197 if (err) 1198 return err; 1199 1200 if (dn->data_blkaddr == NULL_ADDR) 1201 err = f2fs_reserve_new_block(dn); 1202 if (err || need_put) 1203 f2fs_put_dnode(dn); 1204 return err; 1205 } 1206 1207 struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index, 1208 blk_opf_t op_flags, bool for_write, 1209 pgoff_t *next_pgofs) 1210 { 1211 struct address_space *mapping = inode->i_mapping; 1212 struct dnode_of_data dn; 1213 struct page *page; 1214 int err; 1215 1216 page = f2fs_grab_cache_page(mapping, index, for_write); 1217 if (!page) 1218 return ERR_PTR(-ENOMEM); 1219 1220 if (f2fs_lookup_read_extent_cache_block(inode, index, 1221 &dn.data_blkaddr)) { 1222 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr, 1223 DATA_GENERIC_ENHANCE_READ)) { 1224 err = -EFSCORRUPTED; 1225 goto put_err; 1226 } 1227 goto got_it; 1228 } 1229 1230 set_new_dnode(&dn, inode, NULL, NULL, 0); 1231 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 1232 if (err) { 1233 if (err == -ENOENT && next_pgofs) 1234 *next_pgofs = f2fs_get_next_page_offset(&dn, index); 1235 goto put_err; 1236 } 1237 f2fs_put_dnode(&dn); 1238 1239 if (unlikely(dn.data_blkaddr == NULL_ADDR)) { 1240 err = -ENOENT; 1241 if (next_pgofs) 1242 *next_pgofs = index + 1; 1243 goto put_err; 1244 } 1245 if (dn.data_blkaddr != NEW_ADDR && 1246 !f2fs_is_valid_blkaddr(F2FS_I_SB(inode), 1247 dn.data_blkaddr, 1248 DATA_GENERIC_ENHANCE)) { 1249 err = -EFSCORRUPTED; 1250 goto put_err; 1251 } 1252 got_it: 1253 if (PageUptodate(page)) { 1254 unlock_page(page); 1255 return page; 1256 } 1257 1258 /* 1259 * A new dentry page is allocated but not able to be written, since its 1260 * new inode page couldn't be allocated due to -ENOSPC. 1261 * In such the case, its blkaddr can be remained as NEW_ADDR. 1262 * see, f2fs_add_link -> f2fs_get_new_data_page -> 1263 * f2fs_init_inode_metadata. 1264 */ 1265 if (dn.data_blkaddr == NEW_ADDR) { 1266 zero_user_segment(page, 0, PAGE_SIZE); 1267 if (!PageUptodate(page)) 1268 SetPageUptodate(page); 1269 unlock_page(page); 1270 return page; 1271 } 1272 1273 err = f2fs_submit_page_read(inode, page, dn.data_blkaddr, 1274 op_flags, for_write); 1275 if (err) 1276 goto put_err; 1277 return page; 1278 1279 put_err: 1280 f2fs_put_page(page, 1); 1281 return ERR_PTR(err); 1282 } 1283 1284 struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index, 1285 pgoff_t *next_pgofs) 1286 { 1287 struct address_space *mapping = inode->i_mapping; 1288 struct page *page; 1289 1290 page = find_get_page(mapping, index); 1291 if (page && PageUptodate(page)) 1292 return page; 1293 f2fs_put_page(page, 0); 1294 1295 page = f2fs_get_read_data_page(inode, index, 0, false, next_pgofs); 1296 if (IS_ERR(page)) 1297 return page; 1298 1299 if (PageUptodate(page)) 1300 return page; 1301 1302 wait_on_page_locked(page); 1303 if (unlikely(!PageUptodate(page))) { 1304 f2fs_put_page(page, 0); 1305 return ERR_PTR(-EIO); 1306 } 1307 return page; 1308 } 1309 1310 /* 1311 * If it tries to access a hole, return an error. 1312 * Because, the callers, functions in dir.c and GC, should be able to know 1313 * whether this page exists or not. 1314 */ 1315 struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index, 1316 bool for_write) 1317 { 1318 struct address_space *mapping = inode->i_mapping; 1319 struct page *page; 1320 1321 page = f2fs_get_read_data_page(inode, index, 0, for_write, NULL); 1322 if (IS_ERR(page)) 1323 return page; 1324 1325 /* wait for read completion */ 1326 lock_page(page); 1327 if (unlikely(page->mapping != mapping || !PageUptodate(page))) { 1328 f2fs_put_page(page, 1); 1329 return ERR_PTR(-EIO); 1330 } 1331 return page; 1332 } 1333 1334 /* 1335 * Caller ensures that this data page is never allocated. 1336 * A new zero-filled data page is allocated in the page cache. 1337 * 1338 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and 1339 * f2fs_unlock_op(). 1340 * Note that, ipage is set only by make_empty_dir, and if any error occur, 1341 * ipage should be released by this function. 1342 */ 1343 struct page *f2fs_get_new_data_page(struct inode *inode, 1344 struct page *ipage, pgoff_t index, bool new_i_size) 1345 { 1346 struct address_space *mapping = inode->i_mapping; 1347 struct page *page; 1348 struct dnode_of_data dn; 1349 int err; 1350 1351 page = f2fs_grab_cache_page(mapping, index, true); 1352 if (!page) { 1353 /* 1354 * before exiting, we should make sure ipage will be released 1355 * if any error occur. 1356 */ 1357 f2fs_put_page(ipage, 1); 1358 return ERR_PTR(-ENOMEM); 1359 } 1360 1361 set_new_dnode(&dn, inode, ipage, NULL, 0); 1362 err = f2fs_reserve_block(&dn, index); 1363 if (err) { 1364 f2fs_put_page(page, 1); 1365 return ERR_PTR(err); 1366 } 1367 if (!ipage) 1368 f2fs_put_dnode(&dn); 1369 1370 if (PageUptodate(page)) 1371 goto got_it; 1372 1373 if (dn.data_blkaddr == NEW_ADDR) { 1374 zero_user_segment(page, 0, PAGE_SIZE); 1375 if (!PageUptodate(page)) 1376 SetPageUptodate(page); 1377 } else { 1378 f2fs_put_page(page, 1); 1379 1380 /* if ipage exists, blkaddr should be NEW_ADDR */ 1381 f2fs_bug_on(F2FS_I_SB(inode), ipage); 1382 page = f2fs_get_lock_data_page(inode, index, true); 1383 if (IS_ERR(page)) 1384 return page; 1385 } 1386 got_it: 1387 if (new_i_size && i_size_read(inode) < 1388 ((loff_t)(index + 1) << PAGE_SHIFT)) 1389 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT)); 1390 return page; 1391 } 1392 1393 static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) 1394 { 1395 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1396 struct f2fs_summary sum; 1397 struct node_info ni; 1398 block_t old_blkaddr; 1399 blkcnt_t count = 1; 1400 int err; 1401 1402 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) 1403 return -EPERM; 1404 1405 err = f2fs_get_node_info(sbi, dn->nid, &ni, false); 1406 if (err) 1407 return err; 1408 1409 dn->data_blkaddr = f2fs_data_blkaddr(dn); 1410 if (dn->data_blkaddr == NULL_ADDR) { 1411 err = inc_valid_block_count(sbi, dn->inode, &count, true); 1412 if (unlikely(err)) 1413 return err; 1414 } 1415 1416 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 1417 old_blkaddr = dn->data_blkaddr; 1418 err = f2fs_allocate_data_block(sbi, NULL, old_blkaddr, 1419 &dn->data_blkaddr, &sum, seg_type, NULL); 1420 if (err) 1421 return err; 1422 1423 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 1424 f2fs_invalidate_internal_cache(sbi, old_blkaddr); 1425 1426 f2fs_update_data_blkaddr(dn, dn->data_blkaddr); 1427 return 0; 1428 } 1429 1430 static void f2fs_map_lock(struct f2fs_sb_info *sbi, int flag) 1431 { 1432 if (flag == F2FS_GET_BLOCK_PRE_AIO) 1433 f2fs_down_read(&sbi->node_change); 1434 else 1435 f2fs_lock_op(sbi); 1436 } 1437 1438 static void f2fs_map_unlock(struct f2fs_sb_info *sbi, int flag) 1439 { 1440 if (flag == F2FS_GET_BLOCK_PRE_AIO) 1441 f2fs_up_read(&sbi->node_change); 1442 else 1443 f2fs_unlock_op(sbi); 1444 } 1445 1446 int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index) 1447 { 1448 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1449 int err = 0; 1450 1451 f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); 1452 if (!f2fs_lookup_read_extent_cache_block(dn->inode, index, 1453 &dn->data_blkaddr)) 1454 err = f2fs_reserve_block(dn, index); 1455 f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO); 1456 1457 return err; 1458 } 1459 1460 static int f2fs_map_no_dnode(struct inode *inode, 1461 struct f2fs_map_blocks *map, struct dnode_of_data *dn, 1462 pgoff_t pgoff) 1463 { 1464 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1465 1466 /* 1467 * There is one exceptional case that read_node_page() may return 1468 * -ENOENT due to filesystem has been shutdown or cp_error, return 1469 * -EIO in that case. 1470 */ 1471 if (map->m_may_create && 1472 (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || f2fs_cp_error(sbi))) 1473 return -EIO; 1474 1475 if (map->m_next_pgofs) 1476 *map->m_next_pgofs = f2fs_get_next_page_offset(dn, pgoff); 1477 if (map->m_next_extent) 1478 *map->m_next_extent = f2fs_get_next_page_offset(dn, pgoff); 1479 return 0; 1480 } 1481 1482 static bool f2fs_map_blocks_cached(struct inode *inode, 1483 struct f2fs_map_blocks *map, int flag) 1484 { 1485 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1486 unsigned int maxblocks = map->m_len; 1487 pgoff_t pgoff = (pgoff_t)map->m_lblk; 1488 struct extent_info ei = {}; 1489 1490 if (!f2fs_lookup_read_extent_cache(inode, pgoff, &ei)) 1491 return false; 1492 1493 map->m_pblk = ei.blk + pgoff - ei.fofs; 1494 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgoff); 1495 map->m_flags = F2FS_MAP_MAPPED; 1496 if (map->m_next_extent) 1497 *map->m_next_extent = pgoff + map->m_len; 1498 1499 /* for hardware encryption, but to avoid potential issue in future */ 1500 if (flag == F2FS_GET_BLOCK_DIO) 1501 f2fs_wait_on_block_writeback_range(inode, 1502 map->m_pblk, map->m_len); 1503 1504 if (f2fs_allow_multi_device_dio(sbi, flag)) { 1505 int bidx = f2fs_target_device_index(sbi, map->m_pblk); 1506 struct f2fs_dev_info *dev = &sbi->devs[bidx]; 1507 1508 map->m_bdev = dev->bdev; 1509 map->m_pblk -= dev->start_blk; 1510 map->m_len = min(map->m_len, dev->end_blk + 1 - map->m_pblk); 1511 } else { 1512 map->m_bdev = inode->i_sb->s_bdev; 1513 } 1514 return true; 1515 } 1516 1517 static bool map_is_mergeable(struct f2fs_sb_info *sbi, 1518 struct f2fs_map_blocks *map, 1519 block_t blkaddr, int flag, int bidx, 1520 int ofs) 1521 { 1522 if (map->m_multidev_dio && map->m_bdev != FDEV(bidx).bdev) 1523 return false; 1524 if (map->m_pblk != NEW_ADDR && blkaddr == (map->m_pblk + ofs)) 1525 return true; 1526 if (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) 1527 return true; 1528 if (flag == F2FS_GET_BLOCK_PRE_DIO) 1529 return true; 1530 if (flag == F2FS_GET_BLOCK_DIO && 1531 map->m_pblk == NULL_ADDR && blkaddr == NULL_ADDR) 1532 return true; 1533 return false; 1534 } 1535 1536 /* 1537 * f2fs_map_blocks() tries to find or build mapping relationship which 1538 * maps continuous logical blocks to physical blocks, and return such 1539 * info via f2fs_map_blocks structure. 1540 */ 1541 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) 1542 { 1543 unsigned int maxblocks = map->m_len; 1544 struct dnode_of_data dn; 1545 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1546 int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE; 1547 pgoff_t pgofs, end_offset, end; 1548 int err = 0, ofs = 1; 1549 unsigned int ofs_in_node, last_ofs_in_node; 1550 blkcnt_t prealloc; 1551 block_t blkaddr; 1552 unsigned int start_pgofs; 1553 int bidx = 0; 1554 bool is_hole; 1555 1556 if (!maxblocks) 1557 return 0; 1558 1559 if (!map->m_may_create && f2fs_map_blocks_cached(inode, map, flag)) 1560 goto out; 1561 1562 map->m_bdev = inode->i_sb->s_bdev; 1563 map->m_multidev_dio = 1564 f2fs_allow_multi_device_dio(F2FS_I_SB(inode), flag); 1565 1566 map->m_len = 0; 1567 map->m_flags = 0; 1568 1569 /* it only supports block size == page size */ 1570 pgofs = (pgoff_t)map->m_lblk; 1571 end = pgofs + maxblocks; 1572 1573 next_dnode: 1574 if (map->m_may_create) 1575 f2fs_map_lock(sbi, flag); 1576 1577 /* When reading holes, we need its node page */ 1578 set_new_dnode(&dn, inode, NULL, NULL, 0); 1579 err = f2fs_get_dnode_of_data(&dn, pgofs, mode); 1580 if (err) { 1581 if (flag == F2FS_GET_BLOCK_BMAP) 1582 map->m_pblk = 0; 1583 if (err == -ENOENT) 1584 err = f2fs_map_no_dnode(inode, map, &dn, pgofs); 1585 goto unlock_out; 1586 } 1587 1588 start_pgofs = pgofs; 1589 prealloc = 0; 1590 last_ofs_in_node = ofs_in_node = dn.ofs_in_node; 1591 end_offset = ADDRS_PER_PAGE(dn.node_page, inode); 1592 1593 next_block: 1594 blkaddr = f2fs_data_blkaddr(&dn); 1595 is_hole = !__is_valid_data_blkaddr(blkaddr); 1596 if (!is_hole && 1597 !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) { 1598 err = -EFSCORRUPTED; 1599 goto sync_out; 1600 } 1601 1602 /* use out-place-update for direct IO under LFS mode */ 1603 if (map->m_may_create && (is_hole || 1604 (flag == F2FS_GET_BLOCK_DIO && f2fs_lfs_mode(sbi) && 1605 !f2fs_is_pinned_file(inode)))) { 1606 if (unlikely(f2fs_cp_error(sbi))) { 1607 err = -EIO; 1608 goto sync_out; 1609 } 1610 1611 switch (flag) { 1612 case F2FS_GET_BLOCK_PRE_AIO: 1613 if (blkaddr == NULL_ADDR) { 1614 prealloc++; 1615 last_ofs_in_node = dn.ofs_in_node; 1616 } 1617 break; 1618 case F2FS_GET_BLOCK_PRE_DIO: 1619 case F2FS_GET_BLOCK_DIO: 1620 err = __allocate_data_block(&dn, map->m_seg_type); 1621 if (err) 1622 goto sync_out; 1623 if (flag == F2FS_GET_BLOCK_PRE_DIO) 1624 file_need_truncate(inode); 1625 set_inode_flag(inode, FI_APPEND_WRITE); 1626 break; 1627 default: 1628 WARN_ON_ONCE(1); 1629 err = -EIO; 1630 goto sync_out; 1631 } 1632 1633 blkaddr = dn.data_blkaddr; 1634 if (is_hole) 1635 map->m_flags |= F2FS_MAP_NEW; 1636 } else if (is_hole) { 1637 if (f2fs_compressed_file(inode) && 1638 f2fs_sanity_check_cluster(&dn)) { 1639 err = -EFSCORRUPTED; 1640 f2fs_handle_error(sbi, 1641 ERROR_CORRUPTED_CLUSTER); 1642 goto sync_out; 1643 } 1644 1645 switch (flag) { 1646 case F2FS_GET_BLOCK_PRECACHE: 1647 goto sync_out; 1648 case F2FS_GET_BLOCK_BMAP: 1649 map->m_pblk = 0; 1650 goto sync_out; 1651 case F2FS_GET_BLOCK_FIEMAP: 1652 if (blkaddr == NULL_ADDR) { 1653 if (map->m_next_pgofs) 1654 *map->m_next_pgofs = pgofs + 1; 1655 goto sync_out; 1656 } 1657 break; 1658 case F2FS_GET_BLOCK_DIO: 1659 if (map->m_next_pgofs) 1660 *map->m_next_pgofs = pgofs + 1; 1661 break; 1662 default: 1663 /* for defragment case */ 1664 if (map->m_next_pgofs) 1665 *map->m_next_pgofs = pgofs + 1; 1666 goto sync_out; 1667 } 1668 } 1669 1670 if (flag == F2FS_GET_BLOCK_PRE_AIO) 1671 goto skip; 1672 1673 if (map->m_multidev_dio) 1674 bidx = f2fs_target_device_index(sbi, blkaddr); 1675 1676 if (map->m_len == 0) { 1677 /* reserved delalloc block should be mapped for fiemap. */ 1678 if (blkaddr == NEW_ADDR) 1679 map->m_flags |= F2FS_MAP_DELALLOC; 1680 if (flag != F2FS_GET_BLOCK_DIO || !is_hole) 1681 map->m_flags |= F2FS_MAP_MAPPED; 1682 1683 map->m_pblk = blkaddr; 1684 map->m_len = 1; 1685 1686 if (map->m_multidev_dio) 1687 map->m_bdev = FDEV(bidx).bdev; 1688 } else if (map_is_mergeable(sbi, map, blkaddr, flag, bidx, ofs)) { 1689 ofs++; 1690 map->m_len++; 1691 } else { 1692 goto sync_out; 1693 } 1694 1695 skip: 1696 dn.ofs_in_node++; 1697 pgofs++; 1698 1699 /* preallocate blocks in batch for one dnode page */ 1700 if (flag == F2FS_GET_BLOCK_PRE_AIO && 1701 (pgofs == end || dn.ofs_in_node == end_offset)) { 1702 1703 dn.ofs_in_node = ofs_in_node; 1704 err = f2fs_reserve_new_blocks(&dn, prealloc); 1705 if (err) 1706 goto sync_out; 1707 1708 map->m_len += dn.ofs_in_node - ofs_in_node; 1709 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) { 1710 err = -ENOSPC; 1711 goto sync_out; 1712 } 1713 dn.ofs_in_node = end_offset; 1714 } 1715 1716 if (pgofs >= end) 1717 goto sync_out; 1718 else if (dn.ofs_in_node < end_offset) 1719 goto next_block; 1720 1721 if (flag == F2FS_GET_BLOCK_PRECACHE) { 1722 if (map->m_flags & F2FS_MAP_MAPPED) { 1723 unsigned int ofs = start_pgofs - map->m_lblk; 1724 1725 f2fs_update_read_extent_cache_range(&dn, 1726 start_pgofs, map->m_pblk + ofs, 1727 map->m_len - ofs); 1728 } 1729 } 1730 1731 f2fs_put_dnode(&dn); 1732 1733 if (map->m_may_create) { 1734 f2fs_map_unlock(sbi, flag); 1735 f2fs_balance_fs(sbi, dn.node_changed); 1736 } 1737 goto next_dnode; 1738 1739 sync_out: 1740 1741 if (flag == F2FS_GET_BLOCK_DIO && map->m_flags & F2FS_MAP_MAPPED) { 1742 /* 1743 * for hardware encryption, but to avoid potential issue 1744 * in future 1745 */ 1746 f2fs_wait_on_block_writeback_range(inode, 1747 map->m_pblk, map->m_len); 1748 1749 if (map->m_multidev_dio) { 1750 block_t blk_addr = map->m_pblk; 1751 1752 bidx = f2fs_target_device_index(sbi, map->m_pblk); 1753 1754 map->m_bdev = FDEV(bidx).bdev; 1755 map->m_pblk -= FDEV(bidx).start_blk; 1756 1757 if (map->m_may_create) 1758 f2fs_update_device_state(sbi, inode->i_ino, 1759 blk_addr, map->m_len); 1760 1761 f2fs_bug_on(sbi, blk_addr + map->m_len > 1762 FDEV(bidx).end_blk + 1); 1763 } 1764 } 1765 1766 if (flag == F2FS_GET_BLOCK_PRECACHE) { 1767 if (map->m_flags & F2FS_MAP_MAPPED) { 1768 unsigned int ofs = start_pgofs - map->m_lblk; 1769 1770 f2fs_update_read_extent_cache_range(&dn, 1771 start_pgofs, map->m_pblk + ofs, 1772 map->m_len - ofs); 1773 } 1774 if (map->m_next_extent) 1775 *map->m_next_extent = pgofs + 1; 1776 } 1777 f2fs_put_dnode(&dn); 1778 unlock_out: 1779 if (map->m_may_create) { 1780 f2fs_map_unlock(sbi, flag); 1781 f2fs_balance_fs(sbi, dn.node_changed); 1782 } 1783 out: 1784 trace_f2fs_map_blocks(inode, map, flag, err); 1785 return err; 1786 } 1787 1788 bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) 1789 { 1790 struct f2fs_map_blocks map; 1791 block_t last_lblk; 1792 int err; 1793 1794 if (pos + len > i_size_read(inode)) 1795 return false; 1796 1797 map.m_lblk = F2FS_BYTES_TO_BLK(pos); 1798 map.m_next_pgofs = NULL; 1799 map.m_next_extent = NULL; 1800 map.m_seg_type = NO_CHECK_TYPE; 1801 map.m_may_create = false; 1802 last_lblk = F2FS_BLK_ALIGN(pos + len); 1803 1804 while (map.m_lblk < last_lblk) { 1805 map.m_len = last_lblk - map.m_lblk; 1806 err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DEFAULT); 1807 if (err || map.m_len == 0) 1808 return false; 1809 map.m_lblk += map.m_len; 1810 } 1811 return true; 1812 } 1813 1814 static inline u64 bytes_to_blks(struct inode *inode, u64 bytes) 1815 { 1816 return (bytes >> inode->i_blkbits); 1817 } 1818 1819 static inline u64 blks_to_bytes(struct inode *inode, u64 blks) 1820 { 1821 return (blks << inode->i_blkbits); 1822 } 1823 1824 static int f2fs_xattr_fiemap(struct inode *inode, 1825 struct fiemap_extent_info *fieinfo) 1826 { 1827 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1828 struct page *page; 1829 struct node_info ni; 1830 __u64 phys = 0, len; 1831 __u32 flags; 1832 nid_t xnid = F2FS_I(inode)->i_xattr_nid; 1833 int err = 0; 1834 1835 if (f2fs_has_inline_xattr(inode)) { 1836 int offset; 1837 1838 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), 1839 inode->i_ino, false); 1840 if (!page) 1841 return -ENOMEM; 1842 1843 err = f2fs_get_node_info(sbi, inode->i_ino, &ni, false); 1844 if (err) { 1845 f2fs_put_page(page, 1); 1846 return err; 1847 } 1848 1849 phys = blks_to_bytes(inode, ni.blk_addr); 1850 offset = offsetof(struct f2fs_inode, i_addr) + 1851 sizeof(__le32) * (DEF_ADDRS_PER_INODE - 1852 get_inline_xattr_addrs(inode)); 1853 1854 phys += offset; 1855 len = inline_xattr_size(inode); 1856 1857 f2fs_put_page(page, 1); 1858 1859 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED; 1860 1861 if (!xnid) 1862 flags |= FIEMAP_EXTENT_LAST; 1863 1864 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); 1865 trace_f2fs_fiemap(inode, 0, phys, len, flags, err); 1866 if (err) 1867 return err; 1868 } 1869 1870 if (xnid) { 1871 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false); 1872 if (!page) 1873 return -ENOMEM; 1874 1875 err = f2fs_get_node_info(sbi, xnid, &ni, false); 1876 if (err) { 1877 f2fs_put_page(page, 1); 1878 return err; 1879 } 1880 1881 phys = blks_to_bytes(inode, ni.blk_addr); 1882 len = inode->i_sb->s_blocksize; 1883 1884 f2fs_put_page(page, 1); 1885 1886 flags = FIEMAP_EXTENT_LAST; 1887 } 1888 1889 if (phys) { 1890 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); 1891 trace_f2fs_fiemap(inode, 0, phys, len, flags, err); 1892 } 1893 1894 return (err < 0 ? err : 0); 1895 } 1896 1897 static loff_t max_inode_blocks(struct inode *inode) 1898 { 1899 loff_t result = ADDRS_PER_INODE(inode); 1900 loff_t leaf_count = ADDRS_PER_BLOCK(inode); 1901 1902 /* two direct node blocks */ 1903 result += (leaf_count * 2); 1904 1905 /* two indirect node blocks */ 1906 leaf_count *= NIDS_PER_BLOCK; 1907 result += (leaf_count * 2); 1908 1909 /* one double indirect node block */ 1910 leaf_count *= NIDS_PER_BLOCK; 1911 result += leaf_count; 1912 1913 return result; 1914 } 1915 1916 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 1917 u64 start, u64 len) 1918 { 1919 struct f2fs_map_blocks map; 1920 sector_t start_blk, last_blk; 1921 pgoff_t next_pgofs; 1922 u64 logical = 0, phys = 0, size = 0; 1923 u32 flags = 0; 1924 int ret = 0; 1925 bool compr_cluster = false, compr_appended; 1926 unsigned int cluster_size = F2FS_I(inode)->i_cluster_size; 1927 unsigned int count_in_cluster = 0; 1928 loff_t maxbytes; 1929 1930 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) { 1931 ret = f2fs_precache_extents(inode); 1932 if (ret) 1933 return ret; 1934 } 1935 1936 ret = fiemap_prep(inode, fieinfo, start, &len, FIEMAP_FLAG_XATTR); 1937 if (ret) 1938 return ret; 1939 1940 inode_lock_shared(inode); 1941 1942 maxbytes = max_file_blocks(inode) << F2FS_BLKSIZE_BITS; 1943 if (start > maxbytes) { 1944 ret = -EFBIG; 1945 goto out; 1946 } 1947 1948 if (len > maxbytes || (maxbytes - len) < start) 1949 len = maxbytes - start; 1950 1951 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { 1952 ret = f2fs_xattr_fiemap(inode, fieinfo); 1953 goto out; 1954 } 1955 1956 if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) { 1957 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len); 1958 if (ret != -EAGAIN) 1959 goto out; 1960 } 1961 1962 if (bytes_to_blks(inode, len) == 0) 1963 len = blks_to_bytes(inode, 1); 1964 1965 start_blk = bytes_to_blks(inode, start); 1966 last_blk = bytes_to_blks(inode, start + len - 1); 1967 1968 next: 1969 memset(&map, 0, sizeof(map)); 1970 map.m_lblk = start_blk; 1971 map.m_len = bytes_to_blks(inode, len); 1972 map.m_next_pgofs = &next_pgofs; 1973 map.m_seg_type = NO_CHECK_TYPE; 1974 1975 if (compr_cluster) { 1976 map.m_lblk += 1; 1977 map.m_len = cluster_size - count_in_cluster; 1978 } 1979 1980 ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_FIEMAP); 1981 if (ret) 1982 goto out; 1983 1984 /* HOLE */ 1985 if (!compr_cluster && !(map.m_flags & F2FS_MAP_FLAGS)) { 1986 start_blk = next_pgofs; 1987 1988 if (blks_to_bytes(inode, start_blk) < blks_to_bytes(inode, 1989 max_inode_blocks(inode))) 1990 goto prep_next; 1991 1992 flags |= FIEMAP_EXTENT_LAST; 1993 } 1994 1995 compr_appended = false; 1996 /* In a case of compressed cluster, append this to the last extent */ 1997 if (compr_cluster && ((map.m_flags & F2FS_MAP_DELALLOC) || 1998 !(map.m_flags & F2FS_MAP_FLAGS))) { 1999 compr_appended = true; 2000 goto skip_fill; 2001 } 2002 2003 if (size) { 2004 flags |= FIEMAP_EXTENT_MERGED; 2005 if (IS_ENCRYPTED(inode)) 2006 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED; 2007 2008 ret = fiemap_fill_next_extent(fieinfo, logical, 2009 phys, size, flags); 2010 trace_f2fs_fiemap(inode, logical, phys, size, flags, ret); 2011 if (ret) 2012 goto out; 2013 size = 0; 2014 } 2015 2016 if (start_blk > last_blk) 2017 goto out; 2018 2019 skip_fill: 2020 if (map.m_pblk == COMPRESS_ADDR) { 2021 compr_cluster = true; 2022 count_in_cluster = 1; 2023 } else if (compr_appended) { 2024 unsigned int appended_blks = cluster_size - 2025 count_in_cluster + 1; 2026 size += blks_to_bytes(inode, appended_blks); 2027 start_blk += appended_blks; 2028 compr_cluster = false; 2029 } else { 2030 logical = blks_to_bytes(inode, start_blk); 2031 phys = __is_valid_data_blkaddr(map.m_pblk) ? 2032 blks_to_bytes(inode, map.m_pblk) : 0; 2033 size = blks_to_bytes(inode, map.m_len); 2034 flags = 0; 2035 2036 if (compr_cluster) { 2037 flags = FIEMAP_EXTENT_ENCODED; 2038 count_in_cluster += map.m_len; 2039 if (count_in_cluster == cluster_size) { 2040 compr_cluster = false; 2041 size += blks_to_bytes(inode, 1); 2042 } 2043 } else if (map.m_flags & F2FS_MAP_DELALLOC) { 2044 flags = FIEMAP_EXTENT_UNWRITTEN; 2045 } 2046 2047 start_blk += bytes_to_blks(inode, size); 2048 } 2049 2050 prep_next: 2051 cond_resched(); 2052 if (fatal_signal_pending(current)) 2053 ret = -EINTR; 2054 else 2055 goto next; 2056 out: 2057 if (ret == 1) 2058 ret = 0; 2059 2060 inode_unlock_shared(inode); 2061 return ret; 2062 } 2063 2064 static inline loff_t f2fs_readpage_limit(struct inode *inode) 2065 { 2066 if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode)) 2067 return inode->i_sb->s_maxbytes; 2068 2069 return i_size_read(inode); 2070 } 2071 2072 static inline blk_opf_t f2fs_ra_op_flags(struct readahead_control *rac) 2073 { 2074 return rac ? REQ_RAHEAD : 0; 2075 } 2076 2077 static int f2fs_read_single_page(struct inode *inode, struct folio *folio, 2078 unsigned nr_pages, 2079 struct f2fs_map_blocks *map, 2080 struct bio **bio_ret, 2081 sector_t *last_block_in_bio, 2082 struct readahead_control *rac) 2083 { 2084 struct bio *bio = *bio_ret; 2085 const unsigned blocksize = blks_to_bytes(inode, 1); 2086 sector_t block_in_file; 2087 sector_t last_block; 2088 sector_t last_block_in_file; 2089 sector_t block_nr; 2090 pgoff_t index = folio_index(folio); 2091 int ret = 0; 2092 2093 block_in_file = (sector_t)index; 2094 last_block = block_in_file + nr_pages; 2095 last_block_in_file = bytes_to_blks(inode, 2096 f2fs_readpage_limit(inode) + blocksize - 1); 2097 if (last_block > last_block_in_file) 2098 last_block = last_block_in_file; 2099 2100 /* just zeroing out page which is beyond EOF */ 2101 if (block_in_file >= last_block) 2102 goto zero_out; 2103 /* 2104 * Map blocks using the previous result first. 2105 */ 2106 if ((map->m_flags & F2FS_MAP_MAPPED) && 2107 block_in_file > map->m_lblk && 2108 block_in_file < (map->m_lblk + map->m_len)) 2109 goto got_it; 2110 2111 /* 2112 * Then do more f2fs_map_blocks() calls until we are 2113 * done with this page. 2114 */ 2115 map->m_lblk = block_in_file; 2116 map->m_len = last_block - block_in_file; 2117 2118 ret = f2fs_map_blocks(inode, map, F2FS_GET_BLOCK_DEFAULT); 2119 if (ret) 2120 goto out; 2121 got_it: 2122 if ((map->m_flags & F2FS_MAP_MAPPED)) { 2123 block_nr = map->m_pblk + block_in_file - map->m_lblk; 2124 folio_set_mappedtodisk(folio); 2125 2126 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr, 2127 DATA_GENERIC_ENHANCE_READ)) { 2128 ret = -EFSCORRUPTED; 2129 goto out; 2130 } 2131 } else { 2132 zero_out: 2133 folio_zero_segment(folio, 0, folio_size(folio)); 2134 if (f2fs_need_verity(inode, index) && 2135 !fsverity_verify_folio(folio)) { 2136 ret = -EIO; 2137 goto out; 2138 } 2139 if (!folio_test_uptodate(folio)) 2140 folio_mark_uptodate(folio); 2141 folio_unlock(folio); 2142 goto out; 2143 } 2144 2145 /* 2146 * This page will go to BIO. Do we need to send this 2147 * BIO off first? 2148 */ 2149 if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio, 2150 *last_block_in_bio, block_nr) || 2151 !f2fs_crypt_mergeable_bio(bio, inode, index, NULL))) { 2152 submit_and_realloc: 2153 f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); 2154 bio = NULL; 2155 } 2156 if (bio == NULL) { 2157 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages, 2158 f2fs_ra_op_flags(rac), index, 2159 false); 2160 if (IS_ERR(bio)) { 2161 ret = PTR_ERR(bio); 2162 bio = NULL; 2163 goto out; 2164 } 2165 } 2166 2167 /* 2168 * If the page is under writeback, we need to wait for 2169 * its completion to see the correct decrypted data. 2170 */ 2171 f2fs_wait_on_block_writeback(inode, block_nr); 2172 2173 if (!bio_add_folio(bio, folio, blocksize, 0)) 2174 goto submit_and_realloc; 2175 2176 inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA); 2177 f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO, 2178 F2FS_BLKSIZE); 2179 *last_block_in_bio = block_nr; 2180 out: 2181 *bio_ret = bio; 2182 return ret; 2183 } 2184 2185 #ifdef CONFIG_F2FS_FS_COMPRESSION 2186 int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, 2187 unsigned nr_pages, sector_t *last_block_in_bio, 2188 struct readahead_control *rac, bool for_write) 2189 { 2190 struct dnode_of_data dn; 2191 struct inode *inode = cc->inode; 2192 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2193 struct bio *bio = *bio_ret; 2194 unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size; 2195 sector_t last_block_in_file; 2196 const unsigned blocksize = blks_to_bytes(inode, 1); 2197 struct decompress_io_ctx *dic = NULL; 2198 struct extent_info ei = {}; 2199 bool from_dnode = true; 2200 int i; 2201 int ret = 0; 2202 2203 f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc)); 2204 2205 last_block_in_file = bytes_to_blks(inode, 2206 f2fs_readpage_limit(inode) + blocksize - 1); 2207 2208 /* get rid of pages beyond EOF */ 2209 for (i = 0; i < cc->cluster_size; i++) { 2210 struct page *page = cc->rpages[i]; 2211 2212 if (!page) 2213 continue; 2214 if ((sector_t)page->index >= last_block_in_file) { 2215 zero_user_segment(page, 0, PAGE_SIZE); 2216 if (!PageUptodate(page)) 2217 SetPageUptodate(page); 2218 } else if (!PageUptodate(page)) { 2219 continue; 2220 } 2221 unlock_page(page); 2222 if (for_write) 2223 put_page(page); 2224 cc->rpages[i] = NULL; 2225 cc->nr_rpages--; 2226 } 2227 2228 /* we are done since all pages are beyond EOF */ 2229 if (f2fs_cluster_is_empty(cc)) 2230 goto out; 2231 2232 if (f2fs_lookup_read_extent_cache(inode, start_idx, &ei)) 2233 from_dnode = false; 2234 2235 if (!from_dnode) 2236 goto skip_reading_dnode; 2237 2238 set_new_dnode(&dn, inode, NULL, NULL, 0); 2239 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 2240 if (ret) 2241 goto out; 2242 2243 if (unlikely(f2fs_cp_error(sbi))) { 2244 ret = -EIO; 2245 goto out_put_dnode; 2246 } 2247 f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR); 2248 2249 skip_reading_dnode: 2250 for (i = 1; i < cc->cluster_size; i++) { 2251 block_t blkaddr; 2252 2253 blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_page, 2254 dn.ofs_in_node + i) : 2255 ei.blk + i - 1; 2256 2257 if (!__is_valid_data_blkaddr(blkaddr)) 2258 break; 2259 2260 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) { 2261 ret = -EFAULT; 2262 goto out_put_dnode; 2263 } 2264 cc->nr_cpages++; 2265 2266 if (!from_dnode && i >= ei.c_len) 2267 break; 2268 } 2269 2270 /* nothing to decompress */ 2271 if (cc->nr_cpages == 0) { 2272 ret = 0; 2273 goto out_put_dnode; 2274 } 2275 2276 dic = f2fs_alloc_dic(cc); 2277 if (IS_ERR(dic)) { 2278 ret = PTR_ERR(dic); 2279 goto out_put_dnode; 2280 } 2281 2282 for (i = 0; i < cc->nr_cpages; i++) { 2283 struct page *page = dic->cpages[i]; 2284 block_t blkaddr; 2285 struct bio_post_read_ctx *ctx; 2286 2287 blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_page, 2288 dn.ofs_in_node + i + 1) : 2289 ei.blk + i; 2290 2291 f2fs_wait_on_block_writeback(inode, blkaddr); 2292 2293 if (f2fs_load_compressed_page(sbi, page, blkaddr)) { 2294 if (atomic_dec_and_test(&dic->remaining_pages)) { 2295 f2fs_decompress_cluster(dic, true); 2296 break; 2297 } 2298 continue; 2299 } 2300 2301 if (bio && (!page_is_mergeable(sbi, bio, 2302 *last_block_in_bio, blkaddr) || 2303 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) { 2304 submit_and_realloc: 2305 f2fs_submit_read_bio(sbi, bio, DATA); 2306 bio = NULL; 2307 } 2308 2309 if (!bio) { 2310 bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages, 2311 f2fs_ra_op_flags(rac), 2312 page->index, for_write); 2313 if (IS_ERR(bio)) { 2314 ret = PTR_ERR(bio); 2315 f2fs_decompress_end_io(dic, ret, true); 2316 f2fs_put_dnode(&dn); 2317 *bio_ret = NULL; 2318 return ret; 2319 } 2320 } 2321 2322 if (bio_add_page(bio, page, blocksize, 0) < blocksize) 2323 goto submit_and_realloc; 2324 2325 ctx = get_post_read_ctx(bio); 2326 ctx->enabled_steps |= STEP_DECOMPRESS; 2327 refcount_inc(&dic->refcnt); 2328 2329 inc_page_count(sbi, F2FS_RD_DATA); 2330 f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE); 2331 *last_block_in_bio = blkaddr; 2332 } 2333 2334 if (from_dnode) 2335 f2fs_put_dnode(&dn); 2336 2337 *bio_ret = bio; 2338 return 0; 2339 2340 out_put_dnode: 2341 if (from_dnode) 2342 f2fs_put_dnode(&dn); 2343 out: 2344 for (i = 0; i < cc->cluster_size; i++) { 2345 if (cc->rpages[i]) { 2346 ClearPageUptodate(cc->rpages[i]); 2347 unlock_page(cc->rpages[i]); 2348 } 2349 } 2350 *bio_ret = bio; 2351 return ret; 2352 } 2353 #endif 2354 2355 /* 2356 * This function was originally taken from fs/mpage.c, and customized for f2fs. 2357 * Major change was from block_size == page_size in f2fs by default. 2358 */ 2359 static int f2fs_mpage_readpages(struct inode *inode, 2360 struct readahead_control *rac, struct folio *folio) 2361 { 2362 struct bio *bio = NULL; 2363 sector_t last_block_in_bio = 0; 2364 struct f2fs_map_blocks map; 2365 #ifdef CONFIG_F2FS_FS_COMPRESSION 2366 struct compress_ctx cc = { 2367 .inode = inode, 2368 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 2369 .cluster_size = F2FS_I(inode)->i_cluster_size, 2370 .cluster_idx = NULL_CLUSTER, 2371 .rpages = NULL, 2372 .cpages = NULL, 2373 .nr_rpages = 0, 2374 .nr_cpages = 0, 2375 }; 2376 pgoff_t nc_cluster_idx = NULL_CLUSTER; 2377 #endif 2378 unsigned nr_pages = rac ? readahead_count(rac) : 1; 2379 unsigned max_nr_pages = nr_pages; 2380 pgoff_t index; 2381 int ret = 0; 2382 2383 map.m_pblk = 0; 2384 map.m_lblk = 0; 2385 map.m_len = 0; 2386 map.m_flags = 0; 2387 map.m_next_pgofs = NULL; 2388 map.m_next_extent = NULL; 2389 map.m_seg_type = NO_CHECK_TYPE; 2390 map.m_may_create = false; 2391 2392 for (; nr_pages; nr_pages--) { 2393 if (rac) { 2394 folio = readahead_folio(rac); 2395 prefetchw(&folio->flags); 2396 } 2397 2398 index = folio_index(folio); 2399 2400 #ifdef CONFIG_F2FS_FS_COMPRESSION 2401 if (!f2fs_compressed_file(inode)) 2402 goto read_single_page; 2403 2404 /* there are remained compressed pages, submit them */ 2405 if (!f2fs_cluster_can_merge_page(&cc, index)) { 2406 ret = f2fs_read_multi_pages(&cc, &bio, 2407 max_nr_pages, 2408 &last_block_in_bio, 2409 rac, false); 2410 f2fs_destroy_compress_ctx(&cc, false); 2411 if (ret) 2412 goto set_error_page; 2413 } 2414 if (cc.cluster_idx == NULL_CLUSTER) { 2415 if (nc_cluster_idx == index >> cc.log_cluster_size) 2416 goto read_single_page; 2417 2418 ret = f2fs_is_compressed_cluster(inode, index); 2419 if (ret < 0) 2420 goto set_error_page; 2421 else if (!ret) { 2422 nc_cluster_idx = 2423 index >> cc.log_cluster_size; 2424 goto read_single_page; 2425 } 2426 2427 nc_cluster_idx = NULL_CLUSTER; 2428 } 2429 ret = f2fs_init_compress_ctx(&cc); 2430 if (ret) 2431 goto set_error_page; 2432 2433 f2fs_compress_ctx_add_page(&cc, &folio->page); 2434 2435 goto next_page; 2436 read_single_page: 2437 #endif 2438 2439 ret = f2fs_read_single_page(inode, folio, max_nr_pages, &map, 2440 &bio, &last_block_in_bio, rac); 2441 if (ret) { 2442 #ifdef CONFIG_F2FS_FS_COMPRESSION 2443 set_error_page: 2444 #endif 2445 folio_zero_segment(folio, 0, folio_size(folio)); 2446 folio_unlock(folio); 2447 } 2448 #ifdef CONFIG_F2FS_FS_COMPRESSION 2449 next_page: 2450 #endif 2451 2452 #ifdef CONFIG_F2FS_FS_COMPRESSION 2453 if (f2fs_compressed_file(inode)) { 2454 /* last page */ 2455 if (nr_pages == 1 && !f2fs_cluster_is_empty(&cc)) { 2456 ret = f2fs_read_multi_pages(&cc, &bio, 2457 max_nr_pages, 2458 &last_block_in_bio, 2459 rac, false); 2460 f2fs_destroy_compress_ctx(&cc, false); 2461 } 2462 } 2463 #endif 2464 } 2465 if (bio) 2466 f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); 2467 return ret; 2468 } 2469 2470 static int f2fs_read_data_folio(struct file *file, struct folio *folio) 2471 { 2472 struct inode *inode = folio_file_mapping(folio)->host; 2473 int ret = -EAGAIN; 2474 2475 trace_f2fs_readpage(folio, DATA); 2476 2477 if (!f2fs_is_compress_backend_ready(inode)) { 2478 folio_unlock(folio); 2479 return -EOPNOTSUPP; 2480 } 2481 2482 /* If the file has inline data, try to read it directly */ 2483 if (f2fs_has_inline_data(inode)) 2484 ret = f2fs_read_inline_data(inode, folio); 2485 if (ret == -EAGAIN) 2486 ret = f2fs_mpage_readpages(inode, NULL, folio); 2487 return ret; 2488 } 2489 2490 static void f2fs_readahead(struct readahead_control *rac) 2491 { 2492 struct inode *inode = rac->mapping->host; 2493 2494 trace_f2fs_readpages(inode, readahead_index(rac), readahead_count(rac)); 2495 2496 if (!f2fs_is_compress_backend_ready(inode)) 2497 return; 2498 2499 /* If the file has inline data, skip readahead */ 2500 if (f2fs_has_inline_data(inode)) 2501 return; 2502 2503 f2fs_mpage_readpages(inode, rac, NULL); 2504 } 2505 2506 int f2fs_encrypt_one_page(struct f2fs_io_info *fio) 2507 { 2508 struct inode *inode = fio->page->mapping->host; 2509 struct page *mpage, *page; 2510 gfp_t gfp_flags = GFP_NOFS; 2511 2512 if (!f2fs_encrypted_file(inode)) 2513 return 0; 2514 2515 page = fio->compressed_page ? fio->compressed_page : fio->page; 2516 2517 if (fscrypt_inode_uses_inline_crypto(inode)) 2518 return 0; 2519 2520 retry_encrypt: 2521 fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page, 2522 PAGE_SIZE, 0, gfp_flags); 2523 if (IS_ERR(fio->encrypted_page)) { 2524 /* flush pending IOs and wait for a while in the ENOMEM case */ 2525 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) { 2526 f2fs_flush_merged_writes(fio->sbi); 2527 memalloc_retry_wait(GFP_NOFS); 2528 gfp_flags |= __GFP_NOFAIL; 2529 goto retry_encrypt; 2530 } 2531 return PTR_ERR(fio->encrypted_page); 2532 } 2533 2534 mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr); 2535 if (mpage) { 2536 if (PageUptodate(mpage)) 2537 memcpy(page_address(mpage), 2538 page_address(fio->encrypted_page), PAGE_SIZE); 2539 f2fs_put_page(mpage, 1); 2540 } 2541 return 0; 2542 } 2543 2544 static inline bool check_inplace_update_policy(struct inode *inode, 2545 struct f2fs_io_info *fio) 2546 { 2547 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2548 2549 if (IS_F2FS_IPU_HONOR_OPU_WRITE(sbi) && 2550 is_inode_flag_set(inode, FI_OPU_WRITE)) 2551 return false; 2552 if (IS_F2FS_IPU_FORCE(sbi)) 2553 return true; 2554 if (IS_F2FS_IPU_SSR(sbi) && f2fs_need_SSR(sbi)) 2555 return true; 2556 if (IS_F2FS_IPU_UTIL(sbi) && utilization(sbi) > SM_I(sbi)->min_ipu_util) 2557 return true; 2558 if (IS_F2FS_IPU_SSR_UTIL(sbi) && f2fs_need_SSR(sbi) && 2559 utilization(sbi) > SM_I(sbi)->min_ipu_util) 2560 return true; 2561 2562 /* 2563 * IPU for rewrite async pages 2564 */ 2565 if (IS_F2FS_IPU_ASYNC(sbi) && fio && fio->op == REQ_OP_WRITE && 2566 !(fio->op_flags & REQ_SYNC) && !IS_ENCRYPTED(inode)) 2567 return true; 2568 2569 /* this is only set during fdatasync */ 2570 if (IS_F2FS_IPU_FSYNC(sbi) && is_inode_flag_set(inode, FI_NEED_IPU)) 2571 return true; 2572 2573 if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) && 2574 !f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) 2575 return true; 2576 2577 return false; 2578 } 2579 2580 bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio) 2581 { 2582 /* swap file is migrating in aligned write mode */ 2583 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) 2584 return false; 2585 2586 if (f2fs_is_pinned_file(inode)) 2587 return true; 2588 2589 /* if this is cold file, we should overwrite to avoid fragmentation */ 2590 if (file_is_cold(inode) && !is_inode_flag_set(inode, FI_OPU_WRITE)) 2591 return true; 2592 2593 return check_inplace_update_policy(inode, fio); 2594 } 2595 2596 bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) 2597 { 2598 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2599 2600 /* The below cases were checked when setting it. */ 2601 if (f2fs_is_pinned_file(inode)) 2602 return false; 2603 if (fio && is_sbi_flag_set(sbi, SBI_NEED_FSCK)) 2604 return true; 2605 if (f2fs_lfs_mode(sbi)) 2606 return true; 2607 if (S_ISDIR(inode->i_mode)) 2608 return true; 2609 if (IS_NOQUOTA(inode)) 2610 return true; 2611 if (f2fs_used_in_atomic_write(inode)) 2612 return true; 2613 /* rewrite low ratio compress data w/ OPU mode to avoid fragmentation */ 2614 if (f2fs_compressed_file(inode) && 2615 F2FS_OPTION(sbi).compress_mode == COMPR_MODE_USER && 2616 is_inode_flag_set(inode, FI_ENABLE_COMPRESS)) 2617 return true; 2618 2619 /* swap file is migrating in aligned write mode */ 2620 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) 2621 return true; 2622 2623 if (is_inode_flag_set(inode, FI_OPU_WRITE)) 2624 return true; 2625 2626 if (fio) { 2627 if (page_private_gcing(fio->page)) 2628 return true; 2629 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) && 2630 f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) 2631 return true; 2632 } 2633 return false; 2634 } 2635 2636 static inline bool need_inplace_update(struct f2fs_io_info *fio) 2637 { 2638 struct inode *inode = fio->page->mapping->host; 2639 2640 if (f2fs_should_update_outplace(inode, fio)) 2641 return false; 2642 2643 return f2fs_should_update_inplace(inode, fio); 2644 } 2645 2646 int f2fs_do_write_data_page(struct f2fs_io_info *fio) 2647 { 2648 struct page *page = fio->page; 2649 struct inode *inode = page->mapping->host; 2650 struct dnode_of_data dn; 2651 struct node_info ni; 2652 bool ipu_force = false; 2653 int err = 0; 2654 2655 /* Use COW inode to make dnode_of_data for atomic write */ 2656 if (f2fs_is_atomic_file(inode)) 2657 set_new_dnode(&dn, F2FS_I(inode)->cow_inode, NULL, NULL, 0); 2658 else 2659 set_new_dnode(&dn, inode, NULL, NULL, 0); 2660 2661 if (need_inplace_update(fio) && 2662 f2fs_lookup_read_extent_cache_block(inode, page->index, 2663 &fio->old_blkaddr)) { 2664 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, 2665 DATA_GENERIC_ENHANCE)) 2666 return -EFSCORRUPTED; 2667 2668 ipu_force = true; 2669 fio->need_lock = LOCK_DONE; 2670 goto got_it; 2671 } 2672 2673 /* Deadlock due to between page->lock and f2fs_lock_op */ 2674 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi)) 2675 return -EAGAIN; 2676 2677 err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE); 2678 if (err) 2679 goto out; 2680 2681 fio->old_blkaddr = dn.data_blkaddr; 2682 2683 /* This page is already truncated */ 2684 if (fio->old_blkaddr == NULL_ADDR) { 2685 ClearPageUptodate(page); 2686 clear_page_private_gcing(page); 2687 goto out_writepage; 2688 } 2689 got_it: 2690 if (__is_valid_data_blkaddr(fio->old_blkaddr) && 2691 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, 2692 DATA_GENERIC_ENHANCE)) { 2693 err = -EFSCORRUPTED; 2694 goto out_writepage; 2695 } 2696 2697 /* wait for GCed page writeback via META_MAPPING */ 2698 if (fio->meta_gc) 2699 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr); 2700 2701 /* 2702 * If current allocation needs SSR, 2703 * it had better in-place writes for updated data. 2704 */ 2705 if (ipu_force || 2706 (__is_valid_data_blkaddr(fio->old_blkaddr) && 2707 need_inplace_update(fio))) { 2708 err = f2fs_encrypt_one_page(fio); 2709 if (err) 2710 goto out_writepage; 2711 2712 set_page_writeback(page); 2713 f2fs_put_dnode(&dn); 2714 if (fio->need_lock == LOCK_REQ) 2715 f2fs_unlock_op(fio->sbi); 2716 err = f2fs_inplace_write_data(fio); 2717 if (err) { 2718 if (fscrypt_inode_uses_fs_layer_crypto(inode)) 2719 fscrypt_finalize_bounce_page(&fio->encrypted_page); 2720 end_page_writeback(page); 2721 } else { 2722 set_inode_flag(inode, FI_UPDATE_WRITE); 2723 } 2724 trace_f2fs_do_write_data_page(page_folio(page), IPU); 2725 return err; 2726 } 2727 2728 if (fio->need_lock == LOCK_RETRY) { 2729 if (!f2fs_trylock_op(fio->sbi)) { 2730 err = -EAGAIN; 2731 goto out_writepage; 2732 } 2733 fio->need_lock = LOCK_REQ; 2734 } 2735 2736 err = f2fs_get_node_info(fio->sbi, dn.nid, &ni, false); 2737 if (err) 2738 goto out_writepage; 2739 2740 fio->version = ni.version; 2741 2742 err = f2fs_encrypt_one_page(fio); 2743 if (err) 2744 goto out_writepage; 2745 2746 set_page_writeback(page); 2747 2748 if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR) 2749 f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false); 2750 2751 /* LFS mode write path */ 2752 f2fs_outplace_write_data(&dn, fio); 2753 trace_f2fs_do_write_data_page(page_folio(page), OPU); 2754 set_inode_flag(inode, FI_APPEND_WRITE); 2755 out_writepage: 2756 f2fs_put_dnode(&dn); 2757 out: 2758 if (fio->need_lock == LOCK_REQ) 2759 f2fs_unlock_op(fio->sbi); 2760 return err; 2761 } 2762 2763 int f2fs_write_single_data_page(struct page *page, int *submitted, 2764 struct bio **bio, 2765 sector_t *last_block, 2766 struct writeback_control *wbc, 2767 enum iostat_type io_type, 2768 int compr_blocks, 2769 bool allow_balance) 2770 { 2771 struct inode *inode = page->mapping->host; 2772 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2773 loff_t i_size = i_size_read(inode); 2774 const pgoff_t end_index = ((unsigned long long)i_size) 2775 >> PAGE_SHIFT; 2776 loff_t psize = (loff_t)(page->index + 1) << PAGE_SHIFT; 2777 unsigned offset = 0; 2778 bool need_balance_fs = false; 2779 bool quota_inode = IS_NOQUOTA(inode); 2780 int err = 0; 2781 struct f2fs_io_info fio = { 2782 .sbi = sbi, 2783 .ino = inode->i_ino, 2784 .type = DATA, 2785 .op = REQ_OP_WRITE, 2786 .op_flags = wbc_to_write_flags(wbc), 2787 .old_blkaddr = NULL_ADDR, 2788 .page = page, 2789 .encrypted_page = NULL, 2790 .submitted = 0, 2791 .compr_blocks = compr_blocks, 2792 .need_lock = compr_blocks ? LOCK_DONE : LOCK_RETRY, 2793 .meta_gc = f2fs_meta_inode_gc_required(inode) ? 1 : 0, 2794 .io_type = io_type, 2795 .io_wbc = wbc, 2796 .bio = bio, 2797 .last_block = last_block, 2798 }; 2799 2800 trace_f2fs_writepage(page_folio(page), DATA); 2801 2802 /* we should bypass data pages to proceed the kworker jobs */ 2803 if (unlikely(f2fs_cp_error(sbi))) { 2804 mapping_set_error(page->mapping, -EIO); 2805 /* 2806 * don't drop any dirty dentry pages for keeping lastest 2807 * directory structure. 2808 */ 2809 if (S_ISDIR(inode->i_mode) && 2810 !is_sbi_flag_set(sbi, SBI_IS_CLOSE)) 2811 goto redirty_out; 2812 2813 /* keep data pages in remount-ro mode */ 2814 if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY) 2815 goto redirty_out; 2816 goto out; 2817 } 2818 2819 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 2820 goto redirty_out; 2821 2822 if (page->index < end_index || 2823 f2fs_verity_in_progress(inode) || 2824 compr_blocks) 2825 goto write; 2826 2827 /* 2828 * If the offset is out-of-range of file size, 2829 * this page does not have to be written to disk. 2830 */ 2831 offset = i_size & (PAGE_SIZE - 1); 2832 if ((page->index >= end_index + 1) || !offset) 2833 goto out; 2834 2835 zero_user_segment(page, offset, PAGE_SIZE); 2836 write: 2837 /* Dentry/quota blocks are controlled by checkpoint */ 2838 if (S_ISDIR(inode->i_mode) || quota_inode) { 2839 /* 2840 * We need to wait for node_write to avoid block allocation during 2841 * checkpoint. This can only happen to quota writes which can cause 2842 * the below discard race condition. 2843 */ 2844 if (quota_inode) 2845 f2fs_down_read(&sbi->node_write); 2846 2847 fio.need_lock = LOCK_DONE; 2848 err = f2fs_do_write_data_page(&fio); 2849 2850 if (quota_inode) 2851 f2fs_up_read(&sbi->node_write); 2852 2853 goto done; 2854 } 2855 2856 if (!wbc->for_reclaim) 2857 need_balance_fs = true; 2858 else if (has_not_enough_free_secs(sbi, 0, 0)) 2859 goto redirty_out; 2860 else 2861 set_inode_flag(inode, FI_HOT_DATA); 2862 2863 err = -EAGAIN; 2864 if (f2fs_has_inline_data(inode)) { 2865 err = f2fs_write_inline_data(inode, page); 2866 if (!err) 2867 goto out; 2868 } 2869 2870 if (err == -EAGAIN) { 2871 err = f2fs_do_write_data_page(&fio); 2872 if (err == -EAGAIN) { 2873 f2fs_bug_on(sbi, compr_blocks); 2874 fio.need_lock = LOCK_REQ; 2875 err = f2fs_do_write_data_page(&fio); 2876 } 2877 } 2878 2879 if (err) { 2880 file_set_keep_isize(inode); 2881 } else { 2882 spin_lock(&F2FS_I(inode)->i_size_lock); 2883 if (F2FS_I(inode)->last_disk_size < psize) 2884 F2FS_I(inode)->last_disk_size = psize; 2885 spin_unlock(&F2FS_I(inode)->i_size_lock); 2886 } 2887 2888 done: 2889 if (err && err != -ENOENT) 2890 goto redirty_out; 2891 2892 out: 2893 inode_dec_dirty_pages(inode); 2894 if (err) { 2895 ClearPageUptodate(page); 2896 clear_page_private_gcing(page); 2897 } 2898 2899 if (wbc->for_reclaim) { 2900 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, DATA); 2901 clear_inode_flag(inode, FI_HOT_DATA); 2902 f2fs_remove_dirty_inode(inode); 2903 submitted = NULL; 2904 } 2905 unlock_page(page); 2906 if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) && 2907 !F2FS_I(inode)->wb_task && allow_balance) 2908 f2fs_balance_fs(sbi, need_balance_fs); 2909 2910 if (unlikely(f2fs_cp_error(sbi))) { 2911 f2fs_submit_merged_write(sbi, DATA); 2912 if (bio && *bio) 2913 f2fs_submit_merged_ipu_write(sbi, bio, NULL); 2914 submitted = NULL; 2915 } 2916 2917 if (submitted) 2918 *submitted = fio.submitted; 2919 2920 return 0; 2921 2922 redirty_out: 2923 redirty_page_for_writepage(wbc, page); 2924 /* 2925 * pageout() in MM translates EAGAIN, so calls handle_write_error() 2926 * -> mapping_set_error() -> set_bit(AS_EIO, ...). 2927 * file_write_and_wait_range() will see EIO error, which is critical 2928 * to return value of fsync() followed by atomic_write failure to user. 2929 */ 2930 if (!err || wbc->for_reclaim) 2931 return AOP_WRITEPAGE_ACTIVATE; 2932 unlock_page(page); 2933 return err; 2934 } 2935 2936 static int f2fs_write_data_page(struct page *page, 2937 struct writeback_control *wbc) 2938 { 2939 #ifdef CONFIG_F2FS_FS_COMPRESSION 2940 struct inode *inode = page->mapping->host; 2941 2942 if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) 2943 goto out; 2944 2945 if (f2fs_compressed_file(inode)) { 2946 if (f2fs_is_compressed_cluster(inode, page->index)) { 2947 redirty_page_for_writepage(wbc, page); 2948 return AOP_WRITEPAGE_ACTIVATE; 2949 } 2950 } 2951 out: 2952 #endif 2953 2954 return f2fs_write_single_data_page(page, NULL, NULL, NULL, 2955 wbc, FS_DATA_IO, 0, true); 2956 } 2957 2958 /* 2959 * This function was copied from write_cache_pages from mm/page-writeback.c. 2960 * The major change is making write step of cold data page separately from 2961 * warm/hot data page. 2962 */ 2963 static int f2fs_write_cache_pages(struct address_space *mapping, 2964 struct writeback_control *wbc, 2965 enum iostat_type io_type) 2966 { 2967 int ret = 0; 2968 int done = 0, retry = 0; 2969 struct page *pages_local[F2FS_ONSTACK_PAGES]; 2970 struct page **pages = pages_local; 2971 struct folio_batch fbatch; 2972 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); 2973 struct bio *bio = NULL; 2974 sector_t last_block; 2975 #ifdef CONFIG_F2FS_FS_COMPRESSION 2976 struct inode *inode = mapping->host; 2977 struct compress_ctx cc = { 2978 .inode = inode, 2979 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 2980 .cluster_size = F2FS_I(inode)->i_cluster_size, 2981 .cluster_idx = NULL_CLUSTER, 2982 .rpages = NULL, 2983 .nr_rpages = 0, 2984 .cpages = NULL, 2985 .valid_nr_cpages = 0, 2986 .rbuf = NULL, 2987 .cbuf = NULL, 2988 .rlen = PAGE_SIZE * F2FS_I(inode)->i_cluster_size, 2989 .private = NULL, 2990 }; 2991 #endif 2992 int nr_folios, p, idx; 2993 int nr_pages; 2994 unsigned int max_pages = F2FS_ONSTACK_PAGES; 2995 pgoff_t index; 2996 pgoff_t end; /* Inclusive */ 2997 pgoff_t done_index; 2998 int range_whole = 0; 2999 xa_mark_t tag; 3000 int nwritten = 0; 3001 int submitted = 0; 3002 int i; 3003 3004 #ifdef CONFIG_F2FS_FS_COMPRESSION 3005 if (f2fs_compressed_file(inode) && 3006 1 << cc.log_cluster_size > F2FS_ONSTACK_PAGES) { 3007 pages = f2fs_kzalloc(sbi, sizeof(struct page *) << 3008 cc.log_cluster_size, GFP_NOFS | __GFP_NOFAIL); 3009 max_pages = 1 << cc.log_cluster_size; 3010 } 3011 #endif 3012 3013 folio_batch_init(&fbatch); 3014 3015 if (get_dirty_pages(mapping->host) <= 3016 SM_I(F2FS_M_SB(mapping))->min_hot_blocks) 3017 set_inode_flag(mapping->host, FI_HOT_DATA); 3018 else 3019 clear_inode_flag(mapping->host, FI_HOT_DATA); 3020 3021 if (wbc->range_cyclic) { 3022 index = mapping->writeback_index; /* prev offset */ 3023 end = -1; 3024 } else { 3025 index = wbc->range_start >> PAGE_SHIFT; 3026 end = wbc->range_end >> PAGE_SHIFT; 3027 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 3028 range_whole = 1; 3029 } 3030 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 3031 tag = PAGECACHE_TAG_TOWRITE; 3032 else 3033 tag = PAGECACHE_TAG_DIRTY; 3034 retry: 3035 retry = 0; 3036 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 3037 tag_pages_for_writeback(mapping, index, end); 3038 done_index = index; 3039 while (!done && !retry && (index <= end)) { 3040 nr_pages = 0; 3041 again: 3042 nr_folios = filemap_get_folios_tag(mapping, &index, end, 3043 tag, &fbatch); 3044 if (nr_folios == 0) { 3045 if (nr_pages) 3046 goto write; 3047 break; 3048 } 3049 3050 for (i = 0; i < nr_folios; i++) { 3051 struct folio *folio = fbatch.folios[i]; 3052 3053 idx = 0; 3054 p = folio_nr_pages(folio); 3055 add_more: 3056 pages[nr_pages] = folio_page(folio, idx); 3057 folio_get(folio); 3058 if (++nr_pages == max_pages) { 3059 index = folio->index + idx + 1; 3060 folio_batch_release(&fbatch); 3061 goto write; 3062 } 3063 if (++idx < p) 3064 goto add_more; 3065 } 3066 folio_batch_release(&fbatch); 3067 goto again; 3068 write: 3069 for (i = 0; i < nr_pages; i++) { 3070 struct page *page = pages[i]; 3071 struct folio *folio = page_folio(page); 3072 bool need_readd; 3073 readd: 3074 need_readd = false; 3075 #ifdef CONFIG_F2FS_FS_COMPRESSION 3076 if (f2fs_compressed_file(inode)) { 3077 void *fsdata = NULL; 3078 struct page *pagep; 3079 int ret2; 3080 3081 ret = f2fs_init_compress_ctx(&cc); 3082 if (ret) { 3083 done = 1; 3084 break; 3085 } 3086 3087 if (!f2fs_cluster_can_merge_page(&cc, 3088 folio->index)) { 3089 ret = f2fs_write_multi_pages(&cc, 3090 &submitted, wbc, io_type); 3091 if (!ret) 3092 need_readd = true; 3093 goto result; 3094 } 3095 3096 if (unlikely(f2fs_cp_error(sbi))) 3097 goto lock_folio; 3098 3099 if (!f2fs_cluster_is_empty(&cc)) 3100 goto lock_folio; 3101 3102 if (f2fs_all_cluster_page_ready(&cc, 3103 pages, i, nr_pages, true)) 3104 goto lock_folio; 3105 3106 ret2 = f2fs_prepare_compress_overwrite( 3107 inode, &pagep, 3108 folio->index, &fsdata); 3109 if (ret2 < 0) { 3110 ret = ret2; 3111 done = 1; 3112 break; 3113 } else if (ret2 && 3114 (!f2fs_compress_write_end(inode, 3115 fsdata, folio->index, 1) || 3116 !f2fs_all_cluster_page_ready(&cc, 3117 pages, i, nr_pages, 3118 false))) { 3119 retry = 1; 3120 break; 3121 } 3122 } 3123 #endif 3124 /* give a priority to WB_SYNC threads */ 3125 if (atomic_read(&sbi->wb_sync_req[DATA]) && 3126 wbc->sync_mode == WB_SYNC_NONE) { 3127 done = 1; 3128 break; 3129 } 3130 #ifdef CONFIG_F2FS_FS_COMPRESSION 3131 lock_folio: 3132 #endif 3133 done_index = folio->index; 3134 retry_write: 3135 folio_lock(folio); 3136 3137 if (unlikely(folio->mapping != mapping)) { 3138 continue_unlock: 3139 folio_unlock(folio); 3140 continue; 3141 } 3142 3143 if (!folio_test_dirty(folio)) { 3144 /* someone wrote it for us */ 3145 goto continue_unlock; 3146 } 3147 3148 if (folio_test_writeback(folio)) { 3149 if (wbc->sync_mode == WB_SYNC_NONE) 3150 goto continue_unlock; 3151 f2fs_wait_on_page_writeback(&folio->page, DATA, true, true); 3152 } 3153 3154 if (!folio_clear_dirty_for_io(folio)) 3155 goto continue_unlock; 3156 3157 #ifdef CONFIG_F2FS_FS_COMPRESSION 3158 if (f2fs_compressed_file(inode)) { 3159 folio_get(folio); 3160 f2fs_compress_ctx_add_page(&cc, &folio->page); 3161 continue; 3162 } 3163 #endif 3164 ret = f2fs_write_single_data_page(&folio->page, 3165 &submitted, &bio, &last_block, 3166 wbc, io_type, 0, true); 3167 if (ret == AOP_WRITEPAGE_ACTIVATE) 3168 folio_unlock(folio); 3169 #ifdef CONFIG_F2FS_FS_COMPRESSION 3170 result: 3171 #endif 3172 nwritten += submitted; 3173 wbc->nr_to_write -= submitted; 3174 3175 if (unlikely(ret)) { 3176 /* 3177 * keep nr_to_write, since vfs uses this to 3178 * get # of written pages. 3179 */ 3180 if (ret == AOP_WRITEPAGE_ACTIVATE) { 3181 ret = 0; 3182 goto next; 3183 } else if (ret == -EAGAIN) { 3184 ret = 0; 3185 if (wbc->sync_mode == WB_SYNC_ALL) { 3186 f2fs_io_schedule_timeout( 3187 DEFAULT_IO_TIMEOUT); 3188 goto retry_write; 3189 } 3190 goto next; 3191 } 3192 done_index = folio_next_index(folio); 3193 done = 1; 3194 break; 3195 } 3196 3197 if (wbc->nr_to_write <= 0 && 3198 wbc->sync_mode == WB_SYNC_NONE) { 3199 done = 1; 3200 break; 3201 } 3202 next: 3203 if (need_readd) 3204 goto readd; 3205 } 3206 release_pages(pages, nr_pages); 3207 cond_resched(); 3208 } 3209 #ifdef CONFIG_F2FS_FS_COMPRESSION 3210 /* flush remained pages in compress cluster */ 3211 if (f2fs_compressed_file(inode) && !f2fs_cluster_is_empty(&cc)) { 3212 ret = f2fs_write_multi_pages(&cc, &submitted, wbc, io_type); 3213 nwritten += submitted; 3214 wbc->nr_to_write -= submitted; 3215 if (ret) { 3216 done = 1; 3217 retry = 0; 3218 } 3219 } 3220 if (f2fs_compressed_file(inode)) 3221 f2fs_destroy_compress_ctx(&cc, false); 3222 #endif 3223 if (retry) { 3224 index = 0; 3225 end = -1; 3226 goto retry; 3227 } 3228 if (wbc->range_cyclic && !done) 3229 done_index = 0; 3230 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 3231 mapping->writeback_index = done_index; 3232 3233 if (nwritten) 3234 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host, 3235 NULL, 0, DATA); 3236 /* submit cached bio of IPU write */ 3237 if (bio) 3238 f2fs_submit_merged_ipu_write(sbi, &bio, NULL); 3239 3240 #ifdef CONFIG_F2FS_FS_COMPRESSION 3241 if (pages != pages_local) 3242 kfree(pages); 3243 #endif 3244 3245 return ret; 3246 } 3247 3248 static inline bool __should_serialize_io(struct inode *inode, 3249 struct writeback_control *wbc) 3250 { 3251 /* to avoid deadlock in path of data flush */ 3252 if (F2FS_I(inode)->wb_task) 3253 return false; 3254 3255 if (!S_ISREG(inode->i_mode)) 3256 return false; 3257 if (IS_NOQUOTA(inode)) 3258 return false; 3259 3260 if (f2fs_need_compress_data(inode)) 3261 return true; 3262 if (wbc->sync_mode != WB_SYNC_ALL) 3263 return true; 3264 if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks) 3265 return true; 3266 return false; 3267 } 3268 3269 static int __f2fs_write_data_pages(struct address_space *mapping, 3270 struct writeback_control *wbc, 3271 enum iostat_type io_type) 3272 { 3273 struct inode *inode = mapping->host; 3274 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3275 struct blk_plug plug; 3276 int ret; 3277 bool locked = false; 3278 3279 /* deal with chardevs and other special file */ 3280 if (!mapping->a_ops->writepage) 3281 return 0; 3282 3283 /* skip writing if there is no dirty page in this inode */ 3284 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE) 3285 return 0; 3286 3287 /* during POR, we don't need to trigger writepage at all. */ 3288 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 3289 goto skip_write; 3290 3291 if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) && 3292 wbc->sync_mode == WB_SYNC_NONE && 3293 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && 3294 f2fs_available_free_memory(sbi, DIRTY_DENTS)) 3295 goto skip_write; 3296 3297 /* skip writing in file defragment preparing stage */ 3298 if (is_inode_flag_set(inode, FI_SKIP_WRITES)) 3299 goto skip_write; 3300 3301 trace_f2fs_writepages(mapping->host, wbc, DATA); 3302 3303 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */ 3304 if (wbc->sync_mode == WB_SYNC_ALL) 3305 atomic_inc(&sbi->wb_sync_req[DATA]); 3306 else if (atomic_read(&sbi->wb_sync_req[DATA])) { 3307 /* to avoid potential deadlock */ 3308 if (current->plug) 3309 blk_finish_plug(current->plug); 3310 goto skip_write; 3311 } 3312 3313 if (__should_serialize_io(inode, wbc)) { 3314 mutex_lock(&sbi->writepages); 3315 locked = true; 3316 } 3317 3318 blk_start_plug(&plug); 3319 ret = f2fs_write_cache_pages(mapping, wbc, io_type); 3320 blk_finish_plug(&plug); 3321 3322 if (locked) 3323 mutex_unlock(&sbi->writepages); 3324 3325 if (wbc->sync_mode == WB_SYNC_ALL) 3326 atomic_dec(&sbi->wb_sync_req[DATA]); 3327 /* 3328 * if some pages were truncated, we cannot guarantee its mapping->host 3329 * to detect pending bios. 3330 */ 3331 3332 f2fs_remove_dirty_inode(inode); 3333 return ret; 3334 3335 skip_write: 3336 wbc->pages_skipped += get_dirty_pages(inode); 3337 trace_f2fs_writepages(mapping->host, wbc, DATA); 3338 return 0; 3339 } 3340 3341 static int f2fs_write_data_pages(struct address_space *mapping, 3342 struct writeback_control *wbc) 3343 { 3344 struct inode *inode = mapping->host; 3345 3346 return __f2fs_write_data_pages(mapping, wbc, 3347 F2FS_I(inode)->cp_task == current ? 3348 FS_CP_DATA_IO : FS_DATA_IO); 3349 } 3350 3351 void f2fs_write_failed(struct inode *inode, loff_t to) 3352 { 3353 loff_t i_size = i_size_read(inode); 3354 3355 if (IS_NOQUOTA(inode)) 3356 return; 3357 3358 /* In the fs-verity case, f2fs_end_enable_verity() does the truncate */ 3359 if (to > i_size && !f2fs_verity_in_progress(inode)) { 3360 f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3361 filemap_invalidate_lock(inode->i_mapping); 3362 3363 truncate_pagecache(inode, i_size); 3364 f2fs_truncate_blocks(inode, i_size, true); 3365 3366 filemap_invalidate_unlock(inode->i_mapping); 3367 f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3368 } 3369 } 3370 3371 static int prepare_write_begin(struct f2fs_sb_info *sbi, 3372 struct page *page, loff_t pos, unsigned len, 3373 block_t *blk_addr, bool *node_changed) 3374 { 3375 struct inode *inode = page->mapping->host; 3376 pgoff_t index = page->index; 3377 struct dnode_of_data dn; 3378 struct page *ipage; 3379 bool locked = false; 3380 int flag = F2FS_GET_BLOCK_PRE_AIO; 3381 int err = 0; 3382 3383 /* 3384 * If a whole page is being written and we already preallocated all the 3385 * blocks, then there is no need to get a block address now. 3386 */ 3387 if (len == PAGE_SIZE && is_inode_flag_set(inode, FI_PREALLOCATED_ALL)) 3388 return 0; 3389 3390 /* f2fs_lock_op avoids race between write CP and convert_inline_page */ 3391 if (f2fs_has_inline_data(inode)) { 3392 if (pos + len > MAX_INLINE_DATA(inode)) 3393 flag = F2FS_GET_BLOCK_DEFAULT; 3394 f2fs_map_lock(sbi, flag); 3395 locked = true; 3396 } else if ((pos & PAGE_MASK) >= i_size_read(inode)) { 3397 f2fs_map_lock(sbi, flag); 3398 locked = true; 3399 } 3400 3401 restart: 3402 /* check inline_data */ 3403 ipage = f2fs_get_node_page(sbi, inode->i_ino); 3404 if (IS_ERR(ipage)) { 3405 err = PTR_ERR(ipage); 3406 goto unlock_out; 3407 } 3408 3409 set_new_dnode(&dn, inode, ipage, ipage, 0); 3410 3411 if (f2fs_has_inline_data(inode)) { 3412 if (pos + len <= MAX_INLINE_DATA(inode)) { 3413 f2fs_do_read_inline_data(page_folio(page), ipage); 3414 set_inode_flag(inode, FI_DATA_EXIST); 3415 if (inode->i_nlink) 3416 set_page_private_inline(ipage); 3417 goto out; 3418 } 3419 err = f2fs_convert_inline_page(&dn, page); 3420 if (err || dn.data_blkaddr != NULL_ADDR) 3421 goto out; 3422 } 3423 3424 if (!f2fs_lookup_read_extent_cache_block(inode, index, 3425 &dn.data_blkaddr)) { 3426 if (locked) { 3427 err = f2fs_reserve_block(&dn, index); 3428 goto out; 3429 } 3430 3431 /* hole case */ 3432 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 3433 if (!err && dn.data_blkaddr != NULL_ADDR) 3434 goto out; 3435 f2fs_put_dnode(&dn); 3436 f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); 3437 WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO); 3438 locked = true; 3439 goto restart; 3440 } 3441 out: 3442 if (!err) { 3443 /* convert_inline_page can make node_changed */ 3444 *blk_addr = dn.data_blkaddr; 3445 *node_changed = dn.node_changed; 3446 } 3447 f2fs_put_dnode(&dn); 3448 unlock_out: 3449 if (locked) 3450 f2fs_map_unlock(sbi, flag); 3451 return err; 3452 } 3453 3454 static int __find_data_block(struct inode *inode, pgoff_t index, 3455 block_t *blk_addr) 3456 { 3457 struct dnode_of_data dn; 3458 struct page *ipage; 3459 int err = 0; 3460 3461 ipage = f2fs_get_node_page(F2FS_I_SB(inode), inode->i_ino); 3462 if (IS_ERR(ipage)) 3463 return PTR_ERR(ipage); 3464 3465 set_new_dnode(&dn, inode, ipage, ipage, 0); 3466 3467 if (!f2fs_lookup_read_extent_cache_block(inode, index, 3468 &dn.data_blkaddr)) { 3469 /* hole case */ 3470 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 3471 if (err) { 3472 dn.data_blkaddr = NULL_ADDR; 3473 err = 0; 3474 } 3475 } 3476 *blk_addr = dn.data_blkaddr; 3477 f2fs_put_dnode(&dn); 3478 return err; 3479 } 3480 3481 static int __reserve_data_block(struct inode *inode, pgoff_t index, 3482 block_t *blk_addr, bool *node_changed) 3483 { 3484 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3485 struct dnode_of_data dn; 3486 struct page *ipage; 3487 int err = 0; 3488 3489 f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); 3490 3491 ipage = f2fs_get_node_page(sbi, inode->i_ino); 3492 if (IS_ERR(ipage)) { 3493 err = PTR_ERR(ipage); 3494 goto unlock_out; 3495 } 3496 set_new_dnode(&dn, inode, ipage, ipage, 0); 3497 3498 if (!f2fs_lookup_read_extent_cache_block(dn.inode, index, 3499 &dn.data_blkaddr)) 3500 err = f2fs_reserve_block(&dn, index); 3501 3502 *blk_addr = dn.data_blkaddr; 3503 *node_changed = dn.node_changed; 3504 f2fs_put_dnode(&dn); 3505 3506 unlock_out: 3507 f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO); 3508 return err; 3509 } 3510 3511 static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi, 3512 struct page *page, loff_t pos, unsigned int len, 3513 block_t *blk_addr, bool *node_changed, bool *use_cow) 3514 { 3515 struct inode *inode = page->mapping->host; 3516 struct inode *cow_inode = F2FS_I(inode)->cow_inode; 3517 pgoff_t index = page->index; 3518 int err = 0; 3519 block_t ori_blk_addr = NULL_ADDR; 3520 3521 /* If pos is beyond the end of file, reserve a new block in COW inode */ 3522 if ((pos & PAGE_MASK) >= i_size_read(inode)) 3523 goto reserve_block; 3524 3525 /* Look for the block in COW inode first */ 3526 err = __find_data_block(cow_inode, index, blk_addr); 3527 if (err) { 3528 return err; 3529 } else if (*blk_addr != NULL_ADDR) { 3530 *use_cow = true; 3531 return 0; 3532 } 3533 3534 if (is_inode_flag_set(inode, FI_ATOMIC_REPLACE)) 3535 goto reserve_block; 3536 3537 /* Look for the block in the original inode */ 3538 err = __find_data_block(inode, index, &ori_blk_addr); 3539 if (err) 3540 return err; 3541 3542 reserve_block: 3543 /* Finally, we should reserve a new block in COW inode for the update */ 3544 err = __reserve_data_block(cow_inode, index, blk_addr, node_changed); 3545 if (err) 3546 return err; 3547 inc_atomic_write_cnt(inode); 3548 3549 if (ori_blk_addr != NULL_ADDR) 3550 *blk_addr = ori_blk_addr; 3551 return 0; 3552 } 3553 3554 static int f2fs_write_begin(struct file *file, struct address_space *mapping, 3555 loff_t pos, unsigned len, struct page **pagep, void **fsdata) 3556 { 3557 struct inode *inode = mapping->host; 3558 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3559 struct page *page = NULL; 3560 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT; 3561 bool need_balance = false; 3562 bool use_cow = false; 3563 block_t blkaddr = NULL_ADDR; 3564 int err = 0; 3565 3566 trace_f2fs_write_begin(inode, pos, len); 3567 3568 if (!f2fs_is_checkpoint_ready(sbi)) { 3569 err = -ENOSPC; 3570 goto fail; 3571 } 3572 3573 /* 3574 * We should check this at this moment to avoid deadlock on inode page 3575 * and #0 page. The locking rule for inline_data conversion should be: 3576 * lock_page(page #0) -> lock_page(inode_page) 3577 */ 3578 if (index != 0) { 3579 err = f2fs_convert_inline_inode(inode); 3580 if (err) 3581 goto fail; 3582 } 3583 3584 #ifdef CONFIG_F2FS_FS_COMPRESSION 3585 if (f2fs_compressed_file(inode)) { 3586 int ret; 3587 3588 *fsdata = NULL; 3589 3590 if (len == PAGE_SIZE && !(f2fs_is_atomic_file(inode))) 3591 goto repeat; 3592 3593 ret = f2fs_prepare_compress_overwrite(inode, pagep, 3594 index, fsdata); 3595 if (ret < 0) { 3596 err = ret; 3597 goto fail; 3598 } else if (ret) { 3599 return 0; 3600 } 3601 } 3602 #endif 3603 3604 repeat: 3605 /* 3606 * Do not use grab_cache_page_write_begin() to avoid deadlock due to 3607 * wait_for_stable_page. Will wait that below with our IO control. 3608 */ 3609 page = f2fs_pagecache_get_page(mapping, index, 3610 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS); 3611 if (!page) { 3612 err = -ENOMEM; 3613 goto fail; 3614 } 3615 3616 /* TODO: cluster can be compressed due to race with .writepage */ 3617 3618 *pagep = page; 3619 3620 if (f2fs_is_atomic_file(inode)) 3621 err = prepare_atomic_write_begin(sbi, page, pos, len, 3622 &blkaddr, &need_balance, &use_cow); 3623 else 3624 err = prepare_write_begin(sbi, page, pos, len, 3625 &blkaddr, &need_balance); 3626 if (err) 3627 goto fail; 3628 3629 if (need_balance && !IS_NOQUOTA(inode) && 3630 has_not_enough_free_secs(sbi, 0, 0)) { 3631 unlock_page(page); 3632 f2fs_balance_fs(sbi, true); 3633 lock_page(page); 3634 if (page->mapping != mapping) { 3635 /* The page got truncated from under us */ 3636 f2fs_put_page(page, 1); 3637 goto repeat; 3638 } 3639 } 3640 3641 f2fs_wait_on_page_writeback(page, DATA, false, true); 3642 3643 if (len == PAGE_SIZE || PageUptodate(page)) 3644 return 0; 3645 3646 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode) && 3647 !f2fs_verity_in_progress(inode)) { 3648 zero_user_segment(page, len, PAGE_SIZE); 3649 return 0; 3650 } 3651 3652 if (blkaddr == NEW_ADDR) { 3653 zero_user_segment(page, 0, PAGE_SIZE); 3654 SetPageUptodate(page); 3655 } else { 3656 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, 3657 DATA_GENERIC_ENHANCE_READ)) { 3658 err = -EFSCORRUPTED; 3659 goto fail; 3660 } 3661 err = f2fs_submit_page_read(use_cow ? 3662 F2FS_I(inode)->cow_inode : inode, page, 3663 blkaddr, 0, true); 3664 if (err) 3665 goto fail; 3666 3667 lock_page(page); 3668 if (unlikely(page->mapping != mapping)) { 3669 f2fs_put_page(page, 1); 3670 goto repeat; 3671 } 3672 if (unlikely(!PageUptodate(page))) { 3673 err = -EIO; 3674 goto fail; 3675 } 3676 } 3677 return 0; 3678 3679 fail: 3680 f2fs_put_page(page, 1); 3681 f2fs_write_failed(inode, pos + len); 3682 return err; 3683 } 3684 3685 static int f2fs_write_end(struct file *file, 3686 struct address_space *mapping, 3687 loff_t pos, unsigned len, unsigned copied, 3688 struct page *page, void *fsdata) 3689 { 3690 struct inode *inode = page->mapping->host; 3691 3692 trace_f2fs_write_end(inode, pos, len, copied); 3693 3694 /* 3695 * This should be come from len == PAGE_SIZE, and we expect copied 3696 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and 3697 * let generic_perform_write() try to copy data again through copied=0. 3698 */ 3699 if (!PageUptodate(page)) { 3700 if (unlikely(copied != len)) 3701 copied = 0; 3702 else 3703 SetPageUptodate(page); 3704 } 3705 3706 #ifdef CONFIG_F2FS_FS_COMPRESSION 3707 /* overwrite compressed file */ 3708 if (f2fs_compressed_file(inode) && fsdata) { 3709 f2fs_compress_write_end(inode, fsdata, page->index, copied); 3710 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); 3711 3712 if (pos + copied > i_size_read(inode) && 3713 !f2fs_verity_in_progress(inode)) 3714 f2fs_i_size_write(inode, pos + copied); 3715 return copied; 3716 } 3717 #endif 3718 3719 if (!copied) 3720 goto unlock_out; 3721 3722 set_page_dirty(page); 3723 3724 if (pos + copied > i_size_read(inode) && 3725 !f2fs_verity_in_progress(inode)) { 3726 f2fs_i_size_write(inode, pos + copied); 3727 if (f2fs_is_atomic_file(inode)) 3728 f2fs_i_size_write(F2FS_I(inode)->cow_inode, 3729 pos + copied); 3730 } 3731 unlock_out: 3732 f2fs_put_page(page, 1); 3733 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); 3734 return copied; 3735 } 3736 3737 void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length) 3738 { 3739 struct inode *inode = folio->mapping->host; 3740 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3741 3742 if (inode->i_ino >= F2FS_ROOT_INO(sbi) && 3743 (offset || length != folio_size(folio))) 3744 return; 3745 3746 if (folio_test_dirty(folio)) { 3747 if (inode->i_ino == F2FS_META_INO(sbi)) { 3748 dec_page_count(sbi, F2FS_DIRTY_META); 3749 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) { 3750 dec_page_count(sbi, F2FS_DIRTY_NODES); 3751 } else { 3752 inode_dec_dirty_pages(inode); 3753 f2fs_remove_dirty_inode(inode); 3754 } 3755 } 3756 clear_page_private_all(&folio->page); 3757 } 3758 3759 bool f2fs_release_folio(struct folio *folio, gfp_t wait) 3760 { 3761 /* If this is dirty folio, keep private data */ 3762 if (folio_test_dirty(folio)) 3763 return false; 3764 3765 clear_page_private_all(&folio->page); 3766 return true; 3767 } 3768 3769 static bool f2fs_dirty_data_folio(struct address_space *mapping, 3770 struct folio *folio) 3771 { 3772 struct inode *inode = mapping->host; 3773 3774 trace_f2fs_set_page_dirty(folio, DATA); 3775 3776 if (!folio_test_uptodate(folio)) 3777 folio_mark_uptodate(folio); 3778 BUG_ON(folio_test_swapcache(folio)); 3779 3780 if (filemap_dirty_folio(mapping, folio)) { 3781 f2fs_update_dirty_folio(inode, folio); 3782 return true; 3783 } 3784 return false; 3785 } 3786 3787 3788 static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block) 3789 { 3790 #ifdef CONFIG_F2FS_FS_COMPRESSION 3791 struct dnode_of_data dn; 3792 sector_t start_idx, blknr = 0; 3793 int ret; 3794 3795 start_idx = round_down(block, F2FS_I(inode)->i_cluster_size); 3796 3797 set_new_dnode(&dn, inode, NULL, NULL, 0); 3798 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 3799 if (ret) 3800 return 0; 3801 3802 if (dn.data_blkaddr != COMPRESS_ADDR) { 3803 dn.ofs_in_node += block - start_idx; 3804 blknr = f2fs_data_blkaddr(&dn); 3805 if (!__is_valid_data_blkaddr(blknr)) 3806 blknr = 0; 3807 } 3808 3809 f2fs_put_dnode(&dn); 3810 return blknr; 3811 #else 3812 return 0; 3813 #endif 3814 } 3815 3816 3817 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) 3818 { 3819 struct inode *inode = mapping->host; 3820 sector_t blknr = 0; 3821 3822 if (f2fs_has_inline_data(inode)) 3823 goto out; 3824 3825 /* make sure allocating whole blocks */ 3826 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 3827 filemap_write_and_wait(mapping); 3828 3829 /* Block number less than F2FS MAX BLOCKS */ 3830 if (unlikely(block >= max_file_blocks(inode))) 3831 goto out; 3832 3833 if (f2fs_compressed_file(inode)) { 3834 blknr = f2fs_bmap_compress(inode, block); 3835 } else { 3836 struct f2fs_map_blocks map; 3837 3838 memset(&map, 0, sizeof(map)); 3839 map.m_lblk = block; 3840 map.m_len = 1; 3841 map.m_next_pgofs = NULL; 3842 map.m_seg_type = NO_CHECK_TYPE; 3843 3844 if (!f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_BMAP)) 3845 blknr = map.m_pblk; 3846 } 3847 out: 3848 trace_f2fs_bmap(inode, block, blknr); 3849 return blknr; 3850 } 3851 3852 #ifdef CONFIG_SWAP 3853 static int f2fs_migrate_blocks(struct inode *inode, block_t start_blk, 3854 unsigned int blkcnt) 3855 { 3856 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3857 unsigned int blkofs; 3858 unsigned int blk_per_sec = BLKS_PER_SEC(sbi); 3859 unsigned int end_blk = start_blk + blkcnt - 1; 3860 unsigned int secidx = start_blk / blk_per_sec; 3861 unsigned int end_sec; 3862 int ret = 0; 3863 3864 if (!blkcnt) 3865 return 0; 3866 end_sec = end_blk / blk_per_sec; 3867 3868 f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3869 filemap_invalidate_lock(inode->i_mapping); 3870 3871 set_inode_flag(inode, FI_ALIGNED_WRITE); 3872 set_inode_flag(inode, FI_OPU_WRITE); 3873 3874 for (; secidx <= end_sec; secidx++) { 3875 unsigned int blkofs_end = secidx == end_sec ? 3876 end_blk % blk_per_sec : blk_per_sec - 1; 3877 3878 f2fs_down_write(&sbi->pin_sem); 3879 3880 ret = f2fs_allocate_pinning_section(sbi); 3881 if (ret) { 3882 f2fs_up_write(&sbi->pin_sem); 3883 break; 3884 } 3885 3886 set_inode_flag(inode, FI_SKIP_WRITES); 3887 3888 for (blkofs = 0; blkofs <= blkofs_end; blkofs++) { 3889 struct page *page; 3890 unsigned int blkidx = secidx * blk_per_sec + blkofs; 3891 3892 page = f2fs_get_lock_data_page(inode, blkidx, true); 3893 if (IS_ERR(page)) { 3894 f2fs_up_write(&sbi->pin_sem); 3895 ret = PTR_ERR(page); 3896 goto done; 3897 } 3898 3899 set_page_dirty(page); 3900 f2fs_put_page(page, 1); 3901 } 3902 3903 clear_inode_flag(inode, FI_SKIP_WRITES); 3904 3905 ret = filemap_fdatawrite(inode->i_mapping); 3906 3907 f2fs_up_write(&sbi->pin_sem); 3908 3909 if (ret) 3910 break; 3911 } 3912 3913 done: 3914 clear_inode_flag(inode, FI_SKIP_WRITES); 3915 clear_inode_flag(inode, FI_OPU_WRITE); 3916 clear_inode_flag(inode, FI_ALIGNED_WRITE); 3917 3918 filemap_invalidate_unlock(inode->i_mapping); 3919 f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3920 3921 return ret; 3922 } 3923 3924 static int check_swap_activate(struct swap_info_struct *sis, 3925 struct file *swap_file, sector_t *span) 3926 { 3927 struct address_space *mapping = swap_file->f_mapping; 3928 struct inode *inode = mapping->host; 3929 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3930 block_t cur_lblock; 3931 block_t last_lblock; 3932 block_t pblock; 3933 block_t lowest_pblock = -1; 3934 block_t highest_pblock = 0; 3935 int nr_extents = 0; 3936 unsigned int nr_pblocks; 3937 unsigned int blks_per_sec = BLKS_PER_SEC(sbi); 3938 unsigned int not_aligned = 0; 3939 int ret = 0; 3940 3941 /* 3942 * Map all the blocks into the extent list. This code doesn't try 3943 * to be very smart. 3944 */ 3945 cur_lblock = 0; 3946 last_lblock = bytes_to_blks(inode, i_size_read(inode)); 3947 3948 while (cur_lblock < last_lblock && cur_lblock < sis->max) { 3949 struct f2fs_map_blocks map; 3950 retry: 3951 cond_resched(); 3952 3953 memset(&map, 0, sizeof(map)); 3954 map.m_lblk = cur_lblock; 3955 map.m_len = last_lblock - cur_lblock; 3956 map.m_next_pgofs = NULL; 3957 map.m_next_extent = NULL; 3958 map.m_seg_type = NO_CHECK_TYPE; 3959 map.m_may_create = false; 3960 3961 ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_FIEMAP); 3962 if (ret) 3963 goto out; 3964 3965 /* hole */ 3966 if (!(map.m_flags & F2FS_MAP_FLAGS)) { 3967 f2fs_err(sbi, "Swapfile has holes"); 3968 ret = -EINVAL; 3969 goto out; 3970 } 3971 3972 pblock = map.m_pblk; 3973 nr_pblocks = map.m_len; 3974 3975 if ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || 3976 nr_pblocks % blks_per_sec || 3977 !f2fs_valid_pinned_area(sbi, pblock)) { 3978 bool last_extent = false; 3979 3980 not_aligned++; 3981 3982 nr_pblocks = roundup(nr_pblocks, blks_per_sec); 3983 if (cur_lblock + nr_pblocks > sis->max) 3984 nr_pblocks -= blks_per_sec; 3985 3986 /* this extent is last one */ 3987 if (!nr_pblocks) { 3988 nr_pblocks = last_lblock - cur_lblock; 3989 last_extent = true; 3990 } 3991 3992 ret = f2fs_migrate_blocks(inode, cur_lblock, 3993 nr_pblocks); 3994 if (ret) { 3995 if (ret == -ENOENT) 3996 ret = -EINVAL; 3997 goto out; 3998 } 3999 4000 if (!last_extent) 4001 goto retry; 4002 } 4003 4004 if (cur_lblock + nr_pblocks >= sis->max) 4005 nr_pblocks = sis->max - cur_lblock; 4006 4007 if (cur_lblock) { /* exclude the header page */ 4008 if (pblock < lowest_pblock) 4009 lowest_pblock = pblock; 4010 if (pblock + nr_pblocks - 1 > highest_pblock) 4011 highest_pblock = pblock + nr_pblocks - 1; 4012 } 4013 4014 /* 4015 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks 4016 */ 4017 ret = add_swap_extent(sis, cur_lblock, nr_pblocks, pblock); 4018 if (ret < 0) 4019 goto out; 4020 nr_extents += ret; 4021 cur_lblock += nr_pblocks; 4022 } 4023 ret = nr_extents; 4024 *span = 1 + highest_pblock - lowest_pblock; 4025 if (cur_lblock == 0) 4026 cur_lblock = 1; /* force Empty message */ 4027 sis->max = cur_lblock; 4028 sis->pages = cur_lblock - 1; 4029 sis->highest_bit = cur_lblock - 1; 4030 out: 4031 if (not_aligned) 4032 f2fs_warn(sbi, "Swapfile (%u) is not align to section: 1) creat(), 2) ioctl(F2FS_IOC_SET_PIN_FILE), 3) fallocate(%lu * N)", 4033 not_aligned, blks_per_sec * F2FS_BLKSIZE); 4034 return ret; 4035 } 4036 4037 static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, 4038 sector_t *span) 4039 { 4040 struct inode *inode = file_inode(file); 4041 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 4042 int ret; 4043 4044 if (!S_ISREG(inode->i_mode)) 4045 return -EINVAL; 4046 4047 if (f2fs_readonly(sbi->sb)) 4048 return -EROFS; 4049 4050 if (f2fs_lfs_mode(sbi) && !f2fs_sb_has_blkzoned(sbi)) { 4051 f2fs_err(sbi, "Swapfile not supported in LFS mode"); 4052 return -EINVAL; 4053 } 4054 4055 ret = f2fs_convert_inline_inode(inode); 4056 if (ret) 4057 return ret; 4058 4059 if (!f2fs_disable_compressed_file(inode)) 4060 return -EINVAL; 4061 4062 ret = filemap_fdatawrite(inode->i_mapping); 4063 if (ret < 0) 4064 return ret; 4065 4066 f2fs_precache_extents(inode); 4067 4068 ret = check_swap_activate(sis, file, span); 4069 if (ret < 0) 4070 return ret; 4071 4072 stat_inc_swapfile_inode(inode); 4073 set_inode_flag(inode, FI_PIN_FILE); 4074 f2fs_update_time(sbi, REQ_TIME); 4075 return ret; 4076 } 4077 4078 static void f2fs_swap_deactivate(struct file *file) 4079 { 4080 struct inode *inode = file_inode(file); 4081 4082 stat_dec_swapfile_inode(inode); 4083 clear_inode_flag(inode, FI_PIN_FILE); 4084 } 4085 #else 4086 static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, 4087 sector_t *span) 4088 { 4089 return -EOPNOTSUPP; 4090 } 4091 4092 static void f2fs_swap_deactivate(struct file *file) 4093 { 4094 } 4095 #endif 4096 4097 const struct address_space_operations f2fs_dblock_aops = { 4098 .read_folio = f2fs_read_data_folio, 4099 .readahead = f2fs_readahead, 4100 .writepage = f2fs_write_data_page, 4101 .writepages = f2fs_write_data_pages, 4102 .write_begin = f2fs_write_begin, 4103 .write_end = f2fs_write_end, 4104 .dirty_folio = f2fs_dirty_data_folio, 4105 .migrate_folio = filemap_migrate_folio, 4106 .invalidate_folio = f2fs_invalidate_folio, 4107 .release_folio = f2fs_release_folio, 4108 .bmap = f2fs_bmap, 4109 .swap_activate = f2fs_swap_activate, 4110 .swap_deactivate = f2fs_swap_deactivate, 4111 }; 4112 4113 void f2fs_clear_page_cache_dirty_tag(struct page *page) 4114 { 4115 struct folio *folio = page_folio(page); 4116 struct address_space *mapping = folio->mapping; 4117 unsigned long flags; 4118 4119 xa_lock_irqsave(&mapping->i_pages, flags); 4120 __xa_clear_mark(&mapping->i_pages, folio->index, 4121 PAGECACHE_TAG_DIRTY); 4122 xa_unlock_irqrestore(&mapping->i_pages, flags); 4123 } 4124 4125 int __init f2fs_init_post_read_processing(void) 4126 { 4127 bio_post_read_ctx_cache = 4128 kmem_cache_create("f2fs_bio_post_read_ctx", 4129 sizeof(struct bio_post_read_ctx), 0, 0, NULL); 4130 if (!bio_post_read_ctx_cache) 4131 goto fail; 4132 bio_post_read_ctx_pool = 4133 mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS, 4134 bio_post_read_ctx_cache); 4135 if (!bio_post_read_ctx_pool) 4136 goto fail_free_cache; 4137 return 0; 4138 4139 fail_free_cache: 4140 kmem_cache_destroy(bio_post_read_ctx_cache); 4141 fail: 4142 return -ENOMEM; 4143 } 4144 4145 void f2fs_destroy_post_read_processing(void) 4146 { 4147 mempool_destroy(bio_post_read_ctx_pool); 4148 kmem_cache_destroy(bio_post_read_ctx_cache); 4149 } 4150 4151 int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi) 4152 { 4153 if (!f2fs_sb_has_encrypt(sbi) && 4154 !f2fs_sb_has_verity(sbi) && 4155 !f2fs_sb_has_compression(sbi)) 4156 return 0; 4157 4158 sbi->post_read_wq = alloc_workqueue("f2fs_post_read_wq", 4159 WQ_UNBOUND | WQ_HIGHPRI, 4160 num_online_cpus()); 4161 return sbi->post_read_wq ? 0 : -ENOMEM; 4162 } 4163 4164 void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi) 4165 { 4166 if (sbi->post_read_wq) 4167 destroy_workqueue(sbi->post_read_wq); 4168 } 4169 4170 int __init f2fs_init_bio_entry_cache(void) 4171 { 4172 bio_entry_slab = f2fs_kmem_cache_create("f2fs_bio_entry_slab", 4173 sizeof(struct bio_entry)); 4174 return bio_entry_slab ? 0 : -ENOMEM; 4175 } 4176 4177 void f2fs_destroy_bio_entry_cache(void) 4178 { 4179 kmem_cache_destroy(bio_entry_slab); 4180 } 4181 4182 static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, 4183 unsigned int flags, struct iomap *iomap, 4184 struct iomap *srcmap) 4185 { 4186 struct f2fs_map_blocks map = {}; 4187 pgoff_t next_pgofs = 0; 4188 int err; 4189 4190 map.m_lblk = bytes_to_blks(inode, offset); 4191 map.m_len = bytes_to_blks(inode, offset + length - 1) - map.m_lblk + 1; 4192 map.m_next_pgofs = &next_pgofs; 4193 map.m_seg_type = f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode), 4194 inode->i_write_hint); 4195 if (flags & IOMAP_WRITE) 4196 map.m_may_create = true; 4197 4198 err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DIO); 4199 if (err) 4200 return err; 4201 4202 iomap->offset = blks_to_bytes(inode, map.m_lblk); 4203 4204 /* 4205 * When inline encryption is enabled, sometimes I/O to an encrypted file 4206 * has to be broken up to guarantee DUN contiguity. Handle this by 4207 * limiting the length of the mapping returned. 4208 */ 4209 map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len); 4210 4211 /* 4212 * We should never see delalloc or compressed extents here based on 4213 * prior flushing and checks. 4214 */ 4215 if (WARN_ON_ONCE(map.m_pblk == COMPRESS_ADDR)) 4216 return -EINVAL; 4217 4218 if (map.m_flags & F2FS_MAP_MAPPED) { 4219 if (WARN_ON_ONCE(map.m_pblk == NEW_ADDR)) 4220 return -EINVAL; 4221 4222 iomap->length = blks_to_bytes(inode, map.m_len); 4223 iomap->type = IOMAP_MAPPED; 4224 iomap->flags |= IOMAP_F_MERGED; 4225 iomap->bdev = map.m_bdev; 4226 iomap->addr = blks_to_bytes(inode, map.m_pblk); 4227 } else { 4228 if (flags & IOMAP_WRITE) 4229 return -ENOTBLK; 4230 4231 if (map.m_pblk == NULL_ADDR) { 4232 iomap->length = blks_to_bytes(inode, next_pgofs) - 4233 iomap->offset; 4234 iomap->type = IOMAP_HOLE; 4235 } else if (map.m_pblk == NEW_ADDR) { 4236 iomap->length = blks_to_bytes(inode, map.m_len); 4237 iomap->type = IOMAP_UNWRITTEN; 4238 } else { 4239 f2fs_bug_on(F2FS_I_SB(inode), 1); 4240 } 4241 iomap->addr = IOMAP_NULL_ADDR; 4242 } 4243 4244 if (map.m_flags & F2FS_MAP_NEW) 4245 iomap->flags |= IOMAP_F_NEW; 4246 if ((inode->i_state & I_DIRTY_DATASYNC) || 4247 offset + length > i_size_read(inode)) 4248 iomap->flags |= IOMAP_F_DIRTY; 4249 4250 return 0; 4251 } 4252 4253 const struct iomap_ops f2fs_iomap_ops = { 4254 .iomap_begin = f2fs_iomap_begin, 4255 }; 4256