1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fs/f2fs/data.c 4 * 5 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 6 * http://www.samsung.com/ 7 */ 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/sched/mm.h> 11 #include <linux/mpage.h> 12 #include <linux/writeback.h> 13 #include <linux/pagevec.h> 14 #include <linux/blkdev.h> 15 #include <linux/bio.h> 16 #include <linux/blk-crypto.h> 17 #include <linux/swap.h> 18 #include <linux/prefetch.h> 19 #include <linux/uio.h> 20 #include <linux/sched/signal.h> 21 #include <linux/fiemap.h> 22 #include <linux/iomap.h> 23 24 #include "f2fs.h" 25 #include "node.h" 26 #include "segment.h" 27 #include "iostat.h" 28 #include <trace/events/f2fs.h> 29 30 #define NUM_PREALLOC_POST_READ_CTXS 128 31 32 static struct kmem_cache *bio_post_read_ctx_cache; 33 static struct kmem_cache *bio_entry_slab; 34 static mempool_t *bio_post_read_ctx_pool; 35 static struct bio_set f2fs_bioset; 36 37 #define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE 38 39 int __init f2fs_init_bioset(void) 40 { 41 return bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE, 42 0, BIOSET_NEED_BVECS); 43 } 44 45 void f2fs_destroy_bioset(void) 46 { 47 bioset_exit(&f2fs_bioset); 48 } 49 50 bool f2fs_is_cp_guaranteed(struct page *page) 51 { 52 struct address_space *mapping = page->mapping; 53 struct inode *inode; 54 struct f2fs_sb_info *sbi; 55 56 if (!mapping) 57 return false; 58 59 inode = mapping->host; 60 sbi = F2FS_I_SB(inode); 61 62 if (inode->i_ino == F2FS_META_INO(sbi) || 63 inode->i_ino == F2FS_NODE_INO(sbi) || 64 S_ISDIR(inode->i_mode)) 65 return true; 66 67 if ((S_ISREG(inode->i_mode) && IS_NOQUOTA(inode)) || 68 page_private_gcing(page)) 69 return true; 70 return false; 71 } 72 73 static enum count_type __read_io_type(struct folio *folio) 74 { 75 struct address_space *mapping = folio->mapping; 76 77 if (mapping) { 78 struct inode *inode = mapping->host; 79 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 80 81 if (inode->i_ino == F2FS_META_INO(sbi)) 82 return F2FS_RD_META; 83 84 if (inode->i_ino == F2FS_NODE_INO(sbi)) 85 return F2FS_RD_NODE; 86 } 87 return F2FS_RD_DATA; 88 } 89 90 /* postprocessing steps for read bios */ 91 enum bio_post_read_step { 92 #ifdef CONFIG_FS_ENCRYPTION 93 STEP_DECRYPT = BIT(0), 94 #else 95 STEP_DECRYPT = 0, /* compile out the decryption-related code */ 96 #endif 97 #ifdef CONFIG_F2FS_FS_COMPRESSION 98 STEP_DECOMPRESS = BIT(1), 99 #else 100 STEP_DECOMPRESS = 0, /* compile out the decompression-related code */ 101 #endif 102 #ifdef CONFIG_FS_VERITY 103 STEP_VERITY = BIT(2), 104 #else 105 STEP_VERITY = 0, /* compile out the verity-related code */ 106 #endif 107 }; 108 109 struct bio_post_read_ctx { 110 struct bio *bio; 111 struct f2fs_sb_info *sbi; 112 struct work_struct work; 113 unsigned int enabled_steps; 114 /* 115 * decompression_attempted keeps track of whether 116 * f2fs_end_read_compressed_page() has been called on the pages in the 117 * bio that belong to a compressed cluster yet. 118 */ 119 bool decompression_attempted; 120 block_t fs_blkaddr; 121 }; 122 123 /* 124 * Update and unlock a bio's pages, and free the bio. 125 * 126 * This marks pages up-to-date only if there was no error in the bio (I/O error, 127 * decryption error, or verity error), as indicated by bio->bi_status. 128 * 129 * "Compressed pages" (pagecache pages backed by a compressed cluster on-disk) 130 * aren't marked up-to-date here, as decompression is done on a per-compression- 131 * cluster basis rather than a per-bio basis. Instead, we only must do two 132 * things for each compressed page here: call f2fs_end_read_compressed_page() 133 * with failed=true if an error occurred before it would have normally gotten 134 * called (i.e., I/O error or decryption error, but *not* verity error), and 135 * release the bio's reference to the decompress_io_ctx of the page's cluster. 136 */ 137 static void f2fs_finish_read_bio(struct bio *bio, bool in_task) 138 { 139 struct folio_iter fi; 140 struct bio_post_read_ctx *ctx = bio->bi_private; 141 142 bio_for_each_folio_all(fi, bio) { 143 struct folio *folio = fi.folio; 144 145 if (f2fs_is_compressed_page(&folio->page)) { 146 if (ctx && !ctx->decompression_attempted) 147 f2fs_end_read_compressed_page(&folio->page, true, 0, 148 in_task); 149 f2fs_put_page_dic(&folio->page, in_task); 150 continue; 151 } 152 153 dec_page_count(F2FS_F_SB(folio), __read_io_type(folio)); 154 folio_end_read(folio, bio->bi_status == 0); 155 } 156 157 if (ctx) 158 mempool_free(ctx, bio_post_read_ctx_pool); 159 bio_put(bio); 160 } 161 162 static void f2fs_verify_bio(struct work_struct *work) 163 { 164 struct bio_post_read_ctx *ctx = 165 container_of(work, struct bio_post_read_ctx, work); 166 struct bio *bio = ctx->bio; 167 bool may_have_compressed_pages = (ctx->enabled_steps & STEP_DECOMPRESS); 168 169 /* 170 * fsverity_verify_bio() may call readahead() again, and while verity 171 * will be disabled for this, decryption and/or decompression may still 172 * be needed, resulting in another bio_post_read_ctx being allocated. 173 * So to prevent deadlocks we need to release the current ctx to the 174 * mempool first. This assumes that verity is the last post-read step. 175 */ 176 mempool_free(ctx, bio_post_read_ctx_pool); 177 bio->bi_private = NULL; 178 179 /* 180 * Verify the bio's pages with fs-verity. Exclude compressed pages, 181 * as those were handled separately by f2fs_end_read_compressed_page(). 182 */ 183 if (may_have_compressed_pages) { 184 struct bio_vec *bv; 185 struct bvec_iter_all iter_all; 186 187 bio_for_each_segment_all(bv, bio, iter_all) { 188 struct page *page = bv->bv_page; 189 190 if (!f2fs_is_compressed_page(page) && 191 !fsverity_verify_page(page)) { 192 bio->bi_status = BLK_STS_IOERR; 193 break; 194 } 195 } 196 } else { 197 fsverity_verify_bio(bio); 198 } 199 200 f2fs_finish_read_bio(bio, true); 201 } 202 203 /* 204 * If the bio's data needs to be verified with fs-verity, then enqueue the 205 * verity work for the bio. Otherwise finish the bio now. 206 * 207 * Note that to avoid deadlocks, the verity work can't be done on the 208 * decryption/decompression workqueue. This is because verifying the data pages 209 * can involve reading verity metadata pages from the file, and these verity 210 * metadata pages may be encrypted and/or compressed. 211 */ 212 static void f2fs_verify_and_finish_bio(struct bio *bio, bool in_task) 213 { 214 struct bio_post_read_ctx *ctx = bio->bi_private; 215 216 if (ctx && (ctx->enabled_steps & STEP_VERITY)) { 217 INIT_WORK(&ctx->work, f2fs_verify_bio); 218 fsverity_enqueue_verify_work(&ctx->work); 219 } else { 220 f2fs_finish_read_bio(bio, in_task); 221 } 222 } 223 224 /* 225 * Handle STEP_DECOMPRESS by decompressing any compressed clusters whose last 226 * remaining page was read by @ctx->bio. 227 * 228 * Note that a bio may span clusters (even a mix of compressed and uncompressed 229 * clusters) or be for just part of a cluster. STEP_DECOMPRESS just indicates 230 * that the bio includes at least one compressed page. The actual decompression 231 * is done on a per-cluster basis, not a per-bio basis. 232 */ 233 static void f2fs_handle_step_decompress(struct bio_post_read_ctx *ctx, 234 bool in_task) 235 { 236 struct bio_vec *bv; 237 struct bvec_iter_all iter_all; 238 bool all_compressed = true; 239 block_t blkaddr = ctx->fs_blkaddr; 240 241 bio_for_each_segment_all(bv, ctx->bio, iter_all) { 242 struct page *page = bv->bv_page; 243 244 if (f2fs_is_compressed_page(page)) 245 f2fs_end_read_compressed_page(page, false, blkaddr, 246 in_task); 247 else 248 all_compressed = false; 249 250 blkaddr++; 251 } 252 253 ctx->decompression_attempted = true; 254 255 /* 256 * Optimization: if all the bio's pages are compressed, then scheduling 257 * the per-bio verity work is unnecessary, as verity will be fully 258 * handled at the compression cluster level. 259 */ 260 if (all_compressed) 261 ctx->enabled_steps &= ~STEP_VERITY; 262 } 263 264 static void f2fs_post_read_work(struct work_struct *work) 265 { 266 struct bio_post_read_ctx *ctx = 267 container_of(work, struct bio_post_read_ctx, work); 268 struct bio *bio = ctx->bio; 269 270 if ((ctx->enabled_steps & STEP_DECRYPT) && !fscrypt_decrypt_bio(bio)) { 271 f2fs_finish_read_bio(bio, true); 272 return; 273 } 274 275 if (ctx->enabled_steps & STEP_DECOMPRESS) 276 f2fs_handle_step_decompress(ctx, true); 277 278 f2fs_verify_and_finish_bio(bio, true); 279 } 280 281 static void f2fs_read_end_io(struct bio *bio) 282 { 283 struct f2fs_sb_info *sbi = F2FS_P_SB(bio_first_page_all(bio)); 284 struct bio_post_read_ctx *ctx; 285 bool intask = in_task(); 286 287 iostat_update_and_unbind_ctx(bio); 288 ctx = bio->bi_private; 289 290 if (time_to_inject(sbi, FAULT_READ_IO)) 291 bio->bi_status = BLK_STS_IOERR; 292 293 if (bio->bi_status) { 294 f2fs_finish_read_bio(bio, intask); 295 return; 296 } 297 298 if (ctx) { 299 unsigned int enabled_steps = ctx->enabled_steps & 300 (STEP_DECRYPT | STEP_DECOMPRESS); 301 302 /* 303 * If we have only decompression step between decompression and 304 * decrypt, we don't need post processing for this. 305 */ 306 if (enabled_steps == STEP_DECOMPRESS && 307 !f2fs_low_mem_mode(sbi)) { 308 f2fs_handle_step_decompress(ctx, intask); 309 } else if (enabled_steps) { 310 INIT_WORK(&ctx->work, f2fs_post_read_work); 311 queue_work(ctx->sbi->post_read_wq, &ctx->work); 312 return; 313 } 314 } 315 316 f2fs_verify_and_finish_bio(bio, intask); 317 } 318 319 static void f2fs_write_end_io(struct bio *bio) 320 { 321 struct f2fs_sb_info *sbi; 322 struct folio_iter fi; 323 324 iostat_update_and_unbind_ctx(bio); 325 sbi = bio->bi_private; 326 327 if (time_to_inject(sbi, FAULT_WRITE_IO)) 328 bio->bi_status = BLK_STS_IOERR; 329 330 bio_for_each_folio_all(fi, bio) { 331 struct folio *folio = fi.folio; 332 enum count_type type; 333 334 if (fscrypt_is_bounce_folio(folio)) { 335 struct folio *io_folio = folio; 336 337 folio = fscrypt_pagecache_folio(io_folio); 338 fscrypt_free_bounce_page(&io_folio->page); 339 } 340 341 #ifdef CONFIG_F2FS_FS_COMPRESSION 342 if (f2fs_is_compressed_page(&folio->page)) { 343 f2fs_compress_write_end_io(bio, &folio->page); 344 continue; 345 } 346 #endif 347 348 type = WB_DATA_TYPE(&folio->page, false); 349 350 if (unlikely(bio->bi_status)) { 351 mapping_set_error(folio->mapping, -EIO); 352 if (type == F2FS_WB_CP_DATA) 353 f2fs_stop_checkpoint(sbi, true, 354 STOP_CP_REASON_WRITE_FAIL); 355 } 356 357 f2fs_bug_on(sbi, folio->mapping == NODE_MAPPING(sbi) && 358 folio->index != nid_of_node(&folio->page)); 359 360 dec_page_count(sbi, type); 361 if (f2fs_in_warm_node_list(sbi, folio)) 362 f2fs_del_fsync_node_entry(sbi, &folio->page); 363 clear_page_private_gcing(&folio->page); 364 folio_end_writeback(folio); 365 } 366 if (!get_pages(sbi, F2FS_WB_CP_DATA) && 367 wq_has_sleeper(&sbi->cp_wait)) 368 wake_up(&sbi->cp_wait); 369 370 bio_put(bio); 371 } 372 373 #ifdef CONFIG_BLK_DEV_ZONED 374 static void f2fs_zone_write_end_io(struct bio *bio) 375 { 376 struct f2fs_bio_info *io = (struct f2fs_bio_info *)bio->bi_private; 377 378 bio->bi_private = io->bi_private; 379 complete(&io->zone_wait); 380 f2fs_write_end_io(bio); 381 } 382 #endif 383 384 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, 385 block_t blk_addr, sector_t *sector) 386 { 387 struct block_device *bdev = sbi->sb->s_bdev; 388 int i; 389 390 if (f2fs_is_multi_device(sbi)) { 391 for (i = 0; i < sbi->s_ndevs; i++) { 392 if (FDEV(i).start_blk <= blk_addr && 393 FDEV(i).end_blk >= blk_addr) { 394 blk_addr -= FDEV(i).start_blk; 395 bdev = FDEV(i).bdev; 396 break; 397 } 398 } 399 } 400 401 if (sector) 402 *sector = SECTOR_FROM_BLOCK(blk_addr); 403 return bdev; 404 } 405 406 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) 407 { 408 int i; 409 410 if (!f2fs_is_multi_device(sbi)) 411 return 0; 412 413 for (i = 0; i < sbi->s_ndevs; i++) 414 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) 415 return i; 416 return 0; 417 } 418 419 static blk_opf_t f2fs_io_flags(struct f2fs_io_info *fio) 420 { 421 unsigned int temp_mask = GENMASK(NR_TEMP_TYPE - 1, 0); 422 struct folio *fio_folio = page_folio(fio->page); 423 unsigned int fua_flag, meta_flag, io_flag; 424 blk_opf_t op_flags = 0; 425 426 if (fio->op != REQ_OP_WRITE) 427 return 0; 428 if (fio->type == DATA) 429 io_flag = fio->sbi->data_io_flag; 430 else if (fio->type == NODE) 431 io_flag = fio->sbi->node_io_flag; 432 else 433 return 0; 434 435 fua_flag = io_flag & temp_mask; 436 meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask; 437 438 /* 439 * data/node io flag bits per temp: 440 * REQ_META | REQ_FUA | 441 * 5 | 4 | 3 | 2 | 1 | 0 | 442 * Cold | Warm | Hot | Cold | Warm | Hot | 443 */ 444 if (BIT(fio->temp) & meta_flag) 445 op_flags |= REQ_META; 446 if (BIT(fio->temp) & fua_flag) 447 op_flags |= REQ_FUA; 448 449 if (fio->type == DATA && 450 F2FS_I(fio_folio->mapping->host)->ioprio_hint == F2FS_IOPRIO_WRITE) 451 op_flags |= REQ_PRIO; 452 453 return op_flags; 454 } 455 456 static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) 457 { 458 struct f2fs_sb_info *sbi = fio->sbi; 459 struct block_device *bdev; 460 sector_t sector; 461 struct bio *bio; 462 463 bdev = f2fs_target_device(sbi, fio->new_blkaddr, §or); 464 bio = bio_alloc_bioset(bdev, npages, 465 fio->op | fio->op_flags | f2fs_io_flags(fio), 466 GFP_NOIO, &f2fs_bioset); 467 bio->bi_iter.bi_sector = sector; 468 if (is_read_io(fio->op)) { 469 bio->bi_end_io = f2fs_read_end_io; 470 bio->bi_private = NULL; 471 } else { 472 bio->bi_end_io = f2fs_write_end_io; 473 bio->bi_private = sbi; 474 bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, 475 fio->type, fio->temp); 476 } 477 iostat_alloc_and_bind_ctx(sbi, bio, NULL); 478 479 if (fio->io_wbc) 480 wbc_init_bio(fio->io_wbc, bio); 481 482 return bio; 483 } 484 485 static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, 486 pgoff_t first_idx, 487 const struct f2fs_io_info *fio, 488 gfp_t gfp_mask) 489 { 490 /* 491 * The f2fs garbage collector sets ->encrypted_page when it wants to 492 * read/write raw data without encryption. 493 */ 494 if (!fio || !fio->encrypted_page) 495 fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask); 496 } 497 498 static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode, 499 pgoff_t next_idx, 500 const struct f2fs_io_info *fio) 501 { 502 /* 503 * The f2fs garbage collector sets ->encrypted_page when it wants to 504 * read/write raw data without encryption. 505 */ 506 if (fio && fio->encrypted_page) 507 return !bio_has_crypt_ctx(bio); 508 509 return fscrypt_mergeable_bio(bio, inode, next_idx); 510 } 511 512 void f2fs_submit_read_bio(struct f2fs_sb_info *sbi, struct bio *bio, 513 enum page_type type) 514 { 515 WARN_ON_ONCE(!is_read_io(bio_op(bio))); 516 trace_f2fs_submit_read_bio(sbi->sb, type, bio); 517 518 iostat_update_submit_ctx(bio, type); 519 submit_bio(bio); 520 } 521 522 static void f2fs_submit_write_bio(struct f2fs_sb_info *sbi, struct bio *bio, 523 enum page_type type) 524 { 525 WARN_ON_ONCE(is_read_io(bio_op(bio))); 526 trace_f2fs_submit_write_bio(sbi->sb, type, bio); 527 iostat_update_submit_ctx(bio, type); 528 submit_bio(bio); 529 } 530 531 static void __submit_merged_bio(struct f2fs_bio_info *io) 532 { 533 struct f2fs_io_info *fio = &io->fio; 534 535 if (!io->bio) 536 return; 537 538 if (is_read_io(fio->op)) { 539 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); 540 f2fs_submit_read_bio(io->sbi, io->bio, fio->type); 541 } else { 542 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); 543 f2fs_submit_write_bio(io->sbi, io->bio, fio->type); 544 } 545 io->bio = NULL; 546 } 547 548 static bool __has_merged_page(struct bio *bio, struct inode *inode, 549 struct page *page, nid_t ino) 550 { 551 struct bio_vec *bvec; 552 struct bvec_iter_all iter_all; 553 554 if (!bio) 555 return false; 556 557 if (!inode && !page && !ino) 558 return true; 559 560 bio_for_each_segment_all(bvec, bio, iter_all) { 561 struct page *target = bvec->bv_page; 562 563 if (fscrypt_is_bounce_page(target)) { 564 target = fscrypt_pagecache_page(target); 565 if (IS_ERR(target)) 566 continue; 567 } 568 if (f2fs_is_compressed_page(target)) { 569 target = f2fs_compress_control_page(target); 570 if (IS_ERR(target)) 571 continue; 572 } 573 574 if (inode && inode == target->mapping->host) 575 return true; 576 if (page && page == target) 577 return true; 578 if (ino && ino == ino_of_node(target)) 579 return true; 580 } 581 582 return false; 583 } 584 585 int f2fs_init_write_merge_io(struct f2fs_sb_info *sbi) 586 { 587 int i; 588 589 for (i = 0; i < NR_PAGE_TYPE; i++) { 590 int n = (i == META) ? 1 : NR_TEMP_TYPE; 591 int j; 592 593 sbi->write_io[i] = f2fs_kmalloc(sbi, 594 array_size(n, sizeof(struct f2fs_bio_info)), 595 GFP_KERNEL); 596 if (!sbi->write_io[i]) 597 return -ENOMEM; 598 599 for (j = HOT; j < n; j++) { 600 struct f2fs_bio_info *io = &sbi->write_io[i][j]; 601 602 init_f2fs_rwsem(&io->io_rwsem); 603 io->sbi = sbi; 604 io->bio = NULL; 605 io->last_block_in_bio = 0; 606 spin_lock_init(&io->io_lock); 607 INIT_LIST_HEAD(&io->io_list); 608 INIT_LIST_HEAD(&io->bio_list); 609 init_f2fs_rwsem(&io->bio_list_lock); 610 #ifdef CONFIG_BLK_DEV_ZONED 611 init_completion(&io->zone_wait); 612 io->zone_pending_bio = NULL; 613 io->bi_private = NULL; 614 #endif 615 } 616 } 617 618 return 0; 619 } 620 621 static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, 622 enum page_type type, enum temp_type temp) 623 { 624 enum page_type btype = PAGE_TYPE_OF_BIO(type); 625 struct f2fs_bio_info *io = sbi->write_io[btype] + temp; 626 627 f2fs_down_write(&io->io_rwsem); 628 629 if (!io->bio) 630 goto unlock_out; 631 632 /* change META to META_FLUSH in the checkpoint procedure */ 633 if (type >= META_FLUSH) { 634 io->fio.type = META_FLUSH; 635 io->bio->bi_opf |= REQ_META | REQ_PRIO | REQ_SYNC; 636 if (!test_opt(sbi, NOBARRIER)) 637 io->bio->bi_opf |= REQ_PREFLUSH | REQ_FUA; 638 } 639 __submit_merged_bio(io); 640 unlock_out: 641 f2fs_up_write(&io->io_rwsem); 642 } 643 644 static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, 645 struct inode *inode, struct page *page, 646 nid_t ino, enum page_type type, bool force) 647 { 648 enum temp_type temp; 649 bool ret = true; 650 651 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { 652 if (!force) { 653 enum page_type btype = PAGE_TYPE_OF_BIO(type); 654 struct f2fs_bio_info *io = sbi->write_io[btype] + temp; 655 656 f2fs_down_read(&io->io_rwsem); 657 ret = __has_merged_page(io->bio, inode, page, ino); 658 f2fs_up_read(&io->io_rwsem); 659 } 660 if (ret) 661 __f2fs_submit_merged_write(sbi, type, temp); 662 663 /* TODO: use HOT temp only for meta pages now. */ 664 if (type >= META) 665 break; 666 } 667 } 668 669 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type) 670 { 671 __submit_merged_write_cond(sbi, NULL, NULL, 0, type, true); 672 } 673 674 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, 675 struct inode *inode, struct page *page, 676 nid_t ino, enum page_type type) 677 { 678 __submit_merged_write_cond(sbi, inode, page, ino, type, false); 679 } 680 681 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi) 682 { 683 f2fs_submit_merged_write(sbi, DATA); 684 f2fs_submit_merged_write(sbi, NODE); 685 f2fs_submit_merged_write(sbi, META); 686 } 687 688 /* 689 * Fill the locked page with data located in the block address. 690 * A caller needs to unlock the page on failure. 691 */ 692 int f2fs_submit_page_bio(struct f2fs_io_info *fio) 693 { 694 struct bio *bio; 695 struct folio *fio_folio = page_folio(fio->page); 696 struct folio *data_folio = fio->encrypted_page ? 697 page_folio(fio->encrypted_page) : fio_folio; 698 699 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, 700 fio->is_por ? META_POR : (__is_meta_io(fio) ? 701 META_GENERIC : DATA_GENERIC_ENHANCE))) 702 return -EFSCORRUPTED; 703 704 trace_f2fs_submit_folio_bio(data_folio, fio); 705 706 /* Allocate a new bio */ 707 bio = __bio_alloc(fio, 1); 708 709 f2fs_set_bio_crypt_ctx(bio, fio_folio->mapping->host, 710 fio_folio->index, fio, GFP_NOIO); 711 bio_add_folio_nofail(bio, data_folio, folio_size(data_folio), 0); 712 713 if (fio->io_wbc && !is_read_io(fio->op)) 714 wbc_account_cgroup_owner(fio->io_wbc, fio_folio, PAGE_SIZE); 715 716 inc_page_count(fio->sbi, is_read_io(fio->op) ? 717 __read_io_type(data_folio) : WB_DATA_TYPE(fio->page, false)); 718 719 if (is_read_io(bio_op(bio))) 720 f2fs_submit_read_bio(fio->sbi, bio, fio->type); 721 else 722 f2fs_submit_write_bio(fio->sbi, bio, fio->type); 723 return 0; 724 } 725 726 static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, 727 block_t last_blkaddr, block_t cur_blkaddr) 728 { 729 if (unlikely(sbi->max_io_bytes && 730 bio->bi_iter.bi_size >= sbi->max_io_bytes)) 731 return false; 732 if (last_blkaddr + 1 != cur_blkaddr) 733 return false; 734 return bio->bi_bdev == f2fs_target_device(sbi, cur_blkaddr, NULL); 735 } 736 737 static bool io_type_is_mergeable(struct f2fs_bio_info *io, 738 struct f2fs_io_info *fio) 739 { 740 if (io->fio.op != fio->op) 741 return false; 742 return io->fio.op_flags == fio->op_flags; 743 } 744 745 static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, 746 struct f2fs_bio_info *io, 747 struct f2fs_io_info *fio, 748 block_t last_blkaddr, 749 block_t cur_blkaddr) 750 { 751 if (!page_is_mergeable(sbi, bio, last_blkaddr, cur_blkaddr)) 752 return false; 753 return io_type_is_mergeable(io, fio); 754 } 755 756 static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio, 757 struct page *page, enum temp_type temp) 758 { 759 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; 760 struct bio_entry *be; 761 762 be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS, true, NULL); 763 be->bio = bio; 764 bio_get(bio); 765 766 if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE) 767 f2fs_bug_on(sbi, 1); 768 769 f2fs_down_write(&io->bio_list_lock); 770 list_add_tail(&be->list, &io->bio_list); 771 f2fs_up_write(&io->bio_list_lock); 772 } 773 774 static void del_bio_entry(struct bio_entry *be) 775 { 776 list_del(&be->list); 777 kmem_cache_free(bio_entry_slab, be); 778 } 779 780 static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio, 781 struct page *page) 782 { 783 struct f2fs_sb_info *sbi = fio->sbi; 784 enum temp_type temp; 785 bool found = false; 786 int ret = -EAGAIN; 787 788 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) { 789 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; 790 struct list_head *head = &io->bio_list; 791 struct bio_entry *be; 792 793 f2fs_down_write(&io->bio_list_lock); 794 list_for_each_entry(be, head, list) { 795 if (be->bio != *bio) 796 continue; 797 798 found = true; 799 800 f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio, 801 *fio->last_block, 802 fio->new_blkaddr)); 803 if (f2fs_crypt_mergeable_bio(*bio, 804 fio->page->mapping->host, 805 page_folio(fio->page)->index, fio) && 806 bio_add_page(*bio, page, PAGE_SIZE, 0) == 807 PAGE_SIZE) { 808 ret = 0; 809 break; 810 } 811 812 /* page can't be merged into bio; submit the bio */ 813 del_bio_entry(be); 814 f2fs_submit_write_bio(sbi, *bio, DATA); 815 break; 816 } 817 f2fs_up_write(&io->bio_list_lock); 818 } 819 820 if (ret) { 821 bio_put(*bio); 822 *bio = NULL; 823 } 824 825 return ret; 826 } 827 828 void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, 829 struct bio **bio, struct page *page) 830 { 831 enum temp_type temp; 832 bool found = false; 833 struct bio *target = bio ? *bio : NULL; 834 835 f2fs_bug_on(sbi, !target && !page); 836 837 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) { 838 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; 839 struct list_head *head = &io->bio_list; 840 struct bio_entry *be; 841 842 if (list_empty(head)) 843 continue; 844 845 f2fs_down_read(&io->bio_list_lock); 846 list_for_each_entry(be, head, list) { 847 if (target) 848 found = (target == be->bio); 849 else 850 found = __has_merged_page(be->bio, NULL, 851 page, 0); 852 if (found) 853 break; 854 } 855 f2fs_up_read(&io->bio_list_lock); 856 857 if (!found) 858 continue; 859 860 found = false; 861 862 f2fs_down_write(&io->bio_list_lock); 863 list_for_each_entry(be, head, list) { 864 if (target) 865 found = (target == be->bio); 866 else 867 found = __has_merged_page(be->bio, NULL, 868 page, 0); 869 if (found) { 870 target = be->bio; 871 del_bio_entry(be); 872 break; 873 } 874 } 875 f2fs_up_write(&io->bio_list_lock); 876 } 877 878 if (found) 879 f2fs_submit_write_bio(sbi, target, DATA); 880 if (bio && *bio) { 881 bio_put(*bio); 882 *bio = NULL; 883 } 884 } 885 886 int f2fs_merge_page_bio(struct f2fs_io_info *fio) 887 { 888 struct bio *bio = *fio->bio; 889 struct page *page = fio->encrypted_page ? 890 fio->encrypted_page : fio->page; 891 struct folio *folio = page_folio(fio->page); 892 893 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, 894 __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC)) 895 return -EFSCORRUPTED; 896 897 trace_f2fs_submit_folio_bio(page_folio(page), fio); 898 899 if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block, 900 fio->new_blkaddr)) 901 f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL); 902 alloc_new: 903 if (!bio) { 904 bio = __bio_alloc(fio, BIO_MAX_VECS); 905 f2fs_set_bio_crypt_ctx(bio, folio->mapping->host, 906 folio->index, fio, GFP_NOIO); 907 908 add_bio_entry(fio->sbi, bio, page, fio->temp); 909 } else { 910 if (add_ipu_page(fio, &bio, page)) 911 goto alloc_new; 912 } 913 914 if (fio->io_wbc) 915 wbc_account_cgroup_owner(fio->io_wbc, folio, folio_size(folio)); 916 917 inc_page_count(fio->sbi, WB_DATA_TYPE(page, false)); 918 919 *fio->last_block = fio->new_blkaddr; 920 *fio->bio = bio; 921 922 return 0; 923 } 924 925 #ifdef CONFIG_BLK_DEV_ZONED 926 static bool is_end_zone_blkaddr(struct f2fs_sb_info *sbi, block_t blkaddr) 927 { 928 struct block_device *bdev = sbi->sb->s_bdev; 929 int devi = 0; 930 931 if (f2fs_is_multi_device(sbi)) { 932 devi = f2fs_target_device_index(sbi, blkaddr); 933 if (blkaddr < FDEV(devi).start_blk || 934 blkaddr > FDEV(devi).end_blk) { 935 f2fs_err(sbi, "Invalid block %x", blkaddr); 936 return false; 937 } 938 blkaddr -= FDEV(devi).start_blk; 939 bdev = FDEV(devi).bdev; 940 } 941 return bdev_is_zoned(bdev) && 942 f2fs_blkz_is_seq(sbi, devi, blkaddr) && 943 (blkaddr % sbi->blocks_per_blkz == sbi->blocks_per_blkz - 1); 944 } 945 #endif 946 947 void f2fs_submit_page_write(struct f2fs_io_info *fio) 948 { 949 struct f2fs_sb_info *sbi = fio->sbi; 950 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); 951 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; 952 struct page *bio_page; 953 enum count_type type; 954 955 f2fs_bug_on(sbi, is_read_io(fio->op)); 956 957 f2fs_down_write(&io->io_rwsem); 958 next: 959 #ifdef CONFIG_BLK_DEV_ZONED 960 if (f2fs_sb_has_blkzoned(sbi) && btype < META && io->zone_pending_bio) { 961 wait_for_completion_io(&io->zone_wait); 962 bio_put(io->zone_pending_bio); 963 io->zone_pending_bio = NULL; 964 io->bi_private = NULL; 965 } 966 #endif 967 968 if (fio->in_list) { 969 spin_lock(&io->io_lock); 970 if (list_empty(&io->io_list)) { 971 spin_unlock(&io->io_lock); 972 goto out; 973 } 974 fio = list_first_entry(&io->io_list, 975 struct f2fs_io_info, list); 976 list_del(&fio->list); 977 spin_unlock(&io->io_lock); 978 } 979 980 verify_fio_blkaddr(fio); 981 982 if (fio->encrypted_page) 983 bio_page = fio->encrypted_page; 984 else if (fio->compressed_page) 985 bio_page = fio->compressed_page; 986 else 987 bio_page = fio->page; 988 989 /* set submitted = true as a return value */ 990 fio->submitted = 1; 991 992 type = WB_DATA_TYPE(bio_page, fio->compressed_page); 993 inc_page_count(sbi, type); 994 995 if (io->bio && 996 (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio, 997 fio->new_blkaddr) || 998 !f2fs_crypt_mergeable_bio(io->bio, fio->page->mapping->host, 999 page_folio(bio_page)->index, fio))) 1000 __submit_merged_bio(io); 1001 alloc_new: 1002 if (io->bio == NULL) { 1003 io->bio = __bio_alloc(fio, BIO_MAX_VECS); 1004 f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host, 1005 page_folio(bio_page)->index, fio, GFP_NOIO); 1006 io->fio = *fio; 1007 } 1008 1009 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) { 1010 __submit_merged_bio(io); 1011 goto alloc_new; 1012 } 1013 1014 if (fio->io_wbc) 1015 wbc_account_cgroup_owner(fio->io_wbc, page_folio(fio->page), 1016 PAGE_SIZE); 1017 1018 io->last_block_in_bio = fio->new_blkaddr; 1019 1020 trace_f2fs_submit_folio_write(page_folio(fio->page), fio); 1021 #ifdef CONFIG_BLK_DEV_ZONED 1022 if (f2fs_sb_has_blkzoned(sbi) && btype < META && 1023 is_end_zone_blkaddr(sbi, fio->new_blkaddr)) { 1024 bio_get(io->bio); 1025 reinit_completion(&io->zone_wait); 1026 io->bi_private = io->bio->bi_private; 1027 io->bio->bi_private = io; 1028 io->bio->bi_end_io = f2fs_zone_write_end_io; 1029 io->zone_pending_bio = io->bio; 1030 __submit_merged_bio(io); 1031 } 1032 #endif 1033 if (fio->in_list) 1034 goto next; 1035 out: 1036 if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || 1037 !f2fs_is_checkpoint_ready(sbi)) 1038 __submit_merged_bio(io); 1039 f2fs_up_write(&io->io_rwsem); 1040 } 1041 1042 static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, 1043 unsigned nr_pages, blk_opf_t op_flag, 1044 pgoff_t first_idx, bool for_write) 1045 { 1046 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1047 struct bio *bio; 1048 struct bio_post_read_ctx *ctx = NULL; 1049 unsigned int post_read_steps = 0; 1050 sector_t sector; 1051 struct block_device *bdev = f2fs_target_device(sbi, blkaddr, §or); 1052 1053 bio = bio_alloc_bioset(bdev, bio_max_segs(nr_pages), 1054 REQ_OP_READ | op_flag, 1055 for_write ? GFP_NOIO : GFP_KERNEL, &f2fs_bioset); 1056 bio->bi_iter.bi_sector = sector; 1057 f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS); 1058 bio->bi_end_io = f2fs_read_end_io; 1059 1060 if (fscrypt_inode_uses_fs_layer_crypto(inode)) 1061 post_read_steps |= STEP_DECRYPT; 1062 1063 if (f2fs_need_verity(inode, first_idx)) 1064 post_read_steps |= STEP_VERITY; 1065 1066 /* 1067 * STEP_DECOMPRESS is handled specially, since a compressed file might 1068 * contain both compressed and uncompressed clusters. We'll allocate a 1069 * bio_post_read_ctx if the file is compressed, but the caller is 1070 * responsible for enabling STEP_DECOMPRESS if it's actually needed. 1071 */ 1072 1073 if (post_read_steps || f2fs_compressed_file(inode)) { 1074 /* Due to the mempool, this never fails. */ 1075 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS); 1076 ctx->bio = bio; 1077 ctx->sbi = sbi; 1078 ctx->enabled_steps = post_read_steps; 1079 ctx->fs_blkaddr = blkaddr; 1080 ctx->decompression_attempted = false; 1081 bio->bi_private = ctx; 1082 } 1083 iostat_alloc_and_bind_ctx(sbi, bio, ctx); 1084 1085 return bio; 1086 } 1087 1088 /* This can handle encryption stuffs */ 1089 static int f2fs_submit_page_read(struct inode *inode, struct folio *folio, 1090 block_t blkaddr, blk_opf_t op_flags, 1091 bool for_write) 1092 { 1093 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1094 struct bio *bio; 1095 1096 bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags, 1097 folio->index, for_write); 1098 if (IS_ERR(bio)) 1099 return PTR_ERR(bio); 1100 1101 /* wait for GCed page writeback via META_MAPPING */ 1102 f2fs_wait_on_block_writeback(inode, blkaddr); 1103 1104 if (!bio_add_folio(bio, folio, PAGE_SIZE, 0)) { 1105 iostat_update_and_unbind_ctx(bio); 1106 if (bio->bi_private) 1107 mempool_free(bio->bi_private, bio_post_read_ctx_pool); 1108 bio_put(bio); 1109 return -EFAULT; 1110 } 1111 inc_page_count(sbi, F2FS_RD_DATA); 1112 f2fs_update_iostat(sbi, NULL, FS_DATA_READ_IO, F2FS_BLKSIZE); 1113 f2fs_submit_read_bio(sbi, bio, DATA); 1114 return 0; 1115 } 1116 1117 static void __set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) 1118 { 1119 __le32 *addr = get_dnode_addr(dn->inode, dn->node_page); 1120 1121 dn->data_blkaddr = blkaddr; 1122 addr[dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr); 1123 } 1124 1125 /* 1126 * Lock ordering for the change of data block address: 1127 * ->data_page 1128 * ->node_page 1129 * update block addresses in the node page 1130 */ 1131 void f2fs_set_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) 1132 { 1133 f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true); 1134 __set_data_blkaddr(dn, blkaddr); 1135 if (set_page_dirty(dn->node_page)) 1136 dn->node_changed = true; 1137 } 1138 1139 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) 1140 { 1141 f2fs_set_data_blkaddr(dn, blkaddr); 1142 f2fs_update_read_extent_cache(dn); 1143 } 1144 1145 /* dn->ofs_in_node will be returned with up-to-date last block pointer */ 1146 int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count) 1147 { 1148 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1149 int err; 1150 1151 if (!count) 1152 return 0; 1153 1154 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) 1155 return -EPERM; 1156 err = inc_valid_block_count(sbi, dn->inode, &count, true); 1157 if (unlikely(err)) 1158 return err; 1159 1160 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid, 1161 dn->ofs_in_node, count); 1162 1163 f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true); 1164 1165 for (; count > 0; dn->ofs_in_node++) { 1166 block_t blkaddr = f2fs_data_blkaddr(dn); 1167 1168 if (blkaddr == NULL_ADDR) { 1169 __set_data_blkaddr(dn, NEW_ADDR); 1170 count--; 1171 } 1172 } 1173 1174 if (set_page_dirty(dn->node_page)) 1175 dn->node_changed = true; 1176 return 0; 1177 } 1178 1179 /* Should keep dn->ofs_in_node unchanged */ 1180 int f2fs_reserve_new_block(struct dnode_of_data *dn) 1181 { 1182 unsigned int ofs_in_node = dn->ofs_in_node; 1183 int ret; 1184 1185 ret = f2fs_reserve_new_blocks(dn, 1); 1186 dn->ofs_in_node = ofs_in_node; 1187 return ret; 1188 } 1189 1190 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) 1191 { 1192 bool need_put = dn->inode_page ? false : true; 1193 int err; 1194 1195 err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE); 1196 if (err) 1197 return err; 1198 1199 if (dn->data_blkaddr == NULL_ADDR) 1200 err = f2fs_reserve_new_block(dn); 1201 if (err || need_put) 1202 f2fs_put_dnode(dn); 1203 return err; 1204 } 1205 1206 struct folio *f2fs_get_read_data_folio(struct inode *inode, pgoff_t index, 1207 blk_opf_t op_flags, bool for_write, pgoff_t *next_pgofs) 1208 { 1209 struct address_space *mapping = inode->i_mapping; 1210 struct dnode_of_data dn; 1211 struct folio *folio; 1212 int err; 1213 1214 folio = f2fs_grab_cache_folio(mapping, index, for_write); 1215 if (IS_ERR(folio)) 1216 return folio; 1217 1218 if (f2fs_lookup_read_extent_cache_block(inode, index, 1219 &dn.data_blkaddr)) { 1220 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr, 1221 DATA_GENERIC_ENHANCE_READ)) { 1222 err = -EFSCORRUPTED; 1223 goto put_err; 1224 } 1225 goto got_it; 1226 } 1227 1228 set_new_dnode(&dn, inode, NULL, NULL, 0); 1229 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 1230 if (err) { 1231 if (err == -ENOENT && next_pgofs) 1232 *next_pgofs = f2fs_get_next_page_offset(&dn, index); 1233 goto put_err; 1234 } 1235 f2fs_put_dnode(&dn); 1236 1237 if (unlikely(dn.data_blkaddr == NULL_ADDR)) { 1238 err = -ENOENT; 1239 if (next_pgofs) 1240 *next_pgofs = index + 1; 1241 goto put_err; 1242 } 1243 if (dn.data_blkaddr != NEW_ADDR && 1244 !f2fs_is_valid_blkaddr(F2FS_I_SB(inode), 1245 dn.data_blkaddr, 1246 DATA_GENERIC_ENHANCE)) { 1247 err = -EFSCORRUPTED; 1248 goto put_err; 1249 } 1250 got_it: 1251 if (folio_test_uptodate(folio)) { 1252 folio_unlock(folio); 1253 return folio; 1254 } 1255 1256 /* 1257 * A new dentry page is allocated but not able to be written, since its 1258 * new inode page couldn't be allocated due to -ENOSPC. 1259 * In such the case, its blkaddr can be remained as NEW_ADDR. 1260 * see, f2fs_add_link -> f2fs_get_new_data_page -> 1261 * f2fs_init_inode_metadata. 1262 */ 1263 if (dn.data_blkaddr == NEW_ADDR) { 1264 folio_zero_segment(folio, 0, folio_size(folio)); 1265 if (!folio_test_uptodate(folio)) 1266 folio_mark_uptodate(folio); 1267 folio_unlock(folio); 1268 return folio; 1269 } 1270 1271 err = f2fs_submit_page_read(inode, folio, dn.data_blkaddr, 1272 op_flags, for_write); 1273 if (err) 1274 goto put_err; 1275 return folio; 1276 1277 put_err: 1278 f2fs_folio_put(folio, true); 1279 return ERR_PTR(err); 1280 } 1281 1282 struct folio *f2fs_find_data_folio(struct inode *inode, pgoff_t index, 1283 pgoff_t *next_pgofs) 1284 { 1285 struct address_space *mapping = inode->i_mapping; 1286 struct folio *folio; 1287 1288 folio = __filemap_get_folio(mapping, index, FGP_ACCESSED, 0); 1289 if (IS_ERR(folio)) 1290 goto read; 1291 if (folio_test_uptodate(folio)) 1292 return folio; 1293 f2fs_folio_put(folio, false); 1294 1295 read: 1296 folio = f2fs_get_read_data_folio(inode, index, 0, false, next_pgofs); 1297 if (IS_ERR(folio)) 1298 return folio; 1299 1300 if (folio_test_uptodate(folio)) 1301 return folio; 1302 1303 folio_wait_locked(folio); 1304 if (unlikely(!folio_test_uptodate(folio))) { 1305 f2fs_folio_put(folio, false); 1306 return ERR_PTR(-EIO); 1307 } 1308 return folio; 1309 } 1310 1311 /* 1312 * If it tries to access a hole, return an error. 1313 * Because, the callers, functions in dir.c and GC, should be able to know 1314 * whether this page exists or not. 1315 */ 1316 struct folio *f2fs_get_lock_data_folio(struct inode *inode, pgoff_t index, 1317 bool for_write) 1318 { 1319 struct address_space *mapping = inode->i_mapping; 1320 struct folio *folio; 1321 1322 folio = f2fs_get_read_data_folio(inode, index, 0, for_write, NULL); 1323 if (IS_ERR(folio)) 1324 return folio; 1325 1326 /* wait for read completion */ 1327 folio_lock(folio); 1328 if (unlikely(folio->mapping != mapping || !folio_test_uptodate(folio))) { 1329 f2fs_folio_put(folio, true); 1330 return ERR_PTR(-EIO); 1331 } 1332 return folio; 1333 } 1334 1335 /* 1336 * Caller ensures that this data page is never allocated. 1337 * A new zero-filled data page is allocated in the page cache. 1338 * 1339 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and 1340 * f2fs_unlock_op(). 1341 * Note that, ipage is set only by make_empty_dir, and if any error occur, 1342 * ipage should be released by this function. 1343 */ 1344 struct page *f2fs_get_new_data_page(struct inode *inode, 1345 struct page *ipage, pgoff_t index, bool new_i_size) 1346 { 1347 struct address_space *mapping = inode->i_mapping; 1348 struct page *page; 1349 struct dnode_of_data dn; 1350 int err; 1351 1352 page = f2fs_grab_cache_page(mapping, index, true); 1353 if (!page) { 1354 /* 1355 * before exiting, we should make sure ipage will be released 1356 * if any error occur. 1357 */ 1358 f2fs_put_page(ipage, 1); 1359 return ERR_PTR(-ENOMEM); 1360 } 1361 1362 set_new_dnode(&dn, inode, ipage, NULL, 0); 1363 err = f2fs_reserve_block(&dn, index); 1364 if (err) { 1365 f2fs_put_page(page, 1); 1366 return ERR_PTR(err); 1367 } 1368 if (!ipage) 1369 f2fs_put_dnode(&dn); 1370 1371 if (PageUptodate(page)) 1372 goto got_it; 1373 1374 if (dn.data_blkaddr == NEW_ADDR) { 1375 zero_user_segment(page, 0, PAGE_SIZE); 1376 if (!PageUptodate(page)) 1377 SetPageUptodate(page); 1378 } else { 1379 f2fs_put_page(page, 1); 1380 1381 /* if ipage exists, blkaddr should be NEW_ADDR */ 1382 f2fs_bug_on(F2FS_I_SB(inode), ipage); 1383 page = f2fs_get_lock_data_page(inode, index, true); 1384 if (IS_ERR(page)) 1385 return page; 1386 } 1387 got_it: 1388 if (new_i_size && i_size_read(inode) < 1389 ((loff_t)(index + 1) << PAGE_SHIFT)) 1390 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT)); 1391 return page; 1392 } 1393 1394 static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) 1395 { 1396 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1397 struct f2fs_summary sum; 1398 struct node_info ni; 1399 block_t old_blkaddr; 1400 blkcnt_t count = 1; 1401 int err; 1402 1403 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) 1404 return -EPERM; 1405 1406 err = f2fs_get_node_info(sbi, dn->nid, &ni, false); 1407 if (err) 1408 return err; 1409 1410 dn->data_blkaddr = f2fs_data_blkaddr(dn); 1411 if (dn->data_blkaddr == NULL_ADDR) { 1412 err = inc_valid_block_count(sbi, dn->inode, &count, true); 1413 if (unlikely(err)) 1414 return err; 1415 } 1416 1417 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 1418 old_blkaddr = dn->data_blkaddr; 1419 err = f2fs_allocate_data_block(sbi, NULL, old_blkaddr, 1420 &dn->data_blkaddr, &sum, seg_type, NULL); 1421 if (err) 1422 return err; 1423 1424 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 1425 f2fs_invalidate_internal_cache(sbi, old_blkaddr, 1); 1426 1427 f2fs_update_data_blkaddr(dn, dn->data_blkaddr); 1428 return 0; 1429 } 1430 1431 static void f2fs_map_lock(struct f2fs_sb_info *sbi, int flag) 1432 { 1433 if (flag == F2FS_GET_BLOCK_PRE_AIO) 1434 f2fs_down_read(&sbi->node_change); 1435 else 1436 f2fs_lock_op(sbi); 1437 } 1438 1439 static void f2fs_map_unlock(struct f2fs_sb_info *sbi, int flag) 1440 { 1441 if (flag == F2FS_GET_BLOCK_PRE_AIO) 1442 f2fs_up_read(&sbi->node_change); 1443 else 1444 f2fs_unlock_op(sbi); 1445 } 1446 1447 int f2fs_get_block_locked(struct dnode_of_data *dn, pgoff_t index) 1448 { 1449 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1450 int err = 0; 1451 1452 f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); 1453 if (!f2fs_lookup_read_extent_cache_block(dn->inode, index, 1454 &dn->data_blkaddr)) 1455 err = f2fs_reserve_block(dn, index); 1456 f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO); 1457 1458 return err; 1459 } 1460 1461 static int f2fs_map_no_dnode(struct inode *inode, 1462 struct f2fs_map_blocks *map, struct dnode_of_data *dn, 1463 pgoff_t pgoff) 1464 { 1465 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1466 1467 /* 1468 * There is one exceptional case that read_node_page() may return 1469 * -ENOENT due to filesystem has been shutdown or cp_error, return 1470 * -EIO in that case. 1471 */ 1472 if (map->m_may_create && 1473 (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || f2fs_cp_error(sbi))) 1474 return -EIO; 1475 1476 if (map->m_next_pgofs) 1477 *map->m_next_pgofs = f2fs_get_next_page_offset(dn, pgoff); 1478 if (map->m_next_extent) 1479 *map->m_next_extent = f2fs_get_next_page_offset(dn, pgoff); 1480 return 0; 1481 } 1482 1483 static bool f2fs_map_blocks_cached(struct inode *inode, 1484 struct f2fs_map_blocks *map, int flag) 1485 { 1486 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1487 unsigned int maxblocks = map->m_len; 1488 pgoff_t pgoff = (pgoff_t)map->m_lblk; 1489 struct extent_info ei = {}; 1490 1491 if (!f2fs_lookup_read_extent_cache(inode, pgoff, &ei)) 1492 return false; 1493 1494 map->m_pblk = ei.blk + pgoff - ei.fofs; 1495 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgoff); 1496 map->m_flags = F2FS_MAP_MAPPED; 1497 if (map->m_next_extent) 1498 *map->m_next_extent = pgoff + map->m_len; 1499 1500 /* for hardware encryption, but to avoid potential issue in future */ 1501 if (flag == F2FS_GET_BLOCK_DIO) 1502 f2fs_wait_on_block_writeback_range(inode, 1503 map->m_pblk, map->m_len); 1504 1505 if (f2fs_allow_multi_device_dio(sbi, flag)) { 1506 int bidx = f2fs_target_device_index(sbi, map->m_pblk); 1507 struct f2fs_dev_info *dev = &sbi->devs[bidx]; 1508 1509 map->m_bdev = dev->bdev; 1510 map->m_pblk -= dev->start_blk; 1511 map->m_len = min(map->m_len, dev->end_blk + 1 - map->m_pblk); 1512 } else { 1513 map->m_bdev = inode->i_sb->s_bdev; 1514 } 1515 return true; 1516 } 1517 1518 static bool map_is_mergeable(struct f2fs_sb_info *sbi, 1519 struct f2fs_map_blocks *map, 1520 block_t blkaddr, int flag, int bidx, 1521 int ofs) 1522 { 1523 if (map->m_multidev_dio && map->m_bdev != FDEV(bidx).bdev) 1524 return false; 1525 if (map->m_pblk != NEW_ADDR && blkaddr == (map->m_pblk + ofs)) 1526 return true; 1527 if (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) 1528 return true; 1529 if (flag == F2FS_GET_BLOCK_PRE_DIO) 1530 return true; 1531 if (flag == F2FS_GET_BLOCK_DIO && 1532 map->m_pblk == NULL_ADDR && blkaddr == NULL_ADDR) 1533 return true; 1534 return false; 1535 } 1536 1537 /* 1538 * f2fs_map_blocks() tries to find or build mapping relationship which 1539 * maps continuous logical blocks to physical blocks, and return such 1540 * info via f2fs_map_blocks structure. 1541 */ 1542 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, int flag) 1543 { 1544 unsigned int maxblocks = map->m_len; 1545 struct dnode_of_data dn; 1546 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1547 int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE; 1548 pgoff_t pgofs, end_offset, end; 1549 int err = 0, ofs = 1; 1550 unsigned int ofs_in_node, last_ofs_in_node; 1551 blkcnt_t prealloc; 1552 block_t blkaddr; 1553 unsigned int start_pgofs; 1554 int bidx = 0; 1555 bool is_hole; 1556 1557 if (!maxblocks) 1558 return 0; 1559 1560 if (!map->m_may_create && f2fs_map_blocks_cached(inode, map, flag)) 1561 goto out; 1562 1563 map->m_bdev = inode->i_sb->s_bdev; 1564 map->m_multidev_dio = 1565 f2fs_allow_multi_device_dio(F2FS_I_SB(inode), flag); 1566 1567 map->m_len = 0; 1568 map->m_flags = 0; 1569 1570 /* it only supports block size == page size */ 1571 pgofs = (pgoff_t)map->m_lblk; 1572 end = pgofs + maxblocks; 1573 1574 next_dnode: 1575 if (map->m_may_create) 1576 f2fs_map_lock(sbi, flag); 1577 1578 /* When reading holes, we need its node page */ 1579 set_new_dnode(&dn, inode, NULL, NULL, 0); 1580 err = f2fs_get_dnode_of_data(&dn, pgofs, mode); 1581 if (err) { 1582 if (flag == F2FS_GET_BLOCK_BMAP) 1583 map->m_pblk = 0; 1584 if (err == -ENOENT) 1585 err = f2fs_map_no_dnode(inode, map, &dn, pgofs); 1586 goto unlock_out; 1587 } 1588 1589 start_pgofs = pgofs; 1590 prealloc = 0; 1591 last_ofs_in_node = ofs_in_node = dn.ofs_in_node; 1592 end_offset = ADDRS_PER_PAGE(dn.node_page, inode); 1593 1594 next_block: 1595 blkaddr = f2fs_data_blkaddr(&dn); 1596 is_hole = !__is_valid_data_blkaddr(blkaddr); 1597 if (!is_hole && 1598 !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) { 1599 err = -EFSCORRUPTED; 1600 goto sync_out; 1601 } 1602 1603 /* use out-place-update for direct IO under LFS mode */ 1604 if (map->m_may_create && (is_hole || 1605 (flag == F2FS_GET_BLOCK_DIO && f2fs_lfs_mode(sbi) && 1606 !f2fs_is_pinned_file(inode)))) { 1607 if (unlikely(f2fs_cp_error(sbi))) { 1608 err = -EIO; 1609 goto sync_out; 1610 } 1611 1612 switch (flag) { 1613 case F2FS_GET_BLOCK_PRE_AIO: 1614 if (blkaddr == NULL_ADDR) { 1615 prealloc++; 1616 last_ofs_in_node = dn.ofs_in_node; 1617 } 1618 break; 1619 case F2FS_GET_BLOCK_PRE_DIO: 1620 case F2FS_GET_BLOCK_DIO: 1621 err = __allocate_data_block(&dn, map->m_seg_type); 1622 if (err) 1623 goto sync_out; 1624 if (flag == F2FS_GET_BLOCK_PRE_DIO) 1625 file_need_truncate(inode); 1626 set_inode_flag(inode, FI_APPEND_WRITE); 1627 break; 1628 default: 1629 WARN_ON_ONCE(1); 1630 err = -EIO; 1631 goto sync_out; 1632 } 1633 1634 blkaddr = dn.data_blkaddr; 1635 if (is_hole) 1636 map->m_flags |= F2FS_MAP_NEW; 1637 } else if (is_hole) { 1638 if (f2fs_compressed_file(inode) && 1639 f2fs_sanity_check_cluster(&dn)) { 1640 err = -EFSCORRUPTED; 1641 f2fs_handle_error(sbi, 1642 ERROR_CORRUPTED_CLUSTER); 1643 goto sync_out; 1644 } 1645 1646 switch (flag) { 1647 case F2FS_GET_BLOCK_PRECACHE: 1648 goto sync_out; 1649 case F2FS_GET_BLOCK_BMAP: 1650 map->m_pblk = 0; 1651 goto sync_out; 1652 case F2FS_GET_BLOCK_FIEMAP: 1653 if (blkaddr == NULL_ADDR) { 1654 if (map->m_next_pgofs) 1655 *map->m_next_pgofs = pgofs + 1; 1656 goto sync_out; 1657 } 1658 break; 1659 case F2FS_GET_BLOCK_DIO: 1660 if (map->m_next_pgofs) 1661 *map->m_next_pgofs = pgofs + 1; 1662 break; 1663 default: 1664 /* for defragment case */ 1665 if (map->m_next_pgofs) 1666 *map->m_next_pgofs = pgofs + 1; 1667 goto sync_out; 1668 } 1669 } 1670 1671 if (flag == F2FS_GET_BLOCK_PRE_AIO) 1672 goto skip; 1673 1674 if (map->m_multidev_dio) 1675 bidx = f2fs_target_device_index(sbi, blkaddr); 1676 1677 if (map->m_len == 0) { 1678 /* reserved delalloc block should be mapped for fiemap. */ 1679 if (blkaddr == NEW_ADDR) 1680 map->m_flags |= F2FS_MAP_DELALLOC; 1681 /* DIO READ and hole case, should not map the blocks. */ 1682 if (!(flag == F2FS_GET_BLOCK_DIO && is_hole && !map->m_may_create)) 1683 map->m_flags |= F2FS_MAP_MAPPED; 1684 1685 map->m_pblk = blkaddr; 1686 map->m_len = 1; 1687 1688 if (map->m_multidev_dio) 1689 map->m_bdev = FDEV(bidx).bdev; 1690 } else if (map_is_mergeable(sbi, map, blkaddr, flag, bidx, ofs)) { 1691 ofs++; 1692 map->m_len++; 1693 } else { 1694 goto sync_out; 1695 } 1696 1697 skip: 1698 dn.ofs_in_node++; 1699 pgofs++; 1700 1701 /* preallocate blocks in batch for one dnode page */ 1702 if (flag == F2FS_GET_BLOCK_PRE_AIO && 1703 (pgofs == end || dn.ofs_in_node == end_offset)) { 1704 1705 dn.ofs_in_node = ofs_in_node; 1706 err = f2fs_reserve_new_blocks(&dn, prealloc); 1707 if (err) 1708 goto sync_out; 1709 1710 map->m_len += dn.ofs_in_node - ofs_in_node; 1711 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) { 1712 err = -ENOSPC; 1713 goto sync_out; 1714 } 1715 dn.ofs_in_node = end_offset; 1716 } 1717 1718 if (flag == F2FS_GET_BLOCK_DIO && f2fs_lfs_mode(sbi) && 1719 map->m_may_create) { 1720 /* the next block to be allocated may not be contiguous. */ 1721 if (GET_SEGOFF_FROM_SEG0(sbi, blkaddr) % BLKS_PER_SEC(sbi) == 1722 CAP_BLKS_PER_SEC(sbi) - 1) 1723 goto sync_out; 1724 } 1725 1726 if (pgofs >= end) 1727 goto sync_out; 1728 else if (dn.ofs_in_node < end_offset) 1729 goto next_block; 1730 1731 if (flag == F2FS_GET_BLOCK_PRECACHE) { 1732 if (map->m_flags & F2FS_MAP_MAPPED) { 1733 unsigned int ofs = start_pgofs - map->m_lblk; 1734 1735 f2fs_update_read_extent_cache_range(&dn, 1736 start_pgofs, map->m_pblk + ofs, 1737 map->m_len - ofs); 1738 } 1739 } 1740 1741 f2fs_put_dnode(&dn); 1742 1743 if (map->m_may_create) { 1744 f2fs_map_unlock(sbi, flag); 1745 f2fs_balance_fs(sbi, dn.node_changed); 1746 } 1747 goto next_dnode; 1748 1749 sync_out: 1750 1751 if (flag == F2FS_GET_BLOCK_DIO && map->m_flags & F2FS_MAP_MAPPED) { 1752 /* 1753 * for hardware encryption, but to avoid potential issue 1754 * in future 1755 */ 1756 f2fs_wait_on_block_writeback_range(inode, 1757 map->m_pblk, map->m_len); 1758 1759 if (map->m_multidev_dio) { 1760 block_t blk_addr = map->m_pblk; 1761 1762 bidx = f2fs_target_device_index(sbi, map->m_pblk); 1763 1764 map->m_bdev = FDEV(bidx).bdev; 1765 map->m_pblk -= FDEV(bidx).start_blk; 1766 1767 if (map->m_may_create) 1768 f2fs_update_device_state(sbi, inode->i_ino, 1769 blk_addr, map->m_len); 1770 1771 f2fs_bug_on(sbi, blk_addr + map->m_len > 1772 FDEV(bidx).end_blk + 1); 1773 } 1774 } 1775 1776 if (flag == F2FS_GET_BLOCK_PRECACHE) { 1777 if (map->m_flags & F2FS_MAP_MAPPED) { 1778 unsigned int ofs = start_pgofs - map->m_lblk; 1779 1780 f2fs_update_read_extent_cache_range(&dn, 1781 start_pgofs, map->m_pblk + ofs, 1782 map->m_len - ofs); 1783 } 1784 if (map->m_next_extent) 1785 *map->m_next_extent = pgofs + 1; 1786 } 1787 f2fs_put_dnode(&dn); 1788 unlock_out: 1789 if (map->m_may_create) { 1790 f2fs_map_unlock(sbi, flag); 1791 f2fs_balance_fs(sbi, dn.node_changed); 1792 } 1793 out: 1794 trace_f2fs_map_blocks(inode, map, flag, err); 1795 return err; 1796 } 1797 1798 bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) 1799 { 1800 struct f2fs_map_blocks map; 1801 block_t last_lblk; 1802 int err; 1803 1804 if (pos + len > i_size_read(inode)) 1805 return false; 1806 1807 map.m_lblk = F2FS_BYTES_TO_BLK(pos); 1808 map.m_next_pgofs = NULL; 1809 map.m_next_extent = NULL; 1810 map.m_seg_type = NO_CHECK_TYPE; 1811 map.m_may_create = false; 1812 last_lblk = F2FS_BLK_ALIGN(pos + len); 1813 1814 while (map.m_lblk < last_lblk) { 1815 map.m_len = last_lblk - map.m_lblk; 1816 err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DEFAULT); 1817 if (err || map.m_len == 0) 1818 return false; 1819 map.m_lblk += map.m_len; 1820 } 1821 return true; 1822 } 1823 1824 static int f2fs_xattr_fiemap(struct inode *inode, 1825 struct fiemap_extent_info *fieinfo) 1826 { 1827 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1828 struct page *page; 1829 struct node_info ni; 1830 __u64 phys = 0, len; 1831 __u32 flags; 1832 nid_t xnid = F2FS_I(inode)->i_xattr_nid; 1833 int err = 0; 1834 1835 if (f2fs_has_inline_xattr(inode)) { 1836 int offset; 1837 1838 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), 1839 inode->i_ino, false); 1840 if (!page) 1841 return -ENOMEM; 1842 1843 err = f2fs_get_node_info(sbi, inode->i_ino, &ni, false); 1844 if (err) { 1845 f2fs_put_page(page, 1); 1846 return err; 1847 } 1848 1849 phys = F2FS_BLK_TO_BYTES(ni.blk_addr); 1850 offset = offsetof(struct f2fs_inode, i_addr) + 1851 sizeof(__le32) * (DEF_ADDRS_PER_INODE - 1852 get_inline_xattr_addrs(inode)); 1853 1854 phys += offset; 1855 len = inline_xattr_size(inode); 1856 1857 f2fs_put_page(page, 1); 1858 1859 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED; 1860 1861 if (!xnid) 1862 flags |= FIEMAP_EXTENT_LAST; 1863 1864 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); 1865 trace_f2fs_fiemap(inode, 0, phys, len, flags, err); 1866 if (err) 1867 return err; 1868 } 1869 1870 if (xnid) { 1871 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false); 1872 if (!page) 1873 return -ENOMEM; 1874 1875 err = f2fs_get_node_info(sbi, xnid, &ni, false); 1876 if (err) { 1877 f2fs_put_page(page, 1); 1878 return err; 1879 } 1880 1881 phys = F2FS_BLK_TO_BYTES(ni.blk_addr); 1882 len = inode->i_sb->s_blocksize; 1883 1884 f2fs_put_page(page, 1); 1885 1886 flags = FIEMAP_EXTENT_LAST; 1887 } 1888 1889 if (phys) { 1890 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); 1891 trace_f2fs_fiemap(inode, 0, phys, len, flags, err); 1892 } 1893 1894 return (err < 0 ? err : 0); 1895 } 1896 1897 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 1898 u64 start, u64 len) 1899 { 1900 struct f2fs_map_blocks map; 1901 sector_t start_blk, last_blk, blk_len, max_len; 1902 pgoff_t next_pgofs; 1903 u64 logical = 0, phys = 0, size = 0; 1904 u32 flags = 0; 1905 int ret = 0; 1906 bool compr_cluster = false, compr_appended; 1907 unsigned int cluster_size = F2FS_I(inode)->i_cluster_size; 1908 unsigned int count_in_cluster = 0; 1909 loff_t maxbytes; 1910 1911 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) { 1912 ret = f2fs_precache_extents(inode); 1913 if (ret) 1914 return ret; 1915 } 1916 1917 ret = fiemap_prep(inode, fieinfo, start, &len, FIEMAP_FLAG_XATTR); 1918 if (ret) 1919 return ret; 1920 1921 inode_lock_shared(inode); 1922 1923 maxbytes = F2FS_BLK_TO_BYTES(max_file_blocks(inode)); 1924 if (start > maxbytes) { 1925 ret = -EFBIG; 1926 goto out; 1927 } 1928 1929 if (len > maxbytes || (maxbytes - len) < start) 1930 len = maxbytes - start; 1931 1932 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { 1933 ret = f2fs_xattr_fiemap(inode, fieinfo); 1934 goto out; 1935 } 1936 1937 if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) { 1938 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len); 1939 if (ret != -EAGAIN) 1940 goto out; 1941 } 1942 1943 start_blk = F2FS_BYTES_TO_BLK(start); 1944 last_blk = F2FS_BYTES_TO_BLK(start + len - 1); 1945 blk_len = last_blk - start_blk + 1; 1946 max_len = F2FS_BYTES_TO_BLK(maxbytes) - start_blk; 1947 1948 next: 1949 memset(&map, 0, sizeof(map)); 1950 map.m_lblk = start_blk; 1951 map.m_len = blk_len; 1952 map.m_next_pgofs = &next_pgofs; 1953 map.m_seg_type = NO_CHECK_TYPE; 1954 1955 if (compr_cluster) { 1956 map.m_lblk += 1; 1957 map.m_len = cluster_size - count_in_cluster; 1958 } 1959 1960 ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_FIEMAP); 1961 if (ret) 1962 goto out; 1963 1964 /* HOLE */ 1965 if (!compr_cluster && !(map.m_flags & F2FS_MAP_FLAGS)) { 1966 start_blk = next_pgofs; 1967 1968 if (F2FS_BLK_TO_BYTES(start_blk) < maxbytes) 1969 goto prep_next; 1970 1971 flags |= FIEMAP_EXTENT_LAST; 1972 } 1973 1974 /* 1975 * current extent may cross boundary of inquiry, increase len to 1976 * requery. 1977 */ 1978 if (!compr_cluster && (map.m_flags & F2FS_MAP_MAPPED) && 1979 map.m_lblk + map.m_len - 1 == last_blk && 1980 blk_len != max_len) { 1981 blk_len = max_len; 1982 goto next; 1983 } 1984 1985 compr_appended = false; 1986 /* In a case of compressed cluster, append this to the last extent */ 1987 if (compr_cluster && ((map.m_flags & F2FS_MAP_DELALLOC) || 1988 !(map.m_flags & F2FS_MAP_FLAGS))) { 1989 compr_appended = true; 1990 goto skip_fill; 1991 } 1992 1993 if (size) { 1994 flags |= FIEMAP_EXTENT_MERGED; 1995 if (IS_ENCRYPTED(inode)) 1996 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED; 1997 1998 ret = fiemap_fill_next_extent(fieinfo, logical, 1999 phys, size, flags); 2000 trace_f2fs_fiemap(inode, logical, phys, size, flags, ret); 2001 if (ret) 2002 goto out; 2003 size = 0; 2004 } 2005 2006 if (start_blk > last_blk) 2007 goto out; 2008 2009 skip_fill: 2010 if (map.m_pblk == COMPRESS_ADDR) { 2011 compr_cluster = true; 2012 count_in_cluster = 1; 2013 } else if (compr_appended) { 2014 unsigned int appended_blks = cluster_size - 2015 count_in_cluster + 1; 2016 size += F2FS_BLK_TO_BYTES(appended_blks); 2017 start_blk += appended_blks; 2018 compr_cluster = false; 2019 } else { 2020 logical = F2FS_BLK_TO_BYTES(start_blk); 2021 phys = __is_valid_data_blkaddr(map.m_pblk) ? 2022 F2FS_BLK_TO_BYTES(map.m_pblk) : 0; 2023 size = F2FS_BLK_TO_BYTES(map.m_len); 2024 flags = 0; 2025 2026 if (compr_cluster) { 2027 flags = FIEMAP_EXTENT_ENCODED; 2028 count_in_cluster += map.m_len; 2029 if (count_in_cluster == cluster_size) { 2030 compr_cluster = false; 2031 size += F2FS_BLKSIZE; 2032 } 2033 } else if (map.m_flags & F2FS_MAP_DELALLOC) { 2034 flags = FIEMAP_EXTENT_UNWRITTEN; 2035 } 2036 2037 start_blk += F2FS_BYTES_TO_BLK(size); 2038 } 2039 2040 prep_next: 2041 cond_resched(); 2042 if (fatal_signal_pending(current)) 2043 ret = -EINTR; 2044 else 2045 goto next; 2046 out: 2047 if (ret == 1) 2048 ret = 0; 2049 2050 inode_unlock_shared(inode); 2051 return ret; 2052 } 2053 2054 static inline loff_t f2fs_readpage_limit(struct inode *inode) 2055 { 2056 if (IS_ENABLED(CONFIG_FS_VERITY) && IS_VERITY(inode)) 2057 return F2FS_BLK_TO_BYTES(max_file_blocks(inode)); 2058 2059 return i_size_read(inode); 2060 } 2061 2062 static inline blk_opf_t f2fs_ra_op_flags(struct readahead_control *rac) 2063 { 2064 return rac ? REQ_RAHEAD : 0; 2065 } 2066 2067 static int f2fs_read_single_page(struct inode *inode, struct folio *folio, 2068 unsigned nr_pages, 2069 struct f2fs_map_blocks *map, 2070 struct bio **bio_ret, 2071 sector_t *last_block_in_bio, 2072 struct readahead_control *rac) 2073 { 2074 struct bio *bio = *bio_ret; 2075 const unsigned int blocksize = F2FS_BLKSIZE; 2076 sector_t block_in_file; 2077 sector_t last_block; 2078 sector_t last_block_in_file; 2079 sector_t block_nr; 2080 pgoff_t index = folio_index(folio); 2081 int ret = 0; 2082 2083 block_in_file = (sector_t)index; 2084 last_block = block_in_file + nr_pages; 2085 last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) + 2086 blocksize - 1); 2087 if (last_block > last_block_in_file) 2088 last_block = last_block_in_file; 2089 2090 /* just zeroing out page which is beyond EOF */ 2091 if (block_in_file >= last_block) 2092 goto zero_out; 2093 /* 2094 * Map blocks using the previous result first. 2095 */ 2096 if ((map->m_flags & F2FS_MAP_MAPPED) && 2097 block_in_file > map->m_lblk && 2098 block_in_file < (map->m_lblk + map->m_len)) 2099 goto got_it; 2100 2101 /* 2102 * Then do more f2fs_map_blocks() calls until we are 2103 * done with this page. 2104 */ 2105 map->m_lblk = block_in_file; 2106 map->m_len = last_block - block_in_file; 2107 2108 ret = f2fs_map_blocks(inode, map, F2FS_GET_BLOCK_DEFAULT); 2109 if (ret) 2110 goto out; 2111 got_it: 2112 if ((map->m_flags & F2FS_MAP_MAPPED)) { 2113 block_nr = map->m_pblk + block_in_file - map->m_lblk; 2114 folio_set_mappedtodisk(folio); 2115 2116 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr, 2117 DATA_GENERIC_ENHANCE_READ)) { 2118 ret = -EFSCORRUPTED; 2119 goto out; 2120 } 2121 } else { 2122 zero_out: 2123 folio_zero_segment(folio, 0, folio_size(folio)); 2124 if (f2fs_need_verity(inode, index) && 2125 !fsverity_verify_folio(folio)) { 2126 ret = -EIO; 2127 goto out; 2128 } 2129 if (!folio_test_uptodate(folio)) 2130 folio_mark_uptodate(folio); 2131 folio_unlock(folio); 2132 goto out; 2133 } 2134 2135 /* 2136 * This page will go to BIO. Do we need to send this 2137 * BIO off first? 2138 */ 2139 if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio, 2140 *last_block_in_bio, block_nr) || 2141 !f2fs_crypt_mergeable_bio(bio, inode, index, NULL))) { 2142 submit_and_realloc: 2143 f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); 2144 bio = NULL; 2145 } 2146 if (bio == NULL) { 2147 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages, 2148 f2fs_ra_op_flags(rac), index, 2149 false); 2150 if (IS_ERR(bio)) { 2151 ret = PTR_ERR(bio); 2152 bio = NULL; 2153 goto out; 2154 } 2155 } 2156 2157 /* 2158 * If the page is under writeback, we need to wait for 2159 * its completion to see the correct decrypted data. 2160 */ 2161 f2fs_wait_on_block_writeback(inode, block_nr); 2162 2163 if (!bio_add_folio(bio, folio, blocksize, 0)) 2164 goto submit_and_realloc; 2165 2166 inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA); 2167 f2fs_update_iostat(F2FS_I_SB(inode), NULL, FS_DATA_READ_IO, 2168 F2FS_BLKSIZE); 2169 *last_block_in_bio = block_nr; 2170 out: 2171 *bio_ret = bio; 2172 return ret; 2173 } 2174 2175 #ifdef CONFIG_F2FS_FS_COMPRESSION 2176 int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, 2177 unsigned nr_pages, sector_t *last_block_in_bio, 2178 struct readahead_control *rac, bool for_write) 2179 { 2180 struct dnode_of_data dn; 2181 struct inode *inode = cc->inode; 2182 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2183 struct bio *bio = *bio_ret; 2184 unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size; 2185 sector_t last_block_in_file; 2186 const unsigned int blocksize = F2FS_BLKSIZE; 2187 struct decompress_io_ctx *dic = NULL; 2188 struct extent_info ei = {}; 2189 bool from_dnode = true; 2190 int i; 2191 int ret = 0; 2192 2193 if (unlikely(f2fs_cp_error(sbi))) { 2194 ret = -EIO; 2195 from_dnode = false; 2196 goto out_put_dnode; 2197 } 2198 2199 f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc)); 2200 2201 last_block_in_file = F2FS_BYTES_TO_BLK(f2fs_readpage_limit(inode) + 2202 blocksize - 1); 2203 2204 /* get rid of pages beyond EOF */ 2205 for (i = 0; i < cc->cluster_size; i++) { 2206 struct page *page = cc->rpages[i]; 2207 struct folio *folio; 2208 2209 if (!page) 2210 continue; 2211 2212 folio = page_folio(page); 2213 if ((sector_t)folio->index >= last_block_in_file) { 2214 folio_zero_segment(folio, 0, folio_size(folio)); 2215 if (!folio_test_uptodate(folio)) 2216 folio_mark_uptodate(folio); 2217 } else if (!folio_test_uptodate(folio)) { 2218 continue; 2219 } 2220 folio_unlock(folio); 2221 if (for_write) 2222 folio_put(folio); 2223 cc->rpages[i] = NULL; 2224 cc->nr_rpages--; 2225 } 2226 2227 /* we are done since all pages are beyond EOF */ 2228 if (f2fs_cluster_is_empty(cc)) 2229 goto out; 2230 2231 if (f2fs_lookup_read_extent_cache(inode, start_idx, &ei)) 2232 from_dnode = false; 2233 2234 if (!from_dnode) 2235 goto skip_reading_dnode; 2236 2237 set_new_dnode(&dn, inode, NULL, NULL, 0); 2238 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 2239 if (ret) 2240 goto out; 2241 2242 f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR); 2243 2244 skip_reading_dnode: 2245 for (i = 1; i < cc->cluster_size; i++) { 2246 block_t blkaddr; 2247 2248 blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_page, 2249 dn.ofs_in_node + i) : 2250 ei.blk + i - 1; 2251 2252 if (!__is_valid_data_blkaddr(blkaddr)) 2253 break; 2254 2255 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) { 2256 ret = -EFAULT; 2257 goto out_put_dnode; 2258 } 2259 cc->nr_cpages++; 2260 2261 if (!from_dnode && i >= ei.c_len) 2262 break; 2263 } 2264 2265 /* nothing to decompress */ 2266 if (cc->nr_cpages == 0) { 2267 ret = 0; 2268 goto out_put_dnode; 2269 } 2270 2271 dic = f2fs_alloc_dic(cc); 2272 if (IS_ERR(dic)) { 2273 ret = PTR_ERR(dic); 2274 goto out_put_dnode; 2275 } 2276 2277 for (i = 0; i < cc->nr_cpages; i++) { 2278 struct folio *folio = page_folio(dic->cpages[i]); 2279 block_t blkaddr; 2280 struct bio_post_read_ctx *ctx; 2281 2282 blkaddr = from_dnode ? data_blkaddr(dn.inode, dn.node_page, 2283 dn.ofs_in_node + i + 1) : 2284 ei.blk + i; 2285 2286 f2fs_wait_on_block_writeback(inode, blkaddr); 2287 2288 if (f2fs_load_compressed_page(sbi, folio_page(folio, 0), 2289 blkaddr)) { 2290 if (atomic_dec_and_test(&dic->remaining_pages)) { 2291 f2fs_decompress_cluster(dic, true); 2292 break; 2293 } 2294 continue; 2295 } 2296 2297 if (bio && (!page_is_mergeable(sbi, bio, 2298 *last_block_in_bio, blkaddr) || 2299 !f2fs_crypt_mergeable_bio(bio, inode, folio->index, NULL))) { 2300 submit_and_realloc: 2301 f2fs_submit_read_bio(sbi, bio, DATA); 2302 bio = NULL; 2303 } 2304 2305 if (!bio) { 2306 bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages, 2307 f2fs_ra_op_flags(rac), 2308 folio->index, for_write); 2309 if (IS_ERR(bio)) { 2310 ret = PTR_ERR(bio); 2311 f2fs_decompress_end_io(dic, ret, true); 2312 f2fs_put_dnode(&dn); 2313 *bio_ret = NULL; 2314 return ret; 2315 } 2316 } 2317 2318 if (!bio_add_folio(bio, folio, blocksize, 0)) 2319 goto submit_and_realloc; 2320 2321 ctx = get_post_read_ctx(bio); 2322 ctx->enabled_steps |= STEP_DECOMPRESS; 2323 refcount_inc(&dic->refcnt); 2324 2325 inc_page_count(sbi, F2FS_RD_DATA); 2326 f2fs_update_iostat(sbi, inode, FS_DATA_READ_IO, F2FS_BLKSIZE); 2327 *last_block_in_bio = blkaddr; 2328 } 2329 2330 if (from_dnode) 2331 f2fs_put_dnode(&dn); 2332 2333 *bio_ret = bio; 2334 return 0; 2335 2336 out_put_dnode: 2337 if (from_dnode) 2338 f2fs_put_dnode(&dn); 2339 out: 2340 for (i = 0; i < cc->cluster_size; i++) { 2341 if (cc->rpages[i]) { 2342 ClearPageUptodate(cc->rpages[i]); 2343 unlock_page(cc->rpages[i]); 2344 } 2345 } 2346 *bio_ret = bio; 2347 return ret; 2348 } 2349 #endif 2350 2351 /* 2352 * This function was originally taken from fs/mpage.c, and customized for f2fs. 2353 * Major change was from block_size == page_size in f2fs by default. 2354 */ 2355 static int f2fs_mpage_readpages(struct inode *inode, 2356 struct readahead_control *rac, struct folio *folio) 2357 { 2358 struct bio *bio = NULL; 2359 sector_t last_block_in_bio = 0; 2360 struct f2fs_map_blocks map; 2361 #ifdef CONFIG_F2FS_FS_COMPRESSION 2362 struct compress_ctx cc = { 2363 .inode = inode, 2364 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 2365 .cluster_size = F2FS_I(inode)->i_cluster_size, 2366 .cluster_idx = NULL_CLUSTER, 2367 .rpages = NULL, 2368 .cpages = NULL, 2369 .nr_rpages = 0, 2370 .nr_cpages = 0, 2371 }; 2372 pgoff_t nc_cluster_idx = NULL_CLUSTER; 2373 pgoff_t index; 2374 #endif 2375 unsigned nr_pages = rac ? readahead_count(rac) : 1; 2376 unsigned max_nr_pages = nr_pages; 2377 int ret = 0; 2378 2379 map.m_pblk = 0; 2380 map.m_lblk = 0; 2381 map.m_len = 0; 2382 map.m_flags = 0; 2383 map.m_next_pgofs = NULL; 2384 map.m_next_extent = NULL; 2385 map.m_seg_type = NO_CHECK_TYPE; 2386 map.m_may_create = false; 2387 2388 for (; nr_pages; nr_pages--) { 2389 if (rac) { 2390 folio = readahead_folio(rac); 2391 prefetchw(&folio->flags); 2392 } 2393 2394 #ifdef CONFIG_F2FS_FS_COMPRESSION 2395 index = folio_index(folio); 2396 2397 if (!f2fs_compressed_file(inode)) 2398 goto read_single_page; 2399 2400 /* there are remained compressed pages, submit them */ 2401 if (!f2fs_cluster_can_merge_page(&cc, index)) { 2402 ret = f2fs_read_multi_pages(&cc, &bio, 2403 max_nr_pages, 2404 &last_block_in_bio, 2405 rac, false); 2406 f2fs_destroy_compress_ctx(&cc, false); 2407 if (ret) 2408 goto set_error_page; 2409 } 2410 if (cc.cluster_idx == NULL_CLUSTER) { 2411 if (nc_cluster_idx == index >> cc.log_cluster_size) 2412 goto read_single_page; 2413 2414 ret = f2fs_is_compressed_cluster(inode, index); 2415 if (ret < 0) 2416 goto set_error_page; 2417 else if (!ret) { 2418 nc_cluster_idx = 2419 index >> cc.log_cluster_size; 2420 goto read_single_page; 2421 } 2422 2423 nc_cluster_idx = NULL_CLUSTER; 2424 } 2425 ret = f2fs_init_compress_ctx(&cc); 2426 if (ret) 2427 goto set_error_page; 2428 2429 f2fs_compress_ctx_add_page(&cc, folio); 2430 2431 goto next_page; 2432 read_single_page: 2433 #endif 2434 2435 ret = f2fs_read_single_page(inode, folio, max_nr_pages, &map, 2436 &bio, &last_block_in_bio, rac); 2437 if (ret) { 2438 #ifdef CONFIG_F2FS_FS_COMPRESSION 2439 set_error_page: 2440 #endif 2441 folio_zero_segment(folio, 0, folio_size(folio)); 2442 folio_unlock(folio); 2443 } 2444 #ifdef CONFIG_F2FS_FS_COMPRESSION 2445 next_page: 2446 #endif 2447 2448 #ifdef CONFIG_F2FS_FS_COMPRESSION 2449 if (f2fs_compressed_file(inode)) { 2450 /* last page */ 2451 if (nr_pages == 1 && !f2fs_cluster_is_empty(&cc)) { 2452 ret = f2fs_read_multi_pages(&cc, &bio, 2453 max_nr_pages, 2454 &last_block_in_bio, 2455 rac, false); 2456 f2fs_destroy_compress_ctx(&cc, false); 2457 } 2458 } 2459 #endif 2460 } 2461 if (bio) 2462 f2fs_submit_read_bio(F2FS_I_SB(inode), bio, DATA); 2463 return ret; 2464 } 2465 2466 static int f2fs_read_data_folio(struct file *file, struct folio *folio) 2467 { 2468 struct inode *inode = folio->mapping->host; 2469 int ret = -EAGAIN; 2470 2471 trace_f2fs_readpage(folio, DATA); 2472 2473 if (!f2fs_is_compress_backend_ready(inode)) { 2474 folio_unlock(folio); 2475 return -EOPNOTSUPP; 2476 } 2477 2478 /* If the file has inline data, try to read it directly */ 2479 if (f2fs_has_inline_data(inode)) 2480 ret = f2fs_read_inline_data(inode, folio); 2481 if (ret == -EAGAIN) 2482 ret = f2fs_mpage_readpages(inode, NULL, folio); 2483 return ret; 2484 } 2485 2486 static void f2fs_readahead(struct readahead_control *rac) 2487 { 2488 struct inode *inode = rac->mapping->host; 2489 2490 trace_f2fs_readpages(inode, readahead_index(rac), readahead_count(rac)); 2491 2492 if (!f2fs_is_compress_backend_ready(inode)) 2493 return; 2494 2495 /* If the file has inline data, skip readahead */ 2496 if (f2fs_has_inline_data(inode)) 2497 return; 2498 2499 f2fs_mpage_readpages(inode, rac, NULL); 2500 } 2501 2502 int f2fs_encrypt_one_page(struct f2fs_io_info *fio) 2503 { 2504 struct inode *inode = fio->page->mapping->host; 2505 struct page *mpage, *page; 2506 gfp_t gfp_flags = GFP_NOFS; 2507 2508 if (!f2fs_encrypted_file(inode)) 2509 return 0; 2510 2511 page = fio->compressed_page ? fio->compressed_page : fio->page; 2512 2513 if (fscrypt_inode_uses_inline_crypto(inode)) 2514 return 0; 2515 2516 retry_encrypt: 2517 fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page_folio(page), 2518 PAGE_SIZE, 0, gfp_flags); 2519 if (IS_ERR(fio->encrypted_page)) { 2520 /* flush pending IOs and wait for a while in the ENOMEM case */ 2521 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) { 2522 f2fs_flush_merged_writes(fio->sbi); 2523 memalloc_retry_wait(GFP_NOFS); 2524 gfp_flags |= __GFP_NOFAIL; 2525 goto retry_encrypt; 2526 } 2527 return PTR_ERR(fio->encrypted_page); 2528 } 2529 2530 mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr); 2531 if (mpage) { 2532 if (PageUptodate(mpage)) 2533 memcpy(page_address(mpage), 2534 page_address(fio->encrypted_page), PAGE_SIZE); 2535 f2fs_put_page(mpage, 1); 2536 } 2537 return 0; 2538 } 2539 2540 static inline bool check_inplace_update_policy(struct inode *inode, 2541 struct f2fs_io_info *fio) 2542 { 2543 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2544 2545 if (IS_F2FS_IPU_HONOR_OPU_WRITE(sbi) && 2546 is_inode_flag_set(inode, FI_OPU_WRITE)) 2547 return false; 2548 if (IS_F2FS_IPU_FORCE(sbi)) 2549 return true; 2550 if (IS_F2FS_IPU_SSR(sbi) && f2fs_need_SSR(sbi)) 2551 return true; 2552 if (IS_F2FS_IPU_UTIL(sbi) && utilization(sbi) > SM_I(sbi)->min_ipu_util) 2553 return true; 2554 if (IS_F2FS_IPU_SSR_UTIL(sbi) && f2fs_need_SSR(sbi) && 2555 utilization(sbi) > SM_I(sbi)->min_ipu_util) 2556 return true; 2557 2558 /* 2559 * IPU for rewrite async pages 2560 */ 2561 if (IS_F2FS_IPU_ASYNC(sbi) && fio && fio->op == REQ_OP_WRITE && 2562 !(fio->op_flags & REQ_SYNC) && !IS_ENCRYPTED(inode)) 2563 return true; 2564 2565 /* this is only set during fdatasync */ 2566 if (IS_F2FS_IPU_FSYNC(sbi) && is_inode_flag_set(inode, FI_NEED_IPU)) 2567 return true; 2568 2569 if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) && 2570 !f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) 2571 return true; 2572 2573 return false; 2574 } 2575 2576 bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio) 2577 { 2578 /* swap file is migrating in aligned write mode */ 2579 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) 2580 return false; 2581 2582 if (f2fs_is_pinned_file(inode)) 2583 return true; 2584 2585 /* if this is cold file, we should overwrite to avoid fragmentation */ 2586 if (file_is_cold(inode) && !is_inode_flag_set(inode, FI_OPU_WRITE)) 2587 return true; 2588 2589 return check_inplace_update_policy(inode, fio); 2590 } 2591 2592 bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) 2593 { 2594 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2595 2596 /* The below cases were checked when setting it. */ 2597 if (f2fs_is_pinned_file(inode)) 2598 return false; 2599 if (fio && is_sbi_flag_set(sbi, SBI_NEED_FSCK)) 2600 return true; 2601 if (f2fs_lfs_mode(sbi)) 2602 return true; 2603 if (S_ISDIR(inode->i_mode)) 2604 return true; 2605 if (IS_NOQUOTA(inode)) 2606 return true; 2607 if (f2fs_used_in_atomic_write(inode)) 2608 return true; 2609 /* rewrite low ratio compress data w/ OPU mode to avoid fragmentation */ 2610 if (f2fs_compressed_file(inode) && 2611 F2FS_OPTION(sbi).compress_mode == COMPR_MODE_USER && 2612 is_inode_flag_set(inode, FI_ENABLE_COMPRESS)) 2613 return true; 2614 2615 /* swap file is migrating in aligned write mode */ 2616 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE)) 2617 return true; 2618 2619 if (is_inode_flag_set(inode, FI_OPU_WRITE)) 2620 return true; 2621 2622 if (fio) { 2623 if (page_private_gcing(fio->page)) 2624 return true; 2625 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) && 2626 f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) 2627 return true; 2628 } 2629 return false; 2630 } 2631 2632 static inline bool need_inplace_update(struct f2fs_io_info *fio) 2633 { 2634 struct inode *inode = fio->page->mapping->host; 2635 2636 if (f2fs_should_update_outplace(inode, fio)) 2637 return false; 2638 2639 return f2fs_should_update_inplace(inode, fio); 2640 } 2641 2642 int f2fs_do_write_data_page(struct f2fs_io_info *fio) 2643 { 2644 struct folio *folio = page_folio(fio->page); 2645 struct inode *inode = folio->mapping->host; 2646 struct dnode_of_data dn; 2647 struct node_info ni; 2648 bool ipu_force = false; 2649 bool atomic_commit; 2650 int err = 0; 2651 2652 /* Use COW inode to make dnode_of_data for atomic write */ 2653 atomic_commit = f2fs_is_atomic_file(inode) && 2654 page_private_atomic(folio_page(folio, 0)); 2655 if (atomic_commit) 2656 set_new_dnode(&dn, F2FS_I(inode)->cow_inode, NULL, NULL, 0); 2657 else 2658 set_new_dnode(&dn, inode, NULL, NULL, 0); 2659 2660 if (need_inplace_update(fio) && 2661 f2fs_lookup_read_extent_cache_block(inode, folio->index, 2662 &fio->old_blkaddr)) { 2663 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, 2664 DATA_GENERIC_ENHANCE)) 2665 return -EFSCORRUPTED; 2666 2667 ipu_force = true; 2668 fio->need_lock = LOCK_DONE; 2669 goto got_it; 2670 } 2671 2672 /* Deadlock due to between page->lock and f2fs_lock_op */ 2673 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi)) 2674 return -EAGAIN; 2675 2676 err = f2fs_get_dnode_of_data(&dn, folio->index, LOOKUP_NODE); 2677 if (err) 2678 goto out; 2679 2680 fio->old_blkaddr = dn.data_blkaddr; 2681 2682 /* This page is already truncated */ 2683 if (fio->old_blkaddr == NULL_ADDR) { 2684 folio_clear_uptodate(folio); 2685 clear_page_private_gcing(folio_page(folio, 0)); 2686 goto out_writepage; 2687 } 2688 got_it: 2689 if (__is_valid_data_blkaddr(fio->old_blkaddr) && 2690 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, 2691 DATA_GENERIC_ENHANCE)) { 2692 err = -EFSCORRUPTED; 2693 goto out_writepage; 2694 } 2695 2696 /* wait for GCed page writeback via META_MAPPING */ 2697 if (fio->meta_gc) 2698 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr); 2699 2700 /* 2701 * If current allocation needs SSR, 2702 * it had better in-place writes for updated data. 2703 */ 2704 if (ipu_force || 2705 (__is_valid_data_blkaddr(fio->old_blkaddr) && 2706 need_inplace_update(fio))) { 2707 err = f2fs_encrypt_one_page(fio); 2708 if (err) 2709 goto out_writepage; 2710 2711 folio_start_writeback(folio); 2712 f2fs_put_dnode(&dn); 2713 if (fio->need_lock == LOCK_REQ) 2714 f2fs_unlock_op(fio->sbi); 2715 err = f2fs_inplace_write_data(fio); 2716 if (err) { 2717 if (fscrypt_inode_uses_fs_layer_crypto(inode)) 2718 fscrypt_finalize_bounce_page(&fio->encrypted_page); 2719 folio_end_writeback(folio); 2720 } else { 2721 set_inode_flag(inode, FI_UPDATE_WRITE); 2722 } 2723 trace_f2fs_do_write_data_page(folio, IPU); 2724 return err; 2725 } 2726 2727 if (fio->need_lock == LOCK_RETRY) { 2728 if (!f2fs_trylock_op(fio->sbi)) { 2729 err = -EAGAIN; 2730 goto out_writepage; 2731 } 2732 fio->need_lock = LOCK_REQ; 2733 } 2734 2735 err = f2fs_get_node_info(fio->sbi, dn.nid, &ni, false); 2736 if (err) 2737 goto out_writepage; 2738 2739 fio->version = ni.version; 2740 2741 err = f2fs_encrypt_one_page(fio); 2742 if (err) 2743 goto out_writepage; 2744 2745 folio_start_writeback(folio); 2746 2747 if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR) 2748 f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false); 2749 2750 /* LFS mode write path */ 2751 f2fs_outplace_write_data(&dn, fio); 2752 trace_f2fs_do_write_data_page(folio, OPU); 2753 set_inode_flag(inode, FI_APPEND_WRITE); 2754 if (atomic_commit) 2755 clear_page_private_atomic(folio_page(folio, 0)); 2756 out_writepage: 2757 f2fs_put_dnode(&dn); 2758 out: 2759 if (fio->need_lock == LOCK_REQ) 2760 f2fs_unlock_op(fio->sbi); 2761 return err; 2762 } 2763 2764 int f2fs_write_single_data_page(struct folio *folio, int *submitted, 2765 struct bio **bio, 2766 sector_t *last_block, 2767 struct writeback_control *wbc, 2768 enum iostat_type io_type, 2769 int compr_blocks, 2770 bool allow_balance) 2771 { 2772 struct inode *inode = folio->mapping->host; 2773 struct page *page = folio_page(folio, 0); 2774 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2775 loff_t i_size = i_size_read(inode); 2776 const pgoff_t end_index = ((unsigned long long)i_size) 2777 >> PAGE_SHIFT; 2778 loff_t psize = (loff_t)(folio->index + 1) << PAGE_SHIFT; 2779 unsigned offset = 0; 2780 bool need_balance_fs = false; 2781 bool quota_inode = IS_NOQUOTA(inode); 2782 int err = 0; 2783 struct f2fs_io_info fio = { 2784 .sbi = sbi, 2785 .ino = inode->i_ino, 2786 .type = DATA, 2787 .op = REQ_OP_WRITE, 2788 .op_flags = wbc_to_write_flags(wbc), 2789 .old_blkaddr = NULL_ADDR, 2790 .page = page, 2791 .encrypted_page = NULL, 2792 .submitted = 0, 2793 .compr_blocks = compr_blocks, 2794 .need_lock = compr_blocks ? LOCK_DONE : LOCK_RETRY, 2795 .meta_gc = f2fs_meta_inode_gc_required(inode) ? 1 : 0, 2796 .io_type = io_type, 2797 .io_wbc = wbc, 2798 .bio = bio, 2799 .last_block = last_block, 2800 }; 2801 2802 trace_f2fs_writepage(folio, DATA); 2803 2804 /* we should bypass data pages to proceed the kworker jobs */ 2805 if (unlikely(f2fs_cp_error(sbi))) { 2806 mapping_set_error(folio->mapping, -EIO); 2807 /* 2808 * don't drop any dirty dentry pages for keeping lastest 2809 * directory structure. 2810 */ 2811 if (S_ISDIR(inode->i_mode) && 2812 !is_sbi_flag_set(sbi, SBI_IS_CLOSE)) 2813 goto redirty_out; 2814 2815 /* keep data pages in remount-ro mode */ 2816 if (F2FS_OPTION(sbi).errors == MOUNT_ERRORS_READONLY) 2817 goto redirty_out; 2818 goto out; 2819 } 2820 2821 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 2822 goto redirty_out; 2823 2824 if (folio->index < end_index || 2825 f2fs_verity_in_progress(inode) || 2826 compr_blocks) 2827 goto write; 2828 2829 /* 2830 * If the offset is out-of-range of file size, 2831 * this page does not have to be written to disk. 2832 */ 2833 offset = i_size & (PAGE_SIZE - 1); 2834 if ((folio->index >= end_index + 1) || !offset) 2835 goto out; 2836 2837 folio_zero_segment(folio, offset, folio_size(folio)); 2838 write: 2839 /* Dentry/quota blocks are controlled by checkpoint */ 2840 if (S_ISDIR(inode->i_mode) || quota_inode) { 2841 /* 2842 * We need to wait for node_write to avoid block allocation during 2843 * checkpoint. This can only happen to quota writes which can cause 2844 * the below discard race condition. 2845 */ 2846 if (quota_inode) 2847 f2fs_down_read(&sbi->node_write); 2848 2849 fio.need_lock = LOCK_DONE; 2850 err = f2fs_do_write_data_page(&fio); 2851 2852 if (quota_inode) 2853 f2fs_up_read(&sbi->node_write); 2854 2855 goto done; 2856 } 2857 2858 if (!wbc->for_reclaim) 2859 need_balance_fs = true; 2860 else if (has_not_enough_free_secs(sbi, 0, 0)) 2861 goto redirty_out; 2862 else 2863 set_inode_flag(inode, FI_HOT_DATA); 2864 2865 err = -EAGAIN; 2866 if (f2fs_has_inline_data(inode)) { 2867 err = f2fs_write_inline_data(inode, folio); 2868 if (!err) 2869 goto out; 2870 } 2871 2872 if (err == -EAGAIN) { 2873 err = f2fs_do_write_data_page(&fio); 2874 if (err == -EAGAIN) { 2875 f2fs_bug_on(sbi, compr_blocks); 2876 fio.need_lock = LOCK_REQ; 2877 err = f2fs_do_write_data_page(&fio); 2878 } 2879 } 2880 2881 if (err) { 2882 file_set_keep_isize(inode); 2883 } else { 2884 spin_lock(&F2FS_I(inode)->i_size_lock); 2885 if (F2FS_I(inode)->last_disk_size < psize) 2886 F2FS_I(inode)->last_disk_size = psize; 2887 spin_unlock(&F2FS_I(inode)->i_size_lock); 2888 } 2889 2890 done: 2891 if (err && err != -ENOENT) 2892 goto redirty_out; 2893 2894 out: 2895 inode_dec_dirty_pages(inode); 2896 if (err) { 2897 folio_clear_uptodate(folio); 2898 clear_page_private_gcing(page); 2899 } 2900 2901 if (wbc->for_reclaim) { 2902 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, DATA); 2903 clear_inode_flag(inode, FI_HOT_DATA); 2904 f2fs_remove_dirty_inode(inode); 2905 submitted = NULL; 2906 } 2907 folio_unlock(folio); 2908 if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) && 2909 !F2FS_I(inode)->wb_task && allow_balance) 2910 f2fs_balance_fs(sbi, need_balance_fs); 2911 2912 if (unlikely(f2fs_cp_error(sbi))) { 2913 f2fs_submit_merged_write(sbi, DATA); 2914 if (bio && *bio) 2915 f2fs_submit_merged_ipu_write(sbi, bio, NULL); 2916 submitted = NULL; 2917 } 2918 2919 if (submitted) 2920 *submitted = fio.submitted; 2921 2922 return 0; 2923 2924 redirty_out: 2925 folio_redirty_for_writepage(wbc, folio); 2926 /* 2927 * pageout() in MM translates EAGAIN, so calls handle_write_error() 2928 * -> mapping_set_error() -> set_bit(AS_EIO, ...). 2929 * file_write_and_wait_range() will see EIO error, which is critical 2930 * to return value of fsync() followed by atomic_write failure to user. 2931 */ 2932 if (!err || wbc->for_reclaim) 2933 return AOP_WRITEPAGE_ACTIVATE; 2934 folio_unlock(folio); 2935 return err; 2936 } 2937 2938 /* 2939 * This function was copied from write_cache_pages from mm/page-writeback.c. 2940 * The major change is making write step of cold data page separately from 2941 * warm/hot data page. 2942 */ 2943 static int f2fs_write_cache_pages(struct address_space *mapping, 2944 struct writeback_control *wbc, 2945 enum iostat_type io_type) 2946 { 2947 int ret = 0; 2948 int done = 0, retry = 0; 2949 struct page *pages_local[F2FS_ONSTACK_PAGES]; 2950 struct page **pages = pages_local; 2951 struct folio_batch fbatch; 2952 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); 2953 struct bio *bio = NULL; 2954 sector_t last_block; 2955 #ifdef CONFIG_F2FS_FS_COMPRESSION 2956 struct inode *inode = mapping->host; 2957 struct compress_ctx cc = { 2958 .inode = inode, 2959 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 2960 .cluster_size = F2FS_I(inode)->i_cluster_size, 2961 .cluster_idx = NULL_CLUSTER, 2962 .rpages = NULL, 2963 .nr_rpages = 0, 2964 .cpages = NULL, 2965 .valid_nr_cpages = 0, 2966 .rbuf = NULL, 2967 .cbuf = NULL, 2968 .rlen = PAGE_SIZE * F2FS_I(inode)->i_cluster_size, 2969 .private = NULL, 2970 }; 2971 #endif 2972 int nr_folios, p, idx; 2973 int nr_pages; 2974 unsigned int max_pages = F2FS_ONSTACK_PAGES; 2975 pgoff_t index; 2976 pgoff_t end; /* Inclusive */ 2977 pgoff_t done_index; 2978 int range_whole = 0; 2979 xa_mark_t tag; 2980 int nwritten = 0; 2981 int submitted = 0; 2982 int i; 2983 2984 #ifdef CONFIG_F2FS_FS_COMPRESSION 2985 if (f2fs_compressed_file(inode) && 2986 1 << cc.log_cluster_size > F2FS_ONSTACK_PAGES) { 2987 pages = f2fs_kzalloc(sbi, sizeof(struct page *) << 2988 cc.log_cluster_size, GFP_NOFS | __GFP_NOFAIL); 2989 max_pages = 1 << cc.log_cluster_size; 2990 } 2991 #endif 2992 2993 folio_batch_init(&fbatch); 2994 2995 if (get_dirty_pages(mapping->host) <= 2996 SM_I(F2FS_M_SB(mapping))->min_hot_blocks) 2997 set_inode_flag(mapping->host, FI_HOT_DATA); 2998 else 2999 clear_inode_flag(mapping->host, FI_HOT_DATA); 3000 3001 if (wbc->range_cyclic) { 3002 index = mapping->writeback_index; /* prev offset */ 3003 end = -1; 3004 } else { 3005 index = wbc->range_start >> PAGE_SHIFT; 3006 end = wbc->range_end >> PAGE_SHIFT; 3007 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 3008 range_whole = 1; 3009 } 3010 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 3011 tag = PAGECACHE_TAG_TOWRITE; 3012 else 3013 tag = PAGECACHE_TAG_DIRTY; 3014 retry: 3015 retry = 0; 3016 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 3017 tag_pages_for_writeback(mapping, index, end); 3018 done_index = index; 3019 while (!done && !retry && (index <= end)) { 3020 nr_pages = 0; 3021 again: 3022 nr_folios = filemap_get_folios_tag(mapping, &index, end, 3023 tag, &fbatch); 3024 if (nr_folios == 0) { 3025 if (nr_pages) 3026 goto write; 3027 break; 3028 } 3029 3030 for (i = 0; i < nr_folios; i++) { 3031 struct folio *folio = fbatch.folios[i]; 3032 3033 idx = 0; 3034 p = folio_nr_pages(folio); 3035 add_more: 3036 pages[nr_pages] = folio_page(folio, idx); 3037 folio_get(folio); 3038 if (++nr_pages == max_pages) { 3039 index = folio->index + idx + 1; 3040 folio_batch_release(&fbatch); 3041 goto write; 3042 } 3043 if (++idx < p) 3044 goto add_more; 3045 } 3046 folio_batch_release(&fbatch); 3047 goto again; 3048 write: 3049 for (i = 0; i < nr_pages; i++) { 3050 struct page *page = pages[i]; 3051 struct folio *folio = page_folio(page); 3052 bool need_readd; 3053 readd: 3054 need_readd = false; 3055 #ifdef CONFIG_F2FS_FS_COMPRESSION 3056 if (f2fs_compressed_file(inode)) { 3057 void *fsdata = NULL; 3058 struct page *pagep; 3059 int ret2; 3060 3061 ret = f2fs_init_compress_ctx(&cc); 3062 if (ret) { 3063 done = 1; 3064 break; 3065 } 3066 3067 if (!f2fs_cluster_can_merge_page(&cc, 3068 folio->index)) { 3069 ret = f2fs_write_multi_pages(&cc, 3070 &submitted, wbc, io_type); 3071 if (!ret) 3072 need_readd = true; 3073 goto result; 3074 } 3075 3076 if (unlikely(f2fs_cp_error(sbi))) 3077 goto lock_folio; 3078 3079 if (!f2fs_cluster_is_empty(&cc)) 3080 goto lock_folio; 3081 3082 if (f2fs_all_cluster_page_ready(&cc, 3083 pages, i, nr_pages, true)) 3084 goto lock_folio; 3085 3086 ret2 = f2fs_prepare_compress_overwrite( 3087 inode, &pagep, 3088 folio->index, &fsdata); 3089 if (ret2 < 0) { 3090 ret = ret2; 3091 done = 1; 3092 break; 3093 } else if (ret2 && 3094 (!f2fs_compress_write_end(inode, 3095 fsdata, folio->index, 1) || 3096 !f2fs_all_cluster_page_ready(&cc, 3097 pages, i, nr_pages, 3098 false))) { 3099 retry = 1; 3100 break; 3101 } 3102 } 3103 #endif 3104 /* give a priority to WB_SYNC threads */ 3105 if (atomic_read(&sbi->wb_sync_req[DATA]) && 3106 wbc->sync_mode == WB_SYNC_NONE) { 3107 done = 1; 3108 break; 3109 } 3110 #ifdef CONFIG_F2FS_FS_COMPRESSION 3111 lock_folio: 3112 #endif 3113 done_index = folio->index; 3114 retry_write: 3115 folio_lock(folio); 3116 3117 if (unlikely(folio->mapping != mapping)) { 3118 continue_unlock: 3119 folio_unlock(folio); 3120 continue; 3121 } 3122 3123 if (!folio_test_dirty(folio)) { 3124 /* someone wrote it for us */ 3125 goto continue_unlock; 3126 } 3127 3128 if (folio_test_writeback(folio)) { 3129 if (wbc->sync_mode == WB_SYNC_NONE) 3130 goto continue_unlock; 3131 f2fs_wait_on_page_writeback(&folio->page, DATA, true, true); 3132 } 3133 3134 if (!folio_clear_dirty_for_io(folio)) 3135 goto continue_unlock; 3136 3137 #ifdef CONFIG_F2FS_FS_COMPRESSION 3138 if (f2fs_compressed_file(inode)) { 3139 folio_get(folio); 3140 f2fs_compress_ctx_add_page(&cc, folio); 3141 continue; 3142 } 3143 #endif 3144 submitted = 0; 3145 ret = f2fs_write_single_data_page(folio, 3146 &submitted, &bio, &last_block, 3147 wbc, io_type, 0, true); 3148 if (ret == AOP_WRITEPAGE_ACTIVATE) 3149 folio_unlock(folio); 3150 #ifdef CONFIG_F2FS_FS_COMPRESSION 3151 result: 3152 #endif 3153 nwritten += submitted; 3154 wbc->nr_to_write -= submitted; 3155 3156 if (unlikely(ret)) { 3157 /* 3158 * keep nr_to_write, since vfs uses this to 3159 * get # of written pages. 3160 */ 3161 if (ret == AOP_WRITEPAGE_ACTIVATE) { 3162 ret = 0; 3163 goto next; 3164 } else if (ret == -EAGAIN) { 3165 ret = 0; 3166 if (wbc->sync_mode == WB_SYNC_ALL) { 3167 f2fs_io_schedule_timeout( 3168 DEFAULT_IO_TIMEOUT); 3169 goto retry_write; 3170 } 3171 goto next; 3172 } 3173 done_index = folio_next_index(folio); 3174 done = 1; 3175 break; 3176 } 3177 3178 if (wbc->nr_to_write <= 0 && 3179 wbc->sync_mode == WB_SYNC_NONE) { 3180 done = 1; 3181 break; 3182 } 3183 next: 3184 if (need_readd) 3185 goto readd; 3186 } 3187 release_pages(pages, nr_pages); 3188 cond_resched(); 3189 } 3190 #ifdef CONFIG_F2FS_FS_COMPRESSION 3191 /* flush remained pages in compress cluster */ 3192 if (f2fs_compressed_file(inode) && !f2fs_cluster_is_empty(&cc)) { 3193 ret = f2fs_write_multi_pages(&cc, &submitted, wbc, io_type); 3194 nwritten += submitted; 3195 wbc->nr_to_write -= submitted; 3196 if (ret) { 3197 done = 1; 3198 retry = 0; 3199 } 3200 } 3201 if (f2fs_compressed_file(inode)) 3202 f2fs_destroy_compress_ctx(&cc, false); 3203 #endif 3204 if (retry) { 3205 index = 0; 3206 end = -1; 3207 goto retry; 3208 } 3209 if (wbc->range_cyclic && !done) 3210 done_index = 0; 3211 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 3212 mapping->writeback_index = done_index; 3213 3214 if (nwritten) 3215 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host, 3216 NULL, 0, DATA); 3217 /* submit cached bio of IPU write */ 3218 if (bio) 3219 f2fs_submit_merged_ipu_write(sbi, &bio, NULL); 3220 3221 #ifdef CONFIG_F2FS_FS_COMPRESSION 3222 if (pages != pages_local) 3223 kfree(pages); 3224 #endif 3225 3226 return ret; 3227 } 3228 3229 static inline bool __should_serialize_io(struct inode *inode, 3230 struct writeback_control *wbc) 3231 { 3232 /* to avoid deadlock in path of data flush */ 3233 if (F2FS_I(inode)->wb_task) 3234 return false; 3235 3236 if (!S_ISREG(inode->i_mode)) 3237 return false; 3238 if (IS_NOQUOTA(inode)) 3239 return false; 3240 3241 if (f2fs_need_compress_data(inode)) 3242 return true; 3243 if (wbc->sync_mode != WB_SYNC_ALL) 3244 return true; 3245 if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks) 3246 return true; 3247 return false; 3248 } 3249 3250 static int __f2fs_write_data_pages(struct address_space *mapping, 3251 struct writeback_control *wbc, 3252 enum iostat_type io_type) 3253 { 3254 struct inode *inode = mapping->host; 3255 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3256 struct blk_plug plug; 3257 int ret; 3258 bool locked = false; 3259 3260 /* skip writing if there is no dirty page in this inode */ 3261 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE) 3262 return 0; 3263 3264 /* during POR, we don't need to trigger writepage at all. */ 3265 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 3266 goto skip_write; 3267 3268 if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) && 3269 wbc->sync_mode == WB_SYNC_NONE && 3270 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && 3271 f2fs_available_free_memory(sbi, DIRTY_DENTS)) 3272 goto skip_write; 3273 3274 /* skip writing in file defragment preparing stage */ 3275 if (is_inode_flag_set(inode, FI_SKIP_WRITES)) 3276 goto skip_write; 3277 3278 trace_f2fs_writepages(mapping->host, wbc, DATA); 3279 3280 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */ 3281 if (wbc->sync_mode == WB_SYNC_ALL) 3282 atomic_inc(&sbi->wb_sync_req[DATA]); 3283 else if (atomic_read(&sbi->wb_sync_req[DATA])) { 3284 /* to avoid potential deadlock */ 3285 if (current->plug) 3286 blk_finish_plug(current->plug); 3287 goto skip_write; 3288 } 3289 3290 if (__should_serialize_io(inode, wbc)) { 3291 mutex_lock(&sbi->writepages); 3292 locked = true; 3293 } 3294 3295 blk_start_plug(&plug); 3296 ret = f2fs_write_cache_pages(mapping, wbc, io_type); 3297 blk_finish_plug(&plug); 3298 3299 if (locked) 3300 mutex_unlock(&sbi->writepages); 3301 3302 if (wbc->sync_mode == WB_SYNC_ALL) 3303 atomic_dec(&sbi->wb_sync_req[DATA]); 3304 /* 3305 * if some pages were truncated, we cannot guarantee its mapping->host 3306 * to detect pending bios. 3307 */ 3308 3309 f2fs_remove_dirty_inode(inode); 3310 return ret; 3311 3312 skip_write: 3313 wbc->pages_skipped += get_dirty_pages(inode); 3314 trace_f2fs_writepages(mapping->host, wbc, DATA); 3315 return 0; 3316 } 3317 3318 static int f2fs_write_data_pages(struct address_space *mapping, 3319 struct writeback_control *wbc) 3320 { 3321 struct inode *inode = mapping->host; 3322 3323 return __f2fs_write_data_pages(mapping, wbc, 3324 F2FS_I(inode)->cp_task == current ? 3325 FS_CP_DATA_IO : FS_DATA_IO); 3326 } 3327 3328 void f2fs_write_failed(struct inode *inode, loff_t to) 3329 { 3330 loff_t i_size = i_size_read(inode); 3331 3332 if (IS_NOQUOTA(inode)) 3333 return; 3334 3335 /* In the fs-verity case, f2fs_end_enable_verity() does the truncate */ 3336 if (to > i_size && !f2fs_verity_in_progress(inode)) { 3337 f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3338 filemap_invalidate_lock(inode->i_mapping); 3339 3340 truncate_pagecache(inode, i_size); 3341 f2fs_truncate_blocks(inode, i_size, true); 3342 3343 filemap_invalidate_unlock(inode->i_mapping); 3344 f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3345 } 3346 } 3347 3348 static int prepare_write_begin(struct f2fs_sb_info *sbi, 3349 struct folio *folio, loff_t pos, unsigned int len, 3350 block_t *blk_addr, bool *node_changed) 3351 { 3352 struct inode *inode = folio->mapping->host; 3353 pgoff_t index = folio->index; 3354 struct dnode_of_data dn; 3355 struct page *ipage; 3356 bool locked = false; 3357 int flag = F2FS_GET_BLOCK_PRE_AIO; 3358 int err = 0; 3359 3360 /* 3361 * If a whole page is being written and we already preallocated all the 3362 * blocks, then there is no need to get a block address now. 3363 */ 3364 if (len == PAGE_SIZE && is_inode_flag_set(inode, FI_PREALLOCATED_ALL)) 3365 return 0; 3366 3367 /* f2fs_lock_op avoids race between write CP and convert_inline_page */ 3368 if (f2fs_has_inline_data(inode)) { 3369 if (pos + len > MAX_INLINE_DATA(inode)) 3370 flag = F2FS_GET_BLOCK_DEFAULT; 3371 f2fs_map_lock(sbi, flag); 3372 locked = true; 3373 } else if ((pos & PAGE_MASK) >= i_size_read(inode)) { 3374 f2fs_map_lock(sbi, flag); 3375 locked = true; 3376 } 3377 3378 restart: 3379 /* check inline_data */ 3380 ipage = f2fs_get_inode_page(sbi, inode->i_ino); 3381 if (IS_ERR(ipage)) { 3382 err = PTR_ERR(ipage); 3383 goto unlock_out; 3384 } 3385 3386 set_new_dnode(&dn, inode, ipage, ipage, 0); 3387 3388 if (f2fs_has_inline_data(inode)) { 3389 if (pos + len <= MAX_INLINE_DATA(inode)) { 3390 f2fs_do_read_inline_data(folio, ipage); 3391 set_inode_flag(inode, FI_DATA_EXIST); 3392 if (inode->i_nlink) 3393 set_page_private_inline(ipage); 3394 goto out; 3395 } 3396 err = f2fs_convert_inline_page(&dn, folio_page(folio, 0)); 3397 if (err || dn.data_blkaddr != NULL_ADDR) 3398 goto out; 3399 } 3400 3401 if (!f2fs_lookup_read_extent_cache_block(inode, index, 3402 &dn.data_blkaddr)) { 3403 if (IS_DEVICE_ALIASING(inode)) { 3404 err = -ENODATA; 3405 goto out; 3406 } 3407 3408 if (locked) { 3409 err = f2fs_reserve_block(&dn, index); 3410 goto out; 3411 } 3412 3413 /* hole case */ 3414 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 3415 if (!err && dn.data_blkaddr != NULL_ADDR) 3416 goto out; 3417 f2fs_put_dnode(&dn); 3418 f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); 3419 WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO); 3420 locked = true; 3421 goto restart; 3422 } 3423 out: 3424 if (!err) { 3425 /* convert_inline_page can make node_changed */ 3426 *blk_addr = dn.data_blkaddr; 3427 *node_changed = dn.node_changed; 3428 } 3429 f2fs_put_dnode(&dn); 3430 unlock_out: 3431 if (locked) 3432 f2fs_map_unlock(sbi, flag); 3433 return err; 3434 } 3435 3436 static int __find_data_block(struct inode *inode, pgoff_t index, 3437 block_t *blk_addr) 3438 { 3439 struct dnode_of_data dn; 3440 struct page *ipage; 3441 int err = 0; 3442 3443 ipage = f2fs_get_inode_page(F2FS_I_SB(inode), inode->i_ino); 3444 if (IS_ERR(ipage)) 3445 return PTR_ERR(ipage); 3446 3447 set_new_dnode(&dn, inode, ipage, ipage, 0); 3448 3449 if (!f2fs_lookup_read_extent_cache_block(inode, index, 3450 &dn.data_blkaddr)) { 3451 /* hole case */ 3452 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 3453 if (err) { 3454 dn.data_blkaddr = NULL_ADDR; 3455 err = 0; 3456 } 3457 } 3458 *blk_addr = dn.data_blkaddr; 3459 f2fs_put_dnode(&dn); 3460 return err; 3461 } 3462 3463 static int __reserve_data_block(struct inode *inode, pgoff_t index, 3464 block_t *blk_addr, bool *node_changed) 3465 { 3466 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3467 struct dnode_of_data dn; 3468 struct page *ipage; 3469 int err = 0; 3470 3471 f2fs_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO); 3472 3473 ipage = f2fs_get_inode_page(sbi, inode->i_ino); 3474 if (IS_ERR(ipage)) { 3475 err = PTR_ERR(ipage); 3476 goto unlock_out; 3477 } 3478 set_new_dnode(&dn, inode, ipage, ipage, 0); 3479 3480 if (!f2fs_lookup_read_extent_cache_block(dn.inode, index, 3481 &dn.data_blkaddr)) 3482 err = f2fs_reserve_block(&dn, index); 3483 3484 *blk_addr = dn.data_blkaddr; 3485 *node_changed = dn.node_changed; 3486 f2fs_put_dnode(&dn); 3487 3488 unlock_out: 3489 f2fs_map_unlock(sbi, F2FS_GET_BLOCK_PRE_AIO); 3490 return err; 3491 } 3492 3493 static int prepare_atomic_write_begin(struct f2fs_sb_info *sbi, 3494 struct folio *folio, loff_t pos, unsigned int len, 3495 block_t *blk_addr, bool *node_changed, bool *use_cow) 3496 { 3497 struct inode *inode = folio->mapping->host; 3498 struct inode *cow_inode = F2FS_I(inode)->cow_inode; 3499 pgoff_t index = folio->index; 3500 int err = 0; 3501 block_t ori_blk_addr = NULL_ADDR; 3502 3503 /* If pos is beyond the end of file, reserve a new block in COW inode */ 3504 if ((pos & PAGE_MASK) >= i_size_read(inode)) 3505 goto reserve_block; 3506 3507 /* Look for the block in COW inode first */ 3508 err = __find_data_block(cow_inode, index, blk_addr); 3509 if (err) { 3510 return err; 3511 } else if (*blk_addr != NULL_ADDR) { 3512 *use_cow = true; 3513 return 0; 3514 } 3515 3516 if (is_inode_flag_set(inode, FI_ATOMIC_REPLACE)) 3517 goto reserve_block; 3518 3519 /* Look for the block in the original inode */ 3520 err = __find_data_block(inode, index, &ori_blk_addr); 3521 if (err) 3522 return err; 3523 3524 reserve_block: 3525 /* Finally, we should reserve a new block in COW inode for the update */ 3526 err = __reserve_data_block(cow_inode, index, blk_addr, node_changed); 3527 if (err) 3528 return err; 3529 inc_atomic_write_cnt(inode); 3530 3531 if (ori_blk_addr != NULL_ADDR) 3532 *blk_addr = ori_blk_addr; 3533 return 0; 3534 } 3535 3536 static int f2fs_write_begin(struct file *file, struct address_space *mapping, 3537 loff_t pos, unsigned len, struct folio **foliop, void **fsdata) 3538 { 3539 struct inode *inode = mapping->host; 3540 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3541 struct folio *folio; 3542 pgoff_t index = pos >> PAGE_SHIFT; 3543 bool need_balance = false; 3544 bool use_cow = false; 3545 block_t blkaddr = NULL_ADDR; 3546 int err = 0; 3547 3548 trace_f2fs_write_begin(inode, pos, len); 3549 3550 if (!f2fs_is_checkpoint_ready(sbi)) { 3551 err = -ENOSPC; 3552 goto fail; 3553 } 3554 3555 /* 3556 * We should check this at this moment to avoid deadlock on inode page 3557 * and #0 page. The locking rule for inline_data conversion should be: 3558 * folio_lock(folio #0) -> folio_lock(inode_page) 3559 */ 3560 if (index != 0) { 3561 err = f2fs_convert_inline_inode(inode); 3562 if (err) 3563 goto fail; 3564 } 3565 3566 #ifdef CONFIG_F2FS_FS_COMPRESSION 3567 if (f2fs_compressed_file(inode)) { 3568 int ret; 3569 struct page *page; 3570 3571 *fsdata = NULL; 3572 3573 if (len == PAGE_SIZE && !(f2fs_is_atomic_file(inode))) 3574 goto repeat; 3575 3576 ret = f2fs_prepare_compress_overwrite(inode, &page, 3577 index, fsdata); 3578 if (ret < 0) { 3579 err = ret; 3580 goto fail; 3581 } else if (ret) { 3582 *foliop = page_folio(page); 3583 return 0; 3584 } 3585 } 3586 #endif 3587 3588 repeat: 3589 /* 3590 * Do not use FGP_STABLE to avoid deadlock. 3591 * Will wait that below with our IO control. 3592 */ 3593 folio = __filemap_get_folio(mapping, index, 3594 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS); 3595 if (IS_ERR(folio)) { 3596 err = PTR_ERR(folio); 3597 goto fail; 3598 } 3599 3600 /* TODO: cluster can be compressed due to race with .writepage */ 3601 3602 *foliop = folio; 3603 3604 if (f2fs_is_atomic_file(inode)) 3605 err = prepare_atomic_write_begin(sbi, folio, pos, len, 3606 &blkaddr, &need_balance, &use_cow); 3607 else 3608 err = prepare_write_begin(sbi, folio, pos, len, 3609 &blkaddr, &need_balance); 3610 if (err) 3611 goto put_folio; 3612 3613 if (need_balance && !IS_NOQUOTA(inode) && 3614 has_not_enough_free_secs(sbi, 0, 0)) { 3615 folio_unlock(folio); 3616 f2fs_balance_fs(sbi, true); 3617 folio_lock(folio); 3618 if (folio->mapping != mapping) { 3619 /* The folio got truncated from under us */ 3620 folio_unlock(folio); 3621 folio_put(folio); 3622 goto repeat; 3623 } 3624 } 3625 3626 f2fs_wait_on_page_writeback(&folio->page, DATA, false, true); 3627 3628 if (len == folio_size(folio) || folio_test_uptodate(folio)) 3629 return 0; 3630 3631 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode) && 3632 !f2fs_verity_in_progress(inode)) { 3633 folio_zero_segment(folio, len, folio_size(folio)); 3634 return 0; 3635 } 3636 3637 if (blkaddr == NEW_ADDR) { 3638 folio_zero_segment(folio, 0, folio_size(folio)); 3639 folio_mark_uptodate(folio); 3640 } else { 3641 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, 3642 DATA_GENERIC_ENHANCE_READ)) { 3643 err = -EFSCORRUPTED; 3644 goto put_folio; 3645 } 3646 err = f2fs_submit_page_read(use_cow ? 3647 F2FS_I(inode)->cow_inode : inode, 3648 folio, blkaddr, 0, true); 3649 if (err) 3650 goto put_folio; 3651 3652 folio_lock(folio); 3653 if (unlikely(folio->mapping != mapping)) { 3654 folio_unlock(folio); 3655 folio_put(folio); 3656 goto repeat; 3657 } 3658 if (unlikely(!folio_test_uptodate(folio))) { 3659 err = -EIO; 3660 goto put_folio; 3661 } 3662 } 3663 return 0; 3664 3665 put_folio: 3666 folio_unlock(folio); 3667 folio_put(folio); 3668 fail: 3669 f2fs_write_failed(inode, pos + len); 3670 return err; 3671 } 3672 3673 static int f2fs_write_end(struct file *file, 3674 struct address_space *mapping, 3675 loff_t pos, unsigned len, unsigned copied, 3676 struct folio *folio, void *fsdata) 3677 { 3678 struct inode *inode = folio->mapping->host; 3679 3680 trace_f2fs_write_end(inode, pos, len, copied); 3681 3682 /* 3683 * This should be come from len == PAGE_SIZE, and we expect copied 3684 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and 3685 * let generic_perform_write() try to copy data again through copied=0. 3686 */ 3687 if (!folio_test_uptodate(folio)) { 3688 if (unlikely(copied != len)) 3689 copied = 0; 3690 else 3691 folio_mark_uptodate(folio); 3692 } 3693 3694 #ifdef CONFIG_F2FS_FS_COMPRESSION 3695 /* overwrite compressed file */ 3696 if (f2fs_compressed_file(inode) && fsdata) { 3697 f2fs_compress_write_end(inode, fsdata, folio->index, copied); 3698 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); 3699 3700 if (pos + copied > i_size_read(inode) && 3701 !f2fs_verity_in_progress(inode)) 3702 f2fs_i_size_write(inode, pos + copied); 3703 return copied; 3704 } 3705 #endif 3706 3707 if (!copied) 3708 goto unlock_out; 3709 3710 folio_mark_dirty(folio); 3711 3712 if (f2fs_is_atomic_file(inode)) 3713 set_page_private_atomic(folio_page(folio, 0)); 3714 3715 if (pos + copied > i_size_read(inode) && 3716 !f2fs_verity_in_progress(inode)) { 3717 f2fs_i_size_write(inode, pos + copied); 3718 if (f2fs_is_atomic_file(inode)) 3719 f2fs_i_size_write(F2FS_I(inode)->cow_inode, 3720 pos + copied); 3721 } 3722 unlock_out: 3723 folio_unlock(folio); 3724 folio_put(folio); 3725 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); 3726 return copied; 3727 } 3728 3729 void f2fs_invalidate_folio(struct folio *folio, size_t offset, size_t length) 3730 { 3731 struct inode *inode = folio->mapping->host; 3732 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3733 3734 if (inode->i_ino >= F2FS_ROOT_INO(sbi) && 3735 (offset || length != folio_size(folio))) 3736 return; 3737 3738 if (folio_test_dirty(folio)) { 3739 if (inode->i_ino == F2FS_META_INO(sbi)) { 3740 dec_page_count(sbi, F2FS_DIRTY_META); 3741 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) { 3742 dec_page_count(sbi, F2FS_DIRTY_NODES); 3743 } else { 3744 inode_dec_dirty_pages(inode); 3745 f2fs_remove_dirty_inode(inode); 3746 } 3747 } 3748 clear_page_private_all(&folio->page); 3749 } 3750 3751 bool f2fs_release_folio(struct folio *folio, gfp_t wait) 3752 { 3753 /* If this is dirty folio, keep private data */ 3754 if (folio_test_dirty(folio)) 3755 return false; 3756 3757 clear_page_private_all(&folio->page); 3758 return true; 3759 } 3760 3761 static bool f2fs_dirty_data_folio(struct address_space *mapping, 3762 struct folio *folio) 3763 { 3764 struct inode *inode = mapping->host; 3765 3766 trace_f2fs_set_page_dirty(folio, DATA); 3767 3768 if (!folio_test_uptodate(folio)) 3769 folio_mark_uptodate(folio); 3770 BUG_ON(folio_test_swapcache(folio)); 3771 3772 if (filemap_dirty_folio(mapping, folio)) { 3773 f2fs_update_dirty_folio(inode, folio); 3774 return true; 3775 } 3776 return false; 3777 } 3778 3779 3780 static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block) 3781 { 3782 #ifdef CONFIG_F2FS_FS_COMPRESSION 3783 struct dnode_of_data dn; 3784 sector_t start_idx, blknr = 0; 3785 int ret; 3786 3787 start_idx = round_down(block, F2FS_I(inode)->i_cluster_size); 3788 3789 set_new_dnode(&dn, inode, NULL, NULL, 0); 3790 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 3791 if (ret) 3792 return 0; 3793 3794 if (dn.data_blkaddr != COMPRESS_ADDR) { 3795 dn.ofs_in_node += block - start_idx; 3796 blknr = f2fs_data_blkaddr(&dn); 3797 if (!__is_valid_data_blkaddr(blknr)) 3798 blknr = 0; 3799 } 3800 3801 f2fs_put_dnode(&dn); 3802 return blknr; 3803 #else 3804 return 0; 3805 #endif 3806 } 3807 3808 3809 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) 3810 { 3811 struct inode *inode = mapping->host; 3812 sector_t blknr = 0; 3813 3814 if (f2fs_has_inline_data(inode)) 3815 goto out; 3816 3817 /* make sure allocating whole blocks */ 3818 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 3819 filemap_write_and_wait(mapping); 3820 3821 /* Block number less than F2FS MAX BLOCKS */ 3822 if (unlikely(block >= max_file_blocks(inode))) 3823 goto out; 3824 3825 if (f2fs_compressed_file(inode)) { 3826 blknr = f2fs_bmap_compress(inode, block); 3827 } else { 3828 struct f2fs_map_blocks map; 3829 3830 memset(&map, 0, sizeof(map)); 3831 map.m_lblk = block; 3832 map.m_len = 1; 3833 map.m_next_pgofs = NULL; 3834 map.m_seg_type = NO_CHECK_TYPE; 3835 3836 if (!f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_BMAP)) 3837 blknr = map.m_pblk; 3838 } 3839 out: 3840 trace_f2fs_bmap(inode, block, blknr); 3841 return blknr; 3842 } 3843 3844 #ifdef CONFIG_SWAP 3845 static int f2fs_migrate_blocks(struct inode *inode, block_t start_blk, 3846 unsigned int blkcnt) 3847 { 3848 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3849 unsigned int blkofs; 3850 unsigned int blk_per_sec = BLKS_PER_SEC(sbi); 3851 unsigned int end_blk = start_blk + blkcnt - 1; 3852 unsigned int secidx = start_blk / blk_per_sec; 3853 unsigned int end_sec; 3854 int ret = 0; 3855 3856 if (!blkcnt) 3857 return 0; 3858 end_sec = end_blk / blk_per_sec; 3859 3860 f2fs_down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3861 filemap_invalidate_lock(inode->i_mapping); 3862 3863 set_inode_flag(inode, FI_ALIGNED_WRITE); 3864 set_inode_flag(inode, FI_OPU_WRITE); 3865 3866 for (; secidx <= end_sec; secidx++) { 3867 unsigned int blkofs_end = secidx == end_sec ? 3868 end_blk % blk_per_sec : blk_per_sec - 1; 3869 3870 f2fs_down_write(&sbi->pin_sem); 3871 3872 ret = f2fs_allocate_pinning_section(sbi); 3873 if (ret) { 3874 f2fs_up_write(&sbi->pin_sem); 3875 break; 3876 } 3877 3878 set_inode_flag(inode, FI_SKIP_WRITES); 3879 3880 for (blkofs = 0; blkofs <= blkofs_end; blkofs++) { 3881 struct page *page; 3882 unsigned int blkidx = secidx * blk_per_sec + blkofs; 3883 3884 page = f2fs_get_lock_data_page(inode, blkidx, true); 3885 if (IS_ERR(page)) { 3886 f2fs_up_write(&sbi->pin_sem); 3887 ret = PTR_ERR(page); 3888 goto done; 3889 } 3890 3891 set_page_dirty(page); 3892 f2fs_put_page(page, 1); 3893 } 3894 3895 clear_inode_flag(inode, FI_SKIP_WRITES); 3896 3897 ret = filemap_fdatawrite(inode->i_mapping); 3898 3899 f2fs_up_write(&sbi->pin_sem); 3900 3901 if (ret) 3902 break; 3903 } 3904 3905 done: 3906 clear_inode_flag(inode, FI_SKIP_WRITES); 3907 clear_inode_flag(inode, FI_OPU_WRITE); 3908 clear_inode_flag(inode, FI_ALIGNED_WRITE); 3909 3910 filemap_invalidate_unlock(inode->i_mapping); 3911 f2fs_up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3912 3913 return ret; 3914 } 3915 3916 static int check_swap_activate(struct swap_info_struct *sis, 3917 struct file *swap_file, sector_t *span) 3918 { 3919 struct address_space *mapping = swap_file->f_mapping; 3920 struct inode *inode = mapping->host; 3921 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3922 block_t cur_lblock; 3923 block_t last_lblock; 3924 block_t pblock; 3925 block_t lowest_pblock = -1; 3926 block_t highest_pblock = 0; 3927 int nr_extents = 0; 3928 unsigned int nr_pblocks; 3929 unsigned int blks_per_sec = BLKS_PER_SEC(sbi); 3930 unsigned int not_aligned = 0; 3931 int ret = 0; 3932 3933 /* 3934 * Map all the blocks into the extent list. This code doesn't try 3935 * to be very smart. 3936 */ 3937 cur_lblock = 0; 3938 last_lblock = F2FS_BYTES_TO_BLK(i_size_read(inode)); 3939 3940 while (cur_lblock < last_lblock && cur_lblock < sis->max) { 3941 struct f2fs_map_blocks map; 3942 retry: 3943 cond_resched(); 3944 3945 memset(&map, 0, sizeof(map)); 3946 map.m_lblk = cur_lblock; 3947 map.m_len = last_lblock - cur_lblock; 3948 map.m_next_pgofs = NULL; 3949 map.m_next_extent = NULL; 3950 map.m_seg_type = NO_CHECK_TYPE; 3951 map.m_may_create = false; 3952 3953 ret = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_FIEMAP); 3954 if (ret) 3955 goto out; 3956 3957 /* hole */ 3958 if (!(map.m_flags & F2FS_MAP_FLAGS)) { 3959 f2fs_err(sbi, "Swapfile has holes"); 3960 ret = -EINVAL; 3961 goto out; 3962 } 3963 3964 pblock = map.m_pblk; 3965 nr_pblocks = map.m_len; 3966 3967 if ((pblock - SM_I(sbi)->main_blkaddr) % blks_per_sec || 3968 nr_pblocks % blks_per_sec || 3969 !f2fs_valid_pinned_area(sbi, pblock)) { 3970 bool last_extent = false; 3971 3972 not_aligned++; 3973 3974 nr_pblocks = roundup(nr_pblocks, blks_per_sec); 3975 if (cur_lblock + nr_pblocks > sis->max) 3976 nr_pblocks -= blks_per_sec; 3977 3978 /* this extent is last one */ 3979 if (!nr_pblocks) { 3980 nr_pblocks = last_lblock - cur_lblock; 3981 last_extent = true; 3982 } 3983 3984 ret = f2fs_migrate_blocks(inode, cur_lblock, 3985 nr_pblocks); 3986 if (ret) { 3987 if (ret == -ENOENT) 3988 ret = -EINVAL; 3989 goto out; 3990 } 3991 3992 if (!last_extent) 3993 goto retry; 3994 } 3995 3996 if (cur_lblock + nr_pblocks >= sis->max) 3997 nr_pblocks = sis->max - cur_lblock; 3998 3999 if (cur_lblock) { /* exclude the header page */ 4000 if (pblock < lowest_pblock) 4001 lowest_pblock = pblock; 4002 if (pblock + nr_pblocks - 1 > highest_pblock) 4003 highest_pblock = pblock + nr_pblocks - 1; 4004 } 4005 4006 /* 4007 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks 4008 */ 4009 ret = add_swap_extent(sis, cur_lblock, nr_pblocks, pblock); 4010 if (ret < 0) 4011 goto out; 4012 nr_extents += ret; 4013 cur_lblock += nr_pblocks; 4014 } 4015 ret = nr_extents; 4016 *span = 1 + highest_pblock - lowest_pblock; 4017 if (cur_lblock == 0) 4018 cur_lblock = 1; /* force Empty message */ 4019 sis->max = cur_lblock; 4020 sis->pages = cur_lblock - 1; 4021 out: 4022 if (not_aligned) 4023 f2fs_warn(sbi, "Swapfile (%u) is not align to section: 1) creat(), 2) ioctl(F2FS_IOC_SET_PIN_FILE), 3) fallocate(%lu * N)", 4024 not_aligned, blks_per_sec * F2FS_BLKSIZE); 4025 return ret; 4026 } 4027 4028 static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, 4029 sector_t *span) 4030 { 4031 struct inode *inode = file_inode(file); 4032 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 4033 int ret; 4034 4035 if (!S_ISREG(inode->i_mode)) 4036 return -EINVAL; 4037 4038 if (f2fs_readonly(sbi->sb)) 4039 return -EROFS; 4040 4041 if (f2fs_lfs_mode(sbi) && !f2fs_sb_has_blkzoned(sbi)) { 4042 f2fs_err(sbi, "Swapfile not supported in LFS mode"); 4043 return -EINVAL; 4044 } 4045 4046 ret = f2fs_convert_inline_inode(inode); 4047 if (ret) 4048 return ret; 4049 4050 if (!f2fs_disable_compressed_file(inode)) 4051 return -EINVAL; 4052 4053 ret = filemap_fdatawrite(inode->i_mapping); 4054 if (ret < 0) 4055 return ret; 4056 4057 f2fs_precache_extents(inode); 4058 4059 ret = check_swap_activate(sis, file, span); 4060 if (ret < 0) 4061 return ret; 4062 4063 stat_inc_swapfile_inode(inode); 4064 set_inode_flag(inode, FI_PIN_FILE); 4065 f2fs_update_time(sbi, REQ_TIME); 4066 return ret; 4067 } 4068 4069 static void f2fs_swap_deactivate(struct file *file) 4070 { 4071 struct inode *inode = file_inode(file); 4072 4073 stat_dec_swapfile_inode(inode); 4074 clear_inode_flag(inode, FI_PIN_FILE); 4075 } 4076 #else 4077 static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, 4078 sector_t *span) 4079 { 4080 return -EOPNOTSUPP; 4081 } 4082 4083 static void f2fs_swap_deactivate(struct file *file) 4084 { 4085 } 4086 #endif 4087 4088 const struct address_space_operations f2fs_dblock_aops = { 4089 .read_folio = f2fs_read_data_folio, 4090 .readahead = f2fs_readahead, 4091 .writepages = f2fs_write_data_pages, 4092 .write_begin = f2fs_write_begin, 4093 .write_end = f2fs_write_end, 4094 .dirty_folio = f2fs_dirty_data_folio, 4095 .migrate_folio = filemap_migrate_folio, 4096 .invalidate_folio = f2fs_invalidate_folio, 4097 .release_folio = f2fs_release_folio, 4098 .bmap = f2fs_bmap, 4099 .swap_activate = f2fs_swap_activate, 4100 .swap_deactivate = f2fs_swap_deactivate, 4101 }; 4102 4103 void f2fs_clear_page_cache_dirty_tag(struct folio *folio) 4104 { 4105 struct address_space *mapping = folio->mapping; 4106 unsigned long flags; 4107 4108 xa_lock_irqsave(&mapping->i_pages, flags); 4109 __xa_clear_mark(&mapping->i_pages, folio->index, 4110 PAGECACHE_TAG_DIRTY); 4111 xa_unlock_irqrestore(&mapping->i_pages, flags); 4112 } 4113 4114 int __init f2fs_init_post_read_processing(void) 4115 { 4116 bio_post_read_ctx_cache = 4117 kmem_cache_create("f2fs_bio_post_read_ctx", 4118 sizeof(struct bio_post_read_ctx), 0, 0, NULL); 4119 if (!bio_post_read_ctx_cache) 4120 goto fail; 4121 bio_post_read_ctx_pool = 4122 mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS, 4123 bio_post_read_ctx_cache); 4124 if (!bio_post_read_ctx_pool) 4125 goto fail_free_cache; 4126 return 0; 4127 4128 fail_free_cache: 4129 kmem_cache_destroy(bio_post_read_ctx_cache); 4130 fail: 4131 return -ENOMEM; 4132 } 4133 4134 void f2fs_destroy_post_read_processing(void) 4135 { 4136 mempool_destroy(bio_post_read_ctx_pool); 4137 kmem_cache_destroy(bio_post_read_ctx_cache); 4138 } 4139 4140 int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi) 4141 { 4142 if (!f2fs_sb_has_encrypt(sbi) && 4143 !f2fs_sb_has_verity(sbi) && 4144 !f2fs_sb_has_compression(sbi)) 4145 return 0; 4146 4147 sbi->post_read_wq = alloc_workqueue("f2fs_post_read_wq", 4148 WQ_UNBOUND | WQ_HIGHPRI, 4149 num_online_cpus()); 4150 return sbi->post_read_wq ? 0 : -ENOMEM; 4151 } 4152 4153 void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi) 4154 { 4155 if (sbi->post_read_wq) 4156 destroy_workqueue(sbi->post_read_wq); 4157 } 4158 4159 int __init f2fs_init_bio_entry_cache(void) 4160 { 4161 bio_entry_slab = f2fs_kmem_cache_create("f2fs_bio_entry_slab", 4162 sizeof(struct bio_entry)); 4163 return bio_entry_slab ? 0 : -ENOMEM; 4164 } 4165 4166 void f2fs_destroy_bio_entry_cache(void) 4167 { 4168 kmem_cache_destroy(bio_entry_slab); 4169 } 4170 4171 static int f2fs_iomap_begin(struct inode *inode, loff_t offset, loff_t length, 4172 unsigned int flags, struct iomap *iomap, 4173 struct iomap *srcmap) 4174 { 4175 struct f2fs_map_blocks map = {}; 4176 pgoff_t next_pgofs = 0; 4177 int err; 4178 4179 map.m_lblk = F2FS_BYTES_TO_BLK(offset); 4180 map.m_len = F2FS_BYTES_TO_BLK(offset + length - 1) - map.m_lblk + 1; 4181 map.m_next_pgofs = &next_pgofs; 4182 map.m_seg_type = f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode), 4183 inode->i_write_hint); 4184 4185 /* 4186 * If the blocks being overwritten are already allocated, 4187 * f2fs_map_lock and f2fs_balance_fs are not necessary. 4188 */ 4189 if ((flags & IOMAP_WRITE) && 4190 !f2fs_overwrite_io(inode, offset, length)) 4191 map.m_may_create = true; 4192 4193 err = f2fs_map_blocks(inode, &map, F2FS_GET_BLOCK_DIO); 4194 if (err) 4195 return err; 4196 4197 iomap->offset = F2FS_BLK_TO_BYTES(map.m_lblk); 4198 4199 /* 4200 * When inline encryption is enabled, sometimes I/O to an encrypted file 4201 * has to be broken up to guarantee DUN contiguity. Handle this by 4202 * limiting the length of the mapping returned. 4203 */ 4204 map.m_len = fscrypt_limit_io_blocks(inode, map.m_lblk, map.m_len); 4205 4206 /* 4207 * We should never see delalloc or compressed extents here based on 4208 * prior flushing and checks. 4209 */ 4210 if (WARN_ON_ONCE(map.m_pblk == COMPRESS_ADDR)) 4211 return -EINVAL; 4212 4213 if (map.m_flags & F2FS_MAP_MAPPED) { 4214 if (WARN_ON_ONCE(map.m_pblk == NEW_ADDR)) 4215 return -EINVAL; 4216 4217 iomap->length = F2FS_BLK_TO_BYTES(map.m_len); 4218 iomap->type = IOMAP_MAPPED; 4219 iomap->flags |= IOMAP_F_MERGED; 4220 iomap->bdev = map.m_bdev; 4221 iomap->addr = F2FS_BLK_TO_BYTES(map.m_pblk); 4222 } else { 4223 if (flags & IOMAP_WRITE) 4224 return -ENOTBLK; 4225 4226 if (map.m_pblk == NULL_ADDR) { 4227 iomap->length = F2FS_BLK_TO_BYTES(next_pgofs) - 4228 iomap->offset; 4229 iomap->type = IOMAP_HOLE; 4230 } else if (map.m_pblk == NEW_ADDR) { 4231 iomap->length = F2FS_BLK_TO_BYTES(map.m_len); 4232 iomap->type = IOMAP_UNWRITTEN; 4233 } else { 4234 f2fs_bug_on(F2FS_I_SB(inode), 1); 4235 } 4236 iomap->addr = IOMAP_NULL_ADDR; 4237 } 4238 4239 if (map.m_flags & F2FS_MAP_NEW) 4240 iomap->flags |= IOMAP_F_NEW; 4241 if ((inode->i_state & I_DIRTY_DATASYNC) || 4242 offset + length > i_size_read(inode)) 4243 iomap->flags |= IOMAP_F_DIRTY; 4244 4245 return 0; 4246 } 4247 4248 const struct iomap_ops f2fs_iomap_ops = { 4249 .iomap_begin = f2fs_iomap_begin, 4250 }; 4251