1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * fs/f2fs/data.c 4 * 5 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 6 * http://www.samsung.com/ 7 */ 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/buffer_head.h> 11 #include <linux/mpage.h> 12 #include <linux/writeback.h> 13 #include <linux/backing-dev.h> 14 #include <linux/pagevec.h> 15 #include <linux/blkdev.h> 16 #include <linux/bio.h> 17 #include <linux/blk-crypto.h> 18 #include <linux/swap.h> 19 #include <linux/prefetch.h> 20 #include <linux/uio.h> 21 #include <linux/cleancache.h> 22 #include <linux/sched/signal.h> 23 #include <linux/fiemap.h> 24 25 #include "f2fs.h" 26 #include "node.h" 27 #include "segment.h" 28 #include "trace.h" 29 #include <trace/events/f2fs.h> 30 31 #define NUM_PREALLOC_POST_READ_CTXS 128 32 33 static struct kmem_cache *bio_post_read_ctx_cache; 34 static struct kmem_cache *bio_entry_slab; 35 static mempool_t *bio_post_read_ctx_pool; 36 static struct bio_set f2fs_bioset; 37 38 #define F2FS_BIO_POOL_SIZE NR_CURSEG_TYPE 39 40 int __init f2fs_init_bioset(void) 41 { 42 if (bioset_init(&f2fs_bioset, F2FS_BIO_POOL_SIZE, 43 0, BIOSET_NEED_BVECS)) 44 return -ENOMEM; 45 return 0; 46 } 47 48 void f2fs_destroy_bioset(void) 49 { 50 bioset_exit(&f2fs_bioset); 51 } 52 53 static inline struct bio *__f2fs_bio_alloc(gfp_t gfp_mask, 54 unsigned int nr_iovecs) 55 { 56 return bio_alloc_bioset(gfp_mask, nr_iovecs, &f2fs_bioset); 57 } 58 59 struct bio *f2fs_bio_alloc(struct f2fs_sb_info *sbi, int npages, bool noio) 60 { 61 if (noio) { 62 /* No failure on bio allocation */ 63 return __f2fs_bio_alloc(GFP_NOIO, npages); 64 } 65 66 if (time_to_inject(sbi, FAULT_ALLOC_BIO)) { 67 f2fs_show_injection_info(sbi, FAULT_ALLOC_BIO); 68 return NULL; 69 } 70 71 return __f2fs_bio_alloc(GFP_KERNEL, npages); 72 } 73 74 static bool __is_cp_guaranteed(struct page *page) 75 { 76 struct address_space *mapping = page->mapping; 77 struct inode *inode; 78 struct f2fs_sb_info *sbi; 79 80 if (!mapping) 81 return false; 82 83 if (f2fs_is_compressed_page(page)) 84 return false; 85 86 inode = mapping->host; 87 sbi = F2FS_I_SB(inode); 88 89 if (inode->i_ino == F2FS_META_INO(sbi) || 90 inode->i_ino == F2FS_NODE_INO(sbi) || 91 S_ISDIR(inode->i_mode) || 92 (S_ISREG(inode->i_mode) && 93 (f2fs_is_atomic_file(inode) || IS_NOQUOTA(inode))) || 94 is_cold_data(page)) 95 return true; 96 return false; 97 } 98 99 static enum count_type __read_io_type(struct page *page) 100 { 101 struct address_space *mapping = page_file_mapping(page); 102 103 if (mapping) { 104 struct inode *inode = mapping->host; 105 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 106 107 if (inode->i_ino == F2FS_META_INO(sbi)) 108 return F2FS_RD_META; 109 110 if (inode->i_ino == F2FS_NODE_INO(sbi)) 111 return F2FS_RD_NODE; 112 } 113 return F2FS_RD_DATA; 114 } 115 116 /* postprocessing steps for read bios */ 117 enum bio_post_read_step { 118 STEP_DECRYPT, 119 STEP_DECOMPRESS_NOWQ, /* handle normal cluster data inplace */ 120 STEP_DECOMPRESS, /* handle compressed cluster data in workqueue */ 121 STEP_VERITY, 122 }; 123 124 struct bio_post_read_ctx { 125 struct bio *bio; 126 struct f2fs_sb_info *sbi; 127 struct work_struct work; 128 unsigned int enabled_steps; 129 }; 130 131 static void __read_end_io(struct bio *bio, bool compr, bool verity) 132 { 133 struct page *page; 134 struct bio_vec *bv; 135 struct bvec_iter_all iter_all; 136 137 bio_for_each_segment_all(bv, bio, iter_all) { 138 page = bv->bv_page; 139 140 #ifdef CONFIG_F2FS_FS_COMPRESSION 141 if (compr && f2fs_is_compressed_page(page)) { 142 f2fs_decompress_pages(bio, page, verity); 143 continue; 144 } 145 if (verity) 146 continue; 147 #endif 148 149 /* PG_error was set if any post_read step failed */ 150 if (bio->bi_status || PageError(page)) { 151 ClearPageUptodate(page); 152 /* will re-read again later */ 153 ClearPageError(page); 154 } else { 155 SetPageUptodate(page); 156 } 157 dec_page_count(F2FS_P_SB(page), __read_io_type(page)); 158 unlock_page(page); 159 } 160 } 161 162 static void f2fs_release_read_bio(struct bio *bio); 163 static void __f2fs_read_end_io(struct bio *bio, bool compr, bool verity) 164 { 165 if (!compr) 166 __read_end_io(bio, false, verity); 167 f2fs_release_read_bio(bio); 168 } 169 170 static void f2fs_decompress_bio(struct bio *bio, bool verity) 171 { 172 __read_end_io(bio, true, verity); 173 } 174 175 static void bio_post_read_processing(struct bio_post_read_ctx *ctx); 176 177 static void f2fs_decrypt_work(struct bio_post_read_ctx *ctx) 178 { 179 fscrypt_decrypt_bio(ctx->bio); 180 } 181 182 static void f2fs_decompress_work(struct bio_post_read_ctx *ctx) 183 { 184 f2fs_decompress_bio(ctx->bio, ctx->enabled_steps & (1 << STEP_VERITY)); 185 } 186 187 #ifdef CONFIG_F2FS_FS_COMPRESSION 188 static void f2fs_verify_pages(struct page **rpages, unsigned int cluster_size) 189 { 190 f2fs_decompress_end_io(rpages, cluster_size, false, true); 191 } 192 193 static void f2fs_verify_bio(struct bio *bio) 194 { 195 struct bio_vec *bv; 196 struct bvec_iter_all iter_all; 197 198 bio_for_each_segment_all(bv, bio, iter_all) { 199 struct page *page = bv->bv_page; 200 struct decompress_io_ctx *dic; 201 202 dic = (struct decompress_io_ctx *)page_private(page); 203 204 if (dic) { 205 if (atomic_dec_return(&dic->verity_pages)) 206 continue; 207 f2fs_verify_pages(dic->rpages, 208 dic->cluster_size); 209 f2fs_free_dic(dic); 210 continue; 211 } 212 213 if (bio->bi_status || PageError(page)) 214 goto clear_uptodate; 215 216 if (fsverity_verify_page(page)) { 217 SetPageUptodate(page); 218 goto unlock; 219 } 220 clear_uptodate: 221 ClearPageUptodate(page); 222 ClearPageError(page); 223 unlock: 224 dec_page_count(F2FS_P_SB(page), __read_io_type(page)); 225 unlock_page(page); 226 } 227 } 228 #endif 229 230 static void f2fs_verity_work(struct work_struct *work) 231 { 232 struct bio_post_read_ctx *ctx = 233 container_of(work, struct bio_post_read_ctx, work); 234 struct bio *bio = ctx->bio; 235 #ifdef CONFIG_F2FS_FS_COMPRESSION 236 unsigned int enabled_steps = ctx->enabled_steps; 237 #endif 238 239 /* 240 * fsverity_verify_bio() may call readpages() again, and while verity 241 * will be disabled for this, decryption may still be needed, resulting 242 * in another bio_post_read_ctx being allocated. So to prevent 243 * deadlocks we need to release the current ctx to the mempool first. 244 * This assumes that verity is the last post-read step. 245 */ 246 mempool_free(ctx, bio_post_read_ctx_pool); 247 bio->bi_private = NULL; 248 249 #ifdef CONFIG_F2FS_FS_COMPRESSION 250 /* previous step is decompression */ 251 if (enabled_steps & (1 << STEP_DECOMPRESS)) { 252 f2fs_verify_bio(bio); 253 f2fs_release_read_bio(bio); 254 return; 255 } 256 #endif 257 258 fsverity_verify_bio(bio); 259 __f2fs_read_end_io(bio, false, false); 260 } 261 262 static void f2fs_post_read_work(struct work_struct *work) 263 { 264 struct bio_post_read_ctx *ctx = 265 container_of(work, struct bio_post_read_ctx, work); 266 267 if (ctx->enabled_steps & (1 << STEP_DECRYPT)) 268 f2fs_decrypt_work(ctx); 269 270 if (ctx->enabled_steps & (1 << STEP_DECOMPRESS)) 271 f2fs_decompress_work(ctx); 272 273 if (ctx->enabled_steps & (1 << STEP_VERITY)) { 274 INIT_WORK(&ctx->work, f2fs_verity_work); 275 fsverity_enqueue_verify_work(&ctx->work); 276 return; 277 } 278 279 __f2fs_read_end_io(ctx->bio, 280 ctx->enabled_steps & (1 << STEP_DECOMPRESS), false); 281 } 282 283 static void f2fs_enqueue_post_read_work(struct f2fs_sb_info *sbi, 284 struct work_struct *work) 285 { 286 queue_work(sbi->post_read_wq, work); 287 } 288 289 static void bio_post_read_processing(struct bio_post_read_ctx *ctx) 290 { 291 /* 292 * We use different work queues for decryption and for verity because 293 * verity may require reading metadata pages that need decryption, and 294 * we shouldn't recurse to the same workqueue. 295 */ 296 297 if (ctx->enabled_steps & (1 << STEP_DECRYPT) || 298 ctx->enabled_steps & (1 << STEP_DECOMPRESS)) { 299 INIT_WORK(&ctx->work, f2fs_post_read_work); 300 f2fs_enqueue_post_read_work(ctx->sbi, &ctx->work); 301 return; 302 } 303 304 if (ctx->enabled_steps & (1 << STEP_VERITY)) { 305 INIT_WORK(&ctx->work, f2fs_verity_work); 306 fsverity_enqueue_verify_work(&ctx->work); 307 return; 308 } 309 310 __f2fs_read_end_io(ctx->bio, false, false); 311 } 312 313 static bool f2fs_bio_post_read_required(struct bio *bio) 314 { 315 return bio->bi_private; 316 } 317 318 static void f2fs_read_end_io(struct bio *bio) 319 { 320 struct f2fs_sb_info *sbi = F2FS_P_SB(bio_first_page_all(bio)); 321 322 if (time_to_inject(sbi, FAULT_READ_IO)) { 323 f2fs_show_injection_info(sbi, FAULT_READ_IO); 324 bio->bi_status = BLK_STS_IOERR; 325 } 326 327 if (f2fs_bio_post_read_required(bio)) { 328 struct bio_post_read_ctx *ctx = bio->bi_private; 329 330 bio_post_read_processing(ctx); 331 return; 332 } 333 334 __f2fs_read_end_io(bio, false, false); 335 } 336 337 static void f2fs_write_end_io(struct bio *bio) 338 { 339 struct f2fs_sb_info *sbi = bio->bi_private; 340 struct bio_vec *bvec; 341 struct bvec_iter_all iter_all; 342 343 if (time_to_inject(sbi, FAULT_WRITE_IO)) { 344 f2fs_show_injection_info(sbi, FAULT_WRITE_IO); 345 bio->bi_status = BLK_STS_IOERR; 346 } 347 348 bio_for_each_segment_all(bvec, bio, iter_all) { 349 struct page *page = bvec->bv_page; 350 enum count_type type = WB_DATA_TYPE(page); 351 352 if (IS_DUMMY_WRITTEN_PAGE(page)) { 353 set_page_private(page, (unsigned long)NULL); 354 ClearPagePrivate(page); 355 unlock_page(page); 356 mempool_free(page, sbi->write_io_dummy); 357 358 if (unlikely(bio->bi_status)) 359 f2fs_stop_checkpoint(sbi, true); 360 continue; 361 } 362 363 fscrypt_finalize_bounce_page(&page); 364 365 #ifdef CONFIG_F2FS_FS_COMPRESSION 366 if (f2fs_is_compressed_page(page)) { 367 f2fs_compress_write_end_io(bio, page); 368 continue; 369 } 370 #endif 371 372 if (unlikely(bio->bi_status)) { 373 mapping_set_error(page->mapping, -EIO); 374 if (type == F2FS_WB_CP_DATA) 375 f2fs_stop_checkpoint(sbi, true); 376 } 377 378 f2fs_bug_on(sbi, page->mapping == NODE_MAPPING(sbi) && 379 page->index != nid_of_node(page)); 380 381 dec_page_count(sbi, type); 382 if (f2fs_in_warm_node_list(sbi, page)) 383 f2fs_del_fsync_node_entry(sbi, page); 384 clear_cold_data(page); 385 end_page_writeback(page); 386 } 387 if (!get_pages(sbi, F2FS_WB_CP_DATA) && 388 wq_has_sleeper(&sbi->cp_wait)) 389 wake_up(&sbi->cp_wait); 390 391 bio_put(bio); 392 } 393 394 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi, 395 block_t blk_addr, struct bio *bio) 396 { 397 struct block_device *bdev = sbi->sb->s_bdev; 398 int i; 399 400 if (f2fs_is_multi_device(sbi)) { 401 for (i = 0; i < sbi->s_ndevs; i++) { 402 if (FDEV(i).start_blk <= blk_addr && 403 FDEV(i).end_blk >= blk_addr) { 404 blk_addr -= FDEV(i).start_blk; 405 bdev = FDEV(i).bdev; 406 break; 407 } 408 } 409 } 410 if (bio) { 411 bio_set_dev(bio, bdev); 412 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr); 413 } 414 return bdev; 415 } 416 417 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr) 418 { 419 int i; 420 421 if (!f2fs_is_multi_device(sbi)) 422 return 0; 423 424 for (i = 0; i < sbi->s_ndevs; i++) 425 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr) 426 return i; 427 return 0; 428 } 429 430 /* 431 * Return true, if pre_bio's bdev is same as its target device. 432 */ 433 static bool __same_bdev(struct f2fs_sb_info *sbi, 434 block_t blk_addr, struct bio *bio) 435 { 436 struct block_device *b = f2fs_target_device(sbi, blk_addr, NULL); 437 return bio->bi_disk == b->bd_disk && bio->bi_partno == b->bd_partno; 438 } 439 440 static struct bio *__bio_alloc(struct f2fs_io_info *fio, int npages) 441 { 442 struct f2fs_sb_info *sbi = fio->sbi; 443 struct bio *bio; 444 445 bio = f2fs_bio_alloc(sbi, npages, true); 446 447 f2fs_target_device(sbi, fio->new_blkaddr, bio); 448 if (is_read_io(fio->op)) { 449 bio->bi_end_io = f2fs_read_end_io; 450 bio->bi_private = NULL; 451 } else { 452 bio->bi_end_io = f2fs_write_end_io; 453 bio->bi_private = sbi; 454 bio->bi_write_hint = f2fs_io_type_to_rw_hint(sbi, 455 fio->type, fio->temp); 456 } 457 if (fio->io_wbc) 458 wbc_init_bio(fio->io_wbc, bio); 459 460 return bio; 461 } 462 463 static void f2fs_set_bio_crypt_ctx(struct bio *bio, const struct inode *inode, 464 pgoff_t first_idx, 465 const struct f2fs_io_info *fio, 466 gfp_t gfp_mask) 467 { 468 /* 469 * The f2fs garbage collector sets ->encrypted_page when it wants to 470 * read/write raw data without encryption. 471 */ 472 if (!fio || !fio->encrypted_page) 473 fscrypt_set_bio_crypt_ctx(bio, inode, first_idx, gfp_mask); 474 } 475 476 static bool f2fs_crypt_mergeable_bio(struct bio *bio, const struct inode *inode, 477 pgoff_t next_idx, 478 const struct f2fs_io_info *fio) 479 { 480 /* 481 * The f2fs garbage collector sets ->encrypted_page when it wants to 482 * read/write raw data without encryption. 483 */ 484 if (fio && fio->encrypted_page) 485 return !bio_has_crypt_ctx(bio); 486 487 return fscrypt_mergeable_bio(bio, inode, next_idx); 488 } 489 490 static inline void __submit_bio(struct f2fs_sb_info *sbi, 491 struct bio *bio, enum page_type type) 492 { 493 if (!is_read_io(bio_op(bio))) { 494 unsigned int start; 495 496 if (type != DATA && type != NODE) 497 goto submit_io; 498 499 if (f2fs_lfs_mode(sbi) && current->plug) 500 blk_finish_plug(current->plug); 501 502 if (F2FS_IO_ALIGNED(sbi)) 503 goto submit_io; 504 505 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS; 506 start %= F2FS_IO_SIZE(sbi); 507 508 if (start == 0) 509 goto submit_io; 510 511 /* fill dummy pages */ 512 for (; start < F2FS_IO_SIZE(sbi); start++) { 513 struct page *page = 514 mempool_alloc(sbi->write_io_dummy, 515 GFP_NOIO | __GFP_NOFAIL); 516 f2fs_bug_on(sbi, !page); 517 518 zero_user_segment(page, 0, PAGE_SIZE); 519 SetPagePrivate(page); 520 set_page_private(page, DUMMY_WRITTEN_PAGE); 521 lock_page(page); 522 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) 523 f2fs_bug_on(sbi, 1); 524 } 525 /* 526 * In the NODE case, we lose next block address chain. So, we 527 * need to do checkpoint in f2fs_sync_file. 528 */ 529 if (type == NODE) 530 set_sbi_flag(sbi, SBI_NEED_CP); 531 } 532 submit_io: 533 if (is_read_io(bio_op(bio))) 534 trace_f2fs_submit_read_bio(sbi->sb, type, bio); 535 else 536 trace_f2fs_submit_write_bio(sbi->sb, type, bio); 537 submit_bio(bio); 538 } 539 540 void f2fs_submit_bio(struct f2fs_sb_info *sbi, 541 struct bio *bio, enum page_type type) 542 { 543 __submit_bio(sbi, bio, type); 544 } 545 546 static void __attach_io_flag(struct f2fs_io_info *fio) 547 { 548 struct f2fs_sb_info *sbi = fio->sbi; 549 unsigned int temp_mask = (1 << NR_TEMP_TYPE) - 1; 550 unsigned int io_flag, fua_flag, meta_flag; 551 552 if (fio->type == DATA) 553 io_flag = sbi->data_io_flag; 554 else if (fio->type == NODE) 555 io_flag = sbi->node_io_flag; 556 else 557 return; 558 559 fua_flag = io_flag & temp_mask; 560 meta_flag = (io_flag >> NR_TEMP_TYPE) & temp_mask; 561 562 /* 563 * data/node io flag bits per temp: 564 * REQ_META | REQ_FUA | 565 * 5 | 4 | 3 | 2 | 1 | 0 | 566 * Cold | Warm | Hot | Cold | Warm | Hot | 567 */ 568 if ((1 << fio->temp) & meta_flag) 569 fio->op_flags |= REQ_META; 570 if ((1 << fio->temp) & fua_flag) 571 fio->op_flags |= REQ_FUA; 572 } 573 574 static void __submit_merged_bio(struct f2fs_bio_info *io) 575 { 576 struct f2fs_io_info *fio = &io->fio; 577 578 if (!io->bio) 579 return; 580 581 __attach_io_flag(fio); 582 bio_set_op_attrs(io->bio, fio->op, fio->op_flags); 583 584 if (is_read_io(fio->op)) 585 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio); 586 else 587 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio); 588 589 __submit_bio(io->sbi, io->bio, fio->type); 590 io->bio = NULL; 591 } 592 593 static bool __has_merged_page(struct bio *bio, struct inode *inode, 594 struct page *page, nid_t ino) 595 { 596 struct bio_vec *bvec; 597 struct bvec_iter_all iter_all; 598 599 if (!bio) 600 return false; 601 602 if (!inode && !page && !ino) 603 return true; 604 605 bio_for_each_segment_all(bvec, bio, iter_all) { 606 struct page *target = bvec->bv_page; 607 608 if (fscrypt_is_bounce_page(target)) { 609 target = fscrypt_pagecache_page(target); 610 if (IS_ERR(target)) 611 continue; 612 } 613 if (f2fs_is_compressed_page(target)) { 614 target = f2fs_compress_control_page(target); 615 if (IS_ERR(target)) 616 continue; 617 } 618 619 if (inode && inode == target->mapping->host) 620 return true; 621 if (page && page == target) 622 return true; 623 if (ino && ino == ino_of_node(target)) 624 return true; 625 } 626 627 return false; 628 } 629 630 static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi, 631 enum page_type type, enum temp_type temp) 632 { 633 enum page_type btype = PAGE_TYPE_OF_BIO(type); 634 struct f2fs_bio_info *io = sbi->write_io[btype] + temp; 635 636 down_write(&io->io_rwsem); 637 638 /* change META to META_FLUSH in the checkpoint procedure */ 639 if (type >= META_FLUSH) { 640 io->fio.type = META_FLUSH; 641 io->fio.op = REQ_OP_WRITE; 642 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC; 643 if (!test_opt(sbi, NOBARRIER)) 644 io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA; 645 } 646 __submit_merged_bio(io); 647 up_write(&io->io_rwsem); 648 } 649 650 static void __submit_merged_write_cond(struct f2fs_sb_info *sbi, 651 struct inode *inode, struct page *page, 652 nid_t ino, enum page_type type, bool force) 653 { 654 enum temp_type temp; 655 bool ret = true; 656 657 for (temp = HOT; temp < NR_TEMP_TYPE; temp++) { 658 if (!force) { 659 enum page_type btype = PAGE_TYPE_OF_BIO(type); 660 struct f2fs_bio_info *io = sbi->write_io[btype] + temp; 661 662 down_read(&io->io_rwsem); 663 ret = __has_merged_page(io->bio, inode, page, ino); 664 up_read(&io->io_rwsem); 665 } 666 if (ret) 667 __f2fs_submit_merged_write(sbi, type, temp); 668 669 /* TODO: use HOT temp only for meta pages now. */ 670 if (type >= META) 671 break; 672 } 673 } 674 675 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type) 676 { 677 __submit_merged_write_cond(sbi, NULL, NULL, 0, type, true); 678 } 679 680 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi, 681 struct inode *inode, struct page *page, 682 nid_t ino, enum page_type type) 683 { 684 __submit_merged_write_cond(sbi, inode, page, ino, type, false); 685 } 686 687 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi) 688 { 689 f2fs_submit_merged_write(sbi, DATA); 690 f2fs_submit_merged_write(sbi, NODE); 691 f2fs_submit_merged_write(sbi, META); 692 } 693 694 /* 695 * Fill the locked page with data located in the block address. 696 * A caller needs to unlock the page on failure. 697 */ 698 int f2fs_submit_page_bio(struct f2fs_io_info *fio) 699 { 700 struct bio *bio; 701 struct page *page = fio->encrypted_page ? 702 fio->encrypted_page : fio->page; 703 704 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, 705 fio->is_por ? META_POR : (__is_meta_io(fio) ? 706 META_GENERIC : DATA_GENERIC_ENHANCE))) 707 return -EFSCORRUPTED; 708 709 trace_f2fs_submit_page_bio(page, fio); 710 f2fs_trace_ios(fio, 0); 711 712 /* Allocate a new bio */ 713 bio = __bio_alloc(fio, 1); 714 715 f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, 716 fio->page->index, fio, GFP_NOIO); 717 718 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { 719 bio_put(bio); 720 return -EFAULT; 721 } 722 723 if (fio->io_wbc && !is_read_io(fio->op)) 724 wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE); 725 726 __attach_io_flag(fio); 727 bio_set_op_attrs(bio, fio->op, fio->op_flags); 728 729 inc_page_count(fio->sbi, is_read_io(fio->op) ? 730 __read_io_type(page): WB_DATA_TYPE(fio->page)); 731 732 __submit_bio(fio->sbi, bio, fio->type); 733 return 0; 734 } 735 736 static bool page_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, 737 block_t last_blkaddr, block_t cur_blkaddr) 738 { 739 if (unlikely(sbi->max_io_bytes && 740 bio->bi_iter.bi_size >= sbi->max_io_bytes)) 741 return false; 742 if (last_blkaddr + 1 != cur_blkaddr) 743 return false; 744 return __same_bdev(sbi, cur_blkaddr, bio); 745 } 746 747 static bool io_type_is_mergeable(struct f2fs_bio_info *io, 748 struct f2fs_io_info *fio) 749 { 750 if (io->fio.op != fio->op) 751 return false; 752 return io->fio.op_flags == fio->op_flags; 753 } 754 755 static bool io_is_mergeable(struct f2fs_sb_info *sbi, struct bio *bio, 756 struct f2fs_bio_info *io, 757 struct f2fs_io_info *fio, 758 block_t last_blkaddr, 759 block_t cur_blkaddr) 760 { 761 if (F2FS_IO_ALIGNED(sbi) && (fio->type == DATA || fio->type == NODE)) { 762 unsigned int filled_blocks = 763 F2FS_BYTES_TO_BLK(bio->bi_iter.bi_size); 764 unsigned int io_size = F2FS_IO_SIZE(sbi); 765 unsigned int left_vecs = bio->bi_max_vecs - bio->bi_vcnt; 766 767 /* IOs in bio is aligned and left space of vectors is not enough */ 768 if (!(filled_blocks % io_size) && left_vecs < io_size) 769 return false; 770 } 771 if (!page_is_mergeable(sbi, bio, last_blkaddr, cur_blkaddr)) 772 return false; 773 return io_type_is_mergeable(io, fio); 774 } 775 776 static void add_bio_entry(struct f2fs_sb_info *sbi, struct bio *bio, 777 struct page *page, enum temp_type temp) 778 { 779 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; 780 struct bio_entry *be; 781 782 be = f2fs_kmem_cache_alloc(bio_entry_slab, GFP_NOFS); 783 be->bio = bio; 784 bio_get(bio); 785 786 if (bio_add_page(bio, page, PAGE_SIZE, 0) != PAGE_SIZE) 787 f2fs_bug_on(sbi, 1); 788 789 down_write(&io->bio_list_lock); 790 list_add_tail(&be->list, &io->bio_list); 791 up_write(&io->bio_list_lock); 792 } 793 794 static void del_bio_entry(struct bio_entry *be) 795 { 796 list_del(&be->list); 797 kmem_cache_free(bio_entry_slab, be); 798 } 799 800 static int add_ipu_page(struct f2fs_io_info *fio, struct bio **bio, 801 struct page *page) 802 { 803 struct f2fs_sb_info *sbi = fio->sbi; 804 enum temp_type temp; 805 bool found = false; 806 int ret = -EAGAIN; 807 808 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) { 809 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; 810 struct list_head *head = &io->bio_list; 811 struct bio_entry *be; 812 813 down_write(&io->bio_list_lock); 814 list_for_each_entry(be, head, list) { 815 if (be->bio != *bio) 816 continue; 817 818 found = true; 819 820 f2fs_bug_on(sbi, !page_is_mergeable(sbi, *bio, 821 *fio->last_block, 822 fio->new_blkaddr)); 823 if (f2fs_crypt_mergeable_bio(*bio, 824 fio->page->mapping->host, 825 fio->page->index, fio) && 826 bio_add_page(*bio, page, PAGE_SIZE, 0) == 827 PAGE_SIZE) { 828 ret = 0; 829 break; 830 } 831 832 /* page can't be merged into bio; submit the bio */ 833 del_bio_entry(be); 834 __submit_bio(sbi, *bio, DATA); 835 break; 836 } 837 up_write(&io->bio_list_lock); 838 } 839 840 if (ret) { 841 bio_put(*bio); 842 *bio = NULL; 843 } 844 845 return ret; 846 } 847 848 void f2fs_submit_merged_ipu_write(struct f2fs_sb_info *sbi, 849 struct bio **bio, struct page *page) 850 { 851 enum temp_type temp; 852 bool found = false; 853 struct bio *target = bio ? *bio : NULL; 854 855 for (temp = HOT; temp < NR_TEMP_TYPE && !found; temp++) { 856 struct f2fs_bio_info *io = sbi->write_io[DATA] + temp; 857 struct list_head *head = &io->bio_list; 858 struct bio_entry *be; 859 860 if (list_empty(head)) 861 continue; 862 863 down_read(&io->bio_list_lock); 864 list_for_each_entry(be, head, list) { 865 if (target) 866 found = (target == be->bio); 867 else 868 found = __has_merged_page(be->bio, NULL, 869 page, 0); 870 if (found) 871 break; 872 } 873 up_read(&io->bio_list_lock); 874 875 if (!found) 876 continue; 877 878 found = false; 879 880 down_write(&io->bio_list_lock); 881 list_for_each_entry(be, head, list) { 882 if (target) 883 found = (target == be->bio); 884 else 885 found = __has_merged_page(be->bio, NULL, 886 page, 0); 887 if (found) { 888 target = be->bio; 889 del_bio_entry(be); 890 break; 891 } 892 } 893 up_write(&io->bio_list_lock); 894 } 895 896 if (found) 897 __submit_bio(sbi, target, DATA); 898 if (bio && *bio) { 899 bio_put(*bio); 900 *bio = NULL; 901 } 902 } 903 904 int f2fs_merge_page_bio(struct f2fs_io_info *fio) 905 { 906 struct bio *bio = *fio->bio; 907 struct page *page = fio->encrypted_page ? 908 fio->encrypted_page : fio->page; 909 910 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->new_blkaddr, 911 __is_meta_io(fio) ? META_GENERIC : DATA_GENERIC)) 912 return -EFSCORRUPTED; 913 914 trace_f2fs_submit_page_bio(page, fio); 915 f2fs_trace_ios(fio, 0); 916 917 if (bio && !page_is_mergeable(fio->sbi, bio, *fio->last_block, 918 fio->new_blkaddr)) 919 f2fs_submit_merged_ipu_write(fio->sbi, &bio, NULL); 920 alloc_new: 921 if (!bio) { 922 bio = __bio_alloc(fio, BIO_MAX_PAGES); 923 __attach_io_flag(fio); 924 f2fs_set_bio_crypt_ctx(bio, fio->page->mapping->host, 925 fio->page->index, fio, GFP_NOIO); 926 bio_set_op_attrs(bio, fio->op, fio->op_flags); 927 928 add_bio_entry(fio->sbi, bio, page, fio->temp); 929 } else { 930 if (add_ipu_page(fio, &bio, page)) 931 goto alloc_new; 932 } 933 934 if (fio->io_wbc) 935 wbc_account_cgroup_owner(fio->io_wbc, page, PAGE_SIZE); 936 937 inc_page_count(fio->sbi, WB_DATA_TYPE(page)); 938 939 *fio->last_block = fio->new_blkaddr; 940 *fio->bio = bio; 941 942 return 0; 943 } 944 945 void f2fs_submit_page_write(struct f2fs_io_info *fio) 946 { 947 struct f2fs_sb_info *sbi = fio->sbi; 948 enum page_type btype = PAGE_TYPE_OF_BIO(fio->type); 949 struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp; 950 struct page *bio_page; 951 952 f2fs_bug_on(sbi, is_read_io(fio->op)); 953 954 down_write(&io->io_rwsem); 955 next: 956 if (fio->in_list) { 957 spin_lock(&io->io_lock); 958 if (list_empty(&io->io_list)) { 959 spin_unlock(&io->io_lock); 960 goto out; 961 } 962 fio = list_first_entry(&io->io_list, 963 struct f2fs_io_info, list); 964 list_del(&fio->list); 965 spin_unlock(&io->io_lock); 966 } 967 968 verify_fio_blkaddr(fio); 969 970 if (fio->encrypted_page) 971 bio_page = fio->encrypted_page; 972 else if (fio->compressed_page) 973 bio_page = fio->compressed_page; 974 else 975 bio_page = fio->page; 976 977 /* set submitted = true as a return value */ 978 fio->submitted = true; 979 980 inc_page_count(sbi, WB_DATA_TYPE(bio_page)); 981 982 if (io->bio && 983 (!io_is_mergeable(sbi, io->bio, io, fio, io->last_block_in_bio, 984 fio->new_blkaddr) || 985 !f2fs_crypt_mergeable_bio(io->bio, fio->page->mapping->host, 986 bio_page->index, fio))) 987 __submit_merged_bio(io); 988 alloc_new: 989 if (io->bio == NULL) { 990 if (F2FS_IO_ALIGNED(sbi) && 991 (fio->type == DATA || fio->type == NODE) && 992 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) { 993 dec_page_count(sbi, WB_DATA_TYPE(bio_page)); 994 fio->retry = true; 995 goto skip; 996 } 997 io->bio = __bio_alloc(fio, BIO_MAX_PAGES); 998 f2fs_set_bio_crypt_ctx(io->bio, fio->page->mapping->host, 999 bio_page->index, fio, GFP_NOIO); 1000 io->fio = *fio; 1001 } 1002 1003 if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) { 1004 __submit_merged_bio(io); 1005 goto alloc_new; 1006 } 1007 1008 if (fio->io_wbc) 1009 wbc_account_cgroup_owner(fio->io_wbc, bio_page, PAGE_SIZE); 1010 1011 io->last_block_in_bio = fio->new_blkaddr; 1012 f2fs_trace_ios(fio, 0); 1013 1014 trace_f2fs_submit_page_write(fio->page, fio); 1015 skip: 1016 if (fio->in_list) 1017 goto next; 1018 out: 1019 if (is_sbi_flag_set(sbi, SBI_IS_SHUTDOWN) || 1020 !f2fs_is_checkpoint_ready(sbi)) 1021 __submit_merged_bio(io); 1022 up_write(&io->io_rwsem); 1023 } 1024 1025 static inline bool f2fs_need_verity(const struct inode *inode, pgoff_t idx) 1026 { 1027 return fsverity_active(inode) && 1028 idx < DIV_ROUND_UP(inode->i_size, PAGE_SIZE); 1029 } 1030 1031 static struct bio *f2fs_grab_read_bio(struct inode *inode, block_t blkaddr, 1032 unsigned nr_pages, unsigned op_flag, 1033 pgoff_t first_idx, bool for_write, 1034 bool for_verity) 1035 { 1036 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1037 struct bio *bio; 1038 struct bio_post_read_ctx *ctx; 1039 unsigned int post_read_steps = 0; 1040 1041 bio = f2fs_bio_alloc(sbi, min_t(int, nr_pages, BIO_MAX_PAGES), 1042 for_write); 1043 if (!bio) 1044 return ERR_PTR(-ENOMEM); 1045 1046 f2fs_set_bio_crypt_ctx(bio, inode, first_idx, NULL, GFP_NOFS); 1047 1048 f2fs_target_device(sbi, blkaddr, bio); 1049 bio->bi_end_io = f2fs_read_end_io; 1050 bio_set_op_attrs(bio, REQ_OP_READ, op_flag); 1051 1052 if (fscrypt_inode_uses_fs_layer_crypto(inode)) 1053 post_read_steps |= 1 << STEP_DECRYPT; 1054 if (f2fs_compressed_file(inode)) 1055 post_read_steps |= 1 << STEP_DECOMPRESS_NOWQ; 1056 if (for_verity && f2fs_need_verity(inode, first_idx)) 1057 post_read_steps |= 1 << STEP_VERITY; 1058 1059 if (post_read_steps) { 1060 /* Due to the mempool, this never fails. */ 1061 ctx = mempool_alloc(bio_post_read_ctx_pool, GFP_NOFS); 1062 ctx->bio = bio; 1063 ctx->sbi = sbi; 1064 ctx->enabled_steps = post_read_steps; 1065 bio->bi_private = ctx; 1066 } 1067 1068 return bio; 1069 } 1070 1071 static void f2fs_release_read_bio(struct bio *bio) 1072 { 1073 if (bio->bi_private) 1074 mempool_free(bio->bi_private, bio_post_read_ctx_pool); 1075 bio_put(bio); 1076 } 1077 1078 /* This can handle encryption stuffs */ 1079 static int f2fs_submit_page_read(struct inode *inode, struct page *page, 1080 block_t blkaddr, int op_flags, bool for_write) 1081 { 1082 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1083 struct bio *bio; 1084 1085 bio = f2fs_grab_read_bio(inode, blkaddr, 1, op_flags, 1086 page->index, for_write, true); 1087 if (IS_ERR(bio)) 1088 return PTR_ERR(bio); 1089 1090 /* wait for GCed page writeback via META_MAPPING */ 1091 f2fs_wait_on_block_writeback(inode, blkaddr); 1092 1093 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { 1094 bio_put(bio); 1095 return -EFAULT; 1096 } 1097 ClearPageError(page); 1098 inc_page_count(sbi, F2FS_RD_DATA); 1099 f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE); 1100 __submit_bio(sbi, bio, DATA); 1101 return 0; 1102 } 1103 1104 static void __set_data_blkaddr(struct dnode_of_data *dn) 1105 { 1106 struct f2fs_node *rn = F2FS_NODE(dn->node_page); 1107 __le32 *addr_array; 1108 int base = 0; 1109 1110 if (IS_INODE(dn->node_page) && f2fs_has_extra_attr(dn->inode)) 1111 base = get_extra_isize(dn->inode); 1112 1113 /* Get physical address of data block */ 1114 addr_array = blkaddr_in_node(rn); 1115 addr_array[base + dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr); 1116 } 1117 1118 /* 1119 * Lock ordering for the change of data block address: 1120 * ->data_page 1121 * ->node_page 1122 * update block addresses in the node page 1123 */ 1124 void f2fs_set_data_blkaddr(struct dnode_of_data *dn) 1125 { 1126 f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true); 1127 __set_data_blkaddr(dn); 1128 if (set_page_dirty(dn->node_page)) 1129 dn->node_changed = true; 1130 } 1131 1132 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr) 1133 { 1134 dn->data_blkaddr = blkaddr; 1135 f2fs_set_data_blkaddr(dn); 1136 f2fs_update_extent_cache(dn); 1137 } 1138 1139 /* dn->ofs_in_node will be returned with up-to-date last block pointer */ 1140 int f2fs_reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count) 1141 { 1142 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1143 int err; 1144 1145 if (!count) 1146 return 0; 1147 1148 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) 1149 return -EPERM; 1150 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count)))) 1151 return err; 1152 1153 trace_f2fs_reserve_new_blocks(dn->inode, dn->nid, 1154 dn->ofs_in_node, count); 1155 1156 f2fs_wait_on_page_writeback(dn->node_page, NODE, true, true); 1157 1158 for (; count > 0; dn->ofs_in_node++) { 1159 block_t blkaddr = f2fs_data_blkaddr(dn); 1160 if (blkaddr == NULL_ADDR) { 1161 dn->data_blkaddr = NEW_ADDR; 1162 __set_data_blkaddr(dn); 1163 count--; 1164 } 1165 } 1166 1167 if (set_page_dirty(dn->node_page)) 1168 dn->node_changed = true; 1169 return 0; 1170 } 1171 1172 /* Should keep dn->ofs_in_node unchanged */ 1173 int f2fs_reserve_new_block(struct dnode_of_data *dn) 1174 { 1175 unsigned int ofs_in_node = dn->ofs_in_node; 1176 int ret; 1177 1178 ret = f2fs_reserve_new_blocks(dn, 1); 1179 dn->ofs_in_node = ofs_in_node; 1180 return ret; 1181 } 1182 1183 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index) 1184 { 1185 bool need_put = dn->inode_page ? false : true; 1186 int err; 1187 1188 err = f2fs_get_dnode_of_data(dn, index, ALLOC_NODE); 1189 if (err) 1190 return err; 1191 1192 if (dn->data_blkaddr == NULL_ADDR) 1193 err = f2fs_reserve_new_block(dn); 1194 if (err || need_put) 1195 f2fs_put_dnode(dn); 1196 return err; 1197 } 1198 1199 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index) 1200 { 1201 struct extent_info ei = {0, 0, 0}; 1202 struct inode *inode = dn->inode; 1203 1204 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 1205 dn->data_blkaddr = ei.blk + index - ei.fofs; 1206 return 0; 1207 } 1208 1209 return f2fs_reserve_block(dn, index); 1210 } 1211 1212 struct page *f2fs_get_read_data_page(struct inode *inode, pgoff_t index, 1213 int op_flags, bool for_write) 1214 { 1215 struct address_space *mapping = inode->i_mapping; 1216 struct dnode_of_data dn; 1217 struct page *page; 1218 struct extent_info ei = {0,0,0}; 1219 int err; 1220 1221 page = f2fs_grab_cache_page(mapping, index, for_write); 1222 if (!page) 1223 return ERR_PTR(-ENOMEM); 1224 1225 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 1226 dn.data_blkaddr = ei.blk + index - ei.fofs; 1227 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), dn.data_blkaddr, 1228 DATA_GENERIC_ENHANCE_READ)) { 1229 err = -EFSCORRUPTED; 1230 goto put_err; 1231 } 1232 goto got_it; 1233 } 1234 1235 set_new_dnode(&dn, inode, NULL, NULL, 0); 1236 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 1237 if (err) 1238 goto put_err; 1239 f2fs_put_dnode(&dn); 1240 1241 if (unlikely(dn.data_blkaddr == NULL_ADDR)) { 1242 err = -ENOENT; 1243 goto put_err; 1244 } 1245 if (dn.data_blkaddr != NEW_ADDR && 1246 !f2fs_is_valid_blkaddr(F2FS_I_SB(inode), 1247 dn.data_blkaddr, 1248 DATA_GENERIC_ENHANCE)) { 1249 err = -EFSCORRUPTED; 1250 goto put_err; 1251 } 1252 got_it: 1253 if (PageUptodate(page)) { 1254 unlock_page(page); 1255 return page; 1256 } 1257 1258 /* 1259 * A new dentry page is allocated but not able to be written, since its 1260 * new inode page couldn't be allocated due to -ENOSPC. 1261 * In such the case, its blkaddr can be remained as NEW_ADDR. 1262 * see, f2fs_add_link -> f2fs_get_new_data_page -> 1263 * f2fs_init_inode_metadata. 1264 */ 1265 if (dn.data_blkaddr == NEW_ADDR) { 1266 zero_user_segment(page, 0, PAGE_SIZE); 1267 if (!PageUptodate(page)) 1268 SetPageUptodate(page); 1269 unlock_page(page); 1270 return page; 1271 } 1272 1273 err = f2fs_submit_page_read(inode, page, dn.data_blkaddr, 1274 op_flags, for_write); 1275 if (err) 1276 goto put_err; 1277 return page; 1278 1279 put_err: 1280 f2fs_put_page(page, 1); 1281 return ERR_PTR(err); 1282 } 1283 1284 struct page *f2fs_find_data_page(struct inode *inode, pgoff_t index) 1285 { 1286 struct address_space *mapping = inode->i_mapping; 1287 struct page *page; 1288 1289 page = find_get_page(mapping, index); 1290 if (page && PageUptodate(page)) 1291 return page; 1292 f2fs_put_page(page, 0); 1293 1294 page = f2fs_get_read_data_page(inode, index, 0, false); 1295 if (IS_ERR(page)) 1296 return page; 1297 1298 if (PageUptodate(page)) 1299 return page; 1300 1301 wait_on_page_locked(page); 1302 if (unlikely(!PageUptodate(page))) { 1303 f2fs_put_page(page, 0); 1304 return ERR_PTR(-EIO); 1305 } 1306 return page; 1307 } 1308 1309 /* 1310 * If it tries to access a hole, return an error. 1311 * Because, the callers, functions in dir.c and GC, should be able to know 1312 * whether this page exists or not. 1313 */ 1314 struct page *f2fs_get_lock_data_page(struct inode *inode, pgoff_t index, 1315 bool for_write) 1316 { 1317 struct address_space *mapping = inode->i_mapping; 1318 struct page *page; 1319 repeat: 1320 page = f2fs_get_read_data_page(inode, index, 0, for_write); 1321 if (IS_ERR(page)) 1322 return page; 1323 1324 /* wait for read completion */ 1325 lock_page(page); 1326 if (unlikely(page->mapping != mapping)) { 1327 f2fs_put_page(page, 1); 1328 goto repeat; 1329 } 1330 if (unlikely(!PageUptodate(page))) { 1331 f2fs_put_page(page, 1); 1332 return ERR_PTR(-EIO); 1333 } 1334 return page; 1335 } 1336 1337 /* 1338 * Caller ensures that this data page is never allocated. 1339 * A new zero-filled data page is allocated in the page cache. 1340 * 1341 * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and 1342 * f2fs_unlock_op(). 1343 * Note that, ipage is set only by make_empty_dir, and if any error occur, 1344 * ipage should be released by this function. 1345 */ 1346 struct page *f2fs_get_new_data_page(struct inode *inode, 1347 struct page *ipage, pgoff_t index, bool new_i_size) 1348 { 1349 struct address_space *mapping = inode->i_mapping; 1350 struct page *page; 1351 struct dnode_of_data dn; 1352 int err; 1353 1354 page = f2fs_grab_cache_page(mapping, index, true); 1355 if (!page) { 1356 /* 1357 * before exiting, we should make sure ipage will be released 1358 * if any error occur. 1359 */ 1360 f2fs_put_page(ipage, 1); 1361 return ERR_PTR(-ENOMEM); 1362 } 1363 1364 set_new_dnode(&dn, inode, ipage, NULL, 0); 1365 err = f2fs_reserve_block(&dn, index); 1366 if (err) { 1367 f2fs_put_page(page, 1); 1368 return ERR_PTR(err); 1369 } 1370 if (!ipage) 1371 f2fs_put_dnode(&dn); 1372 1373 if (PageUptodate(page)) 1374 goto got_it; 1375 1376 if (dn.data_blkaddr == NEW_ADDR) { 1377 zero_user_segment(page, 0, PAGE_SIZE); 1378 if (!PageUptodate(page)) 1379 SetPageUptodate(page); 1380 } else { 1381 f2fs_put_page(page, 1); 1382 1383 /* if ipage exists, blkaddr should be NEW_ADDR */ 1384 f2fs_bug_on(F2FS_I_SB(inode), ipage); 1385 page = f2fs_get_lock_data_page(inode, index, true); 1386 if (IS_ERR(page)) 1387 return page; 1388 } 1389 got_it: 1390 if (new_i_size && i_size_read(inode) < 1391 ((loff_t)(index + 1) << PAGE_SHIFT)) 1392 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT)); 1393 return page; 1394 } 1395 1396 static int __allocate_data_block(struct dnode_of_data *dn, int seg_type) 1397 { 1398 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 1399 struct f2fs_summary sum; 1400 struct node_info ni; 1401 block_t old_blkaddr; 1402 blkcnt_t count = 1; 1403 int err; 1404 1405 if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC))) 1406 return -EPERM; 1407 1408 err = f2fs_get_node_info(sbi, dn->nid, &ni); 1409 if (err) 1410 return err; 1411 1412 dn->data_blkaddr = f2fs_data_blkaddr(dn); 1413 if (dn->data_blkaddr != NULL_ADDR) 1414 goto alloc; 1415 1416 if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count)))) 1417 return err; 1418 1419 alloc: 1420 set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version); 1421 old_blkaddr = dn->data_blkaddr; 1422 f2fs_allocate_data_block(sbi, NULL, old_blkaddr, &dn->data_blkaddr, 1423 &sum, seg_type, NULL); 1424 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) 1425 invalidate_mapping_pages(META_MAPPING(sbi), 1426 old_blkaddr, old_blkaddr); 1427 f2fs_update_data_blkaddr(dn, dn->data_blkaddr); 1428 1429 /* 1430 * i_size will be updated by direct_IO. Otherwise, we'll get stale 1431 * data from unwritten block via dio_read. 1432 */ 1433 return 0; 1434 } 1435 1436 int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from) 1437 { 1438 struct inode *inode = file_inode(iocb->ki_filp); 1439 struct f2fs_map_blocks map; 1440 int flag; 1441 int err = 0; 1442 bool direct_io = iocb->ki_flags & IOCB_DIRECT; 1443 1444 map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos); 1445 map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from)); 1446 if (map.m_len > map.m_lblk) 1447 map.m_len -= map.m_lblk; 1448 else 1449 map.m_len = 0; 1450 1451 map.m_next_pgofs = NULL; 1452 map.m_next_extent = NULL; 1453 map.m_seg_type = NO_CHECK_TYPE; 1454 map.m_may_create = true; 1455 1456 if (direct_io) { 1457 map.m_seg_type = f2fs_rw_hint_to_seg_type(iocb->ki_hint); 1458 flag = f2fs_force_buffered_io(inode, iocb, from) ? 1459 F2FS_GET_BLOCK_PRE_AIO : 1460 F2FS_GET_BLOCK_PRE_DIO; 1461 goto map_blocks; 1462 } 1463 if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA(inode)) { 1464 err = f2fs_convert_inline_inode(inode); 1465 if (err) 1466 return err; 1467 } 1468 if (f2fs_has_inline_data(inode)) 1469 return err; 1470 1471 flag = F2FS_GET_BLOCK_PRE_AIO; 1472 1473 map_blocks: 1474 err = f2fs_map_blocks(inode, &map, 1, flag); 1475 if (map.m_len > 0 && err == -ENOSPC) { 1476 if (!direct_io) 1477 set_inode_flag(inode, FI_NO_PREALLOC); 1478 err = 0; 1479 } 1480 return err; 1481 } 1482 1483 void f2fs_do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock) 1484 { 1485 if (flag == F2FS_GET_BLOCK_PRE_AIO) { 1486 if (lock) 1487 down_read(&sbi->node_change); 1488 else 1489 up_read(&sbi->node_change); 1490 } else { 1491 if (lock) 1492 f2fs_lock_op(sbi); 1493 else 1494 f2fs_unlock_op(sbi); 1495 } 1496 } 1497 1498 /* 1499 * f2fs_map_blocks() tries to find or build mapping relationship which 1500 * maps continuous logical blocks to physical blocks, and return such 1501 * info via f2fs_map_blocks structure. 1502 */ 1503 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map, 1504 int create, int flag) 1505 { 1506 unsigned int maxblocks = map->m_len; 1507 struct dnode_of_data dn; 1508 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1509 int mode = map->m_may_create ? ALLOC_NODE : LOOKUP_NODE; 1510 pgoff_t pgofs, end_offset, end; 1511 int err = 0, ofs = 1; 1512 unsigned int ofs_in_node, last_ofs_in_node; 1513 blkcnt_t prealloc; 1514 struct extent_info ei = {0,0,0}; 1515 block_t blkaddr; 1516 unsigned int start_pgofs; 1517 1518 if (!maxblocks) 1519 return 0; 1520 1521 map->m_len = 0; 1522 map->m_flags = 0; 1523 1524 /* it only supports block size == page size */ 1525 pgofs = (pgoff_t)map->m_lblk; 1526 end = pgofs + maxblocks; 1527 1528 if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) { 1529 if (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO && 1530 map->m_may_create) 1531 goto next_dnode; 1532 1533 map->m_pblk = ei.blk + pgofs - ei.fofs; 1534 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs); 1535 map->m_flags = F2FS_MAP_MAPPED; 1536 if (map->m_next_extent) 1537 *map->m_next_extent = pgofs + map->m_len; 1538 1539 /* for hardware encryption, but to avoid potential issue in future */ 1540 if (flag == F2FS_GET_BLOCK_DIO) 1541 f2fs_wait_on_block_writeback_range(inode, 1542 map->m_pblk, map->m_len); 1543 goto out; 1544 } 1545 1546 next_dnode: 1547 if (map->m_may_create) 1548 f2fs_do_map_lock(sbi, flag, true); 1549 1550 /* When reading holes, we need its node page */ 1551 set_new_dnode(&dn, inode, NULL, NULL, 0); 1552 err = f2fs_get_dnode_of_data(&dn, pgofs, mode); 1553 if (err) { 1554 if (flag == F2FS_GET_BLOCK_BMAP) 1555 map->m_pblk = 0; 1556 if (err == -ENOENT) { 1557 err = 0; 1558 if (map->m_next_pgofs) 1559 *map->m_next_pgofs = 1560 f2fs_get_next_page_offset(&dn, pgofs); 1561 if (map->m_next_extent) 1562 *map->m_next_extent = 1563 f2fs_get_next_page_offset(&dn, pgofs); 1564 } 1565 goto unlock_out; 1566 } 1567 1568 start_pgofs = pgofs; 1569 prealloc = 0; 1570 last_ofs_in_node = ofs_in_node = dn.ofs_in_node; 1571 end_offset = ADDRS_PER_PAGE(dn.node_page, inode); 1572 1573 next_block: 1574 blkaddr = f2fs_data_blkaddr(&dn); 1575 1576 if (__is_valid_data_blkaddr(blkaddr) && 1577 !f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE)) { 1578 err = -EFSCORRUPTED; 1579 goto sync_out; 1580 } 1581 1582 if (__is_valid_data_blkaddr(blkaddr)) { 1583 /* use out-place-update for driect IO under LFS mode */ 1584 if (f2fs_lfs_mode(sbi) && flag == F2FS_GET_BLOCK_DIO && 1585 map->m_may_create) { 1586 err = __allocate_data_block(&dn, map->m_seg_type); 1587 if (err) 1588 goto sync_out; 1589 blkaddr = dn.data_blkaddr; 1590 set_inode_flag(inode, FI_APPEND_WRITE); 1591 } 1592 } else { 1593 if (create) { 1594 if (unlikely(f2fs_cp_error(sbi))) { 1595 err = -EIO; 1596 goto sync_out; 1597 } 1598 if (flag == F2FS_GET_BLOCK_PRE_AIO) { 1599 if (blkaddr == NULL_ADDR) { 1600 prealloc++; 1601 last_ofs_in_node = dn.ofs_in_node; 1602 } 1603 } else { 1604 WARN_ON(flag != F2FS_GET_BLOCK_PRE_DIO && 1605 flag != F2FS_GET_BLOCK_DIO); 1606 err = __allocate_data_block(&dn, 1607 map->m_seg_type); 1608 if (!err) 1609 set_inode_flag(inode, FI_APPEND_WRITE); 1610 } 1611 if (err) 1612 goto sync_out; 1613 map->m_flags |= F2FS_MAP_NEW; 1614 blkaddr = dn.data_blkaddr; 1615 } else { 1616 if (flag == F2FS_GET_BLOCK_BMAP) { 1617 map->m_pblk = 0; 1618 goto sync_out; 1619 } 1620 if (flag == F2FS_GET_BLOCK_PRECACHE) 1621 goto sync_out; 1622 if (flag == F2FS_GET_BLOCK_FIEMAP && 1623 blkaddr == NULL_ADDR) { 1624 if (map->m_next_pgofs) 1625 *map->m_next_pgofs = pgofs + 1; 1626 goto sync_out; 1627 } 1628 if (flag != F2FS_GET_BLOCK_FIEMAP) { 1629 /* for defragment case */ 1630 if (map->m_next_pgofs) 1631 *map->m_next_pgofs = pgofs + 1; 1632 goto sync_out; 1633 } 1634 } 1635 } 1636 1637 if (flag == F2FS_GET_BLOCK_PRE_AIO) 1638 goto skip; 1639 1640 if (map->m_len == 0) { 1641 /* preallocated unwritten block should be mapped for fiemap. */ 1642 if (blkaddr == NEW_ADDR) 1643 map->m_flags |= F2FS_MAP_UNWRITTEN; 1644 map->m_flags |= F2FS_MAP_MAPPED; 1645 1646 map->m_pblk = blkaddr; 1647 map->m_len = 1; 1648 } else if ((map->m_pblk != NEW_ADDR && 1649 blkaddr == (map->m_pblk + ofs)) || 1650 (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) || 1651 flag == F2FS_GET_BLOCK_PRE_DIO) { 1652 ofs++; 1653 map->m_len++; 1654 } else { 1655 goto sync_out; 1656 } 1657 1658 skip: 1659 dn.ofs_in_node++; 1660 pgofs++; 1661 1662 /* preallocate blocks in batch for one dnode page */ 1663 if (flag == F2FS_GET_BLOCK_PRE_AIO && 1664 (pgofs == end || dn.ofs_in_node == end_offset)) { 1665 1666 dn.ofs_in_node = ofs_in_node; 1667 err = f2fs_reserve_new_blocks(&dn, prealloc); 1668 if (err) 1669 goto sync_out; 1670 1671 map->m_len += dn.ofs_in_node - ofs_in_node; 1672 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) { 1673 err = -ENOSPC; 1674 goto sync_out; 1675 } 1676 dn.ofs_in_node = end_offset; 1677 } 1678 1679 if (pgofs >= end) 1680 goto sync_out; 1681 else if (dn.ofs_in_node < end_offset) 1682 goto next_block; 1683 1684 if (flag == F2FS_GET_BLOCK_PRECACHE) { 1685 if (map->m_flags & F2FS_MAP_MAPPED) { 1686 unsigned int ofs = start_pgofs - map->m_lblk; 1687 1688 f2fs_update_extent_cache_range(&dn, 1689 start_pgofs, map->m_pblk + ofs, 1690 map->m_len - ofs); 1691 } 1692 } 1693 1694 f2fs_put_dnode(&dn); 1695 1696 if (map->m_may_create) { 1697 f2fs_do_map_lock(sbi, flag, false); 1698 f2fs_balance_fs(sbi, dn.node_changed); 1699 } 1700 goto next_dnode; 1701 1702 sync_out: 1703 1704 /* for hardware encryption, but to avoid potential issue in future */ 1705 if (flag == F2FS_GET_BLOCK_DIO && map->m_flags & F2FS_MAP_MAPPED) 1706 f2fs_wait_on_block_writeback_range(inode, 1707 map->m_pblk, map->m_len); 1708 1709 if (flag == F2FS_GET_BLOCK_PRECACHE) { 1710 if (map->m_flags & F2FS_MAP_MAPPED) { 1711 unsigned int ofs = start_pgofs - map->m_lblk; 1712 1713 f2fs_update_extent_cache_range(&dn, 1714 start_pgofs, map->m_pblk + ofs, 1715 map->m_len - ofs); 1716 } 1717 if (map->m_next_extent) 1718 *map->m_next_extent = pgofs + 1; 1719 } 1720 f2fs_put_dnode(&dn); 1721 unlock_out: 1722 if (map->m_may_create) { 1723 f2fs_do_map_lock(sbi, flag, false); 1724 f2fs_balance_fs(sbi, dn.node_changed); 1725 } 1726 out: 1727 trace_f2fs_map_blocks(inode, map, err); 1728 return err; 1729 } 1730 1731 bool f2fs_overwrite_io(struct inode *inode, loff_t pos, size_t len) 1732 { 1733 struct f2fs_map_blocks map; 1734 block_t last_lblk; 1735 int err; 1736 1737 if (pos + len > i_size_read(inode)) 1738 return false; 1739 1740 map.m_lblk = F2FS_BYTES_TO_BLK(pos); 1741 map.m_next_pgofs = NULL; 1742 map.m_next_extent = NULL; 1743 map.m_seg_type = NO_CHECK_TYPE; 1744 map.m_may_create = false; 1745 last_lblk = F2FS_BLK_ALIGN(pos + len); 1746 1747 while (map.m_lblk < last_lblk) { 1748 map.m_len = last_lblk - map.m_lblk; 1749 err = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_DEFAULT); 1750 if (err || map.m_len == 0) 1751 return false; 1752 map.m_lblk += map.m_len; 1753 } 1754 return true; 1755 } 1756 1757 static inline u64 bytes_to_blks(struct inode *inode, u64 bytes) 1758 { 1759 return (bytes >> inode->i_blkbits); 1760 } 1761 1762 static inline u64 blks_to_bytes(struct inode *inode, u64 blks) 1763 { 1764 return (blks << inode->i_blkbits); 1765 } 1766 1767 static int __get_data_block(struct inode *inode, sector_t iblock, 1768 struct buffer_head *bh, int create, int flag, 1769 pgoff_t *next_pgofs, int seg_type, bool may_write) 1770 { 1771 struct f2fs_map_blocks map; 1772 int err; 1773 1774 map.m_lblk = iblock; 1775 map.m_len = bytes_to_blks(inode, bh->b_size); 1776 map.m_next_pgofs = next_pgofs; 1777 map.m_next_extent = NULL; 1778 map.m_seg_type = seg_type; 1779 map.m_may_create = may_write; 1780 1781 err = f2fs_map_blocks(inode, &map, create, flag); 1782 if (!err) { 1783 map_bh(bh, inode->i_sb, map.m_pblk); 1784 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags; 1785 bh->b_size = blks_to_bytes(inode, map.m_len); 1786 } 1787 return err; 1788 } 1789 1790 static int get_data_block_dio_write(struct inode *inode, sector_t iblock, 1791 struct buffer_head *bh_result, int create) 1792 { 1793 return __get_data_block(inode, iblock, bh_result, create, 1794 F2FS_GET_BLOCK_DIO, NULL, 1795 f2fs_rw_hint_to_seg_type(inode->i_write_hint), 1796 IS_SWAPFILE(inode) ? false : true); 1797 } 1798 1799 static int get_data_block_dio(struct inode *inode, sector_t iblock, 1800 struct buffer_head *bh_result, int create) 1801 { 1802 return __get_data_block(inode, iblock, bh_result, create, 1803 F2FS_GET_BLOCK_DIO, NULL, 1804 f2fs_rw_hint_to_seg_type(inode->i_write_hint), 1805 false); 1806 } 1807 1808 static int f2fs_xattr_fiemap(struct inode *inode, 1809 struct fiemap_extent_info *fieinfo) 1810 { 1811 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1812 struct page *page; 1813 struct node_info ni; 1814 __u64 phys = 0, len; 1815 __u32 flags; 1816 nid_t xnid = F2FS_I(inode)->i_xattr_nid; 1817 int err = 0; 1818 1819 if (f2fs_has_inline_xattr(inode)) { 1820 int offset; 1821 1822 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), 1823 inode->i_ino, false); 1824 if (!page) 1825 return -ENOMEM; 1826 1827 err = f2fs_get_node_info(sbi, inode->i_ino, &ni); 1828 if (err) { 1829 f2fs_put_page(page, 1); 1830 return err; 1831 } 1832 1833 phys = blks_to_bytes(inode, ni.blk_addr); 1834 offset = offsetof(struct f2fs_inode, i_addr) + 1835 sizeof(__le32) * (DEF_ADDRS_PER_INODE - 1836 get_inline_xattr_addrs(inode)); 1837 1838 phys += offset; 1839 len = inline_xattr_size(inode); 1840 1841 f2fs_put_page(page, 1); 1842 1843 flags = FIEMAP_EXTENT_DATA_INLINE | FIEMAP_EXTENT_NOT_ALIGNED; 1844 1845 if (!xnid) 1846 flags |= FIEMAP_EXTENT_LAST; 1847 1848 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); 1849 trace_f2fs_fiemap(inode, 0, phys, len, flags, err); 1850 if (err || err == 1) 1851 return err; 1852 } 1853 1854 if (xnid) { 1855 page = f2fs_grab_cache_page(NODE_MAPPING(sbi), xnid, false); 1856 if (!page) 1857 return -ENOMEM; 1858 1859 err = f2fs_get_node_info(sbi, xnid, &ni); 1860 if (err) { 1861 f2fs_put_page(page, 1); 1862 return err; 1863 } 1864 1865 phys = blks_to_bytes(inode, ni.blk_addr); 1866 len = inode->i_sb->s_blocksize; 1867 1868 f2fs_put_page(page, 1); 1869 1870 flags = FIEMAP_EXTENT_LAST; 1871 } 1872 1873 if (phys) { 1874 err = fiemap_fill_next_extent(fieinfo, 0, phys, len, flags); 1875 trace_f2fs_fiemap(inode, 0, phys, len, flags, err); 1876 } 1877 1878 return (err < 0 ? err : 0); 1879 } 1880 1881 static loff_t max_inode_blocks(struct inode *inode) 1882 { 1883 loff_t result = ADDRS_PER_INODE(inode); 1884 loff_t leaf_count = ADDRS_PER_BLOCK(inode); 1885 1886 /* two direct node blocks */ 1887 result += (leaf_count * 2); 1888 1889 /* two indirect node blocks */ 1890 leaf_count *= NIDS_PER_BLOCK; 1891 result += (leaf_count * 2); 1892 1893 /* one double indirect node block */ 1894 leaf_count *= NIDS_PER_BLOCK; 1895 result += leaf_count; 1896 1897 return result; 1898 } 1899 1900 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, 1901 u64 start, u64 len) 1902 { 1903 struct f2fs_map_blocks map; 1904 sector_t start_blk, last_blk; 1905 pgoff_t next_pgofs; 1906 u64 logical = 0, phys = 0, size = 0; 1907 u32 flags = 0; 1908 int ret = 0; 1909 bool compr_cluster = false; 1910 unsigned int cluster_size = F2FS_I(inode)->i_cluster_size; 1911 1912 if (fieinfo->fi_flags & FIEMAP_FLAG_CACHE) { 1913 ret = f2fs_precache_extents(inode); 1914 if (ret) 1915 return ret; 1916 } 1917 1918 ret = fiemap_prep(inode, fieinfo, start, &len, FIEMAP_FLAG_XATTR); 1919 if (ret) 1920 return ret; 1921 1922 inode_lock(inode); 1923 1924 if (fieinfo->fi_flags & FIEMAP_FLAG_XATTR) { 1925 ret = f2fs_xattr_fiemap(inode, fieinfo); 1926 goto out; 1927 } 1928 1929 if (f2fs_has_inline_data(inode) || f2fs_has_inline_dentry(inode)) { 1930 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len); 1931 if (ret != -EAGAIN) 1932 goto out; 1933 } 1934 1935 if (bytes_to_blks(inode, len) == 0) 1936 len = blks_to_bytes(inode, 1); 1937 1938 start_blk = bytes_to_blks(inode, start); 1939 last_blk = bytes_to_blks(inode, start + len - 1); 1940 1941 next: 1942 memset(&map, 0, sizeof(map)); 1943 map.m_lblk = start_blk; 1944 map.m_len = bytes_to_blks(inode, len); 1945 map.m_next_pgofs = &next_pgofs; 1946 map.m_seg_type = NO_CHECK_TYPE; 1947 1948 if (compr_cluster) 1949 map.m_len = cluster_size - 1; 1950 1951 ret = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_FIEMAP); 1952 if (ret) 1953 goto out; 1954 1955 /* HOLE */ 1956 if (!(map.m_flags & F2FS_MAP_FLAGS)) { 1957 start_blk = next_pgofs; 1958 1959 if (blks_to_bytes(inode, start_blk) < blks_to_bytes(inode, 1960 max_inode_blocks(inode))) 1961 goto prep_next; 1962 1963 flags |= FIEMAP_EXTENT_LAST; 1964 } 1965 1966 if (size) { 1967 if (IS_ENCRYPTED(inode)) 1968 flags |= FIEMAP_EXTENT_DATA_ENCRYPTED; 1969 1970 ret = fiemap_fill_next_extent(fieinfo, logical, 1971 phys, size, flags); 1972 trace_f2fs_fiemap(inode, logical, phys, size, flags, ret); 1973 if (ret) 1974 goto out; 1975 size = 0; 1976 } 1977 1978 if (start_blk > last_blk) 1979 goto out; 1980 1981 if (compr_cluster) { 1982 compr_cluster = false; 1983 1984 1985 logical = blks_to_bytes(inode, start_blk - 1); 1986 phys = blks_to_bytes(inode, map.m_pblk); 1987 size = blks_to_bytes(inode, cluster_size); 1988 1989 flags |= FIEMAP_EXTENT_ENCODED; 1990 1991 start_blk += cluster_size - 1; 1992 1993 if (start_blk > last_blk) 1994 goto out; 1995 1996 goto prep_next; 1997 } 1998 1999 if (map.m_pblk == COMPRESS_ADDR) { 2000 compr_cluster = true; 2001 start_blk++; 2002 goto prep_next; 2003 } 2004 2005 logical = blks_to_bytes(inode, start_blk); 2006 phys = blks_to_bytes(inode, map.m_pblk); 2007 size = blks_to_bytes(inode, map.m_len); 2008 flags = 0; 2009 if (map.m_flags & F2FS_MAP_UNWRITTEN) 2010 flags = FIEMAP_EXTENT_UNWRITTEN; 2011 2012 start_blk += bytes_to_blks(inode, size); 2013 2014 prep_next: 2015 cond_resched(); 2016 if (fatal_signal_pending(current)) 2017 ret = -EINTR; 2018 else 2019 goto next; 2020 out: 2021 if (ret == 1) 2022 ret = 0; 2023 2024 inode_unlock(inode); 2025 return ret; 2026 } 2027 2028 static inline loff_t f2fs_readpage_limit(struct inode *inode) 2029 { 2030 if (IS_ENABLED(CONFIG_FS_VERITY) && 2031 (IS_VERITY(inode) || f2fs_verity_in_progress(inode))) 2032 return inode->i_sb->s_maxbytes; 2033 2034 return i_size_read(inode); 2035 } 2036 2037 static int f2fs_read_single_page(struct inode *inode, struct page *page, 2038 unsigned nr_pages, 2039 struct f2fs_map_blocks *map, 2040 struct bio **bio_ret, 2041 sector_t *last_block_in_bio, 2042 bool is_readahead) 2043 { 2044 struct bio *bio = *bio_ret; 2045 const unsigned blocksize = blks_to_bytes(inode, 1); 2046 sector_t block_in_file; 2047 sector_t last_block; 2048 sector_t last_block_in_file; 2049 sector_t block_nr; 2050 int ret = 0; 2051 2052 block_in_file = (sector_t)page_index(page); 2053 last_block = block_in_file + nr_pages; 2054 last_block_in_file = bytes_to_blks(inode, 2055 f2fs_readpage_limit(inode) + blocksize - 1); 2056 if (last_block > last_block_in_file) 2057 last_block = last_block_in_file; 2058 2059 /* just zeroing out page which is beyond EOF */ 2060 if (block_in_file >= last_block) 2061 goto zero_out; 2062 /* 2063 * Map blocks using the previous result first. 2064 */ 2065 if ((map->m_flags & F2FS_MAP_MAPPED) && 2066 block_in_file > map->m_lblk && 2067 block_in_file < (map->m_lblk + map->m_len)) 2068 goto got_it; 2069 2070 /* 2071 * Then do more f2fs_map_blocks() calls until we are 2072 * done with this page. 2073 */ 2074 map->m_lblk = block_in_file; 2075 map->m_len = last_block - block_in_file; 2076 2077 ret = f2fs_map_blocks(inode, map, 0, F2FS_GET_BLOCK_DEFAULT); 2078 if (ret) 2079 goto out; 2080 got_it: 2081 if ((map->m_flags & F2FS_MAP_MAPPED)) { 2082 block_nr = map->m_pblk + block_in_file - map->m_lblk; 2083 SetPageMappedToDisk(page); 2084 2085 if (!PageUptodate(page) && (!PageSwapCache(page) && 2086 !cleancache_get_page(page))) { 2087 SetPageUptodate(page); 2088 goto confused; 2089 } 2090 2091 if (!f2fs_is_valid_blkaddr(F2FS_I_SB(inode), block_nr, 2092 DATA_GENERIC_ENHANCE_READ)) { 2093 ret = -EFSCORRUPTED; 2094 goto out; 2095 } 2096 } else { 2097 zero_out: 2098 zero_user_segment(page, 0, PAGE_SIZE); 2099 if (f2fs_need_verity(inode, page->index) && 2100 !fsverity_verify_page(page)) { 2101 ret = -EIO; 2102 goto out; 2103 } 2104 if (!PageUptodate(page)) 2105 SetPageUptodate(page); 2106 unlock_page(page); 2107 goto out; 2108 } 2109 2110 /* 2111 * This page will go to BIO. Do we need to send this 2112 * BIO off first? 2113 */ 2114 if (bio && (!page_is_mergeable(F2FS_I_SB(inode), bio, 2115 *last_block_in_bio, block_nr) || 2116 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) { 2117 submit_and_realloc: 2118 __submit_bio(F2FS_I_SB(inode), bio, DATA); 2119 bio = NULL; 2120 } 2121 if (bio == NULL) { 2122 bio = f2fs_grab_read_bio(inode, block_nr, nr_pages, 2123 is_readahead ? REQ_RAHEAD : 0, page->index, 2124 false, true); 2125 if (IS_ERR(bio)) { 2126 ret = PTR_ERR(bio); 2127 bio = NULL; 2128 goto out; 2129 } 2130 } 2131 2132 /* 2133 * If the page is under writeback, we need to wait for 2134 * its completion to see the correct decrypted data. 2135 */ 2136 f2fs_wait_on_block_writeback(inode, block_nr); 2137 2138 if (bio_add_page(bio, page, blocksize, 0) < blocksize) 2139 goto submit_and_realloc; 2140 2141 inc_page_count(F2FS_I_SB(inode), F2FS_RD_DATA); 2142 f2fs_update_iostat(F2FS_I_SB(inode), FS_DATA_READ_IO, F2FS_BLKSIZE); 2143 ClearPageError(page); 2144 *last_block_in_bio = block_nr; 2145 goto out; 2146 confused: 2147 if (bio) { 2148 __submit_bio(F2FS_I_SB(inode), bio, DATA); 2149 bio = NULL; 2150 } 2151 unlock_page(page); 2152 out: 2153 *bio_ret = bio; 2154 return ret; 2155 } 2156 2157 #ifdef CONFIG_F2FS_FS_COMPRESSION 2158 int f2fs_read_multi_pages(struct compress_ctx *cc, struct bio **bio_ret, 2159 unsigned nr_pages, sector_t *last_block_in_bio, 2160 bool is_readahead, bool for_write) 2161 { 2162 struct dnode_of_data dn; 2163 struct inode *inode = cc->inode; 2164 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2165 struct bio *bio = *bio_ret; 2166 unsigned int start_idx = cc->cluster_idx << cc->log_cluster_size; 2167 sector_t last_block_in_file; 2168 const unsigned blocksize = blks_to_bytes(inode, 1); 2169 struct decompress_io_ctx *dic = NULL; 2170 struct bio_post_read_ctx *ctx; 2171 bool for_verity = false; 2172 int i; 2173 int ret = 0; 2174 2175 f2fs_bug_on(sbi, f2fs_cluster_is_empty(cc)); 2176 2177 last_block_in_file = bytes_to_blks(inode, 2178 f2fs_readpage_limit(inode) + blocksize - 1); 2179 2180 /* get rid of pages beyond EOF */ 2181 for (i = 0; i < cc->cluster_size; i++) { 2182 struct page *page = cc->rpages[i]; 2183 2184 if (!page) 2185 continue; 2186 if ((sector_t)page->index >= last_block_in_file) { 2187 zero_user_segment(page, 0, PAGE_SIZE); 2188 if (!PageUptodate(page)) 2189 SetPageUptodate(page); 2190 } else if (!PageUptodate(page)) { 2191 continue; 2192 } 2193 unlock_page(page); 2194 cc->rpages[i] = NULL; 2195 cc->nr_rpages--; 2196 } 2197 2198 /* we are done since all pages are beyond EOF */ 2199 if (f2fs_cluster_is_empty(cc)) 2200 goto out; 2201 2202 set_new_dnode(&dn, inode, NULL, NULL, 0); 2203 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 2204 if (ret) 2205 goto out; 2206 2207 f2fs_bug_on(sbi, dn.data_blkaddr != COMPRESS_ADDR); 2208 2209 for (i = 1; i < cc->cluster_size; i++) { 2210 block_t blkaddr; 2211 2212 blkaddr = data_blkaddr(dn.inode, dn.node_page, 2213 dn.ofs_in_node + i); 2214 2215 if (!__is_valid_data_blkaddr(blkaddr)) 2216 break; 2217 2218 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC)) { 2219 ret = -EFAULT; 2220 goto out_put_dnode; 2221 } 2222 cc->nr_cpages++; 2223 } 2224 2225 /* nothing to decompress */ 2226 if (cc->nr_cpages == 0) { 2227 ret = 0; 2228 goto out_put_dnode; 2229 } 2230 2231 dic = f2fs_alloc_dic(cc); 2232 if (IS_ERR(dic)) { 2233 ret = PTR_ERR(dic); 2234 goto out_put_dnode; 2235 } 2236 2237 /* 2238 * It's possible to enable fsverity on the fly when handling a cluster, 2239 * which requires complicated error handling. Instead of adding more 2240 * complexity, let's give a rule where end_io post-processes fsverity 2241 * per cluster. In order to do that, we need to submit bio, if previous 2242 * bio sets a different post-process policy. 2243 */ 2244 if (fsverity_active(cc->inode)) { 2245 atomic_set(&dic->verity_pages, cc->nr_cpages); 2246 for_verity = true; 2247 2248 if (bio) { 2249 ctx = bio->bi_private; 2250 if (!(ctx->enabled_steps & (1 << STEP_VERITY))) { 2251 __submit_bio(sbi, bio, DATA); 2252 bio = NULL; 2253 } 2254 } 2255 } 2256 2257 for (i = 0; i < dic->nr_cpages; i++) { 2258 struct page *page = dic->cpages[i]; 2259 block_t blkaddr; 2260 2261 blkaddr = data_blkaddr(dn.inode, dn.node_page, 2262 dn.ofs_in_node + i + 1); 2263 2264 if (bio && (!page_is_mergeable(sbi, bio, 2265 *last_block_in_bio, blkaddr) || 2266 !f2fs_crypt_mergeable_bio(bio, inode, page->index, NULL))) { 2267 submit_and_realloc: 2268 __submit_bio(sbi, bio, DATA); 2269 bio = NULL; 2270 } 2271 2272 if (!bio) { 2273 bio = f2fs_grab_read_bio(inode, blkaddr, nr_pages, 2274 is_readahead ? REQ_RAHEAD : 0, 2275 page->index, for_write, for_verity); 2276 if (IS_ERR(bio)) { 2277 unsigned int remained = dic->nr_cpages - i; 2278 bool release = false; 2279 2280 ret = PTR_ERR(bio); 2281 dic->failed = true; 2282 2283 if (for_verity) { 2284 if (!atomic_sub_return(remained, 2285 &dic->verity_pages)) 2286 release = true; 2287 } else { 2288 if (!atomic_sub_return(remained, 2289 &dic->pending_pages)) 2290 release = true; 2291 } 2292 2293 if (release) { 2294 f2fs_decompress_end_io(dic->rpages, 2295 cc->cluster_size, true, 2296 false); 2297 f2fs_free_dic(dic); 2298 } 2299 2300 f2fs_put_dnode(&dn); 2301 *bio_ret = NULL; 2302 return ret; 2303 } 2304 } 2305 2306 f2fs_wait_on_block_writeback(inode, blkaddr); 2307 2308 if (bio_add_page(bio, page, blocksize, 0) < blocksize) 2309 goto submit_and_realloc; 2310 2311 /* tag STEP_DECOMPRESS to handle IO in wq */ 2312 ctx = bio->bi_private; 2313 if (!(ctx->enabled_steps & (1 << STEP_DECOMPRESS))) 2314 ctx->enabled_steps |= 1 << STEP_DECOMPRESS; 2315 2316 inc_page_count(sbi, F2FS_RD_DATA); 2317 f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE); 2318 f2fs_update_iostat(sbi, FS_CDATA_READ_IO, F2FS_BLKSIZE); 2319 ClearPageError(page); 2320 *last_block_in_bio = blkaddr; 2321 } 2322 2323 f2fs_put_dnode(&dn); 2324 2325 *bio_ret = bio; 2326 return 0; 2327 2328 out_put_dnode: 2329 f2fs_put_dnode(&dn); 2330 out: 2331 f2fs_decompress_end_io(cc->rpages, cc->cluster_size, true, false); 2332 *bio_ret = bio; 2333 return ret; 2334 } 2335 #endif 2336 2337 /* 2338 * This function was originally taken from fs/mpage.c, and customized for f2fs. 2339 * Major change was from block_size == page_size in f2fs by default. 2340 * 2341 * Note that the aops->readpages() function is ONLY used for read-ahead. If 2342 * this function ever deviates from doing just read-ahead, it should either 2343 * use ->readpage() or do the necessary surgery to decouple ->readpages() 2344 * from read-ahead. 2345 */ 2346 static int f2fs_mpage_readpages(struct inode *inode, 2347 struct readahead_control *rac, struct page *page) 2348 { 2349 struct bio *bio = NULL; 2350 sector_t last_block_in_bio = 0; 2351 struct f2fs_map_blocks map; 2352 #ifdef CONFIG_F2FS_FS_COMPRESSION 2353 struct compress_ctx cc = { 2354 .inode = inode, 2355 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 2356 .cluster_size = F2FS_I(inode)->i_cluster_size, 2357 .cluster_idx = NULL_CLUSTER, 2358 .rpages = NULL, 2359 .cpages = NULL, 2360 .nr_rpages = 0, 2361 .nr_cpages = 0, 2362 }; 2363 #endif 2364 unsigned nr_pages = rac ? readahead_count(rac) : 1; 2365 unsigned max_nr_pages = nr_pages; 2366 int ret = 0; 2367 bool drop_ra = false; 2368 2369 map.m_pblk = 0; 2370 map.m_lblk = 0; 2371 map.m_len = 0; 2372 map.m_flags = 0; 2373 map.m_next_pgofs = NULL; 2374 map.m_next_extent = NULL; 2375 map.m_seg_type = NO_CHECK_TYPE; 2376 map.m_may_create = false; 2377 2378 /* 2379 * Two readahead threads for same address range can cause race condition 2380 * which fragments sequential read IOs. So let's avoid each other. 2381 */ 2382 if (rac && readahead_count(rac)) { 2383 if (READ_ONCE(F2FS_I(inode)->ra_offset) == readahead_index(rac)) 2384 drop_ra = true; 2385 else 2386 WRITE_ONCE(F2FS_I(inode)->ra_offset, 2387 readahead_index(rac)); 2388 } 2389 2390 for (; nr_pages; nr_pages--) { 2391 if (rac) { 2392 page = readahead_page(rac); 2393 prefetchw(&page->flags); 2394 if (drop_ra) { 2395 f2fs_put_page(page, 1); 2396 continue; 2397 } 2398 } 2399 2400 #ifdef CONFIG_F2FS_FS_COMPRESSION 2401 if (f2fs_compressed_file(inode)) { 2402 /* there are remained comressed pages, submit them */ 2403 if (!f2fs_cluster_can_merge_page(&cc, page->index)) { 2404 ret = f2fs_read_multi_pages(&cc, &bio, 2405 max_nr_pages, 2406 &last_block_in_bio, 2407 rac != NULL, false); 2408 f2fs_destroy_compress_ctx(&cc); 2409 if (ret) 2410 goto set_error_page; 2411 } 2412 ret = f2fs_is_compressed_cluster(inode, page->index); 2413 if (ret < 0) 2414 goto set_error_page; 2415 else if (!ret) 2416 goto read_single_page; 2417 2418 ret = f2fs_init_compress_ctx(&cc); 2419 if (ret) 2420 goto set_error_page; 2421 2422 f2fs_compress_ctx_add_page(&cc, page); 2423 2424 goto next_page; 2425 } 2426 read_single_page: 2427 #endif 2428 2429 ret = f2fs_read_single_page(inode, page, max_nr_pages, &map, 2430 &bio, &last_block_in_bio, rac); 2431 if (ret) { 2432 #ifdef CONFIG_F2FS_FS_COMPRESSION 2433 set_error_page: 2434 #endif 2435 SetPageError(page); 2436 zero_user_segment(page, 0, PAGE_SIZE); 2437 unlock_page(page); 2438 } 2439 #ifdef CONFIG_F2FS_FS_COMPRESSION 2440 next_page: 2441 #endif 2442 if (rac) 2443 put_page(page); 2444 2445 #ifdef CONFIG_F2FS_FS_COMPRESSION 2446 if (f2fs_compressed_file(inode)) { 2447 /* last page */ 2448 if (nr_pages == 1 && !f2fs_cluster_is_empty(&cc)) { 2449 ret = f2fs_read_multi_pages(&cc, &bio, 2450 max_nr_pages, 2451 &last_block_in_bio, 2452 rac != NULL, false); 2453 f2fs_destroy_compress_ctx(&cc); 2454 } 2455 } 2456 #endif 2457 } 2458 if (bio) 2459 __submit_bio(F2FS_I_SB(inode), bio, DATA); 2460 2461 if (rac && readahead_count(rac) && !drop_ra) 2462 WRITE_ONCE(F2FS_I(inode)->ra_offset, -1); 2463 return ret; 2464 } 2465 2466 static int f2fs_read_data_page(struct file *file, struct page *page) 2467 { 2468 struct inode *inode = page_file_mapping(page)->host; 2469 int ret = -EAGAIN; 2470 2471 trace_f2fs_readpage(page, DATA); 2472 2473 if (!f2fs_is_compress_backend_ready(inode)) { 2474 unlock_page(page); 2475 return -EOPNOTSUPP; 2476 } 2477 2478 /* If the file has inline data, try to read it directly */ 2479 if (f2fs_has_inline_data(inode)) 2480 ret = f2fs_read_inline_data(inode, page); 2481 if (ret == -EAGAIN) 2482 ret = f2fs_mpage_readpages(inode, NULL, page); 2483 return ret; 2484 } 2485 2486 static void f2fs_readahead(struct readahead_control *rac) 2487 { 2488 struct inode *inode = rac->mapping->host; 2489 2490 trace_f2fs_readpages(inode, readahead_index(rac), readahead_count(rac)); 2491 2492 if (!f2fs_is_compress_backend_ready(inode)) 2493 return; 2494 2495 /* If the file has inline data, skip readpages */ 2496 if (f2fs_has_inline_data(inode)) 2497 return; 2498 2499 f2fs_mpage_readpages(inode, rac, NULL); 2500 } 2501 2502 int f2fs_encrypt_one_page(struct f2fs_io_info *fio) 2503 { 2504 struct inode *inode = fio->page->mapping->host; 2505 struct page *mpage, *page; 2506 gfp_t gfp_flags = GFP_NOFS; 2507 2508 if (!f2fs_encrypted_file(inode)) 2509 return 0; 2510 2511 page = fio->compressed_page ? fio->compressed_page : fio->page; 2512 2513 /* wait for GCed page writeback via META_MAPPING */ 2514 f2fs_wait_on_block_writeback(inode, fio->old_blkaddr); 2515 2516 if (fscrypt_inode_uses_inline_crypto(inode)) 2517 return 0; 2518 2519 retry_encrypt: 2520 fio->encrypted_page = fscrypt_encrypt_pagecache_blocks(page, 2521 PAGE_SIZE, 0, gfp_flags); 2522 if (IS_ERR(fio->encrypted_page)) { 2523 /* flush pending IOs and wait for a while in the ENOMEM case */ 2524 if (PTR_ERR(fio->encrypted_page) == -ENOMEM) { 2525 f2fs_flush_merged_writes(fio->sbi); 2526 congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT); 2527 gfp_flags |= __GFP_NOFAIL; 2528 goto retry_encrypt; 2529 } 2530 return PTR_ERR(fio->encrypted_page); 2531 } 2532 2533 mpage = find_lock_page(META_MAPPING(fio->sbi), fio->old_blkaddr); 2534 if (mpage) { 2535 if (PageUptodate(mpage)) 2536 memcpy(page_address(mpage), 2537 page_address(fio->encrypted_page), PAGE_SIZE); 2538 f2fs_put_page(mpage, 1); 2539 } 2540 return 0; 2541 } 2542 2543 static inline bool check_inplace_update_policy(struct inode *inode, 2544 struct f2fs_io_info *fio) 2545 { 2546 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2547 unsigned int policy = SM_I(sbi)->ipu_policy; 2548 2549 if (policy & (0x1 << F2FS_IPU_FORCE)) 2550 return true; 2551 if (policy & (0x1 << F2FS_IPU_SSR) && f2fs_need_SSR(sbi)) 2552 return true; 2553 if (policy & (0x1 << F2FS_IPU_UTIL) && 2554 utilization(sbi) > SM_I(sbi)->min_ipu_util) 2555 return true; 2556 if (policy & (0x1 << F2FS_IPU_SSR_UTIL) && f2fs_need_SSR(sbi) && 2557 utilization(sbi) > SM_I(sbi)->min_ipu_util) 2558 return true; 2559 2560 /* 2561 * IPU for rewrite async pages 2562 */ 2563 if (policy & (0x1 << F2FS_IPU_ASYNC) && 2564 fio && fio->op == REQ_OP_WRITE && 2565 !(fio->op_flags & REQ_SYNC) && 2566 !IS_ENCRYPTED(inode)) 2567 return true; 2568 2569 /* this is only set during fdatasync */ 2570 if (policy & (0x1 << F2FS_IPU_FSYNC) && 2571 is_inode_flag_set(inode, FI_NEED_IPU)) 2572 return true; 2573 2574 if (unlikely(fio && is_sbi_flag_set(sbi, SBI_CP_DISABLED) && 2575 !f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) 2576 return true; 2577 2578 return false; 2579 } 2580 2581 bool f2fs_should_update_inplace(struct inode *inode, struct f2fs_io_info *fio) 2582 { 2583 if (f2fs_is_pinned_file(inode)) 2584 return true; 2585 2586 /* if this is cold file, we should overwrite to avoid fragmentation */ 2587 if (file_is_cold(inode)) 2588 return true; 2589 2590 return check_inplace_update_policy(inode, fio); 2591 } 2592 2593 bool f2fs_should_update_outplace(struct inode *inode, struct f2fs_io_info *fio) 2594 { 2595 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2596 2597 if (f2fs_lfs_mode(sbi)) 2598 return true; 2599 if (S_ISDIR(inode->i_mode)) 2600 return true; 2601 if (IS_NOQUOTA(inode)) 2602 return true; 2603 if (f2fs_is_atomic_file(inode)) 2604 return true; 2605 if (fio) { 2606 if (is_cold_data(fio->page)) 2607 return true; 2608 if (IS_ATOMIC_WRITTEN_PAGE(fio->page)) 2609 return true; 2610 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) && 2611 f2fs_is_checkpointed_data(sbi, fio->old_blkaddr))) 2612 return true; 2613 } 2614 return false; 2615 } 2616 2617 static inline bool need_inplace_update(struct f2fs_io_info *fio) 2618 { 2619 struct inode *inode = fio->page->mapping->host; 2620 2621 if (f2fs_should_update_outplace(inode, fio)) 2622 return false; 2623 2624 return f2fs_should_update_inplace(inode, fio); 2625 } 2626 2627 int f2fs_do_write_data_page(struct f2fs_io_info *fio) 2628 { 2629 struct page *page = fio->page; 2630 struct inode *inode = page->mapping->host; 2631 struct dnode_of_data dn; 2632 struct extent_info ei = {0,0,0}; 2633 struct node_info ni; 2634 bool ipu_force = false; 2635 int err = 0; 2636 2637 set_new_dnode(&dn, inode, NULL, NULL, 0); 2638 if (need_inplace_update(fio) && 2639 f2fs_lookup_extent_cache(inode, page->index, &ei)) { 2640 fio->old_blkaddr = ei.blk + page->index - ei.fofs; 2641 2642 if (!f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, 2643 DATA_GENERIC_ENHANCE)) 2644 return -EFSCORRUPTED; 2645 2646 ipu_force = true; 2647 fio->need_lock = LOCK_DONE; 2648 goto got_it; 2649 } 2650 2651 /* Deadlock due to between page->lock and f2fs_lock_op */ 2652 if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi)) 2653 return -EAGAIN; 2654 2655 err = f2fs_get_dnode_of_data(&dn, page->index, LOOKUP_NODE); 2656 if (err) 2657 goto out; 2658 2659 fio->old_blkaddr = dn.data_blkaddr; 2660 2661 /* This page is already truncated */ 2662 if (fio->old_blkaddr == NULL_ADDR) { 2663 ClearPageUptodate(page); 2664 clear_cold_data(page); 2665 goto out_writepage; 2666 } 2667 got_it: 2668 if (__is_valid_data_blkaddr(fio->old_blkaddr) && 2669 !f2fs_is_valid_blkaddr(fio->sbi, fio->old_blkaddr, 2670 DATA_GENERIC_ENHANCE)) { 2671 err = -EFSCORRUPTED; 2672 goto out_writepage; 2673 } 2674 /* 2675 * If current allocation needs SSR, 2676 * it had better in-place writes for updated data. 2677 */ 2678 if (ipu_force || 2679 (__is_valid_data_blkaddr(fio->old_blkaddr) && 2680 need_inplace_update(fio))) { 2681 err = f2fs_encrypt_one_page(fio); 2682 if (err) 2683 goto out_writepage; 2684 2685 set_page_writeback(page); 2686 ClearPageError(page); 2687 f2fs_put_dnode(&dn); 2688 if (fio->need_lock == LOCK_REQ) 2689 f2fs_unlock_op(fio->sbi); 2690 err = f2fs_inplace_write_data(fio); 2691 if (err) { 2692 if (fscrypt_inode_uses_fs_layer_crypto(inode)) 2693 fscrypt_finalize_bounce_page(&fio->encrypted_page); 2694 if (PageWriteback(page)) 2695 end_page_writeback(page); 2696 } else { 2697 set_inode_flag(inode, FI_UPDATE_WRITE); 2698 } 2699 trace_f2fs_do_write_data_page(fio->page, IPU); 2700 return err; 2701 } 2702 2703 if (fio->need_lock == LOCK_RETRY) { 2704 if (!f2fs_trylock_op(fio->sbi)) { 2705 err = -EAGAIN; 2706 goto out_writepage; 2707 } 2708 fio->need_lock = LOCK_REQ; 2709 } 2710 2711 err = f2fs_get_node_info(fio->sbi, dn.nid, &ni); 2712 if (err) 2713 goto out_writepage; 2714 2715 fio->version = ni.version; 2716 2717 err = f2fs_encrypt_one_page(fio); 2718 if (err) 2719 goto out_writepage; 2720 2721 set_page_writeback(page); 2722 ClearPageError(page); 2723 2724 if (fio->compr_blocks && fio->old_blkaddr == COMPRESS_ADDR) 2725 f2fs_i_compr_blocks_update(inode, fio->compr_blocks - 1, false); 2726 2727 /* LFS mode write path */ 2728 f2fs_outplace_write_data(&dn, fio); 2729 trace_f2fs_do_write_data_page(page, OPU); 2730 set_inode_flag(inode, FI_APPEND_WRITE); 2731 if (page->index == 0) 2732 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN); 2733 out_writepage: 2734 f2fs_put_dnode(&dn); 2735 out: 2736 if (fio->need_lock == LOCK_REQ) 2737 f2fs_unlock_op(fio->sbi); 2738 return err; 2739 } 2740 2741 int f2fs_write_single_data_page(struct page *page, int *submitted, 2742 struct bio **bio, 2743 sector_t *last_block, 2744 struct writeback_control *wbc, 2745 enum iostat_type io_type, 2746 int compr_blocks) 2747 { 2748 struct inode *inode = page->mapping->host; 2749 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 2750 loff_t i_size = i_size_read(inode); 2751 const pgoff_t end_index = ((unsigned long long)i_size) 2752 >> PAGE_SHIFT; 2753 loff_t psize = (loff_t)(page->index + 1) << PAGE_SHIFT; 2754 unsigned offset = 0; 2755 bool need_balance_fs = false; 2756 int err = 0; 2757 struct f2fs_io_info fio = { 2758 .sbi = sbi, 2759 .ino = inode->i_ino, 2760 .type = DATA, 2761 .op = REQ_OP_WRITE, 2762 .op_flags = wbc_to_write_flags(wbc), 2763 .old_blkaddr = NULL_ADDR, 2764 .page = page, 2765 .encrypted_page = NULL, 2766 .submitted = false, 2767 .compr_blocks = compr_blocks, 2768 .need_lock = LOCK_RETRY, 2769 .io_type = io_type, 2770 .io_wbc = wbc, 2771 .bio = bio, 2772 .last_block = last_block, 2773 }; 2774 2775 trace_f2fs_writepage(page, DATA); 2776 2777 /* we should bypass data pages to proceed the kworkder jobs */ 2778 if (unlikely(f2fs_cp_error(sbi))) { 2779 mapping_set_error(page->mapping, -EIO); 2780 /* 2781 * don't drop any dirty dentry pages for keeping lastest 2782 * directory structure. 2783 */ 2784 if (S_ISDIR(inode->i_mode)) 2785 goto redirty_out; 2786 goto out; 2787 } 2788 2789 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 2790 goto redirty_out; 2791 2792 if (page->index < end_index || 2793 f2fs_verity_in_progress(inode) || 2794 compr_blocks) 2795 goto write; 2796 2797 /* 2798 * If the offset is out-of-range of file size, 2799 * this page does not have to be written to disk. 2800 */ 2801 offset = i_size & (PAGE_SIZE - 1); 2802 if ((page->index >= end_index + 1) || !offset) 2803 goto out; 2804 2805 zero_user_segment(page, offset, PAGE_SIZE); 2806 write: 2807 if (f2fs_is_drop_cache(inode)) 2808 goto out; 2809 /* we should not write 0'th page having journal header */ 2810 if (f2fs_is_volatile_file(inode) && (!page->index || 2811 (!wbc->for_reclaim && 2812 f2fs_available_free_memory(sbi, BASE_CHECK)))) 2813 goto redirty_out; 2814 2815 /* Dentry/quota blocks are controlled by checkpoint */ 2816 if (S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) { 2817 /* 2818 * We need to wait for node_write to avoid block allocation during 2819 * checkpoint. This can only happen to quota writes which can cause 2820 * the below discard race condition. 2821 */ 2822 if (IS_NOQUOTA(inode)) 2823 down_read(&sbi->node_write); 2824 2825 fio.need_lock = LOCK_DONE; 2826 err = f2fs_do_write_data_page(&fio); 2827 2828 if (IS_NOQUOTA(inode)) 2829 up_read(&sbi->node_write); 2830 2831 goto done; 2832 } 2833 2834 if (!wbc->for_reclaim) 2835 need_balance_fs = true; 2836 else if (has_not_enough_free_secs(sbi, 0, 0)) 2837 goto redirty_out; 2838 else 2839 set_inode_flag(inode, FI_HOT_DATA); 2840 2841 err = -EAGAIN; 2842 if (f2fs_has_inline_data(inode)) { 2843 err = f2fs_write_inline_data(inode, page); 2844 if (!err) 2845 goto out; 2846 } 2847 2848 if (err == -EAGAIN) { 2849 err = f2fs_do_write_data_page(&fio); 2850 if (err == -EAGAIN) { 2851 fio.need_lock = LOCK_REQ; 2852 err = f2fs_do_write_data_page(&fio); 2853 } 2854 } 2855 2856 if (err) { 2857 file_set_keep_isize(inode); 2858 } else { 2859 spin_lock(&F2FS_I(inode)->i_size_lock); 2860 if (F2FS_I(inode)->last_disk_size < psize) 2861 F2FS_I(inode)->last_disk_size = psize; 2862 spin_unlock(&F2FS_I(inode)->i_size_lock); 2863 } 2864 2865 done: 2866 if (err && err != -ENOENT) 2867 goto redirty_out; 2868 2869 out: 2870 inode_dec_dirty_pages(inode); 2871 if (err) { 2872 ClearPageUptodate(page); 2873 clear_cold_data(page); 2874 } 2875 2876 if (wbc->for_reclaim) { 2877 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, DATA); 2878 clear_inode_flag(inode, FI_HOT_DATA); 2879 f2fs_remove_dirty_inode(inode); 2880 submitted = NULL; 2881 } 2882 unlock_page(page); 2883 if (!S_ISDIR(inode->i_mode) && !IS_NOQUOTA(inode) && 2884 !F2FS_I(inode)->cp_task) 2885 f2fs_balance_fs(sbi, need_balance_fs); 2886 2887 if (unlikely(f2fs_cp_error(sbi))) { 2888 f2fs_submit_merged_write(sbi, DATA); 2889 f2fs_submit_merged_ipu_write(sbi, bio, NULL); 2890 submitted = NULL; 2891 } 2892 2893 if (submitted) 2894 *submitted = fio.submitted ? 1 : 0; 2895 2896 return 0; 2897 2898 redirty_out: 2899 redirty_page_for_writepage(wbc, page); 2900 /* 2901 * pageout() in MM traslates EAGAIN, so calls handle_write_error() 2902 * -> mapping_set_error() -> set_bit(AS_EIO, ...). 2903 * file_write_and_wait_range() will see EIO error, which is critical 2904 * to return value of fsync() followed by atomic_write failure to user. 2905 */ 2906 if (!err || wbc->for_reclaim) 2907 return AOP_WRITEPAGE_ACTIVATE; 2908 unlock_page(page); 2909 return err; 2910 } 2911 2912 static int f2fs_write_data_page(struct page *page, 2913 struct writeback_control *wbc) 2914 { 2915 #ifdef CONFIG_F2FS_FS_COMPRESSION 2916 struct inode *inode = page->mapping->host; 2917 2918 if (unlikely(f2fs_cp_error(F2FS_I_SB(inode)))) 2919 goto out; 2920 2921 if (f2fs_compressed_file(inode)) { 2922 if (f2fs_is_compressed_cluster(inode, page->index)) { 2923 redirty_page_for_writepage(wbc, page); 2924 return AOP_WRITEPAGE_ACTIVATE; 2925 } 2926 } 2927 out: 2928 #endif 2929 2930 return f2fs_write_single_data_page(page, NULL, NULL, NULL, 2931 wbc, FS_DATA_IO, 0); 2932 } 2933 2934 /* 2935 * This function was copied from write_cche_pages from mm/page-writeback.c. 2936 * The major change is making write step of cold data page separately from 2937 * warm/hot data page. 2938 */ 2939 static int f2fs_write_cache_pages(struct address_space *mapping, 2940 struct writeback_control *wbc, 2941 enum iostat_type io_type) 2942 { 2943 int ret = 0; 2944 int done = 0, retry = 0; 2945 struct pagevec pvec; 2946 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); 2947 struct bio *bio = NULL; 2948 sector_t last_block; 2949 #ifdef CONFIG_F2FS_FS_COMPRESSION 2950 struct inode *inode = mapping->host; 2951 struct compress_ctx cc = { 2952 .inode = inode, 2953 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 2954 .cluster_size = F2FS_I(inode)->i_cluster_size, 2955 .cluster_idx = NULL_CLUSTER, 2956 .rpages = NULL, 2957 .nr_rpages = 0, 2958 .cpages = NULL, 2959 .rbuf = NULL, 2960 .cbuf = NULL, 2961 .rlen = PAGE_SIZE * F2FS_I(inode)->i_cluster_size, 2962 .private = NULL, 2963 }; 2964 #endif 2965 int nr_pages; 2966 pgoff_t index; 2967 pgoff_t end; /* Inclusive */ 2968 pgoff_t done_index; 2969 int range_whole = 0; 2970 xa_mark_t tag; 2971 int nwritten = 0; 2972 int submitted = 0; 2973 int i; 2974 2975 pagevec_init(&pvec); 2976 2977 if (get_dirty_pages(mapping->host) <= 2978 SM_I(F2FS_M_SB(mapping))->min_hot_blocks) 2979 set_inode_flag(mapping->host, FI_HOT_DATA); 2980 else 2981 clear_inode_flag(mapping->host, FI_HOT_DATA); 2982 2983 if (wbc->range_cyclic) { 2984 index = mapping->writeback_index; /* prev offset */ 2985 end = -1; 2986 } else { 2987 index = wbc->range_start >> PAGE_SHIFT; 2988 end = wbc->range_end >> PAGE_SHIFT; 2989 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX) 2990 range_whole = 1; 2991 } 2992 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2993 tag = PAGECACHE_TAG_TOWRITE; 2994 else 2995 tag = PAGECACHE_TAG_DIRTY; 2996 retry: 2997 retry = 0; 2998 if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages) 2999 tag_pages_for_writeback(mapping, index, end); 3000 done_index = index; 3001 while (!done && !retry && (index <= end)) { 3002 nr_pages = pagevec_lookup_range_tag(&pvec, mapping, &index, end, 3003 tag); 3004 if (nr_pages == 0) 3005 break; 3006 3007 for (i = 0; i < nr_pages; i++) { 3008 struct page *page = pvec.pages[i]; 3009 bool need_readd; 3010 readd: 3011 need_readd = false; 3012 #ifdef CONFIG_F2FS_FS_COMPRESSION 3013 if (f2fs_compressed_file(inode)) { 3014 ret = f2fs_init_compress_ctx(&cc); 3015 if (ret) { 3016 done = 1; 3017 break; 3018 } 3019 3020 if (!f2fs_cluster_can_merge_page(&cc, 3021 page->index)) { 3022 ret = f2fs_write_multi_pages(&cc, 3023 &submitted, wbc, io_type); 3024 if (!ret) 3025 need_readd = true; 3026 goto result; 3027 } 3028 3029 if (unlikely(f2fs_cp_error(sbi))) 3030 goto lock_page; 3031 3032 if (f2fs_cluster_is_empty(&cc)) { 3033 void *fsdata = NULL; 3034 struct page *pagep; 3035 int ret2; 3036 3037 ret2 = f2fs_prepare_compress_overwrite( 3038 inode, &pagep, 3039 page->index, &fsdata); 3040 if (ret2 < 0) { 3041 ret = ret2; 3042 done = 1; 3043 break; 3044 } else if (ret2 && 3045 !f2fs_compress_write_end(inode, 3046 fsdata, page->index, 3047 1)) { 3048 retry = 1; 3049 break; 3050 } 3051 } else { 3052 goto lock_page; 3053 } 3054 } 3055 #endif 3056 /* give a priority to WB_SYNC threads */ 3057 if (atomic_read(&sbi->wb_sync_req[DATA]) && 3058 wbc->sync_mode == WB_SYNC_NONE) { 3059 done = 1; 3060 break; 3061 } 3062 #ifdef CONFIG_F2FS_FS_COMPRESSION 3063 lock_page: 3064 #endif 3065 done_index = page->index; 3066 retry_write: 3067 lock_page(page); 3068 3069 if (unlikely(page->mapping != mapping)) { 3070 continue_unlock: 3071 unlock_page(page); 3072 continue; 3073 } 3074 3075 if (!PageDirty(page)) { 3076 /* someone wrote it for us */ 3077 goto continue_unlock; 3078 } 3079 3080 if (PageWriteback(page)) { 3081 if (wbc->sync_mode != WB_SYNC_NONE) 3082 f2fs_wait_on_page_writeback(page, 3083 DATA, true, true); 3084 else 3085 goto continue_unlock; 3086 } 3087 3088 if (!clear_page_dirty_for_io(page)) 3089 goto continue_unlock; 3090 3091 #ifdef CONFIG_F2FS_FS_COMPRESSION 3092 if (f2fs_compressed_file(inode)) { 3093 get_page(page); 3094 f2fs_compress_ctx_add_page(&cc, page); 3095 continue; 3096 } 3097 #endif 3098 ret = f2fs_write_single_data_page(page, &submitted, 3099 &bio, &last_block, wbc, io_type, 0); 3100 if (ret == AOP_WRITEPAGE_ACTIVATE) 3101 unlock_page(page); 3102 #ifdef CONFIG_F2FS_FS_COMPRESSION 3103 result: 3104 #endif 3105 nwritten += submitted; 3106 wbc->nr_to_write -= submitted; 3107 3108 if (unlikely(ret)) { 3109 /* 3110 * keep nr_to_write, since vfs uses this to 3111 * get # of written pages. 3112 */ 3113 if (ret == AOP_WRITEPAGE_ACTIVATE) { 3114 ret = 0; 3115 goto next; 3116 } else if (ret == -EAGAIN) { 3117 ret = 0; 3118 if (wbc->sync_mode == WB_SYNC_ALL) { 3119 cond_resched(); 3120 congestion_wait(BLK_RW_ASYNC, 3121 DEFAULT_IO_TIMEOUT); 3122 goto retry_write; 3123 } 3124 goto next; 3125 } 3126 done_index = page->index + 1; 3127 done = 1; 3128 break; 3129 } 3130 3131 if (wbc->nr_to_write <= 0 && 3132 wbc->sync_mode == WB_SYNC_NONE) { 3133 done = 1; 3134 break; 3135 } 3136 next: 3137 if (need_readd) 3138 goto readd; 3139 } 3140 pagevec_release(&pvec); 3141 cond_resched(); 3142 } 3143 #ifdef CONFIG_F2FS_FS_COMPRESSION 3144 /* flush remained pages in compress cluster */ 3145 if (f2fs_compressed_file(inode) && !f2fs_cluster_is_empty(&cc)) { 3146 ret = f2fs_write_multi_pages(&cc, &submitted, wbc, io_type); 3147 nwritten += submitted; 3148 wbc->nr_to_write -= submitted; 3149 if (ret) { 3150 done = 1; 3151 retry = 0; 3152 } 3153 } 3154 if (f2fs_compressed_file(inode)) 3155 f2fs_destroy_compress_ctx(&cc); 3156 #endif 3157 if (retry) { 3158 index = 0; 3159 end = -1; 3160 goto retry; 3161 } 3162 if (wbc->range_cyclic && !done) 3163 done_index = 0; 3164 if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0)) 3165 mapping->writeback_index = done_index; 3166 3167 if (nwritten) 3168 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host, 3169 NULL, 0, DATA); 3170 /* submit cached bio of IPU write */ 3171 if (bio) 3172 f2fs_submit_merged_ipu_write(sbi, &bio, NULL); 3173 3174 return ret; 3175 } 3176 3177 static inline bool __should_serialize_io(struct inode *inode, 3178 struct writeback_control *wbc) 3179 { 3180 /* to avoid deadlock in path of data flush */ 3181 if (F2FS_I(inode)->cp_task) 3182 return false; 3183 3184 if (!S_ISREG(inode->i_mode)) 3185 return false; 3186 if (IS_NOQUOTA(inode)) 3187 return false; 3188 3189 if (f2fs_need_compress_data(inode)) 3190 return true; 3191 if (wbc->sync_mode != WB_SYNC_ALL) 3192 return true; 3193 if (get_dirty_pages(inode) >= SM_I(F2FS_I_SB(inode))->min_seq_blocks) 3194 return true; 3195 return false; 3196 } 3197 3198 static int __f2fs_write_data_pages(struct address_space *mapping, 3199 struct writeback_control *wbc, 3200 enum iostat_type io_type) 3201 { 3202 struct inode *inode = mapping->host; 3203 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3204 struct blk_plug plug; 3205 int ret; 3206 bool locked = false; 3207 3208 /* deal with chardevs and other special file */ 3209 if (!mapping->a_ops->writepage) 3210 return 0; 3211 3212 /* skip writing if there is no dirty page in this inode */ 3213 if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE) 3214 return 0; 3215 3216 /* during POR, we don't need to trigger writepage at all. */ 3217 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 3218 goto skip_write; 3219 3220 if ((S_ISDIR(inode->i_mode) || IS_NOQUOTA(inode)) && 3221 wbc->sync_mode == WB_SYNC_NONE && 3222 get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) && 3223 f2fs_available_free_memory(sbi, DIRTY_DENTS)) 3224 goto skip_write; 3225 3226 /* skip writing during file defragment */ 3227 if (is_inode_flag_set(inode, FI_DO_DEFRAG)) 3228 goto skip_write; 3229 3230 trace_f2fs_writepages(mapping->host, wbc, DATA); 3231 3232 /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */ 3233 if (wbc->sync_mode == WB_SYNC_ALL) 3234 atomic_inc(&sbi->wb_sync_req[DATA]); 3235 else if (atomic_read(&sbi->wb_sync_req[DATA])) 3236 goto skip_write; 3237 3238 if (__should_serialize_io(inode, wbc)) { 3239 mutex_lock(&sbi->writepages); 3240 locked = true; 3241 } 3242 3243 blk_start_plug(&plug); 3244 ret = f2fs_write_cache_pages(mapping, wbc, io_type); 3245 blk_finish_plug(&plug); 3246 3247 if (locked) 3248 mutex_unlock(&sbi->writepages); 3249 3250 if (wbc->sync_mode == WB_SYNC_ALL) 3251 atomic_dec(&sbi->wb_sync_req[DATA]); 3252 /* 3253 * if some pages were truncated, we cannot guarantee its mapping->host 3254 * to detect pending bios. 3255 */ 3256 3257 f2fs_remove_dirty_inode(inode); 3258 return ret; 3259 3260 skip_write: 3261 wbc->pages_skipped += get_dirty_pages(inode); 3262 trace_f2fs_writepages(mapping->host, wbc, DATA); 3263 return 0; 3264 } 3265 3266 static int f2fs_write_data_pages(struct address_space *mapping, 3267 struct writeback_control *wbc) 3268 { 3269 struct inode *inode = mapping->host; 3270 3271 return __f2fs_write_data_pages(mapping, wbc, 3272 F2FS_I(inode)->cp_task == current ? 3273 FS_CP_DATA_IO : FS_DATA_IO); 3274 } 3275 3276 static void f2fs_write_failed(struct address_space *mapping, loff_t to) 3277 { 3278 struct inode *inode = mapping->host; 3279 loff_t i_size = i_size_read(inode); 3280 3281 if (IS_NOQUOTA(inode)) 3282 return; 3283 3284 /* In the fs-verity case, f2fs_end_enable_verity() does the truncate */ 3285 if (to > i_size && !f2fs_verity_in_progress(inode)) { 3286 down_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3287 down_write(&F2FS_I(inode)->i_mmap_sem); 3288 3289 truncate_pagecache(inode, i_size); 3290 f2fs_truncate_blocks(inode, i_size, true); 3291 3292 up_write(&F2FS_I(inode)->i_mmap_sem); 3293 up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]); 3294 } 3295 } 3296 3297 static int prepare_write_begin(struct f2fs_sb_info *sbi, 3298 struct page *page, loff_t pos, unsigned len, 3299 block_t *blk_addr, bool *node_changed) 3300 { 3301 struct inode *inode = page->mapping->host; 3302 pgoff_t index = page->index; 3303 struct dnode_of_data dn; 3304 struct page *ipage; 3305 bool locked = false; 3306 struct extent_info ei = {0,0,0}; 3307 int err = 0; 3308 int flag; 3309 3310 /* 3311 * we already allocated all the blocks, so we don't need to get 3312 * the block addresses when there is no need to fill the page. 3313 */ 3314 if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE && 3315 !is_inode_flag_set(inode, FI_NO_PREALLOC) && 3316 !f2fs_verity_in_progress(inode)) 3317 return 0; 3318 3319 /* f2fs_lock_op avoids race between write CP and convert_inline_page */ 3320 if (f2fs_has_inline_data(inode) && pos + len > MAX_INLINE_DATA(inode)) 3321 flag = F2FS_GET_BLOCK_DEFAULT; 3322 else 3323 flag = F2FS_GET_BLOCK_PRE_AIO; 3324 3325 if (f2fs_has_inline_data(inode) || 3326 (pos & PAGE_MASK) >= i_size_read(inode)) { 3327 f2fs_do_map_lock(sbi, flag, true); 3328 locked = true; 3329 } 3330 3331 restart: 3332 /* check inline_data */ 3333 ipage = f2fs_get_node_page(sbi, inode->i_ino); 3334 if (IS_ERR(ipage)) { 3335 err = PTR_ERR(ipage); 3336 goto unlock_out; 3337 } 3338 3339 set_new_dnode(&dn, inode, ipage, ipage, 0); 3340 3341 if (f2fs_has_inline_data(inode)) { 3342 if (pos + len <= MAX_INLINE_DATA(inode)) { 3343 f2fs_do_read_inline_data(page, ipage); 3344 set_inode_flag(inode, FI_DATA_EXIST); 3345 if (inode->i_nlink) 3346 set_inline_node(ipage); 3347 } else { 3348 err = f2fs_convert_inline_page(&dn, page); 3349 if (err) 3350 goto out; 3351 if (dn.data_blkaddr == NULL_ADDR) 3352 err = f2fs_get_block(&dn, index); 3353 } 3354 } else if (locked) { 3355 err = f2fs_get_block(&dn, index); 3356 } else { 3357 if (f2fs_lookup_extent_cache(inode, index, &ei)) { 3358 dn.data_blkaddr = ei.blk + index - ei.fofs; 3359 } else { 3360 /* hole case */ 3361 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE); 3362 if (err || dn.data_blkaddr == NULL_ADDR) { 3363 f2fs_put_dnode(&dn); 3364 f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, 3365 true); 3366 WARN_ON(flag != F2FS_GET_BLOCK_PRE_AIO); 3367 locked = true; 3368 goto restart; 3369 } 3370 } 3371 } 3372 3373 /* convert_inline_page can make node_changed */ 3374 *blk_addr = dn.data_blkaddr; 3375 *node_changed = dn.node_changed; 3376 out: 3377 f2fs_put_dnode(&dn); 3378 unlock_out: 3379 if (locked) 3380 f2fs_do_map_lock(sbi, flag, false); 3381 return err; 3382 } 3383 3384 static int f2fs_write_begin(struct file *file, struct address_space *mapping, 3385 loff_t pos, unsigned len, unsigned flags, 3386 struct page **pagep, void **fsdata) 3387 { 3388 struct inode *inode = mapping->host; 3389 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3390 struct page *page = NULL; 3391 pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT; 3392 bool need_balance = false, drop_atomic = false; 3393 block_t blkaddr = NULL_ADDR; 3394 int err = 0; 3395 3396 trace_f2fs_write_begin(inode, pos, len, flags); 3397 3398 if (!f2fs_is_checkpoint_ready(sbi)) { 3399 err = -ENOSPC; 3400 goto fail; 3401 } 3402 3403 if ((f2fs_is_atomic_file(inode) && 3404 !f2fs_available_free_memory(sbi, INMEM_PAGES)) || 3405 is_inode_flag_set(inode, FI_ATOMIC_REVOKE_REQUEST)) { 3406 err = -ENOMEM; 3407 drop_atomic = true; 3408 goto fail; 3409 } 3410 3411 /* 3412 * We should check this at this moment to avoid deadlock on inode page 3413 * and #0 page. The locking rule for inline_data conversion should be: 3414 * lock_page(page #0) -> lock_page(inode_page) 3415 */ 3416 if (index != 0) { 3417 err = f2fs_convert_inline_inode(inode); 3418 if (err) 3419 goto fail; 3420 } 3421 3422 #ifdef CONFIG_F2FS_FS_COMPRESSION 3423 if (f2fs_compressed_file(inode)) { 3424 int ret; 3425 3426 *fsdata = NULL; 3427 3428 ret = f2fs_prepare_compress_overwrite(inode, pagep, 3429 index, fsdata); 3430 if (ret < 0) { 3431 err = ret; 3432 goto fail; 3433 } else if (ret) { 3434 return 0; 3435 } 3436 } 3437 #endif 3438 3439 repeat: 3440 /* 3441 * Do not use grab_cache_page_write_begin() to avoid deadlock due to 3442 * wait_for_stable_page. Will wait that below with our IO control. 3443 */ 3444 page = f2fs_pagecache_get_page(mapping, index, 3445 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS); 3446 if (!page) { 3447 err = -ENOMEM; 3448 goto fail; 3449 } 3450 3451 /* TODO: cluster can be compressed due to race with .writepage */ 3452 3453 *pagep = page; 3454 3455 err = prepare_write_begin(sbi, page, pos, len, 3456 &blkaddr, &need_balance); 3457 if (err) 3458 goto fail; 3459 3460 if (need_balance && !IS_NOQUOTA(inode) && 3461 has_not_enough_free_secs(sbi, 0, 0)) { 3462 unlock_page(page); 3463 f2fs_balance_fs(sbi, true); 3464 lock_page(page); 3465 if (page->mapping != mapping) { 3466 /* The page got truncated from under us */ 3467 f2fs_put_page(page, 1); 3468 goto repeat; 3469 } 3470 } 3471 3472 f2fs_wait_on_page_writeback(page, DATA, false, true); 3473 3474 if (len == PAGE_SIZE || PageUptodate(page)) 3475 return 0; 3476 3477 if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode) && 3478 !f2fs_verity_in_progress(inode)) { 3479 zero_user_segment(page, len, PAGE_SIZE); 3480 return 0; 3481 } 3482 3483 if (blkaddr == NEW_ADDR) { 3484 zero_user_segment(page, 0, PAGE_SIZE); 3485 SetPageUptodate(page); 3486 } else { 3487 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, 3488 DATA_GENERIC_ENHANCE_READ)) { 3489 err = -EFSCORRUPTED; 3490 goto fail; 3491 } 3492 err = f2fs_submit_page_read(inode, page, blkaddr, 0, true); 3493 if (err) 3494 goto fail; 3495 3496 lock_page(page); 3497 if (unlikely(page->mapping != mapping)) { 3498 f2fs_put_page(page, 1); 3499 goto repeat; 3500 } 3501 if (unlikely(!PageUptodate(page))) { 3502 err = -EIO; 3503 goto fail; 3504 } 3505 } 3506 return 0; 3507 3508 fail: 3509 f2fs_put_page(page, 1); 3510 f2fs_write_failed(mapping, pos + len); 3511 if (drop_atomic) 3512 f2fs_drop_inmem_pages_all(sbi, false); 3513 return err; 3514 } 3515 3516 static int f2fs_write_end(struct file *file, 3517 struct address_space *mapping, 3518 loff_t pos, unsigned len, unsigned copied, 3519 struct page *page, void *fsdata) 3520 { 3521 struct inode *inode = page->mapping->host; 3522 3523 trace_f2fs_write_end(inode, pos, len, copied); 3524 3525 /* 3526 * This should be come from len == PAGE_SIZE, and we expect copied 3527 * should be PAGE_SIZE. Otherwise, we treat it with zero copied and 3528 * let generic_perform_write() try to copy data again through copied=0. 3529 */ 3530 if (!PageUptodate(page)) { 3531 if (unlikely(copied != len)) 3532 copied = 0; 3533 else 3534 SetPageUptodate(page); 3535 } 3536 3537 #ifdef CONFIG_F2FS_FS_COMPRESSION 3538 /* overwrite compressed file */ 3539 if (f2fs_compressed_file(inode) && fsdata) { 3540 f2fs_compress_write_end(inode, fsdata, page->index, copied); 3541 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); 3542 3543 if (pos + copied > i_size_read(inode) && 3544 !f2fs_verity_in_progress(inode)) 3545 f2fs_i_size_write(inode, pos + copied); 3546 return copied; 3547 } 3548 #endif 3549 3550 if (!copied) 3551 goto unlock_out; 3552 3553 set_page_dirty(page); 3554 3555 if (pos + copied > i_size_read(inode) && 3556 !f2fs_verity_in_progress(inode)) 3557 f2fs_i_size_write(inode, pos + copied); 3558 unlock_out: 3559 f2fs_put_page(page, 1); 3560 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); 3561 return copied; 3562 } 3563 3564 static int check_direct_IO(struct inode *inode, struct iov_iter *iter, 3565 loff_t offset) 3566 { 3567 unsigned i_blkbits = READ_ONCE(inode->i_blkbits); 3568 unsigned blkbits = i_blkbits; 3569 unsigned blocksize_mask = (1 << blkbits) - 1; 3570 unsigned long align = offset | iov_iter_alignment(iter); 3571 struct block_device *bdev = inode->i_sb->s_bdev; 3572 3573 if (iov_iter_rw(iter) == READ && offset >= i_size_read(inode)) 3574 return 1; 3575 3576 if (align & blocksize_mask) { 3577 if (bdev) 3578 blkbits = blksize_bits(bdev_logical_block_size(bdev)); 3579 blocksize_mask = (1 << blkbits) - 1; 3580 if (align & blocksize_mask) 3581 return -EINVAL; 3582 return 1; 3583 } 3584 return 0; 3585 } 3586 3587 static void f2fs_dio_end_io(struct bio *bio) 3588 { 3589 struct f2fs_private_dio *dio = bio->bi_private; 3590 3591 dec_page_count(F2FS_I_SB(dio->inode), 3592 dio->write ? F2FS_DIO_WRITE : F2FS_DIO_READ); 3593 3594 bio->bi_private = dio->orig_private; 3595 bio->bi_end_io = dio->orig_end_io; 3596 3597 kfree(dio); 3598 3599 bio_endio(bio); 3600 } 3601 3602 static void f2fs_dio_submit_bio(struct bio *bio, struct inode *inode, 3603 loff_t file_offset) 3604 { 3605 struct f2fs_private_dio *dio; 3606 bool write = (bio_op(bio) == REQ_OP_WRITE); 3607 3608 dio = f2fs_kzalloc(F2FS_I_SB(inode), 3609 sizeof(struct f2fs_private_dio), GFP_NOFS); 3610 if (!dio) 3611 goto out; 3612 3613 dio->inode = inode; 3614 dio->orig_end_io = bio->bi_end_io; 3615 dio->orig_private = bio->bi_private; 3616 dio->write = write; 3617 3618 bio->bi_end_io = f2fs_dio_end_io; 3619 bio->bi_private = dio; 3620 3621 inc_page_count(F2FS_I_SB(inode), 3622 write ? F2FS_DIO_WRITE : F2FS_DIO_READ); 3623 3624 submit_bio(bio); 3625 return; 3626 out: 3627 bio->bi_status = BLK_STS_IOERR; 3628 bio_endio(bio); 3629 } 3630 3631 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 3632 { 3633 struct address_space *mapping = iocb->ki_filp->f_mapping; 3634 struct inode *inode = mapping->host; 3635 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3636 struct f2fs_inode_info *fi = F2FS_I(inode); 3637 size_t count = iov_iter_count(iter); 3638 loff_t offset = iocb->ki_pos; 3639 int rw = iov_iter_rw(iter); 3640 int err; 3641 enum rw_hint hint = iocb->ki_hint; 3642 int whint_mode = F2FS_OPTION(sbi).whint_mode; 3643 bool do_opu; 3644 3645 err = check_direct_IO(inode, iter, offset); 3646 if (err) 3647 return err < 0 ? err : 0; 3648 3649 if (f2fs_force_buffered_io(inode, iocb, iter)) 3650 return 0; 3651 3652 do_opu = allow_outplace_dio(inode, iocb, iter); 3653 3654 trace_f2fs_direct_IO_enter(inode, offset, count, rw); 3655 3656 if (rw == WRITE && whint_mode == WHINT_MODE_OFF) 3657 iocb->ki_hint = WRITE_LIFE_NOT_SET; 3658 3659 if (iocb->ki_flags & IOCB_NOWAIT) { 3660 if (!down_read_trylock(&fi->i_gc_rwsem[rw])) { 3661 iocb->ki_hint = hint; 3662 err = -EAGAIN; 3663 goto out; 3664 } 3665 if (do_opu && !down_read_trylock(&fi->i_gc_rwsem[READ])) { 3666 up_read(&fi->i_gc_rwsem[rw]); 3667 iocb->ki_hint = hint; 3668 err = -EAGAIN; 3669 goto out; 3670 } 3671 } else { 3672 down_read(&fi->i_gc_rwsem[rw]); 3673 if (do_opu) 3674 down_read(&fi->i_gc_rwsem[READ]); 3675 } 3676 3677 err = __blockdev_direct_IO(iocb, inode, inode->i_sb->s_bdev, 3678 iter, rw == WRITE ? get_data_block_dio_write : 3679 get_data_block_dio, NULL, f2fs_dio_submit_bio, 3680 rw == WRITE ? DIO_LOCKING | DIO_SKIP_HOLES : 3681 DIO_SKIP_HOLES); 3682 3683 if (do_opu) 3684 up_read(&fi->i_gc_rwsem[READ]); 3685 3686 up_read(&fi->i_gc_rwsem[rw]); 3687 3688 if (rw == WRITE) { 3689 if (whint_mode == WHINT_MODE_OFF) 3690 iocb->ki_hint = hint; 3691 if (err > 0) { 3692 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO, 3693 err); 3694 if (!do_opu) 3695 set_inode_flag(inode, FI_UPDATE_WRITE); 3696 } else if (err == -EIOCBQUEUED) { 3697 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_IO, 3698 count - iov_iter_count(iter)); 3699 } else if (err < 0) { 3700 f2fs_write_failed(mapping, offset + count); 3701 } 3702 } else { 3703 if (err > 0) 3704 f2fs_update_iostat(sbi, APP_DIRECT_READ_IO, err); 3705 else if (err == -EIOCBQUEUED) 3706 f2fs_update_iostat(F2FS_I_SB(inode), APP_DIRECT_READ_IO, 3707 count - iov_iter_count(iter)); 3708 } 3709 3710 out: 3711 trace_f2fs_direct_IO_exit(inode, offset, count, rw, err); 3712 3713 return err; 3714 } 3715 3716 void f2fs_invalidate_page(struct page *page, unsigned int offset, 3717 unsigned int length) 3718 { 3719 struct inode *inode = page->mapping->host; 3720 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 3721 3722 if (inode->i_ino >= F2FS_ROOT_INO(sbi) && 3723 (offset % PAGE_SIZE || length != PAGE_SIZE)) 3724 return; 3725 3726 if (PageDirty(page)) { 3727 if (inode->i_ino == F2FS_META_INO(sbi)) { 3728 dec_page_count(sbi, F2FS_DIRTY_META); 3729 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) { 3730 dec_page_count(sbi, F2FS_DIRTY_NODES); 3731 } else { 3732 inode_dec_dirty_pages(inode); 3733 f2fs_remove_dirty_inode(inode); 3734 } 3735 } 3736 3737 clear_cold_data(page); 3738 3739 if (IS_ATOMIC_WRITTEN_PAGE(page)) 3740 return f2fs_drop_inmem_page(inode, page); 3741 3742 f2fs_clear_page_private(page); 3743 } 3744 3745 int f2fs_release_page(struct page *page, gfp_t wait) 3746 { 3747 /* If this is dirty page, keep PagePrivate */ 3748 if (PageDirty(page)) 3749 return 0; 3750 3751 /* This is atomic written page, keep Private */ 3752 if (IS_ATOMIC_WRITTEN_PAGE(page)) 3753 return 0; 3754 3755 clear_cold_data(page); 3756 f2fs_clear_page_private(page); 3757 return 1; 3758 } 3759 3760 static int f2fs_set_data_page_dirty(struct page *page) 3761 { 3762 struct inode *inode = page_file_mapping(page)->host; 3763 3764 trace_f2fs_set_page_dirty(page, DATA); 3765 3766 if (!PageUptodate(page)) 3767 SetPageUptodate(page); 3768 if (PageSwapCache(page)) 3769 return __set_page_dirty_nobuffers(page); 3770 3771 if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) { 3772 if (!IS_ATOMIC_WRITTEN_PAGE(page)) { 3773 f2fs_register_inmem_page(inode, page); 3774 return 1; 3775 } 3776 /* 3777 * Previously, this page has been registered, we just 3778 * return here. 3779 */ 3780 return 0; 3781 } 3782 3783 if (!PageDirty(page)) { 3784 __set_page_dirty_nobuffers(page); 3785 f2fs_update_dirty_page(inode, page); 3786 return 1; 3787 } 3788 return 0; 3789 } 3790 3791 3792 static sector_t f2fs_bmap_compress(struct inode *inode, sector_t block) 3793 { 3794 #ifdef CONFIG_F2FS_FS_COMPRESSION 3795 struct dnode_of_data dn; 3796 sector_t start_idx, blknr = 0; 3797 int ret; 3798 3799 start_idx = round_down(block, F2FS_I(inode)->i_cluster_size); 3800 3801 set_new_dnode(&dn, inode, NULL, NULL, 0); 3802 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 3803 if (ret) 3804 return 0; 3805 3806 if (dn.data_blkaddr != COMPRESS_ADDR) { 3807 dn.ofs_in_node += block - start_idx; 3808 blknr = f2fs_data_blkaddr(&dn); 3809 if (!__is_valid_data_blkaddr(blknr)) 3810 blknr = 0; 3811 } 3812 3813 f2fs_put_dnode(&dn); 3814 return blknr; 3815 #else 3816 return 0; 3817 #endif 3818 } 3819 3820 3821 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block) 3822 { 3823 struct inode *inode = mapping->host; 3824 sector_t blknr = 0; 3825 3826 if (f2fs_has_inline_data(inode)) 3827 goto out; 3828 3829 /* make sure allocating whole blocks */ 3830 if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY)) 3831 filemap_write_and_wait(mapping); 3832 3833 /* Block number less than F2FS MAX BLOCKS */ 3834 if (unlikely(block >= F2FS_I_SB(inode)->max_file_blocks)) 3835 goto out; 3836 3837 if (f2fs_compressed_file(inode)) { 3838 blknr = f2fs_bmap_compress(inode, block); 3839 } else { 3840 struct f2fs_map_blocks map; 3841 3842 memset(&map, 0, sizeof(map)); 3843 map.m_lblk = block; 3844 map.m_len = 1; 3845 map.m_next_pgofs = NULL; 3846 map.m_seg_type = NO_CHECK_TYPE; 3847 3848 if (!f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_BMAP)) 3849 blknr = map.m_pblk; 3850 } 3851 out: 3852 trace_f2fs_bmap(inode, block, blknr); 3853 return blknr; 3854 } 3855 3856 #ifdef CONFIG_MIGRATION 3857 #include <linux/migrate.h> 3858 3859 int f2fs_migrate_page(struct address_space *mapping, 3860 struct page *newpage, struct page *page, enum migrate_mode mode) 3861 { 3862 int rc, extra_count; 3863 struct f2fs_inode_info *fi = F2FS_I(mapping->host); 3864 bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page); 3865 3866 BUG_ON(PageWriteback(page)); 3867 3868 /* migrating an atomic written page is safe with the inmem_lock hold */ 3869 if (atomic_written) { 3870 if (mode != MIGRATE_SYNC) 3871 return -EBUSY; 3872 if (!mutex_trylock(&fi->inmem_lock)) 3873 return -EAGAIN; 3874 } 3875 3876 /* one extra reference was held for atomic_write page */ 3877 extra_count = atomic_written ? 1 : 0; 3878 rc = migrate_page_move_mapping(mapping, newpage, 3879 page, extra_count); 3880 if (rc != MIGRATEPAGE_SUCCESS) { 3881 if (atomic_written) 3882 mutex_unlock(&fi->inmem_lock); 3883 return rc; 3884 } 3885 3886 if (atomic_written) { 3887 struct inmem_pages *cur; 3888 list_for_each_entry(cur, &fi->inmem_pages, list) 3889 if (cur->page == page) { 3890 cur->page = newpage; 3891 break; 3892 } 3893 mutex_unlock(&fi->inmem_lock); 3894 put_page(page); 3895 get_page(newpage); 3896 } 3897 3898 if (PagePrivate(page)) { 3899 f2fs_set_page_private(newpage, page_private(page)); 3900 f2fs_clear_page_private(page); 3901 } 3902 3903 if (mode != MIGRATE_SYNC_NO_COPY) 3904 migrate_page_copy(newpage, page); 3905 else 3906 migrate_page_states(newpage, page); 3907 3908 return MIGRATEPAGE_SUCCESS; 3909 } 3910 #endif 3911 3912 #ifdef CONFIG_SWAP 3913 static int check_swap_activate_fast(struct swap_info_struct *sis, 3914 struct file *swap_file, sector_t *span) 3915 { 3916 struct address_space *mapping = swap_file->f_mapping; 3917 struct inode *inode = mapping->host; 3918 sector_t cur_lblock; 3919 sector_t last_lblock; 3920 sector_t pblock; 3921 sector_t lowest_pblock = -1; 3922 sector_t highest_pblock = 0; 3923 int nr_extents = 0; 3924 unsigned long nr_pblocks; 3925 u64 len; 3926 int ret; 3927 3928 /* 3929 * Map all the blocks into the extent list. This code doesn't try 3930 * to be very smart. 3931 */ 3932 cur_lblock = 0; 3933 last_lblock = bytes_to_blks(inode, i_size_read(inode)); 3934 len = i_size_read(inode); 3935 3936 while (cur_lblock <= last_lblock && cur_lblock < sis->max) { 3937 struct f2fs_map_blocks map; 3938 pgoff_t next_pgofs; 3939 3940 cond_resched(); 3941 3942 memset(&map, 0, sizeof(map)); 3943 map.m_lblk = cur_lblock; 3944 map.m_len = bytes_to_blks(inode, len) - cur_lblock; 3945 map.m_next_pgofs = &next_pgofs; 3946 map.m_seg_type = NO_CHECK_TYPE; 3947 3948 ret = f2fs_map_blocks(inode, &map, 0, F2FS_GET_BLOCK_FIEMAP); 3949 if (ret) 3950 goto err_out; 3951 3952 /* hole */ 3953 if (!(map.m_flags & F2FS_MAP_FLAGS)) 3954 goto err_out; 3955 3956 pblock = map.m_pblk; 3957 nr_pblocks = map.m_len; 3958 3959 if (cur_lblock + nr_pblocks >= sis->max) 3960 nr_pblocks = sis->max - cur_lblock; 3961 3962 if (cur_lblock) { /* exclude the header page */ 3963 if (pblock < lowest_pblock) 3964 lowest_pblock = pblock; 3965 if (pblock + nr_pblocks - 1 > highest_pblock) 3966 highest_pblock = pblock + nr_pblocks - 1; 3967 } 3968 3969 /* 3970 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks 3971 */ 3972 ret = add_swap_extent(sis, cur_lblock, nr_pblocks, pblock); 3973 if (ret < 0) 3974 goto out; 3975 nr_extents += ret; 3976 cur_lblock += nr_pblocks; 3977 } 3978 ret = nr_extents; 3979 *span = 1 + highest_pblock - lowest_pblock; 3980 if (cur_lblock == 0) 3981 cur_lblock = 1; /* force Empty message */ 3982 sis->max = cur_lblock; 3983 sis->pages = cur_lblock - 1; 3984 sis->highest_bit = cur_lblock - 1; 3985 out: 3986 return ret; 3987 err_out: 3988 pr_err("swapon: swapfile has holes\n"); 3989 return -EINVAL; 3990 } 3991 3992 /* Copied from generic_swapfile_activate() to check any holes */ 3993 static int check_swap_activate(struct swap_info_struct *sis, 3994 struct file *swap_file, sector_t *span) 3995 { 3996 struct address_space *mapping = swap_file->f_mapping; 3997 struct inode *inode = mapping->host; 3998 unsigned blocks_per_page; 3999 unsigned long page_no; 4000 sector_t probe_block; 4001 sector_t last_block; 4002 sector_t lowest_block = -1; 4003 sector_t highest_block = 0; 4004 int nr_extents = 0; 4005 int ret; 4006 4007 if (PAGE_SIZE == F2FS_BLKSIZE) 4008 return check_swap_activate_fast(sis, swap_file, span); 4009 4010 blocks_per_page = bytes_to_blks(inode, PAGE_SIZE); 4011 4012 /* 4013 * Map all the blocks into the extent list. This code doesn't try 4014 * to be very smart. 4015 */ 4016 probe_block = 0; 4017 page_no = 0; 4018 last_block = bytes_to_blks(inode, i_size_read(inode)); 4019 while ((probe_block + blocks_per_page) <= last_block && 4020 page_no < sis->max) { 4021 unsigned block_in_page; 4022 sector_t first_block; 4023 sector_t block = 0; 4024 int err = 0; 4025 4026 cond_resched(); 4027 4028 block = probe_block; 4029 err = bmap(inode, &block); 4030 if (err || !block) 4031 goto bad_bmap; 4032 first_block = block; 4033 4034 /* 4035 * It must be PAGE_SIZE aligned on-disk 4036 */ 4037 if (first_block & (blocks_per_page - 1)) { 4038 probe_block++; 4039 goto reprobe; 4040 } 4041 4042 for (block_in_page = 1; block_in_page < blocks_per_page; 4043 block_in_page++) { 4044 4045 block = probe_block + block_in_page; 4046 err = bmap(inode, &block); 4047 4048 if (err || !block) 4049 goto bad_bmap; 4050 4051 if (block != first_block + block_in_page) { 4052 /* Discontiguity */ 4053 probe_block++; 4054 goto reprobe; 4055 } 4056 } 4057 4058 first_block >>= (PAGE_SHIFT - inode->i_blkbits); 4059 if (page_no) { /* exclude the header page */ 4060 if (first_block < lowest_block) 4061 lowest_block = first_block; 4062 if (first_block > highest_block) 4063 highest_block = first_block; 4064 } 4065 4066 /* 4067 * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks 4068 */ 4069 ret = add_swap_extent(sis, page_no, 1, first_block); 4070 if (ret < 0) 4071 goto out; 4072 nr_extents += ret; 4073 page_no++; 4074 probe_block += blocks_per_page; 4075 reprobe: 4076 continue; 4077 } 4078 ret = nr_extents; 4079 *span = 1 + highest_block - lowest_block; 4080 if (page_no == 0) 4081 page_no = 1; /* force Empty message */ 4082 sis->max = page_no; 4083 sis->pages = page_no - 1; 4084 sis->highest_bit = page_no - 1; 4085 out: 4086 return ret; 4087 bad_bmap: 4088 pr_err("swapon: swapfile has holes\n"); 4089 return -EINVAL; 4090 } 4091 4092 static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, 4093 sector_t *span) 4094 { 4095 struct inode *inode = file_inode(file); 4096 int ret; 4097 4098 if (!S_ISREG(inode->i_mode)) 4099 return -EINVAL; 4100 4101 if (f2fs_readonly(F2FS_I_SB(inode)->sb)) 4102 return -EROFS; 4103 4104 ret = f2fs_convert_inline_inode(inode); 4105 if (ret) 4106 return ret; 4107 4108 if (!f2fs_disable_compressed_file(inode)) 4109 return -EINVAL; 4110 4111 ret = check_swap_activate(sis, file, span); 4112 if (ret < 0) 4113 return ret; 4114 4115 set_inode_flag(inode, FI_PIN_FILE); 4116 f2fs_precache_extents(inode); 4117 f2fs_update_time(F2FS_I_SB(inode), REQ_TIME); 4118 return ret; 4119 } 4120 4121 static void f2fs_swap_deactivate(struct file *file) 4122 { 4123 struct inode *inode = file_inode(file); 4124 4125 clear_inode_flag(inode, FI_PIN_FILE); 4126 } 4127 #else 4128 static int f2fs_swap_activate(struct swap_info_struct *sis, struct file *file, 4129 sector_t *span) 4130 { 4131 return -EOPNOTSUPP; 4132 } 4133 4134 static void f2fs_swap_deactivate(struct file *file) 4135 { 4136 } 4137 #endif 4138 4139 const struct address_space_operations f2fs_dblock_aops = { 4140 .readpage = f2fs_read_data_page, 4141 .readahead = f2fs_readahead, 4142 .writepage = f2fs_write_data_page, 4143 .writepages = f2fs_write_data_pages, 4144 .write_begin = f2fs_write_begin, 4145 .write_end = f2fs_write_end, 4146 .set_page_dirty = f2fs_set_data_page_dirty, 4147 .invalidatepage = f2fs_invalidate_page, 4148 .releasepage = f2fs_release_page, 4149 .direct_IO = f2fs_direct_IO, 4150 .bmap = f2fs_bmap, 4151 .swap_activate = f2fs_swap_activate, 4152 .swap_deactivate = f2fs_swap_deactivate, 4153 #ifdef CONFIG_MIGRATION 4154 .migratepage = f2fs_migrate_page, 4155 #endif 4156 }; 4157 4158 void f2fs_clear_page_cache_dirty_tag(struct page *page) 4159 { 4160 struct address_space *mapping = page_mapping(page); 4161 unsigned long flags; 4162 4163 xa_lock_irqsave(&mapping->i_pages, flags); 4164 __xa_clear_mark(&mapping->i_pages, page_index(page), 4165 PAGECACHE_TAG_DIRTY); 4166 xa_unlock_irqrestore(&mapping->i_pages, flags); 4167 } 4168 4169 int __init f2fs_init_post_read_processing(void) 4170 { 4171 bio_post_read_ctx_cache = 4172 kmem_cache_create("f2fs_bio_post_read_ctx", 4173 sizeof(struct bio_post_read_ctx), 0, 0, NULL); 4174 if (!bio_post_read_ctx_cache) 4175 goto fail; 4176 bio_post_read_ctx_pool = 4177 mempool_create_slab_pool(NUM_PREALLOC_POST_READ_CTXS, 4178 bio_post_read_ctx_cache); 4179 if (!bio_post_read_ctx_pool) 4180 goto fail_free_cache; 4181 return 0; 4182 4183 fail_free_cache: 4184 kmem_cache_destroy(bio_post_read_ctx_cache); 4185 fail: 4186 return -ENOMEM; 4187 } 4188 4189 void f2fs_destroy_post_read_processing(void) 4190 { 4191 mempool_destroy(bio_post_read_ctx_pool); 4192 kmem_cache_destroy(bio_post_read_ctx_cache); 4193 } 4194 4195 int f2fs_init_post_read_wq(struct f2fs_sb_info *sbi) 4196 { 4197 if (!f2fs_sb_has_encrypt(sbi) && 4198 !f2fs_sb_has_verity(sbi) && 4199 !f2fs_sb_has_compression(sbi)) 4200 return 0; 4201 4202 sbi->post_read_wq = alloc_workqueue("f2fs_post_read_wq", 4203 WQ_UNBOUND | WQ_HIGHPRI, 4204 num_online_cpus()); 4205 if (!sbi->post_read_wq) 4206 return -ENOMEM; 4207 return 0; 4208 } 4209 4210 void f2fs_destroy_post_read_wq(struct f2fs_sb_info *sbi) 4211 { 4212 if (sbi->post_read_wq) 4213 destroy_workqueue(sbi->post_read_wq); 4214 } 4215 4216 int __init f2fs_init_bio_entry_cache(void) 4217 { 4218 bio_entry_slab = f2fs_kmem_cache_create("f2fs_bio_entry_slab", 4219 sizeof(struct bio_entry)); 4220 if (!bio_entry_slab) 4221 return -ENOMEM; 4222 return 0; 4223 } 4224 4225 void f2fs_destroy_bio_entry_cache(void) 4226 { 4227 kmem_cache_destroy(bio_entry_slab); 4228 } 4229