1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * f2fs compress support 4 * 5 * Copyright (c) 2019 Chao Yu <chao@kernel.org> 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/moduleparam.h> 11 #include <linux/writeback.h> 12 #include <linux/backing-dev.h> 13 #include <linux/lzo.h> 14 #include <linux/lz4.h> 15 #include <linux/zstd.h> 16 #include <linux/folio_batch.h> 17 18 #include "f2fs.h" 19 #include "node.h" 20 #include "segment.h" 21 #include <trace/events/f2fs.h> 22 23 static struct kmem_cache *cic_entry_slab; 24 static struct kmem_cache *dic_entry_slab; 25 26 static void *page_array_alloc(struct f2fs_sb_info *sbi, int nr) 27 { 28 unsigned int size = sizeof(struct page *) * nr; 29 30 if (likely(size <= sbi->page_array_slab_size)) 31 return f2fs_kmem_cache_alloc(sbi->page_array_slab, 32 GFP_F2FS_ZERO, false, sbi); 33 return f2fs_kzalloc(sbi, size, GFP_NOFS); 34 } 35 36 static void page_array_free(struct f2fs_sb_info *sbi, void *pages, int nr) 37 { 38 unsigned int size = sizeof(struct page *) * nr; 39 40 if (!pages) 41 return; 42 43 if (likely(size <= sbi->page_array_slab_size)) 44 kmem_cache_free(sbi->page_array_slab, pages); 45 else 46 kfree(pages); 47 } 48 49 struct f2fs_compress_ops { 50 int (*init_compress_ctx)(struct compress_ctx *cc); 51 void (*destroy_compress_ctx)(struct compress_ctx *cc); 52 int (*compress_pages)(struct compress_ctx *cc); 53 int (*init_decompress_ctx)(struct decompress_io_ctx *dic); 54 void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic); 55 int (*decompress_pages)(struct decompress_io_ctx *dic); 56 bool (*is_level_valid)(int level); 57 }; 58 59 static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index) 60 { 61 return index & (cc->cluster_size - 1); 62 } 63 64 static pgoff_t cluster_idx(struct compress_ctx *cc, pgoff_t index) 65 { 66 return index >> cc->log_cluster_size; 67 } 68 69 static pgoff_t start_idx_of_cluster(struct compress_ctx *cc) 70 { 71 return cc->cluster_idx << cc->log_cluster_size; 72 } 73 74 bool f2fs_is_compressed_page(struct folio *folio) 75 { 76 if (!folio->private) 77 return false; 78 if (folio_test_f2fs_nonpointer(folio)) 79 return false; 80 81 f2fs_bug_on(F2FS_F_SB(folio), 82 *((u32 *)folio->private) != F2FS_COMPRESSED_PAGE_MAGIC); 83 return true; 84 } 85 86 static void f2fs_set_compressed_page(struct page *page, 87 struct inode *inode, pgoff_t index, void *data) 88 { 89 struct folio *folio = page_folio(page); 90 91 folio_attach_private(folio, (void *)data); 92 93 /* i_crypto_info and iv index */ 94 folio->index = index; 95 folio->mapping = inode->i_mapping; 96 } 97 98 static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock) 99 { 100 int i; 101 102 for (i = 0; i < len; i++) { 103 if (!cc->rpages[i]) 104 continue; 105 if (unlock) 106 unlock_page(cc->rpages[i]); 107 else 108 put_page(cc->rpages[i]); 109 } 110 } 111 112 static void f2fs_put_rpages(struct compress_ctx *cc) 113 { 114 f2fs_drop_rpages(cc, cc->cluster_size, false); 115 } 116 117 static void f2fs_unlock_rpages(struct compress_ctx *cc, int len) 118 { 119 f2fs_drop_rpages(cc, len, true); 120 } 121 122 static void f2fs_put_rpages_wbc(struct compress_ctx *cc, 123 struct writeback_control *wbc, bool redirty, bool unlock) 124 { 125 unsigned int i; 126 127 for (i = 0; i < cc->cluster_size; i++) { 128 if (!cc->rpages[i]) 129 continue; 130 if (redirty) 131 redirty_page_for_writepage(wbc, cc->rpages[i]); 132 f2fs_put_page(cc->rpages[i], unlock); 133 } 134 } 135 136 struct folio *f2fs_compress_control_folio(struct folio *folio) 137 { 138 struct compress_io_ctx *ctx = folio->private; 139 140 return page_folio(ctx->rpages[0]); 141 } 142 143 int f2fs_init_compress_ctx(struct compress_ctx *cc) 144 { 145 if (cc->rpages) 146 return 0; 147 148 cc->rpages = page_array_alloc(F2FS_I_SB(cc->inode), cc->cluster_size); 149 return cc->rpages ? 0 : -ENOMEM; 150 } 151 152 void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse) 153 { 154 page_array_free(F2FS_I_SB(cc->inode), cc->rpages, cc->cluster_size); 155 cc->rpages = NULL; 156 cc->nr_rpages = 0; 157 cc->nr_cpages = 0; 158 cc->valid_nr_cpages = 0; 159 if (!reuse) 160 cc->cluster_idx = NULL_CLUSTER; 161 } 162 163 void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct folio *folio) 164 { 165 unsigned int cluster_ofs; 166 167 if (!f2fs_cluster_can_merge_page(cc, folio->index)) 168 f2fs_bug_on(F2FS_I_SB(cc->inode), 1); 169 170 cluster_ofs = offset_in_cluster(cc, folio->index); 171 cc->rpages[cluster_ofs] = folio_page(folio, 0); 172 cc->nr_rpages++; 173 cc->cluster_idx = cluster_idx(cc, folio->index); 174 } 175 176 #ifdef CONFIG_F2FS_FS_LZO 177 static int lzo_init_compress_ctx(struct compress_ctx *cc) 178 { 179 cc->private = f2fs_vmalloc(F2FS_I_SB(cc->inode), 180 LZO1X_MEM_COMPRESS); 181 if (!cc->private) 182 return -ENOMEM; 183 184 cc->clen = lzo1x_worst_compress(PAGE_SIZE << cc->log_cluster_size); 185 return 0; 186 } 187 188 static void lzo_destroy_compress_ctx(struct compress_ctx *cc) 189 { 190 vfree(cc->private); 191 cc->private = NULL; 192 } 193 194 static int lzo_compress_pages(struct compress_ctx *cc) 195 { 196 int ret; 197 198 ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, 199 &cc->clen, cc->private); 200 if (ret != LZO_E_OK) { 201 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 202 "lzo compress failed, ret:%d", ret); 203 return -EIO; 204 } 205 return 0; 206 } 207 208 static int lzo_decompress_pages(struct decompress_io_ctx *dic) 209 { 210 int ret; 211 212 ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen, 213 dic->rbuf, &dic->rlen); 214 if (ret != LZO_E_OK) { 215 f2fs_err_ratelimited(dic->sbi, 216 "lzo decompress failed, ret:%d", ret); 217 return -EIO; 218 } 219 220 if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) { 221 f2fs_err_ratelimited(dic->sbi, 222 "lzo invalid rlen:%zu, expected:%lu", 223 dic->rlen, PAGE_SIZE << dic->log_cluster_size); 224 return -EIO; 225 } 226 return 0; 227 } 228 229 static const struct f2fs_compress_ops f2fs_lzo_ops = { 230 .init_compress_ctx = lzo_init_compress_ctx, 231 .destroy_compress_ctx = lzo_destroy_compress_ctx, 232 .compress_pages = lzo_compress_pages, 233 .decompress_pages = lzo_decompress_pages, 234 }; 235 #endif 236 237 #ifdef CONFIG_F2FS_FS_LZ4 238 static int lz4_init_compress_ctx(struct compress_ctx *cc) 239 { 240 unsigned int size = LZ4_MEM_COMPRESS; 241 242 #ifdef CONFIG_F2FS_FS_LZ4HC 243 if (F2FS_I(cc->inode)->i_compress_level) 244 size = LZ4HC_MEM_COMPRESS; 245 #endif 246 247 cc->private = f2fs_vmalloc(F2FS_I_SB(cc->inode), size); 248 if (!cc->private) 249 return -ENOMEM; 250 251 /* 252 * we do not change cc->clen to LZ4_compressBound(inputsize) to 253 * adapt worst compress case, because lz4 compressor can handle 254 * output budget properly. 255 */ 256 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; 257 return 0; 258 } 259 260 static void lz4_destroy_compress_ctx(struct compress_ctx *cc) 261 { 262 vfree(cc->private); 263 cc->private = NULL; 264 } 265 266 static int lz4_compress_pages(struct compress_ctx *cc) 267 { 268 int len = -EINVAL; 269 unsigned char level = F2FS_I(cc->inode)->i_compress_level; 270 271 if (!level) 272 len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen, 273 cc->clen, cc->private); 274 #ifdef CONFIG_F2FS_FS_LZ4HC 275 else 276 len = LZ4_compress_HC(cc->rbuf, cc->cbuf->cdata, cc->rlen, 277 cc->clen, level, cc->private); 278 #endif 279 if (len < 0) 280 return len; 281 if (!len) 282 return -EAGAIN; 283 284 cc->clen = len; 285 return 0; 286 } 287 288 static int lz4_decompress_pages(struct decompress_io_ctx *dic) 289 { 290 int ret; 291 292 ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf, 293 dic->clen, dic->rlen); 294 if (ret < 0) { 295 f2fs_err_ratelimited(dic->sbi, 296 "lz4 decompress failed, ret:%d", ret); 297 return -EIO; 298 } 299 300 if (ret != PAGE_SIZE << dic->log_cluster_size) { 301 f2fs_err_ratelimited(dic->sbi, 302 "lz4 invalid ret:%d, expected:%lu", 303 ret, PAGE_SIZE << dic->log_cluster_size); 304 return -EIO; 305 } 306 return 0; 307 } 308 309 static bool lz4_is_level_valid(int lvl) 310 { 311 #ifdef CONFIG_F2FS_FS_LZ4HC 312 return !lvl || (lvl >= LZ4HC_MIN_CLEVEL && lvl <= LZ4HC_MAX_CLEVEL); 313 #else 314 return lvl == 0; 315 #endif 316 } 317 318 static const struct f2fs_compress_ops f2fs_lz4_ops = { 319 .init_compress_ctx = lz4_init_compress_ctx, 320 .destroy_compress_ctx = lz4_destroy_compress_ctx, 321 .compress_pages = lz4_compress_pages, 322 .decompress_pages = lz4_decompress_pages, 323 .is_level_valid = lz4_is_level_valid, 324 }; 325 #endif 326 327 #ifdef CONFIG_F2FS_FS_ZSTD 328 static int zstd_init_compress_ctx(struct compress_ctx *cc) 329 { 330 zstd_parameters params; 331 zstd_cstream *stream; 332 void *workspace; 333 unsigned int workspace_size; 334 unsigned char level = F2FS_I(cc->inode)->i_compress_level; 335 336 /* Need to remain this for backward compatibility */ 337 if (!level) 338 level = F2FS_ZSTD_DEFAULT_CLEVEL; 339 340 params = zstd_get_params(level, cc->rlen); 341 workspace_size = zstd_cstream_workspace_bound(¶ms.cParams); 342 343 workspace = f2fs_vmalloc(F2FS_I_SB(cc->inode), workspace_size); 344 if (!workspace) 345 return -ENOMEM; 346 347 stream = zstd_init_cstream(¶ms, 0, workspace, workspace_size); 348 if (!stream) { 349 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 350 "%s zstd_init_cstream failed", __func__); 351 vfree(workspace); 352 return -EIO; 353 } 354 355 cc->private = workspace; 356 cc->private2 = stream; 357 358 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; 359 return 0; 360 } 361 362 static void zstd_destroy_compress_ctx(struct compress_ctx *cc) 363 { 364 vfree(cc->private); 365 cc->private = NULL; 366 cc->private2 = NULL; 367 } 368 369 static int zstd_compress_pages(struct compress_ctx *cc) 370 { 371 zstd_cstream *stream = cc->private2; 372 zstd_in_buffer inbuf; 373 zstd_out_buffer outbuf; 374 int src_size = cc->rlen; 375 int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE; 376 int ret; 377 378 inbuf.pos = 0; 379 inbuf.src = cc->rbuf; 380 inbuf.size = src_size; 381 382 outbuf.pos = 0; 383 outbuf.dst = cc->cbuf->cdata; 384 outbuf.size = dst_size; 385 386 ret = zstd_compress_stream(stream, &outbuf, &inbuf); 387 if (zstd_is_error(ret)) { 388 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 389 "%s zstd_compress_stream failed, ret: %d", 390 __func__, zstd_get_error_code(ret)); 391 return -EIO; 392 } 393 394 ret = zstd_end_stream(stream, &outbuf); 395 if (zstd_is_error(ret)) { 396 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 397 "%s zstd_end_stream returned %d", 398 __func__, zstd_get_error_code(ret)); 399 return -EIO; 400 } 401 402 /* 403 * there is compressed data remained in intermediate buffer due to 404 * no more space in cbuf.cdata 405 */ 406 if (ret) 407 return -EAGAIN; 408 409 cc->clen = outbuf.pos; 410 return 0; 411 } 412 413 static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic) 414 { 415 zstd_dstream *stream; 416 void *workspace; 417 unsigned int workspace_size; 418 unsigned int max_window_size = 419 MAX_COMPRESS_WINDOW_SIZE(dic->log_cluster_size); 420 421 workspace_size = zstd_dstream_workspace_bound(max_window_size); 422 423 workspace = f2fs_vmalloc(dic->sbi, workspace_size); 424 if (!workspace) 425 return -ENOMEM; 426 427 stream = zstd_init_dstream(max_window_size, workspace, workspace_size); 428 if (!stream) { 429 f2fs_err_ratelimited(dic->sbi, 430 "%s zstd_init_dstream failed", __func__); 431 vfree(workspace); 432 return -EIO; 433 } 434 435 dic->private = workspace; 436 dic->private2 = stream; 437 438 return 0; 439 } 440 441 static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic) 442 { 443 vfree(dic->private); 444 dic->private = NULL; 445 dic->private2 = NULL; 446 } 447 448 static int zstd_decompress_pages(struct decompress_io_ctx *dic) 449 { 450 zstd_dstream *stream = dic->private2; 451 zstd_in_buffer inbuf; 452 zstd_out_buffer outbuf; 453 int ret; 454 455 inbuf.pos = 0; 456 inbuf.src = dic->cbuf->cdata; 457 inbuf.size = dic->clen; 458 459 outbuf.pos = 0; 460 outbuf.dst = dic->rbuf; 461 outbuf.size = dic->rlen; 462 463 ret = zstd_decompress_stream(stream, &outbuf, &inbuf); 464 if (zstd_is_error(ret)) { 465 f2fs_err_ratelimited(dic->sbi, 466 "%s zstd_decompress_stream failed, ret: %d", 467 __func__, zstd_get_error_code(ret)); 468 return -EIO; 469 } 470 471 if (dic->rlen != outbuf.pos) { 472 f2fs_err_ratelimited(dic->sbi, 473 "%s ZSTD invalid rlen:%zu, expected:%lu", 474 __func__, dic->rlen, 475 PAGE_SIZE << dic->log_cluster_size); 476 return -EIO; 477 } 478 479 return 0; 480 } 481 482 static bool zstd_is_level_valid(int lvl) 483 { 484 return lvl >= zstd_min_clevel() && lvl <= zstd_max_clevel(); 485 } 486 487 static const struct f2fs_compress_ops f2fs_zstd_ops = { 488 .init_compress_ctx = zstd_init_compress_ctx, 489 .destroy_compress_ctx = zstd_destroy_compress_ctx, 490 .compress_pages = zstd_compress_pages, 491 .init_decompress_ctx = zstd_init_decompress_ctx, 492 .destroy_decompress_ctx = zstd_destroy_decompress_ctx, 493 .decompress_pages = zstd_decompress_pages, 494 .is_level_valid = zstd_is_level_valid, 495 }; 496 #endif 497 498 #ifdef CONFIG_F2FS_FS_LZO 499 #ifdef CONFIG_F2FS_FS_LZORLE 500 static int lzorle_compress_pages(struct compress_ctx *cc) 501 { 502 int ret; 503 504 ret = lzorle1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, 505 &cc->clen, cc->private); 506 if (ret != LZO_E_OK) { 507 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 508 "lzo-rle compress failed, ret:%d", ret); 509 return -EIO; 510 } 511 return 0; 512 } 513 514 static const struct f2fs_compress_ops f2fs_lzorle_ops = { 515 .init_compress_ctx = lzo_init_compress_ctx, 516 .destroy_compress_ctx = lzo_destroy_compress_ctx, 517 .compress_pages = lzorle_compress_pages, 518 .decompress_pages = lzo_decompress_pages, 519 }; 520 #endif 521 #endif 522 523 static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = { 524 #ifdef CONFIG_F2FS_FS_LZO 525 &f2fs_lzo_ops, 526 #else 527 NULL, 528 #endif 529 #ifdef CONFIG_F2FS_FS_LZ4 530 &f2fs_lz4_ops, 531 #else 532 NULL, 533 #endif 534 #ifdef CONFIG_F2FS_FS_ZSTD 535 &f2fs_zstd_ops, 536 #else 537 NULL, 538 #endif 539 #if defined(CONFIG_F2FS_FS_LZO) && defined(CONFIG_F2FS_FS_LZORLE) 540 &f2fs_lzorle_ops, 541 #else 542 NULL, 543 #endif 544 }; 545 546 bool f2fs_is_compress_backend_ready(struct inode *inode) 547 { 548 if (!f2fs_compressed_file(inode)) 549 return true; 550 return f2fs_cops[F2FS_I(inode)->i_compress_algorithm]; 551 } 552 553 bool f2fs_is_compress_level_valid(int alg, int lvl) 554 { 555 const struct f2fs_compress_ops *cops = f2fs_cops[alg]; 556 557 if (cops->is_level_valid) 558 return cops->is_level_valid(lvl); 559 560 return lvl == 0; 561 } 562 563 static mempool_t *compress_page_pool; 564 static int num_compress_pages = 512; 565 module_param(num_compress_pages, uint, 0444); 566 MODULE_PARM_DESC(num_compress_pages, 567 "Number of intermediate compress pages to preallocate"); 568 569 int __init f2fs_init_compress_mempool(void) 570 { 571 compress_page_pool = mempool_create_page_pool(num_compress_pages, 0); 572 return compress_page_pool ? 0 : -ENOMEM; 573 } 574 575 void f2fs_destroy_compress_mempool(void) 576 { 577 mempool_destroy(compress_page_pool); 578 } 579 580 static struct page *f2fs_compress_alloc_page(void) 581 { 582 struct page *page; 583 584 page = mempool_alloc(compress_page_pool, GFP_NOFS); 585 lock_page(page); 586 587 return page; 588 } 589 590 static void f2fs_compress_free_page(struct page *page) 591 { 592 struct folio *folio; 593 594 if (!page) 595 return; 596 folio = page_folio(page); 597 folio_detach_private(folio); 598 folio->mapping = NULL; 599 folio_unlock(folio); 600 mempool_free(page, compress_page_pool); 601 } 602 603 #define MAX_VMAP_RETRIES 3 604 605 static void *f2fs_vmap(struct page **pages, unsigned int count) 606 { 607 int i; 608 void *buf = NULL; 609 610 for (i = 0; i < MAX_VMAP_RETRIES; i++) { 611 buf = vm_map_ram(pages, count, -1); 612 if (buf) 613 break; 614 vm_unmap_aliases(); 615 } 616 return buf; 617 } 618 619 static int f2fs_compress_pages(struct compress_ctx *cc) 620 { 621 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 622 struct f2fs_inode_info *fi = F2FS_I(cc->inode); 623 const struct f2fs_compress_ops *cops = 624 f2fs_cops[fi->i_compress_algorithm]; 625 unsigned int max_len, new_nr_cpages; 626 u32 chksum = 0; 627 int i, ret; 628 629 trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx, 630 cc->cluster_size, fi->i_compress_algorithm); 631 632 if (cops->init_compress_ctx) { 633 ret = cops->init_compress_ctx(cc); 634 if (ret) 635 goto out; 636 } 637 638 max_len = COMPRESS_HEADER_SIZE + cc->clen; 639 cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE); 640 cc->valid_nr_cpages = cc->nr_cpages; 641 642 cc->cpages = page_array_alloc(sbi, cc->nr_cpages); 643 if (!cc->cpages) { 644 ret = -ENOMEM; 645 goto destroy_compress_ctx; 646 } 647 648 for (i = 0; i < cc->nr_cpages; i++) 649 cc->cpages[i] = f2fs_compress_alloc_page(); 650 651 cc->rbuf = f2fs_vmap(cc->rpages, cc->cluster_size); 652 if (!cc->rbuf) { 653 ret = -ENOMEM; 654 goto out_free_cpages; 655 } 656 657 cc->cbuf = f2fs_vmap(cc->cpages, cc->nr_cpages); 658 if (!cc->cbuf) { 659 ret = -ENOMEM; 660 goto out_vunmap_rbuf; 661 } 662 663 ret = cops->compress_pages(cc); 664 if (ret) 665 goto out_vunmap_cbuf; 666 667 max_len = PAGE_SIZE * (cc->cluster_size - 1) - COMPRESS_HEADER_SIZE; 668 669 if (cc->clen > max_len) { 670 ret = -EAGAIN; 671 goto out_vunmap_cbuf; 672 } 673 674 cc->cbuf->clen = cpu_to_le32(cc->clen); 675 676 if (fi->i_compress_flag & BIT(COMPRESS_CHKSUM)) 677 chksum = f2fs_crc32(cc->cbuf->cdata, cc->clen); 678 cc->cbuf->chksum = cpu_to_le32(chksum); 679 680 for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++) 681 cc->cbuf->reserved[i] = cpu_to_le32(0); 682 683 new_nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE); 684 685 /* zero out any unused part of the last page */ 686 memset(&cc->cbuf->cdata[cc->clen], 0, 687 (new_nr_cpages * PAGE_SIZE) - 688 (cc->clen + COMPRESS_HEADER_SIZE)); 689 690 vm_unmap_ram(cc->cbuf, cc->nr_cpages); 691 vm_unmap_ram(cc->rbuf, cc->cluster_size); 692 693 for (i = new_nr_cpages; i < cc->nr_cpages; i++) { 694 f2fs_compress_free_page(cc->cpages[i]); 695 cc->cpages[i] = NULL; 696 } 697 698 if (cops->destroy_compress_ctx) 699 cops->destroy_compress_ctx(cc); 700 701 cc->valid_nr_cpages = new_nr_cpages; 702 703 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, 704 cc->clen, ret); 705 return 0; 706 707 out_vunmap_cbuf: 708 vm_unmap_ram(cc->cbuf, cc->nr_cpages); 709 out_vunmap_rbuf: 710 vm_unmap_ram(cc->rbuf, cc->cluster_size); 711 out_free_cpages: 712 for (i = 0; i < cc->nr_cpages; i++) { 713 if (cc->cpages[i]) 714 f2fs_compress_free_page(cc->cpages[i]); 715 } 716 page_array_free(sbi, cc->cpages, cc->nr_cpages); 717 cc->cpages = NULL; 718 destroy_compress_ctx: 719 if (cops->destroy_compress_ctx) 720 cops->destroy_compress_ctx(cc); 721 out: 722 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, 723 cc->clen, ret); 724 return ret; 725 } 726 727 static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, 728 bool pre_alloc); 729 static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic, 730 bool bypass_destroy_callback, bool pre_alloc); 731 732 void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task) 733 { 734 struct f2fs_sb_info *sbi = dic->sbi; 735 struct f2fs_inode_info *fi = F2FS_I(dic->inode); 736 const struct f2fs_compress_ops *cops = 737 f2fs_cops[fi->i_compress_algorithm]; 738 bool bypass_callback = false; 739 int ret; 740 741 trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx, 742 dic->cluster_size, fi->i_compress_algorithm); 743 744 if (dic->failed) { 745 ret = -EIO; 746 goto out_end_io; 747 } 748 749 ret = f2fs_prepare_decomp_mem(dic, false); 750 if (ret) { 751 bypass_callback = true; 752 goto out_release; 753 } 754 755 dic->clen = le32_to_cpu(dic->cbuf->clen); 756 dic->rlen = PAGE_SIZE << dic->log_cluster_size; 757 758 if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) { 759 ret = -EFSCORRUPTED; 760 761 /* Avoid f2fs_commit_super in irq context */ 762 f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION); 763 goto out_release; 764 } 765 766 ret = cops->decompress_pages(dic); 767 768 if (!ret && (fi->i_compress_flag & BIT(COMPRESS_CHKSUM))) { 769 u32 provided = le32_to_cpu(dic->cbuf->chksum); 770 u32 calculated = f2fs_crc32(dic->cbuf->cdata, dic->clen); 771 772 if (provided != calculated) { 773 if (!is_inode_flag_set(dic->inode, FI_COMPRESS_CORRUPT)) { 774 set_inode_flag(dic->inode, FI_COMPRESS_CORRUPT); 775 f2fs_info_ratelimited(sbi, 776 "checksum invalid, nid = %llu, %x vs %x", 777 dic->inode->i_ino, 778 provided, calculated); 779 } 780 set_sbi_flag(sbi, SBI_NEED_FSCK); 781 } 782 } 783 784 out_release: 785 f2fs_release_decomp_mem(dic, bypass_callback, false); 786 787 out_end_io: 788 trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx, 789 dic->clen, ret); 790 f2fs_decompress_end_io(dic, ret, in_task); 791 } 792 793 static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, 794 struct folio *folio, nid_t ino, block_t blkaddr); 795 796 /* 797 * This is called when a page of a compressed cluster has been read from disk 798 * (or failed to be read from disk). It checks whether this page was the last 799 * page being waited on in the cluster, and if so, it decompresses the cluster 800 * (or in the case of a failure, cleans up without actually decompressing). 801 */ 802 void f2fs_end_read_compressed_page(struct folio *folio, bool failed, 803 block_t blkaddr, bool in_task) 804 { 805 struct decompress_io_ctx *dic = folio->private; 806 struct f2fs_sb_info *sbi = dic->sbi; 807 808 dec_page_count(sbi, F2FS_RD_DATA); 809 810 if (failed) 811 WRITE_ONCE(dic->failed, true); 812 else if (blkaddr && in_task) 813 f2fs_cache_compressed_page(sbi, folio, 814 dic->inode->i_ino, blkaddr); 815 816 if (atomic_dec_and_test(&dic->remaining_pages)) 817 f2fs_decompress_cluster(dic, in_task); 818 } 819 820 static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index) 821 { 822 if (cc->cluster_idx == NULL_CLUSTER) 823 return true; 824 return cc->cluster_idx == cluster_idx(cc, index); 825 } 826 827 bool f2fs_cluster_is_empty(struct compress_ctx *cc) 828 { 829 return cc->nr_rpages == 0; 830 } 831 832 static bool f2fs_cluster_is_full(struct compress_ctx *cc) 833 { 834 return cc->cluster_size == cc->nr_rpages; 835 } 836 837 bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index) 838 { 839 if (f2fs_cluster_is_empty(cc)) 840 return true; 841 return is_page_in_cluster(cc, index); 842 } 843 844 bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, 845 int index, int nr_pages, bool uptodate) 846 { 847 unsigned long pgidx = page_folio(pages[index])->index; 848 int i = uptodate ? 0 : 1; 849 850 /* 851 * when uptodate set to true, try to check all pages in cluster is 852 * uptodate or not. 853 */ 854 if (uptodate && (pgidx % cc->cluster_size)) 855 return false; 856 857 if (nr_pages - index < cc->cluster_size) 858 return false; 859 860 for (; i < cc->cluster_size; i++) { 861 struct folio *folio = page_folio(pages[index + i]); 862 863 if (folio->index != pgidx + i) 864 return false; 865 if (uptodate && !folio_test_uptodate(folio)) 866 return false; 867 } 868 869 return true; 870 } 871 872 static bool cluster_has_invalid_data(struct compress_ctx *cc) 873 { 874 loff_t i_size = i_size_read(cc->inode); 875 unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE); 876 int i; 877 878 for (i = 0; i < cc->cluster_size; i++) { 879 struct page *page = cc->rpages[i]; 880 881 f2fs_bug_on(F2FS_I_SB(cc->inode), !page); 882 883 /* beyond EOF */ 884 if (page_folio(page)->index >= nr_pages) 885 return true; 886 } 887 return false; 888 } 889 890 bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) 891 { 892 #ifdef CONFIG_F2FS_CHECK_FS 893 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 894 unsigned int cluster_size = F2FS_I(dn->inode)->i_cluster_size; 895 int cluster_end = 0; 896 unsigned int count; 897 int i; 898 char *reason = ""; 899 900 if (dn->data_blkaddr != COMPRESS_ADDR) 901 return false; 902 903 /* [..., COMPR_ADDR, ...] */ 904 if (dn->ofs_in_node % cluster_size) { 905 reason = "[*|C|*|*]"; 906 goto out; 907 } 908 909 for (i = 1, count = 1; i < cluster_size; i++, count++) { 910 block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio, 911 dn->ofs_in_node + i); 912 913 /* [COMPR_ADDR, ..., COMPR_ADDR] */ 914 if (blkaddr == COMPRESS_ADDR) { 915 reason = "[C|*|C|*]"; 916 goto out; 917 } 918 if (!__is_valid_data_blkaddr(blkaddr)) { 919 if (!cluster_end) 920 cluster_end = i; 921 continue; 922 } 923 /* [COMPR_ADDR, NULL_ADDR or NEW_ADDR, valid_blkaddr] */ 924 if (cluster_end) { 925 reason = "[C|N|N|V]"; 926 goto out; 927 } 928 } 929 930 f2fs_bug_on(F2FS_I_SB(dn->inode), count != cluster_size && 931 !is_inode_flag_set(dn->inode, FI_COMPRESS_RELEASED)); 932 933 return false; 934 out: 935 f2fs_warn(sbi, "access invalid cluster, ino:%llu, nid:%u, ofs_in_node:%u, reason:%s", 936 dn->inode->i_ino, dn->nid, dn->ofs_in_node, reason); 937 set_sbi_flag(sbi, SBI_NEED_FSCK); 938 return true; 939 #else 940 return false; 941 #endif 942 } 943 944 static int __f2fs_get_cluster_blocks(struct inode *inode, 945 struct dnode_of_data *dn) 946 { 947 unsigned int cluster_size = F2FS_I(inode)->i_cluster_size; 948 int count, i; 949 950 for (i = 0, count = 0; i < cluster_size; i++) { 951 block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio, 952 dn->ofs_in_node + i); 953 954 if (__is_valid_data_blkaddr(blkaddr)) 955 count++; 956 } 957 958 return count; 959 } 960 961 static int __f2fs_cluster_blocks(struct inode *inode, unsigned int cluster_idx, 962 enum cluster_check_type type) 963 { 964 struct dnode_of_data dn; 965 unsigned int start_idx = cluster_idx << 966 F2FS_I(inode)->i_log_cluster_size; 967 int ret; 968 969 set_new_dnode(&dn, inode, NULL, NULL, 0); 970 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 971 if (ret) { 972 if (ret == -ENOENT) 973 ret = 0; 974 goto fail; 975 } 976 977 if (f2fs_sanity_check_cluster(&dn)) { 978 ret = -EFSCORRUPTED; 979 goto fail; 980 } 981 982 if (dn.data_blkaddr == COMPRESS_ADDR) { 983 if (type == CLUSTER_COMPR_BLKS) 984 ret = 1 + __f2fs_get_cluster_blocks(inode, &dn); 985 else if (type == CLUSTER_IS_COMPR) 986 ret = 1; 987 } else if (type == CLUSTER_RAW_BLKS) { 988 ret = __f2fs_get_cluster_blocks(inode, &dn); 989 } 990 fail: 991 f2fs_put_dnode(&dn); 992 return ret; 993 } 994 995 /* return # of compressed blocks in compressed cluster */ 996 static int f2fs_compressed_blocks(struct compress_ctx *cc) 997 { 998 return __f2fs_cluster_blocks(cc->inode, cc->cluster_idx, 999 CLUSTER_COMPR_BLKS); 1000 } 1001 1002 /* return # of raw blocks in non-compressed cluster */ 1003 static int f2fs_decompressed_blocks(struct inode *inode, 1004 unsigned int cluster_idx) 1005 { 1006 return __f2fs_cluster_blocks(inode, cluster_idx, 1007 CLUSTER_RAW_BLKS); 1008 } 1009 1010 /* return whether cluster is compressed one or not */ 1011 int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index) 1012 { 1013 return __f2fs_cluster_blocks(inode, 1014 index >> F2FS_I(inode)->i_log_cluster_size, 1015 CLUSTER_IS_COMPR); 1016 } 1017 1018 /* return whether cluster contains non raw blocks or not */ 1019 bool f2fs_is_sparse_cluster(struct inode *inode, pgoff_t index) 1020 { 1021 unsigned int cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size; 1022 1023 return f2fs_decompressed_blocks(inode, cluster_idx) != 1024 F2FS_I(inode)->i_cluster_size; 1025 } 1026 1027 static bool cluster_may_compress(struct compress_ctx *cc) 1028 { 1029 if (!f2fs_need_compress_data(cc->inode)) 1030 return false; 1031 if (f2fs_is_atomic_file(cc->inode)) 1032 return false; 1033 if (!f2fs_cluster_is_full(cc)) 1034 return false; 1035 if (unlikely(f2fs_cp_error(F2FS_I_SB(cc->inode)))) 1036 return false; 1037 return !cluster_has_invalid_data(cc); 1038 } 1039 1040 static void set_cluster_writeback(struct compress_ctx *cc) 1041 { 1042 int i; 1043 1044 for (i = 0; i < cc->cluster_size; i++) { 1045 if (cc->rpages[i]) 1046 set_page_writeback(cc->rpages[i]); 1047 } 1048 } 1049 1050 static void cancel_cluster_writeback(struct compress_ctx *cc, 1051 struct compress_io_ctx *cic, int submitted) 1052 { 1053 int i; 1054 1055 /* Wait for submitted IOs. */ 1056 if (submitted > 1) { 1057 f2fs_submit_merged_write(F2FS_I_SB(cc->inode), DATA); 1058 while (atomic_read(&cic->pending_pages) != 1059 (cc->valid_nr_cpages - submitted + 1)) 1060 f2fs_io_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); 1061 } 1062 1063 /* Cancel writeback and stay locked. */ 1064 for (i = 0; i < cc->cluster_size; i++) { 1065 if (i < submitted) { 1066 inode_inc_dirty_pages(cc->inode); 1067 lock_page(cc->rpages[i]); 1068 } 1069 clear_page_private_gcing(cc->rpages[i]); 1070 if (folio_test_writeback(page_folio(cc->rpages[i]))) 1071 end_page_writeback(cc->rpages[i]); 1072 } 1073 } 1074 1075 static void set_cluster_dirty(struct compress_ctx *cc) 1076 { 1077 int i; 1078 1079 for (i = 0; i < cc->cluster_size; i++) 1080 if (cc->rpages[i]) { 1081 set_page_dirty(cc->rpages[i]); 1082 set_page_private_gcing(cc->rpages[i]); 1083 } 1084 } 1085 1086 static int prepare_compress_overwrite(struct compress_ctx *cc, 1087 struct page **pagep, pgoff_t index, void **fsdata) 1088 { 1089 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 1090 struct address_space *mapping = cc->inode->i_mapping; 1091 struct folio *folio; 1092 sector_t last_block_in_bio; 1093 fgf_t fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT; 1094 pgoff_t start_idx = start_idx_of_cluster(cc); 1095 int i, ret; 1096 1097 retry: 1098 ret = f2fs_is_compressed_cluster(cc->inode, start_idx); 1099 if (ret <= 0) 1100 return ret; 1101 1102 ret = f2fs_init_compress_ctx(cc); 1103 if (ret) 1104 return ret; 1105 1106 /* keep folio reference to avoid page reclaim */ 1107 for (i = 0; i < cc->cluster_size; i++) { 1108 folio = f2fs_filemap_get_folio(mapping, start_idx + i, 1109 fgp_flag, GFP_NOFS); 1110 if (IS_ERR(folio)) { 1111 ret = PTR_ERR(folio); 1112 goto unlock_pages; 1113 } 1114 1115 if (folio_test_uptodate(folio)) 1116 f2fs_folio_put(folio, true); 1117 else 1118 f2fs_compress_ctx_add_page(cc, folio); 1119 } 1120 1121 if (!f2fs_cluster_is_empty(cc)) { 1122 struct bio *bio = NULL; 1123 1124 ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size, 1125 &last_block_in_bio, NULL, true); 1126 f2fs_put_rpages(cc); 1127 f2fs_destroy_compress_ctx(cc, true); 1128 if (ret) 1129 goto out; 1130 if (bio) 1131 f2fs_submit_read_bio(sbi, bio, DATA); 1132 1133 ret = f2fs_init_compress_ctx(cc); 1134 if (ret) 1135 goto out; 1136 } 1137 1138 for (i = 0; i < cc->cluster_size; i++) { 1139 f2fs_bug_on(sbi, cc->rpages[i]); 1140 1141 folio = filemap_lock_folio(mapping, start_idx + i); 1142 if (IS_ERR(folio)) { 1143 /* folio could be truncated */ 1144 goto release_and_retry; 1145 } 1146 1147 f2fs_folio_wait_writeback(folio, DATA, true, true); 1148 f2fs_compress_ctx_add_page(cc, folio); 1149 1150 if (!folio_test_uptodate(folio)) { 1151 f2fs_handle_page_eio(sbi, folio, DATA); 1152 release_and_retry: 1153 f2fs_put_rpages(cc); 1154 f2fs_unlock_rpages(cc, i + 1); 1155 f2fs_destroy_compress_ctx(cc, true); 1156 goto retry; 1157 } 1158 } 1159 1160 if (likely(!ret)) { 1161 *fsdata = cc->rpages; 1162 *pagep = cc->rpages[offset_in_cluster(cc, index)]; 1163 return cc->cluster_size; 1164 } 1165 1166 unlock_pages: 1167 f2fs_put_rpages(cc); 1168 f2fs_unlock_rpages(cc, i); 1169 f2fs_destroy_compress_ctx(cc, true); 1170 out: 1171 return ret; 1172 } 1173 1174 int f2fs_prepare_compress_overwrite(struct inode *inode, 1175 struct page **pagep, pgoff_t index, void **fsdata) 1176 { 1177 struct compress_ctx cc = { 1178 .inode = inode, 1179 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 1180 .cluster_size = F2FS_I(inode)->i_cluster_size, 1181 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size, 1182 .rpages = NULL, 1183 .nr_rpages = 0, 1184 .vi = NULL, /* can't write to fsverity files */ 1185 }; 1186 1187 return prepare_compress_overwrite(&cc, pagep, index, fsdata); 1188 } 1189 1190 bool f2fs_compress_write_end(struct inode *inode, void *fsdata, 1191 pgoff_t index, unsigned copied) 1192 1193 { 1194 struct compress_ctx cc = { 1195 .inode = inode, 1196 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 1197 .cluster_size = F2FS_I(inode)->i_cluster_size, 1198 .rpages = fsdata, 1199 }; 1200 struct folio *folio = page_folio(cc.rpages[0]); 1201 bool first_index = (index == folio->index); 1202 1203 if (copied) 1204 set_cluster_dirty(&cc); 1205 1206 f2fs_put_rpages_wbc(&cc, NULL, false, true); 1207 f2fs_destroy_compress_ctx(&cc, false); 1208 1209 return first_index; 1210 } 1211 1212 int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock) 1213 { 1214 void *fsdata = NULL; 1215 struct page *pagep; 1216 struct page **rpages; 1217 int log_cluster_size = F2FS_I(inode)->i_log_cluster_size; 1218 pgoff_t start_idx = from >> (PAGE_SHIFT + log_cluster_size) << 1219 log_cluster_size; 1220 int i; 1221 int err; 1222 1223 err = f2fs_is_compressed_cluster(inode, start_idx); 1224 if (err < 0) 1225 return err; 1226 1227 /* truncate normal cluster */ 1228 if (!err) 1229 return f2fs_do_truncate_blocks(inode, from, lock); 1230 1231 /* truncate compressed cluster */ 1232 err = f2fs_prepare_compress_overwrite(inode, &pagep, 1233 start_idx, &fsdata); 1234 1235 /* should not be a normal cluster */ 1236 f2fs_bug_on(F2FS_I_SB(inode), err == 0); 1237 1238 if (err <= 0) 1239 return err; 1240 1241 rpages = fsdata; 1242 1243 for (i = (1 << log_cluster_size) - 1; i >= 0; i--) { 1244 struct folio *folio = page_folio(rpages[i]); 1245 loff_t start = (loff_t)folio->index << PAGE_SHIFT; 1246 loff_t offset = from > start ? from - start : 0; 1247 1248 folio_zero_segment(folio, offset, folio_size(folio)); 1249 1250 if (from >= start) 1251 break; 1252 } 1253 1254 f2fs_compress_write_end(inode, fsdata, start_idx, true); 1255 1256 err = filemap_write_and_wait_range(inode->i_mapping, 1257 round_down(from, 1 << log_cluster_size << PAGE_SHIFT), 1258 LLONG_MAX); 1259 if (err) 1260 return err; 1261 1262 truncate_pagecache(inode, from); 1263 1264 return f2fs_do_truncate_blocks(inode, round_up(from, PAGE_SIZE), lock); 1265 } 1266 1267 static int f2fs_write_compressed_pages(struct compress_ctx *cc, 1268 int *submitted, 1269 struct writeback_control *wbc, 1270 enum iostat_type io_type) 1271 { 1272 struct inode *inode = cc->inode; 1273 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1274 struct f2fs_inode_info *fi = F2FS_I(inode); 1275 struct f2fs_io_info fio = { 1276 .sbi = sbi, 1277 .ino = cc->inode->i_ino, 1278 .type = DATA, 1279 .op = REQ_OP_WRITE, 1280 .op_flags = wbc_to_write_flags(wbc), 1281 .old_blkaddr = NEW_ADDR, 1282 .page = NULL, 1283 .encrypted_page = NULL, 1284 .compressed_page = NULL, 1285 .io_type = io_type, 1286 .io_wbc = wbc, 1287 .encrypted = fscrypt_inode_uses_fs_layer_crypto(cc->inode) ? 1288 1 : 0, 1289 }; 1290 struct folio *folio; 1291 struct dnode_of_data dn; 1292 struct node_info ni; 1293 struct compress_io_ctx *cic; 1294 struct f2fs_lock_context lc; 1295 pgoff_t start_idx = start_idx_of_cluster(cc); 1296 unsigned int last_index = cc->cluster_size - 1; 1297 loff_t psize; 1298 int i, err; 1299 bool quota_inode = IS_NOQUOTA(inode); 1300 1301 /* we should bypass data pages to proceed the kworker jobs */ 1302 if (unlikely(f2fs_cp_error(sbi))) { 1303 mapping_set_error(inode->i_mapping, -EIO); 1304 goto out_free; 1305 } 1306 1307 if (quota_inode) { 1308 /* 1309 * We need to wait for node_write to avoid block allocation during 1310 * checkpoint. This can only happen to quota writes which can cause 1311 * the below discard race condition. 1312 */ 1313 f2fs_down_read_trace(&sbi->node_write, &lc); 1314 } else if (!f2fs_trylock_op(sbi, &lc)) { 1315 goto out_free; 1316 } 1317 1318 set_new_dnode(&dn, cc->inode, NULL, NULL, 0); 1319 1320 err = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 1321 if (err) 1322 goto out_unlock_op; 1323 1324 for (i = 0; i < cc->cluster_size; i++) { 1325 if (data_blkaddr(dn.inode, dn.node_folio, 1326 dn.ofs_in_node + i) == NULL_ADDR) 1327 goto out_put_dnode; 1328 } 1329 1330 folio = page_folio(cc->rpages[last_index]); 1331 psize = folio_next_pos(folio); 1332 1333 err = f2fs_get_node_info(fio.sbi, dn.nid, &ni, false); 1334 if (err) 1335 goto out_put_dnode; 1336 1337 fio.version = ni.version; 1338 1339 cic = f2fs_kmem_cache_alloc(cic_entry_slab, GFP_F2FS_ZERO, false, sbi); 1340 if (!cic) 1341 goto out_put_dnode; 1342 1343 cic->magic = F2FS_COMPRESSED_PAGE_MAGIC; 1344 cic->inode = inode; 1345 atomic_set(&cic->pending_pages, cc->valid_nr_cpages); 1346 cic->rpages = page_array_alloc(sbi, cc->cluster_size); 1347 if (!cic->rpages) 1348 goto out_put_cic; 1349 1350 cic->nr_rpages = cc->cluster_size; 1351 1352 for (i = 0; i < cc->valid_nr_cpages; i++) { 1353 f2fs_set_compressed_page(cc->cpages[i], inode, 1354 page_folio(cc->rpages[i + 1])->index, cic); 1355 fio.compressed_page = cc->cpages[i]; 1356 1357 fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_folio, 1358 dn.ofs_in_node + i + 1); 1359 1360 /* wait for GCed page writeback via META_MAPPING */ 1361 f2fs_wait_on_block_writeback(inode, fio.old_blkaddr); 1362 1363 if (fio.encrypted) { 1364 fio.page = cc->rpages[i + 1]; 1365 err = f2fs_encrypt_one_page(&fio); 1366 if (err) 1367 goto out_destroy_crypt; 1368 cc->cpages[i] = fio.encrypted_page; 1369 } 1370 } 1371 1372 set_cluster_writeback(cc); 1373 1374 for (i = 0; i < cc->cluster_size; i++) 1375 cic->rpages[i] = cc->rpages[i]; 1376 1377 for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) { 1378 block_t blkaddr; 1379 1380 blkaddr = f2fs_data_blkaddr(&dn); 1381 fio.page = cc->rpages[i]; 1382 fio.old_blkaddr = blkaddr; 1383 1384 /* cluster header */ 1385 if (i == 0) { 1386 if (blkaddr == COMPRESS_ADDR) 1387 fio.compr_blocks++; 1388 if (__is_valid_data_blkaddr(blkaddr)) 1389 f2fs_invalidate_blocks(sbi, blkaddr, 1); 1390 f2fs_update_data_blkaddr(&dn, COMPRESS_ADDR); 1391 goto unlock_continue; 1392 } 1393 1394 if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr)) 1395 fio.compr_blocks++; 1396 1397 if (i > cc->valid_nr_cpages) { 1398 if (__is_valid_data_blkaddr(blkaddr)) { 1399 f2fs_invalidate_blocks(sbi, blkaddr, 1); 1400 f2fs_update_data_blkaddr(&dn, NEW_ADDR); 1401 } 1402 goto unlock_continue; 1403 } 1404 1405 f2fs_bug_on(fio.sbi, blkaddr == NULL_ADDR); 1406 1407 if (fio.encrypted) 1408 fio.encrypted_page = cc->cpages[i - 1]; 1409 else 1410 fio.compressed_page = cc->cpages[i - 1]; 1411 1412 cc->cpages[i - 1] = NULL; 1413 fio.submitted = 0; 1414 f2fs_outplace_write_data(&dn, &fio); 1415 if (unlikely(!fio.submitted)) { 1416 cancel_cluster_writeback(cc, cic, i); 1417 1418 /* To call fscrypt_finalize_bounce_page */ 1419 i = cc->valid_nr_cpages; 1420 *submitted = 0; 1421 goto out_destroy_crypt; 1422 } 1423 (*submitted)++; 1424 unlock_continue: 1425 inode_dec_dirty_pages(cc->inode); 1426 folio_unlock(fio.folio); 1427 } 1428 1429 if (fio.compr_blocks) 1430 f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false); 1431 f2fs_i_compr_blocks_update(inode, cc->valid_nr_cpages, true); 1432 add_compr_block_stat(inode, cc->valid_nr_cpages); 1433 1434 set_inode_flag(cc->inode, FI_APPEND_WRITE); 1435 1436 f2fs_put_dnode(&dn); 1437 if (quota_inode) 1438 f2fs_up_read_trace(&sbi->node_write, &lc); 1439 else 1440 f2fs_unlock_op(sbi, &lc); 1441 1442 spin_lock(&fi->i_size_lock); 1443 if (fi->last_disk_size < psize) 1444 fi->last_disk_size = psize; 1445 spin_unlock(&fi->i_size_lock); 1446 1447 f2fs_put_rpages(cc); 1448 page_array_free(sbi, cc->cpages, cc->nr_cpages); 1449 cc->cpages = NULL; 1450 f2fs_destroy_compress_ctx(cc, false); 1451 return 0; 1452 1453 out_destroy_crypt: 1454 page_array_free(sbi, cic->rpages, cc->cluster_size); 1455 1456 for (--i; i >= 0; i--) { 1457 if (!cc->cpages[i]) 1458 continue; 1459 fscrypt_finalize_bounce_page(&cc->cpages[i]); 1460 } 1461 out_put_cic: 1462 kmem_cache_free(cic_entry_slab, cic); 1463 out_put_dnode: 1464 f2fs_put_dnode(&dn); 1465 out_unlock_op: 1466 if (quota_inode) 1467 f2fs_up_read_trace(&sbi->node_write, &lc); 1468 else 1469 f2fs_unlock_op(sbi, &lc); 1470 out_free: 1471 for (i = 0; i < cc->valid_nr_cpages; i++) { 1472 f2fs_compress_free_page(cc->cpages[i]); 1473 cc->cpages[i] = NULL; 1474 } 1475 page_array_free(sbi, cc->cpages, cc->nr_cpages); 1476 cc->cpages = NULL; 1477 return -EAGAIN; 1478 } 1479 1480 void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio) 1481 { 1482 struct page *page = &folio->page; 1483 struct f2fs_sb_info *sbi = bio->bi_private; 1484 struct compress_io_ctx *cic = folio->private; 1485 enum count_type type = WB_DATA_TYPE(folio, 1486 f2fs_is_compressed_page(folio)); 1487 int i; 1488 1489 if (unlikely(bio->bi_status != BLK_STS_OK)) 1490 mapping_set_error(cic->inode->i_mapping, -EIO); 1491 1492 f2fs_compress_free_page(page); 1493 1494 if (atomic_dec_return(&cic->pending_pages)) { 1495 dec_page_count(sbi, type); 1496 return; 1497 } 1498 1499 for (i = 0; i < cic->nr_rpages; i++) { 1500 WARN_ON(!cic->rpages[i]); 1501 clear_page_private_gcing(cic->rpages[i]); 1502 end_page_writeback(cic->rpages[i]); 1503 } 1504 1505 page_array_free(sbi, cic->rpages, cic->nr_rpages); 1506 kmem_cache_free(cic_entry_slab, cic); 1507 1508 /* 1509 * Make sure dec_page_count() is the last access to sbi. 1510 * Once it drops the F2FS_WB_CP_DATA counter to zero, the 1511 * unmount thread can proceed to destroy sbi and 1512 * sbi->page_array_slab. 1513 */ 1514 dec_page_count(sbi, type); 1515 } 1516 1517 static int f2fs_write_raw_pages(struct compress_ctx *cc, 1518 int *submitted_p, 1519 struct writeback_control *wbc, 1520 enum iostat_type io_type) 1521 { 1522 struct address_space *mapping = cc->inode->i_mapping; 1523 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); 1524 struct f2fs_lock_context lc; 1525 int submitted, compr_blocks, i; 1526 int ret = 0; 1527 1528 compr_blocks = f2fs_compressed_blocks(cc); 1529 1530 for (i = 0; i < cc->cluster_size; i++) { 1531 if (!cc->rpages[i]) 1532 continue; 1533 1534 redirty_page_for_writepage(wbc, cc->rpages[i]); 1535 unlock_page(cc->rpages[i]); 1536 } 1537 1538 if (compr_blocks < 0) 1539 return compr_blocks; 1540 1541 /* overwrite compressed cluster w/ normal cluster */ 1542 if (compr_blocks > 0) 1543 f2fs_lock_op(sbi, &lc); 1544 1545 for (i = 0; i < cc->cluster_size; i++) { 1546 struct folio *folio; 1547 1548 if (!cc->rpages[i]) 1549 continue; 1550 folio = page_folio(cc->rpages[i]); 1551 retry_write: 1552 folio_lock(folio); 1553 1554 if (folio->mapping != mapping) { 1555 continue_unlock: 1556 folio_unlock(folio); 1557 continue; 1558 } 1559 1560 if (!folio_test_dirty(folio)) 1561 goto continue_unlock; 1562 1563 if (folio_test_writeback(folio)) { 1564 if (wbc->sync_mode == WB_SYNC_NONE) 1565 goto continue_unlock; 1566 f2fs_folio_wait_writeback(folio, DATA, true, true); 1567 } 1568 1569 if (!folio_clear_dirty_for_io(folio)) 1570 goto continue_unlock; 1571 1572 submitted = 0; 1573 ret = f2fs_write_single_data_page(folio, &submitted, 1574 NULL, NULL, wbc, io_type, 1575 compr_blocks, false); 1576 if (ret) { 1577 if (ret == 1) { 1578 ret = 0; 1579 } else if (ret == -EAGAIN) { 1580 ret = 0; 1581 /* 1582 * for quota file, just redirty left pages to 1583 * avoid deadlock caused by cluster update race 1584 * from foreground operation. 1585 */ 1586 if (IS_NOQUOTA(cc->inode)) 1587 goto out; 1588 f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); 1589 goto retry_write; 1590 } 1591 goto out; 1592 } 1593 1594 *submitted_p += submitted; 1595 } 1596 1597 out: 1598 if (compr_blocks > 0) 1599 f2fs_unlock_op(sbi, &lc); 1600 1601 f2fs_balance_fs(sbi, true); 1602 return ret; 1603 } 1604 1605 int f2fs_write_multi_pages(struct compress_ctx *cc, 1606 int *submitted, 1607 struct writeback_control *wbc, 1608 enum iostat_type io_type) 1609 { 1610 int err; 1611 1612 *submitted = 0; 1613 if (cluster_may_compress(cc)) { 1614 err = f2fs_compress_pages(cc); 1615 if (err == -EAGAIN) { 1616 add_compr_block_stat(cc->inode, cc->cluster_size); 1617 goto write; 1618 } else if (err) { 1619 f2fs_put_rpages_wbc(cc, wbc, true, true); 1620 goto destroy_out; 1621 } 1622 1623 err = f2fs_write_compressed_pages(cc, submitted, 1624 wbc, io_type); 1625 if (!err) 1626 return 0; 1627 f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN); 1628 } 1629 write: 1630 f2fs_bug_on(F2FS_I_SB(cc->inode), *submitted); 1631 1632 err = f2fs_write_raw_pages(cc, submitted, wbc, io_type); 1633 f2fs_put_rpages_wbc(cc, wbc, false, false); 1634 destroy_out: 1635 f2fs_destroy_compress_ctx(cc, false); 1636 return err; 1637 } 1638 1639 static inline bool allow_memalloc_for_decomp(struct f2fs_sb_info *sbi, 1640 bool pre_alloc) 1641 { 1642 return pre_alloc ^ f2fs_low_mem_mode(sbi); 1643 } 1644 1645 static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, 1646 bool pre_alloc) 1647 { 1648 const struct f2fs_compress_ops *cops = f2fs_cops[dic->compress_algorithm]; 1649 int i; 1650 1651 if (!allow_memalloc_for_decomp(dic->sbi, pre_alloc)) 1652 return 0; 1653 1654 dic->tpages = page_array_alloc(dic->sbi, dic->cluster_size); 1655 if (!dic->tpages) 1656 return -ENOMEM; 1657 1658 for (i = 0; i < dic->cluster_size; i++) { 1659 if (dic->rpages[i]) { 1660 dic->tpages[i] = dic->rpages[i]; 1661 continue; 1662 } 1663 1664 dic->tpages[i] = f2fs_compress_alloc_page(); 1665 } 1666 1667 dic->rbuf = f2fs_vmap(dic->tpages, dic->cluster_size); 1668 if (!dic->rbuf) 1669 return -ENOMEM; 1670 1671 dic->cbuf = f2fs_vmap(dic->cpages, dic->nr_cpages); 1672 if (!dic->cbuf) 1673 return -ENOMEM; 1674 1675 if (cops->init_decompress_ctx) 1676 return cops->init_decompress_ctx(dic); 1677 1678 return 0; 1679 } 1680 1681 static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic, 1682 bool bypass_destroy_callback, bool pre_alloc) 1683 { 1684 const struct f2fs_compress_ops *cops = f2fs_cops[dic->compress_algorithm]; 1685 1686 if (!allow_memalloc_for_decomp(dic->sbi, pre_alloc)) 1687 return; 1688 1689 if (!bypass_destroy_callback && cops->destroy_decompress_ctx) 1690 cops->destroy_decompress_ctx(dic); 1691 1692 if (dic->cbuf) 1693 vm_unmap_ram(dic->cbuf, dic->nr_cpages); 1694 1695 if (dic->rbuf) 1696 vm_unmap_ram(dic->rbuf, dic->cluster_size); 1697 } 1698 1699 static void f2fs_free_dic(struct decompress_io_ctx *dic, 1700 bool bypass_destroy_callback); 1701 1702 struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) 1703 { 1704 struct decompress_io_ctx *dic; 1705 pgoff_t start_idx = start_idx_of_cluster(cc); 1706 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 1707 int i, ret; 1708 1709 dic = f2fs_kmem_cache_alloc(dic_entry_slab, GFP_F2FS_ZERO, false, sbi); 1710 if (!dic) 1711 return ERR_PTR(-ENOMEM); 1712 1713 dic->rpages = page_array_alloc(sbi, cc->cluster_size); 1714 if (!dic->rpages) { 1715 kmem_cache_free(dic_entry_slab, dic); 1716 return ERR_PTR(-ENOMEM); 1717 } 1718 1719 dic->magic = F2FS_COMPRESSED_PAGE_MAGIC; 1720 dic->inode = cc->inode; 1721 dic->sbi = sbi; 1722 dic->compress_algorithm = F2FS_I(cc->inode)->i_compress_algorithm; 1723 atomic_set(&dic->remaining_pages, cc->nr_cpages); 1724 dic->cluster_idx = cc->cluster_idx; 1725 dic->cluster_size = cc->cluster_size; 1726 dic->log_cluster_size = cc->log_cluster_size; 1727 dic->nr_cpages = cc->nr_cpages; 1728 refcount_set(&dic->refcnt, 1); 1729 dic->failed = false; 1730 dic->vi = cc->vi; 1731 1732 for (i = 0; i < dic->cluster_size; i++) 1733 dic->rpages[i] = cc->rpages[i]; 1734 dic->nr_rpages = cc->cluster_size; 1735 1736 dic->cpages = page_array_alloc(sbi, dic->nr_cpages); 1737 if (!dic->cpages) { 1738 ret = -ENOMEM; 1739 goto out_free; 1740 } 1741 1742 for (i = 0; i < dic->nr_cpages; i++) { 1743 struct page *page; 1744 1745 page = f2fs_compress_alloc_page(); 1746 f2fs_set_compressed_page(page, cc->inode, 1747 start_idx + i + 1, dic); 1748 dic->cpages[i] = page; 1749 } 1750 1751 ret = f2fs_prepare_decomp_mem(dic, true); 1752 if (ret) 1753 goto out_free; 1754 1755 return dic; 1756 1757 out_free: 1758 f2fs_free_dic(dic, true); 1759 return ERR_PTR(ret); 1760 } 1761 1762 static void f2fs_free_dic(struct decompress_io_ctx *dic, 1763 bool bypass_destroy_callback) 1764 { 1765 int i; 1766 /* use sbi in dic to avoid UFA of dic->inode*/ 1767 struct f2fs_sb_info *sbi = dic->sbi; 1768 1769 f2fs_release_decomp_mem(dic, bypass_destroy_callback, true); 1770 1771 if (dic->tpages) { 1772 for (i = 0; i < dic->cluster_size; i++) { 1773 if (dic->rpages[i]) 1774 continue; 1775 if (!dic->tpages[i]) 1776 continue; 1777 f2fs_compress_free_page(dic->tpages[i]); 1778 } 1779 page_array_free(sbi, dic->tpages, dic->cluster_size); 1780 } 1781 1782 if (dic->cpages) { 1783 for (i = 0; i < dic->nr_cpages; i++) { 1784 if (!dic->cpages[i]) 1785 continue; 1786 f2fs_compress_free_page(dic->cpages[i]); 1787 } 1788 page_array_free(sbi, dic->cpages, dic->nr_cpages); 1789 } 1790 1791 page_array_free(sbi, dic->rpages, dic->nr_rpages); 1792 kmem_cache_free(dic_entry_slab, dic); 1793 } 1794 1795 static void f2fs_late_free_dic(struct work_struct *work) 1796 { 1797 struct decompress_io_ctx *dic = 1798 container_of(work, struct decompress_io_ctx, free_work); 1799 1800 f2fs_free_dic(dic, false); 1801 } 1802 1803 static void f2fs_put_dic(struct decompress_io_ctx *dic, bool in_task) 1804 { 1805 if (refcount_dec_and_test(&dic->refcnt)) { 1806 if (in_task) { 1807 f2fs_free_dic(dic, false); 1808 } else { 1809 INIT_WORK(&dic->free_work, f2fs_late_free_dic); 1810 queue_work(dic->sbi->post_read_wq, &dic->free_work); 1811 } 1812 } 1813 } 1814 1815 static void f2fs_verify_cluster(struct work_struct *work) 1816 { 1817 struct decompress_io_ctx *dic = 1818 container_of(work, struct decompress_io_ctx, verity_work); 1819 int i; 1820 1821 /* Verify, update, and unlock the decompressed pages. */ 1822 for (i = 0; i < dic->cluster_size; i++) { 1823 struct page *rpage = dic->rpages[i]; 1824 struct folio *rfolio; 1825 1826 if (!rpage) 1827 continue; 1828 rfolio = page_folio(rpage); 1829 if (fsverity_verify_folio(dic->vi, rfolio)) 1830 folio_mark_uptodate(rfolio); 1831 folio_unlock(rfolio); 1832 } 1833 1834 f2fs_put_dic(dic, true); 1835 } 1836 1837 /* 1838 * This is called when a compressed cluster has been decompressed 1839 * (or failed to be read and/or decompressed). 1840 */ 1841 void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed, 1842 bool in_task) 1843 { 1844 int i; 1845 1846 if (IS_ENABLED(CONFIG_FS_VERITY) && !failed && dic->vi) { 1847 /* 1848 * Note that to avoid deadlocks, the verity work can't be done 1849 * on the decompression workqueue. This is because verifying 1850 * the data pages can involve reading metadata pages from the 1851 * file, and these metadata pages may be compressed. 1852 */ 1853 INIT_WORK(&dic->verity_work, f2fs_verify_cluster); 1854 fsverity_enqueue_verify_work(&dic->verity_work); 1855 return; 1856 } 1857 1858 /* Update and unlock the cluster's pagecache pages. */ 1859 for (i = 0; i < dic->cluster_size; i++) { 1860 struct page *rpage = dic->rpages[i]; 1861 1862 if (!rpage) 1863 continue; 1864 1865 if (failed) 1866 ClearPageUptodate(rpage); 1867 else 1868 SetPageUptodate(rpage); 1869 unlock_page(rpage); 1870 } 1871 1872 /* 1873 * Release the reference to the decompress_io_ctx that was being held 1874 * for I/O completion. 1875 */ 1876 f2fs_put_dic(dic, in_task); 1877 } 1878 1879 /* 1880 * Put a reference to a compressed folio's decompress_io_ctx. 1881 * 1882 * This is called when the folio is no longer needed and can be freed. 1883 */ 1884 void f2fs_put_folio_dic(struct folio *folio, bool in_task) 1885 { 1886 struct decompress_io_ctx *dic = folio->private; 1887 1888 f2fs_put_dic(dic, in_task); 1889 } 1890 1891 /* 1892 * check whether cluster blocks are contiguous, and add extent cache entry 1893 * only if cluster blocks are logically and physically contiguous. 1894 */ 1895 unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn, 1896 unsigned int ofs_in_node) 1897 { 1898 bool compressed = data_blkaddr(dn->inode, dn->node_folio, 1899 ofs_in_node) == COMPRESS_ADDR; 1900 int i = compressed ? 1 : 0; 1901 block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_folio, 1902 ofs_in_node + i); 1903 1904 for (i += 1; i < F2FS_I(dn->inode)->i_cluster_size; i++) { 1905 block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio, 1906 ofs_in_node + i); 1907 1908 if (!__is_valid_data_blkaddr(blkaddr)) 1909 break; 1910 if (first_blkaddr + i - (compressed ? 1 : 0) != blkaddr) 1911 return 0; 1912 } 1913 1914 return compressed ? i - 1 : i; 1915 } 1916 1917 const struct address_space_operations f2fs_compress_aops = { 1918 .release_folio = f2fs_release_folio, 1919 .invalidate_folio = f2fs_invalidate_folio, 1920 .migrate_folio = filemap_migrate_folio, 1921 }; 1922 1923 struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi) 1924 { 1925 return sbi->compress_inode->i_mapping; 1926 } 1927 1928 void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi, 1929 block_t blkaddr, unsigned int len) 1930 { 1931 if (!sbi->compress_inode) 1932 return; 1933 invalidate_mapping_pages(COMPRESS_MAPPING(sbi), blkaddr, blkaddr + len - 1); 1934 } 1935 1936 static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, 1937 struct folio *folio, nid_t ino, block_t blkaddr) 1938 { 1939 struct folio *cfolio; 1940 int ret; 1941 1942 if (!test_opt(sbi, COMPRESS_CACHE)) 1943 return; 1944 1945 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ)) 1946 return; 1947 1948 if (!f2fs_available_free_memory(sbi, COMPRESS_PAGE)) 1949 return; 1950 1951 cfolio = filemap_get_folio(COMPRESS_MAPPING(sbi), blkaddr); 1952 if (!IS_ERR(cfolio)) { 1953 f2fs_folio_put(cfolio, false); 1954 return; 1955 } 1956 1957 cfolio = filemap_alloc_folio(__GFP_NOWARN | __GFP_IO, 0, NULL); 1958 if (!cfolio) 1959 return; 1960 1961 ret = filemap_add_folio(COMPRESS_MAPPING(sbi), cfolio, 1962 blkaddr, GFP_NOFS); 1963 if (ret) { 1964 f2fs_folio_put(cfolio, false); 1965 return; 1966 } 1967 1968 folio_set_f2fs_data(cfolio, ino); 1969 1970 memcpy(folio_address(cfolio), folio_address(folio), PAGE_SIZE); 1971 folio_mark_uptodate(cfolio); 1972 f2fs_folio_put(cfolio, true); 1973 } 1974 1975 bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, struct folio *folio, 1976 block_t blkaddr) 1977 { 1978 struct folio *cfolio; 1979 bool hitted = false; 1980 1981 if (!test_opt(sbi, COMPRESS_CACHE)) 1982 return false; 1983 1984 cfolio = f2fs_filemap_get_folio(COMPRESS_MAPPING(sbi), 1985 blkaddr, FGP_LOCK | FGP_NOWAIT, GFP_NOFS); 1986 if (!IS_ERR(cfolio)) { 1987 if (folio_test_uptodate(cfolio)) { 1988 atomic_inc(&sbi->compress_page_hit); 1989 memcpy(folio_address(folio), 1990 folio_address(cfolio), folio_size(folio)); 1991 hitted = true; 1992 } 1993 f2fs_folio_put(cfolio, true); 1994 } 1995 1996 return hitted; 1997 } 1998 1999 void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino) 2000 { 2001 struct address_space *mapping = COMPRESS_MAPPING(sbi); 2002 struct folio_batch fbatch; 2003 pgoff_t index = 0; 2004 pgoff_t end = MAX_BLKADDR(sbi); 2005 2006 if (!mapping->nrpages) 2007 return; 2008 2009 folio_batch_init(&fbatch); 2010 2011 do { 2012 unsigned int nr, i; 2013 2014 nr = filemap_get_folios(mapping, &index, end - 1, &fbatch); 2015 if (!nr) 2016 break; 2017 2018 for (i = 0; i < nr; i++) { 2019 struct folio *folio = fbatch.folios[i]; 2020 2021 folio_lock(folio); 2022 if (folio->mapping != mapping) { 2023 folio_unlock(folio); 2024 continue; 2025 } 2026 2027 if (ino != folio_get_f2fs_data(folio)) { 2028 folio_unlock(folio); 2029 continue; 2030 } 2031 2032 generic_error_remove_folio(mapping, folio); 2033 folio_unlock(folio); 2034 } 2035 folio_batch_release(&fbatch); 2036 cond_resched(); 2037 } while (index < end); 2038 } 2039 2040 int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) 2041 { 2042 struct inode *inode; 2043 2044 if (!test_opt(sbi, COMPRESS_CACHE)) 2045 return 0; 2046 2047 inode = f2fs_iget(sbi->sb, F2FS_COMPRESS_INO(sbi)); 2048 if (IS_ERR(inode)) 2049 return PTR_ERR(inode); 2050 sbi->compress_inode = inode; 2051 2052 sbi->compress_percent = COMPRESS_PERCENT; 2053 sbi->compress_watermark = COMPRESS_WATERMARK; 2054 2055 atomic_set(&sbi->compress_page_hit, 0); 2056 2057 return 0; 2058 } 2059 2060 void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) 2061 { 2062 if (!sbi->compress_inode) 2063 return; 2064 iput(sbi->compress_inode); 2065 sbi->compress_inode = NULL; 2066 } 2067 2068 int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) 2069 { 2070 dev_t dev = sbi->sb->s_bdev->bd_dev; 2071 char slab_name[35]; 2072 2073 if (!f2fs_sb_has_compression(sbi)) 2074 return 0; 2075 2076 sprintf(slab_name, "f2fs_page_array_entry-%u:%u", MAJOR(dev), MINOR(dev)); 2077 2078 sbi->page_array_slab_size = sizeof(struct page *) << 2079 F2FS_OPTION(sbi).compress_log_size; 2080 2081 sbi->page_array_slab = f2fs_kmem_cache_create(slab_name, 2082 sbi->page_array_slab_size); 2083 return sbi->page_array_slab ? 0 : -ENOMEM; 2084 } 2085 2086 void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) 2087 { 2088 kmem_cache_destroy(sbi->page_array_slab); 2089 } 2090 2091 int __init f2fs_init_compress_cache(void) 2092 { 2093 cic_entry_slab = f2fs_kmem_cache_create("f2fs_cic_entry", 2094 sizeof(struct compress_io_ctx)); 2095 if (!cic_entry_slab) 2096 return -ENOMEM; 2097 dic_entry_slab = f2fs_kmem_cache_create("f2fs_dic_entry", 2098 sizeof(struct decompress_io_ctx)); 2099 if (!dic_entry_slab) 2100 goto free_cic; 2101 return 0; 2102 free_cic: 2103 kmem_cache_destroy(cic_entry_slab); 2104 return -ENOMEM; 2105 } 2106 2107 void f2fs_destroy_compress_cache(void) 2108 { 2109 kmem_cache_destroy(dic_entry_slab); 2110 kmem_cache_destroy(cic_entry_slab); 2111 } 2112