1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * f2fs compress support 4 * 5 * Copyright (c) 2019 Chao Yu <chao@kernel.org> 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/moduleparam.h> 11 #include <linux/writeback.h> 12 #include <linux/backing-dev.h> 13 #include <linux/lzo.h> 14 #include <linux/lz4.h> 15 #include <linux/zstd.h> 16 #include <linux/pagevec.h> 17 18 #include "f2fs.h" 19 #include "node.h" 20 #include "segment.h" 21 #include <trace/events/f2fs.h> 22 23 static struct kmem_cache *cic_entry_slab; 24 static struct kmem_cache *dic_entry_slab; 25 26 static void *page_array_alloc(struct f2fs_sb_info *sbi, int nr) 27 { 28 unsigned int size = sizeof(struct page *) * nr; 29 30 if (likely(size <= sbi->page_array_slab_size)) 31 return f2fs_kmem_cache_alloc(sbi->page_array_slab, 32 GFP_F2FS_ZERO, false, sbi); 33 return f2fs_kzalloc(sbi, size, GFP_NOFS); 34 } 35 36 static void page_array_free(struct f2fs_sb_info *sbi, void *pages, int nr) 37 { 38 unsigned int size = sizeof(struct page *) * nr; 39 40 if (!pages) 41 return; 42 43 if (likely(size <= sbi->page_array_slab_size)) 44 kmem_cache_free(sbi->page_array_slab, pages); 45 else 46 kfree(pages); 47 } 48 49 struct f2fs_compress_ops { 50 int (*init_compress_ctx)(struct compress_ctx *cc); 51 void (*destroy_compress_ctx)(struct compress_ctx *cc); 52 int (*compress_pages)(struct compress_ctx *cc); 53 int (*init_decompress_ctx)(struct decompress_io_ctx *dic); 54 void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic); 55 int (*decompress_pages)(struct decompress_io_ctx *dic); 56 bool (*is_level_valid)(int level); 57 }; 58 59 static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index) 60 { 61 return index & (cc->cluster_size - 1); 62 } 63 64 static pgoff_t cluster_idx(struct compress_ctx *cc, pgoff_t index) 65 { 66 return index >> cc->log_cluster_size; 67 } 68 69 static pgoff_t start_idx_of_cluster(struct compress_ctx *cc) 70 { 71 return cc->cluster_idx << cc->log_cluster_size; 72 } 73 74 bool f2fs_is_compressed_page(struct folio *folio) 75 { 76 if (!folio->private) 77 return false; 78 if (folio_test_f2fs_nonpointer(folio)) 79 return false; 80 81 f2fs_bug_on(F2FS_F_SB(folio), 82 *((u32 *)folio->private) != F2FS_COMPRESSED_PAGE_MAGIC); 83 return true; 84 } 85 86 static void f2fs_set_compressed_page(struct page *page, 87 struct inode *inode, pgoff_t index, void *data) 88 { 89 struct folio *folio = page_folio(page); 90 91 folio_attach_private(folio, (void *)data); 92 93 /* i_crypto_info and iv index */ 94 folio->index = index; 95 folio->mapping = inode->i_mapping; 96 } 97 98 static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock) 99 { 100 int i; 101 102 for (i = 0; i < len; i++) { 103 if (!cc->rpages[i]) 104 continue; 105 if (unlock) 106 unlock_page(cc->rpages[i]); 107 else 108 put_page(cc->rpages[i]); 109 } 110 } 111 112 static void f2fs_put_rpages(struct compress_ctx *cc) 113 { 114 f2fs_drop_rpages(cc, cc->cluster_size, false); 115 } 116 117 static void f2fs_unlock_rpages(struct compress_ctx *cc, int len) 118 { 119 f2fs_drop_rpages(cc, len, true); 120 } 121 122 static void f2fs_put_rpages_wbc(struct compress_ctx *cc, 123 struct writeback_control *wbc, bool redirty, bool unlock) 124 { 125 unsigned int i; 126 127 for (i = 0; i < cc->cluster_size; i++) { 128 if (!cc->rpages[i]) 129 continue; 130 if (redirty) 131 redirty_page_for_writepage(wbc, cc->rpages[i]); 132 f2fs_put_page(cc->rpages[i], unlock); 133 } 134 } 135 136 struct folio *f2fs_compress_control_folio(struct folio *folio) 137 { 138 struct compress_io_ctx *ctx = folio->private; 139 140 return page_folio(ctx->rpages[0]); 141 } 142 143 int f2fs_init_compress_ctx(struct compress_ctx *cc) 144 { 145 if (cc->rpages) 146 return 0; 147 148 cc->rpages = page_array_alloc(F2FS_I_SB(cc->inode), cc->cluster_size); 149 return cc->rpages ? 0 : -ENOMEM; 150 } 151 152 void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse) 153 { 154 page_array_free(F2FS_I_SB(cc->inode), cc->rpages, cc->cluster_size); 155 cc->rpages = NULL; 156 cc->nr_rpages = 0; 157 cc->nr_cpages = 0; 158 cc->valid_nr_cpages = 0; 159 if (!reuse) 160 cc->cluster_idx = NULL_CLUSTER; 161 } 162 163 void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct folio *folio) 164 { 165 unsigned int cluster_ofs; 166 167 if (!f2fs_cluster_can_merge_page(cc, folio->index)) 168 f2fs_bug_on(F2FS_I_SB(cc->inode), 1); 169 170 cluster_ofs = offset_in_cluster(cc, folio->index); 171 cc->rpages[cluster_ofs] = folio_page(folio, 0); 172 cc->nr_rpages++; 173 cc->cluster_idx = cluster_idx(cc, folio->index); 174 } 175 176 #ifdef CONFIG_F2FS_FS_LZO 177 static int lzo_init_compress_ctx(struct compress_ctx *cc) 178 { 179 cc->private = f2fs_vmalloc(F2FS_I_SB(cc->inode), 180 LZO1X_MEM_COMPRESS); 181 if (!cc->private) 182 return -ENOMEM; 183 184 cc->clen = lzo1x_worst_compress(PAGE_SIZE << cc->log_cluster_size); 185 return 0; 186 } 187 188 static void lzo_destroy_compress_ctx(struct compress_ctx *cc) 189 { 190 vfree(cc->private); 191 cc->private = NULL; 192 } 193 194 static int lzo_compress_pages(struct compress_ctx *cc) 195 { 196 int ret; 197 198 ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, 199 &cc->clen, cc->private); 200 if (ret != LZO_E_OK) { 201 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 202 "lzo compress failed, ret:%d", ret); 203 return -EIO; 204 } 205 return 0; 206 } 207 208 static int lzo_decompress_pages(struct decompress_io_ctx *dic) 209 { 210 int ret; 211 212 ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen, 213 dic->rbuf, &dic->rlen); 214 if (ret != LZO_E_OK) { 215 f2fs_err_ratelimited(dic->sbi, 216 "lzo decompress failed, ret:%d", ret); 217 return -EIO; 218 } 219 220 if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) { 221 f2fs_err_ratelimited(dic->sbi, 222 "lzo invalid rlen:%zu, expected:%lu", 223 dic->rlen, PAGE_SIZE << dic->log_cluster_size); 224 return -EIO; 225 } 226 return 0; 227 } 228 229 static const struct f2fs_compress_ops f2fs_lzo_ops = { 230 .init_compress_ctx = lzo_init_compress_ctx, 231 .destroy_compress_ctx = lzo_destroy_compress_ctx, 232 .compress_pages = lzo_compress_pages, 233 .decompress_pages = lzo_decompress_pages, 234 }; 235 #endif 236 237 #ifdef CONFIG_F2FS_FS_LZ4 238 static int lz4_init_compress_ctx(struct compress_ctx *cc) 239 { 240 unsigned int size = LZ4_MEM_COMPRESS; 241 242 #ifdef CONFIG_F2FS_FS_LZ4HC 243 if (F2FS_I(cc->inode)->i_compress_level) 244 size = LZ4HC_MEM_COMPRESS; 245 #endif 246 247 cc->private = f2fs_vmalloc(F2FS_I_SB(cc->inode), size); 248 if (!cc->private) 249 return -ENOMEM; 250 251 /* 252 * we do not change cc->clen to LZ4_compressBound(inputsize) to 253 * adapt worst compress case, because lz4 compressor can handle 254 * output budget properly. 255 */ 256 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; 257 return 0; 258 } 259 260 static void lz4_destroy_compress_ctx(struct compress_ctx *cc) 261 { 262 vfree(cc->private); 263 cc->private = NULL; 264 } 265 266 static int lz4_compress_pages(struct compress_ctx *cc) 267 { 268 int len = -EINVAL; 269 unsigned char level = F2FS_I(cc->inode)->i_compress_level; 270 271 if (!level) 272 len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen, 273 cc->clen, cc->private); 274 #ifdef CONFIG_F2FS_FS_LZ4HC 275 else 276 len = LZ4_compress_HC(cc->rbuf, cc->cbuf->cdata, cc->rlen, 277 cc->clen, level, cc->private); 278 #endif 279 if (len < 0) 280 return len; 281 if (!len) 282 return -EAGAIN; 283 284 cc->clen = len; 285 return 0; 286 } 287 288 static int lz4_decompress_pages(struct decompress_io_ctx *dic) 289 { 290 int ret; 291 292 ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf, 293 dic->clen, dic->rlen); 294 if (ret < 0) { 295 f2fs_err_ratelimited(dic->sbi, 296 "lz4 decompress failed, ret:%d", ret); 297 return -EIO; 298 } 299 300 if (ret != PAGE_SIZE << dic->log_cluster_size) { 301 f2fs_err_ratelimited(dic->sbi, 302 "lz4 invalid ret:%d, expected:%lu", 303 ret, PAGE_SIZE << dic->log_cluster_size); 304 return -EIO; 305 } 306 return 0; 307 } 308 309 static bool lz4_is_level_valid(int lvl) 310 { 311 #ifdef CONFIG_F2FS_FS_LZ4HC 312 return !lvl || (lvl >= LZ4HC_MIN_CLEVEL && lvl <= LZ4HC_MAX_CLEVEL); 313 #else 314 return lvl == 0; 315 #endif 316 } 317 318 static const struct f2fs_compress_ops f2fs_lz4_ops = { 319 .init_compress_ctx = lz4_init_compress_ctx, 320 .destroy_compress_ctx = lz4_destroy_compress_ctx, 321 .compress_pages = lz4_compress_pages, 322 .decompress_pages = lz4_decompress_pages, 323 .is_level_valid = lz4_is_level_valid, 324 }; 325 #endif 326 327 #ifdef CONFIG_F2FS_FS_ZSTD 328 static int zstd_init_compress_ctx(struct compress_ctx *cc) 329 { 330 zstd_parameters params; 331 zstd_cstream *stream; 332 void *workspace; 333 unsigned int workspace_size; 334 unsigned char level = F2FS_I(cc->inode)->i_compress_level; 335 336 /* Need to remain this for backward compatibility */ 337 if (!level) 338 level = F2FS_ZSTD_DEFAULT_CLEVEL; 339 340 params = zstd_get_params(level, cc->rlen); 341 workspace_size = zstd_cstream_workspace_bound(¶ms.cParams); 342 343 workspace = f2fs_vmalloc(F2FS_I_SB(cc->inode), workspace_size); 344 if (!workspace) 345 return -ENOMEM; 346 347 stream = zstd_init_cstream(¶ms, 0, workspace, workspace_size); 348 if (!stream) { 349 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 350 "%s zstd_init_cstream failed", __func__); 351 vfree(workspace); 352 return -EIO; 353 } 354 355 cc->private = workspace; 356 cc->private2 = stream; 357 358 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; 359 return 0; 360 } 361 362 static void zstd_destroy_compress_ctx(struct compress_ctx *cc) 363 { 364 vfree(cc->private); 365 cc->private = NULL; 366 cc->private2 = NULL; 367 } 368 369 static int zstd_compress_pages(struct compress_ctx *cc) 370 { 371 zstd_cstream *stream = cc->private2; 372 zstd_in_buffer inbuf; 373 zstd_out_buffer outbuf; 374 int src_size = cc->rlen; 375 int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE; 376 int ret; 377 378 inbuf.pos = 0; 379 inbuf.src = cc->rbuf; 380 inbuf.size = src_size; 381 382 outbuf.pos = 0; 383 outbuf.dst = cc->cbuf->cdata; 384 outbuf.size = dst_size; 385 386 ret = zstd_compress_stream(stream, &outbuf, &inbuf); 387 if (zstd_is_error(ret)) { 388 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 389 "%s zstd_compress_stream failed, ret: %d", 390 __func__, zstd_get_error_code(ret)); 391 return -EIO; 392 } 393 394 ret = zstd_end_stream(stream, &outbuf); 395 if (zstd_is_error(ret)) { 396 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 397 "%s zstd_end_stream returned %d", 398 __func__, zstd_get_error_code(ret)); 399 return -EIO; 400 } 401 402 /* 403 * there is compressed data remained in intermediate buffer due to 404 * no more space in cbuf.cdata 405 */ 406 if (ret) 407 return -EAGAIN; 408 409 cc->clen = outbuf.pos; 410 return 0; 411 } 412 413 static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic) 414 { 415 zstd_dstream *stream; 416 void *workspace; 417 unsigned int workspace_size; 418 unsigned int max_window_size = 419 MAX_COMPRESS_WINDOW_SIZE(dic->log_cluster_size); 420 421 workspace_size = zstd_dstream_workspace_bound(max_window_size); 422 423 workspace = f2fs_vmalloc(dic->sbi, workspace_size); 424 if (!workspace) 425 return -ENOMEM; 426 427 stream = zstd_init_dstream(max_window_size, workspace, workspace_size); 428 if (!stream) { 429 f2fs_err_ratelimited(dic->sbi, 430 "%s zstd_init_dstream failed", __func__); 431 vfree(workspace); 432 return -EIO; 433 } 434 435 dic->private = workspace; 436 dic->private2 = stream; 437 438 return 0; 439 } 440 441 static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic) 442 { 443 vfree(dic->private); 444 dic->private = NULL; 445 dic->private2 = NULL; 446 } 447 448 static int zstd_decompress_pages(struct decompress_io_ctx *dic) 449 { 450 zstd_dstream *stream = dic->private2; 451 zstd_in_buffer inbuf; 452 zstd_out_buffer outbuf; 453 int ret; 454 455 inbuf.pos = 0; 456 inbuf.src = dic->cbuf->cdata; 457 inbuf.size = dic->clen; 458 459 outbuf.pos = 0; 460 outbuf.dst = dic->rbuf; 461 outbuf.size = dic->rlen; 462 463 ret = zstd_decompress_stream(stream, &outbuf, &inbuf); 464 if (zstd_is_error(ret)) { 465 f2fs_err_ratelimited(dic->sbi, 466 "%s zstd_decompress_stream failed, ret: %d", 467 __func__, zstd_get_error_code(ret)); 468 return -EIO; 469 } 470 471 if (dic->rlen != outbuf.pos) { 472 f2fs_err_ratelimited(dic->sbi, 473 "%s ZSTD invalid rlen:%zu, expected:%lu", 474 __func__, dic->rlen, 475 PAGE_SIZE << dic->log_cluster_size); 476 return -EIO; 477 } 478 479 return 0; 480 } 481 482 static bool zstd_is_level_valid(int lvl) 483 { 484 return lvl >= zstd_min_clevel() && lvl <= zstd_max_clevel(); 485 } 486 487 static const struct f2fs_compress_ops f2fs_zstd_ops = { 488 .init_compress_ctx = zstd_init_compress_ctx, 489 .destroy_compress_ctx = zstd_destroy_compress_ctx, 490 .compress_pages = zstd_compress_pages, 491 .init_decompress_ctx = zstd_init_decompress_ctx, 492 .destroy_decompress_ctx = zstd_destroy_decompress_ctx, 493 .decompress_pages = zstd_decompress_pages, 494 .is_level_valid = zstd_is_level_valid, 495 }; 496 #endif 497 498 #ifdef CONFIG_F2FS_FS_LZO 499 #ifdef CONFIG_F2FS_FS_LZORLE 500 static int lzorle_compress_pages(struct compress_ctx *cc) 501 { 502 int ret; 503 504 ret = lzorle1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, 505 &cc->clen, cc->private); 506 if (ret != LZO_E_OK) { 507 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 508 "lzo-rle compress failed, ret:%d", ret); 509 return -EIO; 510 } 511 return 0; 512 } 513 514 static const struct f2fs_compress_ops f2fs_lzorle_ops = { 515 .init_compress_ctx = lzo_init_compress_ctx, 516 .destroy_compress_ctx = lzo_destroy_compress_ctx, 517 .compress_pages = lzorle_compress_pages, 518 .decompress_pages = lzo_decompress_pages, 519 }; 520 #endif 521 #endif 522 523 static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = { 524 #ifdef CONFIG_F2FS_FS_LZO 525 &f2fs_lzo_ops, 526 #else 527 NULL, 528 #endif 529 #ifdef CONFIG_F2FS_FS_LZ4 530 &f2fs_lz4_ops, 531 #else 532 NULL, 533 #endif 534 #ifdef CONFIG_F2FS_FS_ZSTD 535 &f2fs_zstd_ops, 536 #else 537 NULL, 538 #endif 539 #if defined(CONFIG_F2FS_FS_LZO) && defined(CONFIG_F2FS_FS_LZORLE) 540 &f2fs_lzorle_ops, 541 #else 542 NULL, 543 #endif 544 }; 545 546 bool f2fs_is_compress_backend_ready(struct inode *inode) 547 { 548 if (!f2fs_compressed_file(inode)) 549 return true; 550 return f2fs_cops[F2FS_I(inode)->i_compress_algorithm]; 551 } 552 553 bool f2fs_is_compress_level_valid(int alg, int lvl) 554 { 555 const struct f2fs_compress_ops *cops = f2fs_cops[alg]; 556 557 if (cops->is_level_valid) 558 return cops->is_level_valid(lvl); 559 560 return lvl == 0; 561 } 562 563 static mempool_t *compress_page_pool; 564 static int num_compress_pages = 512; 565 module_param(num_compress_pages, uint, 0444); 566 MODULE_PARM_DESC(num_compress_pages, 567 "Number of intermediate compress pages to preallocate"); 568 569 int __init f2fs_init_compress_mempool(void) 570 { 571 compress_page_pool = mempool_create_page_pool(num_compress_pages, 0); 572 return compress_page_pool ? 0 : -ENOMEM; 573 } 574 575 void f2fs_destroy_compress_mempool(void) 576 { 577 mempool_destroy(compress_page_pool); 578 } 579 580 static struct page *f2fs_compress_alloc_page(void) 581 { 582 struct page *page; 583 584 page = mempool_alloc(compress_page_pool, GFP_NOFS); 585 lock_page(page); 586 587 return page; 588 } 589 590 static void f2fs_compress_free_page(struct page *page) 591 { 592 struct folio *folio; 593 594 if (!page) 595 return; 596 folio = page_folio(page); 597 folio_detach_private(folio); 598 folio->mapping = NULL; 599 folio_unlock(folio); 600 mempool_free(page, compress_page_pool); 601 } 602 603 #define MAX_VMAP_RETRIES 3 604 605 static void *f2fs_vmap(struct page **pages, unsigned int count) 606 { 607 int i; 608 void *buf = NULL; 609 610 for (i = 0; i < MAX_VMAP_RETRIES; i++) { 611 buf = vm_map_ram(pages, count, -1); 612 if (buf) 613 break; 614 vm_unmap_aliases(); 615 } 616 return buf; 617 } 618 619 static int f2fs_compress_pages(struct compress_ctx *cc) 620 { 621 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 622 struct f2fs_inode_info *fi = F2FS_I(cc->inode); 623 const struct f2fs_compress_ops *cops = 624 f2fs_cops[fi->i_compress_algorithm]; 625 unsigned int max_len, new_nr_cpages; 626 u32 chksum = 0; 627 int i, ret; 628 629 trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx, 630 cc->cluster_size, fi->i_compress_algorithm); 631 632 if (cops->init_compress_ctx) { 633 ret = cops->init_compress_ctx(cc); 634 if (ret) 635 goto out; 636 } 637 638 max_len = COMPRESS_HEADER_SIZE + cc->clen; 639 cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE); 640 cc->valid_nr_cpages = cc->nr_cpages; 641 642 cc->cpages = page_array_alloc(sbi, cc->nr_cpages); 643 if (!cc->cpages) { 644 ret = -ENOMEM; 645 goto destroy_compress_ctx; 646 } 647 648 for (i = 0; i < cc->nr_cpages; i++) 649 cc->cpages[i] = f2fs_compress_alloc_page(); 650 651 cc->rbuf = f2fs_vmap(cc->rpages, cc->cluster_size); 652 if (!cc->rbuf) { 653 ret = -ENOMEM; 654 goto out_free_cpages; 655 } 656 657 cc->cbuf = f2fs_vmap(cc->cpages, cc->nr_cpages); 658 if (!cc->cbuf) { 659 ret = -ENOMEM; 660 goto out_vunmap_rbuf; 661 } 662 663 ret = cops->compress_pages(cc); 664 if (ret) 665 goto out_vunmap_cbuf; 666 667 max_len = PAGE_SIZE * (cc->cluster_size - 1) - COMPRESS_HEADER_SIZE; 668 669 if (cc->clen > max_len) { 670 ret = -EAGAIN; 671 goto out_vunmap_cbuf; 672 } 673 674 cc->cbuf->clen = cpu_to_le32(cc->clen); 675 676 if (fi->i_compress_flag & BIT(COMPRESS_CHKSUM)) 677 chksum = f2fs_crc32(cc->cbuf->cdata, cc->clen); 678 cc->cbuf->chksum = cpu_to_le32(chksum); 679 680 for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++) 681 cc->cbuf->reserved[i] = cpu_to_le32(0); 682 683 new_nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE); 684 685 /* zero out any unused part of the last page */ 686 memset(&cc->cbuf->cdata[cc->clen], 0, 687 (new_nr_cpages * PAGE_SIZE) - 688 (cc->clen + COMPRESS_HEADER_SIZE)); 689 690 vm_unmap_ram(cc->cbuf, cc->nr_cpages); 691 vm_unmap_ram(cc->rbuf, cc->cluster_size); 692 693 for (i = new_nr_cpages; i < cc->nr_cpages; i++) { 694 f2fs_compress_free_page(cc->cpages[i]); 695 cc->cpages[i] = NULL; 696 } 697 698 if (cops->destroy_compress_ctx) 699 cops->destroy_compress_ctx(cc); 700 701 cc->valid_nr_cpages = new_nr_cpages; 702 703 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, 704 cc->clen, ret); 705 return 0; 706 707 out_vunmap_cbuf: 708 vm_unmap_ram(cc->cbuf, cc->nr_cpages); 709 out_vunmap_rbuf: 710 vm_unmap_ram(cc->rbuf, cc->cluster_size); 711 out_free_cpages: 712 for (i = 0; i < cc->nr_cpages; i++) { 713 if (cc->cpages[i]) 714 f2fs_compress_free_page(cc->cpages[i]); 715 } 716 page_array_free(sbi, cc->cpages, cc->nr_cpages); 717 cc->cpages = NULL; 718 destroy_compress_ctx: 719 if (cops->destroy_compress_ctx) 720 cops->destroy_compress_ctx(cc); 721 out: 722 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, 723 cc->clen, ret); 724 return ret; 725 } 726 727 static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, 728 bool pre_alloc); 729 static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic, 730 bool bypass_destroy_callback, bool pre_alloc); 731 732 void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task) 733 { 734 struct f2fs_sb_info *sbi = dic->sbi; 735 struct f2fs_inode_info *fi = F2FS_I(dic->inode); 736 const struct f2fs_compress_ops *cops = 737 f2fs_cops[fi->i_compress_algorithm]; 738 bool bypass_callback = false; 739 int ret; 740 741 trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx, 742 dic->cluster_size, fi->i_compress_algorithm); 743 744 if (dic->failed) { 745 ret = -EIO; 746 goto out_end_io; 747 } 748 749 ret = f2fs_prepare_decomp_mem(dic, false); 750 if (ret) { 751 bypass_callback = true; 752 goto out_release; 753 } 754 755 dic->clen = le32_to_cpu(dic->cbuf->clen); 756 dic->rlen = PAGE_SIZE << dic->log_cluster_size; 757 758 if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) { 759 ret = -EFSCORRUPTED; 760 761 /* Avoid f2fs_commit_super in irq context */ 762 f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION); 763 goto out_release; 764 } 765 766 ret = cops->decompress_pages(dic); 767 768 if (!ret && (fi->i_compress_flag & BIT(COMPRESS_CHKSUM))) { 769 u32 provided = le32_to_cpu(dic->cbuf->chksum); 770 u32 calculated = f2fs_crc32(dic->cbuf->cdata, dic->clen); 771 772 if (provided != calculated) { 773 if (!is_inode_flag_set(dic->inode, FI_COMPRESS_CORRUPT)) { 774 set_inode_flag(dic->inode, FI_COMPRESS_CORRUPT); 775 f2fs_info_ratelimited(sbi, 776 "checksum invalid, nid = %lu, %x vs %x", 777 dic->inode->i_ino, 778 provided, calculated); 779 } 780 set_sbi_flag(sbi, SBI_NEED_FSCK); 781 } 782 } 783 784 out_release: 785 f2fs_release_decomp_mem(dic, bypass_callback, false); 786 787 out_end_io: 788 trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx, 789 dic->clen, ret); 790 f2fs_decompress_end_io(dic, ret, in_task); 791 } 792 793 static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, 794 struct folio *folio, nid_t ino, block_t blkaddr); 795 796 /* 797 * This is called when a page of a compressed cluster has been read from disk 798 * (or failed to be read from disk). It checks whether this page was the last 799 * page being waited on in the cluster, and if so, it decompresses the cluster 800 * (or in the case of a failure, cleans up without actually decompressing). 801 */ 802 void f2fs_end_read_compressed_page(struct folio *folio, bool failed, 803 block_t blkaddr, bool in_task) 804 { 805 struct decompress_io_ctx *dic = folio->private; 806 struct f2fs_sb_info *sbi = dic->sbi; 807 808 dec_page_count(sbi, F2FS_RD_DATA); 809 810 if (failed) 811 WRITE_ONCE(dic->failed, true); 812 else if (blkaddr && in_task) 813 f2fs_cache_compressed_page(sbi, folio, 814 dic->inode->i_ino, blkaddr); 815 816 if (atomic_dec_and_test(&dic->remaining_pages)) 817 f2fs_decompress_cluster(dic, in_task); 818 } 819 820 static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index) 821 { 822 if (cc->cluster_idx == NULL_CLUSTER) 823 return true; 824 return cc->cluster_idx == cluster_idx(cc, index); 825 } 826 827 bool f2fs_cluster_is_empty(struct compress_ctx *cc) 828 { 829 return cc->nr_rpages == 0; 830 } 831 832 static bool f2fs_cluster_is_full(struct compress_ctx *cc) 833 { 834 return cc->cluster_size == cc->nr_rpages; 835 } 836 837 bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index) 838 { 839 if (f2fs_cluster_is_empty(cc)) 840 return true; 841 return is_page_in_cluster(cc, index); 842 } 843 844 bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, 845 int index, int nr_pages, bool uptodate) 846 { 847 unsigned long pgidx = page_folio(pages[index])->index; 848 int i = uptodate ? 0 : 1; 849 850 /* 851 * when uptodate set to true, try to check all pages in cluster is 852 * uptodate or not. 853 */ 854 if (uptodate && (pgidx % cc->cluster_size)) 855 return false; 856 857 if (nr_pages - index < cc->cluster_size) 858 return false; 859 860 for (; i < cc->cluster_size; i++) { 861 struct folio *folio = page_folio(pages[index + i]); 862 863 if (folio->index != pgidx + i) 864 return false; 865 if (uptodate && !folio_test_uptodate(folio)) 866 return false; 867 } 868 869 return true; 870 } 871 872 static bool cluster_has_invalid_data(struct compress_ctx *cc) 873 { 874 loff_t i_size = i_size_read(cc->inode); 875 unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE); 876 int i; 877 878 for (i = 0; i < cc->cluster_size; i++) { 879 struct page *page = cc->rpages[i]; 880 881 f2fs_bug_on(F2FS_I_SB(cc->inode), !page); 882 883 /* beyond EOF */ 884 if (page_folio(page)->index >= nr_pages) 885 return true; 886 } 887 return false; 888 } 889 890 bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) 891 { 892 #ifdef CONFIG_F2FS_CHECK_FS 893 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 894 unsigned int cluster_size = F2FS_I(dn->inode)->i_cluster_size; 895 int cluster_end = 0; 896 unsigned int count; 897 int i; 898 char *reason = ""; 899 900 if (dn->data_blkaddr != COMPRESS_ADDR) 901 return false; 902 903 /* [..., COMPR_ADDR, ...] */ 904 if (dn->ofs_in_node % cluster_size) { 905 reason = "[*|C|*|*]"; 906 goto out; 907 } 908 909 for (i = 1, count = 1; i < cluster_size; i++, count++) { 910 block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio, 911 dn->ofs_in_node + i); 912 913 /* [COMPR_ADDR, ..., COMPR_ADDR] */ 914 if (blkaddr == COMPRESS_ADDR) { 915 reason = "[C|*|C|*]"; 916 goto out; 917 } 918 if (!__is_valid_data_blkaddr(blkaddr)) { 919 if (!cluster_end) 920 cluster_end = i; 921 continue; 922 } 923 /* [COMPR_ADDR, NULL_ADDR or NEW_ADDR, valid_blkaddr] */ 924 if (cluster_end) { 925 reason = "[C|N|N|V]"; 926 goto out; 927 } 928 } 929 930 f2fs_bug_on(F2FS_I_SB(dn->inode), count != cluster_size && 931 !is_inode_flag_set(dn->inode, FI_COMPRESS_RELEASED)); 932 933 return false; 934 out: 935 f2fs_warn(sbi, "access invalid cluster, ino:%lu, nid:%u, ofs_in_node:%u, reason:%s", 936 dn->inode->i_ino, dn->nid, dn->ofs_in_node, reason); 937 set_sbi_flag(sbi, SBI_NEED_FSCK); 938 return true; 939 #else 940 return false; 941 #endif 942 } 943 944 static int __f2fs_get_cluster_blocks(struct inode *inode, 945 struct dnode_of_data *dn) 946 { 947 unsigned int cluster_size = F2FS_I(inode)->i_cluster_size; 948 int count, i; 949 950 for (i = 0, count = 0; i < cluster_size; i++) { 951 block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio, 952 dn->ofs_in_node + i); 953 954 if (__is_valid_data_blkaddr(blkaddr)) 955 count++; 956 } 957 958 return count; 959 } 960 961 static int __f2fs_cluster_blocks(struct inode *inode, unsigned int cluster_idx, 962 enum cluster_check_type type) 963 { 964 struct dnode_of_data dn; 965 unsigned int start_idx = cluster_idx << 966 F2FS_I(inode)->i_log_cluster_size; 967 int ret; 968 969 set_new_dnode(&dn, inode, NULL, NULL, 0); 970 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 971 if (ret) { 972 if (ret == -ENOENT) 973 ret = 0; 974 goto fail; 975 } 976 977 if (f2fs_sanity_check_cluster(&dn)) { 978 ret = -EFSCORRUPTED; 979 goto fail; 980 } 981 982 if (dn.data_blkaddr == COMPRESS_ADDR) { 983 if (type == CLUSTER_COMPR_BLKS) 984 ret = 1 + __f2fs_get_cluster_blocks(inode, &dn); 985 else if (type == CLUSTER_IS_COMPR) 986 ret = 1; 987 } else if (type == CLUSTER_RAW_BLKS) { 988 ret = __f2fs_get_cluster_blocks(inode, &dn); 989 } 990 fail: 991 f2fs_put_dnode(&dn); 992 return ret; 993 } 994 995 /* return # of compressed blocks in compressed cluster */ 996 static int f2fs_compressed_blocks(struct compress_ctx *cc) 997 { 998 return __f2fs_cluster_blocks(cc->inode, cc->cluster_idx, 999 CLUSTER_COMPR_BLKS); 1000 } 1001 1002 /* return # of raw blocks in non-compressed cluster */ 1003 static int f2fs_decompressed_blocks(struct inode *inode, 1004 unsigned int cluster_idx) 1005 { 1006 return __f2fs_cluster_blocks(inode, cluster_idx, 1007 CLUSTER_RAW_BLKS); 1008 } 1009 1010 /* return whether cluster is compressed one or not */ 1011 int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index) 1012 { 1013 return __f2fs_cluster_blocks(inode, 1014 index >> F2FS_I(inode)->i_log_cluster_size, 1015 CLUSTER_IS_COMPR); 1016 } 1017 1018 /* return whether cluster contains non raw blocks or not */ 1019 bool f2fs_is_sparse_cluster(struct inode *inode, pgoff_t index) 1020 { 1021 unsigned int cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size; 1022 1023 return f2fs_decompressed_blocks(inode, cluster_idx) != 1024 F2FS_I(inode)->i_cluster_size; 1025 } 1026 1027 static bool cluster_may_compress(struct compress_ctx *cc) 1028 { 1029 if (!f2fs_need_compress_data(cc->inode)) 1030 return false; 1031 if (f2fs_is_atomic_file(cc->inode)) 1032 return false; 1033 if (!f2fs_cluster_is_full(cc)) 1034 return false; 1035 if (unlikely(f2fs_cp_error(F2FS_I_SB(cc->inode)))) 1036 return false; 1037 return !cluster_has_invalid_data(cc); 1038 } 1039 1040 static void set_cluster_writeback(struct compress_ctx *cc) 1041 { 1042 int i; 1043 1044 for (i = 0; i < cc->cluster_size; i++) { 1045 if (cc->rpages[i]) 1046 set_page_writeback(cc->rpages[i]); 1047 } 1048 } 1049 1050 static void cancel_cluster_writeback(struct compress_ctx *cc, 1051 struct compress_io_ctx *cic, int submitted) 1052 { 1053 int i; 1054 1055 /* Wait for submitted IOs. */ 1056 if (submitted > 1) { 1057 f2fs_submit_merged_write(F2FS_I_SB(cc->inode), DATA); 1058 while (atomic_read(&cic->pending_pages) != 1059 (cc->valid_nr_cpages - submitted + 1)) 1060 f2fs_io_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); 1061 } 1062 1063 /* Cancel writeback and stay locked. */ 1064 for (i = 0; i < cc->cluster_size; i++) { 1065 if (i < submitted) { 1066 inode_inc_dirty_pages(cc->inode); 1067 lock_page(cc->rpages[i]); 1068 } 1069 clear_page_private_gcing(cc->rpages[i]); 1070 if (folio_test_writeback(page_folio(cc->rpages[i]))) 1071 end_page_writeback(cc->rpages[i]); 1072 } 1073 } 1074 1075 static void set_cluster_dirty(struct compress_ctx *cc) 1076 { 1077 int i; 1078 1079 for (i = 0; i < cc->cluster_size; i++) 1080 if (cc->rpages[i]) { 1081 set_page_dirty(cc->rpages[i]); 1082 set_page_private_gcing(cc->rpages[i]); 1083 } 1084 } 1085 1086 static int prepare_compress_overwrite(struct compress_ctx *cc, 1087 struct page **pagep, pgoff_t index, void **fsdata) 1088 { 1089 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 1090 struct address_space *mapping = cc->inode->i_mapping; 1091 struct folio *folio; 1092 sector_t last_block_in_bio; 1093 fgf_t fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT; 1094 pgoff_t start_idx = start_idx_of_cluster(cc); 1095 int i, ret; 1096 1097 retry: 1098 ret = f2fs_is_compressed_cluster(cc->inode, start_idx); 1099 if (ret <= 0) 1100 return ret; 1101 1102 ret = f2fs_init_compress_ctx(cc); 1103 if (ret) 1104 return ret; 1105 1106 /* keep folio reference to avoid page reclaim */ 1107 for (i = 0; i < cc->cluster_size; i++) { 1108 folio = f2fs_filemap_get_folio(mapping, start_idx + i, 1109 fgp_flag, GFP_NOFS); 1110 if (IS_ERR(folio)) { 1111 ret = PTR_ERR(folio); 1112 goto unlock_pages; 1113 } 1114 1115 if (folio_test_uptodate(folio)) 1116 f2fs_folio_put(folio, true); 1117 else 1118 f2fs_compress_ctx_add_page(cc, folio); 1119 } 1120 1121 if (!f2fs_cluster_is_empty(cc)) { 1122 struct bio *bio = NULL; 1123 1124 ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size, 1125 &last_block_in_bio, NULL, true); 1126 f2fs_put_rpages(cc); 1127 f2fs_destroy_compress_ctx(cc, true); 1128 if (ret) 1129 goto out; 1130 if (bio) 1131 f2fs_submit_read_bio(sbi, bio, DATA); 1132 1133 ret = f2fs_init_compress_ctx(cc); 1134 if (ret) 1135 goto out; 1136 } 1137 1138 for (i = 0; i < cc->cluster_size; i++) { 1139 f2fs_bug_on(sbi, cc->rpages[i]); 1140 1141 folio = filemap_lock_folio(mapping, start_idx + i); 1142 if (IS_ERR(folio)) { 1143 /* folio could be truncated */ 1144 goto release_and_retry; 1145 } 1146 1147 f2fs_folio_wait_writeback(folio, DATA, true, true); 1148 f2fs_compress_ctx_add_page(cc, folio); 1149 1150 if (!folio_test_uptodate(folio)) { 1151 f2fs_handle_page_eio(sbi, folio, DATA); 1152 release_and_retry: 1153 f2fs_put_rpages(cc); 1154 f2fs_unlock_rpages(cc, i + 1); 1155 f2fs_destroy_compress_ctx(cc, true); 1156 goto retry; 1157 } 1158 } 1159 1160 if (likely(!ret)) { 1161 *fsdata = cc->rpages; 1162 *pagep = cc->rpages[offset_in_cluster(cc, index)]; 1163 return cc->cluster_size; 1164 } 1165 1166 unlock_pages: 1167 f2fs_put_rpages(cc); 1168 f2fs_unlock_rpages(cc, i); 1169 f2fs_destroy_compress_ctx(cc, true); 1170 out: 1171 return ret; 1172 } 1173 1174 int f2fs_prepare_compress_overwrite(struct inode *inode, 1175 struct page **pagep, pgoff_t index, void **fsdata) 1176 { 1177 struct compress_ctx cc = { 1178 .inode = inode, 1179 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 1180 .cluster_size = F2FS_I(inode)->i_cluster_size, 1181 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size, 1182 .rpages = NULL, 1183 .nr_rpages = 0, 1184 .vi = NULL, /* can't write to fsverity files */ 1185 }; 1186 1187 return prepare_compress_overwrite(&cc, pagep, index, fsdata); 1188 } 1189 1190 bool f2fs_compress_write_end(struct inode *inode, void *fsdata, 1191 pgoff_t index, unsigned copied) 1192 1193 { 1194 struct compress_ctx cc = { 1195 .inode = inode, 1196 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 1197 .cluster_size = F2FS_I(inode)->i_cluster_size, 1198 .rpages = fsdata, 1199 }; 1200 struct folio *folio = page_folio(cc.rpages[0]); 1201 bool first_index = (index == folio->index); 1202 1203 if (copied) 1204 set_cluster_dirty(&cc); 1205 1206 f2fs_put_rpages_wbc(&cc, NULL, false, true); 1207 f2fs_destroy_compress_ctx(&cc, false); 1208 1209 return first_index; 1210 } 1211 1212 int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock) 1213 { 1214 void *fsdata = NULL; 1215 struct page *pagep; 1216 struct page **rpages; 1217 int log_cluster_size = F2FS_I(inode)->i_log_cluster_size; 1218 pgoff_t start_idx = from >> (PAGE_SHIFT + log_cluster_size) << 1219 log_cluster_size; 1220 int i; 1221 int err; 1222 1223 err = f2fs_is_compressed_cluster(inode, start_idx); 1224 if (err < 0) 1225 return err; 1226 1227 /* truncate normal cluster */ 1228 if (!err) 1229 return f2fs_do_truncate_blocks(inode, from, lock); 1230 1231 /* truncate compressed cluster */ 1232 err = f2fs_prepare_compress_overwrite(inode, &pagep, 1233 start_idx, &fsdata); 1234 1235 /* should not be a normal cluster */ 1236 f2fs_bug_on(F2FS_I_SB(inode), err == 0); 1237 1238 if (err <= 0) 1239 return err; 1240 1241 rpages = fsdata; 1242 1243 for (i = (1 << log_cluster_size) - 1; i >= 0; i--) { 1244 struct folio *folio = page_folio(rpages[i]); 1245 loff_t start = (loff_t)folio->index << PAGE_SHIFT; 1246 loff_t offset = from > start ? from - start : 0; 1247 1248 folio_zero_segment(folio, offset, folio_size(folio)); 1249 1250 if (from >= start) 1251 break; 1252 } 1253 1254 f2fs_compress_write_end(inode, fsdata, start_idx, true); 1255 1256 err = filemap_write_and_wait_range(inode->i_mapping, 1257 round_down(from, 1 << log_cluster_size << PAGE_SHIFT), 1258 LLONG_MAX); 1259 if (err) 1260 return err; 1261 1262 truncate_pagecache(inode, from); 1263 1264 return f2fs_do_truncate_blocks(inode, round_up(from, PAGE_SIZE), lock); 1265 } 1266 1267 static int f2fs_write_compressed_pages(struct compress_ctx *cc, 1268 int *submitted, 1269 struct writeback_control *wbc, 1270 enum iostat_type io_type) 1271 { 1272 struct inode *inode = cc->inode; 1273 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1274 struct f2fs_inode_info *fi = F2FS_I(inode); 1275 struct f2fs_io_info fio = { 1276 .sbi = sbi, 1277 .ino = cc->inode->i_ino, 1278 .type = DATA, 1279 .op = REQ_OP_WRITE, 1280 .op_flags = wbc_to_write_flags(wbc), 1281 .old_blkaddr = NEW_ADDR, 1282 .page = NULL, 1283 .encrypted_page = NULL, 1284 .compressed_page = NULL, 1285 .io_type = io_type, 1286 .io_wbc = wbc, 1287 .encrypted = fscrypt_inode_uses_fs_layer_crypto(cc->inode) ? 1288 1 : 0, 1289 }; 1290 struct folio *folio; 1291 struct dnode_of_data dn; 1292 struct node_info ni; 1293 struct compress_io_ctx *cic; 1294 struct f2fs_lock_context lc; 1295 pgoff_t start_idx = start_idx_of_cluster(cc); 1296 unsigned int last_index = cc->cluster_size - 1; 1297 loff_t psize; 1298 int i, err; 1299 bool quota_inode = IS_NOQUOTA(inode); 1300 1301 /* we should bypass data pages to proceed the kworker jobs */ 1302 if (unlikely(f2fs_cp_error(sbi))) { 1303 mapping_set_error(inode->i_mapping, -EIO); 1304 goto out_free; 1305 } 1306 1307 if (quota_inode) { 1308 /* 1309 * We need to wait for node_write to avoid block allocation during 1310 * checkpoint. This can only happen to quota writes which can cause 1311 * the below discard race condition. 1312 */ 1313 f2fs_down_read_trace(&sbi->node_write, &lc); 1314 } else if (!f2fs_trylock_op(sbi, &lc)) { 1315 goto out_free; 1316 } 1317 1318 set_new_dnode(&dn, cc->inode, NULL, NULL, 0); 1319 1320 err = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 1321 if (err) 1322 goto out_unlock_op; 1323 1324 for (i = 0; i < cc->cluster_size; i++) { 1325 if (data_blkaddr(dn.inode, dn.node_folio, 1326 dn.ofs_in_node + i) == NULL_ADDR) 1327 goto out_put_dnode; 1328 } 1329 1330 folio = page_folio(cc->rpages[last_index]); 1331 psize = folio_next_pos(folio); 1332 1333 err = f2fs_get_node_info(fio.sbi, dn.nid, &ni, false); 1334 if (err) 1335 goto out_put_dnode; 1336 1337 fio.version = ni.version; 1338 1339 cic = f2fs_kmem_cache_alloc(cic_entry_slab, GFP_F2FS_ZERO, false, sbi); 1340 if (!cic) 1341 goto out_put_dnode; 1342 1343 cic->magic = F2FS_COMPRESSED_PAGE_MAGIC; 1344 cic->inode = inode; 1345 atomic_set(&cic->pending_pages, cc->valid_nr_cpages); 1346 cic->rpages = page_array_alloc(sbi, cc->cluster_size); 1347 if (!cic->rpages) 1348 goto out_put_cic; 1349 1350 cic->nr_rpages = cc->cluster_size; 1351 1352 for (i = 0; i < cc->valid_nr_cpages; i++) { 1353 f2fs_set_compressed_page(cc->cpages[i], inode, 1354 page_folio(cc->rpages[i + 1])->index, cic); 1355 fio.compressed_page = cc->cpages[i]; 1356 1357 fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_folio, 1358 dn.ofs_in_node + i + 1); 1359 1360 /* wait for GCed page writeback via META_MAPPING */ 1361 f2fs_wait_on_block_writeback(inode, fio.old_blkaddr); 1362 1363 if (fio.encrypted) { 1364 fio.page = cc->rpages[i + 1]; 1365 err = f2fs_encrypt_one_page(&fio); 1366 if (err) 1367 goto out_destroy_crypt; 1368 cc->cpages[i] = fio.encrypted_page; 1369 } 1370 } 1371 1372 set_cluster_writeback(cc); 1373 1374 for (i = 0; i < cc->cluster_size; i++) 1375 cic->rpages[i] = cc->rpages[i]; 1376 1377 for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) { 1378 block_t blkaddr; 1379 1380 blkaddr = f2fs_data_blkaddr(&dn); 1381 fio.page = cc->rpages[i]; 1382 fio.old_blkaddr = blkaddr; 1383 1384 /* cluster header */ 1385 if (i == 0) { 1386 if (blkaddr == COMPRESS_ADDR) 1387 fio.compr_blocks++; 1388 if (__is_valid_data_blkaddr(blkaddr)) 1389 f2fs_invalidate_blocks(sbi, blkaddr, 1); 1390 f2fs_update_data_blkaddr(&dn, COMPRESS_ADDR); 1391 goto unlock_continue; 1392 } 1393 1394 if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr)) 1395 fio.compr_blocks++; 1396 1397 if (i > cc->valid_nr_cpages) { 1398 if (__is_valid_data_blkaddr(blkaddr)) { 1399 f2fs_invalidate_blocks(sbi, blkaddr, 1); 1400 f2fs_update_data_blkaddr(&dn, NEW_ADDR); 1401 } 1402 goto unlock_continue; 1403 } 1404 1405 f2fs_bug_on(fio.sbi, blkaddr == NULL_ADDR); 1406 1407 if (fio.encrypted) 1408 fio.encrypted_page = cc->cpages[i - 1]; 1409 else 1410 fio.compressed_page = cc->cpages[i - 1]; 1411 1412 cc->cpages[i - 1] = NULL; 1413 fio.submitted = 0; 1414 f2fs_outplace_write_data(&dn, &fio); 1415 if (unlikely(!fio.submitted)) { 1416 cancel_cluster_writeback(cc, cic, i); 1417 1418 /* To call fscrypt_finalize_bounce_page */ 1419 i = cc->valid_nr_cpages; 1420 *submitted = 0; 1421 goto out_destroy_crypt; 1422 } 1423 (*submitted)++; 1424 unlock_continue: 1425 inode_dec_dirty_pages(cc->inode); 1426 folio_unlock(fio.folio); 1427 } 1428 1429 if (fio.compr_blocks) 1430 f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false); 1431 f2fs_i_compr_blocks_update(inode, cc->valid_nr_cpages, true); 1432 add_compr_block_stat(inode, cc->valid_nr_cpages); 1433 1434 set_inode_flag(cc->inode, FI_APPEND_WRITE); 1435 1436 f2fs_put_dnode(&dn); 1437 if (quota_inode) 1438 f2fs_up_read_trace(&sbi->node_write, &lc); 1439 else 1440 f2fs_unlock_op(sbi, &lc); 1441 1442 spin_lock(&fi->i_size_lock); 1443 if (fi->last_disk_size < psize) 1444 fi->last_disk_size = psize; 1445 spin_unlock(&fi->i_size_lock); 1446 1447 f2fs_put_rpages(cc); 1448 page_array_free(sbi, cc->cpages, cc->nr_cpages); 1449 cc->cpages = NULL; 1450 f2fs_destroy_compress_ctx(cc, false); 1451 return 0; 1452 1453 out_destroy_crypt: 1454 page_array_free(sbi, cic->rpages, cc->cluster_size); 1455 1456 for (--i; i >= 0; i--) { 1457 if (!cc->cpages[i]) 1458 continue; 1459 fscrypt_finalize_bounce_page(&cc->cpages[i]); 1460 } 1461 out_put_cic: 1462 kmem_cache_free(cic_entry_slab, cic); 1463 out_put_dnode: 1464 f2fs_put_dnode(&dn); 1465 out_unlock_op: 1466 if (quota_inode) 1467 f2fs_up_read_trace(&sbi->node_write, &lc); 1468 else 1469 f2fs_unlock_op(sbi, &lc); 1470 out_free: 1471 for (i = 0; i < cc->valid_nr_cpages; i++) { 1472 f2fs_compress_free_page(cc->cpages[i]); 1473 cc->cpages[i] = NULL; 1474 } 1475 page_array_free(sbi, cc->cpages, cc->nr_cpages); 1476 cc->cpages = NULL; 1477 return -EAGAIN; 1478 } 1479 1480 void f2fs_compress_write_end_io(struct bio *bio, struct folio *folio) 1481 { 1482 struct page *page = &folio->page; 1483 struct f2fs_sb_info *sbi = bio->bi_private; 1484 struct compress_io_ctx *cic = folio->private; 1485 enum count_type type = WB_DATA_TYPE(folio, 1486 f2fs_is_compressed_page(folio)); 1487 int i; 1488 1489 if (unlikely(bio->bi_status != BLK_STS_OK)) 1490 mapping_set_error(cic->inode->i_mapping, -EIO); 1491 1492 f2fs_compress_free_page(page); 1493 1494 dec_page_count(sbi, type); 1495 1496 if (atomic_dec_return(&cic->pending_pages)) 1497 return; 1498 1499 for (i = 0; i < cic->nr_rpages; i++) { 1500 WARN_ON(!cic->rpages[i]); 1501 clear_page_private_gcing(cic->rpages[i]); 1502 end_page_writeback(cic->rpages[i]); 1503 } 1504 1505 page_array_free(sbi, cic->rpages, cic->nr_rpages); 1506 kmem_cache_free(cic_entry_slab, cic); 1507 } 1508 1509 static int f2fs_write_raw_pages(struct compress_ctx *cc, 1510 int *submitted_p, 1511 struct writeback_control *wbc, 1512 enum iostat_type io_type) 1513 { 1514 struct address_space *mapping = cc->inode->i_mapping; 1515 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); 1516 struct f2fs_lock_context lc; 1517 int submitted, compr_blocks, i; 1518 int ret = 0; 1519 1520 compr_blocks = f2fs_compressed_blocks(cc); 1521 1522 for (i = 0; i < cc->cluster_size; i++) { 1523 if (!cc->rpages[i]) 1524 continue; 1525 1526 redirty_page_for_writepage(wbc, cc->rpages[i]); 1527 unlock_page(cc->rpages[i]); 1528 } 1529 1530 if (compr_blocks < 0) 1531 return compr_blocks; 1532 1533 /* overwrite compressed cluster w/ normal cluster */ 1534 if (compr_blocks > 0) 1535 f2fs_lock_op(sbi, &lc); 1536 1537 for (i = 0; i < cc->cluster_size; i++) { 1538 struct folio *folio; 1539 1540 if (!cc->rpages[i]) 1541 continue; 1542 folio = page_folio(cc->rpages[i]); 1543 retry_write: 1544 folio_lock(folio); 1545 1546 if (folio->mapping != mapping) { 1547 continue_unlock: 1548 folio_unlock(folio); 1549 continue; 1550 } 1551 1552 if (!folio_test_dirty(folio)) 1553 goto continue_unlock; 1554 1555 if (folio_test_writeback(folio)) { 1556 if (wbc->sync_mode == WB_SYNC_NONE) 1557 goto continue_unlock; 1558 f2fs_folio_wait_writeback(folio, DATA, true, true); 1559 } 1560 1561 if (!folio_clear_dirty_for_io(folio)) 1562 goto continue_unlock; 1563 1564 submitted = 0; 1565 ret = f2fs_write_single_data_page(folio, &submitted, 1566 NULL, NULL, wbc, io_type, 1567 compr_blocks, false); 1568 if (ret) { 1569 if (ret == 1) { 1570 ret = 0; 1571 } else if (ret == -EAGAIN) { 1572 ret = 0; 1573 /* 1574 * for quota file, just redirty left pages to 1575 * avoid deadlock caused by cluster update race 1576 * from foreground operation. 1577 */ 1578 if (IS_NOQUOTA(cc->inode)) 1579 goto out; 1580 f2fs_schedule_timeout(DEFAULT_SCHEDULE_TIMEOUT); 1581 goto retry_write; 1582 } 1583 goto out; 1584 } 1585 1586 *submitted_p += submitted; 1587 } 1588 1589 out: 1590 if (compr_blocks > 0) 1591 f2fs_unlock_op(sbi, &lc); 1592 1593 f2fs_balance_fs(sbi, true); 1594 return ret; 1595 } 1596 1597 int f2fs_write_multi_pages(struct compress_ctx *cc, 1598 int *submitted, 1599 struct writeback_control *wbc, 1600 enum iostat_type io_type) 1601 { 1602 int err; 1603 1604 *submitted = 0; 1605 if (cluster_may_compress(cc)) { 1606 err = f2fs_compress_pages(cc); 1607 if (err == -EAGAIN) { 1608 add_compr_block_stat(cc->inode, cc->cluster_size); 1609 goto write; 1610 } else if (err) { 1611 f2fs_put_rpages_wbc(cc, wbc, true, true); 1612 goto destroy_out; 1613 } 1614 1615 err = f2fs_write_compressed_pages(cc, submitted, 1616 wbc, io_type); 1617 if (!err) 1618 return 0; 1619 f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN); 1620 } 1621 write: 1622 f2fs_bug_on(F2FS_I_SB(cc->inode), *submitted); 1623 1624 err = f2fs_write_raw_pages(cc, submitted, wbc, io_type); 1625 f2fs_put_rpages_wbc(cc, wbc, false, false); 1626 destroy_out: 1627 f2fs_destroy_compress_ctx(cc, false); 1628 return err; 1629 } 1630 1631 static inline bool allow_memalloc_for_decomp(struct f2fs_sb_info *sbi, 1632 bool pre_alloc) 1633 { 1634 return pre_alloc ^ f2fs_low_mem_mode(sbi); 1635 } 1636 1637 static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, 1638 bool pre_alloc) 1639 { 1640 const struct f2fs_compress_ops *cops = f2fs_cops[dic->compress_algorithm]; 1641 int i; 1642 1643 if (!allow_memalloc_for_decomp(dic->sbi, pre_alloc)) 1644 return 0; 1645 1646 dic->tpages = page_array_alloc(dic->sbi, dic->cluster_size); 1647 if (!dic->tpages) 1648 return -ENOMEM; 1649 1650 for (i = 0; i < dic->cluster_size; i++) { 1651 if (dic->rpages[i]) { 1652 dic->tpages[i] = dic->rpages[i]; 1653 continue; 1654 } 1655 1656 dic->tpages[i] = f2fs_compress_alloc_page(); 1657 } 1658 1659 dic->rbuf = f2fs_vmap(dic->tpages, dic->cluster_size); 1660 if (!dic->rbuf) 1661 return -ENOMEM; 1662 1663 dic->cbuf = f2fs_vmap(dic->cpages, dic->nr_cpages); 1664 if (!dic->cbuf) 1665 return -ENOMEM; 1666 1667 if (cops->init_decompress_ctx) 1668 return cops->init_decompress_ctx(dic); 1669 1670 return 0; 1671 } 1672 1673 static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic, 1674 bool bypass_destroy_callback, bool pre_alloc) 1675 { 1676 const struct f2fs_compress_ops *cops = f2fs_cops[dic->compress_algorithm]; 1677 1678 if (!allow_memalloc_for_decomp(dic->sbi, pre_alloc)) 1679 return; 1680 1681 if (!bypass_destroy_callback && cops->destroy_decompress_ctx) 1682 cops->destroy_decompress_ctx(dic); 1683 1684 if (dic->cbuf) 1685 vm_unmap_ram(dic->cbuf, dic->nr_cpages); 1686 1687 if (dic->rbuf) 1688 vm_unmap_ram(dic->rbuf, dic->cluster_size); 1689 } 1690 1691 static void f2fs_free_dic(struct decompress_io_ctx *dic, 1692 bool bypass_destroy_callback); 1693 1694 struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) 1695 { 1696 struct decompress_io_ctx *dic; 1697 pgoff_t start_idx = start_idx_of_cluster(cc); 1698 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 1699 int i, ret; 1700 1701 dic = f2fs_kmem_cache_alloc(dic_entry_slab, GFP_F2FS_ZERO, false, sbi); 1702 if (!dic) 1703 return ERR_PTR(-ENOMEM); 1704 1705 dic->rpages = page_array_alloc(sbi, cc->cluster_size); 1706 if (!dic->rpages) { 1707 kmem_cache_free(dic_entry_slab, dic); 1708 return ERR_PTR(-ENOMEM); 1709 } 1710 1711 dic->magic = F2FS_COMPRESSED_PAGE_MAGIC; 1712 dic->inode = cc->inode; 1713 dic->sbi = sbi; 1714 dic->compress_algorithm = F2FS_I(cc->inode)->i_compress_algorithm; 1715 atomic_set(&dic->remaining_pages, cc->nr_cpages); 1716 dic->cluster_idx = cc->cluster_idx; 1717 dic->cluster_size = cc->cluster_size; 1718 dic->log_cluster_size = cc->log_cluster_size; 1719 dic->nr_cpages = cc->nr_cpages; 1720 refcount_set(&dic->refcnt, 1); 1721 dic->failed = false; 1722 dic->vi = cc->vi; 1723 1724 for (i = 0; i < dic->cluster_size; i++) 1725 dic->rpages[i] = cc->rpages[i]; 1726 dic->nr_rpages = cc->cluster_size; 1727 1728 dic->cpages = page_array_alloc(sbi, dic->nr_cpages); 1729 if (!dic->cpages) { 1730 ret = -ENOMEM; 1731 goto out_free; 1732 } 1733 1734 for (i = 0; i < dic->nr_cpages; i++) { 1735 struct page *page; 1736 1737 page = f2fs_compress_alloc_page(); 1738 f2fs_set_compressed_page(page, cc->inode, 1739 start_idx + i + 1, dic); 1740 dic->cpages[i] = page; 1741 } 1742 1743 ret = f2fs_prepare_decomp_mem(dic, true); 1744 if (ret) 1745 goto out_free; 1746 1747 return dic; 1748 1749 out_free: 1750 f2fs_free_dic(dic, true); 1751 return ERR_PTR(ret); 1752 } 1753 1754 static void f2fs_free_dic(struct decompress_io_ctx *dic, 1755 bool bypass_destroy_callback) 1756 { 1757 int i; 1758 /* use sbi in dic to avoid UFA of dic->inode*/ 1759 struct f2fs_sb_info *sbi = dic->sbi; 1760 1761 f2fs_release_decomp_mem(dic, bypass_destroy_callback, true); 1762 1763 if (dic->tpages) { 1764 for (i = 0; i < dic->cluster_size; i++) { 1765 if (dic->rpages[i]) 1766 continue; 1767 if (!dic->tpages[i]) 1768 continue; 1769 f2fs_compress_free_page(dic->tpages[i]); 1770 } 1771 page_array_free(sbi, dic->tpages, dic->cluster_size); 1772 } 1773 1774 if (dic->cpages) { 1775 for (i = 0; i < dic->nr_cpages; i++) { 1776 if (!dic->cpages[i]) 1777 continue; 1778 f2fs_compress_free_page(dic->cpages[i]); 1779 } 1780 page_array_free(sbi, dic->cpages, dic->nr_cpages); 1781 } 1782 1783 page_array_free(sbi, dic->rpages, dic->nr_rpages); 1784 kmem_cache_free(dic_entry_slab, dic); 1785 } 1786 1787 static void f2fs_late_free_dic(struct work_struct *work) 1788 { 1789 struct decompress_io_ctx *dic = 1790 container_of(work, struct decompress_io_ctx, free_work); 1791 1792 f2fs_free_dic(dic, false); 1793 } 1794 1795 static void f2fs_put_dic(struct decompress_io_ctx *dic, bool in_task) 1796 { 1797 if (refcount_dec_and_test(&dic->refcnt)) { 1798 if (in_task) { 1799 f2fs_free_dic(dic, false); 1800 } else { 1801 INIT_WORK(&dic->free_work, f2fs_late_free_dic); 1802 queue_work(dic->sbi->post_read_wq, &dic->free_work); 1803 } 1804 } 1805 } 1806 1807 static void f2fs_verify_cluster(struct work_struct *work) 1808 { 1809 struct decompress_io_ctx *dic = 1810 container_of(work, struct decompress_io_ctx, verity_work); 1811 int i; 1812 1813 /* Verify, update, and unlock the decompressed pages. */ 1814 for (i = 0; i < dic->cluster_size; i++) { 1815 struct page *rpage = dic->rpages[i]; 1816 1817 if (!rpage) 1818 continue; 1819 1820 if (fsverity_verify_page(dic->vi, rpage)) 1821 SetPageUptodate(rpage); 1822 else 1823 ClearPageUptodate(rpage); 1824 unlock_page(rpage); 1825 } 1826 1827 f2fs_put_dic(dic, true); 1828 } 1829 1830 /* 1831 * This is called when a compressed cluster has been decompressed 1832 * (or failed to be read and/or decompressed). 1833 */ 1834 void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed, 1835 bool in_task) 1836 { 1837 int i; 1838 1839 if (IS_ENABLED(CONFIG_FS_VERITY) && !failed && dic->vi) { 1840 /* 1841 * Note that to avoid deadlocks, the verity work can't be done 1842 * on the decompression workqueue. This is because verifying 1843 * the data pages can involve reading metadata pages from the 1844 * file, and these metadata pages may be compressed. 1845 */ 1846 INIT_WORK(&dic->verity_work, f2fs_verify_cluster); 1847 fsverity_enqueue_verify_work(&dic->verity_work); 1848 return; 1849 } 1850 1851 /* Update and unlock the cluster's pagecache pages. */ 1852 for (i = 0; i < dic->cluster_size; i++) { 1853 struct page *rpage = dic->rpages[i]; 1854 1855 if (!rpage) 1856 continue; 1857 1858 if (failed) 1859 ClearPageUptodate(rpage); 1860 else 1861 SetPageUptodate(rpage); 1862 unlock_page(rpage); 1863 } 1864 1865 /* 1866 * Release the reference to the decompress_io_ctx that was being held 1867 * for I/O completion. 1868 */ 1869 f2fs_put_dic(dic, in_task); 1870 } 1871 1872 /* 1873 * Put a reference to a compressed folio's decompress_io_ctx. 1874 * 1875 * This is called when the folio is no longer needed and can be freed. 1876 */ 1877 void f2fs_put_folio_dic(struct folio *folio, bool in_task) 1878 { 1879 struct decompress_io_ctx *dic = folio->private; 1880 1881 f2fs_put_dic(dic, in_task); 1882 } 1883 1884 /* 1885 * check whether cluster blocks are contiguous, and add extent cache entry 1886 * only if cluster blocks are logically and physically contiguous. 1887 */ 1888 unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn, 1889 unsigned int ofs_in_node) 1890 { 1891 bool compressed = data_blkaddr(dn->inode, dn->node_folio, 1892 ofs_in_node) == COMPRESS_ADDR; 1893 int i = compressed ? 1 : 0; 1894 block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_folio, 1895 ofs_in_node + i); 1896 1897 for (i += 1; i < F2FS_I(dn->inode)->i_cluster_size; i++) { 1898 block_t blkaddr = data_blkaddr(dn->inode, dn->node_folio, 1899 ofs_in_node + i); 1900 1901 if (!__is_valid_data_blkaddr(blkaddr)) 1902 break; 1903 if (first_blkaddr + i - (compressed ? 1 : 0) != blkaddr) 1904 return 0; 1905 } 1906 1907 return compressed ? i - 1 : i; 1908 } 1909 1910 const struct address_space_operations f2fs_compress_aops = { 1911 .release_folio = f2fs_release_folio, 1912 .invalidate_folio = f2fs_invalidate_folio, 1913 .migrate_folio = filemap_migrate_folio, 1914 }; 1915 1916 struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi) 1917 { 1918 return sbi->compress_inode->i_mapping; 1919 } 1920 1921 void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi, 1922 block_t blkaddr, unsigned int len) 1923 { 1924 if (!sbi->compress_inode) 1925 return; 1926 invalidate_mapping_pages(COMPRESS_MAPPING(sbi), blkaddr, blkaddr + len - 1); 1927 } 1928 1929 static void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, 1930 struct folio *folio, nid_t ino, block_t blkaddr) 1931 { 1932 struct folio *cfolio; 1933 int ret; 1934 1935 if (!test_opt(sbi, COMPRESS_CACHE)) 1936 return; 1937 1938 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ)) 1939 return; 1940 1941 if (!f2fs_available_free_memory(sbi, COMPRESS_PAGE)) 1942 return; 1943 1944 cfolio = filemap_get_folio(COMPRESS_MAPPING(sbi), blkaddr); 1945 if (!IS_ERR(cfolio)) { 1946 f2fs_folio_put(cfolio, false); 1947 return; 1948 } 1949 1950 cfolio = filemap_alloc_folio(__GFP_NOWARN | __GFP_IO, 0, NULL); 1951 if (!cfolio) 1952 return; 1953 1954 ret = filemap_add_folio(COMPRESS_MAPPING(sbi), cfolio, 1955 blkaddr, GFP_NOFS); 1956 if (ret) { 1957 f2fs_folio_put(cfolio, false); 1958 return; 1959 } 1960 1961 folio_set_f2fs_data(cfolio, ino); 1962 1963 memcpy(folio_address(cfolio), folio_address(folio), PAGE_SIZE); 1964 folio_mark_uptodate(cfolio); 1965 f2fs_folio_put(cfolio, true); 1966 } 1967 1968 bool f2fs_load_compressed_folio(struct f2fs_sb_info *sbi, struct folio *folio, 1969 block_t blkaddr) 1970 { 1971 struct folio *cfolio; 1972 bool hitted = false; 1973 1974 if (!test_opt(sbi, COMPRESS_CACHE)) 1975 return false; 1976 1977 cfolio = f2fs_filemap_get_folio(COMPRESS_MAPPING(sbi), 1978 blkaddr, FGP_LOCK | FGP_NOWAIT, GFP_NOFS); 1979 if (!IS_ERR(cfolio)) { 1980 if (folio_test_uptodate(cfolio)) { 1981 atomic_inc(&sbi->compress_page_hit); 1982 memcpy(folio_address(folio), 1983 folio_address(cfolio), folio_size(folio)); 1984 hitted = true; 1985 } 1986 f2fs_folio_put(cfolio, true); 1987 } 1988 1989 return hitted; 1990 } 1991 1992 void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino) 1993 { 1994 struct address_space *mapping = COMPRESS_MAPPING(sbi); 1995 struct folio_batch fbatch; 1996 pgoff_t index = 0; 1997 pgoff_t end = MAX_BLKADDR(sbi); 1998 1999 if (!mapping->nrpages) 2000 return; 2001 2002 folio_batch_init(&fbatch); 2003 2004 do { 2005 unsigned int nr, i; 2006 2007 nr = filemap_get_folios(mapping, &index, end - 1, &fbatch); 2008 if (!nr) 2009 break; 2010 2011 for (i = 0; i < nr; i++) { 2012 struct folio *folio = fbatch.folios[i]; 2013 2014 folio_lock(folio); 2015 if (folio->mapping != mapping) { 2016 folio_unlock(folio); 2017 continue; 2018 } 2019 2020 if (ino != folio_get_f2fs_data(folio)) { 2021 folio_unlock(folio); 2022 continue; 2023 } 2024 2025 generic_error_remove_folio(mapping, folio); 2026 folio_unlock(folio); 2027 } 2028 folio_batch_release(&fbatch); 2029 cond_resched(); 2030 } while (index < end); 2031 } 2032 2033 int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) 2034 { 2035 struct inode *inode; 2036 2037 if (!test_opt(sbi, COMPRESS_CACHE)) 2038 return 0; 2039 2040 inode = f2fs_iget(sbi->sb, F2FS_COMPRESS_INO(sbi)); 2041 if (IS_ERR(inode)) 2042 return PTR_ERR(inode); 2043 sbi->compress_inode = inode; 2044 2045 sbi->compress_percent = COMPRESS_PERCENT; 2046 sbi->compress_watermark = COMPRESS_WATERMARK; 2047 2048 atomic_set(&sbi->compress_page_hit, 0); 2049 2050 return 0; 2051 } 2052 2053 void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) 2054 { 2055 if (!sbi->compress_inode) 2056 return; 2057 iput(sbi->compress_inode); 2058 sbi->compress_inode = NULL; 2059 } 2060 2061 int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) 2062 { 2063 dev_t dev = sbi->sb->s_bdev->bd_dev; 2064 char slab_name[35]; 2065 2066 if (!f2fs_sb_has_compression(sbi)) 2067 return 0; 2068 2069 sprintf(slab_name, "f2fs_page_array_entry-%u:%u", MAJOR(dev), MINOR(dev)); 2070 2071 sbi->page_array_slab_size = sizeof(struct page *) << 2072 F2FS_OPTION(sbi).compress_log_size; 2073 2074 sbi->page_array_slab = f2fs_kmem_cache_create(slab_name, 2075 sbi->page_array_slab_size); 2076 return sbi->page_array_slab ? 0 : -ENOMEM; 2077 } 2078 2079 void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) 2080 { 2081 kmem_cache_destroy(sbi->page_array_slab); 2082 } 2083 2084 int __init f2fs_init_compress_cache(void) 2085 { 2086 cic_entry_slab = f2fs_kmem_cache_create("f2fs_cic_entry", 2087 sizeof(struct compress_io_ctx)); 2088 if (!cic_entry_slab) 2089 return -ENOMEM; 2090 dic_entry_slab = f2fs_kmem_cache_create("f2fs_dic_entry", 2091 sizeof(struct decompress_io_ctx)); 2092 if (!dic_entry_slab) 2093 goto free_cic; 2094 return 0; 2095 free_cic: 2096 kmem_cache_destroy(cic_entry_slab); 2097 return -ENOMEM; 2098 } 2099 2100 void f2fs_destroy_compress_cache(void) 2101 { 2102 kmem_cache_destroy(dic_entry_slab); 2103 kmem_cache_destroy(cic_entry_slab); 2104 } 2105