1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * f2fs compress support 4 * 5 * Copyright (c) 2019 Chao Yu <chao@kernel.org> 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/moduleparam.h> 11 #include <linux/writeback.h> 12 #include <linux/backing-dev.h> 13 #include <linux/lzo.h> 14 #include <linux/lz4.h> 15 #include <linux/zstd.h> 16 #include <linux/pagevec.h> 17 18 #include "f2fs.h" 19 #include "node.h" 20 #include "segment.h" 21 #include <trace/events/f2fs.h> 22 23 static struct kmem_cache *cic_entry_slab; 24 static struct kmem_cache *dic_entry_slab; 25 26 static void *page_array_alloc(struct inode *inode, int nr) 27 { 28 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 29 unsigned int size = sizeof(struct page *) * nr; 30 31 if (likely(size <= sbi->page_array_slab_size)) 32 return f2fs_kmem_cache_alloc(sbi->page_array_slab, 33 GFP_F2FS_ZERO, false, F2FS_I_SB(inode)); 34 return f2fs_kzalloc(sbi, size, GFP_NOFS); 35 } 36 37 static void page_array_free(struct inode *inode, void *pages, int nr) 38 { 39 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 40 unsigned int size = sizeof(struct page *) * nr; 41 42 if (!pages) 43 return; 44 45 if (likely(size <= sbi->page_array_slab_size)) 46 kmem_cache_free(sbi->page_array_slab, pages); 47 else 48 kfree(pages); 49 } 50 51 struct f2fs_compress_ops { 52 int (*init_compress_ctx)(struct compress_ctx *cc); 53 void (*destroy_compress_ctx)(struct compress_ctx *cc); 54 int (*compress_pages)(struct compress_ctx *cc); 55 int (*init_decompress_ctx)(struct decompress_io_ctx *dic); 56 void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic); 57 int (*decompress_pages)(struct decompress_io_ctx *dic); 58 bool (*is_level_valid)(int level); 59 }; 60 61 static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index) 62 { 63 return index & (cc->cluster_size - 1); 64 } 65 66 static pgoff_t cluster_idx(struct compress_ctx *cc, pgoff_t index) 67 { 68 return index >> cc->log_cluster_size; 69 } 70 71 static pgoff_t start_idx_of_cluster(struct compress_ctx *cc) 72 { 73 return cc->cluster_idx << cc->log_cluster_size; 74 } 75 76 bool f2fs_is_compressed_page(struct page *page) 77 { 78 if (!PagePrivate(page)) 79 return false; 80 if (!page_private(page)) 81 return false; 82 if (page_private_nonpointer(page)) 83 return false; 84 85 f2fs_bug_on(F2FS_M_SB(page->mapping), 86 *((u32 *)page_private(page)) != F2FS_COMPRESSED_PAGE_MAGIC); 87 return true; 88 } 89 90 static void f2fs_set_compressed_page(struct page *page, 91 struct inode *inode, pgoff_t index, void *data) 92 { 93 struct folio *folio = page_folio(page); 94 95 folio_attach_private(folio, (void *)data); 96 97 /* i_crypto_info and iv index */ 98 folio->index = index; 99 folio->mapping = inode->i_mapping; 100 } 101 102 static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock) 103 { 104 int i; 105 106 for (i = 0; i < len; i++) { 107 if (!cc->rpages[i]) 108 continue; 109 if (unlock) 110 unlock_page(cc->rpages[i]); 111 else 112 put_page(cc->rpages[i]); 113 } 114 } 115 116 static void f2fs_put_rpages(struct compress_ctx *cc) 117 { 118 f2fs_drop_rpages(cc, cc->cluster_size, false); 119 } 120 121 static void f2fs_unlock_rpages(struct compress_ctx *cc, int len) 122 { 123 f2fs_drop_rpages(cc, len, true); 124 } 125 126 static void f2fs_put_rpages_wbc(struct compress_ctx *cc, 127 struct writeback_control *wbc, bool redirty, int unlock) 128 { 129 unsigned int i; 130 131 for (i = 0; i < cc->cluster_size; i++) { 132 if (!cc->rpages[i]) 133 continue; 134 if (redirty) 135 redirty_page_for_writepage(wbc, cc->rpages[i]); 136 f2fs_put_page(cc->rpages[i], unlock); 137 } 138 } 139 140 struct page *f2fs_compress_control_page(struct page *page) 141 { 142 return ((struct compress_io_ctx *)page_private(page))->rpages[0]; 143 } 144 145 int f2fs_init_compress_ctx(struct compress_ctx *cc) 146 { 147 if (cc->rpages) 148 return 0; 149 150 cc->rpages = page_array_alloc(cc->inode, cc->cluster_size); 151 return cc->rpages ? 0 : -ENOMEM; 152 } 153 154 void f2fs_destroy_compress_ctx(struct compress_ctx *cc, bool reuse) 155 { 156 page_array_free(cc->inode, cc->rpages, cc->cluster_size); 157 cc->rpages = NULL; 158 cc->nr_rpages = 0; 159 cc->nr_cpages = 0; 160 cc->valid_nr_cpages = 0; 161 if (!reuse) 162 cc->cluster_idx = NULL_CLUSTER; 163 } 164 165 void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct folio *folio) 166 { 167 unsigned int cluster_ofs; 168 169 if (!f2fs_cluster_can_merge_page(cc, folio->index)) 170 f2fs_bug_on(F2FS_I_SB(cc->inode), 1); 171 172 cluster_ofs = offset_in_cluster(cc, folio->index); 173 cc->rpages[cluster_ofs] = folio_page(folio, 0); 174 cc->nr_rpages++; 175 cc->cluster_idx = cluster_idx(cc, folio->index); 176 } 177 178 #ifdef CONFIG_F2FS_FS_LZO 179 static int lzo_init_compress_ctx(struct compress_ctx *cc) 180 { 181 cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), 182 LZO1X_MEM_COMPRESS, GFP_NOFS); 183 if (!cc->private) 184 return -ENOMEM; 185 186 cc->clen = lzo1x_worst_compress(PAGE_SIZE << cc->log_cluster_size); 187 return 0; 188 } 189 190 static void lzo_destroy_compress_ctx(struct compress_ctx *cc) 191 { 192 kvfree(cc->private); 193 cc->private = NULL; 194 } 195 196 static int lzo_compress_pages(struct compress_ctx *cc) 197 { 198 int ret; 199 200 ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, 201 &cc->clen, cc->private); 202 if (ret != LZO_E_OK) { 203 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 204 "lzo compress failed, ret:%d", ret); 205 return -EIO; 206 } 207 return 0; 208 } 209 210 static int lzo_decompress_pages(struct decompress_io_ctx *dic) 211 { 212 int ret; 213 214 ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen, 215 dic->rbuf, &dic->rlen); 216 if (ret != LZO_E_OK) { 217 f2fs_err_ratelimited(F2FS_I_SB(dic->inode), 218 "lzo decompress failed, ret:%d", ret); 219 return -EIO; 220 } 221 222 if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) { 223 f2fs_err_ratelimited(F2FS_I_SB(dic->inode), 224 "lzo invalid rlen:%zu, expected:%lu", 225 dic->rlen, PAGE_SIZE << dic->log_cluster_size); 226 return -EIO; 227 } 228 return 0; 229 } 230 231 static const struct f2fs_compress_ops f2fs_lzo_ops = { 232 .init_compress_ctx = lzo_init_compress_ctx, 233 .destroy_compress_ctx = lzo_destroy_compress_ctx, 234 .compress_pages = lzo_compress_pages, 235 .decompress_pages = lzo_decompress_pages, 236 }; 237 #endif 238 239 #ifdef CONFIG_F2FS_FS_LZ4 240 static int lz4_init_compress_ctx(struct compress_ctx *cc) 241 { 242 unsigned int size = LZ4_MEM_COMPRESS; 243 244 #ifdef CONFIG_F2FS_FS_LZ4HC 245 if (F2FS_I(cc->inode)->i_compress_level) 246 size = LZ4HC_MEM_COMPRESS; 247 #endif 248 249 cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), size, GFP_NOFS); 250 if (!cc->private) 251 return -ENOMEM; 252 253 /* 254 * we do not change cc->clen to LZ4_compressBound(inputsize) to 255 * adapt worst compress case, because lz4 compressor can handle 256 * output budget properly. 257 */ 258 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; 259 return 0; 260 } 261 262 static void lz4_destroy_compress_ctx(struct compress_ctx *cc) 263 { 264 kvfree(cc->private); 265 cc->private = NULL; 266 } 267 268 static int lz4_compress_pages(struct compress_ctx *cc) 269 { 270 int len = -EINVAL; 271 unsigned char level = F2FS_I(cc->inode)->i_compress_level; 272 273 if (!level) 274 len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen, 275 cc->clen, cc->private); 276 #ifdef CONFIG_F2FS_FS_LZ4HC 277 else 278 len = LZ4_compress_HC(cc->rbuf, cc->cbuf->cdata, cc->rlen, 279 cc->clen, level, cc->private); 280 #endif 281 if (len < 0) 282 return len; 283 if (!len) 284 return -EAGAIN; 285 286 cc->clen = len; 287 return 0; 288 } 289 290 static int lz4_decompress_pages(struct decompress_io_ctx *dic) 291 { 292 int ret; 293 294 ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf, 295 dic->clen, dic->rlen); 296 if (ret < 0) { 297 f2fs_err_ratelimited(F2FS_I_SB(dic->inode), 298 "lz4 decompress failed, ret:%d", ret); 299 return -EIO; 300 } 301 302 if (ret != PAGE_SIZE << dic->log_cluster_size) { 303 f2fs_err_ratelimited(F2FS_I_SB(dic->inode), 304 "lz4 invalid ret:%d, expected:%lu", 305 ret, PAGE_SIZE << dic->log_cluster_size); 306 return -EIO; 307 } 308 return 0; 309 } 310 311 static bool lz4_is_level_valid(int lvl) 312 { 313 #ifdef CONFIG_F2FS_FS_LZ4HC 314 return !lvl || (lvl >= LZ4HC_MIN_CLEVEL && lvl <= LZ4HC_MAX_CLEVEL); 315 #else 316 return lvl == 0; 317 #endif 318 } 319 320 static const struct f2fs_compress_ops f2fs_lz4_ops = { 321 .init_compress_ctx = lz4_init_compress_ctx, 322 .destroy_compress_ctx = lz4_destroy_compress_ctx, 323 .compress_pages = lz4_compress_pages, 324 .decompress_pages = lz4_decompress_pages, 325 .is_level_valid = lz4_is_level_valid, 326 }; 327 #endif 328 329 #ifdef CONFIG_F2FS_FS_ZSTD 330 static int zstd_init_compress_ctx(struct compress_ctx *cc) 331 { 332 zstd_parameters params; 333 zstd_cstream *stream; 334 void *workspace; 335 unsigned int workspace_size; 336 unsigned char level = F2FS_I(cc->inode)->i_compress_level; 337 338 /* Need to remain this for backward compatibility */ 339 if (!level) 340 level = F2FS_ZSTD_DEFAULT_CLEVEL; 341 342 params = zstd_get_params(level, cc->rlen); 343 workspace_size = zstd_cstream_workspace_bound(¶ms.cParams); 344 345 workspace = f2fs_kvmalloc(F2FS_I_SB(cc->inode), 346 workspace_size, GFP_NOFS); 347 if (!workspace) 348 return -ENOMEM; 349 350 stream = zstd_init_cstream(¶ms, 0, workspace, workspace_size); 351 if (!stream) { 352 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 353 "%s zstd_init_cstream failed", __func__); 354 kvfree(workspace); 355 return -EIO; 356 } 357 358 cc->private = workspace; 359 cc->private2 = stream; 360 361 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; 362 return 0; 363 } 364 365 static void zstd_destroy_compress_ctx(struct compress_ctx *cc) 366 { 367 kvfree(cc->private); 368 cc->private = NULL; 369 cc->private2 = NULL; 370 } 371 372 static int zstd_compress_pages(struct compress_ctx *cc) 373 { 374 zstd_cstream *stream = cc->private2; 375 zstd_in_buffer inbuf; 376 zstd_out_buffer outbuf; 377 int src_size = cc->rlen; 378 int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE; 379 int ret; 380 381 inbuf.pos = 0; 382 inbuf.src = cc->rbuf; 383 inbuf.size = src_size; 384 385 outbuf.pos = 0; 386 outbuf.dst = cc->cbuf->cdata; 387 outbuf.size = dst_size; 388 389 ret = zstd_compress_stream(stream, &outbuf, &inbuf); 390 if (zstd_is_error(ret)) { 391 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 392 "%s zstd_compress_stream failed, ret: %d", 393 __func__, zstd_get_error_code(ret)); 394 return -EIO; 395 } 396 397 ret = zstd_end_stream(stream, &outbuf); 398 if (zstd_is_error(ret)) { 399 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 400 "%s zstd_end_stream returned %d", 401 __func__, zstd_get_error_code(ret)); 402 return -EIO; 403 } 404 405 /* 406 * there is compressed data remained in intermediate buffer due to 407 * no more space in cbuf.cdata 408 */ 409 if (ret) 410 return -EAGAIN; 411 412 cc->clen = outbuf.pos; 413 return 0; 414 } 415 416 static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic) 417 { 418 zstd_dstream *stream; 419 void *workspace; 420 unsigned int workspace_size; 421 unsigned int max_window_size = 422 MAX_COMPRESS_WINDOW_SIZE(dic->log_cluster_size); 423 424 workspace_size = zstd_dstream_workspace_bound(max_window_size); 425 426 workspace = f2fs_kvmalloc(F2FS_I_SB(dic->inode), 427 workspace_size, GFP_NOFS); 428 if (!workspace) 429 return -ENOMEM; 430 431 stream = zstd_init_dstream(max_window_size, workspace, workspace_size); 432 if (!stream) { 433 f2fs_err_ratelimited(F2FS_I_SB(dic->inode), 434 "%s zstd_init_dstream failed", __func__); 435 kvfree(workspace); 436 return -EIO; 437 } 438 439 dic->private = workspace; 440 dic->private2 = stream; 441 442 return 0; 443 } 444 445 static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic) 446 { 447 kvfree(dic->private); 448 dic->private = NULL; 449 dic->private2 = NULL; 450 } 451 452 static int zstd_decompress_pages(struct decompress_io_ctx *dic) 453 { 454 zstd_dstream *stream = dic->private2; 455 zstd_in_buffer inbuf; 456 zstd_out_buffer outbuf; 457 int ret; 458 459 inbuf.pos = 0; 460 inbuf.src = dic->cbuf->cdata; 461 inbuf.size = dic->clen; 462 463 outbuf.pos = 0; 464 outbuf.dst = dic->rbuf; 465 outbuf.size = dic->rlen; 466 467 ret = zstd_decompress_stream(stream, &outbuf, &inbuf); 468 if (zstd_is_error(ret)) { 469 f2fs_err_ratelimited(F2FS_I_SB(dic->inode), 470 "%s zstd_decompress_stream failed, ret: %d", 471 __func__, zstd_get_error_code(ret)); 472 return -EIO; 473 } 474 475 if (dic->rlen != outbuf.pos) { 476 f2fs_err_ratelimited(F2FS_I_SB(dic->inode), 477 "%s ZSTD invalid rlen:%zu, expected:%lu", 478 __func__, dic->rlen, 479 PAGE_SIZE << dic->log_cluster_size); 480 return -EIO; 481 } 482 483 return 0; 484 } 485 486 static bool zstd_is_level_valid(int lvl) 487 { 488 return lvl >= zstd_min_clevel() && lvl <= zstd_max_clevel(); 489 } 490 491 static const struct f2fs_compress_ops f2fs_zstd_ops = { 492 .init_compress_ctx = zstd_init_compress_ctx, 493 .destroy_compress_ctx = zstd_destroy_compress_ctx, 494 .compress_pages = zstd_compress_pages, 495 .init_decompress_ctx = zstd_init_decompress_ctx, 496 .destroy_decompress_ctx = zstd_destroy_decompress_ctx, 497 .decompress_pages = zstd_decompress_pages, 498 .is_level_valid = zstd_is_level_valid, 499 }; 500 #endif 501 502 #ifdef CONFIG_F2FS_FS_LZO 503 #ifdef CONFIG_F2FS_FS_LZORLE 504 static int lzorle_compress_pages(struct compress_ctx *cc) 505 { 506 int ret; 507 508 ret = lzorle1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, 509 &cc->clen, cc->private); 510 if (ret != LZO_E_OK) { 511 f2fs_err_ratelimited(F2FS_I_SB(cc->inode), 512 "lzo-rle compress failed, ret:%d", ret); 513 return -EIO; 514 } 515 return 0; 516 } 517 518 static const struct f2fs_compress_ops f2fs_lzorle_ops = { 519 .init_compress_ctx = lzo_init_compress_ctx, 520 .destroy_compress_ctx = lzo_destroy_compress_ctx, 521 .compress_pages = lzorle_compress_pages, 522 .decompress_pages = lzo_decompress_pages, 523 }; 524 #endif 525 #endif 526 527 static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = { 528 #ifdef CONFIG_F2FS_FS_LZO 529 &f2fs_lzo_ops, 530 #else 531 NULL, 532 #endif 533 #ifdef CONFIG_F2FS_FS_LZ4 534 &f2fs_lz4_ops, 535 #else 536 NULL, 537 #endif 538 #ifdef CONFIG_F2FS_FS_ZSTD 539 &f2fs_zstd_ops, 540 #else 541 NULL, 542 #endif 543 #if defined(CONFIG_F2FS_FS_LZO) && defined(CONFIG_F2FS_FS_LZORLE) 544 &f2fs_lzorle_ops, 545 #else 546 NULL, 547 #endif 548 }; 549 550 bool f2fs_is_compress_backend_ready(struct inode *inode) 551 { 552 if (!f2fs_compressed_file(inode)) 553 return true; 554 return f2fs_cops[F2FS_I(inode)->i_compress_algorithm]; 555 } 556 557 bool f2fs_is_compress_level_valid(int alg, int lvl) 558 { 559 const struct f2fs_compress_ops *cops = f2fs_cops[alg]; 560 561 if (cops->is_level_valid) 562 return cops->is_level_valid(lvl); 563 564 return lvl == 0; 565 } 566 567 static mempool_t *compress_page_pool; 568 static int num_compress_pages = 512; 569 module_param(num_compress_pages, uint, 0444); 570 MODULE_PARM_DESC(num_compress_pages, 571 "Number of intermediate compress pages to preallocate"); 572 573 int __init f2fs_init_compress_mempool(void) 574 { 575 compress_page_pool = mempool_create_page_pool(num_compress_pages, 0); 576 return compress_page_pool ? 0 : -ENOMEM; 577 } 578 579 void f2fs_destroy_compress_mempool(void) 580 { 581 mempool_destroy(compress_page_pool); 582 } 583 584 static struct page *f2fs_compress_alloc_page(void) 585 { 586 struct page *page; 587 588 page = mempool_alloc(compress_page_pool, GFP_NOFS); 589 lock_page(page); 590 591 return page; 592 } 593 594 static void f2fs_compress_free_page(struct page *page) 595 { 596 if (!page) 597 return; 598 detach_page_private(page); 599 page->mapping = NULL; 600 unlock_page(page); 601 mempool_free(page, compress_page_pool); 602 } 603 604 #define MAX_VMAP_RETRIES 3 605 606 static void *f2fs_vmap(struct page **pages, unsigned int count) 607 { 608 int i; 609 void *buf = NULL; 610 611 for (i = 0; i < MAX_VMAP_RETRIES; i++) { 612 buf = vm_map_ram(pages, count, -1); 613 if (buf) 614 break; 615 vm_unmap_aliases(); 616 } 617 return buf; 618 } 619 620 static int f2fs_compress_pages(struct compress_ctx *cc) 621 { 622 struct f2fs_inode_info *fi = F2FS_I(cc->inode); 623 const struct f2fs_compress_ops *cops = 624 f2fs_cops[fi->i_compress_algorithm]; 625 unsigned int max_len, new_nr_cpages; 626 u32 chksum = 0; 627 int i, ret; 628 629 trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx, 630 cc->cluster_size, fi->i_compress_algorithm); 631 632 if (cops->init_compress_ctx) { 633 ret = cops->init_compress_ctx(cc); 634 if (ret) 635 goto out; 636 } 637 638 max_len = COMPRESS_HEADER_SIZE + cc->clen; 639 cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE); 640 cc->valid_nr_cpages = cc->nr_cpages; 641 642 cc->cpages = page_array_alloc(cc->inode, cc->nr_cpages); 643 if (!cc->cpages) { 644 ret = -ENOMEM; 645 goto destroy_compress_ctx; 646 } 647 648 for (i = 0; i < cc->nr_cpages; i++) 649 cc->cpages[i] = f2fs_compress_alloc_page(); 650 651 cc->rbuf = f2fs_vmap(cc->rpages, cc->cluster_size); 652 if (!cc->rbuf) { 653 ret = -ENOMEM; 654 goto out_free_cpages; 655 } 656 657 cc->cbuf = f2fs_vmap(cc->cpages, cc->nr_cpages); 658 if (!cc->cbuf) { 659 ret = -ENOMEM; 660 goto out_vunmap_rbuf; 661 } 662 663 ret = cops->compress_pages(cc); 664 if (ret) 665 goto out_vunmap_cbuf; 666 667 max_len = PAGE_SIZE * (cc->cluster_size - 1) - COMPRESS_HEADER_SIZE; 668 669 if (cc->clen > max_len) { 670 ret = -EAGAIN; 671 goto out_vunmap_cbuf; 672 } 673 674 cc->cbuf->clen = cpu_to_le32(cc->clen); 675 676 if (fi->i_compress_flag & BIT(COMPRESS_CHKSUM)) 677 chksum = f2fs_crc32(F2FS_I_SB(cc->inode), 678 cc->cbuf->cdata, cc->clen); 679 cc->cbuf->chksum = cpu_to_le32(chksum); 680 681 for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++) 682 cc->cbuf->reserved[i] = cpu_to_le32(0); 683 684 new_nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE); 685 686 /* zero out any unused part of the last page */ 687 memset(&cc->cbuf->cdata[cc->clen], 0, 688 (new_nr_cpages * PAGE_SIZE) - 689 (cc->clen + COMPRESS_HEADER_SIZE)); 690 691 vm_unmap_ram(cc->cbuf, cc->nr_cpages); 692 vm_unmap_ram(cc->rbuf, cc->cluster_size); 693 694 for (i = new_nr_cpages; i < cc->nr_cpages; i++) { 695 f2fs_compress_free_page(cc->cpages[i]); 696 cc->cpages[i] = NULL; 697 } 698 699 if (cops->destroy_compress_ctx) 700 cops->destroy_compress_ctx(cc); 701 702 cc->valid_nr_cpages = new_nr_cpages; 703 704 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, 705 cc->clen, ret); 706 return 0; 707 708 out_vunmap_cbuf: 709 vm_unmap_ram(cc->cbuf, cc->nr_cpages); 710 out_vunmap_rbuf: 711 vm_unmap_ram(cc->rbuf, cc->cluster_size); 712 out_free_cpages: 713 for (i = 0; i < cc->nr_cpages; i++) { 714 if (cc->cpages[i]) 715 f2fs_compress_free_page(cc->cpages[i]); 716 } 717 page_array_free(cc->inode, cc->cpages, cc->nr_cpages); 718 cc->cpages = NULL; 719 destroy_compress_ctx: 720 if (cops->destroy_compress_ctx) 721 cops->destroy_compress_ctx(cc); 722 out: 723 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, 724 cc->clen, ret); 725 return ret; 726 } 727 728 static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, 729 bool pre_alloc); 730 static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic, 731 bool bypass_destroy_callback, bool pre_alloc); 732 733 void f2fs_decompress_cluster(struct decompress_io_ctx *dic, bool in_task) 734 { 735 struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode); 736 struct f2fs_inode_info *fi = F2FS_I(dic->inode); 737 const struct f2fs_compress_ops *cops = 738 f2fs_cops[fi->i_compress_algorithm]; 739 bool bypass_callback = false; 740 int ret; 741 742 trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx, 743 dic->cluster_size, fi->i_compress_algorithm); 744 745 if (dic->failed) { 746 ret = -EIO; 747 goto out_end_io; 748 } 749 750 ret = f2fs_prepare_decomp_mem(dic, false); 751 if (ret) { 752 bypass_callback = true; 753 goto out_release; 754 } 755 756 dic->clen = le32_to_cpu(dic->cbuf->clen); 757 dic->rlen = PAGE_SIZE << dic->log_cluster_size; 758 759 if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) { 760 ret = -EFSCORRUPTED; 761 762 /* Avoid f2fs_commit_super in irq context */ 763 if (!in_task) 764 f2fs_handle_error_async(sbi, ERROR_FAIL_DECOMPRESSION); 765 else 766 f2fs_handle_error(sbi, ERROR_FAIL_DECOMPRESSION); 767 goto out_release; 768 } 769 770 ret = cops->decompress_pages(dic); 771 772 if (!ret && (fi->i_compress_flag & BIT(COMPRESS_CHKSUM))) { 773 u32 provided = le32_to_cpu(dic->cbuf->chksum); 774 u32 calculated = f2fs_crc32(sbi, dic->cbuf->cdata, dic->clen); 775 776 if (provided != calculated) { 777 if (!is_inode_flag_set(dic->inode, FI_COMPRESS_CORRUPT)) { 778 set_inode_flag(dic->inode, FI_COMPRESS_CORRUPT); 779 f2fs_info_ratelimited(sbi, 780 "checksum invalid, nid = %lu, %x vs %x", 781 dic->inode->i_ino, 782 provided, calculated); 783 } 784 set_sbi_flag(sbi, SBI_NEED_FSCK); 785 } 786 } 787 788 out_release: 789 f2fs_release_decomp_mem(dic, bypass_callback, false); 790 791 out_end_io: 792 trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx, 793 dic->clen, ret); 794 f2fs_decompress_end_io(dic, ret, in_task); 795 } 796 797 /* 798 * This is called when a page of a compressed cluster has been read from disk 799 * (or failed to be read from disk). It checks whether this page was the last 800 * page being waited on in the cluster, and if so, it decompresses the cluster 801 * (or in the case of a failure, cleans up without actually decompressing). 802 */ 803 void f2fs_end_read_compressed_page(struct page *page, bool failed, 804 block_t blkaddr, bool in_task) 805 { 806 struct decompress_io_ctx *dic = 807 (struct decompress_io_ctx *)page_private(page); 808 struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode); 809 810 dec_page_count(sbi, F2FS_RD_DATA); 811 812 if (failed) 813 WRITE_ONCE(dic->failed, true); 814 else if (blkaddr && in_task) 815 f2fs_cache_compressed_page(sbi, page, 816 dic->inode->i_ino, blkaddr); 817 818 if (atomic_dec_and_test(&dic->remaining_pages)) 819 f2fs_decompress_cluster(dic, in_task); 820 } 821 822 static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index) 823 { 824 if (cc->cluster_idx == NULL_CLUSTER) 825 return true; 826 return cc->cluster_idx == cluster_idx(cc, index); 827 } 828 829 bool f2fs_cluster_is_empty(struct compress_ctx *cc) 830 { 831 return cc->nr_rpages == 0; 832 } 833 834 static bool f2fs_cluster_is_full(struct compress_ctx *cc) 835 { 836 return cc->cluster_size == cc->nr_rpages; 837 } 838 839 bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index) 840 { 841 if (f2fs_cluster_is_empty(cc)) 842 return true; 843 return is_page_in_cluster(cc, index); 844 } 845 846 bool f2fs_all_cluster_page_ready(struct compress_ctx *cc, struct page **pages, 847 int index, int nr_pages, bool uptodate) 848 { 849 unsigned long pgidx = page_folio(pages[index])->index; 850 int i = uptodate ? 0 : 1; 851 852 /* 853 * when uptodate set to true, try to check all pages in cluster is 854 * uptodate or not. 855 */ 856 if (uptodate && (pgidx % cc->cluster_size)) 857 return false; 858 859 if (nr_pages - index < cc->cluster_size) 860 return false; 861 862 for (; i < cc->cluster_size; i++) { 863 struct folio *folio = page_folio(pages[index + i]); 864 865 if (folio->index != pgidx + i) 866 return false; 867 if (uptodate && !folio_test_uptodate(folio)) 868 return false; 869 } 870 871 return true; 872 } 873 874 static bool cluster_has_invalid_data(struct compress_ctx *cc) 875 { 876 loff_t i_size = i_size_read(cc->inode); 877 unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE); 878 int i; 879 880 for (i = 0; i < cc->cluster_size; i++) { 881 struct page *page = cc->rpages[i]; 882 883 f2fs_bug_on(F2FS_I_SB(cc->inode), !page); 884 885 /* beyond EOF */ 886 if (page_folio(page)->index >= nr_pages) 887 return true; 888 } 889 return false; 890 } 891 892 bool f2fs_sanity_check_cluster(struct dnode_of_data *dn) 893 { 894 #ifdef CONFIG_F2FS_CHECK_FS 895 struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode); 896 unsigned int cluster_size = F2FS_I(dn->inode)->i_cluster_size; 897 int cluster_end = 0; 898 unsigned int count; 899 int i; 900 char *reason = ""; 901 902 if (dn->data_blkaddr != COMPRESS_ADDR) 903 return false; 904 905 /* [..., COMPR_ADDR, ...] */ 906 if (dn->ofs_in_node % cluster_size) { 907 reason = "[*|C|*|*]"; 908 goto out; 909 } 910 911 for (i = 1, count = 1; i < cluster_size; i++, count++) { 912 block_t blkaddr = data_blkaddr(dn->inode, dn->node_page, 913 dn->ofs_in_node + i); 914 915 /* [COMPR_ADDR, ..., COMPR_ADDR] */ 916 if (blkaddr == COMPRESS_ADDR) { 917 reason = "[C|*|C|*]"; 918 goto out; 919 } 920 if (!__is_valid_data_blkaddr(blkaddr)) { 921 if (!cluster_end) 922 cluster_end = i; 923 continue; 924 } 925 /* [COMPR_ADDR, NULL_ADDR or NEW_ADDR, valid_blkaddr] */ 926 if (cluster_end) { 927 reason = "[C|N|N|V]"; 928 goto out; 929 } 930 } 931 932 f2fs_bug_on(F2FS_I_SB(dn->inode), count != cluster_size && 933 !is_inode_flag_set(dn->inode, FI_COMPRESS_RELEASED)); 934 935 return false; 936 out: 937 f2fs_warn(sbi, "access invalid cluster, ino:%lu, nid:%u, ofs_in_node:%u, reason:%s", 938 dn->inode->i_ino, dn->nid, dn->ofs_in_node, reason); 939 set_sbi_flag(sbi, SBI_NEED_FSCK); 940 return true; 941 #else 942 return false; 943 #endif 944 } 945 946 static int __f2fs_get_cluster_blocks(struct inode *inode, 947 struct dnode_of_data *dn) 948 { 949 unsigned int cluster_size = F2FS_I(inode)->i_cluster_size; 950 int count, i; 951 952 for (i = 0, count = 0; i < cluster_size; i++) { 953 block_t blkaddr = data_blkaddr(dn->inode, dn->node_page, 954 dn->ofs_in_node + i); 955 956 if (__is_valid_data_blkaddr(blkaddr)) 957 count++; 958 } 959 960 return count; 961 } 962 963 static int __f2fs_cluster_blocks(struct inode *inode, unsigned int cluster_idx, 964 enum cluster_check_type type) 965 { 966 struct dnode_of_data dn; 967 unsigned int start_idx = cluster_idx << 968 F2FS_I(inode)->i_log_cluster_size; 969 int ret; 970 971 set_new_dnode(&dn, inode, NULL, NULL, 0); 972 ret = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 973 if (ret) { 974 if (ret == -ENOENT) 975 ret = 0; 976 goto fail; 977 } 978 979 if (f2fs_sanity_check_cluster(&dn)) { 980 ret = -EFSCORRUPTED; 981 goto fail; 982 } 983 984 if (dn.data_blkaddr == COMPRESS_ADDR) { 985 if (type == CLUSTER_COMPR_BLKS) 986 ret = 1 + __f2fs_get_cluster_blocks(inode, &dn); 987 else if (type == CLUSTER_IS_COMPR) 988 ret = 1; 989 } else if (type == CLUSTER_RAW_BLKS) { 990 ret = __f2fs_get_cluster_blocks(inode, &dn); 991 } 992 fail: 993 f2fs_put_dnode(&dn); 994 return ret; 995 } 996 997 /* return # of compressed blocks in compressed cluster */ 998 static int f2fs_compressed_blocks(struct compress_ctx *cc) 999 { 1000 return __f2fs_cluster_blocks(cc->inode, cc->cluster_idx, 1001 CLUSTER_COMPR_BLKS); 1002 } 1003 1004 /* return # of raw blocks in non-compressed cluster */ 1005 static int f2fs_decompressed_blocks(struct inode *inode, 1006 unsigned int cluster_idx) 1007 { 1008 return __f2fs_cluster_blocks(inode, cluster_idx, 1009 CLUSTER_RAW_BLKS); 1010 } 1011 1012 /* return whether cluster is compressed one or not */ 1013 int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index) 1014 { 1015 return __f2fs_cluster_blocks(inode, 1016 index >> F2FS_I(inode)->i_log_cluster_size, 1017 CLUSTER_IS_COMPR); 1018 } 1019 1020 /* return whether cluster contains non raw blocks or not */ 1021 bool f2fs_is_sparse_cluster(struct inode *inode, pgoff_t index) 1022 { 1023 unsigned int cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size; 1024 1025 return f2fs_decompressed_blocks(inode, cluster_idx) != 1026 F2FS_I(inode)->i_cluster_size; 1027 } 1028 1029 static bool cluster_may_compress(struct compress_ctx *cc) 1030 { 1031 if (!f2fs_need_compress_data(cc->inode)) 1032 return false; 1033 if (f2fs_is_atomic_file(cc->inode)) 1034 return false; 1035 if (!f2fs_cluster_is_full(cc)) 1036 return false; 1037 if (unlikely(f2fs_cp_error(F2FS_I_SB(cc->inode)))) 1038 return false; 1039 return !cluster_has_invalid_data(cc); 1040 } 1041 1042 static void set_cluster_writeback(struct compress_ctx *cc) 1043 { 1044 int i; 1045 1046 for (i = 0; i < cc->cluster_size; i++) { 1047 if (cc->rpages[i]) 1048 set_page_writeback(cc->rpages[i]); 1049 } 1050 } 1051 1052 static void cancel_cluster_writeback(struct compress_ctx *cc, 1053 struct compress_io_ctx *cic, int submitted) 1054 { 1055 int i; 1056 1057 /* Wait for submitted IOs. */ 1058 if (submitted > 1) { 1059 f2fs_submit_merged_write(F2FS_I_SB(cc->inode), DATA); 1060 while (atomic_read(&cic->pending_pages) != 1061 (cc->valid_nr_cpages - submitted + 1)) 1062 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT); 1063 } 1064 1065 /* Cancel writeback and stay locked. */ 1066 for (i = 0; i < cc->cluster_size; i++) { 1067 if (i < submitted) { 1068 inode_inc_dirty_pages(cc->inode); 1069 lock_page(cc->rpages[i]); 1070 } 1071 clear_page_private_gcing(cc->rpages[i]); 1072 if (folio_test_writeback(page_folio(cc->rpages[i]))) 1073 end_page_writeback(cc->rpages[i]); 1074 } 1075 } 1076 1077 static void set_cluster_dirty(struct compress_ctx *cc) 1078 { 1079 int i; 1080 1081 for (i = 0; i < cc->cluster_size; i++) 1082 if (cc->rpages[i]) { 1083 set_page_dirty(cc->rpages[i]); 1084 set_page_private_gcing(cc->rpages[i]); 1085 } 1086 } 1087 1088 static int prepare_compress_overwrite(struct compress_ctx *cc, 1089 struct page **pagep, pgoff_t index, void **fsdata) 1090 { 1091 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 1092 struct address_space *mapping = cc->inode->i_mapping; 1093 struct page *page; 1094 sector_t last_block_in_bio; 1095 fgf_t fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT; 1096 pgoff_t start_idx = start_idx_of_cluster(cc); 1097 int i, ret; 1098 1099 retry: 1100 ret = f2fs_is_compressed_cluster(cc->inode, start_idx); 1101 if (ret <= 0) 1102 return ret; 1103 1104 ret = f2fs_init_compress_ctx(cc); 1105 if (ret) 1106 return ret; 1107 1108 /* keep page reference to avoid page reclaim */ 1109 for (i = 0; i < cc->cluster_size; i++) { 1110 page = f2fs_pagecache_get_page(mapping, start_idx + i, 1111 fgp_flag, GFP_NOFS); 1112 if (!page) { 1113 ret = -ENOMEM; 1114 goto unlock_pages; 1115 } 1116 1117 if (PageUptodate(page)) 1118 f2fs_put_page(page, 1); 1119 else 1120 f2fs_compress_ctx_add_page(cc, page_folio(page)); 1121 } 1122 1123 if (!f2fs_cluster_is_empty(cc)) { 1124 struct bio *bio = NULL; 1125 1126 ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size, 1127 &last_block_in_bio, NULL, true); 1128 f2fs_put_rpages(cc); 1129 f2fs_destroy_compress_ctx(cc, true); 1130 if (ret) 1131 goto out; 1132 if (bio) 1133 f2fs_submit_read_bio(sbi, bio, DATA); 1134 1135 ret = f2fs_init_compress_ctx(cc); 1136 if (ret) 1137 goto out; 1138 } 1139 1140 for (i = 0; i < cc->cluster_size; i++) { 1141 f2fs_bug_on(sbi, cc->rpages[i]); 1142 1143 page = find_lock_page(mapping, start_idx + i); 1144 if (!page) { 1145 /* page can be truncated */ 1146 goto release_and_retry; 1147 } 1148 1149 f2fs_wait_on_page_writeback(page, DATA, true, true); 1150 f2fs_compress_ctx_add_page(cc, page_folio(page)); 1151 1152 if (!PageUptodate(page)) { 1153 f2fs_handle_page_eio(sbi, page_folio(page), DATA); 1154 release_and_retry: 1155 f2fs_put_rpages(cc); 1156 f2fs_unlock_rpages(cc, i + 1); 1157 f2fs_destroy_compress_ctx(cc, true); 1158 goto retry; 1159 } 1160 } 1161 1162 if (likely(!ret)) { 1163 *fsdata = cc->rpages; 1164 *pagep = cc->rpages[offset_in_cluster(cc, index)]; 1165 return cc->cluster_size; 1166 } 1167 1168 unlock_pages: 1169 f2fs_put_rpages(cc); 1170 f2fs_unlock_rpages(cc, i); 1171 f2fs_destroy_compress_ctx(cc, true); 1172 out: 1173 return ret; 1174 } 1175 1176 int f2fs_prepare_compress_overwrite(struct inode *inode, 1177 struct page **pagep, pgoff_t index, void **fsdata) 1178 { 1179 struct compress_ctx cc = { 1180 .inode = inode, 1181 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 1182 .cluster_size = F2FS_I(inode)->i_cluster_size, 1183 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size, 1184 .rpages = NULL, 1185 .nr_rpages = 0, 1186 }; 1187 1188 return prepare_compress_overwrite(&cc, pagep, index, fsdata); 1189 } 1190 1191 bool f2fs_compress_write_end(struct inode *inode, void *fsdata, 1192 pgoff_t index, unsigned copied) 1193 1194 { 1195 struct compress_ctx cc = { 1196 .inode = inode, 1197 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 1198 .cluster_size = F2FS_I(inode)->i_cluster_size, 1199 .rpages = fsdata, 1200 }; 1201 struct folio *folio = page_folio(cc.rpages[0]); 1202 bool first_index = (index == folio->index); 1203 1204 if (copied) 1205 set_cluster_dirty(&cc); 1206 1207 f2fs_put_rpages_wbc(&cc, NULL, false, 1); 1208 f2fs_destroy_compress_ctx(&cc, false); 1209 1210 return first_index; 1211 } 1212 1213 int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock) 1214 { 1215 void *fsdata = NULL; 1216 struct page *pagep; 1217 int log_cluster_size = F2FS_I(inode)->i_log_cluster_size; 1218 pgoff_t start_idx = from >> (PAGE_SHIFT + log_cluster_size) << 1219 log_cluster_size; 1220 int err; 1221 1222 err = f2fs_is_compressed_cluster(inode, start_idx); 1223 if (err < 0) 1224 return err; 1225 1226 /* truncate normal cluster */ 1227 if (!err) 1228 return f2fs_do_truncate_blocks(inode, from, lock); 1229 1230 /* truncate compressed cluster */ 1231 err = f2fs_prepare_compress_overwrite(inode, &pagep, 1232 start_idx, &fsdata); 1233 1234 /* should not be a normal cluster */ 1235 f2fs_bug_on(F2FS_I_SB(inode), err == 0); 1236 1237 if (err <= 0) 1238 return err; 1239 1240 if (err > 0) { 1241 struct page **rpages = fsdata; 1242 int cluster_size = F2FS_I(inode)->i_cluster_size; 1243 int i; 1244 1245 for (i = cluster_size - 1; i >= 0; i--) { 1246 struct folio *folio = page_folio(rpages[i]); 1247 loff_t start = folio->index << PAGE_SHIFT; 1248 1249 if (from <= start) { 1250 folio_zero_segment(folio, 0, folio_size(folio)); 1251 } else { 1252 folio_zero_segment(folio, from - start, 1253 folio_size(folio)); 1254 break; 1255 } 1256 } 1257 1258 f2fs_compress_write_end(inode, fsdata, start_idx, true); 1259 } 1260 return 0; 1261 } 1262 1263 static int f2fs_write_compressed_pages(struct compress_ctx *cc, 1264 int *submitted, 1265 struct writeback_control *wbc, 1266 enum iostat_type io_type) 1267 { 1268 struct inode *inode = cc->inode; 1269 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1270 struct f2fs_inode_info *fi = F2FS_I(inode); 1271 struct f2fs_io_info fio = { 1272 .sbi = sbi, 1273 .ino = cc->inode->i_ino, 1274 .type = DATA, 1275 .op = REQ_OP_WRITE, 1276 .op_flags = wbc_to_write_flags(wbc), 1277 .old_blkaddr = NEW_ADDR, 1278 .page = NULL, 1279 .encrypted_page = NULL, 1280 .compressed_page = NULL, 1281 .io_type = io_type, 1282 .io_wbc = wbc, 1283 .encrypted = fscrypt_inode_uses_fs_layer_crypto(cc->inode) ? 1284 1 : 0, 1285 }; 1286 struct folio *folio; 1287 struct dnode_of_data dn; 1288 struct node_info ni; 1289 struct compress_io_ctx *cic; 1290 pgoff_t start_idx = start_idx_of_cluster(cc); 1291 unsigned int last_index = cc->cluster_size - 1; 1292 loff_t psize; 1293 int i, err; 1294 bool quota_inode = IS_NOQUOTA(inode); 1295 1296 /* we should bypass data pages to proceed the kworker jobs */ 1297 if (unlikely(f2fs_cp_error(sbi))) { 1298 mapping_set_error(inode->i_mapping, -EIO); 1299 goto out_free; 1300 } 1301 1302 if (quota_inode) { 1303 /* 1304 * We need to wait for node_write to avoid block allocation during 1305 * checkpoint. This can only happen to quota writes which can cause 1306 * the below discard race condition. 1307 */ 1308 f2fs_down_read(&sbi->node_write); 1309 } else if (!f2fs_trylock_op(sbi)) { 1310 goto out_free; 1311 } 1312 1313 set_new_dnode(&dn, cc->inode, NULL, NULL, 0); 1314 1315 err = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 1316 if (err) 1317 goto out_unlock_op; 1318 1319 for (i = 0; i < cc->cluster_size; i++) { 1320 if (data_blkaddr(dn.inode, dn.node_page, 1321 dn.ofs_in_node + i) == NULL_ADDR) 1322 goto out_put_dnode; 1323 } 1324 1325 folio = page_folio(cc->rpages[last_index]); 1326 psize = folio_pos(folio) + folio_size(folio); 1327 1328 err = f2fs_get_node_info(fio.sbi, dn.nid, &ni, false); 1329 if (err) 1330 goto out_put_dnode; 1331 1332 fio.version = ni.version; 1333 1334 cic = f2fs_kmem_cache_alloc(cic_entry_slab, GFP_F2FS_ZERO, false, sbi); 1335 if (!cic) 1336 goto out_put_dnode; 1337 1338 cic->magic = F2FS_COMPRESSED_PAGE_MAGIC; 1339 cic->inode = inode; 1340 atomic_set(&cic->pending_pages, cc->valid_nr_cpages); 1341 cic->rpages = page_array_alloc(cc->inode, cc->cluster_size); 1342 if (!cic->rpages) 1343 goto out_put_cic; 1344 1345 cic->nr_rpages = cc->cluster_size; 1346 1347 for (i = 0; i < cc->valid_nr_cpages; i++) { 1348 f2fs_set_compressed_page(cc->cpages[i], inode, 1349 page_folio(cc->rpages[i + 1])->index, cic); 1350 fio.compressed_page = cc->cpages[i]; 1351 1352 fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_page, 1353 dn.ofs_in_node + i + 1); 1354 1355 /* wait for GCed page writeback via META_MAPPING */ 1356 f2fs_wait_on_block_writeback(inode, fio.old_blkaddr); 1357 1358 if (fio.encrypted) { 1359 fio.page = cc->rpages[i + 1]; 1360 err = f2fs_encrypt_one_page(&fio); 1361 if (err) 1362 goto out_destroy_crypt; 1363 cc->cpages[i] = fio.encrypted_page; 1364 } 1365 } 1366 1367 set_cluster_writeback(cc); 1368 1369 for (i = 0; i < cc->cluster_size; i++) 1370 cic->rpages[i] = cc->rpages[i]; 1371 1372 for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) { 1373 block_t blkaddr; 1374 1375 blkaddr = f2fs_data_blkaddr(&dn); 1376 fio.page = cc->rpages[i]; 1377 fio.old_blkaddr = blkaddr; 1378 1379 /* cluster header */ 1380 if (i == 0) { 1381 if (blkaddr == COMPRESS_ADDR) 1382 fio.compr_blocks++; 1383 if (__is_valid_data_blkaddr(blkaddr)) 1384 f2fs_invalidate_blocks(sbi, blkaddr, 1); 1385 f2fs_update_data_blkaddr(&dn, COMPRESS_ADDR); 1386 goto unlock_continue; 1387 } 1388 1389 if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr)) 1390 fio.compr_blocks++; 1391 1392 if (i > cc->valid_nr_cpages) { 1393 if (__is_valid_data_blkaddr(blkaddr)) { 1394 f2fs_invalidate_blocks(sbi, blkaddr, 1); 1395 f2fs_update_data_blkaddr(&dn, NEW_ADDR); 1396 } 1397 goto unlock_continue; 1398 } 1399 1400 f2fs_bug_on(fio.sbi, blkaddr == NULL_ADDR); 1401 1402 if (fio.encrypted) 1403 fio.encrypted_page = cc->cpages[i - 1]; 1404 else 1405 fio.compressed_page = cc->cpages[i - 1]; 1406 1407 cc->cpages[i - 1] = NULL; 1408 fio.submitted = 0; 1409 f2fs_outplace_write_data(&dn, &fio); 1410 if (unlikely(!fio.submitted)) { 1411 cancel_cluster_writeback(cc, cic, i); 1412 1413 /* To call fscrypt_finalize_bounce_page */ 1414 i = cc->valid_nr_cpages; 1415 *submitted = 0; 1416 goto out_destroy_crypt; 1417 } 1418 (*submitted)++; 1419 unlock_continue: 1420 inode_dec_dirty_pages(cc->inode); 1421 unlock_page(fio.page); 1422 } 1423 1424 if (fio.compr_blocks) 1425 f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false); 1426 f2fs_i_compr_blocks_update(inode, cc->valid_nr_cpages, true); 1427 add_compr_block_stat(inode, cc->valid_nr_cpages); 1428 1429 set_inode_flag(cc->inode, FI_APPEND_WRITE); 1430 1431 f2fs_put_dnode(&dn); 1432 if (quota_inode) 1433 f2fs_up_read(&sbi->node_write); 1434 else 1435 f2fs_unlock_op(sbi); 1436 1437 spin_lock(&fi->i_size_lock); 1438 if (fi->last_disk_size < psize) 1439 fi->last_disk_size = psize; 1440 spin_unlock(&fi->i_size_lock); 1441 1442 f2fs_put_rpages(cc); 1443 page_array_free(cc->inode, cc->cpages, cc->nr_cpages); 1444 cc->cpages = NULL; 1445 f2fs_destroy_compress_ctx(cc, false); 1446 return 0; 1447 1448 out_destroy_crypt: 1449 page_array_free(cc->inode, cic->rpages, cc->cluster_size); 1450 1451 for (--i; i >= 0; i--) { 1452 if (!cc->cpages[i]) 1453 continue; 1454 fscrypt_finalize_bounce_page(&cc->cpages[i]); 1455 } 1456 out_put_cic: 1457 kmem_cache_free(cic_entry_slab, cic); 1458 out_put_dnode: 1459 f2fs_put_dnode(&dn); 1460 out_unlock_op: 1461 if (quota_inode) 1462 f2fs_up_read(&sbi->node_write); 1463 else 1464 f2fs_unlock_op(sbi); 1465 out_free: 1466 for (i = 0; i < cc->valid_nr_cpages; i++) { 1467 f2fs_compress_free_page(cc->cpages[i]); 1468 cc->cpages[i] = NULL; 1469 } 1470 page_array_free(cc->inode, cc->cpages, cc->nr_cpages); 1471 cc->cpages = NULL; 1472 return -EAGAIN; 1473 } 1474 1475 void f2fs_compress_write_end_io(struct bio *bio, struct page *page) 1476 { 1477 struct f2fs_sb_info *sbi = bio->bi_private; 1478 struct compress_io_ctx *cic = 1479 (struct compress_io_ctx *)page_private(page); 1480 enum count_type type = WB_DATA_TYPE(page, 1481 f2fs_is_compressed_page(page)); 1482 int i; 1483 1484 if (unlikely(bio->bi_status)) 1485 mapping_set_error(cic->inode->i_mapping, -EIO); 1486 1487 f2fs_compress_free_page(page); 1488 1489 dec_page_count(sbi, type); 1490 1491 if (atomic_dec_return(&cic->pending_pages)) 1492 return; 1493 1494 for (i = 0; i < cic->nr_rpages; i++) { 1495 WARN_ON(!cic->rpages[i]); 1496 clear_page_private_gcing(cic->rpages[i]); 1497 end_page_writeback(cic->rpages[i]); 1498 } 1499 1500 page_array_free(cic->inode, cic->rpages, cic->nr_rpages); 1501 kmem_cache_free(cic_entry_slab, cic); 1502 } 1503 1504 static int f2fs_write_raw_pages(struct compress_ctx *cc, 1505 int *submitted_p, 1506 struct writeback_control *wbc, 1507 enum iostat_type io_type) 1508 { 1509 struct address_space *mapping = cc->inode->i_mapping; 1510 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping); 1511 int submitted, compr_blocks, i; 1512 int ret = 0; 1513 1514 compr_blocks = f2fs_compressed_blocks(cc); 1515 1516 for (i = 0; i < cc->cluster_size; i++) { 1517 if (!cc->rpages[i]) 1518 continue; 1519 1520 redirty_page_for_writepage(wbc, cc->rpages[i]); 1521 unlock_page(cc->rpages[i]); 1522 } 1523 1524 if (compr_blocks < 0) 1525 return compr_blocks; 1526 1527 /* overwrite compressed cluster w/ normal cluster */ 1528 if (compr_blocks > 0) 1529 f2fs_lock_op(sbi); 1530 1531 for (i = 0; i < cc->cluster_size; i++) { 1532 if (!cc->rpages[i]) 1533 continue; 1534 retry_write: 1535 lock_page(cc->rpages[i]); 1536 1537 if (cc->rpages[i]->mapping != mapping) { 1538 continue_unlock: 1539 unlock_page(cc->rpages[i]); 1540 continue; 1541 } 1542 1543 if (!PageDirty(cc->rpages[i])) 1544 goto continue_unlock; 1545 1546 if (folio_test_writeback(page_folio(cc->rpages[i]))) { 1547 if (wbc->sync_mode == WB_SYNC_NONE) 1548 goto continue_unlock; 1549 f2fs_wait_on_page_writeback(cc->rpages[i], DATA, true, true); 1550 } 1551 1552 if (!clear_page_dirty_for_io(cc->rpages[i])) 1553 goto continue_unlock; 1554 1555 submitted = 0; 1556 ret = f2fs_write_single_data_page(page_folio(cc->rpages[i]), 1557 &submitted, 1558 NULL, NULL, wbc, io_type, 1559 compr_blocks, false); 1560 if (ret) { 1561 if (ret == AOP_WRITEPAGE_ACTIVATE) { 1562 unlock_page(cc->rpages[i]); 1563 ret = 0; 1564 } else if (ret == -EAGAIN) { 1565 ret = 0; 1566 /* 1567 * for quota file, just redirty left pages to 1568 * avoid deadlock caused by cluster update race 1569 * from foreground operation. 1570 */ 1571 if (IS_NOQUOTA(cc->inode)) 1572 goto out; 1573 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT); 1574 goto retry_write; 1575 } 1576 goto out; 1577 } 1578 1579 *submitted_p += submitted; 1580 } 1581 1582 out: 1583 if (compr_blocks > 0) 1584 f2fs_unlock_op(sbi); 1585 1586 f2fs_balance_fs(sbi, true); 1587 return ret; 1588 } 1589 1590 int f2fs_write_multi_pages(struct compress_ctx *cc, 1591 int *submitted, 1592 struct writeback_control *wbc, 1593 enum iostat_type io_type) 1594 { 1595 int err; 1596 1597 *submitted = 0; 1598 if (cluster_may_compress(cc)) { 1599 err = f2fs_compress_pages(cc); 1600 if (err == -EAGAIN) { 1601 add_compr_block_stat(cc->inode, cc->cluster_size); 1602 goto write; 1603 } else if (err) { 1604 f2fs_put_rpages_wbc(cc, wbc, true, 1); 1605 goto destroy_out; 1606 } 1607 1608 err = f2fs_write_compressed_pages(cc, submitted, 1609 wbc, io_type); 1610 if (!err) 1611 return 0; 1612 f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN); 1613 } 1614 write: 1615 f2fs_bug_on(F2FS_I_SB(cc->inode), *submitted); 1616 1617 err = f2fs_write_raw_pages(cc, submitted, wbc, io_type); 1618 f2fs_put_rpages_wbc(cc, wbc, false, 0); 1619 destroy_out: 1620 f2fs_destroy_compress_ctx(cc, false); 1621 return err; 1622 } 1623 1624 static inline bool allow_memalloc_for_decomp(struct f2fs_sb_info *sbi, 1625 bool pre_alloc) 1626 { 1627 return pre_alloc ^ f2fs_low_mem_mode(sbi); 1628 } 1629 1630 static int f2fs_prepare_decomp_mem(struct decompress_io_ctx *dic, 1631 bool pre_alloc) 1632 { 1633 const struct f2fs_compress_ops *cops = 1634 f2fs_cops[F2FS_I(dic->inode)->i_compress_algorithm]; 1635 int i; 1636 1637 if (!allow_memalloc_for_decomp(F2FS_I_SB(dic->inode), pre_alloc)) 1638 return 0; 1639 1640 dic->tpages = page_array_alloc(dic->inode, dic->cluster_size); 1641 if (!dic->tpages) 1642 return -ENOMEM; 1643 1644 for (i = 0; i < dic->cluster_size; i++) { 1645 if (dic->rpages[i]) { 1646 dic->tpages[i] = dic->rpages[i]; 1647 continue; 1648 } 1649 1650 dic->tpages[i] = f2fs_compress_alloc_page(); 1651 } 1652 1653 dic->rbuf = f2fs_vmap(dic->tpages, dic->cluster_size); 1654 if (!dic->rbuf) 1655 return -ENOMEM; 1656 1657 dic->cbuf = f2fs_vmap(dic->cpages, dic->nr_cpages); 1658 if (!dic->cbuf) 1659 return -ENOMEM; 1660 1661 if (cops->init_decompress_ctx) 1662 return cops->init_decompress_ctx(dic); 1663 1664 return 0; 1665 } 1666 1667 static void f2fs_release_decomp_mem(struct decompress_io_ctx *dic, 1668 bool bypass_destroy_callback, bool pre_alloc) 1669 { 1670 const struct f2fs_compress_ops *cops = 1671 f2fs_cops[F2FS_I(dic->inode)->i_compress_algorithm]; 1672 1673 if (!allow_memalloc_for_decomp(F2FS_I_SB(dic->inode), pre_alloc)) 1674 return; 1675 1676 if (!bypass_destroy_callback && cops->destroy_decompress_ctx) 1677 cops->destroy_decompress_ctx(dic); 1678 1679 if (dic->cbuf) 1680 vm_unmap_ram(dic->cbuf, dic->nr_cpages); 1681 1682 if (dic->rbuf) 1683 vm_unmap_ram(dic->rbuf, dic->cluster_size); 1684 } 1685 1686 static void f2fs_free_dic(struct decompress_io_ctx *dic, 1687 bool bypass_destroy_callback); 1688 1689 struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) 1690 { 1691 struct decompress_io_ctx *dic; 1692 pgoff_t start_idx = start_idx_of_cluster(cc); 1693 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 1694 int i, ret; 1695 1696 dic = f2fs_kmem_cache_alloc(dic_entry_slab, GFP_F2FS_ZERO, false, sbi); 1697 if (!dic) 1698 return ERR_PTR(-ENOMEM); 1699 1700 dic->rpages = page_array_alloc(cc->inode, cc->cluster_size); 1701 if (!dic->rpages) { 1702 kmem_cache_free(dic_entry_slab, dic); 1703 return ERR_PTR(-ENOMEM); 1704 } 1705 1706 dic->magic = F2FS_COMPRESSED_PAGE_MAGIC; 1707 dic->inode = cc->inode; 1708 atomic_set(&dic->remaining_pages, cc->nr_cpages); 1709 dic->cluster_idx = cc->cluster_idx; 1710 dic->cluster_size = cc->cluster_size; 1711 dic->log_cluster_size = cc->log_cluster_size; 1712 dic->nr_cpages = cc->nr_cpages; 1713 refcount_set(&dic->refcnt, 1); 1714 dic->failed = false; 1715 dic->need_verity = f2fs_need_verity(cc->inode, start_idx); 1716 1717 for (i = 0; i < dic->cluster_size; i++) 1718 dic->rpages[i] = cc->rpages[i]; 1719 dic->nr_rpages = cc->cluster_size; 1720 1721 dic->cpages = page_array_alloc(dic->inode, dic->nr_cpages); 1722 if (!dic->cpages) { 1723 ret = -ENOMEM; 1724 goto out_free; 1725 } 1726 1727 for (i = 0; i < dic->nr_cpages; i++) { 1728 struct page *page; 1729 1730 page = f2fs_compress_alloc_page(); 1731 f2fs_set_compressed_page(page, cc->inode, 1732 start_idx + i + 1, dic); 1733 dic->cpages[i] = page; 1734 } 1735 1736 ret = f2fs_prepare_decomp_mem(dic, true); 1737 if (ret) 1738 goto out_free; 1739 1740 return dic; 1741 1742 out_free: 1743 f2fs_free_dic(dic, true); 1744 return ERR_PTR(ret); 1745 } 1746 1747 static void f2fs_free_dic(struct decompress_io_ctx *dic, 1748 bool bypass_destroy_callback) 1749 { 1750 int i; 1751 1752 f2fs_release_decomp_mem(dic, bypass_destroy_callback, true); 1753 1754 if (dic->tpages) { 1755 for (i = 0; i < dic->cluster_size; i++) { 1756 if (dic->rpages[i]) 1757 continue; 1758 if (!dic->tpages[i]) 1759 continue; 1760 f2fs_compress_free_page(dic->tpages[i]); 1761 } 1762 page_array_free(dic->inode, dic->tpages, dic->cluster_size); 1763 } 1764 1765 if (dic->cpages) { 1766 for (i = 0; i < dic->nr_cpages; i++) { 1767 if (!dic->cpages[i]) 1768 continue; 1769 f2fs_compress_free_page(dic->cpages[i]); 1770 } 1771 page_array_free(dic->inode, dic->cpages, dic->nr_cpages); 1772 } 1773 1774 page_array_free(dic->inode, dic->rpages, dic->nr_rpages); 1775 kmem_cache_free(dic_entry_slab, dic); 1776 } 1777 1778 static void f2fs_late_free_dic(struct work_struct *work) 1779 { 1780 struct decompress_io_ctx *dic = 1781 container_of(work, struct decompress_io_ctx, free_work); 1782 1783 f2fs_free_dic(dic, false); 1784 } 1785 1786 static void f2fs_put_dic(struct decompress_io_ctx *dic, bool in_task) 1787 { 1788 if (refcount_dec_and_test(&dic->refcnt)) { 1789 if (in_task) { 1790 f2fs_free_dic(dic, false); 1791 } else { 1792 INIT_WORK(&dic->free_work, f2fs_late_free_dic); 1793 queue_work(F2FS_I_SB(dic->inode)->post_read_wq, 1794 &dic->free_work); 1795 } 1796 } 1797 } 1798 1799 static void f2fs_verify_cluster(struct work_struct *work) 1800 { 1801 struct decompress_io_ctx *dic = 1802 container_of(work, struct decompress_io_ctx, verity_work); 1803 int i; 1804 1805 /* Verify, update, and unlock the decompressed pages. */ 1806 for (i = 0; i < dic->cluster_size; i++) { 1807 struct page *rpage = dic->rpages[i]; 1808 1809 if (!rpage) 1810 continue; 1811 1812 if (fsverity_verify_page(rpage)) 1813 SetPageUptodate(rpage); 1814 else 1815 ClearPageUptodate(rpage); 1816 unlock_page(rpage); 1817 } 1818 1819 f2fs_put_dic(dic, true); 1820 } 1821 1822 /* 1823 * This is called when a compressed cluster has been decompressed 1824 * (or failed to be read and/or decompressed). 1825 */ 1826 void f2fs_decompress_end_io(struct decompress_io_ctx *dic, bool failed, 1827 bool in_task) 1828 { 1829 int i; 1830 1831 if (!failed && dic->need_verity) { 1832 /* 1833 * Note that to avoid deadlocks, the verity work can't be done 1834 * on the decompression workqueue. This is because verifying 1835 * the data pages can involve reading metadata pages from the 1836 * file, and these metadata pages may be compressed. 1837 */ 1838 INIT_WORK(&dic->verity_work, f2fs_verify_cluster); 1839 fsverity_enqueue_verify_work(&dic->verity_work); 1840 return; 1841 } 1842 1843 /* Update and unlock the cluster's pagecache pages. */ 1844 for (i = 0; i < dic->cluster_size; i++) { 1845 struct page *rpage = dic->rpages[i]; 1846 1847 if (!rpage) 1848 continue; 1849 1850 if (failed) 1851 ClearPageUptodate(rpage); 1852 else 1853 SetPageUptodate(rpage); 1854 unlock_page(rpage); 1855 } 1856 1857 /* 1858 * Release the reference to the decompress_io_ctx that was being held 1859 * for I/O completion. 1860 */ 1861 f2fs_put_dic(dic, in_task); 1862 } 1863 1864 /* 1865 * Put a reference to a compressed page's decompress_io_ctx. 1866 * 1867 * This is called when the page is no longer needed and can be freed. 1868 */ 1869 void f2fs_put_page_dic(struct page *page, bool in_task) 1870 { 1871 struct decompress_io_ctx *dic = 1872 (struct decompress_io_ctx *)page_private(page); 1873 1874 f2fs_put_dic(dic, in_task); 1875 } 1876 1877 /* 1878 * check whether cluster blocks are contiguous, and add extent cache entry 1879 * only if cluster blocks are logically and physically contiguous. 1880 */ 1881 unsigned int f2fs_cluster_blocks_are_contiguous(struct dnode_of_data *dn, 1882 unsigned int ofs_in_node) 1883 { 1884 bool compressed = data_blkaddr(dn->inode, dn->node_page, 1885 ofs_in_node) == COMPRESS_ADDR; 1886 int i = compressed ? 1 : 0; 1887 block_t first_blkaddr = data_blkaddr(dn->inode, dn->node_page, 1888 ofs_in_node + i); 1889 1890 for (i += 1; i < F2FS_I(dn->inode)->i_cluster_size; i++) { 1891 block_t blkaddr = data_blkaddr(dn->inode, dn->node_page, 1892 ofs_in_node + i); 1893 1894 if (!__is_valid_data_blkaddr(blkaddr)) 1895 break; 1896 if (first_blkaddr + i - (compressed ? 1 : 0) != blkaddr) 1897 return 0; 1898 } 1899 1900 return compressed ? i - 1 : i; 1901 } 1902 1903 const struct address_space_operations f2fs_compress_aops = { 1904 .release_folio = f2fs_release_folio, 1905 .invalidate_folio = f2fs_invalidate_folio, 1906 .migrate_folio = filemap_migrate_folio, 1907 }; 1908 1909 struct address_space *COMPRESS_MAPPING(struct f2fs_sb_info *sbi) 1910 { 1911 return sbi->compress_inode->i_mapping; 1912 } 1913 1914 void f2fs_invalidate_compress_pages_range(struct f2fs_sb_info *sbi, 1915 block_t blkaddr, unsigned int len) 1916 { 1917 if (!sbi->compress_inode) 1918 return; 1919 invalidate_mapping_pages(COMPRESS_MAPPING(sbi), blkaddr, blkaddr + len - 1); 1920 } 1921 1922 void f2fs_cache_compressed_page(struct f2fs_sb_info *sbi, struct page *page, 1923 nid_t ino, block_t blkaddr) 1924 { 1925 struct page *cpage; 1926 int ret; 1927 1928 if (!test_opt(sbi, COMPRESS_CACHE)) 1929 return; 1930 1931 if (!f2fs_is_valid_blkaddr(sbi, blkaddr, DATA_GENERIC_ENHANCE_READ)) 1932 return; 1933 1934 if (!f2fs_available_free_memory(sbi, COMPRESS_PAGE)) 1935 return; 1936 1937 cpage = find_get_page(COMPRESS_MAPPING(sbi), blkaddr); 1938 if (cpage) { 1939 f2fs_put_page(cpage, 0); 1940 return; 1941 } 1942 1943 cpage = alloc_page(__GFP_NOWARN | __GFP_IO); 1944 if (!cpage) 1945 return; 1946 1947 ret = add_to_page_cache_lru(cpage, COMPRESS_MAPPING(sbi), 1948 blkaddr, GFP_NOFS); 1949 if (ret) { 1950 f2fs_put_page(cpage, 0); 1951 return; 1952 } 1953 1954 set_page_private_data(cpage, ino); 1955 1956 memcpy(page_address(cpage), page_address(page), PAGE_SIZE); 1957 SetPageUptodate(cpage); 1958 f2fs_put_page(cpage, 1); 1959 } 1960 1961 bool f2fs_load_compressed_page(struct f2fs_sb_info *sbi, struct page *page, 1962 block_t blkaddr) 1963 { 1964 struct page *cpage; 1965 bool hitted = false; 1966 1967 if (!test_opt(sbi, COMPRESS_CACHE)) 1968 return false; 1969 1970 cpage = f2fs_pagecache_get_page(COMPRESS_MAPPING(sbi), 1971 blkaddr, FGP_LOCK | FGP_NOWAIT, GFP_NOFS); 1972 if (cpage) { 1973 if (PageUptodate(cpage)) { 1974 atomic_inc(&sbi->compress_page_hit); 1975 memcpy(page_address(page), 1976 page_address(cpage), PAGE_SIZE); 1977 hitted = true; 1978 } 1979 f2fs_put_page(cpage, 1); 1980 } 1981 1982 return hitted; 1983 } 1984 1985 void f2fs_invalidate_compress_pages(struct f2fs_sb_info *sbi, nid_t ino) 1986 { 1987 struct address_space *mapping = COMPRESS_MAPPING(sbi); 1988 struct folio_batch fbatch; 1989 pgoff_t index = 0; 1990 pgoff_t end = MAX_BLKADDR(sbi); 1991 1992 if (!mapping->nrpages) 1993 return; 1994 1995 folio_batch_init(&fbatch); 1996 1997 do { 1998 unsigned int nr, i; 1999 2000 nr = filemap_get_folios(mapping, &index, end - 1, &fbatch); 2001 if (!nr) 2002 break; 2003 2004 for (i = 0; i < nr; i++) { 2005 struct folio *folio = fbatch.folios[i]; 2006 2007 folio_lock(folio); 2008 if (folio->mapping != mapping) { 2009 folio_unlock(folio); 2010 continue; 2011 } 2012 2013 if (ino != get_page_private_data(&folio->page)) { 2014 folio_unlock(folio); 2015 continue; 2016 } 2017 2018 generic_error_remove_folio(mapping, folio); 2019 folio_unlock(folio); 2020 } 2021 folio_batch_release(&fbatch); 2022 cond_resched(); 2023 } while (index < end); 2024 } 2025 2026 int f2fs_init_compress_inode(struct f2fs_sb_info *sbi) 2027 { 2028 struct inode *inode; 2029 2030 if (!test_opt(sbi, COMPRESS_CACHE)) 2031 return 0; 2032 2033 inode = f2fs_iget(sbi->sb, F2FS_COMPRESS_INO(sbi)); 2034 if (IS_ERR(inode)) 2035 return PTR_ERR(inode); 2036 sbi->compress_inode = inode; 2037 2038 sbi->compress_percent = COMPRESS_PERCENT; 2039 sbi->compress_watermark = COMPRESS_WATERMARK; 2040 2041 atomic_set(&sbi->compress_page_hit, 0); 2042 2043 return 0; 2044 } 2045 2046 void f2fs_destroy_compress_inode(struct f2fs_sb_info *sbi) 2047 { 2048 if (!sbi->compress_inode) 2049 return; 2050 iput(sbi->compress_inode); 2051 sbi->compress_inode = NULL; 2052 } 2053 2054 int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) 2055 { 2056 dev_t dev = sbi->sb->s_bdev->bd_dev; 2057 char slab_name[35]; 2058 2059 if (!f2fs_sb_has_compression(sbi)) 2060 return 0; 2061 2062 sprintf(slab_name, "f2fs_page_array_entry-%u:%u", MAJOR(dev), MINOR(dev)); 2063 2064 sbi->page_array_slab_size = sizeof(struct page *) << 2065 F2FS_OPTION(sbi).compress_log_size; 2066 2067 sbi->page_array_slab = f2fs_kmem_cache_create(slab_name, 2068 sbi->page_array_slab_size); 2069 return sbi->page_array_slab ? 0 : -ENOMEM; 2070 } 2071 2072 void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) 2073 { 2074 kmem_cache_destroy(sbi->page_array_slab); 2075 } 2076 2077 int __init f2fs_init_compress_cache(void) 2078 { 2079 cic_entry_slab = f2fs_kmem_cache_create("f2fs_cic_entry", 2080 sizeof(struct compress_io_ctx)); 2081 if (!cic_entry_slab) 2082 return -ENOMEM; 2083 dic_entry_slab = f2fs_kmem_cache_create("f2fs_dic_entry", 2084 sizeof(struct decompress_io_ctx)); 2085 if (!dic_entry_slab) 2086 goto free_cic; 2087 return 0; 2088 free_cic: 2089 kmem_cache_destroy(cic_entry_slab); 2090 return -ENOMEM; 2091 } 2092 2093 void f2fs_destroy_compress_cache(void) 2094 { 2095 kmem_cache_destroy(dic_entry_slab); 2096 kmem_cache_destroy(cic_entry_slab); 2097 } 2098