1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * f2fs compress support 4 * 5 * Copyright (c) 2019 Chao Yu <chao@kernel.org> 6 */ 7 8 #include <linux/fs.h> 9 #include <linux/f2fs_fs.h> 10 #include <linux/writeback.h> 11 #include <linux/backing-dev.h> 12 #include <linux/lzo.h> 13 #include <linux/lz4.h> 14 #include <linux/zstd.h> 15 16 #include "f2fs.h" 17 #include "node.h" 18 #include <trace/events/f2fs.h> 19 20 static struct kmem_cache *cic_entry_slab; 21 static struct kmem_cache *dic_entry_slab; 22 23 static void *page_array_alloc(struct inode *inode, int nr) 24 { 25 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 26 unsigned int size = sizeof(struct page *) * nr; 27 28 if (likely(size <= sbi->page_array_slab_size)) 29 return kmem_cache_zalloc(sbi->page_array_slab, GFP_NOFS); 30 return f2fs_kzalloc(sbi, size, GFP_NOFS); 31 } 32 33 static void page_array_free(struct inode *inode, void *pages, int nr) 34 { 35 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 36 unsigned int size = sizeof(struct page *) * nr; 37 38 if (!pages) 39 return; 40 41 if (likely(size <= sbi->page_array_slab_size)) 42 kmem_cache_free(sbi->page_array_slab, pages); 43 else 44 kfree(pages); 45 } 46 47 struct f2fs_compress_ops { 48 int (*init_compress_ctx)(struct compress_ctx *cc); 49 void (*destroy_compress_ctx)(struct compress_ctx *cc); 50 int (*compress_pages)(struct compress_ctx *cc); 51 int (*init_decompress_ctx)(struct decompress_io_ctx *dic); 52 void (*destroy_decompress_ctx)(struct decompress_io_ctx *dic); 53 int (*decompress_pages)(struct decompress_io_ctx *dic); 54 }; 55 56 static unsigned int offset_in_cluster(struct compress_ctx *cc, pgoff_t index) 57 { 58 return index & (cc->cluster_size - 1); 59 } 60 61 static pgoff_t cluster_idx(struct compress_ctx *cc, pgoff_t index) 62 { 63 return index >> cc->log_cluster_size; 64 } 65 66 static pgoff_t start_idx_of_cluster(struct compress_ctx *cc) 67 { 68 return cc->cluster_idx << cc->log_cluster_size; 69 } 70 71 bool f2fs_is_compressed_page(struct page *page) 72 { 73 if (!PagePrivate(page)) 74 return false; 75 if (!page_private(page)) 76 return false; 77 if (IS_ATOMIC_WRITTEN_PAGE(page) || IS_DUMMY_WRITTEN_PAGE(page)) 78 return false; 79 /* 80 * page->private may be set with pid. 81 * pid_max is enough to check if it is traced. 82 */ 83 if (IS_IO_TRACED_PAGE(page)) 84 return false; 85 86 f2fs_bug_on(F2FS_M_SB(page->mapping), 87 *((u32 *)page_private(page)) != F2FS_COMPRESSED_PAGE_MAGIC); 88 return true; 89 } 90 91 static void f2fs_set_compressed_page(struct page *page, 92 struct inode *inode, pgoff_t index, void *data) 93 { 94 SetPagePrivate(page); 95 set_page_private(page, (unsigned long)data); 96 97 /* i_crypto_info and iv index */ 98 page->index = index; 99 page->mapping = inode->i_mapping; 100 } 101 102 static void f2fs_drop_rpages(struct compress_ctx *cc, int len, bool unlock) 103 { 104 int i; 105 106 for (i = 0; i < len; i++) { 107 if (!cc->rpages[i]) 108 continue; 109 if (unlock) 110 unlock_page(cc->rpages[i]); 111 else 112 put_page(cc->rpages[i]); 113 } 114 } 115 116 static void f2fs_put_rpages(struct compress_ctx *cc) 117 { 118 f2fs_drop_rpages(cc, cc->cluster_size, false); 119 } 120 121 static void f2fs_unlock_rpages(struct compress_ctx *cc, int len) 122 { 123 f2fs_drop_rpages(cc, len, true); 124 } 125 126 static void f2fs_put_rpages_mapping(struct address_space *mapping, 127 pgoff_t start, int len) 128 { 129 int i; 130 131 for (i = 0; i < len; i++) { 132 struct page *page = find_get_page(mapping, start + i); 133 134 put_page(page); 135 put_page(page); 136 } 137 } 138 139 static void f2fs_put_rpages_wbc(struct compress_ctx *cc, 140 struct writeback_control *wbc, bool redirty, int unlock) 141 { 142 unsigned int i; 143 144 for (i = 0; i < cc->cluster_size; i++) { 145 if (!cc->rpages[i]) 146 continue; 147 if (redirty) 148 redirty_page_for_writepage(wbc, cc->rpages[i]); 149 f2fs_put_page(cc->rpages[i], unlock); 150 } 151 } 152 153 struct page *f2fs_compress_control_page(struct page *page) 154 { 155 return ((struct compress_io_ctx *)page_private(page))->rpages[0]; 156 } 157 158 int f2fs_init_compress_ctx(struct compress_ctx *cc) 159 { 160 if (cc->rpages) 161 return 0; 162 163 cc->rpages = page_array_alloc(cc->inode, cc->cluster_size); 164 return cc->rpages ? 0 : -ENOMEM; 165 } 166 167 void f2fs_destroy_compress_ctx(struct compress_ctx *cc) 168 { 169 page_array_free(cc->inode, cc->rpages, cc->cluster_size); 170 cc->rpages = NULL; 171 cc->nr_rpages = 0; 172 cc->nr_cpages = 0; 173 cc->cluster_idx = NULL_CLUSTER; 174 } 175 176 void f2fs_compress_ctx_add_page(struct compress_ctx *cc, struct page *page) 177 { 178 unsigned int cluster_ofs; 179 180 if (!f2fs_cluster_can_merge_page(cc, page->index)) 181 f2fs_bug_on(F2FS_I_SB(cc->inode), 1); 182 183 cluster_ofs = offset_in_cluster(cc, page->index); 184 cc->rpages[cluster_ofs] = page; 185 cc->nr_rpages++; 186 cc->cluster_idx = cluster_idx(cc, page->index); 187 } 188 189 #ifdef CONFIG_F2FS_FS_LZO 190 static int lzo_init_compress_ctx(struct compress_ctx *cc) 191 { 192 cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), 193 LZO1X_MEM_COMPRESS, GFP_NOFS); 194 if (!cc->private) 195 return -ENOMEM; 196 197 cc->clen = lzo1x_worst_compress(PAGE_SIZE << cc->log_cluster_size); 198 return 0; 199 } 200 201 static void lzo_destroy_compress_ctx(struct compress_ctx *cc) 202 { 203 kvfree(cc->private); 204 cc->private = NULL; 205 } 206 207 static int lzo_compress_pages(struct compress_ctx *cc) 208 { 209 int ret; 210 211 ret = lzo1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, 212 &cc->clen, cc->private); 213 if (ret != LZO_E_OK) { 214 printk_ratelimited("%sF2FS-fs (%s): lzo compress failed, ret:%d\n", 215 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret); 216 return -EIO; 217 } 218 return 0; 219 } 220 221 static int lzo_decompress_pages(struct decompress_io_ctx *dic) 222 { 223 int ret; 224 225 ret = lzo1x_decompress_safe(dic->cbuf->cdata, dic->clen, 226 dic->rbuf, &dic->rlen); 227 if (ret != LZO_E_OK) { 228 printk_ratelimited("%sF2FS-fs (%s): lzo decompress failed, ret:%d\n", 229 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret); 230 return -EIO; 231 } 232 233 if (dic->rlen != PAGE_SIZE << dic->log_cluster_size) { 234 printk_ratelimited("%sF2FS-fs (%s): lzo invalid rlen:%zu, " 235 "expected:%lu\n", KERN_ERR, 236 F2FS_I_SB(dic->inode)->sb->s_id, 237 dic->rlen, 238 PAGE_SIZE << dic->log_cluster_size); 239 return -EIO; 240 } 241 return 0; 242 } 243 244 static const struct f2fs_compress_ops f2fs_lzo_ops = { 245 .init_compress_ctx = lzo_init_compress_ctx, 246 .destroy_compress_ctx = lzo_destroy_compress_ctx, 247 .compress_pages = lzo_compress_pages, 248 .decompress_pages = lzo_decompress_pages, 249 }; 250 #endif 251 252 #ifdef CONFIG_F2FS_FS_LZ4 253 static int lz4_init_compress_ctx(struct compress_ctx *cc) 254 { 255 cc->private = f2fs_kvmalloc(F2FS_I_SB(cc->inode), 256 LZ4_MEM_COMPRESS, GFP_NOFS); 257 if (!cc->private) 258 return -ENOMEM; 259 260 /* 261 * we do not change cc->clen to LZ4_compressBound(inputsize) to 262 * adapt worst compress case, because lz4 compressor can handle 263 * output budget properly. 264 */ 265 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; 266 return 0; 267 } 268 269 static void lz4_destroy_compress_ctx(struct compress_ctx *cc) 270 { 271 kvfree(cc->private); 272 cc->private = NULL; 273 } 274 275 static int lz4_compress_pages(struct compress_ctx *cc) 276 { 277 int len; 278 279 len = LZ4_compress_default(cc->rbuf, cc->cbuf->cdata, cc->rlen, 280 cc->clen, cc->private); 281 if (!len) 282 return -EAGAIN; 283 284 cc->clen = len; 285 return 0; 286 } 287 288 static int lz4_decompress_pages(struct decompress_io_ctx *dic) 289 { 290 int ret; 291 292 ret = LZ4_decompress_safe(dic->cbuf->cdata, dic->rbuf, 293 dic->clen, dic->rlen); 294 if (ret < 0) { 295 printk_ratelimited("%sF2FS-fs (%s): lz4 decompress failed, ret:%d\n", 296 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, ret); 297 return -EIO; 298 } 299 300 if (ret != PAGE_SIZE << dic->log_cluster_size) { 301 printk_ratelimited("%sF2FS-fs (%s): lz4 invalid rlen:%zu, " 302 "expected:%lu\n", KERN_ERR, 303 F2FS_I_SB(dic->inode)->sb->s_id, 304 dic->rlen, 305 PAGE_SIZE << dic->log_cluster_size); 306 return -EIO; 307 } 308 return 0; 309 } 310 311 static const struct f2fs_compress_ops f2fs_lz4_ops = { 312 .init_compress_ctx = lz4_init_compress_ctx, 313 .destroy_compress_ctx = lz4_destroy_compress_ctx, 314 .compress_pages = lz4_compress_pages, 315 .decompress_pages = lz4_decompress_pages, 316 }; 317 #endif 318 319 #ifdef CONFIG_F2FS_FS_ZSTD 320 #define F2FS_ZSTD_DEFAULT_CLEVEL 1 321 322 static int zstd_init_compress_ctx(struct compress_ctx *cc) 323 { 324 ZSTD_parameters params; 325 ZSTD_CStream *stream; 326 void *workspace; 327 unsigned int workspace_size; 328 329 params = ZSTD_getParams(F2FS_ZSTD_DEFAULT_CLEVEL, cc->rlen, 0); 330 workspace_size = ZSTD_CStreamWorkspaceBound(params.cParams); 331 332 workspace = f2fs_kvmalloc(F2FS_I_SB(cc->inode), 333 workspace_size, GFP_NOFS); 334 if (!workspace) 335 return -ENOMEM; 336 337 stream = ZSTD_initCStream(params, 0, workspace, workspace_size); 338 if (!stream) { 339 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initCStream failed\n", 340 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, 341 __func__); 342 kvfree(workspace); 343 return -EIO; 344 } 345 346 cc->private = workspace; 347 cc->private2 = stream; 348 349 cc->clen = cc->rlen - PAGE_SIZE - COMPRESS_HEADER_SIZE; 350 return 0; 351 } 352 353 static void zstd_destroy_compress_ctx(struct compress_ctx *cc) 354 { 355 kvfree(cc->private); 356 cc->private = NULL; 357 cc->private2 = NULL; 358 } 359 360 static int zstd_compress_pages(struct compress_ctx *cc) 361 { 362 ZSTD_CStream *stream = cc->private2; 363 ZSTD_inBuffer inbuf; 364 ZSTD_outBuffer outbuf; 365 int src_size = cc->rlen; 366 int dst_size = src_size - PAGE_SIZE - COMPRESS_HEADER_SIZE; 367 int ret; 368 369 inbuf.pos = 0; 370 inbuf.src = cc->rbuf; 371 inbuf.size = src_size; 372 373 outbuf.pos = 0; 374 outbuf.dst = cc->cbuf->cdata; 375 outbuf.size = dst_size; 376 377 ret = ZSTD_compressStream(stream, &outbuf, &inbuf); 378 if (ZSTD_isError(ret)) { 379 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n", 380 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, 381 __func__, ZSTD_getErrorCode(ret)); 382 return -EIO; 383 } 384 385 ret = ZSTD_endStream(stream, &outbuf); 386 if (ZSTD_isError(ret)) { 387 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_endStream returned %d\n", 388 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, 389 __func__, ZSTD_getErrorCode(ret)); 390 return -EIO; 391 } 392 393 /* 394 * there is compressed data remained in intermediate buffer due to 395 * no more space in cbuf.cdata 396 */ 397 if (ret) 398 return -EAGAIN; 399 400 cc->clen = outbuf.pos; 401 return 0; 402 } 403 404 static int zstd_init_decompress_ctx(struct decompress_io_ctx *dic) 405 { 406 ZSTD_DStream *stream; 407 void *workspace; 408 unsigned int workspace_size; 409 unsigned int max_window_size = 410 MAX_COMPRESS_WINDOW_SIZE(dic->log_cluster_size); 411 412 workspace_size = ZSTD_DStreamWorkspaceBound(max_window_size); 413 414 workspace = f2fs_kvmalloc(F2FS_I_SB(dic->inode), 415 workspace_size, GFP_NOFS); 416 if (!workspace) 417 return -ENOMEM; 418 419 stream = ZSTD_initDStream(max_window_size, workspace, workspace_size); 420 if (!stream) { 421 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_initDStream failed\n", 422 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, 423 __func__); 424 kvfree(workspace); 425 return -EIO; 426 } 427 428 dic->private = workspace; 429 dic->private2 = stream; 430 431 return 0; 432 } 433 434 static void zstd_destroy_decompress_ctx(struct decompress_io_ctx *dic) 435 { 436 kvfree(dic->private); 437 dic->private = NULL; 438 dic->private2 = NULL; 439 } 440 441 static int zstd_decompress_pages(struct decompress_io_ctx *dic) 442 { 443 ZSTD_DStream *stream = dic->private2; 444 ZSTD_inBuffer inbuf; 445 ZSTD_outBuffer outbuf; 446 int ret; 447 448 inbuf.pos = 0; 449 inbuf.src = dic->cbuf->cdata; 450 inbuf.size = dic->clen; 451 452 outbuf.pos = 0; 453 outbuf.dst = dic->rbuf; 454 outbuf.size = dic->rlen; 455 456 ret = ZSTD_decompressStream(stream, &outbuf, &inbuf); 457 if (ZSTD_isError(ret)) { 458 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD_compressStream failed, ret: %d\n", 459 KERN_ERR, F2FS_I_SB(dic->inode)->sb->s_id, 460 __func__, ZSTD_getErrorCode(ret)); 461 return -EIO; 462 } 463 464 if (dic->rlen != outbuf.pos) { 465 printk_ratelimited("%sF2FS-fs (%s): %s ZSTD invalid rlen:%zu, " 466 "expected:%lu\n", KERN_ERR, 467 F2FS_I_SB(dic->inode)->sb->s_id, 468 __func__, dic->rlen, 469 PAGE_SIZE << dic->log_cluster_size); 470 return -EIO; 471 } 472 473 return 0; 474 } 475 476 static const struct f2fs_compress_ops f2fs_zstd_ops = { 477 .init_compress_ctx = zstd_init_compress_ctx, 478 .destroy_compress_ctx = zstd_destroy_compress_ctx, 479 .compress_pages = zstd_compress_pages, 480 .init_decompress_ctx = zstd_init_decompress_ctx, 481 .destroy_decompress_ctx = zstd_destroy_decompress_ctx, 482 .decompress_pages = zstd_decompress_pages, 483 }; 484 #endif 485 486 #ifdef CONFIG_F2FS_FS_LZO 487 #ifdef CONFIG_F2FS_FS_LZORLE 488 static int lzorle_compress_pages(struct compress_ctx *cc) 489 { 490 int ret; 491 492 ret = lzorle1x_1_compress(cc->rbuf, cc->rlen, cc->cbuf->cdata, 493 &cc->clen, cc->private); 494 if (ret != LZO_E_OK) { 495 printk_ratelimited("%sF2FS-fs (%s): lzo-rle compress failed, ret:%d\n", 496 KERN_ERR, F2FS_I_SB(cc->inode)->sb->s_id, ret); 497 return -EIO; 498 } 499 return 0; 500 } 501 502 static const struct f2fs_compress_ops f2fs_lzorle_ops = { 503 .init_compress_ctx = lzo_init_compress_ctx, 504 .destroy_compress_ctx = lzo_destroy_compress_ctx, 505 .compress_pages = lzorle_compress_pages, 506 .decompress_pages = lzo_decompress_pages, 507 }; 508 #endif 509 #endif 510 511 static const struct f2fs_compress_ops *f2fs_cops[COMPRESS_MAX] = { 512 #ifdef CONFIG_F2FS_FS_LZO 513 &f2fs_lzo_ops, 514 #else 515 NULL, 516 #endif 517 #ifdef CONFIG_F2FS_FS_LZ4 518 &f2fs_lz4_ops, 519 #else 520 NULL, 521 #endif 522 #ifdef CONFIG_F2FS_FS_ZSTD 523 &f2fs_zstd_ops, 524 #else 525 NULL, 526 #endif 527 #if defined(CONFIG_F2FS_FS_LZO) && defined(CONFIG_F2FS_FS_LZORLE) 528 &f2fs_lzorle_ops, 529 #else 530 NULL, 531 #endif 532 }; 533 534 bool f2fs_is_compress_backend_ready(struct inode *inode) 535 { 536 if (!f2fs_compressed_file(inode)) 537 return true; 538 return f2fs_cops[F2FS_I(inode)->i_compress_algorithm]; 539 } 540 541 static mempool_t *compress_page_pool; 542 static int num_compress_pages = 512; 543 module_param(num_compress_pages, uint, 0444); 544 MODULE_PARM_DESC(num_compress_pages, 545 "Number of intermediate compress pages to preallocate"); 546 547 int f2fs_init_compress_mempool(void) 548 { 549 compress_page_pool = mempool_create_page_pool(num_compress_pages, 0); 550 if (!compress_page_pool) 551 return -ENOMEM; 552 553 return 0; 554 } 555 556 void f2fs_destroy_compress_mempool(void) 557 { 558 mempool_destroy(compress_page_pool); 559 } 560 561 static struct page *f2fs_compress_alloc_page(void) 562 { 563 struct page *page; 564 565 page = mempool_alloc(compress_page_pool, GFP_NOFS); 566 lock_page(page); 567 568 return page; 569 } 570 571 static void f2fs_compress_free_page(struct page *page) 572 { 573 if (!page) 574 return; 575 set_page_private(page, (unsigned long)NULL); 576 ClearPagePrivate(page); 577 page->mapping = NULL; 578 unlock_page(page); 579 mempool_free(page, compress_page_pool); 580 } 581 582 #define MAX_VMAP_RETRIES 3 583 584 static void *f2fs_vmap(struct page **pages, unsigned int count) 585 { 586 int i; 587 void *buf = NULL; 588 589 for (i = 0; i < MAX_VMAP_RETRIES; i++) { 590 buf = vm_map_ram(pages, count, -1); 591 if (buf) 592 break; 593 vm_unmap_aliases(); 594 } 595 return buf; 596 } 597 598 static int f2fs_compress_pages(struct compress_ctx *cc) 599 { 600 struct f2fs_inode_info *fi = F2FS_I(cc->inode); 601 const struct f2fs_compress_ops *cops = 602 f2fs_cops[fi->i_compress_algorithm]; 603 unsigned int max_len, new_nr_cpages; 604 struct page **new_cpages; 605 u32 chksum = 0; 606 int i, ret; 607 608 trace_f2fs_compress_pages_start(cc->inode, cc->cluster_idx, 609 cc->cluster_size, fi->i_compress_algorithm); 610 611 if (cops->init_compress_ctx) { 612 ret = cops->init_compress_ctx(cc); 613 if (ret) 614 goto out; 615 } 616 617 max_len = COMPRESS_HEADER_SIZE + cc->clen; 618 cc->nr_cpages = DIV_ROUND_UP(max_len, PAGE_SIZE); 619 620 cc->cpages = page_array_alloc(cc->inode, cc->nr_cpages); 621 if (!cc->cpages) { 622 ret = -ENOMEM; 623 goto destroy_compress_ctx; 624 } 625 626 for (i = 0; i < cc->nr_cpages; i++) { 627 cc->cpages[i] = f2fs_compress_alloc_page(); 628 if (!cc->cpages[i]) { 629 ret = -ENOMEM; 630 goto out_free_cpages; 631 } 632 } 633 634 cc->rbuf = f2fs_vmap(cc->rpages, cc->cluster_size); 635 if (!cc->rbuf) { 636 ret = -ENOMEM; 637 goto out_free_cpages; 638 } 639 640 cc->cbuf = f2fs_vmap(cc->cpages, cc->nr_cpages); 641 if (!cc->cbuf) { 642 ret = -ENOMEM; 643 goto out_vunmap_rbuf; 644 } 645 646 ret = cops->compress_pages(cc); 647 if (ret) 648 goto out_vunmap_cbuf; 649 650 max_len = PAGE_SIZE * (cc->cluster_size - 1) - COMPRESS_HEADER_SIZE; 651 652 if (cc->clen > max_len) { 653 ret = -EAGAIN; 654 goto out_vunmap_cbuf; 655 } 656 657 cc->cbuf->clen = cpu_to_le32(cc->clen); 658 659 if (fi->i_compress_flag & 1 << COMPRESS_CHKSUM) 660 chksum = f2fs_crc32(F2FS_I_SB(cc->inode), 661 cc->cbuf->cdata, cc->clen); 662 cc->cbuf->chksum = cpu_to_le32(chksum); 663 664 for (i = 0; i < COMPRESS_DATA_RESERVED_SIZE; i++) 665 cc->cbuf->reserved[i] = cpu_to_le32(0); 666 667 new_nr_cpages = DIV_ROUND_UP(cc->clen + COMPRESS_HEADER_SIZE, PAGE_SIZE); 668 669 /* Now we're going to cut unnecessary tail pages */ 670 new_cpages = page_array_alloc(cc->inode, new_nr_cpages); 671 if (!new_cpages) { 672 ret = -ENOMEM; 673 goto out_vunmap_cbuf; 674 } 675 676 /* zero out any unused part of the last page */ 677 memset(&cc->cbuf->cdata[cc->clen], 0, 678 (new_nr_cpages * PAGE_SIZE) - 679 (cc->clen + COMPRESS_HEADER_SIZE)); 680 681 vm_unmap_ram(cc->cbuf, cc->nr_cpages); 682 vm_unmap_ram(cc->rbuf, cc->cluster_size); 683 684 for (i = 0; i < cc->nr_cpages; i++) { 685 if (i < new_nr_cpages) { 686 new_cpages[i] = cc->cpages[i]; 687 continue; 688 } 689 f2fs_compress_free_page(cc->cpages[i]); 690 cc->cpages[i] = NULL; 691 } 692 693 if (cops->destroy_compress_ctx) 694 cops->destroy_compress_ctx(cc); 695 696 page_array_free(cc->inode, cc->cpages, cc->nr_cpages); 697 cc->cpages = new_cpages; 698 cc->nr_cpages = new_nr_cpages; 699 700 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, 701 cc->clen, ret); 702 return 0; 703 704 out_vunmap_cbuf: 705 vm_unmap_ram(cc->cbuf, cc->nr_cpages); 706 out_vunmap_rbuf: 707 vm_unmap_ram(cc->rbuf, cc->cluster_size); 708 out_free_cpages: 709 for (i = 0; i < cc->nr_cpages; i++) { 710 if (cc->cpages[i]) 711 f2fs_compress_free_page(cc->cpages[i]); 712 } 713 page_array_free(cc->inode, cc->cpages, cc->nr_cpages); 714 cc->cpages = NULL; 715 destroy_compress_ctx: 716 if (cops->destroy_compress_ctx) 717 cops->destroy_compress_ctx(cc); 718 out: 719 trace_f2fs_compress_pages_end(cc->inode, cc->cluster_idx, 720 cc->clen, ret); 721 return ret; 722 } 723 724 void f2fs_decompress_pages(struct bio *bio, struct page *page, bool verity) 725 { 726 struct decompress_io_ctx *dic = 727 (struct decompress_io_ctx *)page_private(page); 728 struct f2fs_sb_info *sbi = F2FS_I_SB(dic->inode); 729 struct f2fs_inode_info *fi= F2FS_I(dic->inode); 730 const struct f2fs_compress_ops *cops = 731 f2fs_cops[fi->i_compress_algorithm]; 732 int ret; 733 int i; 734 735 dec_page_count(sbi, F2FS_RD_DATA); 736 737 if (bio->bi_status || PageError(page)) 738 dic->failed = true; 739 740 if (atomic_dec_return(&dic->pending_pages)) 741 return; 742 743 trace_f2fs_decompress_pages_start(dic->inode, dic->cluster_idx, 744 dic->cluster_size, fi->i_compress_algorithm); 745 746 /* submit partial compressed pages */ 747 if (dic->failed) { 748 ret = -EIO; 749 goto out_free_dic; 750 } 751 752 dic->tpages = page_array_alloc(dic->inode, dic->cluster_size); 753 if (!dic->tpages) { 754 ret = -ENOMEM; 755 goto out_free_dic; 756 } 757 758 for (i = 0; i < dic->cluster_size; i++) { 759 if (dic->rpages[i]) { 760 dic->tpages[i] = dic->rpages[i]; 761 continue; 762 } 763 764 dic->tpages[i] = f2fs_compress_alloc_page(); 765 if (!dic->tpages[i]) { 766 ret = -ENOMEM; 767 goto out_free_dic; 768 } 769 } 770 771 if (cops->init_decompress_ctx) { 772 ret = cops->init_decompress_ctx(dic); 773 if (ret) 774 goto out_free_dic; 775 } 776 777 dic->rbuf = f2fs_vmap(dic->tpages, dic->cluster_size); 778 if (!dic->rbuf) { 779 ret = -ENOMEM; 780 goto destroy_decompress_ctx; 781 } 782 783 dic->cbuf = f2fs_vmap(dic->cpages, dic->nr_cpages); 784 if (!dic->cbuf) { 785 ret = -ENOMEM; 786 goto out_vunmap_rbuf; 787 } 788 789 dic->clen = le32_to_cpu(dic->cbuf->clen); 790 dic->rlen = PAGE_SIZE << dic->log_cluster_size; 791 792 if (dic->clen > PAGE_SIZE * dic->nr_cpages - COMPRESS_HEADER_SIZE) { 793 ret = -EFSCORRUPTED; 794 goto out_vunmap_cbuf; 795 } 796 797 ret = cops->decompress_pages(dic); 798 799 if (!ret && fi->i_compress_flag & 1 << COMPRESS_CHKSUM) { 800 u32 provided = le32_to_cpu(dic->cbuf->chksum); 801 u32 calculated = f2fs_crc32(sbi, dic->cbuf->cdata, dic->clen); 802 803 if (provided != calculated) { 804 if (!is_inode_flag_set(dic->inode, FI_COMPRESS_CORRUPT)) { 805 set_inode_flag(dic->inode, FI_COMPRESS_CORRUPT); 806 printk_ratelimited( 807 "%sF2FS-fs (%s): checksum invalid, nid = %lu, %x vs %x", 808 KERN_INFO, sbi->sb->s_id, dic->inode->i_ino, 809 provided, calculated); 810 } 811 set_sbi_flag(sbi, SBI_NEED_FSCK); 812 WARN_ON_ONCE(1); 813 } 814 } 815 816 out_vunmap_cbuf: 817 vm_unmap_ram(dic->cbuf, dic->nr_cpages); 818 out_vunmap_rbuf: 819 vm_unmap_ram(dic->rbuf, dic->cluster_size); 820 destroy_decompress_ctx: 821 if (cops->destroy_decompress_ctx) 822 cops->destroy_decompress_ctx(dic); 823 out_free_dic: 824 if (verity) 825 atomic_set(&dic->pending_pages, dic->nr_cpages); 826 if (!verity) 827 f2fs_decompress_end_io(dic->rpages, dic->cluster_size, 828 ret, false); 829 830 trace_f2fs_decompress_pages_end(dic->inode, dic->cluster_idx, 831 dic->clen, ret); 832 if (!verity) 833 f2fs_free_dic(dic); 834 } 835 836 static bool is_page_in_cluster(struct compress_ctx *cc, pgoff_t index) 837 { 838 if (cc->cluster_idx == NULL_CLUSTER) 839 return true; 840 return cc->cluster_idx == cluster_idx(cc, index); 841 } 842 843 bool f2fs_cluster_is_empty(struct compress_ctx *cc) 844 { 845 return cc->nr_rpages == 0; 846 } 847 848 static bool f2fs_cluster_is_full(struct compress_ctx *cc) 849 { 850 return cc->cluster_size == cc->nr_rpages; 851 } 852 853 bool f2fs_cluster_can_merge_page(struct compress_ctx *cc, pgoff_t index) 854 { 855 if (f2fs_cluster_is_empty(cc)) 856 return true; 857 return is_page_in_cluster(cc, index); 858 } 859 860 static bool __cluster_may_compress(struct compress_ctx *cc) 861 { 862 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 863 loff_t i_size = i_size_read(cc->inode); 864 unsigned nr_pages = DIV_ROUND_UP(i_size, PAGE_SIZE); 865 int i; 866 867 for (i = 0; i < cc->cluster_size; i++) { 868 struct page *page = cc->rpages[i]; 869 870 f2fs_bug_on(sbi, !page); 871 872 if (unlikely(f2fs_cp_error(sbi))) 873 return false; 874 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) 875 return false; 876 877 /* beyond EOF */ 878 if (page->index >= nr_pages) 879 return false; 880 } 881 return true; 882 } 883 884 static int __f2fs_cluster_blocks(struct compress_ctx *cc, bool compr) 885 { 886 struct dnode_of_data dn; 887 int ret; 888 889 set_new_dnode(&dn, cc->inode, NULL, NULL, 0); 890 ret = f2fs_get_dnode_of_data(&dn, start_idx_of_cluster(cc), 891 LOOKUP_NODE); 892 if (ret) { 893 if (ret == -ENOENT) 894 ret = 0; 895 goto fail; 896 } 897 898 if (dn.data_blkaddr == COMPRESS_ADDR) { 899 int i; 900 901 ret = 1; 902 for (i = 1; i < cc->cluster_size; i++) { 903 block_t blkaddr; 904 905 blkaddr = data_blkaddr(dn.inode, 906 dn.node_page, dn.ofs_in_node + i); 907 if (compr) { 908 if (__is_valid_data_blkaddr(blkaddr)) 909 ret++; 910 } else { 911 if (blkaddr != NULL_ADDR) 912 ret++; 913 } 914 } 915 } 916 fail: 917 f2fs_put_dnode(&dn); 918 return ret; 919 } 920 921 /* return # of compressed blocks in compressed cluster */ 922 static int f2fs_compressed_blocks(struct compress_ctx *cc) 923 { 924 return __f2fs_cluster_blocks(cc, true); 925 } 926 927 /* return # of valid blocks in compressed cluster */ 928 static int f2fs_cluster_blocks(struct compress_ctx *cc) 929 { 930 return __f2fs_cluster_blocks(cc, false); 931 } 932 933 int f2fs_is_compressed_cluster(struct inode *inode, pgoff_t index) 934 { 935 struct compress_ctx cc = { 936 .inode = inode, 937 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 938 .cluster_size = F2FS_I(inode)->i_cluster_size, 939 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size, 940 }; 941 942 return f2fs_cluster_blocks(&cc); 943 } 944 945 static bool cluster_may_compress(struct compress_ctx *cc) 946 { 947 if (!f2fs_need_compress_data(cc->inode)) 948 return false; 949 if (f2fs_is_atomic_file(cc->inode)) 950 return false; 951 if (f2fs_is_mmap_file(cc->inode)) 952 return false; 953 if (!f2fs_cluster_is_full(cc)) 954 return false; 955 if (unlikely(f2fs_cp_error(F2FS_I_SB(cc->inode)))) 956 return false; 957 return __cluster_may_compress(cc); 958 } 959 960 static void set_cluster_writeback(struct compress_ctx *cc) 961 { 962 int i; 963 964 for (i = 0; i < cc->cluster_size; i++) { 965 if (cc->rpages[i]) 966 set_page_writeback(cc->rpages[i]); 967 } 968 } 969 970 static void set_cluster_dirty(struct compress_ctx *cc) 971 { 972 int i; 973 974 for (i = 0; i < cc->cluster_size; i++) 975 if (cc->rpages[i]) 976 set_page_dirty(cc->rpages[i]); 977 } 978 979 static int prepare_compress_overwrite(struct compress_ctx *cc, 980 struct page **pagep, pgoff_t index, void **fsdata) 981 { 982 struct f2fs_sb_info *sbi = F2FS_I_SB(cc->inode); 983 struct address_space *mapping = cc->inode->i_mapping; 984 struct page *page; 985 struct dnode_of_data dn; 986 sector_t last_block_in_bio; 987 unsigned fgp_flag = FGP_LOCK | FGP_WRITE | FGP_CREAT; 988 pgoff_t start_idx = start_idx_of_cluster(cc); 989 int i, ret; 990 bool prealloc; 991 992 retry: 993 ret = f2fs_cluster_blocks(cc); 994 if (ret <= 0) 995 return ret; 996 997 /* compressed case */ 998 prealloc = (ret < cc->cluster_size); 999 1000 ret = f2fs_init_compress_ctx(cc); 1001 if (ret) 1002 return ret; 1003 1004 /* keep page reference to avoid page reclaim */ 1005 for (i = 0; i < cc->cluster_size; i++) { 1006 page = f2fs_pagecache_get_page(mapping, start_idx + i, 1007 fgp_flag, GFP_NOFS); 1008 if (!page) { 1009 ret = -ENOMEM; 1010 goto unlock_pages; 1011 } 1012 1013 if (PageUptodate(page)) 1014 unlock_page(page); 1015 else 1016 f2fs_compress_ctx_add_page(cc, page); 1017 } 1018 1019 if (!f2fs_cluster_is_empty(cc)) { 1020 struct bio *bio = NULL; 1021 1022 ret = f2fs_read_multi_pages(cc, &bio, cc->cluster_size, 1023 &last_block_in_bio, false, true); 1024 f2fs_destroy_compress_ctx(cc); 1025 if (ret) 1026 goto release_pages; 1027 if (bio) 1028 f2fs_submit_bio(sbi, bio, DATA); 1029 1030 ret = f2fs_init_compress_ctx(cc); 1031 if (ret) 1032 goto release_pages; 1033 } 1034 1035 for (i = 0; i < cc->cluster_size; i++) { 1036 f2fs_bug_on(sbi, cc->rpages[i]); 1037 1038 page = find_lock_page(mapping, start_idx + i); 1039 f2fs_bug_on(sbi, !page); 1040 1041 f2fs_wait_on_page_writeback(page, DATA, true, true); 1042 1043 f2fs_compress_ctx_add_page(cc, page); 1044 f2fs_put_page(page, 0); 1045 1046 if (!PageUptodate(page)) { 1047 f2fs_unlock_rpages(cc, i + 1); 1048 f2fs_put_rpages_mapping(mapping, start_idx, 1049 cc->cluster_size); 1050 f2fs_destroy_compress_ctx(cc); 1051 goto retry; 1052 } 1053 } 1054 1055 if (prealloc) { 1056 f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true); 1057 1058 set_new_dnode(&dn, cc->inode, NULL, NULL, 0); 1059 1060 for (i = cc->cluster_size - 1; i > 0; i--) { 1061 ret = f2fs_get_block(&dn, start_idx + i); 1062 if (ret) { 1063 i = cc->cluster_size; 1064 break; 1065 } 1066 1067 if (dn.data_blkaddr != NEW_ADDR) 1068 break; 1069 } 1070 1071 f2fs_do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false); 1072 } 1073 1074 if (likely(!ret)) { 1075 *fsdata = cc->rpages; 1076 *pagep = cc->rpages[offset_in_cluster(cc, index)]; 1077 return cc->cluster_size; 1078 } 1079 1080 unlock_pages: 1081 f2fs_unlock_rpages(cc, i); 1082 release_pages: 1083 f2fs_put_rpages_mapping(mapping, start_idx, i); 1084 f2fs_destroy_compress_ctx(cc); 1085 return ret; 1086 } 1087 1088 int f2fs_prepare_compress_overwrite(struct inode *inode, 1089 struct page **pagep, pgoff_t index, void **fsdata) 1090 { 1091 struct compress_ctx cc = { 1092 .inode = inode, 1093 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 1094 .cluster_size = F2FS_I(inode)->i_cluster_size, 1095 .cluster_idx = index >> F2FS_I(inode)->i_log_cluster_size, 1096 .rpages = NULL, 1097 .nr_rpages = 0, 1098 }; 1099 1100 return prepare_compress_overwrite(&cc, pagep, index, fsdata); 1101 } 1102 1103 bool f2fs_compress_write_end(struct inode *inode, void *fsdata, 1104 pgoff_t index, unsigned copied) 1105 1106 { 1107 struct compress_ctx cc = { 1108 .inode = inode, 1109 .log_cluster_size = F2FS_I(inode)->i_log_cluster_size, 1110 .cluster_size = F2FS_I(inode)->i_cluster_size, 1111 .rpages = fsdata, 1112 }; 1113 bool first_index = (index == cc.rpages[0]->index); 1114 1115 if (copied) 1116 set_cluster_dirty(&cc); 1117 1118 f2fs_put_rpages_wbc(&cc, NULL, false, 1); 1119 f2fs_destroy_compress_ctx(&cc); 1120 1121 return first_index; 1122 } 1123 1124 int f2fs_truncate_partial_cluster(struct inode *inode, u64 from, bool lock) 1125 { 1126 void *fsdata = NULL; 1127 struct page *pagep; 1128 int log_cluster_size = F2FS_I(inode)->i_log_cluster_size; 1129 pgoff_t start_idx = from >> (PAGE_SHIFT + log_cluster_size) << 1130 log_cluster_size; 1131 int err; 1132 1133 err = f2fs_is_compressed_cluster(inode, start_idx); 1134 if (err < 0) 1135 return err; 1136 1137 /* truncate normal cluster */ 1138 if (!err) 1139 return f2fs_do_truncate_blocks(inode, from, lock); 1140 1141 /* truncate compressed cluster */ 1142 err = f2fs_prepare_compress_overwrite(inode, &pagep, 1143 start_idx, &fsdata); 1144 1145 /* should not be a normal cluster */ 1146 f2fs_bug_on(F2FS_I_SB(inode), err == 0); 1147 1148 if (err <= 0) 1149 return err; 1150 1151 if (err > 0) { 1152 struct page **rpages = fsdata; 1153 int cluster_size = F2FS_I(inode)->i_cluster_size; 1154 int i; 1155 1156 for (i = cluster_size - 1; i >= 0; i--) { 1157 loff_t start = rpages[i]->index << PAGE_SHIFT; 1158 1159 if (from <= start) { 1160 zero_user_segment(rpages[i], 0, PAGE_SIZE); 1161 } else { 1162 zero_user_segment(rpages[i], from - start, 1163 PAGE_SIZE); 1164 break; 1165 } 1166 } 1167 1168 f2fs_compress_write_end(inode, fsdata, start_idx, true); 1169 } 1170 return 0; 1171 } 1172 1173 static int f2fs_write_compressed_pages(struct compress_ctx *cc, 1174 int *submitted, 1175 struct writeback_control *wbc, 1176 enum iostat_type io_type) 1177 { 1178 struct inode *inode = cc->inode; 1179 struct f2fs_sb_info *sbi = F2FS_I_SB(inode); 1180 struct f2fs_inode_info *fi = F2FS_I(inode); 1181 struct f2fs_io_info fio = { 1182 .sbi = sbi, 1183 .ino = cc->inode->i_ino, 1184 .type = DATA, 1185 .op = REQ_OP_WRITE, 1186 .op_flags = wbc_to_write_flags(wbc), 1187 .old_blkaddr = NEW_ADDR, 1188 .page = NULL, 1189 .encrypted_page = NULL, 1190 .compressed_page = NULL, 1191 .submitted = false, 1192 .io_type = io_type, 1193 .io_wbc = wbc, 1194 .encrypted = fscrypt_inode_uses_fs_layer_crypto(cc->inode), 1195 }; 1196 struct dnode_of_data dn; 1197 struct node_info ni; 1198 struct compress_io_ctx *cic; 1199 pgoff_t start_idx = start_idx_of_cluster(cc); 1200 unsigned int last_index = cc->cluster_size - 1; 1201 loff_t psize; 1202 int i, err; 1203 1204 if (IS_NOQUOTA(inode)) { 1205 /* 1206 * We need to wait for node_write to avoid block allocation during 1207 * checkpoint. This can only happen to quota writes which can cause 1208 * the below discard race condition. 1209 */ 1210 down_read(&sbi->node_write); 1211 } else if (!f2fs_trylock_op(sbi)) { 1212 goto out_free; 1213 } 1214 1215 set_new_dnode(&dn, cc->inode, NULL, NULL, 0); 1216 1217 err = f2fs_get_dnode_of_data(&dn, start_idx, LOOKUP_NODE); 1218 if (err) 1219 goto out_unlock_op; 1220 1221 for (i = 0; i < cc->cluster_size; i++) { 1222 if (data_blkaddr(dn.inode, dn.node_page, 1223 dn.ofs_in_node + i) == NULL_ADDR) 1224 goto out_put_dnode; 1225 } 1226 1227 psize = (loff_t)(cc->rpages[last_index]->index + 1) << PAGE_SHIFT; 1228 1229 err = f2fs_get_node_info(fio.sbi, dn.nid, &ni); 1230 if (err) 1231 goto out_put_dnode; 1232 1233 fio.version = ni.version; 1234 1235 cic = kmem_cache_zalloc(cic_entry_slab, GFP_NOFS); 1236 if (!cic) 1237 goto out_put_dnode; 1238 1239 cic->magic = F2FS_COMPRESSED_PAGE_MAGIC; 1240 cic->inode = inode; 1241 atomic_set(&cic->pending_pages, cc->nr_cpages); 1242 cic->rpages = page_array_alloc(cc->inode, cc->cluster_size); 1243 if (!cic->rpages) 1244 goto out_put_cic; 1245 1246 cic->nr_rpages = cc->cluster_size; 1247 1248 for (i = 0; i < cc->nr_cpages; i++) { 1249 f2fs_set_compressed_page(cc->cpages[i], inode, 1250 cc->rpages[i + 1]->index, cic); 1251 fio.compressed_page = cc->cpages[i]; 1252 1253 fio.old_blkaddr = data_blkaddr(dn.inode, dn.node_page, 1254 dn.ofs_in_node + i + 1); 1255 1256 /* wait for GCed page writeback via META_MAPPING */ 1257 f2fs_wait_on_block_writeback(inode, fio.old_blkaddr); 1258 1259 if (fio.encrypted) { 1260 fio.page = cc->rpages[i + 1]; 1261 err = f2fs_encrypt_one_page(&fio); 1262 if (err) 1263 goto out_destroy_crypt; 1264 cc->cpages[i] = fio.encrypted_page; 1265 } 1266 } 1267 1268 set_cluster_writeback(cc); 1269 1270 for (i = 0; i < cc->cluster_size; i++) 1271 cic->rpages[i] = cc->rpages[i]; 1272 1273 for (i = 0; i < cc->cluster_size; i++, dn.ofs_in_node++) { 1274 block_t blkaddr; 1275 1276 blkaddr = f2fs_data_blkaddr(&dn); 1277 fio.page = cc->rpages[i]; 1278 fio.old_blkaddr = blkaddr; 1279 1280 /* cluster header */ 1281 if (i == 0) { 1282 if (blkaddr == COMPRESS_ADDR) 1283 fio.compr_blocks++; 1284 if (__is_valid_data_blkaddr(blkaddr)) 1285 f2fs_invalidate_blocks(sbi, blkaddr); 1286 f2fs_update_data_blkaddr(&dn, COMPRESS_ADDR); 1287 goto unlock_continue; 1288 } 1289 1290 if (fio.compr_blocks && __is_valid_data_blkaddr(blkaddr)) 1291 fio.compr_blocks++; 1292 1293 if (i > cc->nr_cpages) { 1294 if (__is_valid_data_blkaddr(blkaddr)) { 1295 f2fs_invalidate_blocks(sbi, blkaddr); 1296 f2fs_update_data_blkaddr(&dn, NEW_ADDR); 1297 } 1298 goto unlock_continue; 1299 } 1300 1301 f2fs_bug_on(fio.sbi, blkaddr == NULL_ADDR); 1302 1303 if (fio.encrypted) 1304 fio.encrypted_page = cc->cpages[i - 1]; 1305 else 1306 fio.compressed_page = cc->cpages[i - 1]; 1307 1308 cc->cpages[i - 1] = NULL; 1309 f2fs_outplace_write_data(&dn, &fio); 1310 (*submitted)++; 1311 unlock_continue: 1312 inode_dec_dirty_pages(cc->inode); 1313 unlock_page(fio.page); 1314 } 1315 1316 if (fio.compr_blocks) 1317 f2fs_i_compr_blocks_update(inode, fio.compr_blocks - 1, false); 1318 f2fs_i_compr_blocks_update(inode, cc->nr_cpages, true); 1319 1320 set_inode_flag(cc->inode, FI_APPEND_WRITE); 1321 if (cc->cluster_idx == 0) 1322 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN); 1323 1324 f2fs_put_dnode(&dn); 1325 if (IS_NOQUOTA(inode)) 1326 up_read(&sbi->node_write); 1327 else 1328 f2fs_unlock_op(sbi); 1329 1330 spin_lock(&fi->i_size_lock); 1331 if (fi->last_disk_size < psize) 1332 fi->last_disk_size = psize; 1333 spin_unlock(&fi->i_size_lock); 1334 1335 f2fs_put_rpages(cc); 1336 page_array_free(cc->inode, cc->cpages, cc->nr_cpages); 1337 cc->cpages = NULL; 1338 f2fs_destroy_compress_ctx(cc); 1339 return 0; 1340 1341 out_destroy_crypt: 1342 page_array_free(cc->inode, cic->rpages, cc->cluster_size); 1343 1344 for (--i; i >= 0; i--) 1345 fscrypt_finalize_bounce_page(&cc->cpages[i]); 1346 for (i = 0; i < cc->nr_cpages; i++) { 1347 if (!cc->cpages[i]) 1348 continue; 1349 f2fs_put_page(cc->cpages[i], 1); 1350 } 1351 out_put_cic: 1352 kmem_cache_free(cic_entry_slab, cic); 1353 out_put_dnode: 1354 f2fs_put_dnode(&dn); 1355 out_unlock_op: 1356 if (IS_NOQUOTA(inode)) 1357 up_read(&sbi->node_write); 1358 else 1359 f2fs_unlock_op(sbi); 1360 out_free: 1361 page_array_free(cc->inode, cc->cpages, cc->nr_cpages); 1362 cc->cpages = NULL; 1363 return -EAGAIN; 1364 } 1365 1366 void f2fs_compress_write_end_io(struct bio *bio, struct page *page) 1367 { 1368 struct f2fs_sb_info *sbi = bio->bi_private; 1369 struct compress_io_ctx *cic = 1370 (struct compress_io_ctx *)page_private(page); 1371 int i; 1372 1373 if (unlikely(bio->bi_status)) 1374 mapping_set_error(cic->inode->i_mapping, -EIO); 1375 1376 f2fs_compress_free_page(page); 1377 1378 dec_page_count(sbi, F2FS_WB_DATA); 1379 1380 if (atomic_dec_return(&cic->pending_pages)) 1381 return; 1382 1383 for (i = 0; i < cic->nr_rpages; i++) { 1384 WARN_ON(!cic->rpages[i]); 1385 clear_cold_data(cic->rpages[i]); 1386 end_page_writeback(cic->rpages[i]); 1387 } 1388 1389 page_array_free(cic->inode, cic->rpages, cic->nr_rpages); 1390 kmem_cache_free(cic_entry_slab, cic); 1391 } 1392 1393 static int f2fs_write_raw_pages(struct compress_ctx *cc, 1394 int *submitted, 1395 struct writeback_control *wbc, 1396 enum iostat_type io_type) 1397 { 1398 struct address_space *mapping = cc->inode->i_mapping; 1399 int _submitted, compr_blocks, ret; 1400 int i = -1, err = 0; 1401 1402 compr_blocks = f2fs_compressed_blocks(cc); 1403 if (compr_blocks < 0) { 1404 err = compr_blocks; 1405 goto out_err; 1406 } 1407 1408 for (i = 0; i < cc->cluster_size; i++) { 1409 if (!cc->rpages[i]) 1410 continue; 1411 retry_write: 1412 if (cc->rpages[i]->mapping != mapping) { 1413 unlock_page(cc->rpages[i]); 1414 continue; 1415 } 1416 1417 BUG_ON(!PageLocked(cc->rpages[i])); 1418 1419 ret = f2fs_write_single_data_page(cc->rpages[i], &_submitted, 1420 NULL, NULL, wbc, io_type, 1421 compr_blocks); 1422 if (ret) { 1423 if (ret == AOP_WRITEPAGE_ACTIVATE) { 1424 unlock_page(cc->rpages[i]); 1425 ret = 0; 1426 } else if (ret == -EAGAIN) { 1427 /* 1428 * for quota file, just redirty left pages to 1429 * avoid deadlock caused by cluster update race 1430 * from foreground operation. 1431 */ 1432 if (IS_NOQUOTA(cc->inode)) { 1433 err = 0; 1434 goto out_err; 1435 } 1436 ret = 0; 1437 cond_resched(); 1438 congestion_wait(BLK_RW_ASYNC, 1439 DEFAULT_IO_TIMEOUT); 1440 lock_page(cc->rpages[i]); 1441 1442 if (!PageDirty(cc->rpages[i])) { 1443 unlock_page(cc->rpages[i]); 1444 continue; 1445 } 1446 1447 clear_page_dirty_for_io(cc->rpages[i]); 1448 goto retry_write; 1449 } 1450 err = ret; 1451 goto out_err; 1452 } 1453 1454 *submitted += _submitted; 1455 } 1456 return 0; 1457 out_err: 1458 for (++i; i < cc->cluster_size; i++) { 1459 if (!cc->rpages[i]) 1460 continue; 1461 redirty_page_for_writepage(wbc, cc->rpages[i]); 1462 unlock_page(cc->rpages[i]); 1463 } 1464 return err; 1465 } 1466 1467 int f2fs_write_multi_pages(struct compress_ctx *cc, 1468 int *submitted, 1469 struct writeback_control *wbc, 1470 enum iostat_type io_type) 1471 { 1472 int err; 1473 1474 *submitted = 0; 1475 if (cluster_may_compress(cc)) { 1476 err = f2fs_compress_pages(cc); 1477 if (err == -EAGAIN) { 1478 goto write; 1479 } else if (err) { 1480 f2fs_put_rpages_wbc(cc, wbc, true, 1); 1481 goto destroy_out; 1482 } 1483 1484 err = f2fs_write_compressed_pages(cc, submitted, 1485 wbc, io_type); 1486 if (!err) 1487 return 0; 1488 f2fs_bug_on(F2FS_I_SB(cc->inode), err != -EAGAIN); 1489 } 1490 write: 1491 f2fs_bug_on(F2FS_I_SB(cc->inode), *submitted); 1492 1493 err = f2fs_write_raw_pages(cc, submitted, wbc, io_type); 1494 f2fs_put_rpages_wbc(cc, wbc, false, 0); 1495 destroy_out: 1496 f2fs_destroy_compress_ctx(cc); 1497 return err; 1498 } 1499 1500 struct decompress_io_ctx *f2fs_alloc_dic(struct compress_ctx *cc) 1501 { 1502 struct decompress_io_ctx *dic; 1503 pgoff_t start_idx = start_idx_of_cluster(cc); 1504 int i; 1505 1506 dic = kmem_cache_zalloc(dic_entry_slab, GFP_NOFS); 1507 if (!dic) 1508 return ERR_PTR(-ENOMEM); 1509 1510 dic->rpages = page_array_alloc(cc->inode, cc->cluster_size); 1511 if (!dic->rpages) { 1512 kmem_cache_free(dic_entry_slab, dic); 1513 return ERR_PTR(-ENOMEM); 1514 } 1515 1516 dic->magic = F2FS_COMPRESSED_PAGE_MAGIC; 1517 dic->inode = cc->inode; 1518 atomic_set(&dic->pending_pages, cc->nr_cpages); 1519 dic->cluster_idx = cc->cluster_idx; 1520 dic->cluster_size = cc->cluster_size; 1521 dic->log_cluster_size = cc->log_cluster_size; 1522 dic->nr_cpages = cc->nr_cpages; 1523 dic->failed = false; 1524 1525 for (i = 0; i < dic->cluster_size; i++) 1526 dic->rpages[i] = cc->rpages[i]; 1527 dic->nr_rpages = cc->cluster_size; 1528 1529 dic->cpages = page_array_alloc(dic->inode, dic->nr_cpages); 1530 if (!dic->cpages) 1531 goto out_free; 1532 1533 for (i = 0; i < dic->nr_cpages; i++) { 1534 struct page *page; 1535 1536 page = f2fs_compress_alloc_page(); 1537 if (!page) 1538 goto out_free; 1539 1540 f2fs_set_compressed_page(page, cc->inode, 1541 start_idx + i + 1, dic); 1542 dic->cpages[i] = page; 1543 } 1544 1545 return dic; 1546 1547 out_free: 1548 f2fs_free_dic(dic); 1549 return ERR_PTR(-ENOMEM); 1550 } 1551 1552 void f2fs_free_dic(struct decompress_io_ctx *dic) 1553 { 1554 int i; 1555 1556 if (dic->tpages) { 1557 for (i = 0; i < dic->cluster_size; i++) { 1558 if (dic->rpages[i]) 1559 continue; 1560 if (!dic->tpages[i]) 1561 continue; 1562 f2fs_compress_free_page(dic->tpages[i]); 1563 } 1564 page_array_free(dic->inode, dic->tpages, dic->cluster_size); 1565 } 1566 1567 if (dic->cpages) { 1568 for (i = 0; i < dic->nr_cpages; i++) { 1569 if (!dic->cpages[i]) 1570 continue; 1571 f2fs_compress_free_page(dic->cpages[i]); 1572 } 1573 page_array_free(dic->inode, dic->cpages, dic->nr_cpages); 1574 } 1575 1576 page_array_free(dic->inode, dic->rpages, dic->nr_rpages); 1577 kmem_cache_free(dic_entry_slab, dic); 1578 } 1579 1580 void f2fs_decompress_end_io(struct page **rpages, 1581 unsigned int cluster_size, bool err, bool verity) 1582 { 1583 int i; 1584 1585 for (i = 0; i < cluster_size; i++) { 1586 struct page *rpage = rpages[i]; 1587 1588 if (!rpage) 1589 continue; 1590 1591 if (err || PageError(rpage)) 1592 goto clear_uptodate; 1593 1594 if (!verity || fsverity_verify_page(rpage)) { 1595 SetPageUptodate(rpage); 1596 goto unlock; 1597 } 1598 clear_uptodate: 1599 ClearPageUptodate(rpage); 1600 ClearPageError(rpage); 1601 unlock: 1602 unlock_page(rpage); 1603 } 1604 } 1605 1606 int f2fs_init_page_array_cache(struct f2fs_sb_info *sbi) 1607 { 1608 dev_t dev = sbi->sb->s_bdev->bd_dev; 1609 char slab_name[32]; 1610 1611 sprintf(slab_name, "f2fs_page_array_entry-%u:%u", MAJOR(dev), MINOR(dev)); 1612 1613 sbi->page_array_slab_size = sizeof(struct page *) << 1614 F2FS_OPTION(sbi).compress_log_size; 1615 1616 sbi->page_array_slab = f2fs_kmem_cache_create(slab_name, 1617 sbi->page_array_slab_size); 1618 if (!sbi->page_array_slab) 1619 return -ENOMEM; 1620 return 0; 1621 } 1622 1623 void f2fs_destroy_page_array_cache(struct f2fs_sb_info *sbi) 1624 { 1625 kmem_cache_destroy(sbi->page_array_slab); 1626 } 1627 1628 static int __init f2fs_init_cic_cache(void) 1629 { 1630 cic_entry_slab = f2fs_kmem_cache_create("f2fs_cic_entry", 1631 sizeof(struct compress_io_ctx)); 1632 if (!cic_entry_slab) 1633 return -ENOMEM; 1634 return 0; 1635 } 1636 1637 static void f2fs_destroy_cic_cache(void) 1638 { 1639 kmem_cache_destroy(cic_entry_slab); 1640 } 1641 1642 static int __init f2fs_init_dic_cache(void) 1643 { 1644 dic_entry_slab = f2fs_kmem_cache_create("f2fs_dic_entry", 1645 sizeof(struct decompress_io_ctx)); 1646 if (!dic_entry_slab) 1647 return -ENOMEM; 1648 return 0; 1649 } 1650 1651 static void f2fs_destroy_dic_cache(void) 1652 { 1653 kmem_cache_destroy(dic_entry_slab); 1654 } 1655 1656 int __init f2fs_init_compress_cache(void) 1657 { 1658 int err; 1659 1660 err = f2fs_init_cic_cache(); 1661 if (err) 1662 goto out; 1663 err = f2fs_init_dic_cache(); 1664 if (err) 1665 goto free_cic; 1666 return 0; 1667 free_cic: 1668 f2fs_destroy_cic_cache(); 1669 out: 1670 return -ENOMEM; 1671 } 1672 1673 void f2fs_destroy_compress_cache(void) 1674 { 1675 f2fs_destroy_dic_cache(); 1676 f2fs_destroy_cic_cache(); 1677 } 1678