1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2016-present, Facebook, Inc. 4 * All rights reserved. 5 * 6 */ 7 8 #include <linux/bio.h> 9 #include <linux/bitmap.h> 10 #include <linux/err.h> 11 #include <linux/init.h> 12 #include <linux/kernel.h> 13 #include <linux/mm.h> 14 #include <linux/sched/mm.h> 15 #include <linux/pagemap.h> 16 #include <linux/refcount.h> 17 #include <linux/sched.h> 18 #include <linux/slab.h> 19 #include <linux/zstd.h> 20 #include "misc.h" 21 #include "fs.h" 22 #include "btrfs_inode.h" 23 #include "compression.h" 24 #include "super.h" 25 26 #define ZSTD_BTRFS_MAX_WINDOWLOG 17 27 #define ZSTD_BTRFS_MAX_INPUT (1U << ZSTD_BTRFS_MAX_WINDOWLOG) 28 #define ZSTD_BTRFS_DEFAULT_LEVEL 3 29 #define ZSTD_BTRFS_MIN_LEVEL -15 30 #define ZSTD_BTRFS_MAX_LEVEL 15 31 /* 307s to avoid pathologically clashing with transaction commit */ 32 #define ZSTD_BTRFS_RECLAIM_JIFFIES (307 * HZ) 33 34 static zstd_parameters zstd_get_btrfs_parameters(int level, 35 size_t src_len) 36 { 37 zstd_parameters params = zstd_get_params(level, src_len); 38 39 if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG) 40 params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG; 41 WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT); 42 return params; 43 } 44 45 struct workspace { 46 void *mem; 47 size_t size; 48 char *buf; 49 int level; 50 int req_level; 51 unsigned long last_used; /* jiffies */ 52 struct list_head list; 53 struct list_head lru_list; 54 zstd_in_buffer in_buf; 55 zstd_out_buffer out_buf; 56 zstd_parameters params; 57 }; 58 59 /* 60 * Zstd Workspace Management 61 * 62 * Zstd workspaces have different memory requirements depending on the level. 63 * The zstd workspaces are managed by having individual lists for each level 64 * and a global lru. Forward progress is maintained by protecting a max level 65 * workspace. 66 * 67 * Getting a workspace is done by using the bitmap to identify the levels that 68 * have available workspaces and scans up. This lets us recycle higher level 69 * workspaces because of the monotonic memory guarantee. A workspace's 70 * last_used is only updated if it is being used by the corresponding memory 71 * level. Putting a workspace involves adding it back to the appropriate places 72 * and adding it back to the lru if necessary. 73 * 74 * A timer is used to reclaim workspaces if they have not been used for 75 * ZSTD_BTRFS_RECLAIM_JIFFIES. This helps keep only active workspaces around. 76 * The upper bound is provided by the workqueue limit which is 2 (percpu limit). 77 */ 78 79 struct zstd_workspace_manager { 80 spinlock_t lock; 81 struct list_head lru_list; 82 struct list_head idle_ws[ZSTD_BTRFS_MAX_LEVEL]; 83 unsigned long active_map; 84 wait_queue_head_t wait; 85 struct timer_list timer; 86 }; 87 88 static size_t zstd_ws_mem_sizes[ZSTD_BTRFS_MAX_LEVEL]; 89 90 static inline struct workspace *list_to_workspace(struct list_head *list) 91 { 92 return container_of(list, struct workspace, list); 93 } 94 95 static inline int clip_level(int level) 96 { 97 return max(0, level - 1); 98 } 99 100 /* 101 * Timer callback to free unused workspaces. 102 * 103 * @t: timer 104 * 105 * This scans the lru_list and attempts to reclaim any workspace that hasn't 106 * been used for ZSTD_BTRFS_RECLAIM_JIFFIES. 107 * 108 * The context is softirq and does not need the _bh locking primitives. 109 */ 110 static void zstd_reclaim_timer_fn(struct timer_list *timer) 111 { 112 struct zstd_workspace_manager *zwsm = 113 container_of(timer, struct zstd_workspace_manager, timer); 114 unsigned long reclaim_threshold = jiffies - ZSTD_BTRFS_RECLAIM_JIFFIES; 115 struct list_head *pos, *next; 116 117 spin_lock(&zwsm->lock); 118 119 if (list_empty(&zwsm->lru_list)) { 120 spin_unlock(&zwsm->lock); 121 return; 122 } 123 124 list_for_each_prev_safe(pos, next, &zwsm->lru_list) { 125 struct workspace *victim = container_of(pos, struct workspace, 126 lru_list); 127 int level; 128 129 if (time_after(victim->last_used, reclaim_threshold)) 130 break; 131 132 /* workspace is in use */ 133 if (victim->req_level) 134 continue; 135 136 level = victim->level; 137 list_del(&victim->lru_list); 138 list_del(&victim->list); 139 zstd_free_workspace(&victim->list); 140 141 if (list_empty(&zwsm->idle_ws[level])) 142 clear_bit(level, &zwsm->active_map); 143 144 } 145 146 if (!list_empty(&zwsm->lru_list)) 147 mod_timer(&zwsm->timer, jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES); 148 149 spin_unlock(&zwsm->lock); 150 } 151 152 /* 153 * Calculate monotonic memory bounds. 154 * 155 * It is possible based on the level configurations that a higher level 156 * workspace uses less memory than a lower level workspace. In order to reuse 157 * workspaces, this must be made a monotonic relationship. This precomputes 158 * the required memory for each level and enforces the monotonicity between 159 * level and memory required. 160 */ 161 static void zstd_calc_ws_mem_sizes(void) 162 { 163 size_t max_size = 0; 164 int level; 165 166 for (level = ZSTD_BTRFS_MIN_LEVEL; level <= ZSTD_BTRFS_MAX_LEVEL; level++) { 167 if (level == 0) 168 continue; 169 zstd_parameters params = 170 zstd_get_btrfs_parameters(level, ZSTD_BTRFS_MAX_INPUT); 171 size_t level_size = 172 max_t(size_t, 173 zstd_cstream_workspace_bound(¶ms.cParams), 174 zstd_dstream_workspace_bound(ZSTD_BTRFS_MAX_INPUT)); 175 176 max_size = max_t(size_t, max_size, level_size); 177 /* Use level 1 workspace size for all the fast mode negative levels. */ 178 zstd_ws_mem_sizes[clip_level(level)] = max_size; 179 } 180 } 181 182 int zstd_alloc_workspace_manager(struct btrfs_fs_info *fs_info) 183 { 184 struct zstd_workspace_manager *zwsm; 185 struct list_head *ws; 186 187 ASSERT(fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] == NULL); 188 zwsm = kzalloc_obj(*zwsm); 189 if (!zwsm) 190 return -ENOMEM; 191 zstd_calc_ws_mem_sizes(); 192 spin_lock_init(&zwsm->lock); 193 init_waitqueue_head(&zwsm->wait); 194 timer_setup(&zwsm->timer, zstd_reclaim_timer_fn, 0); 195 196 INIT_LIST_HEAD(&zwsm->lru_list); 197 for (int i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) 198 INIT_LIST_HEAD(&zwsm->idle_ws[i]); 199 fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] = zwsm; 200 201 ws = zstd_alloc_workspace(fs_info, ZSTD_BTRFS_MAX_LEVEL); 202 if (IS_ERR(ws)) { 203 btrfs_warn(NULL, "cannot preallocate zstd compression workspace"); 204 } else { 205 set_bit(ZSTD_BTRFS_MAX_LEVEL - 1, &zwsm->active_map); 206 list_add(ws, &zwsm->idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1]); 207 } 208 return 0; 209 } 210 211 void zstd_free_workspace_manager(struct btrfs_fs_info *fs_info) 212 { 213 struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD]; 214 struct workspace *workspace; 215 216 if (!zwsm) 217 return; 218 fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] = NULL; 219 spin_lock_bh(&zwsm->lock); 220 for (int i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) { 221 while (!list_empty(&zwsm->idle_ws[i])) { 222 workspace = container_of(zwsm->idle_ws[i].next, 223 struct workspace, list); 224 list_del(&workspace->list); 225 list_del(&workspace->lru_list); 226 zstd_free_workspace(&workspace->list); 227 } 228 } 229 spin_unlock_bh(&zwsm->lock); 230 timer_delete_sync(&zwsm->timer); 231 kfree(zwsm); 232 } 233 234 /* 235 * Find workspace for given level. 236 * 237 * @level: compression level 238 * 239 * This iterates over the set bits in the active_map beginning at the requested 240 * compression level. This lets us utilize already allocated workspaces before 241 * allocating a new one. If the workspace is of a larger size, it is used, but 242 * the place in the lru_list and last_used times are not updated. This is to 243 * offer the opportunity to reclaim the workspace in favor of allocating an 244 * appropriately sized one in the future. 245 */ 246 static struct list_head *zstd_find_workspace(struct btrfs_fs_info *fs_info, int level) 247 { 248 struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD]; 249 struct list_head *ws; 250 struct workspace *workspace; 251 int i = clip_level(level); 252 253 ASSERT(zwsm); 254 spin_lock_bh(&zwsm->lock); 255 for_each_set_bit_from(i, &zwsm->active_map, ZSTD_BTRFS_MAX_LEVEL) { 256 if (!list_empty(&zwsm->idle_ws[i])) { 257 ws = zwsm->idle_ws[i].next; 258 workspace = list_to_workspace(ws); 259 list_del_init(ws); 260 /* keep its place if it's a lower level using this */ 261 workspace->req_level = level; 262 if (clip_level(level) == workspace->level) 263 list_del(&workspace->lru_list); 264 if (list_empty(&zwsm->idle_ws[i])) 265 clear_bit(i, &zwsm->active_map); 266 spin_unlock_bh(&zwsm->lock); 267 return ws; 268 } 269 } 270 spin_unlock_bh(&zwsm->lock); 271 272 return NULL; 273 } 274 275 /* 276 * Zstd get_workspace for level. 277 * 278 * @level: compression level 279 * 280 * If @level is 0, then any compression level can be used. Therefore, we begin 281 * scanning from 1. We first scan through possible workspaces and then after 282 * attempt to allocate a new workspace. If we fail to allocate one due to 283 * memory pressure, go to sleep waiting for the max level workspace to free up. 284 */ 285 struct list_head *zstd_get_workspace(struct btrfs_fs_info *fs_info, int level) 286 { 287 struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD]; 288 struct list_head *ws; 289 unsigned int nofs_flag; 290 291 ASSERT(zwsm); 292 293 /* level == 0 means we can use any workspace */ 294 if (!level) 295 level = 1; 296 297 again: 298 ws = zstd_find_workspace(fs_info, level); 299 if (ws) 300 return ws; 301 302 nofs_flag = memalloc_nofs_save(); 303 ws = zstd_alloc_workspace(fs_info, level); 304 memalloc_nofs_restore(nofs_flag); 305 306 if (IS_ERR(ws)) { 307 DEFINE_WAIT(wait); 308 309 prepare_to_wait(&zwsm->wait, &wait, TASK_UNINTERRUPTIBLE); 310 /* 311 * Re-check after being queued: zstd_put_workspace() only wakes 312 * a queue that already has a sleeper, so a workspace returned 313 * since the failed allocation woke nobody. 314 */ 315 ws = zstd_find_workspace(fs_info, level); 316 if (!ws) 317 schedule(); 318 finish_wait(&zwsm->wait, &wait); 319 if (ws) 320 return ws; 321 322 goto again; 323 } 324 325 return ws; 326 } 327 328 /* 329 * Zstd put_workspace. 330 * 331 * @ws: list_head for the workspace 332 * 333 * When putting back a workspace, we only need to update the LRU if we are of 334 * the requested compression level. Here is where we continue to protect the 335 * max level workspace or update last_used accordingly. If the reclaim timer 336 * isn't set, it is also set here. Only the max level workspace tries and wakes 337 * up waiting workspaces. 338 */ 339 void zstd_put_workspace(struct btrfs_fs_info *fs_info, struct list_head *ws) 340 { 341 struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD]; 342 struct workspace *workspace = list_to_workspace(ws); 343 344 ASSERT(zwsm); 345 spin_lock_bh(&zwsm->lock); 346 347 /* A node is only taken off the lru if we are the corresponding level */ 348 if (clip_level(workspace->req_level) == workspace->level) { 349 /* Hide a max level workspace from reclaim */ 350 if (workspace->level == ZSTD_BTRFS_MAX_LEVEL - 1 && 351 list_empty(&zwsm->idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1])) { 352 INIT_LIST_HEAD(&workspace->lru_list); 353 } else { 354 workspace->last_used = jiffies; 355 list_add(&workspace->lru_list, &zwsm->lru_list); 356 if (!timer_pending(&zwsm->timer)) 357 mod_timer(&zwsm->timer, 358 jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES); 359 } 360 } 361 362 set_bit(workspace->level, &zwsm->active_map); 363 list_add(&workspace->list, &zwsm->idle_ws[workspace->level]); 364 workspace->req_level = 0; 365 366 spin_unlock_bh(&zwsm->lock); 367 368 if (workspace->level == clip_level(ZSTD_BTRFS_MAX_LEVEL)) 369 cond_wake_up(&zwsm->wait); 370 } 371 372 void zstd_free_workspace(struct list_head *ws) 373 { 374 struct workspace *workspace = list_entry(ws, struct workspace, list); 375 376 kvfree(workspace->mem); 377 kvfree(workspace->buf); 378 kfree(workspace); 379 } 380 381 struct list_head *zstd_alloc_workspace(struct btrfs_fs_info *fs_info, int level) 382 { 383 struct workspace *workspace; 384 385 workspace = kzalloc_obj(*workspace); 386 if (!workspace) 387 return ERR_PTR(-ENOMEM); 388 389 /* Use level 1 workspace size for all the fast mode negative levels. */ 390 workspace->size = zstd_ws_mem_sizes[clip_level(level)]; 391 workspace->level = clip_level(level); 392 workspace->req_level = level; 393 workspace->last_used = jiffies; 394 workspace->mem = kvmalloc(workspace->size, GFP_KERNEL | __GFP_NOWARN); 395 workspace->buf = kvmalloc(fs_info->sectorsize, GFP_KERNEL); 396 if (!workspace->mem || !workspace->buf) 397 goto fail; 398 399 INIT_LIST_HEAD(&workspace->list); 400 INIT_LIST_HEAD(&workspace->lru_list); 401 402 return &workspace->list; 403 fail: 404 zstd_free_workspace(&workspace->list); 405 return ERR_PTR(-ENOMEM); 406 } 407 408 int zstd_compress_bio(struct list_head *ws, struct compressed_bio *cb) 409 { 410 struct btrfs_inode *inode = cb->bbio.inode; 411 struct btrfs_fs_info *fs_info = inode->root->fs_info; 412 struct workspace *workspace = list_entry(ws, struct workspace, list); 413 struct address_space *mapping = inode->vfs_inode.i_mapping; 414 struct bio *bio = &cb->bbio.bio; 415 zstd_cstream *stream; 416 int ret = 0; 417 /* The current folio to read. */ 418 struct folio *in_folio = NULL; 419 /* The current folio to write to. */ 420 struct folio *out_folio = NULL; 421 unsigned long tot_in = 0; 422 unsigned long tot_out = 0; 423 const u64 start = cb->start; 424 const u32 len = cb->len; 425 const u64 end = start + len; 426 const u32 min_folio_size = btrfs_min_folio_size(fs_info); 427 428 workspace->params = zstd_get_btrfs_parameters(workspace->req_level, len); 429 430 /* Initialize the stream. */ 431 stream = zstd_init_cstream(&workspace->params, len, workspace->mem, workspace->size); 432 if (unlikely(!stream)) { 433 btrfs_err(fs_info, 434 "zstd compression init level %d failed, root %llu inode %llu offset %llu", 435 workspace->req_level, btrfs_root_id(inode->root), 436 btrfs_ino(inode), start); 437 ret = -EIO; 438 goto out; 439 } 440 441 /* Map in the first page of input data. */ 442 ret = btrfs_compress_filemap_get_folio(mapping, start, &in_folio); 443 if (ret < 0) 444 goto out; 445 workspace->in_buf.src = kmap_local_folio(in_folio, offset_in_folio(in_folio, start)); 446 workspace->in_buf.pos = 0; 447 workspace->in_buf.size = btrfs_calc_input_length(in_folio, end, start); 448 449 /* Allocate and map in the output buffer. */ 450 out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); 451 if (out_folio == NULL) { 452 ret = -ENOMEM; 453 goto out; 454 } 455 workspace->out_buf.dst = folio_address(out_folio); 456 workspace->out_buf.pos = 0; 457 workspace->out_buf.size = min_folio_size; 458 459 while (1) { 460 size_t ret2; 461 462 ret2 = zstd_compress_stream(stream, &workspace->out_buf, &workspace->in_buf); 463 if (unlikely(zstd_is_error(ret2))) { 464 btrfs_warn(fs_info, 465 "zstd compression level %d failed, error %d root %llu inode %llu offset %llu", 466 workspace->req_level, zstd_get_error_code(ret2), 467 btrfs_root_id(inode->root), btrfs_ino(inode), 468 start + tot_in); 469 ret = -EIO; 470 goto out; 471 } 472 473 /* Check to see if we are making it bigger. */ 474 if (tot_in + workspace->in_buf.pos > fs_info->sectorsize * 2 && 475 tot_in + workspace->in_buf.pos < tot_out + workspace->out_buf.pos) { 476 ret = -E2BIG; 477 goto out; 478 } 479 480 /* Check if we need more output space. */ 481 if (workspace->out_buf.pos >= workspace->out_buf.size) { 482 tot_out += min_folio_size; 483 if (tot_out >= len) { 484 ret = -E2BIG; 485 goto out; 486 } 487 /* Queue the current foliot into the bio. */ 488 if (!bio_add_folio(bio, out_folio, folio_size(out_folio), 0)) { 489 ret = -E2BIG; 490 goto out; 491 } 492 493 out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); 494 if (out_folio == NULL) { 495 ret = -ENOMEM; 496 goto out; 497 } 498 workspace->out_buf.dst = folio_address(out_folio); 499 workspace->out_buf.pos = 0; 500 workspace->out_buf.size = min_folio_size; 501 } 502 503 /* We've reached the end of the input. */ 504 if (tot_in + workspace->in_buf.pos >= len) { 505 tot_in += workspace->in_buf.pos; 506 break; 507 } 508 509 /* Check if we need more input. */ 510 if (workspace->in_buf.pos >= workspace->in_buf.size) { 511 u64 cur; 512 513 tot_in += workspace->in_buf.size; 514 cur = start + tot_in; 515 516 kunmap_local(workspace->in_buf.src); 517 workspace->in_buf.src = NULL; 518 folio_put(in_folio); 519 520 ret = btrfs_compress_filemap_get_folio(mapping, cur, &in_folio); 521 if (ret < 0) 522 goto out; 523 workspace->in_buf.src = kmap_local_folio(in_folio, 524 offset_in_folio(in_folio, cur)); 525 workspace->in_buf.pos = 0; 526 workspace->in_buf.size = btrfs_calc_input_length(in_folio, end, cur); 527 } 528 } 529 530 while (1) { 531 size_t ret2; 532 533 ret2 = zstd_end_stream(stream, &workspace->out_buf); 534 if (unlikely(zstd_is_error(ret2))) { 535 btrfs_err(fs_info, 536 "zstd compression end level %d failed, error %d root %llu inode %llu offset %llu", 537 workspace->req_level, zstd_get_error_code(ret2), 538 btrfs_root_id(inode->root), btrfs_ino(inode), 539 start + tot_in); 540 ret = -EIO; 541 goto out; 542 } 543 /* Queue the remaining part of the output folio into bio. */ 544 if (ret2 == 0) { 545 tot_out += workspace->out_buf.pos; 546 if (tot_out >= len) { 547 ret = -E2BIG; 548 goto out; 549 } 550 if (!bio_add_folio(bio, out_folio, workspace->out_buf.pos, 0)) { 551 ret = -E2BIG; 552 goto out; 553 } 554 out_folio = NULL; 555 break; 556 } 557 tot_out += min_folio_size; 558 if (tot_out >= len) { 559 ret = -E2BIG; 560 goto out; 561 } 562 if (!bio_add_folio(bio, out_folio, folio_size(out_folio), 0)) { 563 ret = -E2BIG; 564 goto out; 565 } 566 out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS); 567 if (out_folio == NULL) { 568 ret = -ENOMEM; 569 goto out; 570 } 571 workspace->out_buf.dst = folio_address(out_folio); 572 workspace->out_buf.pos = 0; 573 workspace->out_buf.size = min_folio_size; 574 } 575 576 if (tot_out >= tot_in) { 577 ret = -E2BIG; 578 goto out; 579 } 580 581 ret = 0; 582 ASSERT(tot_out == bio->bi_iter.bi_size); 583 out: 584 if (out_folio) 585 btrfs_free_compr_folio(out_folio); 586 if (workspace->in_buf.src) { 587 kunmap_local(workspace->in_buf.src); 588 folio_put(in_folio); 589 } 590 return ret; 591 } 592 593 /* 594 * Map the destination for the next chunk of output. 595 * 596 * @decompressed is the offset of the next output byte inside the fully 597 * decompressed extent. If that offset has reached the current destination 598 * segment, its page-bounded bio_vec is kmapped so that zstd can write into the 599 * page cache directly, and the number of bytes writable there is returned. 600 * Otherwise @kaddr_ret is set to NULL and the number of bytes to skip before 601 * that segment is returned. This covers both the initial prefix and gaps in 602 * the destination bio. 603 */ 604 static u32 zstd_map_dest(struct compressed_bio *cb, u32 decompressed, 605 void **kaddr_ret) 606 { 607 struct bio *orig_bio = &cb->orig_bbio->bio; 608 struct bio_vec bvec; 609 u32 bvec_offset; 610 u32 off; 611 612 bvec = bio_iter_iovec(orig_bio, orig_bio->bi_iter); 613 /* 614 * cb->start may underflow, but subtracting that value can still give us 615 * the correct offset inside the full decompressed extent. 616 */ 617 bvec_offset = page_offset(bvec.bv_page) + bvec.bv_offset - cb->start; 618 619 if (decompressed < bvec_offset) { 620 *kaddr_ret = NULL; 621 return bvec_offset - decompressed; 622 } 623 624 off = decompressed - bvec_offset; 625 ASSERT(off < bvec.bv_len); 626 *kaddr_ret = bvec_kmap_local(&bvec) + off; 627 return bvec.bv_len - off; 628 } 629 630 int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb) 631 { 632 struct btrfs_fs_info *fs_info = cb_to_fs_info(cb); 633 struct workspace *workspace = list_entry(ws, struct workspace, list); 634 struct bio *orig_bio = &cb->orig_bbio->bio; 635 struct folio_iter fi; 636 size_t srclen = bio_get_size(&cb->bbio.bio); 637 zstd_dstream *stream; 638 int ret = 0; 639 const unsigned int min_folio_size = btrfs_min_folio_size(fs_info); 640 unsigned long folio_in_index = 0; 641 unsigned long total_folios_in = DIV_ROUND_UP(srclen, min_folio_size); 642 unsigned long total_out = 0; 643 644 bio_first_folio(&fi, &cb->bbio.bio, 0); 645 if (unlikely(!fi.folio)) 646 return -EINVAL; 647 ASSERT(folio_size(fi.folio) == min_folio_size); 648 649 stream = zstd_init_dstream( 650 ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size); 651 if (unlikely(!stream)) { 652 struct btrfs_inode *inode = cb->bbio.inode; 653 654 btrfs_err(inode->root->fs_info, 655 "zstd decompression init failed, root %llu inode %llu offset %llu", 656 btrfs_root_id(inode->root), btrfs_ino(inode), cb->start); 657 ret = -EIO; 658 goto done; 659 } 660 661 workspace->in_buf.src = kmap_local_folio(fi.folio, 0); 662 workspace->in_buf.pos = 0; 663 workspace->in_buf.size = min_t(size_t, srclen, min_folio_size); 664 665 while (orig_bio->bi_iter.bi_size) { 666 size_t ret2; 667 void *kaddr; 668 u32 dstlen; 669 670 dstlen = zstd_map_dest(cb, total_out, &kaddr); 671 if (kaddr) { 672 workspace->out_buf.dst = kaddr; 673 workspace->out_buf.size = dstlen; 674 } else { 675 workspace->out_buf.dst = workspace->buf; 676 workspace->out_buf.size = min_t(u32, dstlen, 677 fs_info->sectorsize); 678 } 679 workspace->out_buf.pos = 0; 680 681 ret2 = zstd_decompress_stream(stream, &workspace->out_buf, 682 &workspace->in_buf); 683 if (kaddr) 684 kunmap_local(kaddr); 685 if (unlikely(zstd_is_error(ret2))) { 686 struct btrfs_inode *inode = cb->bbio.inode; 687 688 btrfs_err(inode->root->fs_info, 689 "zstd decompression failed, error %d root %llu inode %llu offset %llu", 690 zstd_get_error_code(ret2), btrfs_root_id(inode->root), 691 btrfs_ino(inode), cb->start); 692 ret = -EIO; 693 goto done; 694 } 695 total_out += workspace->out_buf.pos; 696 if (kaddr) 697 bio_advance(orig_bio, workspace->out_buf.pos); 698 699 if (workspace->in_buf.pos >= srclen) 700 break; 701 702 /* Check if we've hit the end of a frame */ 703 if (ret2 == 0) 704 break; 705 706 if (workspace->in_buf.pos == workspace->in_buf.size) { 707 kunmap_local(workspace->in_buf.src); 708 folio_in_index++; 709 if (unlikely(folio_in_index >= total_folios_in)) { 710 workspace->in_buf.src = NULL; 711 ret = -EIO; 712 goto done; 713 } 714 srclen -= min_folio_size; 715 bio_next_folio(&fi, &cb->bbio.bio); 716 ASSERT(fi.folio); 717 workspace->in_buf.src = kmap_local_folio(fi.folio, 0); 718 workspace->in_buf.pos = 0; 719 workspace->in_buf.size = min_t(size_t, srclen, min_folio_size); 720 } 721 } 722 ret = 0; 723 done: 724 if (workspace->in_buf.src) 725 kunmap_local(workspace->in_buf.src); 726 return ret; 727 } 728 729 int zstd_decompress(struct list_head *ws, const u8 *data_in, 730 struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen, 731 size_t destlen) 732 { 733 struct workspace *workspace = list_entry(ws, struct workspace, list); 734 struct btrfs_fs_info *fs_info = btrfs_sb(folio_inode(dest_folio)->i_sb); 735 zstd_dstream *stream; 736 int ret = 0; 737 unsigned long to_copy = 0; 738 739 stream = zstd_init_dstream( 740 ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size); 741 if (unlikely(!stream)) { 742 struct btrfs_inode *inode = folio_to_inode(dest_folio); 743 744 btrfs_err(inode->root->fs_info, 745 "zstd decompression init failed, root %llu inode %llu offset %llu", 746 btrfs_root_id(inode->root), btrfs_ino(inode), 747 folio_pos(dest_folio)); 748 ret = -EIO; 749 goto finish; 750 } 751 752 workspace->in_buf.src = data_in; 753 workspace->in_buf.pos = 0; 754 workspace->in_buf.size = srclen; 755 756 workspace->out_buf.dst = workspace->buf; 757 workspace->out_buf.pos = 0; 758 workspace->out_buf.size = fs_info->sectorsize; 759 760 /* 761 * Since both input and output buffers should not exceed one sector, 762 * one call should end the decompression. 763 */ 764 ret = zstd_decompress_stream(stream, &workspace->out_buf, &workspace->in_buf); 765 if (unlikely(zstd_is_error(ret))) { 766 struct btrfs_inode *inode = folio_to_inode(dest_folio); 767 768 btrfs_err(inode->root->fs_info, 769 "zstd decompression failed, error %d root %llu inode %llu offset %llu", 770 zstd_get_error_code(ret), btrfs_root_id(inode->root), 771 btrfs_ino(inode), folio_pos(dest_folio)); 772 goto finish; 773 } 774 to_copy = workspace->out_buf.pos; 775 memcpy_to_folio(dest_folio, dest_pgoff, workspace->out_buf.dst, to_copy); 776 finish: 777 /* Error or early end. */ 778 if (unlikely(to_copy < destlen)) { 779 ret = -EIO; 780 folio_zero_range(dest_folio, dest_pgoff + to_copy, destlen - to_copy); 781 } 782 return ret; 783 } 784 785 const struct btrfs_compress_levels btrfs_zstd_compress = { 786 .min_level = ZSTD_BTRFS_MIN_LEVEL, 787 .max_level = ZSTD_BTRFS_MAX_LEVEL, 788 .default_level = ZSTD_BTRFS_DEFAULT_LEVEL, 789 }; 790