1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2016-present, Facebook, Inc.
4 * All rights reserved.
5 *
6 */
7
8 #include <linux/bio.h>
9 #include <linux/bitmap.h>
10 #include <linux/err.h>
11 #include <linux/init.h>
12 #include <linux/kernel.h>
13 #include <linux/mm.h>
14 #include <linux/sched/mm.h>
15 #include <linux/pagemap.h>
16 #include <linux/refcount.h>
17 #include <linux/sched.h>
18 #include <linux/slab.h>
19 #include <linux/zstd.h>
20 #include "misc.h"
21 #include "fs.h"
22 #include "btrfs_inode.h"
23 #include "compression.h"
24 #include "super.h"
25
26 #define ZSTD_BTRFS_MAX_WINDOWLOG 17
27 #define ZSTD_BTRFS_MAX_INPUT (1U << ZSTD_BTRFS_MAX_WINDOWLOG)
28 #define ZSTD_BTRFS_DEFAULT_LEVEL 3
29 #define ZSTD_BTRFS_MIN_LEVEL -15
30 #define ZSTD_BTRFS_MAX_LEVEL 15
31 /* 307s to avoid pathologically clashing with transaction commit */
32 #define ZSTD_BTRFS_RECLAIM_JIFFIES (307 * HZ)
33
zstd_get_btrfs_parameters(int level,size_t src_len)34 static zstd_parameters zstd_get_btrfs_parameters(int level,
35 size_t src_len)
36 {
37 zstd_parameters params = zstd_get_params(level, src_len);
38
39 if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
40 params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
41 WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT);
42 return params;
43 }
44
45 struct workspace {
46 void *mem;
47 size_t size;
48 char *buf;
49 int level;
50 int req_level;
51 unsigned long last_used; /* jiffies */
52 struct list_head list;
53 struct list_head lru_list;
54 zstd_in_buffer in_buf;
55 zstd_out_buffer out_buf;
56 zstd_parameters params;
57 };
58
59 /*
60 * Zstd Workspace Management
61 *
62 * Zstd workspaces have different memory requirements depending on the level.
63 * The zstd workspaces are managed by having individual lists for each level
64 * and a global lru. Forward progress is maintained by protecting a max level
65 * workspace.
66 *
67 * Getting a workspace is done by using the bitmap to identify the levels that
68 * have available workspaces and scans up. This lets us recycle higher level
69 * workspaces because of the monotonic memory guarantee. A workspace's
70 * last_used is only updated if it is being used by the corresponding memory
71 * level. Putting a workspace involves adding it back to the appropriate places
72 * and adding it back to the lru if necessary.
73 *
74 * A timer is used to reclaim workspaces if they have not been used for
75 * ZSTD_BTRFS_RECLAIM_JIFFIES. This helps keep only active workspaces around.
76 * The upper bound is provided by the workqueue limit which is 2 (percpu limit).
77 */
78
79 struct zstd_workspace_manager {
80 spinlock_t lock;
81 struct list_head lru_list;
82 struct list_head idle_ws[ZSTD_BTRFS_MAX_LEVEL];
83 unsigned long active_map;
84 wait_queue_head_t wait;
85 struct timer_list timer;
86 };
87
88 static size_t zstd_ws_mem_sizes[ZSTD_BTRFS_MAX_LEVEL];
89
list_to_workspace(struct list_head * list)90 static inline struct workspace *list_to_workspace(struct list_head *list)
91 {
92 return container_of(list, struct workspace, list);
93 }
94
clip_level(int level)95 static inline int clip_level(int level)
96 {
97 return max(0, level - 1);
98 }
99
100 /*
101 * Timer callback to free unused workspaces.
102 *
103 * @t: timer
104 *
105 * This scans the lru_list and attempts to reclaim any workspace that hasn't
106 * been used for ZSTD_BTRFS_RECLAIM_JIFFIES.
107 *
108 * The context is softirq and does not need the _bh locking primitives.
109 */
zstd_reclaim_timer_fn(struct timer_list * timer)110 static void zstd_reclaim_timer_fn(struct timer_list *timer)
111 {
112 struct zstd_workspace_manager *zwsm =
113 container_of(timer, struct zstd_workspace_manager, timer);
114 unsigned long reclaim_threshold = jiffies - ZSTD_BTRFS_RECLAIM_JIFFIES;
115 struct list_head *pos, *next;
116
117 spin_lock(&zwsm->lock);
118
119 if (list_empty(&zwsm->lru_list)) {
120 spin_unlock(&zwsm->lock);
121 return;
122 }
123
124 list_for_each_prev_safe(pos, next, &zwsm->lru_list) {
125 struct workspace *victim = container_of(pos, struct workspace,
126 lru_list);
127 int level;
128
129 if (time_after(victim->last_used, reclaim_threshold))
130 break;
131
132 /* workspace is in use */
133 if (victim->req_level)
134 continue;
135
136 level = victim->level;
137 list_del(&victim->lru_list);
138 list_del(&victim->list);
139 zstd_free_workspace(&victim->list);
140
141 if (list_empty(&zwsm->idle_ws[level]))
142 clear_bit(level, &zwsm->active_map);
143
144 }
145
146 if (!list_empty(&zwsm->lru_list))
147 mod_timer(&zwsm->timer, jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES);
148
149 spin_unlock(&zwsm->lock);
150 }
151
152 /*
153 * Calculate monotonic memory bounds.
154 *
155 * It is possible based on the level configurations that a higher level
156 * workspace uses less memory than a lower level workspace. In order to reuse
157 * workspaces, this must be made a monotonic relationship. This precomputes
158 * the required memory for each level and enforces the monotonicity between
159 * level and memory required.
160 */
zstd_calc_ws_mem_sizes(void)161 static void zstd_calc_ws_mem_sizes(void)
162 {
163 size_t max_size = 0;
164 int level;
165
166 for (level = ZSTD_BTRFS_MIN_LEVEL; level <= ZSTD_BTRFS_MAX_LEVEL; level++) {
167 if (level == 0)
168 continue;
169 zstd_parameters params =
170 zstd_get_btrfs_parameters(level, ZSTD_BTRFS_MAX_INPUT);
171 size_t level_size =
172 max_t(size_t,
173 zstd_cstream_workspace_bound(¶ms.cParams),
174 zstd_dstream_workspace_bound(ZSTD_BTRFS_MAX_INPUT));
175
176 max_size = max_t(size_t, max_size, level_size);
177 /* Use level 1 workspace size for all the fast mode negative levels. */
178 zstd_ws_mem_sizes[clip_level(level)] = max_size;
179 }
180 }
181
zstd_alloc_workspace_manager(struct btrfs_fs_info * fs_info)182 int zstd_alloc_workspace_manager(struct btrfs_fs_info *fs_info)
183 {
184 struct zstd_workspace_manager *zwsm;
185 struct list_head *ws;
186
187 ASSERT(fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] == NULL);
188 zwsm = kzalloc_obj(*zwsm);
189 if (!zwsm)
190 return -ENOMEM;
191 zstd_calc_ws_mem_sizes();
192 spin_lock_init(&zwsm->lock);
193 init_waitqueue_head(&zwsm->wait);
194 timer_setup(&zwsm->timer, zstd_reclaim_timer_fn, 0);
195
196 INIT_LIST_HEAD(&zwsm->lru_list);
197 for (int i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++)
198 INIT_LIST_HEAD(&zwsm->idle_ws[i]);
199 fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] = zwsm;
200
201 ws = zstd_alloc_workspace(fs_info, ZSTD_BTRFS_MAX_LEVEL);
202 if (IS_ERR(ws)) {
203 btrfs_warn(NULL, "cannot preallocate zstd compression workspace");
204 } else {
205 set_bit(ZSTD_BTRFS_MAX_LEVEL - 1, &zwsm->active_map);
206 list_add(ws, &zwsm->idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1]);
207 }
208 return 0;
209 }
210
zstd_free_workspace_manager(struct btrfs_fs_info * fs_info)211 void zstd_free_workspace_manager(struct btrfs_fs_info *fs_info)
212 {
213 struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD];
214 struct workspace *workspace;
215
216 if (!zwsm)
217 return;
218 fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] = NULL;
219 spin_lock_bh(&zwsm->lock);
220 for (int i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) {
221 while (!list_empty(&zwsm->idle_ws[i])) {
222 workspace = container_of(zwsm->idle_ws[i].next,
223 struct workspace, list);
224 list_del(&workspace->list);
225 list_del(&workspace->lru_list);
226 zstd_free_workspace(&workspace->list);
227 }
228 }
229 spin_unlock_bh(&zwsm->lock);
230 timer_delete_sync(&zwsm->timer);
231 kfree(zwsm);
232 }
233
234 /*
235 * Find workspace for given level.
236 *
237 * @level: compression level
238 *
239 * This iterates over the set bits in the active_map beginning at the requested
240 * compression level. This lets us utilize already allocated workspaces before
241 * allocating a new one. If the workspace is of a larger size, it is used, but
242 * the place in the lru_list and last_used times are not updated. This is to
243 * offer the opportunity to reclaim the workspace in favor of allocating an
244 * appropriately sized one in the future.
245 */
zstd_find_workspace(struct btrfs_fs_info * fs_info,int level)246 static struct list_head *zstd_find_workspace(struct btrfs_fs_info *fs_info, int level)
247 {
248 struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD];
249 struct list_head *ws;
250 struct workspace *workspace;
251 int i = clip_level(level);
252
253 ASSERT(zwsm);
254 spin_lock_bh(&zwsm->lock);
255 for_each_set_bit_from(i, &zwsm->active_map, ZSTD_BTRFS_MAX_LEVEL) {
256 if (!list_empty(&zwsm->idle_ws[i])) {
257 ws = zwsm->idle_ws[i].next;
258 workspace = list_to_workspace(ws);
259 list_del_init(ws);
260 /* keep its place if it's a lower level using this */
261 workspace->req_level = level;
262 if (clip_level(level) == workspace->level)
263 list_del(&workspace->lru_list);
264 if (list_empty(&zwsm->idle_ws[i]))
265 clear_bit(i, &zwsm->active_map);
266 spin_unlock_bh(&zwsm->lock);
267 return ws;
268 }
269 }
270 spin_unlock_bh(&zwsm->lock);
271
272 return NULL;
273 }
274
275 /*
276 * Zstd get_workspace for level.
277 *
278 * @level: compression level
279 *
280 * If @level is 0, then any compression level can be used. Therefore, we begin
281 * scanning from 1. We first scan through possible workspaces and then after
282 * attempt to allocate a new workspace. If we fail to allocate one due to
283 * memory pressure, go to sleep waiting for the max level workspace to free up.
284 */
zstd_get_workspace(struct btrfs_fs_info * fs_info,int level)285 struct list_head *zstd_get_workspace(struct btrfs_fs_info *fs_info, int level)
286 {
287 struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD];
288 struct list_head *ws;
289 unsigned int nofs_flag;
290
291 ASSERT(zwsm);
292
293 /* level == 0 means we can use any workspace */
294 if (!level)
295 level = 1;
296
297 again:
298 ws = zstd_find_workspace(fs_info, level);
299 if (ws)
300 return ws;
301
302 nofs_flag = memalloc_nofs_save();
303 ws = zstd_alloc_workspace(fs_info, level);
304 memalloc_nofs_restore(nofs_flag);
305
306 if (IS_ERR(ws)) {
307 DEFINE_WAIT(wait);
308
309 prepare_to_wait(&zwsm->wait, &wait, TASK_UNINTERRUPTIBLE);
310 /*
311 * Re-check after being queued: zstd_put_workspace() only wakes
312 * a queue that already has a sleeper, so a workspace returned
313 * since the failed allocation woke nobody.
314 */
315 ws = zstd_find_workspace(fs_info, level);
316 if (!ws)
317 schedule();
318 finish_wait(&zwsm->wait, &wait);
319 if (ws)
320 return ws;
321
322 goto again;
323 }
324
325 return ws;
326 }
327
328 /*
329 * Zstd put_workspace.
330 *
331 * @ws: list_head for the workspace
332 *
333 * When putting back a workspace, we only need to update the LRU if we are of
334 * the requested compression level. Here is where we continue to protect the
335 * max level workspace or update last_used accordingly. If the reclaim timer
336 * isn't set, it is also set here. Only the max level workspace tries and wakes
337 * up waiting workspaces.
338 */
zstd_put_workspace(struct btrfs_fs_info * fs_info,struct list_head * ws)339 void zstd_put_workspace(struct btrfs_fs_info *fs_info, struct list_head *ws)
340 {
341 struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD];
342 struct workspace *workspace = list_to_workspace(ws);
343
344 ASSERT(zwsm);
345 spin_lock_bh(&zwsm->lock);
346
347 /* A node is only taken off the lru if we are the corresponding level */
348 if (clip_level(workspace->req_level) == workspace->level) {
349 /* Hide a max level workspace from reclaim */
350 if (list_empty(&zwsm->idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1])) {
351 INIT_LIST_HEAD(&workspace->lru_list);
352 } else {
353 workspace->last_used = jiffies;
354 list_add(&workspace->lru_list, &zwsm->lru_list);
355 if (!timer_pending(&zwsm->timer))
356 mod_timer(&zwsm->timer,
357 jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES);
358 }
359 }
360
361 set_bit(workspace->level, &zwsm->active_map);
362 list_add(&workspace->list, &zwsm->idle_ws[workspace->level]);
363 workspace->req_level = 0;
364
365 spin_unlock_bh(&zwsm->lock);
366
367 if (workspace->level == clip_level(ZSTD_BTRFS_MAX_LEVEL))
368 cond_wake_up(&zwsm->wait);
369 }
370
zstd_free_workspace(struct list_head * ws)371 void zstd_free_workspace(struct list_head *ws)
372 {
373 struct workspace *workspace = list_entry(ws, struct workspace, list);
374
375 kvfree(workspace->mem);
376 kfree(workspace->buf);
377 kfree(workspace);
378 }
379
zstd_alloc_workspace(struct btrfs_fs_info * fs_info,int level)380 struct list_head *zstd_alloc_workspace(struct btrfs_fs_info *fs_info, int level)
381 {
382 struct workspace *workspace;
383
384 workspace = kzalloc_obj(*workspace);
385 if (!workspace)
386 return ERR_PTR(-ENOMEM);
387
388 /* Use level 1 workspace size for all the fast mode negative levels. */
389 workspace->size = zstd_ws_mem_sizes[clip_level(level)];
390 workspace->level = clip_level(level);
391 workspace->req_level = level;
392 workspace->last_used = jiffies;
393 workspace->mem = kvmalloc(workspace->size, GFP_KERNEL | __GFP_NOWARN);
394 workspace->buf = kmalloc(fs_info->sectorsize, GFP_KERNEL);
395 if (!workspace->mem || !workspace->buf)
396 goto fail;
397
398 INIT_LIST_HEAD(&workspace->list);
399 INIT_LIST_HEAD(&workspace->lru_list);
400
401 return &workspace->list;
402 fail:
403 zstd_free_workspace(&workspace->list);
404 return ERR_PTR(-ENOMEM);
405 }
406
zstd_compress_bio(struct list_head * ws,struct compressed_bio * cb)407 int zstd_compress_bio(struct list_head *ws, struct compressed_bio *cb)
408 {
409 struct btrfs_inode *inode = cb->bbio.inode;
410 struct btrfs_fs_info *fs_info = inode->root->fs_info;
411 struct workspace *workspace = list_entry(ws, struct workspace, list);
412 struct address_space *mapping = inode->vfs_inode.i_mapping;
413 struct bio *bio = &cb->bbio.bio;
414 zstd_cstream *stream;
415 int ret = 0;
416 /* The current folio to read. */
417 struct folio *in_folio = NULL;
418 /* The current folio to write to. */
419 struct folio *out_folio = NULL;
420 unsigned long tot_in = 0;
421 unsigned long tot_out = 0;
422 const u64 start = cb->start;
423 const u32 len = cb->len;
424 const u64 end = start + len;
425 const u32 min_folio_size = btrfs_min_folio_size(fs_info);
426
427 workspace->params = zstd_get_btrfs_parameters(workspace->req_level, len);
428
429 /* Initialize the stream. */
430 stream = zstd_init_cstream(&workspace->params, len, workspace->mem, workspace->size);
431 if (unlikely(!stream)) {
432 btrfs_err(fs_info,
433 "zstd compression init level %d failed, root %llu inode %llu offset %llu",
434 workspace->req_level, btrfs_root_id(inode->root),
435 btrfs_ino(inode), start);
436 ret = -EIO;
437 goto out;
438 }
439
440 /* Map in the first page of input data. */
441 ret = btrfs_compress_filemap_get_folio(mapping, start, &in_folio);
442 if (ret < 0)
443 goto out;
444 workspace->in_buf.src = kmap_local_folio(in_folio, offset_in_folio(in_folio, start));
445 workspace->in_buf.pos = 0;
446 workspace->in_buf.size = btrfs_calc_input_length(in_folio, end, start);
447
448 /* Allocate and map in the output buffer. */
449 out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS);
450 if (out_folio == NULL) {
451 ret = -ENOMEM;
452 goto out;
453 }
454 workspace->out_buf.dst = folio_address(out_folio);
455 workspace->out_buf.pos = 0;
456 workspace->out_buf.size = min_folio_size;
457
458 while (1) {
459 size_t ret2;
460
461 ret2 = zstd_compress_stream(stream, &workspace->out_buf, &workspace->in_buf);
462 if (unlikely(zstd_is_error(ret2))) {
463 btrfs_warn(fs_info,
464 "zstd compression level %d failed, error %d root %llu inode %llu offset %llu",
465 workspace->req_level, zstd_get_error_code(ret2),
466 btrfs_root_id(inode->root), btrfs_ino(inode),
467 start + tot_in);
468 ret = -EIO;
469 goto out;
470 }
471
472 /* Check to see if we are making it bigger. */
473 if (tot_in + workspace->in_buf.pos > fs_info->sectorsize * 2 &&
474 tot_in + workspace->in_buf.pos < tot_out + workspace->out_buf.pos) {
475 ret = -E2BIG;
476 goto out;
477 }
478
479 /* Check if we need more output space. */
480 if (workspace->out_buf.pos >= workspace->out_buf.size) {
481 tot_out += min_folio_size;
482 if (tot_out >= len) {
483 ret = -E2BIG;
484 goto out;
485 }
486 /* Queue the current foliot into the bio. */
487 if (!bio_add_folio(bio, out_folio, folio_size(out_folio), 0)) {
488 ret = -E2BIG;
489 goto out;
490 }
491
492 out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS);
493 if (out_folio == NULL) {
494 ret = -ENOMEM;
495 goto out;
496 }
497 workspace->out_buf.dst = folio_address(out_folio);
498 workspace->out_buf.pos = 0;
499 workspace->out_buf.size = min_folio_size;
500 }
501
502 /* We've reached the end of the input. */
503 if (tot_in + workspace->in_buf.pos >= len) {
504 tot_in += workspace->in_buf.pos;
505 break;
506 }
507
508 /* Check if we need more input. */
509 if (workspace->in_buf.pos >= workspace->in_buf.size) {
510 u64 cur;
511
512 tot_in += workspace->in_buf.size;
513 cur = start + tot_in;
514
515 kunmap_local(workspace->in_buf.src);
516 workspace->in_buf.src = NULL;
517 folio_put(in_folio);
518
519 ret = btrfs_compress_filemap_get_folio(mapping, cur, &in_folio);
520 if (ret < 0)
521 goto out;
522 workspace->in_buf.src = kmap_local_folio(in_folio,
523 offset_in_folio(in_folio, cur));
524 workspace->in_buf.pos = 0;
525 workspace->in_buf.size = btrfs_calc_input_length(in_folio, end, cur);
526 }
527 }
528
529 while (1) {
530 size_t ret2;
531
532 ret2 = zstd_end_stream(stream, &workspace->out_buf);
533 if (unlikely(zstd_is_error(ret2))) {
534 btrfs_err(fs_info,
535 "zstd compression end level %d failed, error %d root %llu inode %llu offset %llu",
536 workspace->req_level, zstd_get_error_code(ret2),
537 btrfs_root_id(inode->root), btrfs_ino(inode),
538 start + tot_in);
539 ret = -EIO;
540 goto out;
541 }
542 /* Queue the remaining part of the output folio into bio. */
543 if (ret2 == 0) {
544 tot_out += workspace->out_buf.pos;
545 if (tot_out >= len) {
546 ret = -E2BIG;
547 goto out;
548 }
549 if (!bio_add_folio(bio, out_folio, workspace->out_buf.pos, 0)) {
550 ret = -E2BIG;
551 goto out;
552 }
553 out_folio = NULL;
554 break;
555 }
556 tot_out += min_folio_size;
557 if (tot_out >= len) {
558 ret = -E2BIG;
559 goto out;
560 }
561 if (!bio_add_folio(bio, out_folio, folio_size(out_folio), 0)) {
562 ret = -E2BIG;
563 goto out;
564 }
565 out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS);
566 if (out_folio == NULL) {
567 ret = -ENOMEM;
568 goto out;
569 }
570 workspace->out_buf.dst = folio_address(out_folio);
571 workspace->out_buf.pos = 0;
572 workspace->out_buf.size = min_folio_size;
573 }
574
575 if (tot_out >= tot_in) {
576 ret = -E2BIG;
577 goto out;
578 }
579
580 ret = 0;
581 ASSERT(tot_out == bio->bi_iter.bi_size);
582 out:
583 if (out_folio)
584 btrfs_free_compr_folio(out_folio);
585 if (workspace->in_buf.src) {
586 kunmap_local(workspace->in_buf.src);
587 folio_put(in_folio);
588 }
589 return ret;
590 }
591
zstd_decompress_bio(struct list_head * ws,struct compressed_bio * cb)592 int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
593 {
594 struct btrfs_fs_info *fs_info = cb_to_fs_info(cb);
595 struct workspace *workspace = list_entry(ws, struct workspace, list);
596 struct folio_iter fi;
597 size_t srclen = bio_get_size(&cb->bbio.bio);
598 zstd_dstream *stream;
599 int ret = 0;
600 const unsigned int min_folio_size = btrfs_min_folio_size(fs_info);
601 unsigned long folio_in_index = 0;
602 unsigned long total_folios_in = DIV_ROUND_UP(srclen, min_folio_size);
603 unsigned long buf_start;
604 unsigned long total_out = 0;
605
606 bio_first_folio(&fi, &cb->bbio.bio, 0);
607 if (unlikely(!fi.folio))
608 return -EINVAL;
609 ASSERT(folio_size(fi.folio) == min_folio_size);
610
611 stream = zstd_init_dstream(
612 ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
613 if (unlikely(!stream)) {
614 struct btrfs_inode *inode = cb->bbio.inode;
615
616 btrfs_err(inode->root->fs_info,
617 "zstd decompression init failed, root %llu inode %llu offset %llu",
618 btrfs_root_id(inode->root), btrfs_ino(inode), cb->start);
619 ret = -EIO;
620 goto done;
621 }
622
623 workspace->in_buf.src = kmap_local_folio(fi.folio, 0);
624 workspace->in_buf.pos = 0;
625 workspace->in_buf.size = min_t(size_t, srclen, min_folio_size);
626
627 workspace->out_buf.dst = workspace->buf;
628 workspace->out_buf.pos = 0;
629 workspace->out_buf.size = fs_info->sectorsize;
630
631 while (1) {
632 size_t ret2;
633
634 ret2 = zstd_decompress_stream(stream, &workspace->out_buf,
635 &workspace->in_buf);
636 if (unlikely(zstd_is_error(ret2))) {
637 struct btrfs_inode *inode = cb->bbio.inode;
638
639 btrfs_err(inode->root->fs_info,
640 "zstd decompression failed, error %d root %llu inode %llu offset %llu",
641 zstd_get_error_code(ret2), btrfs_root_id(inode->root),
642 btrfs_ino(inode), cb->start);
643 ret = -EIO;
644 goto done;
645 }
646 buf_start = total_out;
647 total_out += workspace->out_buf.pos;
648 workspace->out_buf.pos = 0;
649
650 ret = btrfs_decompress_buf2page(workspace->out_buf.dst,
651 total_out - buf_start, cb, buf_start);
652 if (ret == 0)
653 break;
654
655 if (workspace->in_buf.pos >= srclen)
656 break;
657
658 /* Check if we've hit the end of a frame */
659 if (ret2 == 0)
660 break;
661
662 if (workspace->in_buf.pos == workspace->in_buf.size) {
663 kunmap_local(workspace->in_buf.src);
664 folio_in_index++;
665 if (unlikely(folio_in_index >= total_folios_in)) {
666 workspace->in_buf.src = NULL;
667 ret = -EIO;
668 goto done;
669 }
670 srclen -= min_folio_size;
671 bio_next_folio(&fi, &cb->bbio.bio);
672 ASSERT(fi.folio);
673 workspace->in_buf.src = kmap_local_folio(fi.folio, 0);
674 workspace->in_buf.pos = 0;
675 workspace->in_buf.size = min_t(size_t, srclen, min_folio_size);
676 }
677 }
678 ret = 0;
679 done:
680 if (workspace->in_buf.src)
681 kunmap_local(workspace->in_buf.src);
682 return ret;
683 }
684
zstd_decompress(struct list_head * ws,const u8 * data_in,struct folio * dest_folio,unsigned long dest_pgoff,size_t srclen,size_t destlen)685 int zstd_decompress(struct list_head *ws, const u8 *data_in,
686 struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen,
687 size_t destlen)
688 {
689 struct workspace *workspace = list_entry(ws, struct workspace, list);
690 struct btrfs_fs_info *fs_info = btrfs_sb(folio_inode(dest_folio)->i_sb);
691 zstd_dstream *stream;
692 int ret = 0;
693 unsigned long to_copy = 0;
694
695 stream = zstd_init_dstream(
696 ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
697 if (unlikely(!stream)) {
698 struct btrfs_inode *inode = folio_to_inode(dest_folio);
699
700 btrfs_err(inode->root->fs_info,
701 "zstd decompression init failed, root %llu inode %llu offset %llu",
702 btrfs_root_id(inode->root), btrfs_ino(inode),
703 folio_pos(dest_folio));
704 ret = -EIO;
705 goto finish;
706 }
707
708 workspace->in_buf.src = data_in;
709 workspace->in_buf.pos = 0;
710 workspace->in_buf.size = srclen;
711
712 workspace->out_buf.dst = workspace->buf;
713 workspace->out_buf.pos = 0;
714 workspace->out_buf.size = fs_info->sectorsize;
715
716 /*
717 * Since both input and output buffers should not exceed one sector,
718 * one call should end the decompression.
719 */
720 ret = zstd_decompress_stream(stream, &workspace->out_buf, &workspace->in_buf);
721 if (unlikely(zstd_is_error(ret))) {
722 struct btrfs_inode *inode = folio_to_inode(dest_folio);
723
724 btrfs_err(inode->root->fs_info,
725 "zstd decompression failed, error %d root %llu inode %llu offset %llu",
726 zstd_get_error_code(ret), btrfs_root_id(inode->root),
727 btrfs_ino(inode), folio_pos(dest_folio));
728 goto finish;
729 }
730 to_copy = workspace->out_buf.pos;
731 memcpy_to_folio(dest_folio, dest_pgoff, workspace->out_buf.dst, to_copy);
732 finish:
733 /* Error or early end. */
734 if (unlikely(to_copy < destlen)) {
735 ret = -EIO;
736 folio_zero_range(dest_folio, dest_pgoff + to_copy, destlen - to_copy);
737 }
738 return ret;
739 }
740
741 const struct btrfs_compress_levels btrfs_zstd_compress = {
742 .min_level = ZSTD_BTRFS_MIN_LEVEL,
743 .max_level = ZSTD_BTRFS_MAX_LEVEL,
744 .default_level = ZSTD_BTRFS_DEFAULT_LEVEL,
745 };
746