xref: /linux/fs/btrfs/zstd.c (revision bb66a38f5812f2c04abd4eb6e586a12b5a95172a)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (c) 2016-present, Facebook, Inc.
4  * All rights reserved.
5  *
6  */
7 
8 #include <linux/bio.h>
9 #include <linux/bitmap.h>
10 #include <linux/err.h>
11 #include <linux/init.h>
12 #include <linux/kernel.h>
13 #include <linux/mm.h>
14 #include <linux/sched/mm.h>
15 #include <linux/pagemap.h>
16 #include <linux/refcount.h>
17 #include <linux/sched.h>
18 #include <linux/slab.h>
19 #include <linux/zstd.h>
20 #include "misc.h"
21 #include "fs.h"
22 #include "btrfs_inode.h"
23 #include "compression.h"
24 #include "super.h"
25 
26 #define ZSTD_BTRFS_MAX_WINDOWLOG 17
27 #define ZSTD_BTRFS_MAX_INPUT (1U << ZSTD_BTRFS_MAX_WINDOWLOG)
28 #define ZSTD_BTRFS_DEFAULT_LEVEL 3
29 #define ZSTD_BTRFS_MIN_LEVEL -15
30 #define ZSTD_BTRFS_MAX_LEVEL 15
31 /* 307s to avoid pathologically clashing with transaction commit */
32 #define ZSTD_BTRFS_RECLAIM_JIFFIES (307 * HZ)
33 
34 static zstd_parameters zstd_get_btrfs_parameters(int level,
35 						 size_t src_len)
36 {
37 	zstd_parameters params = zstd_get_params(level, src_len);
38 
39 	if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
40 		params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
41 	WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT);
42 	return params;
43 }
44 
45 struct workspace {
46 	void *mem;
47 	size_t size;
48 	char *buf;
49 	int level;
50 	int req_level;
51 	unsigned long last_used; /* jiffies */
52 	struct list_head list;
53 	struct list_head lru_list;
54 	zstd_in_buffer in_buf;
55 	zstd_out_buffer out_buf;
56 	zstd_parameters params;
57 };
58 
59 /*
60  * Zstd Workspace Management
61  *
62  * Zstd workspaces have different memory requirements depending on the level.
63  * The zstd workspaces are managed by having individual lists for each level
64  * and a global lru.  Forward progress is maintained by protecting a max level
65  * workspace.
66  *
67  * Getting a workspace is done by using the bitmap to identify the levels that
68  * have available workspaces and scans up.  This lets us recycle higher level
69  * workspaces because of the monotonic memory guarantee.  A workspace's
70  * last_used is only updated if it is being used by the corresponding memory
71  * level.  Putting a workspace involves adding it back to the appropriate places
72  * and adding it back to the lru if necessary.
73  *
74  * A timer is used to reclaim workspaces if they have not been used for
75  * ZSTD_BTRFS_RECLAIM_JIFFIES.  This helps keep only active workspaces around.
76  * The upper bound is provided by the workqueue limit which is 2 (percpu limit).
77  */
78 
79 struct zstd_workspace_manager {
80 	spinlock_t lock;
81 	struct list_head lru_list;
82 	struct list_head idle_ws[ZSTD_BTRFS_MAX_LEVEL];
83 	unsigned long active_map;
84 	wait_queue_head_t wait;
85 	struct timer_list timer;
86 };
87 
88 static size_t zstd_ws_mem_sizes[ZSTD_BTRFS_MAX_LEVEL];
89 
90 static inline struct workspace *list_to_workspace(struct list_head *list)
91 {
92 	return container_of(list, struct workspace, list);
93 }
94 
95 static inline int clip_level(int level)
96 {
97 	return max(0, level - 1);
98 }
99 
100 /*
101  * Timer callback to free unused workspaces.
102  *
103  * @t: timer
104  *
105  * This scans the lru_list and attempts to reclaim any workspace that hasn't
106  * been used for ZSTD_BTRFS_RECLAIM_JIFFIES.
107  *
108  * The context is softirq and does not need the _bh locking primitives.
109  */
110 static void zstd_reclaim_timer_fn(struct timer_list *timer)
111 {
112 	struct zstd_workspace_manager *zwsm =
113 		container_of(timer, struct zstd_workspace_manager, timer);
114 	unsigned long reclaim_threshold = jiffies - ZSTD_BTRFS_RECLAIM_JIFFIES;
115 	struct list_head *pos, *next;
116 
117 	spin_lock(&zwsm->lock);
118 
119 	if (list_empty(&zwsm->lru_list)) {
120 		spin_unlock(&zwsm->lock);
121 		return;
122 	}
123 
124 	list_for_each_prev_safe(pos, next, &zwsm->lru_list) {
125 		struct workspace *victim = container_of(pos, struct workspace,
126 							lru_list);
127 		int level;
128 
129 		if (time_after(victim->last_used, reclaim_threshold))
130 			break;
131 
132 		/* workspace is in use */
133 		if (victim->req_level)
134 			continue;
135 
136 		level = victim->level;
137 		list_del(&victim->lru_list);
138 		list_del(&victim->list);
139 		zstd_free_workspace(&victim->list);
140 
141 		if (list_empty(&zwsm->idle_ws[level]))
142 			clear_bit(level, &zwsm->active_map);
143 
144 	}
145 
146 	if (!list_empty(&zwsm->lru_list))
147 		mod_timer(&zwsm->timer, jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES);
148 
149 	spin_unlock(&zwsm->lock);
150 }
151 
152 /*
153  * Calculate monotonic memory bounds.
154  *
155  * It is possible based on the level configurations that a higher level
156  * workspace uses less memory than a lower level workspace.  In order to reuse
157  * workspaces, this must be made a monotonic relationship.  This precomputes
158  * the required memory for each level and enforces the monotonicity between
159  * level and memory required.
160  */
161 static void zstd_calc_ws_mem_sizes(void)
162 {
163 	size_t max_size = 0;
164 	int level;
165 
166 	for (level = ZSTD_BTRFS_MIN_LEVEL; level <= ZSTD_BTRFS_MAX_LEVEL; level++) {
167 		if (level == 0)
168 			continue;
169 		zstd_parameters params =
170 			zstd_get_btrfs_parameters(level, ZSTD_BTRFS_MAX_INPUT);
171 		size_t level_size =
172 			max_t(size_t,
173 			      zstd_cstream_workspace_bound(&params.cParams),
174 			      zstd_dstream_workspace_bound(ZSTD_BTRFS_MAX_INPUT));
175 
176 		max_size = max_t(size_t, max_size, level_size);
177 		/* Use level 1 workspace size for all the fast mode negative levels. */
178 		zstd_ws_mem_sizes[clip_level(level)] = max_size;
179 	}
180 }
181 
182 int zstd_alloc_workspace_manager(struct btrfs_fs_info *fs_info)
183 {
184 	struct zstd_workspace_manager *zwsm;
185 	struct list_head *ws;
186 
187 	ASSERT(fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] == NULL);
188 	zwsm = kzalloc_obj(*zwsm);
189 	if (!zwsm)
190 		return -ENOMEM;
191 	zstd_calc_ws_mem_sizes();
192 	spin_lock_init(&zwsm->lock);
193 	init_waitqueue_head(&zwsm->wait);
194 	timer_setup(&zwsm->timer, zstd_reclaim_timer_fn, 0);
195 
196 	INIT_LIST_HEAD(&zwsm->lru_list);
197 	for (int i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++)
198 		INIT_LIST_HEAD(&zwsm->idle_ws[i]);
199 	fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] = zwsm;
200 
201 	ws = zstd_alloc_workspace(fs_info, ZSTD_BTRFS_MAX_LEVEL);
202 	if (IS_ERR(ws)) {
203 		btrfs_warn(NULL, "cannot preallocate zstd compression workspace");
204 	} else {
205 		set_bit(ZSTD_BTRFS_MAX_LEVEL - 1, &zwsm->active_map);
206 		list_add(ws, &zwsm->idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1]);
207 	}
208 	return 0;
209 }
210 
211 void zstd_free_workspace_manager(struct btrfs_fs_info *fs_info)
212 {
213 	struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD];
214 	struct workspace *workspace;
215 
216 	if (!zwsm)
217 		return;
218 	fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD] = NULL;
219 	spin_lock_bh(&zwsm->lock);
220 	for (int i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) {
221 		while (!list_empty(&zwsm->idle_ws[i])) {
222 			workspace = container_of(zwsm->idle_ws[i].next,
223 						 struct workspace, list);
224 			list_del(&workspace->list);
225 			list_del(&workspace->lru_list);
226 			zstd_free_workspace(&workspace->list);
227 		}
228 	}
229 	spin_unlock_bh(&zwsm->lock);
230 	timer_delete_sync(&zwsm->timer);
231 	kfree(zwsm);
232 }
233 
234 /*
235  * Find workspace for given level.
236  *
237  * @level: compression level
238  *
239  * This iterates over the set bits in the active_map beginning at the requested
240  * compression level.  This lets us utilize already allocated workspaces before
241  * allocating a new one.  If the workspace is of a larger size, it is used, but
242  * the place in the lru_list and last_used times are not updated.  This is to
243  * offer the opportunity to reclaim the workspace in favor of allocating an
244  * appropriately sized one in the future.
245  */
246 static struct list_head *zstd_find_workspace(struct btrfs_fs_info *fs_info, int level)
247 {
248 	struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD];
249 	struct list_head *ws;
250 	struct workspace *workspace;
251 	int i = clip_level(level);
252 
253 	ASSERT(zwsm);
254 	spin_lock_bh(&zwsm->lock);
255 	for_each_set_bit_from(i, &zwsm->active_map, ZSTD_BTRFS_MAX_LEVEL) {
256 		if (!list_empty(&zwsm->idle_ws[i])) {
257 			ws = zwsm->idle_ws[i].next;
258 			workspace = list_to_workspace(ws);
259 			list_del_init(ws);
260 			/* keep its place if it's a lower level using this */
261 			workspace->req_level = level;
262 			if (clip_level(level) == workspace->level)
263 				list_del(&workspace->lru_list);
264 			if (list_empty(&zwsm->idle_ws[i]))
265 				clear_bit(i, &zwsm->active_map);
266 			spin_unlock_bh(&zwsm->lock);
267 			return ws;
268 		}
269 	}
270 	spin_unlock_bh(&zwsm->lock);
271 
272 	return NULL;
273 }
274 
275 /*
276  * Zstd get_workspace for level.
277  *
278  * @level: compression level
279  *
280  * If @level is 0, then any compression level can be used.  Therefore, we begin
281  * scanning from 1.  We first scan through possible workspaces and then after
282  * attempt to allocate a new workspace.  If we fail to allocate one due to
283  * memory pressure, go to sleep waiting for the max level workspace to free up.
284  */
285 struct list_head *zstd_get_workspace(struct btrfs_fs_info *fs_info, int level)
286 {
287 	struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD];
288 	struct list_head *ws;
289 	unsigned int nofs_flag;
290 
291 	ASSERT(zwsm);
292 
293 	/* level == 0 means we can use any workspace */
294 	if (!level)
295 		level = 1;
296 
297 again:
298 	ws = zstd_find_workspace(fs_info, level);
299 	if (ws)
300 		return ws;
301 
302 	nofs_flag = memalloc_nofs_save();
303 	ws = zstd_alloc_workspace(fs_info, level);
304 	memalloc_nofs_restore(nofs_flag);
305 
306 	if (IS_ERR(ws)) {
307 		DEFINE_WAIT(wait);
308 
309 		prepare_to_wait(&zwsm->wait, &wait, TASK_UNINTERRUPTIBLE);
310 		/*
311 		 * Re-check after being queued: zstd_put_workspace() only wakes
312 		 * a queue that already has a sleeper, so a workspace returned
313 		 * since the failed allocation woke nobody.
314 		 */
315 		ws = zstd_find_workspace(fs_info, level);
316 		if (!ws)
317 			schedule();
318 		finish_wait(&zwsm->wait, &wait);
319 		if (ws)
320 			return ws;
321 
322 		goto again;
323 	}
324 
325 	return ws;
326 }
327 
328 /*
329  * Zstd put_workspace.
330  *
331  * @ws: list_head for the workspace
332  *
333  * When putting back a workspace, we only need to update the LRU if we are of
334  * the requested compression level.  Here is where we continue to protect the
335  * max level workspace or update last_used accordingly.  If the reclaim timer
336  * isn't set, it is also set here.  Only the max level workspace tries and wakes
337  * up waiting workspaces.
338  */
339 void zstd_put_workspace(struct btrfs_fs_info *fs_info, struct list_head *ws)
340 {
341 	struct zstd_workspace_manager *zwsm = fs_info->compr_wsm[BTRFS_COMPRESS_ZSTD];
342 	struct workspace *workspace = list_to_workspace(ws);
343 
344 	ASSERT(zwsm);
345 	spin_lock_bh(&zwsm->lock);
346 
347 	/* A node is only taken off the lru if we are the corresponding level */
348 	if (clip_level(workspace->req_level) == workspace->level) {
349 		/* Hide a max level workspace from reclaim */
350 		if (workspace->level == ZSTD_BTRFS_MAX_LEVEL - 1 &&
351 		    list_empty(&zwsm->idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1])) {
352 			INIT_LIST_HEAD(&workspace->lru_list);
353 		} else {
354 			workspace->last_used = jiffies;
355 			list_add(&workspace->lru_list, &zwsm->lru_list);
356 			if (!timer_pending(&zwsm->timer))
357 				mod_timer(&zwsm->timer,
358 					  jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES);
359 		}
360 	}
361 
362 	set_bit(workspace->level, &zwsm->active_map);
363 	list_add(&workspace->list, &zwsm->idle_ws[workspace->level]);
364 	workspace->req_level = 0;
365 
366 	spin_unlock_bh(&zwsm->lock);
367 
368 	if (workspace->level == clip_level(ZSTD_BTRFS_MAX_LEVEL))
369 		cond_wake_up(&zwsm->wait);
370 }
371 
372 void zstd_free_workspace(struct list_head *ws)
373 {
374 	struct workspace *workspace = list_entry(ws, struct workspace, list);
375 
376 	kvfree(workspace->mem);
377 	kvfree(workspace->buf);
378 	kfree(workspace);
379 }
380 
381 struct list_head *zstd_alloc_workspace(struct btrfs_fs_info *fs_info, int level)
382 {
383 	struct workspace *workspace;
384 
385 	workspace = kzalloc_obj(*workspace);
386 	if (!workspace)
387 		return ERR_PTR(-ENOMEM);
388 
389 	/* Use level 1 workspace size for all the fast mode negative levels. */
390 	workspace->size = zstd_ws_mem_sizes[clip_level(level)];
391 	workspace->level = clip_level(level);
392 	workspace->req_level = level;
393 	workspace->last_used = jiffies;
394 	workspace->mem = kvmalloc(workspace->size, GFP_KERNEL | __GFP_NOWARN);
395 	workspace->buf = kvmalloc(fs_info->sectorsize, GFP_KERNEL);
396 	if (!workspace->mem || !workspace->buf)
397 		goto fail;
398 
399 	INIT_LIST_HEAD(&workspace->list);
400 	INIT_LIST_HEAD(&workspace->lru_list);
401 
402 	return &workspace->list;
403 fail:
404 	zstd_free_workspace(&workspace->list);
405 	return ERR_PTR(-ENOMEM);
406 }
407 
408 int zstd_compress_bio(struct list_head *ws, struct compressed_bio *cb)
409 {
410 	struct btrfs_inode *inode = cb->bbio.inode;
411 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
412 	struct workspace *workspace = list_entry(ws, struct workspace, list);
413 	struct address_space *mapping = inode->vfs_inode.i_mapping;
414 	struct bio *bio = &cb->bbio.bio;
415 	zstd_cstream *stream;
416 	int ret = 0;
417 	/* The current folio to read. */
418 	struct folio *in_folio = NULL;
419 	/* The current folio to write to. */
420 	struct folio *out_folio = NULL;
421 	unsigned long tot_in = 0;
422 	unsigned long tot_out = 0;
423 	const u64 start = cb->start;
424 	const u32 len = cb->len;
425 	const u64 end = start + len;
426 	const u32 min_folio_size = btrfs_min_folio_size(fs_info);
427 
428 	workspace->params = zstd_get_btrfs_parameters(workspace->req_level, len);
429 
430 	/* Initialize the stream. */
431 	stream = zstd_init_cstream(&workspace->params, len, workspace->mem, workspace->size);
432 	if (unlikely(!stream)) {
433 		btrfs_err(fs_info,
434 	"zstd compression init level %d failed, root %llu inode %llu offset %llu",
435 			  workspace->req_level, btrfs_root_id(inode->root),
436 			  btrfs_ino(inode), start);
437 		ret = -EIO;
438 		goto out;
439 	}
440 
441 	/* Map in the first page of input data. */
442 	ret = btrfs_compress_filemap_get_folio(mapping, start, &in_folio);
443 	if (ret < 0)
444 		goto out;
445 	workspace->in_buf.src = kmap_local_folio(in_folio, offset_in_folio(in_folio, start));
446 	workspace->in_buf.pos = 0;
447 	workspace->in_buf.size = btrfs_calc_input_length(in_folio, end, start);
448 
449 	/* Allocate and map in the output buffer. */
450 	out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS);
451 	if (out_folio == NULL) {
452 		ret = -ENOMEM;
453 		goto out;
454 	}
455 	workspace->out_buf.dst = folio_address(out_folio);
456 	workspace->out_buf.pos = 0;
457 	workspace->out_buf.size = min_folio_size;
458 
459 	while (1) {
460 		size_t ret2;
461 
462 		ret2 = zstd_compress_stream(stream, &workspace->out_buf, &workspace->in_buf);
463 		if (unlikely(zstd_is_error(ret2))) {
464 			btrfs_warn(fs_info,
465 "zstd compression level %d failed, error %d root %llu inode %llu offset %llu",
466 				   workspace->req_level, zstd_get_error_code(ret2),
467 				   btrfs_root_id(inode->root), btrfs_ino(inode),
468 				   start + tot_in);
469 			ret = -EIO;
470 			goto out;
471 		}
472 
473 		/* Check to see if we are making it bigger. */
474 		if (tot_in + workspace->in_buf.pos > fs_info->sectorsize * 2 &&
475 		    tot_in + workspace->in_buf.pos < tot_out + workspace->out_buf.pos) {
476 			ret = -E2BIG;
477 			goto out;
478 		}
479 
480 		/* Check if we need more output space. */
481 		if (workspace->out_buf.pos >= workspace->out_buf.size) {
482 			tot_out += min_folio_size;
483 			if (tot_out >= len) {
484 				ret = -E2BIG;
485 				goto out;
486 			}
487 			/* Queue the current foliot into the bio. */
488 			if (!bio_add_folio(bio, out_folio, folio_size(out_folio), 0)) {
489 				ret = -E2BIG;
490 				goto out;
491 			}
492 
493 			out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS);
494 			if (out_folio == NULL) {
495 				ret = -ENOMEM;
496 				goto out;
497 			}
498 			workspace->out_buf.dst = folio_address(out_folio);
499 			workspace->out_buf.pos = 0;
500 			workspace->out_buf.size = min_folio_size;
501 		}
502 
503 		/* We've reached the end of the input. */
504 		if (tot_in + workspace->in_buf.pos >= len) {
505 			tot_in += workspace->in_buf.pos;
506 			break;
507 		}
508 
509 		/* Check if we need more input. */
510 		if (workspace->in_buf.pos >= workspace->in_buf.size) {
511 			u64 cur;
512 
513 			tot_in += workspace->in_buf.size;
514 			cur = start + tot_in;
515 
516 			kunmap_local(workspace->in_buf.src);
517 			workspace->in_buf.src = NULL;
518 			folio_put(in_folio);
519 
520 			ret = btrfs_compress_filemap_get_folio(mapping, cur, &in_folio);
521 			if (ret < 0)
522 				goto out;
523 			workspace->in_buf.src = kmap_local_folio(in_folio,
524 							 offset_in_folio(in_folio, cur));
525 			workspace->in_buf.pos = 0;
526 			workspace->in_buf.size = btrfs_calc_input_length(in_folio, end, cur);
527 		}
528 	}
529 
530 	while (1) {
531 		size_t ret2;
532 
533 		ret2 = zstd_end_stream(stream, &workspace->out_buf);
534 		if (unlikely(zstd_is_error(ret2))) {
535 			btrfs_err(fs_info,
536 "zstd compression end level %d failed, error %d root %llu inode %llu offset %llu",
537 				  workspace->req_level, zstd_get_error_code(ret2),
538 				  btrfs_root_id(inode->root), btrfs_ino(inode),
539 				  start + tot_in);
540 			ret = -EIO;
541 			goto out;
542 		}
543 		/* Queue the remaining part of the output folio into bio. */
544 		if (ret2 == 0) {
545 			tot_out += workspace->out_buf.pos;
546 			if (tot_out >= len) {
547 				ret = -E2BIG;
548 				goto out;
549 			}
550 			if (!bio_add_folio(bio, out_folio, workspace->out_buf.pos, 0)) {
551 				ret = -E2BIG;
552 				goto out;
553 			}
554 			out_folio = NULL;
555 			break;
556 		}
557 		tot_out += min_folio_size;
558 		if (tot_out >= len) {
559 			ret = -E2BIG;
560 			goto out;
561 		}
562 		if (!bio_add_folio(bio, out_folio, folio_size(out_folio), 0)) {
563 			ret = -E2BIG;
564 			goto out;
565 		}
566 		out_folio = btrfs_alloc_compr_folio(fs_info, GFP_NOFS);
567 		if (out_folio == NULL) {
568 			ret = -ENOMEM;
569 			goto out;
570 		}
571 		workspace->out_buf.dst = folio_address(out_folio);
572 		workspace->out_buf.pos = 0;
573 		workspace->out_buf.size = min_folio_size;
574 	}
575 
576 	if (tot_out >= tot_in) {
577 		ret = -E2BIG;
578 		goto out;
579 	}
580 
581 	ret = 0;
582 	ASSERT(tot_out == bio->bi_iter.bi_size);
583 out:
584 	if (out_folio)
585 		btrfs_free_compr_folio(out_folio);
586 	if (workspace->in_buf.src) {
587 		kunmap_local(workspace->in_buf.src);
588 		folio_put(in_folio);
589 	}
590 	return ret;
591 }
592 
593 /*
594  * Map the destination for the next chunk of output.
595  *
596  * @decompressed is the offset of the next output byte inside the fully
597  * decompressed extent.  If that offset has reached the current destination
598  * segment, its page-bounded bio_vec is kmapped so that zstd can write into the
599  * page cache directly, and the number of bytes writable there is returned.
600  * Otherwise @kaddr_ret is set to NULL and the number of bytes to skip before
601  * that segment is returned.  This covers both the initial prefix and gaps in
602  * the destination bio.
603  */
604 static u32 zstd_map_dest(struct compressed_bio *cb, u32 decompressed,
605 			 void **kaddr_ret)
606 {
607 	struct bio *orig_bio = &cb->orig_bbio->bio;
608 	struct bio_vec bvec;
609 	u32 bvec_offset;
610 	u32 off;
611 
612 	bvec = bio_iter_iovec(orig_bio, orig_bio->bi_iter);
613 	/*
614 	 * cb->start may underflow, but subtracting that value can still give us
615 	 * the correct offset inside the full decompressed extent.
616 	 */
617 	bvec_offset = page_offset(bvec.bv_page) + bvec.bv_offset - cb->start;
618 
619 	if (decompressed < bvec_offset) {
620 		*kaddr_ret = NULL;
621 		return bvec_offset - decompressed;
622 	}
623 
624 	off = decompressed - bvec_offset;
625 	ASSERT(off < bvec.bv_len);
626 	*kaddr_ret = bvec_kmap_local(&bvec) + off;
627 	return bvec.bv_len - off;
628 }
629 
630 int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
631 {
632 	struct btrfs_fs_info *fs_info = cb_to_fs_info(cb);
633 	struct workspace *workspace = list_entry(ws, struct workspace, list);
634 	struct bio *orig_bio = &cb->orig_bbio->bio;
635 	struct folio_iter fi;
636 	size_t srclen = bio_get_size(&cb->bbio.bio);
637 	zstd_dstream *stream;
638 	int ret = 0;
639 	const unsigned int min_folio_size = btrfs_min_folio_size(fs_info);
640 	unsigned long folio_in_index = 0;
641 	unsigned long total_folios_in = DIV_ROUND_UP(srclen, min_folio_size);
642 	unsigned long total_out = 0;
643 
644 	bio_first_folio(&fi, &cb->bbio.bio, 0);
645 	if (unlikely(!fi.folio))
646 		return -EINVAL;
647 	ASSERT(folio_size(fi.folio) == min_folio_size);
648 
649 	stream = zstd_init_dstream(
650 			ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
651 	if (unlikely(!stream)) {
652 		struct btrfs_inode *inode = cb->bbio.inode;
653 
654 		btrfs_err(inode->root->fs_info,
655 		"zstd decompression init failed, root %llu inode %llu offset %llu",
656 			  btrfs_root_id(inode->root), btrfs_ino(inode), cb->start);
657 		ret = -EIO;
658 		goto done;
659 	}
660 
661 	workspace->in_buf.src = kmap_local_folio(fi.folio, 0);
662 	workspace->in_buf.pos = 0;
663 	workspace->in_buf.size = min_t(size_t, srclen, min_folio_size);
664 
665 	while (orig_bio->bi_iter.bi_size) {
666 		size_t ret2;
667 		void *kaddr;
668 		u32 dstlen;
669 
670 		dstlen = zstd_map_dest(cb, total_out, &kaddr);
671 		if (kaddr) {
672 			workspace->out_buf.dst = kaddr;
673 			workspace->out_buf.size = dstlen;
674 		} else {
675 			workspace->out_buf.dst = workspace->buf;
676 			workspace->out_buf.size = min_t(u32, dstlen,
677 							fs_info->sectorsize);
678 		}
679 		workspace->out_buf.pos = 0;
680 
681 		ret2 = zstd_decompress_stream(stream, &workspace->out_buf,
682 				&workspace->in_buf);
683 		if (kaddr)
684 			kunmap_local(kaddr);
685 		if (unlikely(zstd_is_error(ret2))) {
686 			struct btrfs_inode *inode = cb->bbio.inode;
687 
688 			btrfs_err(inode->root->fs_info,
689 		"zstd decompression failed, error %d root %llu inode %llu offset %llu",
690 				  zstd_get_error_code(ret2), btrfs_root_id(inode->root),
691 				  btrfs_ino(inode), cb->start);
692 			ret = -EIO;
693 			goto done;
694 		}
695 		total_out += workspace->out_buf.pos;
696 		if (kaddr)
697 			bio_advance(orig_bio, workspace->out_buf.pos);
698 
699 		if (workspace->in_buf.pos >= srclen)
700 			break;
701 
702 		/* Check if we've hit the end of a frame */
703 		if (ret2 == 0)
704 			break;
705 
706 		if (workspace->in_buf.pos == workspace->in_buf.size) {
707 			kunmap_local(workspace->in_buf.src);
708 			folio_in_index++;
709 			if (unlikely(folio_in_index >= total_folios_in)) {
710 				workspace->in_buf.src = NULL;
711 				ret = -EIO;
712 				goto done;
713 			}
714 			srclen -= min_folio_size;
715 			bio_next_folio(&fi, &cb->bbio.bio);
716 			ASSERT(fi.folio);
717 			workspace->in_buf.src = kmap_local_folio(fi.folio, 0);
718 			workspace->in_buf.pos = 0;
719 			workspace->in_buf.size = min_t(size_t, srclen, min_folio_size);
720 		}
721 	}
722 	ret = 0;
723 done:
724 	if (workspace->in_buf.src)
725 		kunmap_local(workspace->in_buf.src);
726 	return ret;
727 }
728 
729 int zstd_decompress(struct list_head *ws, const u8 *data_in,
730 		struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen,
731 		size_t destlen)
732 {
733 	struct workspace *workspace = list_entry(ws, struct workspace, list);
734 	struct btrfs_fs_info *fs_info = btrfs_sb(folio_inode(dest_folio)->i_sb);
735 	zstd_dstream *stream;
736 	int ret = 0;
737 	unsigned long to_copy = 0;
738 
739 	stream = zstd_init_dstream(
740 			ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
741 	if (unlikely(!stream)) {
742 		struct btrfs_inode *inode = folio_to_inode(dest_folio);
743 
744 		btrfs_err(inode->root->fs_info,
745 		"zstd decompression init failed, root %llu inode %llu offset %llu",
746 			  btrfs_root_id(inode->root), btrfs_ino(inode),
747 			  folio_pos(dest_folio));
748 		ret = -EIO;
749 		goto finish;
750 	}
751 
752 	workspace->in_buf.src = data_in;
753 	workspace->in_buf.pos = 0;
754 	workspace->in_buf.size = srclen;
755 
756 	workspace->out_buf.dst = workspace->buf;
757 	workspace->out_buf.pos = 0;
758 	workspace->out_buf.size = fs_info->sectorsize;
759 
760 	/*
761 	 * Since both input and output buffers should not exceed one sector,
762 	 * one call should end the decompression.
763 	 */
764 	ret = zstd_decompress_stream(stream, &workspace->out_buf, &workspace->in_buf);
765 	if (unlikely(zstd_is_error(ret))) {
766 		struct btrfs_inode *inode = folio_to_inode(dest_folio);
767 
768 		btrfs_err(inode->root->fs_info,
769 		"zstd decompression failed, error %d root %llu inode %llu offset %llu",
770 			  zstd_get_error_code(ret), btrfs_root_id(inode->root),
771 			  btrfs_ino(inode), folio_pos(dest_folio));
772 		goto finish;
773 	}
774 	to_copy = workspace->out_buf.pos;
775 	memcpy_to_folio(dest_folio, dest_pgoff, workspace->out_buf.dst, to_copy);
776 finish:
777 	/* Error or early end. */
778 	if (unlikely(to_copy < destlen)) {
779 		ret = -EIO;
780 		folio_zero_range(dest_folio, dest_pgoff + to_copy, destlen - to_copy);
781 	}
782 	return ret;
783 }
784 
785 const struct btrfs_compress_levels btrfs_zstd_compress = {
786 	.min_level	= ZSTD_BTRFS_MIN_LEVEL,
787 	.max_level	= ZSTD_BTRFS_MAX_LEVEL,
788 	.default_level	= ZSTD_BTRFS_DEFAULT_LEVEL,
789 };
790