1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <linux/zstd.h>
3 #include "compress.h"
4
5 struct z_erofs_zstd {
6 struct z_erofs_zstd *next;
7 u8 bounce[PAGE_SIZE];
8 void *wksp;
9 unsigned int wkspsz;
10 };
11
12 static DEFINE_SPINLOCK(z_erofs_zstd_lock);
13 static unsigned int z_erofs_zstd_max_dictsize;
14 static unsigned int z_erofs_zstd_nstrms, z_erofs_zstd_avail_strms;
15 static struct z_erofs_zstd *z_erofs_zstd_head;
16 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_zstd_wq);
17
18 module_param_named(zstd_streams, z_erofs_zstd_nstrms, uint, 0444);
19
z_erofs_isolate_strms(bool all)20 static struct z_erofs_zstd *z_erofs_isolate_strms(bool all)
21 {
22 struct z_erofs_zstd *strm;
23
24 again:
25 spin_lock(&z_erofs_zstd_lock);
26 strm = z_erofs_zstd_head;
27 if (!strm) {
28 spin_unlock(&z_erofs_zstd_lock);
29 wait_event(z_erofs_zstd_wq, READ_ONCE(z_erofs_zstd_head));
30 goto again;
31 }
32 z_erofs_zstd_head = all ? NULL : strm->next;
33 spin_unlock(&z_erofs_zstd_lock);
34 return strm;
35 }
36
z_erofs_zstd_exit(void)37 static void z_erofs_zstd_exit(void)
38 {
39 while (z_erofs_zstd_avail_strms) {
40 struct z_erofs_zstd *strm, *n;
41
42 for (strm = z_erofs_isolate_strms(true); strm; strm = n) {
43 n = strm->next;
44
45 kvfree(strm->wksp);
46 kfree(strm);
47 --z_erofs_zstd_avail_strms;
48 }
49 }
50 }
51
z_erofs_zstd_init(void)52 static int __init z_erofs_zstd_init(void)
53 {
54 /* by default, use # of possible CPUs instead */
55 if (!z_erofs_zstd_nstrms)
56 z_erofs_zstd_nstrms = num_possible_cpus();
57
58 for (; z_erofs_zstd_avail_strms < z_erofs_zstd_nstrms;
59 ++z_erofs_zstd_avail_strms) {
60 struct z_erofs_zstd *strm;
61
62 strm = kzalloc(sizeof(*strm), GFP_KERNEL);
63 if (!strm) {
64 z_erofs_zstd_exit();
65 return -ENOMEM;
66 }
67 spin_lock(&z_erofs_zstd_lock);
68 strm->next = z_erofs_zstd_head;
69 z_erofs_zstd_head = strm;
70 spin_unlock(&z_erofs_zstd_lock);
71 }
72 return 0;
73 }
74
z_erofs_load_zstd_config(struct super_block * sb,struct erofs_super_block * dsb,void * data,int size)75 static int z_erofs_load_zstd_config(struct super_block *sb,
76 struct erofs_super_block *dsb, void *data, int size)
77 {
78 static DEFINE_MUTEX(zstd_resize_mutex);
79 struct z_erofs_zstd_cfgs *zstd = data;
80 unsigned int dict_size, wkspsz;
81 struct z_erofs_zstd *strm, *head = NULL;
82 void *wksp;
83
84 if (!zstd || size < sizeof(struct z_erofs_zstd_cfgs) || zstd->format) {
85 erofs_err(sb, "unsupported zstd format, size=%u", size);
86 return -EINVAL;
87 }
88
89 if (zstd->windowlog > ilog2(Z_EROFS_ZSTD_MAX_DICT_SIZE) - 10) {
90 erofs_err(sb, "unsupported zstd window log %u", zstd->windowlog);
91 return -EINVAL;
92 }
93 dict_size = 1U << (zstd->windowlog + 10);
94
95 /* in case 2 z_erofs_load_zstd_config() race to avoid deadlock */
96 mutex_lock(&zstd_resize_mutex);
97 if (z_erofs_zstd_max_dictsize >= dict_size) {
98 mutex_unlock(&zstd_resize_mutex);
99 return 0;
100 }
101
102 /* 1. collect/isolate all streams for the following check */
103 while (z_erofs_zstd_avail_strms) {
104 struct z_erofs_zstd *n;
105
106 for (strm = z_erofs_isolate_strms(true); strm; strm = n) {
107 n = strm->next;
108 strm->next = head;
109 head = strm;
110 --z_erofs_zstd_avail_strms;
111 }
112 }
113
114 /* 2. walk each isolated stream and grow max dict_size if needed */
115 wkspsz = zstd_dstream_workspace_bound(dict_size);
116 for (strm = head; strm; strm = strm->next) {
117 wksp = kvmalloc(wkspsz, GFP_KERNEL);
118 if (!wksp)
119 break;
120 kvfree(strm->wksp);
121 strm->wksp = wksp;
122 strm->wkspsz = wkspsz;
123 }
124
125 /* 3. push back all to the global list and update max dict_size */
126 spin_lock(&z_erofs_zstd_lock);
127 DBG_BUGON(z_erofs_zstd_head);
128 z_erofs_zstd_head = head;
129 spin_unlock(&z_erofs_zstd_lock);
130 z_erofs_zstd_avail_strms = z_erofs_zstd_nstrms;
131 wake_up_all(&z_erofs_zstd_wq);
132 if (!strm)
133 z_erofs_zstd_max_dictsize = dict_size;
134 mutex_unlock(&zstd_resize_mutex);
135 return strm ? -ENOMEM : 0;
136 }
137
z_erofs_zstd_decompress(struct z_erofs_decompress_req * rq,struct page ** pgpl)138 static int z_erofs_zstd_decompress(struct z_erofs_decompress_req *rq,
139 struct page **pgpl)
140 {
141 struct super_block *sb = rq->sb;
142 struct z_erofs_stream_dctx dctx = { .rq = rq, .no = -1, .ni = 0 };
143 zstd_in_buffer in_buf = { NULL, 0, 0 };
144 zstd_out_buffer out_buf = { NULL, 0, 0 };
145 struct z_erofs_zstd *strm;
146 zstd_dstream *stream;
147 int zerr, err;
148
149 /* 1. get the exact compressed size */
150 dctx.kin = kmap_local_page(*rq->in);
151 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
152 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
153 if (err) {
154 kunmap_local(dctx.kin);
155 return err;
156 }
157
158 /* 2. get an available ZSTD context */
159 strm = z_erofs_isolate_strms(false);
160
161 /* 3. multi-call decompress */
162 stream = zstd_init_dstream(z_erofs_zstd_max_dictsize, strm->wksp, strm->wkspsz);
163 if (!stream) {
164 err = -EIO;
165 goto failed_zinit;
166 }
167
168 rq->fillgaps = true; /* ZSTD doesn't support NULL output buffer */
169 in_buf.size = min_t(u32, rq->inputsize, PAGE_SIZE - rq->pageofs_in);
170 rq->inputsize -= in_buf.size;
171 in_buf.src = dctx.kin + rq->pageofs_in;
172 dctx.bounce = strm->bounce;
173
174 do {
175 dctx.avail_out = out_buf.size - out_buf.pos;
176 dctx.inbuf_sz = in_buf.size;
177 dctx.inbuf_pos = in_buf.pos;
178 err = z_erofs_stream_switch_bufs(&dctx, &out_buf.dst,
179 (void **)&in_buf.src, pgpl);
180 if (err)
181 break;
182
183 if (out_buf.size == out_buf.pos) {
184 out_buf.size = dctx.avail_out;
185 out_buf.pos = 0;
186 }
187 in_buf.size = dctx.inbuf_sz;
188 in_buf.pos = dctx.inbuf_pos;
189
190 zerr = zstd_decompress_stream(stream, &out_buf, &in_buf);
191 if (zstd_is_error(zerr) || (!zerr && rq->outputsize)) {
192 erofs_err(sb, "failed to decompress in[%u] out[%u]: %s",
193 rq->inputsize, rq->outputsize,
194 zerr ? zstd_get_error_name(zerr) : "unexpected end of stream");
195 err = -EFSCORRUPTED;
196 break;
197 }
198 } while (rq->outputsize || out_buf.pos < out_buf.size);
199
200 if (dctx.kout)
201 kunmap_local(dctx.kout);
202 failed_zinit:
203 kunmap_local(dctx.kin);
204 /* 4. push back ZSTD stream context to the global list */
205 spin_lock(&z_erofs_zstd_lock);
206 strm->next = z_erofs_zstd_head;
207 z_erofs_zstd_head = strm;
208 spin_unlock(&z_erofs_zstd_lock);
209 wake_up(&z_erofs_zstd_wq);
210 return err;
211 }
212
213 const struct z_erofs_decompressor z_erofs_zstd_decomp = {
214 .config = z_erofs_load_zstd_config,
215 .decompress = z_erofs_zstd_decompress,
216 .init = z_erofs_zstd_init,
217 .exit = z_erofs_zstd_exit,
218 .name = "zstd",
219 };
220