1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <linux/zstd.h>
3 #include "compress.h"
4
5 struct z_erofs_zstd {
6 struct z_erofs_zstd *next;
7 u8 bounce[PAGE_SIZE];
8 void *wksp;
9 unsigned int wkspsz;
10 };
11
12 static DEFINE_SPINLOCK(z_erofs_zstd_lock);
13 static unsigned int z_erofs_zstd_max_dictsize;
14 static unsigned int z_erofs_zstd_nstrms, z_erofs_zstd_avail_strms;
15 static struct z_erofs_zstd *z_erofs_zstd_head;
16 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_zstd_wq);
17
18 module_param_named(zstd_streams, z_erofs_zstd_nstrms, uint, 0444);
19
z_erofs_isolate_strms(bool all)20 static struct z_erofs_zstd *z_erofs_isolate_strms(bool all)
21 {
22 struct z_erofs_zstd *strm;
23
24 again:
25 spin_lock(&z_erofs_zstd_lock);
26 strm = z_erofs_zstd_head;
27 if (!strm) {
28 spin_unlock(&z_erofs_zstd_lock);
29 wait_event(z_erofs_zstd_wq, READ_ONCE(z_erofs_zstd_head));
30 goto again;
31 }
32 z_erofs_zstd_head = all ? NULL : strm->next;
33 spin_unlock(&z_erofs_zstd_lock);
34 return strm;
35 }
36
z_erofs_zstd_exit(void)37 static void z_erofs_zstd_exit(void)
38 {
39 while (z_erofs_zstd_avail_strms) {
40 struct z_erofs_zstd *strm, *n;
41
42 for (strm = z_erofs_isolate_strms(true); strm; strm = n) {
43 n = strm->next;
44
45 kvfree(strm->wksp);
46 kfree(strm);
47 --z_erofs_zstd_avail_strms;
48 }
49 }
50 }
51
z_erofs_zstd_init(void)52 static int __init z_erofs_zstd_init(void)
53 {
54 /* by default, use # of possible CPUs instead */
55 if (!z_erofs_zstd_nstrms)
56 z_erofs_zstd_nstrms = num_possible_cpus();
57
58 for (; z_erofs_zstd_avail_strms < z_erofs_zstd_nstrms;
59 ++z_erofs_zstd_avail_strms) {
60 struct z_erofs_zstd *strm;
61
62 strm = kzalloc(sizeof(*strm), GFP_KERNEL);
63 if (!strm) {
64 z_erofs_zstd_exit();
65 return -ENOMEM;
66 }
67 spin_lock(&z_erofs_zstd_lock);
68 strm->next = z_erofs_zstd_head;
69 z_erofs_zstd_head = strm;
70 spin_unlock(&z_erofs_zstd_lock);
71 }
72 return 0;
73 }
74
z_erofs_load_zstd_config(struct super_block * sb,struct erofs_super_block * dsb,void * data,int size)75 static int z_erofs_load_zstd_config(struct super_block *sb,
76 struct erofs_super_block *dsb, void *data, int size)
77 {
78 static DEFINE_MUTEX(zstd_resize_mutex);
79 struct z_erofs_zstd_cfgs *zstd = data;
80 unsigned int dict_size, wkspsz;
81 struct z_erofs_zstd *strm, *head = NULL;
82 void *wksp;
83
84 if (!zstd || size < sizeof(struct z_erofs_zstd_cfgs) || zstd->format) {
85 erofs_err(sb, "unsupported zstd format, size=%u", size);
86 return -EINVAL;
87 }
88
89 if (zstd->windowlog > ilog2(Z_EROFS_ZSTD_MAX_DICT_SIZE) - 10) {
90 erofs_err(sb, "unsupported zstd window log %u", zstd->windowlog);
91 return -EINVAL;
92 }
93 dict_size = 1U << (zstd->windowlog + 10);
94
95 /* in case 2 z_erofs_load_zstd_config() race to avoid deadlock */
96 mutex_lock(&zstd_resize_mutex);
97 if (z_erofs_zstd_max_dictsize >= dict_size) {
98 mutex_unlock(&zstd_resize_mutex);
99 return 0;
100 }
101
102 /* 1. collect/isolate all streams for the following check */
103 while (z_erofs_zstd_avail_strms) {
104 struct z_erofs_zstd *n;
105
106 for (strm = z_erofs_isolate_strms(true); strm; strm = n) {
107 n = strm->next;
108 strm->next = head;
109 head = strm;
110 --z_erofs_zstd_avail_strms;
111 }
112 }
113
114 /* 2. walk each isolated stream and grow max dict_size if needed */
115 wkspsz = zstd_dstream_workspace_bound(dict_size);
116 for (strm = head; strm; strm = strm->next) {
117 wksp = kvmalloc(wkspsz, GFP_KERNEL);
118 if (!wksp)
119 break;
120 kvfree(strm->wksp);
121 strm->wksp = wksp;
122 strm->wkspsz = wkspsz;
123 }
124
125 /* 3. push back all to the global list and update max dict_size */
126 spin_lock(&z_erofs_zstd_lock);
127 DBG_BUGON(z_erofs_zstd_head);
128 z_erofs_zstd_head = head;
129 spin_unlock(&z_erofs_zstd_lock);
130 z_erofs_zstd_avail_strms = z_erofs_zstd_nstrms;
131 wake_up_all(&z_erofs_zstd_wq);
132 if (!strm)
133 z_erofs_zstd_max_dictsize = dict_size;
134 mutex_unlock(&zstd_resize_mutex);
135 return strm ? -ENOMEM : 0;
136 }
137
z_erofs_zstd_decompress(struct z_erofs_decompress_req * rq,struct page ** pgpl)138 static int z_erofs_zstd_decompress(struct z_erofs_decompress_req *rq,
139 struct page **pgpl)
140 {
141 struct super_block *sb = rq->sb;
142 struct z_erofs_stream_dctx dctx = {
143 .rq = rq,
144 .inpages = PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT,
145 .outpages = PAGE_ALIGN(rq->pageofs_out + rq->outputsize)
146 >> PAGE_SHIFT,
147 .no = -1, .ni = 0,
148 };
149 zstd_in_buffer in_buf = { NULL, 0, 0 };
150 zstd_out_buffer out_buf = { NULL, 0, 0 };
151 struct z_erofs_zstd *strm;
152 zstd_dstream *stream;
153 int zerr, err;
154
155 /* 1. get the exact compressed size */
156 dctx.kin = kmap_local_page(*rq->in);
157 err = z_erofs_fixup_insize(rq, dctx.kin + rq->pageofs_in,
158 min(rq->inputsize, sb->s_blocksize - rq->pageofs_in));
159 if (err) {
160 kunmap_local(dctx.kin);
161 return err;
162 }
163
164 /* 2. get an available ZSTD context */
165 strm = z_erofs_isolate_strms(false);
166
167 /* 3. multi-call decompress */
168 stream = zstd_init_dstream(z_erofs_zstd_max_dictsize, strm->wksp, strm->wkspsz);
169 if (!stream) {
170 err = -EIO;
171 goto failed_zinit;
172 }
173
174 rq->fillgaps = true; /* ZSTD doesn't support NULL output buffer */
175 in_buf.size = min_t(u32, rq->inputsize, PAGE_SIZE - rq->pageofs_in);
176 rq->inputsize -= in_buf.size;
177 in_buf.src = dctx.kin + rq->pageofs_in;
178 dctx.bounce = strm->bounce;
179
180 do {
181 dctx.avail_out = out_buf.size - out_buf.pos;
182 dctx.inbuf_sz = in_buf.size;
183 dctx.inbuf_pos = in_buf.pos;
184 err = z_erofs_stream_switch_bufs(&dctx, &out_buf.dst,
185 (void **)&in_buf.src, pgpl);
186 if (err)
187 break;
188
189 if (out_buf.size == out_buf.pos) {
190 out_buf.size = dctx.avail_out;
191 out_buf.pos = 0;
192 }
193 in_buf.size = dctx.inbuf_sz;
194 in_buf.pos = dctx.inbuf_pos;
195
196 zerr = zstd_decompress_stream(stream, &out_buf, &in_buf);
197 if (zstd_is_error(zerr) || (!zerr && rq->outputsize)) {
198 erofs_err(sb, "failed to decompress in[%u] out[%u]: %s",
199 rq->inputsize, rq->outputsize,
200 zerr ? zstd_get_error_name(zerr) : "unexpected end of stream");
201 err = -EFSCORRUPTED;
202 break;
203 }
204 } while (rq->outputsize || out_buf.pos < out_buf.size);
205
206 if (dctx.kout)
207 kunmap_local(dctx.kout);
208 failed_zinit:
209 kunmap_local(dctx.kin);
210 /* 4. push back ZSTD stream context to the global list */
211 spin_lock(&z_erofs_zstd_lock);
212 strm->next = z_erofs_zstd_head;
213 z_erofs_zstd_head = strm;
214 spin_unlock(&z_erofs_zstd_lock);
215 wake_up(&z_erofs_zstd_wq);
216 return err;
217 }
218
219 const struct z_erofs_decompressor z_erofs_zstd_decomp = {
220 .config = z_erofs_load_zstd_config,
221 .decompress = z_erofs_zstd_decompress,
222 .init = z_erofs_zstd_init,
223 .exit = z_erofs_zstd_exit,
224 .name = "zstd",
225 };
226