1 // SPDX-License-Identifier: GPL-2.0-or-later
2
3 #include <linux/kernel.h>
4 #include <linux/slab.h>
5 #include <linux/vmalloc.h>
6 #include <linux/zstd.h>
7
8 #include "backend_zstd.h"
9
10 struct zstd_ctx {
11 zstd_cctx *cctx;
12 zstd_dctx *dctx;
13 void *cctx_mem;
14 void *dctx_mem;
15 };
16
17 struct zstd_params {
18 zstd_custom_mem custom_mem;
19 zstd_cdict *cdict;
20 zstd_ddict *ddict;
21 zstd_parameters cprm;
22 };
23
24 /*
25 * For C/D dictionaries we need to provide zstd with zstd_custom_mem,
26 * which zstd uses internally to allocate/free memory when needed.
27 *
28 * This means that allocator.customAlloc() can be called from zcomp_compress()
29 * under local-lock (per-CPU compression stream), in which case we must use
30 * GFP_ATOMIC.
31 *
32 * Another complication here is that we can be configured as a swap device.
33 */
zstd_custom_alloc(void * opaque,size_t size)34 static void *zstd_custom_alloc(void *opaque, size_t size)
35 {
36 if (!preemptible())
37 return kvzalloc(size, GFP_ATOMIC);
38
39 return kvzalloc(size, __GFP_KSWAPD_RECLAIM | __GFP_NOWARN);
40 }
41
zstd_custom_free(void * opaque,void * address)42 static void zstd_custom_free(void *opaque, void *address)
43 {
44 kvfree(address);
45 }
46
zstd_release_params(struct zcomp_params * params)47 static void zstd_release_params(struct zcomp_params *params)
48 {
49 struct zstd_params *zp = params->drv_data;
50
51 params->drv_data = NULL;
52 if (!zp)
53 return;
54
55 zstd_free_cdict(zp->cdict);
56 zstd_free_ddict(zp->ddict);
57 kfree(zp);
58 }
59
zstd_setup_params(struct zcomp_params * params)60 static int zstd_setup_params(struct zcomp_params *params)
61 {
62 zstd_compression_parameters prm;
63 struct zstd_params *zp;
64
65 zp = kzalloc(sizeof(*zp), GFP_KERNEL);
66 if (!zp)
67 return -ENOMEM;
68
69 params->drv_data = zp;
70 if (params->level == ZCOMP_PARAM_NO_LEVEL)
71 params->level = zstd_default_clevel();
72
73 zp->cprm = zstd_get_params(params->level, PAGE_SIZE);
74
75 zp->custom_mem.customAlloc = zstd_custom_alloc;
76 zp->custom_mem.customFree = zstd_custom_free;
77
78 prm = zstd_get_cparams(params->level, PAGE_SIZE,
79 params->dict_sz);
80
81 zp->cdict = zstd_create_cdict_byreference(params->dict,
82 params->dict_sz,
83 prm,
84 zp->custom_mem);
85 if (!zp->cdict)
86 goto error;
87
88 zp->ddict = zstd_create_ddict_byreference(params->dict,
89 params->dict_sz,
90 zp->custom_mem);
91 if (!zp->ddict)
92 goto error;
93
94 return 0;
95
96 error:
97 zstd_release_params(params);
98 return -EINVAL;
99 }
100
zstd_destroy(struct zcomp_ctx * ctx)101 static void zstd_destroy(struct zcomp_ctx *ctx)
102 {
103 struct zstd_ctx *zctx = ctx->context;
104
105 if (!zctx)
106 return;
107
108 /*
109 * If ->cctx_mem and ->dctx_mem were allocated then we didn't use
110 * C/D dictionary and ->cctx / ->dctx were "embedded" into these
111 * buffers.
112 *
113 * If otherwise then we need to explicitly release ->cctx / ->dctx.
114 */
115 if (zctx->cctx_mem)
116 vfree(zctx->cctx_mem);
117 else
118 zstd_free_cctx(zctx->cctx);
119
120 if (zctx->dctx_mem)
121 vfree(zctx->dctx_mem);
122 else
123 zstd_free_dctx(zctx->dctx);
124
125 kfree(zctx);
126 }
127
zstd_create(struct zcomp_params * params,struct zcomp_ctx * ctx)128 static int zstd_create(struct zcomp_params *params, struct zcomp_ctx *ctx)
129 {
130 struct zstd_ctx *zctx;
131 zstd_parameters prm;
132 size_t sz;
133
134 zctx = kzalloc(sizeof(*zctx), GFP_KERNEL);
135 if (!zctx)
136 return -ENOMEM;
137
138 ctx->context = zctx;
139 if (params->dict_sz == 0) {
140 prm = zstd_get_params(params->level, PAGE_SIZE);
141 sz = zstd_cctx_workspace_bound(&prm.cParams);
142 zctx->cctx_mem = vzalloc(sz);
143 if (!zctx->cctx_mem)
144 goto error;
145
146 zctx->cctx = zstd_init_cctx(zctx->cctx_mem, sz);
147 if (!zctx->cctx)
148 goto error;
149
150 sz = zstd_dctx_workspace_bound();
151 zctx->dctx_mem = vzalloc(sz);
152 if (!zctx->dctx_mem)
153 goto error;
154
155 zctx->dctx = zstd_init_dctx(zctx->dctx_mem, sz);
156 if (!zctx->dctx)
157 goto error;
158 } else {
159 struct zstd_params *zp = params->drv_data;
160
161 zctx->cctx = zstd_create_cctx_advanced(zp->custom_mem);
162 if (!zctx->cctx)
163 goto error;
164
165 zctx->dctx = zstd_create_dctx_advanced(zp->custom_mem);
166 if (!zctx->dctx)
167 goto error;
168 }
169
170 return 0;
171
172 error:
173 zstd_release_params(params);
174 zstd_destroy(ctx);
175 return -EINVAL;
176 }
177
zstd_compress(struct zcomp_params * params,struct zcomp_ctx * ctx,struct zcomp_req * req)178 static int zstd_compress(struct zcomp_params *params, struct zcomp_ctx *ctx,
179 struct zcomp_req *req)
180 {
181 struct zstd_params *zp = params->drv_data;
182 struct zstd_ctx *zctx = ctx->context;
183 size_t ret;
184
185 if (params->dict_sz == 0)
186 ret = zstd_compress_cctx(zctx->cctx, req->dst, req->dst_len,
187 req->src, req->src_len, &zp->cprm);
188 else
189 ret = zstd_compress_using_cdict(zctx->cctx, req->dst,
190 req->dst_len, req->src,
191 req->src_len,
192 zp->cdict);
193 if (zstd_is_error(ret))
194 return -EINVAL;
195 req->dst_len = ret;
196 return 0;
197 }
198
zstd_decompress(struct zcomp_params * params,struct zcomp_ctx * ctx,struct zcomp_req * req)199 static int zstd_decompress(struct zcomp_params *params, struct zcomp_ctx *ctx,
200 struct zcomp_req *req)
201 {
202 struct zstd_params *zp = params->drv_data;
203 struct zstd_ctx *zctx = ctx->context;
204 size_t ret;
205
206 if (params->dict_sz == 0)
207 ret = zstd_decompress_dctx(zctx->dctx, req->dst, req->dst_len,
208 req->src, req->src_len);
209 else
210 ret = zstd_decompress_using_ddict(zctx->dctx, req->dst,
211 req->dst_len, req->src,
212 req->src_len, zp->ddict);
213 if (zstd_is_error(ret))
214 return -EINVAL;
215 return 0;
216 }
217
218 const struct zcomp_ops backend_zstd = {
219 .compress = zstd_compress,
220 .decompress = zstd_decompress,
221 .create_ctx = zstd_create,
222 .destroy_ctx = zstd_destroy,
223 .setup_params = zstd_setup_params,
224 .release_params = zstd_release_params,
225 .name = "zstd",
226 };
227