1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Synchronous Compression operations
4 *
5 * Copyright 2015 LG Electronics Inc.
6 * Copyright (c) 2016, Intel Corporation
7 * Author: Giovanni Cabiddu <giovanni.cabiddu@intel.com>
8 */
9
10 #include <crypto/internal/scompress.h>
11 #include <crypto/scatterwalk.h>
12 #include <linux/cpumask.h>
13 #include <linux/cryptouser.h>
14 #include <linux/err.h>
15 #include <linux/highmem.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18 #include <linux/overflow.h>
19 #include <linux/scatterlist.h>
20 #include <linux/seq_file.h>
21 #include <linux/slab.h>
22 #include <linux/string.h>
23 #include <linux/workqueue.h>
24 #include <net/netlink.h>
25
26 #include "compress.h"
27
28 struct scomp_scratch {
29 spinlock_t lock;
30 union {
31 void *src;
32 unsigned long saddr;
33 };
34 };
35
36 static DEFINE_PER_CPU(struct scomp_scratch, scomp_scratch) = {
37 .lock = __SPIN_LOCK_UNLOCKED(scomp_scratch.lock),
38 };
39
40 static const struct crypto_type crypto_scomp_type;
41 static int scomp_scratch_users;
42 static DEFINE_MUTEX(scomp_lock);
43
44 static cpumask_t scomp_scratch_want;
45 static void scomp_scratch_workfn(struct work_struct *work);
46 static DECLARE_WORK(scomp_scratch_work, scomp_scratch_workfn);
47
crypto_scomp_report(struct sk_buff * skb,struct crypto_alg * alg)48 static int __maybe_unused crypto_scomp_report(
49 struct sk_buff *skb, struct crypto_alg *alg)
50 {
51 struct crypto_report_comp rscomp;
52
53 memset(&rscomp, 0, sizeof(rscomp));
54
55 strscpy(rscomp.type, "scomp", sizeof(rscomp.type));
56
57 return nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS,
58 sizeof(rscomp), &rscomp);
59 }
60
61 static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
62 __maybe_unused;
63
crypto_scomp_show(struct seq_file * m,struct crypto_alg * alg)64 static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg)
65 {
66 seq_puts(m, "type : scomp\n");
67 }
68
crypto_scomp_free_scratches(void)69 static void crypto_scomp_free_scratches(void)
70 {
71 struct scomp_scratch *scratch;
72 int i;
73
74 for_each_possible_cpu(i) {
75 scratch = per_cpu_ptr(&scomp_scratch, i);
76
77 free_page(scratch->saddr);
78 scratch->src = NULL;
79 }
80 }
81
scomp_alloc_scratch(struct scomp_scratch * scratch,int cpu)82 static int scomp_alloc_scratch(struct scomp_scratch *scratch, int cpu)
83 {
84 int node = cpu_to_node(cpu);
85 struct page *page;
86
87 page = alloc_pages_node(node, GFP_KERNEL, 0);
88 if (!page)
89 return -ENOMEM;
90 spin_lock_bh(&scratch->lock);
91 scratch->src = page_address(page);
92 spin_unlock_bh(&scratch->lock);
93 return 0;
94 }
95
scomp_scratch_workfn(struct work_struct * work)96 static void scomp_scratch_workfn(struct work_struct *work)
97 {
98 int cpu;
99
100 for_each_cpu(cpu, &scomp_scratch_want) {
101 struct scomp_scratch *scratch;
102
103 scratch = per_cpu_ptr(&scomp_scratch, cpu);
104 if (scratch->src)
105 continue;
106 if (scomp_alloc_scratch(scratch, cpu))
107 break;
108
109 cpumask_clear_cpu(cpu, &scomp_scratch_want);
110 }
111 }
112
crypto_scomp_alloc_scratches(void)113 static int crypto_scomp_alloc_scratches(void)
114 {
115 unsigned int i = cpumask_first(cpu_possible_mask);
116 struct scomp_scratch *scratch;
117
118 scratch = per_cpu_ptr(&scomp_scratch, i);
119 return scomp_alloc_scratch(scratch, i);
120 }
121
crypto_scomp_init_tfm(struct crypto_tfm * tfm)122 static int crypto_scomp_init_tfm(struct crypto_tfm *tfm)
123 {
124 struct scomp_alg *alg = crypto_scomp_alg(__crypto_scomp_tfm(tfm));
125 int ret = 0;
126
127 mutex_lock(&scomp_lock);
128 ret = crypto_acomp_alloc_streams(&alg->streams);
129 if (ret)
130 goto unlock;
131 if (!scomp_scratch_users++) {
132 ret = crypto_scomp_alloc_scratches();
133 if (ret)
134 scomp_scratch_users--;
135 }
136 unlock:
137 mutex_unlock(&scomp_lock);
138
139 return ret;
140 }
141
scomp_lock_scratch(void)142 static struct scomp_scratch *scomp_lock_scratch(void) __acquires(scratch)
143 {
144 int cpu = raw_smp_processor_id();
145 struct scomp_scratch *scratch;
146
147 scratch = per_cpu_ptr(&scomp_scratch, cpu);
148 spin_lock(&scratch->lock);
149 if (likely(scratch->src))
150 return scratch;
151 spin_unlock(&scratch->lock);
152
153 cpumask_set_cpu(cpu, &scomp_scratch_want);
154 schedule_work(&scomp_scratch_work);
155
156 scratch = per_cpu_ptr(&scomp_scratch, cpumask_first(cpu_possible_mask));
157 spin_lock(&scratch->lock);
158 return scratch;
159 }
160
scomp_unlock_scratch(struct scomp_scratch * scratch)161 static inline void scomp_unlock_scratch(struct scomp_scratch *scratch)
162 __releases(scratch)
163 {
164 spin_unlock(&scratch->lock);
165 }
166
scomp_acomp_comp_decomp(struct acomp_req * req,int dir)167 static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir)
168 {
169 struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
170 struct crypto_scomp **tfm_ctx = acomp_tfm_ctx(tfm);
171 bool src_isvirt = acomp_request_src_isvirt(req);
172 bool dst_isvirt = acomp_request_dst_isvirt(req);
173 struct crypto_scomp *scomp = *tfm_ctx;
174 struct crypto_acomp_stream *stream;
175 struct scomp_scratch *scratch;
176 unsigned int slen = req->slen;
177 unsigned int dlen = req->dlen;
178 struct page *spage, *dpage;
179 unsigned int n;
180 const u8 *src;
181 size_t soff;
182 size_t doff;
183 u8 *dst;
184 int ret;
185
186 if (!req->src || !slen)
187 return -EINVAL;
188
189 if (!req->dst || !dlen)
190 return -EINVAL;
191
192 if (dst_isvirt)
193 dst = req->dvirt;
194 else {
195 if (dlen <= req->dst->length) {
196 dpage = sg_page(req->dst);
197 doff = req->dst->offset;
198 } else
199 return -ENOSYS;
200
201 dpage = nth_page(dpage, doff / PAGE_SIZE);
202 doff = offset_in_page(doff);
203
204 n = (dlen - 1) / PAGE_SIZE;
205 n += (offset_in_page(dlen - 1) + doff) / PAGE_SIZE;
206 if (PageHighMem(dpage + n) &&
207 size_add(doff, dlen) > PAGE_SIZE)
208 return -ENOSYS;
209 dst = kmap_local_page(dpage) + doff;
210 }
211
212 if (src_isvirt)
213 src = req->svirt;
214 else {
215 src = NULL;
216 do {
217 if (slen <= req->src->length) {
218 spage = sg_page(req->src);
219 soff = req->src->offset;
220 } else
221 break;
222
223 spage = nth_page(spage, soff / PAGE_SIZE);
224 soff = offset_in_page(soff);
225
226 n = (slen - 1) / PAGE_SIZE;
227 n += (offset_in_page(slen - 1) + soff) / PAGE_SIZE;
228 if (PageHighMem(nth_page(spage, n)) &&
229 size_add(soff, slen) > PAGE_SIZE)
230 break;
231 src = kmap_local_page(spage) + soff;
232 } while (0);
233 }
234
235 stream = crypto_acomp_lock_stream_bh(&crypto_scomp_alg(scomp)->streams);
236
237 if (!src_isvirt && !src) {
238 const u8 *src;
239
240 scratch = scomp_lock_scratch();
241 src = scratch->src;
242 memcpy_from_sglist(scratch->src, req->src, 0, slen);
243
244 if (dir)
245 ret = crypto_scomp_compress(scomp, src, slen,
246 dst, &dlen, stream->ctx);
247 else
248 ret = crypto_scomp_decompress(scomp, src, slen,
249 dst, &dlen, stream->ctx);
250
251 scomp_unlock_scratch(scratch);
252 } else if (dir)
253 ret = crypto_scomp_compress(scomp, src, slen,
254 dst, &dlen, stream->ctx);
255 else
256 ret = crypto_scomp_decompress(scomp, src, slen,
257 dst, &dlen, stream->ctx);
258
259 crypto_acomp_unlock_stream_bh(stream);
260
261 req->dlen = dlen;
262
263 if (!src_isvirt && src)
264 kunmap_local(src);
265 if (!dst_isvirt) {
266 kunmap_local(dst);
267 dlen += doff;
268 for (;;) {
269 flush_dcache_page(dpage);
270 if (dlen <= PAGE_SIZE)
271 break;
272 dlen -= PAGE_SIZE;
273 dpage = nth_page(dpage, 1);
274 }
275 }
276
277 return ret;
278 }
279
scomp_acomp_compress(struct acomp_req * req)280 static int scomp_acomp_compress(struct acomp_req *req)
281 {
282 return scomp_acomp_comp_decomp(req, 1);
283 }
284
scomp_acomp_decompress(struct acomp_req * req)285 static int scomp_acomp_decompress(struct acomp_req *req)
286 {
287 return scomp_acomp_comp_decomp(req, 0);
288 }
289
crypto_exit_scomp_ops_async(struct crypto_tfm * tfm)290 static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm)
291 {
292 struct crypto_scomp **ctx = crypto_tfm_ctx(tfm);
293
294 crypto_free_scomp(*ctx);
295
296 flush_work(&scomp_scratch_work);
297 mutex_lock(&scomp_lock);
298 if (!--scomp_scratch_users)
299 crypto_scomp_free_scratches();
300 mutex_unlock(&scomp_lock);
301 }
302
crypto_init_scomp_ops_async(struct crypto_tfm * tfm)303 int crypto_init_scomp_ops_async(struct crypto_tfm *tfm)
304 {
305 struct crypto_alg *calg = tfm->__crt_alg;
306 struct crypto_acomp *crt = __crypto_acomp_tfm(tfm);
307 struct crypto_scomp **ctx = crypto_tfm_ctx(tfm);
308 struct crypto_scomp *scomp;
309
310 if (!crypto_mod_get(calg))
311 return -EAGAIN;
312
313 scomp = crypto_create_tfm(calg, &crypto_scomp_type);
314 if (IS_ERR(scomp)) {
315 crypto_mod_put(calg);
316 return PTR_ERR(scomp);
317 }
318
319 *ctx = scomp;
320 tfm->exit = crypto_exit_scomp_ops_async;
321
322 crt->compress = scomp_acomp_compress;
323 crt->decompress = scomp_acomp_decompress;
324
325 return 0;
326 }
327
crypto_scomp_destroy(struct crypto_alg * alg)328 static void crypto_scomp_destroy(struct crypto_alg *alg)
329 {
330 struct scomp_alg *scomp = __crypto_scomp_alg(alg);
331
332 crypto_acomp_free_streams(&scomp->streams);
333 }
334
335 static const struct crypto_type crypto_scomp_type = {
336 .extsize = crypto_alg_extsize,
337 .init_tfm = crypto_scomp_init_tfm,
338 .destroy = crypto_scomp_destroy,
339 #ifdef CONFIG_PROC_FS
340 .show = crypto_scomp_show,
341 #endif
342 #if IS_ENABLED(CONFIG_CRYPTO_USER)
343 .report = crypto_scomp_report,
344 #endif
345 .maskclear = ~CRYPTO_ALG_TYPE_MASK,
346 .maskset = CRYPTO_ALG_TYPE_MASK,
347 .type = CRYPTO_ALG_TYPE_SCOMPRESS,
348 .tfmsize = offsetof(struct crypto_scomp, base),
349 .algsize = offsetof(struct scomp_alg, base),
350 };
351
scomp_prepare_alg(struct scomp_alg * alg)352 static void scomp_prepare_alg(struct scomp_alg *alg)
353 {
354 struct crypto_alg *base = &alg->calg.base;
355
356 comp_prepare_alg(&alg->calg);
357
358 base->cra_flags |= CRYPTO_ALG_REQ_VIRT;
359 }
360
crypto_register_scomp(struct scomp_alg * alg)361 int crypto_register_scomp(struct scomp_alg *alg)
362 {
363 struct crypto_alg *base = &alg->calg.base;
364
365 scomp_prepare_alg(alg);
366
367 base->cra_type = &crypto_scomp_type;
368 base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS;
369
370 return crypto_register_alg(base);
371 }
372 EXPORT_SYMBOL_GPL(crypto_register_scomp);
373
crypto_unregister_scomp(struct scomp_alg * alg)374 void crypto_unregister_scomp(struct scomp_alg *alg)
375 {
376 crypto_unregister_alg(&alg->base);
377 }
378 EXPORT_SYMBOL_GPL(crypto_unregister_scomp);
379
crypto_register_scomps(struct scomp_alg * algs,int count)380 int crypto_register_scomps(struct scomp_alg *algs, int count)
381 {
382 int i, ret;
383
384 for (i = 0; i < count; i++) {
385 ret = crypto_register_scomp(&algs[i]);
386 if (ret)
387 goto err;
388 }
389
390 return 0;
391
392 err:
393 for (--i; i >= 0; --i)
394 crypto_unregister_scomp(&algs[i]);
395
396 return ret;
397 }
398 EXPORT_SYMBOL_GPL(crypto_register_scomps);
399
crypto_unregister_scomps(struct scomp_alg * algs,int count)400 void crypto_unregister_scomps(struct scomp_alg *algs, int count)
401 {
402 int i;
403
404 for (i = count - 1; i >= 0; --i)
405 crypto_unregister_scomp(&algs[i]);
406 }
407 EXPORT_SYMBOL_GPL(crypto_unregister_scomps);
408
409 MODULE_LICENSE("GPL");
410 MODULE_DESCRIPTION("Synchronous compression type");
411