xref: /linux/crypto/acompress.c (revision bba2c3615bd6cfee7456d1130f2e6b01b3f4e9ba) 
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Asynchronous Compression operations
4  *
5  * Copyright (c) 2016, Intel Corporation
6  * Authors: Weigang Li <weigang.li@intel.com>
7  *          Giovanni Cabiddu <giovanni.cabiddu@intel.com>
8  */
9 
10 #include <crypto/internal/acompress.h>
11 #include <crypto/scatterwalk.h>
12 #include <linux/cryptouser.h>
13 #include <linux/cpumask.h>
14 #include <linux/err.h>
15 #include <linux/kernel.h>
16 #include <linux/module.h>
17 #include <linux/percpu.h>
18 #include <linux/scatterlist.h>
19 #include <linux/sched.h>
20 #include <linux/seq_file.h>
21 #include <linux/smp.h>
22 #include <linux/spinlock.h>
23 #include <linux/string.h>
24 #include <linux/workqueue.h>
25 #include <net/netlink.h>
26 
27 #include "compress.h"
28 
29 struct crypto_scomp;
30 
31 enum {
32 	ACOMP_WALK_SLEEP = 1 << 0,
33 	ACOMP_WALK_SRC_LINEAR = 1 << 1,
34 	ACOMP_WALK_DST_LINEAR = 1 << 2,
35 };
36 
37 static const struct crypto_type crypto_acomp_type;
38 
39 static void acomp_reqchain_done(void *data, int err);
40 
41 static inline struct acomp_alg *__crypto_acomp_alg(struct crypto_alg *alg)
42 {
43 	return container_of(alg, struct acomp_alg, calg.base);
44 }
45 
46 static inline struct acomp_alg *crypto_acomp_alg(struct crypto_acomp *tfm)
47 {
48 	return __crypto_acomp_alg(crypto_acomp_tfm(tfm)->__crt_alg);
49 }
50 
51 static int __maybe_unused crypto_acomp_report(
52 	struct sk_buff *skb, struct crypto_alg *alg)
53 {
54 	struct crypto_report_acomp racomp = {
55 		.type = "acomp",
56 	};
57 
58 	return nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, sizeof(racomp), &racomp);
59 }
60 
61 static void __maybe_unused crypto_acomp_show(struct seq_file *m,
62 					     struct crypto_alg *alg)
63 {
64 	seq_puts(m, "type         : acomp\n");
65 }
66 
67 static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm)
68 {
69 	struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
70 	struct acomp_alg *alg = crypto_acomp_alg(acomp);
71 
72 	if (alg->exit)
73 		alg->exit(acomp);
74 
75 	if (acomp_is_async(acomp))
76 		crypto_free_acomp(crypto_acomp_fb(acomp));
77 }
78 
79 static int crypto_acomp_init_tfm(struct crypto_tfm *tfm)
80 {
81 	struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm);
82 	struct acomp_alg *alg = crypto_acomp_alg(acomp);
83 	struct crypto_acomp *fb = NULL;
84 	int err;
85 
86 	if (tfm->__crt_alg->cra_type != &crypto_acomp_type)
87 		return crypto_init_scomp_ops_async(tfm);
88 
89 	if (acomp_is_async(acomp)) {
90 		fb = crypto_alloc_acomp(crypto_acomp_alg_name(acomp), 0,
91 					CRYPTO_ALG_ASYNC);
92 		if (IS_ERR(fb))
93 			return PTR_ERR(fb);
94 
95 		err = -EINVAL;
96 		if (crypto_acomp_reqsize(fb) > MAX_SYNC_COMP_REQSIZE)
97 			goto out_free_fb;
98 
99 		tfm->fb = crypto_acomp_tfm(fb);
100 	}
101 
102 	acomp->compress = alg->compress;
103 	acomp->decompress = alg->decompress;
104 	acomp->reqsize = alg->base.cra_reqsize;
105 
106 	acomp->base.exit = crypto_acomp_exit_tfm;
107 
108 	if (!alg->init)
109 		return 0;
110 
111 	err = alg->init(acomp);
112 	if (err)
113 		goto out_free_fb;
114 
115 	return 0;
116 
117 out_free_fb:
118 	crypto_free_acomp(fb);
119 	return err;
120 }
121 
122 static unsigned int crypto_acomp_extsize(struct crypto_alg *alg)
123 {
124 	int extsize = crypto_alg_extsize(alg);
125 
126 	if (alg->cra_type != &crypto_acomp_type)
127 		extsize += sizeof(struct crypto_scomp *);
128 
129 	return extsize;
130 }
131 
132 static const struct crypto_type crypto_acomp_type = {
133 	.extsize = crypto_acomp_extsize,
134 	.init_tfm = crypto_acomp_init_tfm,
135 #ifdef CONFIG_PROC_FS
136 	.show = crypto_acomp_show,
137 #endif
138 #if IS_ENABLED(CONFIG_CRYPTO_USER)
139 	.report = crypto_acomp_report,
140 #endif
141 	.maskclear = ~CRYPTO_ALG_TYPE_MASK,
142 	.maskset = CRYPTO_ALG_TYPE_ACOMPRESS_MASK,
143 	.type = CRYPTO_ALG_TYPE_ACOMPRESS,
144 	.tfmsize = offsetof(struct crypto_acomp, base),
145 	.algsize = offsetof(struct acomp_alg, base),
146 };
147 
148 struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type,
149 					u32 mask)
150 {
151 	return crypto_alloc_tfm(alg_name, &crypto_acomp_type, type, mask);
152 }
153 EXPORT_SYMBOL_GPL(crypto_alloc_acomp);
154 
155 struct crypto_acomp *crypto_alloc_acomp_node(const char *alg_name, u32 type,
156 					u32 mask, int node)
157 {
158 	return crypto_alloc_tfm_node(alg_name, &crypto_acomp_type, type, mask,
159 				node);
160 }
161 EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node);
162 
163 static void acomp_save_req(struct acomp_req *req, crypto_completion_t cplt)
164 {
165 	struct acomp_req_chain *state = &req->chain;
166 
167 	state->compl = req->base.complete;
168 	state->data = req->base.data;
169 	req->base.complete = cplt;
170 	req->base.data = req;
171 }
172 
173 static void acomp_restore_req(struct acomp_req *req)
174 {
175 	req->base.complete = req->chain.compl;
176 	req->base.data = req->chain.data;
177 }
178 
179 static void acomp_reqchain_virt(struct acomp_req *req)
180 {
181 	struct acomp_req_chain *state = &req->chain;
182 	unsigned int slen = req->slen;
183 	unsigned int dlen = req->dlen;
184 
185 	if (state->flags & CRYPTO_ACOMP_REQ_SRC_VIRT)
186 		acomp_request_set_src_dma(req, state->src, slen);
187 	if (state->flags & CRYPTO_ACOMP_REQ_DST_VIRT)
188 		acomp_request_set_dst_dma(req, state->dst, dlen);
189 }
190 
191 static void acomp_virt_to_sg(struct acomp_req *req)
192 {
193 	struct acomp_req_chain *state = &req->chain;
194 
195 	state->flags = req->base.flags & (CRYPTO_ACOMP_REQ_SRC_VIRT |
196 					  CRYPTO_ACOMP_REQ_DST_VIRT);
197 
198 	if (acomp_request_src_isvirt(req)) {
199 		unsigned int slen = req->slen;
200 		const u8 *svirt = req->svirt;
201 
202 		state->src = svirt;
203 		sg_init_one(&state->ssg, svirt, slen);
204 		acomp_request_set_src_sg(req, &state->ssg, slen);
205 	}
206 
207 	if (acomp_request_dst_isvirt(req)) {
208 		unsigned int dlen = req->dlen;
209 		u8 *dvirt = req->dvirt;
210 
211 		state->dst = dvirt;
212 		sg_init_one(&state->dsg, dvirt, dlen);
213 		acomp_request_set_dst_sg(req, &state->dsg, dlen);
214 	}
215 }
216 
217 static int acomp_do_nondma(struct acomp_req *req, bool comp)
218 {
219 	ACOMP_FBREQ_ON_STACK(fbreq, req);
220 	int err;
221 
222 	if (comp)
223 		err = crypto_acomp_compress(fbreq);
224 	else
225 		err = crypto_acomp_decompress(fbreq);
226 
227 	req->dlen = fbreq->dlen;
228 	return err;
229 }
230 
231 static int acomp_do_one_req(struct acomp_req *req, bool comp)
232 {
233 	if (acomp_request_isnondma(req))
234 		return acomp_do_nondma(req, comp);
235 
236 	acomp_virt_to_sg(req);
237 	return comp ? crypto_acomp_reqtfm(req)->compress(req) :
238 		      crypto_acomp_reqtfm(req)->decompress(req);
239 }
240 
241 static int acomp_reqchain_finish(struct acomp_req *req, int err)
242 {
243 	acomp_reqchain_virt(req);
244 	acomp_restore_req(req);
245 	return err;
246 }
247 
248 static void acomp_reqchain_done(void *data, int err)
249 {
250 	struct acomp_req *req = data;
251 	crypto_completion_t compl;
252 
253 	compl = req->chain.compl;
254 	data = req->chain.data;
255 
256 	if (err == -EINPROGRESS)
257 		goto notify;
258 
259 	err = acomp_reqchain_finish(req, err);
260 
261 notify:
262 	compl(data, err);
263 }
264 
265 static int acomp_do_req_chain(struct acomp_req *req, bool comp)
266 {
267 	int err;
268 
269 	acomp_save_req(req, acomp_reqchain_done);
270 
271 	err = acomp_do_one_req(req, comp);
272 	if (err == -EBUSY || err == -EINPROGRESS)
273 		return err;
274 
275 	return acomp_reqchain_finish(req, err);
276 }
277 
278 int crypto_acomp_compress(struct acomp_req *req)
279 {
280 	struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
281 
282 	if (acomp_req_on_stack(req) && acomp_is_async(tfm))
283 		return -EAGAIN;
284 	if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req))
285 		return crypto_acomp_reqtfm(req)->compress(req);
286 	return acomp_do_req_chain(req, true);
287 }
288 EXPORT_SYMBOL_GPL(crypto_acomp_compress);
289 
290 int crypto_acomp_decompress(struct acomp_req *req)
291 {
292 	struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
293 
294 	if (acomp_req_on_stack(req) && acomp_is_async(tfm))
295 		return -EAGAIN;
296 	if (crypto_acomp_req_virt(tfm) || acomp_request_issg(req))
297 		return crypto_acomp_reqtfm(req)->decompress(req);
298 	return acomp_do_req_chain(req, false);
299 }
300 EXPORT_SYMBOL_GPL(crypto_acomp_decompress);
301 
302 void comp_prepare_alg(struct comp_alg_common *alg)
303 {
304 	struct crypto_alg *base = &alg->base;
305 
306 	base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK;
307 }
308 
309 int crypto_register_acomp(struct acomp_alg *alg)
310 {
311 	struct crypto_alg *base = &alg->calg.base;
312 
313 	comp_prepare_alg(&alg->calg);
314 
315 	base->cra_type = &crypto_acomp_type;
316 	base->cra_flags |= CRYPTO_ALG_TYPE_ACOMPRESS;
317 
318 	return crypto_register_alg(base);
319 }
320 EXPORT_SYMBOL_GPL(crypto_register_acomp);
321 
322 void crypto_unregister_acomp(struct acomp_alg *alg)
323 {
324 	crypto_unregister_alg(&alg->base);
325 }
326 EXPORT_SYMBOL_GPL(crypto_unregister_acomp);
327 
328 int crypto_register_acomps(struct acomp_alg *algs, int count)
329 {
330 	int i, ret;
331 
332 	for (i = 0; i < count; i++) {
333 		ret = crypto_register_acomp(&algs[i]);
334 		if (ret) {
335 			crypto_unregister_acomps(algs, i);
336 			return ret;
337 		}
338 	}
339 
340 	return 0;
341 }
342 EXPORT_SYMBOL_GPL(crypto_register_acomps);
343 
344 void crypto_unregister_acomps(struct acomp_alg *algs, int count)
345 {
346 	int i;
347 
348 	for (i = count - 1; i >= 0; --i)
349 		crypto_unregister_acomp(&algs[i]);
350 }
351 EXPORT_SYMBOL_GPL(crypto_unregister_acomps);
352 
353 static void acomp_stream_workfn(struct work_struct *work)
354 {
355 	struct crypto_acomp_streams *s =
356 		container_of(work, struct crypto_acomp_streams, stream_work);
357 	struct crypto_acomp_stream __percpu *streams = s->streams;
358 	int cpu;
359 
360 	for_each_cpu(cpu, &s->stream_want) {
361 		struct crypto_acomp_stream *ps;
362 		void *ctx;
363 
364 		ps = per_cpu_ptr(streams, cpu);
365 		if (ps->ctx)
366 			continue;
367 
368 		ctx = s->alloc_ctx();
369 		if (IS_ERR(ctx))
370 			break;
371 
372 		spin_lock_bh(&ps->lock);
373 		ps->ctx = ctx;
374 		spin_unlock_bh(&ps->lock);
375 
376 		cpumask_clear_cpu(cpu, &s->stream_want);
377 	}
378 }
379 
380 void crypto_acomp_free_streams(struct crypto_acomp_streams *s)
381 {
382 	struct crypto_acomp_stream __percpu *streams = s->streams;
383 	void (*free_ctx)(void *);
384 	int i;
385 
386 	s->streams = NULL;
387 	if (!streams)
388 		return;
389 
390 	cancel_work_sync(&s->stream_work);
391 	free_ctx = s->free_ctx;
392 
393 	for_each_possible_cpu(i) {
394 		struct crypto_acomp_stream *ps = per_cpu_ptr(streams, i);
395 
396 		if (!ps->ctx)
397 			continue;
398 
399 		free_ctx(ps->ctx);
400 	}
401 
402 	free_percpu(streams);
403 }
404 EXPORT_SYMBOL_GPL(crypto_acomp_free_streams);
405 
406 int crypto_acomp_alloc_streams(struct crypto_acomp_streams *s)
407 {
408 	struct crypto_acomp_stream __percpu *streams;
409 	struct crypto_acomp_stream *ps;
410 	unsigned int i;
411 	void *ctx;
412 
413 	if (s->streams)
414 		return 0;
415 
416 	streams = alloc_percpu(struct crypto_acomp_stream);
417 	if (!streams)
418 		return -ENOMEM;
419 
420 	ctx = s->alloc_ctx();
421 	if (IS_ERR(ctx)) {
422 		free_percpu(streams);
423 		return PTR_ERR(ctx);
424 	}
425 
426 	i = cpumask_first(cpu_possible_mask);
427 	ps = per_cpu_ptr(streams, i);
428 	ps->ctx = ctx;
429 
430 	for_each_possible_cpu(i) {
431 		ps = per_cpu_ptr(streams, i);
432 		spin_lock_init(&ps->lock);
433 	}
434 
435 	s->streams = streams;
436 
437 	INIT_WORK(&s->stream_work, acomp_stream_workfn);
438 	return 0;
439 }
440 EXPORT_SYMBOL_GPL(crypto_acomp_alloc_streams);
441 
442 struct crypto_acomp_stream *_crypto_acomp_lock_stream_bh(
443 	struct crypto_acomp_streams *s)
444 {
445 	struct crypto_acomp_stream __percpu *streams = s->streams;
446 	int cpu = raw_smp_processor_id();
447 	struct crypto_acomp_stream *ps;
448 
449 	ps = per_cpu_ptr(streams, cpu);
450 	spin_lock_bh(&ps->lock);
451 	if (likely(ps->ctx))
452 		return ps;
453 	spin_unlock(&ps->lock);
454 
455 	cpumask_set_cpu(cpu, &s->stream_want);
456 	schedule_work(&s->stream_work);
457 
458 	ps = per_cpu_ptr(streams, cpumask_first(cpu_possible_mask));
459 	spin_lock(&ps->lock);
460 	return ps;
461 }
462 EXPORT_SYMBOL_GPL(_crypto_acomp_lock_stream_bh);
463 
464 void acomp_walk_done_src(struct acomp_walk *walk, int used)
465 {
466 	walk->slen -= used;
467 	if ((walk->flags & ACOMP_WALK_SRC_LINEAR))
468 		scatterwalk_advance(&walk->in, used);
469 	else
470 		scatterwalk_done_src(&walk->in, used);
471 
472 	if ((walk->flags & ACOMP_WALK_SLEEP))
473 		cond_resched();
474 }
475 EXPORT_SYMBOL_GPL(acomp_walk_done_src);
476 
477 void acomp_walk_done_dst(struct acomp_walk *walk, int used)
478 {
479 	walk->dlen -= used;
480 	if ((walk->flags & ACOMP_WALK_DST_LINEAR))
481 		scatterwalk_advance(&walk->out, used);
482 	else
483 		scatterwalk_done_dst(&walk->out, used);
484 
485 	if ((walk->flags & ACOMP_WALK_SLEEP))
486 		cond_resched();
487 }
488 EXPORT_SYMBOL_GPL(acomp_walk_done_dst);
489 
490 int acomp_walk_next_src(struct acomp_walk *walk)
491 {
492 	unsigned int slen = walk->slen;
493 	unsigned int max = UINT_MAX;
494 
495 	if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP))
496 		max = PAGE_SIZE;
497 	if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) {
498 		walk->in.__addr = (void *)(((u8 *)walk->in.sg) +
499 					   walk->in.offset);
500 		return min(slen, max);
501 	}
502 
503 	return slen ? scatterwalk_next(&walk->in, slen) : 0;
504 }
505 EXPORT_SYMBOL_GPL(acomp_walk_next_src);
506 
507 int acomp_walk_next_dst(struct acomp_walk *walk)
508 {
509 	unsigned int dlen = walk->dlen;
510 	unsigned int max = UINT_MAX;
511 
512 	if (!preempt_model_preemptible() && (walk->flags & ACOMP_WALK_SLEEP))
513 		max = PAGE_SIZE;
514 	if ((walk->flags & ACOMP_WALK_DST_LINEAR)) {
515 		walk->out.__addr = (void *)(((u8 *)walk->out.sg) +
516 					    walk->out.offset);
517 		return min(dlen, max);
518 	}
519 
520 	return dlen ? scatterwalk_next(&walk->out, dlen) : 0;
521 }
522 EXPORT_SYMBOL_GPL(acomp_walk_next_dst);
523 
524 int acomp_walk_virt(struct acomp_walk *__restrict walk,
525 		    struct acomp_req *__restrict req, bool atomic)
526 {
527 	struct scatterlist *src = req->src;
528 	struct scatterlist *dst = req->dst;
529 
530 	walk->slen = req->slen;
531 	walk->dlen = req->dlen;
532 
533 	if (!walk->slen || !walk->dlen)
534 		return -EINVAL;
535 
536 	walk->flags = 0;
537 	if ((req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) && !atomic)
538 		walk->flags |= ACOMP_WALK_SLEEP;
539 	if ((req->base.flags & CRYPTO_ACOMP_REQ_SRC_VIRT))
540 		walk->flags |= ACOMP_WALK_SRC_LINEAR;
541 	if ((req->base.flags & CRYPTO_ACOMP_REQ_DST_VIRT))
542 		walk->flags |= ACOMP_WALK_DST_LINEAR;
543 
544 	if ((walk->flags & ACOMP_WALK_SRC_LINEAR)) {
545 		walk->in.sg = (void *)req->svirt;
546 		walk->in.offset = 0;
547 	} else
548 		scatterwalk_start(&walk->in, src);
549 	if ((walk->flags & ACOMP_WALK_DST_LINEAR)) {
550 		walk->out.sg = (void *)req->dvirt;
551 		walk->out.offset = 0;
552 	} else
553 		scatterwalk_start(&walk->out, dst);
554 
555 	return 0;
556 }
557 EXPORT_SYMBOL_GPL(acomp_walk_virt);
558 
559 struct acomp_req *acomp_request_clone(struct acomp_req *req,
560 				      size_t total, gfp_t gfp)
561 {
562 	struct acomp_req *nreq;
563 
564 	nreq = container_of(crypto_request_clone(&req->base, total, gfp),
565 			    struct acomp_req, base);
566 	if (nreq == req)
567 		return req;
568 
569 	if (req->src == &req->chain.ssg)
570 		nreq->src = &nreq->chain.ssg;
571 	if (req->dst == &req->chain.dsg)
572 		nreq->dst = &nreq->chain.dsg;
573 	return nreq;
574 }
575 EXPORT_SYMBOL_GPL(acomp_request_clone);
576 
577 MODULE_LICENSE("GPL");
578 MODULE_DESCRIPTION("Asynchronous compression type");
579