1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation */
3 #include <linux/crypto.h>
4 #include <crypto/acompress.h>
5 #include <crypto/internal/acompress.h>
6 #include <crypto/scatterwalk.h>
7 #include <linux/dma-mapping.h>
8 #include <linux/workqueue.h>
9 #include <linux/zstd.h>
10 #include "adf_accel_devices.h"
11 #include "adf_common_drv.h"
12 #include "adf_dc.h"
13 #include "qat_bl.h"
14 #include "qat_comp_req.h"
15 #include "qat_compression.h"
16 #include "qat_algs_send.h"
17 #include "qat_comp_zstd_utils.h"
18
19 #define QAT_ZSTD_SCRATCH_SIZE 524288
20 #define QAT_ZSTD_MAX_BLOCK_SIZE 65535
21 #define QAT_ZSTD_MAX_CONTENT_SIZE 4096
22 #define QAT_LZ4S_MIN_INPUT_SIZE 8192
23 #define QAT_LZ4S_MAX_OUTPUT_SIZE QAT_ZSTD_SCRATCH_SIZE
24 #define QAT_MAX_SEQUENCES (128 * 1024)
25
26 static DEFINE_MUTEX(algs_lock);
27 static unsigned int active_devs_deflate;
28 static unsigned int active_devs_lz4s;
29 static unsigned int active_devs_zstd;
30
31 struct qat_zstd_scratch {
32 size_t cctx_buffer_size;
33 void *lz4s;
34 void *literals;
35 void *out_seqs;
36 void *workspace;
37 ZSTD_CCtx *ctx;
38 };
39
qat_zstd_alloc_scratch(void)40 static void *qat_zstd_alloc_scratch(void)
41 {
42 struct qat_zstd_scratch *scratch;
43 ZSTD_parameters params;
44 size_t cctx_size;
45 ZSTD_CCtx *ctx;
46 size_t zret;
47 int ret;
48
49 ret = -ENOMEM;
50 scratch = kzalloc_obj(*scratch);
51 if (!scratch)
52 return ERR_PTR(ret);
53
54 scratch->lz4s = kvmalloc(QAT_ZSTD_SCRATCH_SIZE, GFP_KERNEL);
55 if (!scratch->lz4s)
56 goto error;
57
58 scratch->literals = kvmalloc(QAT_ZSTD_SCRATCH_SIZE, GFP_KERNEL);
59 if (!scratch->literals)
60 goto error;
61
62 scratch->out_seqs = kvzalloc_objs(ZSTD_Sequence, QAT_MAX_SEQUENCES);
63 if (!scratch->out_seqs)
64 goto error;
65
66 params = zstd_get_params(zstd_max_clevel(), QAT_ZSTD_SCRATCH_SIZE);
67 cctx_size = zstd_cctx_workspace_bound(¶ms.cParams);
68
69 scratch->workspace = kvmalloc(cctx_size, GFP_KERNEL | __GFP_ZERO);
70 if (!scratch->workspace)
71 goto error;
72
73 ret = -EINVAL;
74 ctx = zstd_init_cctx(scratch->workspace, cctx_size);
75 if (!ctx)
76 goto error;
77
78 scratch->ctx = ctx;
79 scratch->cctx_buffer_size = cctx_size;
80
81 zret = zstd_cctx_set_param(ctx, ZSTD_c_blockDelimiters, ZSTD_sf_explicitBlockDelimiters);
82 if (zstd_is_error(zret))
83 goto error;
84
85 return scratch;
86
87 error:
88 kvfree(scratch->lz4s);
89 kvfree(scratch->literals);
90 kvfree(scratch->out_seqs);
91 kvfree(scratch->workspace);
92 kfree(scratch);
93 return ERR_PTR(ret);
94 }
95
qat_zstd_free_scratch(void * ctx)96 static void qat_zstd_free_scratch(void *ctx)
97 {
98 struct qat_zstd_scratch *scratch = ctx;
99
100 if (!scratch)
101 return;
102
103 kvfree(scratch->lz4s);
104 kvfree(scratch->literals);
105 kvfree(scratch->out_seqs);
106 kvfree(scratch->workspace);
107 kfree(scratch);
108 }
109
110 static struct crypto_acomp_streams qat_zstd_streams = {
111 .alloc_ctx = qat_zstd_alloc_scratch,
112 .free_ctx = qat_zstd_free_scratch,
113 };
114
115 enum direction {
116 DECOMPRESSION = 0,
117 COMPRESSION = 1,
118 };
119
120 struct qat_compression_req;
121
122 struct qat_callback_params {
123 unsigned int produced;
124 unsigned int dlen;
125 bool plain;
126 };
127
128 struct qat_compression_ctx {
129 u8 comp_ctx[QAT_COMP_CTX_SIZE];
130 struct qat_compression_instance *inst;
131 int (*qat_comp_callback)(struct qat_compression_req *qat_req, void *resp,
132 struct qat_callback_params *params);
133 struct crypto_acomp *ftfm;
134 };
135
136 struct qat_compression_req {
137 u8 req[QAT_COMP_REQ_SIZE];
138 struct qat_compression_ctx *qat_compression_ctx;
139 struct acomp_req *acompress_req;
140 struct qat_request_buffs buf;
141 enum direction dir;
142 int actual_dlen;
143 struct qat_alg_req alg_req;
144 };
145
qat_alg_send_dc_message(struct qat_compression_req * qat_req,struct qat_compression_instance * inst,struct crypto_async_request * base)146 static int qat_alg_send_dc_message(struct qat_compression_req *qat_req,
147 struct qat_compression_instance *inst,
148 struct crypto_async_request *base)
149 {
150 struct qat_alg_req *alg_req = &qat_req->alg_req;
151
152 alg_req->fw_req = (u32 *)&qat_req->req;
153 alg_req->tx_ring = inst->dc_tx;
154 alg_req->base = base;
155 alg_req->backlog = &inst->backlog;
156
157 return qat_alg_send_message(alg_req);
158 }
159
qat_comp_generic_callback(struct qat_compression_req * qat_req,void * resp)160 static void qat_comp_generic_callback(struct qat_compression_req *qat_req,
161 void *resp)
162 {
163 struct acomp_req *areq = qat_req->acompress_req;
164 struct qat_compression_ctx *ctx = qat_req->qat_compression_ctx;
165 struct adf_accel_dev *accel_dev = ctx->inst->accel_dev;
166 struct crypto_acomp *tfm = crypto_acomp_reqtfm(areq);
167 struct qat_compression_instance *inst = ctx->inst;
168 struct qat_callback_params params = { };
169 int consumed, produced;
170 s8 cmp_err, xlt_err;
171 int res = -EBADMSG;
172 int status;
173 u8 cnv;
174
175 status = qat_comp_get_cmp_status(resp);
176 status |= qat_comp_get_xlt_status(resp);
177 cmp_err = qat_comp_get_cmp_err(resp);
178 xlt_err = qat_comp_get_xlt_err(resp);
179
180 consumed = qat_comp_get_consumed_ctr(resp);
181 produced = qat_comp_get_produced_ctr(resp);
182
183 /* Cache parameters for algorithm specific callback */
184 params.produced = produced;
185 params.dlen = areq->dlen;
186
187 dev_dbg(&GET_DEV(accel_dev),
188 "[%s][%s][%s] slen = %8d dlen = %8d consumed = %8d produced = %8d cmp_err = %3d xlt_err = %3d",
189 crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)),
190 qat_req->dir == COMPRESSION ? "comp " : "decomp",
191 status ? "ERR" : "OK ",
192 areq->slen, areq->dlen, consumed, produced, cmp_err, xlt_err);
193
194 if (unlikely(status != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) {
195 if (cmp_err == ERR_CODE_OVERFLOW_ERROR || xlt_err == ERR_CODE_OVERFLOW_ERROR)
196 res = -E2BIG;
197
198 areq->dlen = 0;
199 goto end;
200 }
201
202 if (qat_req->dir == COMPRESSION) {
203 cnv = qat_comp_get_cmp_cnv_flag(resp);
204 if (unlikely(!cnv)) {
205 dev_err(&GET_DEV(accel_dev),
206 "Verified compression not supported\n");
207 areq->dlen = 0;
208 goto end;
209 }
210
211 if (unlikely(produced > qat_req->actual_dlen)) {
212 memset(inst->dc_data->ovf_buff, 0,
213 inst->dc_data->ovf_buff_sz);
214 dev_dbg(&GET_DEV(accel_dev),
215 "Actual buffer overflow: produced=%d, dlen=%d\n",
216 produced, qat_req->actual_dlen);
217
218 res = -E2BIG;
219 areq->dlen = 0;
220 goto end;
221 }
222
223 params.plain = !!qat_comp_get_cmp_uncomp_flag(resp);
224 }
225
226 res = 0;
227 areq->dlen = produced;
228
229 if (ctx->qat_comp_callback)
230 res = ctx->qat_comp_callback(qat_req, resp, ¶ms);
231
232 end:
233 qat_bl_free_bufl(accel_dev, &qat_req->buf);
234 acomp_request_complete(areq, res);
235 qat_alg_send_backlog(qat_req->alg_req.backlog);
236 }
237
qat_comp_alg_callback(void * resp)238 void qat_comp_alg_callback(void *resp)
239 {
240 struct qat_compression_req *qat_req =
241 (void *)(__force long)qat_comp_get_opaque(resp);
242
243 qat_comp_generic_callback(qat_req, resp);
244 }
245
qat_comp_alg_init_tfm(struct crypto_acomp * acomp_tfm,int alg)246 static int qat_comp_alg_init_tfm(struct crypto_acomp *acomp_tfm, int alg)
247 {
248 struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
249 struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
250 struct qat_compression_instance *inst;
251 int node, ret;
252
253 if (tfm->node == NUMA_NO_NODE)
254 node = numa_node_id();
255 else
256 node = tfm->node;
257
258 memset(ctx, 0, sizeof(*ctx));
259 inst = qat_compression_get_instance_node(node, alg);
260 if (!inst)
261 return -EINVAL;
262 ctx->inst = inst;
263
264 ret = qat_comp_build_ctx(inst->accel_dev, ctx->comp_ctx, alg);
265 if (ret) {
266 qat_compression_put_instance(inst);
267 memset(ctx, 0, sizeof(*ctx));
268 }
269
270 return ret;
271 }
272
qat_comp_alg_deflate_init_tfm(struct crypto_acomp * acomp_tfm)273 static int qat_comp_alg_deflate_init_tfm(struct crypto_acomp *acomp_tfm)
274 {
275 return qat_comp_alg_init_tfm(acomp_tfm, QAT_DEFLATE);
276 }
277
qat_comp_alg_exit_tfm(struct crypto_acomp * acomp_tfm)278 static void qat_comp_alg_exit_tfm(struct crypto_acomp *acomp_tfm)
279 {
280 struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
281
282 qat_compression_put_instance(ctx->inst);
283 memset(ctx, 0, sizeof(*ctx));
284 }
285
qat_comp_alg_compress_decompress(struct acomp_req * areq,enum direction dir,unsigned int shdr,unsigned int sftr,unsigned int dhdr,unsigned int dftr)286 static int qat_comp_alg_compress_decompress(struct acomp_req *areq, enum direction dir,
287 unsigned int shdr, unsigned int sftr,
288 unsigned int dhdr, unsigned int dftr)
289 {
290 struct qat_compression_req *qat_req = acomp_request_ctx(areq);
291 struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(areq);
292 struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
293 struct qat_compression_instance *inst = ctx->inst;
294 gfp_t f = qat_algs_alloc_flags(&areq->base);
295 struct qat_sgl_to_bufl_params params = {0};
296 int slen = areq->slen - shdr - sftr;
297 int dlen = areq->dlen - dhdr - dftr;
298 dma_addr_t sfbuf, dfbuf;
299 u8 *req = qat_req->req;
300 size_t ovf_buff_sz;
301 int ret;
302
303 params.sskip = shdr;
304 params.dskip = dhdr;
305
306 if (!areq->src || !slen)
307 return -EINVAL;
308
309 if (!areq->dst || !dlen)
310 return -EINVAL;
311
312 if (dir == COMPRESSION) {
313 params.extra_dst_buff = inst->dc_data->ovf_buff_p;
314 ovf_buff_sz = inst->dc_data->ovf_buff_sz;
315 params.sz_extra_dst_buff = ovf_buff_sz;
316 }
317
318 ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst,
319 &qat_req->buf, ¶ms, f);
320 if (unlikely(ret))
321 return ret;
322
323 sfbuf = qat_req->buf.blp;
324 dfbuf = qat_req->buf.bloutp;
325 qat_req->qat_compression_ctx = ctx;
326 qat_req->acompress_req = areq;
327 qat_req->dir = dir;
328
329 if (dir == COMPRESSION) {
330 qat_req->actual_dlen = dlen;
331 dlen += ovf_buff_sz;
332 qat_comp_create_compression_req(ctx->comp_ctx, req,
333 (u64)(__force long)sfbuf, slen,
334 (u64)(__force long)dfbuf, dlen,
335 (u64)(__force long)qat_req);
336 } else {
337 qat_comp_create_decompression_req(ctx->comp_ctx, req,
338 (u64)(__force long)sfbuf, slen,
339 (u64)(__force long)dfbuf, dlen,
340 (u64)(__force long)qat_req);
341 }
342
343 ret = qat_alg_send_dc_message(qat_req, inst, &areq->base);
344 if (ret == -ENOSPC)
345 qat_bl_free_bufl(inst->accel_dev, &qat_req->buf);
346
347 return ret;
348 }
349
qat_comp_alg_compress(struct acomp_req * req)350 static int qat_comp_alg_compress(struct acomp_req *req)
351 {
352 return qat_comp_alg_compress_decompress(req, COMPRESSION, 0, 0, 0, 0);
353 }
354
qat_comp_alg_decompress(struct acomp_req * req)355 static int qat_comp_alg_decompress(struct acomp_req *req)
356 {
357 return qat_comp_alg_compress_decompress(req, DECOMPRESSION, 0, 0, 0, 0);
358 }
359
qat_comp_alg_zstd_decompress(struct acomp_req * req)360 static int qat_comp_alg_zstd_decompress(struct acomp_req *req)
361 {
362 struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(req);
363 struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
364 struct acomp_req *nreq = acomp_request_ctx(req);
365 zstd_frame_header header;
366 void *buffer;
367 size_t zret;
368 int ret;
369
370 buffer = kmap_local_page(sg_page(req->src)) + req->src->offset;
371 zret = zstd_get_frame_header(&header, buffer, req->src->length);
372 kunmap_local(buffer);
373 if (zret) {
374 dev_err(&GET_DEV(ctx->inst->accel_dev),
375 "ZSTD-compressed data has an incomplete frame header\n");
376 return -EINVAL;
377 }
378
379 if (header.windowSize > QAT_ZSTD_MAX_BLOCK_SIZE ||
380 header.frameContentSize >= QAT_ZSTD_MAX_CONTENT_SIZE) {
381 dev_dbg(&GET_DEV(ctx->inst->accel_dev), "Window size=0x%llx\n",
382 header.windowSize);
383
384 memcpy(nreq, req, sizeof(*req));
385 acomp_request_set_tfm(nreq, ctx->ftfm);
386
387 ret = crypto_acomp_decompress(nreq);
388 req->dlen = nreq->dlen;
389
390 return ret;
391 }
392
393 return qat_comp_alg_compress_decompress(req, DECOMPRESSION, 0, 0, 0, 0);
394 }
395
qat_comp_lz4s_zstd_callback(struct qat_compression_req * qat_req,void * resp,struct qat_callback_params * params)396 static int qat_comp_lz4s_zstd_callback(struct qat_compression_req *qat_req, void *resp,
397 struct qat_callback_params *params)
398 {
399 struct qat_compression_ctx *qat_ctx = qat_req->qat_compression_ctx;
400 struct acomp_req *areq = qat_req->acompress_req;
401 struct qat_zstd_scratch *scratch;
402 struct crypto_acomp_stream *s;
403 unsigned int lit_len = 0;
404 ZSTD_Sequence *out_seqs;
405 void *lz4s, *zstd;
406 size_t comp_size;
407 ZSTD_CCtx *ctx;
408 void *literals;
409 int seq_count;
410 int ret = 0;
411
412 if (params->produced + QAT_ZSTD_LIT_COPY_LEN > QAT_ZSTD_SCRATCH_SIZE) {
413 dev_dbg(&GET_DEV(qat_ctx->inst->accel_dev),
414 "LZ4s-ZSTD: produced size (%u) + COPY_SIZE > QAT_ZSTD_SCRATCH_SIZE (%u)\n",
415 params->produced, QAT_ZSTD_SCRATCH_SIZE);
416 areq->dlen = 0;
417 return -E2BIG;
418 }
419
420 s = crypto_acomp_lock_stream_bh(&qat_zstd_streams);
421 scratch = s->ctx;
422
423 lz4s = scratch->lz4s;
424 zstd = lz4s; /* Output buffer is same as lz4s */
425 out_seqs = scratch->out_seqs;
426 ctx = scratch->ctx;
427 literals = scratch->literals;
428
429 if (likely(!params->plain)) {
430 if (likely(sg_nents(areq->dst) == 1)) {
431 zstd = sg_virt(areq->dst);
432 lz4s = zstd;
433 } else {
434 memcpy_from_sglist(lz4s, areq->dst, 0, params->produced);
435 }
436
437 seq_count = qat_alg_dec_lz4s(out_seqs, QAT_MAX_SEQUENCES, lz4s,
438 params->produced, literals, &lit_len);
439 if (seq_count < 0) {
440 ret = seq_count;
441 comp_size = 0;
442 goto out;
443 }
444 } else {
445 out_seqs[0].litLength = areq->slen;
446 out_seqs[0].offset = 0;
447 out_seqs[0].matchLength = 0;
448
449 seq_count = 1;
450 }
451
452 comp_size = zstd_compress_sequences_and_literals(ctx, zstd, params->dlen,
453 out_seqs, seq_count,
454 literals, lit_len,
455 QAT_ZSTD_SCRATCH_SIZE,
456 areq->slen);
457 if (zstd_is_error(comp_size)) {
458 if (comp_size == ZSTD_error_cannotProduce_uncompressedBlock)
459 ret = -E2BIG;
460 else
461 ret = -EOPNOTSUPP;
462
463 comp_size = 0;
464 goto out;
465 }
466
467 if (comp_size > params->dlen) {
468 dev_dbg(&GET_DEV(qat_ctx->inst->accel_dev),
469 "LZ4s-ZSTD: compressed_size (%u) > output buffer size (%u)\n",
470 (unsigned int)comp_size, params->dlen);
471 ret = -EOVERFLOW;
472 goto out;
473 }
474
475 if (unlikely(sg_nents(areq->dst) != 1))
476 memcpy_to_sglist(areq->dst, 0, zstd, comp_size);
477
478 out:
479 areq->dlen = comp_size;
480 crypto_acomp_unlock_stream_bh(s);
481
482 return ret;
483 }
484
qat_comp_alg_lz4s_zstd_init_tfm(struct crypto_acomp * acomp_tfm)485 static int qat_comp_alg_lz4s_zstd_init_tfm(struct crypto_acomp *acomp_tfm)
486 {
487 struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
488 struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
489 int reqsize;
490 int ret;
491
492 /* qat_comp_alg_init_tfm() wipes out the ctx */
493 ret = qat_comp_alg_init_tfm(acomp_tfm, QAT_LZ4S);
494 if (ret)
495 return ret;
496
497 ctx->ftfm = crypto_alloc_acomp_node("zstd", 0, CRYPTO_ALG_NEED_FALLBACK,
498 tfm->node);
499 if (IS_ERR(ctx->ftfm)) {
500 qat_comp_alg_exit_tfm(acomp_tfm);
501 return PTR_ERR(ctx->ftfm);
502 }
503
504 reqsize = max(sizeof(struct qat_compression_req),
505 sizeof(struct acomp_req) + crypto_acomp_reqsize(ctx->ftfm));
506
507 acomp_tfm->reqsize = reqsize;
508
509 ctx->qat_comp_callback = qat_comp_lz4s_zstd_callback;
510
511 return 0;
512 }
513
qat_comp_alg_zstd_init_tfm(struct crypto_acomp * acomp_tfm)514 static int qat_comp_alg_zstd_init_tfm(struct crypto_acomp *acomp_tfm)
515 {
516 struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
517 struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
518 int reqsize;
519 int ret;
520
521 /* qat_comp_alg_init_tfm() wipes out the ctx */
522 ret = qat_comp_alg_init_tfm(acomp_tfm, QAT_ZSTD);
523 if (ret)
524 return ret;
525
526 ctx->ftfm = crypto_alloc_acomp_node("zstd", 0, CRYPTO_ALG_NEED_FALLBACK,
527 tfm->node);
528 if (IS_ERR(ctx->ftfm)) {
529 qat_comp_alg_exit_tfm(acomp_tfm);
530 return PTR_ERR(ctx->ftfm);
531 }
532
533 reqsize = max(sizeof(struct qat_compression_req),
534 sizeof(struct acomp_req) + crypto_acomp_reqsize(ctx->ftfm));
535
536 acomp_tfm->reqsize = reqsize;
537
538 return 0;
539 }
540
qat_comp_alg_zstd_exit_tfm(struct crypto_acomp * acomp_tfm)541 static void qat_comp_alg_zstd_exit_tfm(struct crypto_acomp *acomp_tfm)
542 {
543 struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
544
545 if (ctx->ftfm)
546 crypto_free_acomp(ctx->ftfm);
547
548 qat_comp_alg_exit_tfm(acomp_tfm);
549 }
550
qat_comp_alg_lz4s_zstd_compress(struct acomp_req * req)551 static int qat_comp_alg_lz4s_zstd_compress(struct acomp_req *req)
552 {
553 struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(req);
554 struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
555 struct acomp_req *nreq = acomp_request_ctx(req);
556 int ret;
557
558 if (req->slen >= QAT_LZ4S_MIN_INPUT_SIZE && req->dlen >= QAT_LZ4S_MIN_INPUT_SIZE &&
559 req->slen <= QAT_LZ4S_MAX_OUTPUT_SIZE && req->dlen <= QAT_LZ4S_MAX_OUTPUT_SIZE)
560 return qat_comp_alg_compress(req);
561
562 memcpy(nreq, req, sizeof(*req));
563 acomp_request_set_tfm(nreq, ctx->ftfm);
564
565 ret = crypto_acomp_compress(nreq);
566 req->dlen = nreq->dlen;
567
568 return ret;
569 }
570
qat_comp_alg_sw_decompress(struct acomp_req * req)571 static int qat_comp_alg_sw_decompress(struct acomp_req *req)
572 {
573 struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(req);
574 struct qat_compression_ctx *ctx = acomp_tfm_ctx(acomp_tfm);
575 struct acomp_req *nreq = acomp_request_ctx(req);
576 int ret;
577
578 memcpy(nreq, req, sizeof(*req));
579 acomp_request_set_tfm(nreq, ctx->ftfm);
580
581 ret = crypto_acomp_decompress(nreq);
582 req->dlen = nreq->dlen;
583
584 return ret;
585 }
586
587 static struct acomp_alg qat_acomp_deflate[] = { {
588 .base = {
589 .cra_name = "deflate",
590 .cra_driver_name = "qat_deflate",
591 .cra_priority = 4001,
592 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
593 .cra_ctxsize = sizeof(struct qat_compression_ctx),
594 .cra_reqsize = sizeof(struct qat_compression_req),
595 .cra_module = THIS_MODULE,
596 },
597 .init = qat_comp_alg_deflate_init_tfm,
598 .exit = qat_comp_alg_exit_tfm,
599 .compress = qat_comp_alg_compress,
600 .decompress = qat_comp_alg_decompress,
601 }};
602
603 static struct acomp_alg qat_acomp_zstd_lz4s = {
604 .base = {
605 .cra_name = "zstd",
606 .cra_driver_name = "qat_zstd",
607 .cra_priority = 4001,
608 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
609 CRYPTO_ALG_NEED_FALLBACK,
610 .cra_reqsize = sizeof(struct qat_compression_req),
611 .cra_ctxsize = sizeof(struct qat_compression_ctx),
612 .cra_module = THIS_MODULE,
613 },
614 .init = qat_comp_alg_lz4s_zstd_init_tfm,
615 .exit = qat_comp_alg_zstd_exit_tfm,
616 .compress = qat_comp_alg_lz4s_zstd_compress,
617 .decompress = qat_comp_alg_sw_decompress,
618 };
619
620 static struct acomp_alg qat_acomp_zstd_native = {
621 .base = {
622 .cra_name = "zstd",
623 .cra_driver_name = "qat_zstd",
624 .cra_priority = 4001,
625 .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY |
626 CRYPTO_ALG_NEED_FALLBACK,
627 .cra_reqsize = sizeof(struct qat_compression_req),
628 .cra_ctxsize = sizeof(struct qat_compression_ctx),
629 .cra_module = THIS_MODULE,
630 },
631 .init = qat_comp_alg_zstd_init_tfm,
632 .exit = qat_comp_alg_zstd_exit_tfm,
633 .compress = qat_comp_alg_compress,
634 .decompress = qat_comp_alg_zstd_decompress,
635 };
636
qat_comp_algs_register_deflate(void)637 static int qat_comp_algs_register_deflate(void)
638 {
639 int ret = 0;
640
641 mutex_lock(&algs_lock);
642 if (++active_devs_deflate == 1) {
643 ret = crypto_register_acomps(qat_acomp_deflate,
644 ARRAY_SIZE(qat_acomp_deflate));
645 if (ret)
646 active_devs_deflate--;
647 }
648 mutex_unlock(&algs_lock);
649
650 return ret;
651 }
652
qat_comp_algs_unregister_deflate(void)653 static void qat_comp_algs_unregister_deflate(void)
654 {
655 mutex_lock(&algs_lock);
656 if (--active_devs_deflate == 0)
657 crypto_unregister_acomps(qat_acomp_deflate, ARRAY_SIZE(qat_acomp_deflate));
658 mutex_unlock(&algs_lock);
659 }
660
qat_comp_algs_register_lz4s(void)661 static int qat_comp_algs_register_lz4s(void)
662 {
663 int ret = 0;
664
665 mutex_lock(&algs_lock);
666 if (++active_devs_lz4s == 1) {
667 ret = crypto_acomp_alloc_streams(&qat_zstd_streams);
668 if (ret) {
669 active_devs_lz4s--;
670 goto unlock;
671 }
672
673 ret = crypto_register_acomp(&qat_acomp_zstd_lz4s);
674 if (ret) {
675 crypto_acomp_free_streams(&qat_zstd_streams);
676 active_devs_lz4s--;
677 }
678 }
679 unlock:
680 mutex_unlock(&algs_lock);
681
682 return ret;
683 }
684
qat_comp_algs_unregister_lz4s(void)685 static void qat_comp_algs_unregister_lz4s(void)
686 {
687 mutex_lock(&algs_lock);
688 if (--active_devs_lz4s == 0) {
689 crypto_unregister_acomp(&qat_acomp_zstd_lz4s);
690 crypto_acomp_free_streams(&qat_zstd_streams);
691 }
692 mutex_unlock(&algs_lock);
693 }
694
qat_comp_algs_register_zstd(void)695 static int qat_comp_algs_register_zstd(void)
696 {
697 int ret = 0;
698
699 mutex_lock(&algs_lock);
700 if (++active_devs_zstd == 1) {
701 ret = crypto_register_acomp(&qat_acomp_zstd_native);
702 if (ret)
703 active_devs_zstd--;
704 }
705 mutex_unlock(&algs_lock);
706
707 return ret;
708 }
709
qat_comp_algs_unregister_zstd(void)710 static void qat_comp_algs_unregister_zstd(void)
711 {
712 mutex_lock(&algs_lock);
713 if (--active_devs_zstd == 0)
714 crypto_unregister_acomp(&qat_acomp_zstd_native);
715 mutex_unlock(&algs_lock);
716 }
717
qat_comp_algs_register(u32 caps)718 int qat_comp_algs_register(u32 caps)
719 {
720 int ret;
721
722 ret = qat_comp_algs_register_deflate();
723 if (ret)
724 return ret;
725
726 if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S) {
727 ret = qat_comp_algs_register_lz4s();
728 if (ret)
729 goto err_unregister_deflate;
730 }
731
732 if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD) {
733 ret = qat_comp_algs_register_zstd();
734 if (ret)
735 goto err_unregister_lz4s;
736 }
737
738 return ret;
739
740 err_unregister_lz4s:
741 if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S)
742 qat_comp_algs_unregister_lz4s();
743 err_unregister_deflate:
744 qat_comp_algs_unregister_deflate();
745
746 return ret;
747 }
748
qat_comp_algs_unregister(u32 caps)749 void qat_comp_algs_unregister(u32 caps)
750 {
751 qat_comp_algs_unregister_deflate();
752
753 if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S)
754 qat_comp_algs_unregister_lz4s();
755
756 if (caps & ADF_ACCEL_CAPABILITIES_EXT_ZSTD)
757 qat_comp_algs_unregister_zstd();
758 }
759