xref: /linux/drivers/crypto/intel/qat/qat_common/qat_comp_algs.c (revision 3df692169e8486fc3dd91fcd5ea81c27a0bac033)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright(c) 2022 Intel Corporation */
3 #include <linux/crypto.h>
4 #include <crypto/acompress.h>
5 #include <crypto/internal/acompress.h>
6 #include <crypto/scatterwalk.h>
7 #include <linux/dma-mapping.h>
8 #include <linux/workqueue.h>
9 #include "adf_accel_devices.h"
10 #include "adf_common_drv.h"
11 #include "qat_bl.h"
12 #include "qat_comp_req.h"
13 #include "qat_compression.h"
14 #include "qat_algs_send.h"
15 
16 #define QAT_RFC_1950_HDR_SIZE 2
17 #define QAT_RFC_1950_FOOTER_SIZE 4
18 #define QAT_RFC_1950_CM_DEFLATE 8
19 #define QAT_RFC_1950_CM_DEFLATE_CINFO_32K 7
20 #define QAT_RFC_1950_CM_MASK 0x0f
21 #define QAT_RFC_1950_CM_OFFSET 4
22 #define QAT_RFC_1950_DICT_MASK 0x20
23 #define QAT_RFC_1950_COMP_HDR 0x785e
24 
25 static DEFINE_MUTEX(algs_lock);
26 static unsigned int active_devs;
27 
28 enum direction {
29 	DECOMPRESSION = 0,
30 	COMPRESSION = 1,
31 };
32 
33 struct qat_compression_req;
34 
35 struct qat_compression_ctx {
36 	u8 comp_ctx[QAT_COMP_CTX_SIZE];
37 	struct qat_compression_instance *inst;
38 	int (*qat_comp_callback)(struct qat_compression_req *qat_req, void *resp);
39 };
40 
41 struct qat_dst {
42 	bool is_null;
43 	int resubmitted;
44 };
45 
46 struct qat_compression_req {
47 	u8 req[QAT_COMP_REQ_SIZE];
48 	struct qat_compression_ctx *qat_compression_ctx;
49 	struct acomp_req *acompress_req;
50 	struct qat_request_buffs buf;
51 	enum direction dir;
52 	int actual_dlen;
53 	struct qat_alg_req alg_req;
54 	struct work_struct resubmit;
55 	struct qat_dst dst;
56 };
57 
58 static int qat_alg_send_dc_message(struct qat_compression_req *qat_req,
59 				   struct qat_compression_instance *inst,
60 				   struct crypto_async_request *base)
61 {
62 	struct qat_alg_req *alg_req = &qat_req->alg_req;
63 
64 	alg_req->fw_req = (u32 *)&qat_req->req;
65 	alg_req->tx_ring = inst->dc_tx;
66 	alg_req->base = base;
67 	alg_req->backlog = &inst->backlog;
68 
69 	return qat_alg_send_message(alg_req);
70 }
71 
72 static void qat_comp_resubmit(struct work_struct *work)
73 {
74 	struct qat_compression_req *qat_req =
75 		container_of(work, struct qat_compression_req, resubmit);
76 	struct qat_compression_ctx *ctx = qat_req->qat_compression_ctx;
77 	struct adf_accel_dev *accel_dev = ctx->inst->accel_dev;
78 	struct qat_request_buffs *qat_bufs = &qat_req->buf;
79 	struct qat_compression_instance *inst = ctx->inst;
80 	struct acomp_req *areq = qat_req->acompress_req;
81 	struct crypto_acomp *tfm = crypto_acomp_reqtfm(areq);
82 	unsigned int dlen = CRYPTO_ACOMP_DST_MAX;
83 	u8 *req = qat_req->req;
84 	dma_addr_t dfbuf;
85 	int ret;
86 
87 	areq->dlen = dlen;
88 
89 	dev_dbg(&GET_DEV(accel_dev), "[%s][%s] retry NULL dst request - dlen = %d\n",
90 		crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)),
91 		qat_req->dir == COMPRESSION ? "comp" : "decomp", dlen);
92 
93 	ret = qat_bl_realloc_map_new_dst(accel_dev, &areq->dst, dlen, qat_bufs,
94 					 qat_algs_alloc_flags(&areq->base));
95 	if (ret)
96 		goto err;
97 
98 	qat_req->dst.resubmitted = true;
99 
100 	dfbuf = qat_req->buf.bloutp;
101 	qat_comp_override_dst(req, dfbuf, dlen);
102 
103 	ret = qat_alg_send_dc_message(qat_req, inst, &areq->base);
104 	if (ret != -ENOSPC)
105 		return;
106 
107 err:
108 	qat_bl_free_bufl(accel_dev, qat_bufs);
109 	acomp_request_complete(areq, ret);
110 }
111 
112 static void qat_comp_generic_callback(struct qat_compression_req *qat_req,
113 				      void *resp)
114 {
115 	struct acomp_req *areq = qat_req->acompress_req;
116 	struct qat_compression_ctx *ctx = qat_req->qat_compression_ctx;
117 	struct adf_accel_dev *accel_dev = ctx->inst->accel_dev;
118 	struct crypto_acomp *tfm = crypto_acomp_reqtfm(areq);
119 	struct qat_compression_instance *inst = ctx->inst;
120 	int consumed, produced;
121 	s8 cmp_err, xlt_err;
122 	int res = -EBADMSG;
123 	int status;
124 	u8 cnv;
125 
126 	status = qat_comp_get_cmp_status(resp);
127 	status |= qat_comp_get_xlt_status(resp);
128 	cmp_err = qat_comp_get_cmp_err(resp);
129 	xlt_err = qat_comp_get_xlt_err(resp);
130 
131 	consumed = qat_comp_get_consumed_ctr(resp);
132 	produced = qat_comp_get_produced_ctr(resp);
133 
134 	dev_dbg(&GET_DEV(accel_dev),
135 		"[%s][%s][%s] slen = %8d dlen = %8d consumed = %8d produced = %8d cmp_err = %3d xlt_err = %3d",
136 		crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)),
137 		qat_req->dir == COMPRESSION ? "comp  " : "decomp",
138 		status ? "ERR" : "OK ",
139 		areq->slen, areq->dlen, consumed, produced, cmp_err, xlt_err);
140 
141 	areq->dlen = 0;
142 
143 	if (qat_req->dir == DECOMPRESSION && qat_req->dst.is_null) {
144 		if (cmp_err == ERR_CODE_OVERFLOW_ERROR) {
145 			if (qat_req->dst.resubmitted) {
146 				dev_dbg(&GET_DEV(accel_dev),
147 					"Output does not fit destination buffer\n");
148 				res = -EOVERFLOW;
149 				goto end;
150 			}
151 
152 			INIT_WORK(&qat_req->resubmit, qat_comp_resubmit);
153 			adf_misc_wq_queue_work(&qat_req->resubmit);
154 			return;
155 		}
156 	}
157 
158 	if (unlikely(status != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
159 		goto end;
160 
161 	if (qat_req->dir == COMPRESSION) {
162 		cnv = qat_comp_get_cmp_cnv_flag(resp);
163 		if (unlikely(!cnv)) {
164 			dev_err(&GET_DEV(accel_dev),
165 				"Verified compression not supported\n");
166 			goto end;
167 		}
168 
169 		if (unlikely(produced > qat_req->actual_dlen)) {
170 			memset(inst->dc_data->ovf_buff, 0,
171 			       inst->dc_data->ovf_buff_sz);
172 			dev_dbg(&GET_DEV(accel_dev),
173 				"Actual buffer overflow: produced=%d, dlen=%d\n",
174 				produced, qat_req->actual_dlen);
175 			goto end;
176 		}
177 	}
178 
179 	res = 0;
180 	areq->dlen = produced;
181 
182 	if (ctx->qat_comp_callback)
183 		res = ctx->qat_comp_callback(qat_req, resp);
184 
185 end:
186 	qat_bl_free_bufl(accel_dev, &qat_req->buf);
187 	acomp_request_complete(areq, res);
188 }
189 
190 void qat_comp_alg_callback(void *resp)
191 {
192 	struct qat_compression_req *qat_req =
193 			(void *)(__force long)qat_comp_get_opaque(resp);
194 	struct qat_instance_backlog *backlog = qat_req->alg_req.backlog;
195 
196 	qat_comp_generic_callback(qat_req, resp);
197 
198 	qat_alg_send_backlog(backlog);
199 }
200 
201 static int qat_comp_alg_init_tfm(struct crypto_acomp *acomp_tfm)
202 {
203 	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
204 	struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm);
205 	struct qat_compression_instance *inst;
206 	int node;
207 
208 	if (tfm->node == NUMA_NO_NODE)
209 		node = numa_node_id();
210 	else
211 		node = tfm->node;
212 
213 	memset(ctx, 0, sizeof(*ctx));
214 	inst = qat_compression_get_instance_node(node);
215 	if (!inst)
216 		return -EINVAL;
217 	ctx->inst = inst;
218 
219 	ctx->inst->build_deflate_ctx(ctx->comp_ctx);
220 
221 	return 0;
222 }
223 
224 static void qat_comp_alg_exit_tfm(struct crypto_acomp *acomp_tfm)
225 {
226 	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
227 	struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm);
228 
229 	qat_compression_put_instance(ctx->inst);
230 	memset(ctx, 0, sizeof(*ctx));
231 }
232 
233 static int qat_comp_alg_compress_decompress(struct acomp_req *areq, enum direction dir,
234 					    unsigned int shdr, unsigned int sftr,
235 					    unsigned int dhdr, unsigned int dftr)
236 {
237 	struct qat_compression_req *qat_req = acomp_request_ctx(areq);
238 	struct crypto_acomp *acomp_tfm = crypto_acomp_reqtfm(areq);
239 	struct crypto_tfm *tfm = crypto_acomp_tfm(acomp_tfm);
240 	struct qat_compression_ctx *ctx = crypto_tfm_ctx(tfm);
241 	struct qat_compression_instance *inst = ctx->inst;
242 	gfp_t f = qat_algs_alloc_flags(&areq->base);
243 	struct qat_sgl_to_bufl_params params = {0};
244 	int slen = areq->slen - shdr - sftr;
245 	int dlen = areq->dlen - dhdr - dftr;
246 	dma_addr_t sfbuf, dfbuf;
247 	u8 *req = qat_req->req;
248 	size_t ovf_buff_sz;
249 	int ret;
250 
251 	params.sskip = shdr;
252 	params.dskip = dhdr;
253 
254 	if (!areq->src || !slen)
255 		return -EINVAL;
256 
257 	if (areq->dst && !dlen)
258 		return -EINVAL;
259 
260 	qat_req->dst.is_null = false;
261 
262 	/* Handle acomp requests that require the allocation of a destination
263 	 * buffer. The size of the destination buffer is double the source
264 	 * buffer (rounded up to the size of a page) to fit the decompressed
265 	 * output or an expansion on the data for compression.
266 	 */
267 	if (!areq->dst) {
268 		qat_req->dst.is_null = true;
269 
270 		dlen = round_up(2 * slen, PAGE_SIZE);
271 		areq->dst = sgl_alloc(dlen, f, NULL);
272 		if (!areq->dst)
273 			return -ENOMEM;
274 
275 		dlen -= dhdr + dftr;
276 		areq->dlen = dlen;
277 		qat_req->dst.resubmitted = false;
278 	}
279 
280 	if (dir == COMPRESSION) {
281 		params.extra_dst_buff = inst->dc_data->ovf_buff_p;
282 		ovf_buff_sz = inst->dc_data->ovf_buff_sz;
283 		params.sz_extra_dst_buff = ovf_buff_sz;
284 	}
285 
286 	ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst,
287 				 &qat_req->buf, &params, f);
288 	if (unlikely(ret))
289 		return ret;
290 
291 	sfbuf = qat_req->buf.blp;
292 	dfbuf = qat_req->buf.bloutp;
293 	qat_req->qat_compression_ctx = ctx;
294 	qat_req->acompress_req = areq;
295 	qat_req->dir = dir;
296 
297 	if (dir == COMPRESSION) {
298 		qat_req->actual_dlen = dlen;
299 		dlen += ovf_buff_sz;
300 		qat_comp_create_compression_req(ctx->comp_ctx, req,
301 						(u64)(__force long)sfbuf, slen,
302 						(u64)(__force long)dfbuf, dlen,
303 						(u64)(__force long)qat_req);
304 	} else {
305 		qat_comp_create_decompression_req(ctx->comp_ctx, req,
306 						  (u64)(__force long)sfbuf, slen,
307 						  (u64)(__force long)dfbuf, dlen,
308 						  (u64)(__force long)qat_req);
309 	}
310 
311 	ret = qat_alg_send_dc_message(qat_req, inst, &areq->base);
312 	if (ret == -ENOSPC)
313 		qat_bl_free_bufl(inst->accel_dev, &qat_req->buf);
314 
315 	return ret;
316 }
317 
318 static int qat_comp_alg_compress(struct acomp_req *req)
319 {
320 	return qat_comp_alg_compress_decompress(req, COMPRESSION, 0, 0, 0, 0);
321 }
322 
323 static int qat_comp_alg_decompress(struct acomp_req *req)
324 {
325 	return qat_comp_alg_compress_decompress(req, DECOMPRESSION, 0, 0, 0, 0);
326 }
327 
328 static struct acomp_alg qat_acomp[] = { {
329 	.base = {
330 		.cra_name = "deflate",
331 		.cra_driver_name = "qat_deflate",
332 		.cra_priority = 4001,
333 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
334 		.cra_ctxsize = sizeof(struct qat_compression_ctx),
335 		.cra_module = THIS_MODULE,
336 	},
337 	.init = qat_comp_alg_init_tfm,
338 	.exit = qat_comp_alg_exit_tfm,
339 	.compress = qat_comp_alg_compress,
340 	.decompress = qat_comp_alg_decompress,
341 	.dst_free = sgl_free,
342 	.reqsize = sizeof(struct qat_compression_req),
343 }};
344 
345 int qat_comp_algs_register(void)
346 {
347 	int ret = 0;
348 
349 	mutex_lock(&algs_lock);
350 	if (++active_devs == 1)
351 		ret = crypto_register_acomps(qat_acomp, ARRAY_SIZE(qat_acomp));
352 	mutex_unlock(&algs_lock);
353 	return ret;
354 }
355 
356 void qat_comp_algs_unregister(void)
357 {
358 	mutex_lock(&algs_lock);
359 	if (--active_devs == 0)
360 		crypto_unregister_acomps(qat_acomp, ARRAY_SIZE(qat_acomp));
361 	mutex_unlock(&algs_lock);
362 }
363