Lines Matching +full:k +full:- +full:to +full:- +full:j
1 // SPDX-License-Identifier: GPL-2.0
3 * sun8i-ss-hash.c - hardware cryptographic offloader for
6 * Copyright (C) 2015-2020 Corentin Labbe <clabbe@baylibre.com>
20 #include <linux/dma-mapping.h>
27 #include "sun8i-ss.h"
39 ret = crypto_shash_tfm_digest(xtfm, key, keylen, tfmctx->key); in sun8i_ss_hashkey()
41 dev_err(tfmctx->ss->dev, "shash digest error ret=%d\n", ret); in sun8i_ss_hashkey()
61 tfmctx->keylen = digestsize; in sun8i_ss_hmac_setkey()
63 tfmctx->keylen = keylen; in sun8i_ss_hmac_setkey()
64 memcpy(tfmctx->key, key, keylen); in sun8i_ss_hmac_setkey()
67 tfmctx->ipad = kzalloc(bs, GFP_KERNEL); in sun8i_ss_hmac_setkey()
68 if (!tfmctx->ipad) in sun8i_ss_hmac_setkey()
69 return -ENOMEM; in sun8i_ss_hmac_setkey()
70 tfmctx->opad = kzalloc(bs, GFP_KERNEL); in sun8i_ss_hmac_setkey()
71 if (!tfmctx->opad) { in sun8i_ss_hmac_setkey()
72 ret = -ENOMEM; in sun8i_ss_hmac_setkey()
76 memset(tfmctx->key + tfmctx->keylen, 0, bs - tfmctx->keylen); in sun8i_ss_hmac_setkey()
77 memcpy(tfmctx->ipad, tfmctx->key, tfmctx->keylen); in sun8i_ss_hmac_setkey()
78 memcpy(tfmctx->opad, tfmctx->key, tfmctx->keylen); in sun8i_ss_hmac_setkey()
80 tfmctx->ipad[i] ^= HMAC_IPAD_VALUE; in sun8i_ss_hmac_setkey()
81 tfmctx->opad[i] ^= HMAC_OPAD_VALUE; in sun8i_ss_hmac_setkey()
84 ret = crypto_ahash_setkey(tfmctx->fallback_tfm, key, keylen); in sun8i_ss_hmac_setkey()
88 memzero_explicit(tfmctx->key, keylen); in sun8i_ss_hmac_setkey()
89 kfree_sensitive(tfmctx->opad); in sun8i_ss_hmac_setkey()
91 kfree_sensitive(tfmctx->ipad); in sun8i_ss_hmac_setkey()
103 op->ss = algt->ss; in sun8i_ss_hash_init_tfm()
106 op->fallback_tfm = crypto_alloc_ahash(crypto_ahash_alg_name(tfm), 0, in sun8i_ss_hash_init_tfm()
108 if (IS_ERR(op->fallback_tfm)) { in sun8i_ss_hash_init_tfm()
109 dev_err(algt->ss->dev, "Fallback driver could no be loaded\n"); in sun8i_ss_hash_init_tfm()
110 return PTR_ERR(op->fallback_tfm); in sun8i_ss_hash_init_tfm()
114 crypto_ahash_statesize(op->fallback_tfm)); in sun8i_ss_hash_init_tfm()
118 crypto_ahash_reqsize(op->fallback_tfm)); in sun8i_ss_hash_init_tfm()
120 memcpy(algt->fbname, crypto_ahash_driver_name(op->fallback_tfm), in sun8i_ss_hash_init_tfm()
123 err = pm_runtime_get_sync(op->ss->dev); in sun8i_ss_hash_init_tfm()
128 pm_runtime_put_noidle(op->ss->dev); in sun8i_ss_hash_init_tfm()
129 crypto_free_ahash(op->fallback_tfm); in sun8i_ss_hash_init_tfm()
137 kfree_sensitive(tfmctx->ipad); in sun8i_ss_hash_exit_tfm()
138 kfree_sensitive(tfmctx->opad); in sun8i_ss_hash_exit_tfm()
140 crypto_free_ahash(tfmctx->fallback_tfm); in sun8i_ss_hash_exit_tfm()
141 pm_runtime_put_sync_suspend(tfmctx->ss->dev); in sun8i_ss_hash_exit_tfm()
152 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); in sun8i_ss_hash_init()
153 ahash_request_set_callback(&rctx->fallback_req, in sun8i_ss_hash_init()
154 areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, in sun8i_ss_hash_init()
155 areq->base.complete, areq->base.data); in sun8i_ss_hash_init()
157 return crypto_ahash_init(&rctx->fallback_req); in sun8i_ss_hash_init()
166 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); in sun8i_ss_hash_export()
167 ahash_request_set_callback(&rctx->fallback_req, in sun8i_ss_hash_export()
168 areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, in sun8i_ss_hash_export()
169 areq->base.complete, areq->base.data); in sun8i_ss_hash_export()
171 return crypto_ahash_export(&rctx->fallback_req, out); in sun8i_ss_hash_export()
180 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); in sun8i_ss_hash_import()
181 ahash_request_set_callback(&rctx->fallback_req, in sun8i_ss_hash_import()
182 areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, in sun8i_ss_hash_import()
183 areq->base.complete, areq->base.data); in sun8i_ss_hash_import()
185 return crypto_ahash_import(&rctx->fallback_req, in); in sun8i_ss_hash_import()
194 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); in sun8i_ss_hash_final()
195 ahash_request_set_callback(&rctx->fallback_req, in sun8i_ss_hash_final()
196 areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, in sun8i_ss_hash_final()
197 areq->base.complete, areq->base.data); in sun8i_ss_hash_final()
198 ahash_request_set_crypt(&rctx->fallback_req, NULL, areq->result, 0); in sun8i_ss_hash_final()
208 algt->stat_fb++; in sun8i_ss_hash_final()
212 return crypto_ahash_final(&rctx->fallback_req); in sun8i_ss_hash_final()
221 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); in sun8i_ss_hash_update()
222 ahash_request_set_callback(&rctx->fallback_req, in sun8i_ss_hash_update()
223 areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, in sun8i_ss_hash_update()
224 areq->base.complete, areq->base.data); in sun8i_ss_hash_update()
225 ahash_request_set_crypt(&rctx->fallback_req, areq->src, NULL, areq->nbytes); in sun8i_ss_hash_update()
227 return crypto_ahash_update(&rctx->fallback_req); in sun8i_ss_hash_update()
236 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); in sun8i_ss_hash_finup()
237 ahash_request_set_callback(&rctx->fallback_req, in sun8i_ss_hash_finup()
238 areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, in sun8i_ss_hash_finup()
239 areq->base.complete, areq->base.data); in sun8i_ss_hash_finup()
240 ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, in sun8i_ss_hash_finup()
241 areq->nbytes); in sun8i_ss_hash_finup()
251 algt->stat_fb++; in sun8i_ss_hash_finup()
255 return crypto_ahash_finup(&rctx->fallback_req); in sun8i_ss_hash_finup()
264 ahash_request_set_tfm(&rctx->fallback_req, tfmctx->fallback_tfm); in sun8i_ss_hash_digest_fb()
265 ahash_request_set_callback(&rctx->fallback_req, in sun8i_ss_hash_digest_fb()
266 areq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP, in sun8i_ss_hash_digest_fb()
267 areq->base.complete, areq->base.data); in sun8i_ss_hash_digest_fb()
268 ahash_request_set_crypt(&rctx->fallback_req, areq->src, areq->result, in sun8i_ss_hash_digest_fb()
269 areq->nbytes); in sun8i_ss_hash_digest_fb()
279 algt->stat_fb++; in sun8i_ss_hash_digest_fb()
283 return crypto_ahash_digest(&rctx->fallback_req); in sun8i_ss_hash_digest_fb()
290 int flow = rctx->flow; in sun8i_ss_run_hash_task()
295 ss->flows[flow].stat_req++; in sun8i_ss_run_hash_task()
304 v |= rctx->method; in sun8i_ss_run_hash_task()
307 if (!rctx->t_dst[i].addr) in sun8i_ss_run_hash_task()
310 mutex_lock(&ss->mlock); in sun8i_ss_run_hash_task()
313 writel(rctx->t_dst[i - 1].addr, ss->base + SS_KEY_ADR_REG); in sun8i_ss_run_hash_task()
314 writel(rctx->t_dst[i - 1].addr, ss->base + SS_IV_ADR_REG); in sun8i_ss_run_hash_task()
317 dev_dbg(ss->dev, in sun8i_ss_run_hash_task()
318 "Processing SG %d on flow %d %s ctl=%x %d to %d method=%x src=%x dst=%x\n", in sun8i_ss_run_hash_task()
320 rctx->t_src[i].len, rctx->t_dst[i].len, in sun8i_ss_run_hash_task()
321 rctx->method, rctx->t_src[i].addr, rctx->t_dst[i].addr); in sun8i_ss_run_hash_task()
323 writel(rctx->t_src[i].addr, ss->base + SS_SRC_ADR_REG); in sun8i_ss_run_hash_task()
324 writel(rctx->t_dst[i].addr, ss->base + SS_DST_ADR_REG); in sun8i_ss_run_hash_task()
325 writel(rctx->t_src[i].len, ss->base + SS_LEN_ADR_REG); in sun8i_ss_run_hash_task()
326 writel(BIT(0) | BIT(1), ss->base + SS_INT_CTL_REG); in sun8i_ss_run_hash_task()
328 reinit_completion(&ss->flows[flow].complete); in sun8i_ss_run_hash_task()
329 ss->flows[flow].status = 0; in sun8i_ss_run_hash_task()
332 writel(v, ss->base + SS_CTL_REG); in sun8i_ss_run_hash_task()
333 mutex_unlock(&ss->mlock); in sun8i_ss_run_hash_task()
334 wait_for_completion_interruptible_timeout(&ss->flows[flow].complete, in sun8i_ss_run_hash_task()
336 if (ss->flows[flow].status == 0) { in sun8i_ss_run_hash_task()
337 dev_err(ss->dev, "DMA timeout for %s\n", name); in sun8i_ss_run_hash_task()
338 return -EFAULT; in sun8i_ss_run_hash_task()
354 if (areq->nbytes == 0) { in sun8i_ss_hash_need_fallback()
355 algt->stat_fb_len++; in sun8i_ss_hash_need_fallback()
359 if (areq->nbytes >= MAX_PAD_SIZE - 64) { in sun8i_ss_hash_need_fallback()
360 algt->stat_fb_len++; in sun8i_ss_hash_need_fallback()
364 /* we need to reserve one SG for the padding one */ in sun8i_ss_hash_need_fallback()
365 if (sg_nents(areq->src) > MAX_SG - 1) { in sun8i_ss_hash_need_fallback()
366 algt->stat_fb_sgnum++; in sun8i_ss_hash_need_fallback()
370 sg = areq->src; in sun8i_ss_hash_need_fallback()
376 /* Only the last block could be bounced to the pad buffer */ in sun8i_ss_hash_need_fallback()
377 if (sg->length % 64 && sg_next(sg)) { in sun8i_ss_hash_need_fallback()
378 algt->stat_fb_sglen++; in sun8i_ss_hash_need_fallback()
381 if (!IS_ALIGNED(sg->offset, sizeof(u32))) { in sun8i_ss_hash_need_fallback()
382 algt->stat_fb_align++; in sun8i_ss_hash_need_fallback()
385 if (sg->length % 4) { in sun8i_ss_hash_need_fallback()
386 algt->stat_fb_sglen++; in sun8i_ss_hash_need_fallback()
408 ss = algt->ss; in sun8i_ss_hash_digest()
411 rctx->flow = e; in sun8i_ss_hash_digest()
412 engine = ss->flows[e].engine; in sun8i_ss_hash_digest()
419 u64 fill, min_fill, j, k; in hash_pad() local
423 j = padi; in hash_pad()
424 buf[j++] = cpu_to_le32(0x80); in hash_pad()
427 fill = 64 - (byte_count % 64); in hash_pad()
430 fill = 128 - (byte_count % 128); in hash_pad()
437 k = j; in hash_pad()
438 j += (fill - min_fill) / sizeof(u32); in hash_pad()
439 if (j * 4 > bufsize) { in hash_pad()
440 pr_err("%s OVERFLOW %llu\n", __func__, j); in hash_pad()
443 for (; k < j; k++) in hash_pad()
444 buf[k] = 0; in hash_pad()
448 lebits = (__le64 *)&buf[j]; in hash_pad()
450 j += 2; in hash_pad()
454 bebits = (__be64 *)&buf[j]; in hash_pad()
456 j += 2; in hash_pad()
459 bebits = (__be64 *)&buf[j]; in hash_pad()
461 j += 2; in hash_pad()
462 bebits = (__be64 *)&buf[j]; in hash_pad()
464 j += 2; in hash_pad()
467 if (j * 4 > bufsize) { in hash_pad()
468 pr_err("%s OVERFLOW %llu\n", __func__, j); in hash_pad()
472 return j; in hash_pad()
475 /* sun8i_ss_hash_run - run an ahash request
476 * Send the data of the request to the SS along with an extra SG with padding
493 int j, i, k, todo; in sun8i_ss_hash_run() local
504 ss = algt->ss; in sun8i_ss_hash_run()
505 j = 0; in sun8i_ss_hash_run()
511 result = ss->flows[rctx->flow].result; in sun8i_ss_hash_run()
512 pad = ss->flows[rctx->flow].pad; in sun8i_ss_hash_run()
516 rctx->t_dst[i].addr = 0; in sun8i_ss_hash_run()
517 rctx->t_dst[i].len = 0; in sun8i_ss_hash_run()
521 algt->stat_req++; in sun8i_ss_hash_run()
524 rctx->method = ss->variant->alg_hash[algt->ss_algo_id]; in sun8i_ss_hash_run()
526 nr_sgs = dma_map_sg(ss->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); in sun8i_ss_hash_run()
528 dev_err(ss->dev, "Invalid sg number %d\n", nr_sgs); in sun8i_ss_hash_run()
529 err = -EINVAL; in sun8i_ss_hash_run()
533 addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE); in sun8i_ss_hash_run()
534 if (dma_mapping_error(ss->dev, addr_res)) { in sun8i_ss_hash_run()
535 dev_err(ss->dev, "DMA map dest\n"); in sun8i_ss_hash_run()
536 err = -EINVAL; in sun8i_ss_hash_run()
540 len = areq->nbytes; in sun8i_ss_hash_run()
541 sg = areq->src; in sun8i_ss_hash_run()
551 rctx->t_src[i].addr = sg_dma_address(sg); in sun8i_ss_hash_run()
552 rctx->t_src[i].len = todo / 4; in sun8i_ss_hash_run()
553 rctx->t_dst[i].addr = addr_res; in sun8i_ss_hash_run()
554 rctx->t_dst[i].len = digestsize / 4; in sun8i_ss_hash_run()
555 len -= todo; in sun8i_ss_hash_run()
558 j += todo / 4; in sun8i_ss_hash_run()
559 len -= todo; in sun8i_ss_hash_run()
565 dev_err(ss->dev, "remaining len %d\n", len); in sun8i_ss_hash_run()
566 err = -EINVAL; in sun8i_ss_hash_run()
570 if (j > 0) in sun8i_ss_hash_run()
571 i--; in sun8i_ss_hash_run()
574 byte_count = areq->nbytes; in sun8i_ss_hash_run()
575 if (tfmctx->keylen && hmac == 0) { in sun8i_ss_hash_run()
577 /* shift all SG one slot up, to free slot 0 for IPAD */ in sun8i_ss_hash_run()
578 for (k = 6; k >= 0; k--) { in sun8i_ss_hash_run()
579 rctx->t_src[k + 1].addr = rctx->t_src[k].addr; in sun8i_ss_hash_run()
580 rctx->t_src[k + 1].len = rctx->t_src[k].len; in sun8i_ss_hash_run()
581 rctx->t_dst[k + 1].addr = rctx->t_dst[k].addr; in sun8i_ss_hash_run()
582 rctx->t_dst[k + 1].len = rctx->t_dst[k].len; in sun8i_ss_hash_run()
584 addr_xpad = dma_map_single(ss->dev, tfmctx->ipad, bs, DMA_TO_DEVICE); in sun8i_ss_hash_run()
585 err = dma_mapping_error(ss->dev, addr_xpad); in sun8i_ss_hash_run()
587 dev_err(ss->dev, "Fail to create DMA mapping of ipad\n"); in sun8i_ss_hash_run()
590 rctx->t_src[0].addr = addr_xpad; in sun8i_ss_hash_run()
591 rctx->t_src[0].len = bs / 4; in sun8i_ss_hash_run()
592 rctx->t_dst[0].addr = addr_res; in sun8i_ss_hash_run()
593 rctx->t_dst[0].len = digestsize / 4; in sun8i_ss_hash_run()
595 byte_count = areq->nbytes + bs; in sun8i_ss_hash_run()
597 if (tfmctx->keylen && hmac == 2) { in sun8i_ss_hash_run()
599 rctx->t_src[i].addr = 0; in sun8i_ss_hash_run()
600 rctx->t_src[i].len = 0; in sun8i_ss_hash_run()
601 rctx->t_dst[i].addr = 0; in sun8i_ss_hash_run()
602 rctx->t_dst[i].len = 0; in sun8i_ss_hash_run()
605 addr_res = dma_map_single(ss->dev, result, digestsize, DMA_FROM_DEVICE); in sun8i_ss_hash_run()
606 if (dma_mapping_error(ss->dev, addr_res)) { in sun8i_ss_hash_run()
607 dev_err(ss->dev, "Fail to create DMA mapping of result\n"); in sun8i_ss_hash_run()
608 err = -EINVAL; in sun8i_ss_hash_run()
611 addr_xpad = dma_map_single(ss->dev, tfmctx->opad, bs, DMA_TO_DEVICE); in sun8i_ss_hash_run()
612 err = dma_mapping_error(ss->dev, addr_xpad); in sun8i_ss_hash_run()
614 dev_err(ss->dev, "Fail to create DMA mapping of opad\n"); in sun8i_ss_hash_run()
617 rctx->t_src[0].addr = addr_xpad; in sun8i_ss_hash_run()
618 rctx->t_src[0].len = bs / 4; in sun8i_ss_hash_run()
621 j = digestsize / 4; in sun8i_ss_hash_run()
625 rctx->t_dst[0].addr = addr_res; in sun8i_ss_hash_run()
626 rctx->t_dst[0].len = digestsize / 4; in sun8i_ss_hash_run()
629 switch (algt->ss_algo_id) { in sun8i_ss_hash_run()
631 j = hash_pad(bf, 4096, j, byte_count, true, bs); in sun8i_ss_hash_run()
636 j = hash_pad(bf, 4096, j, byte_count, false, bs); in sun8i_ss_hash_run()
639 if (!j) { in sun8i_ss_hash_run()
640 err = -EINVAL; in sun8i_ss_hash_run()
644 addr_pad = dma_map_single(ss->dev, pad, j * 4, DMA_TO_DEVICE); in sun8i_ss_hash_run()
645 if (dma_mapping_error(ss->dev, addr_pad)) { in sun8i_ss_hash_run()
646 dev_err(ss->dev, "DMA error on padding SG\n"); in sun8i_ss_hash_run()
647 err = -EINVAL; in sun8i_ss_hash_run()
650 rctx->t_src[i].addr = addr_pad; in sun8i_ss_hash_run()
651 rctx->t_src[i].len = j; in sun8i_ss_hash_run()
652 rctx->t_dst[i].addr = addr_res; in sun8i_ss_hash_run()
653 rctx->t_dst[i].len = digestsize / 4; in sun8i_ss_hash_run()
655 err = sun8i_ss_run_hash_task(ss, rctx, crypto_tfm_alg_name(areq->base.tfm)); in sun8i_ss_hash_run()
682 dma_unmap_single(ss->dev, addr_pad, j * 4, DMA_TO_DEVICE); in sun8i_ss_hash_run()
686 dma_unmap_single(ss->dev, addr_xpad, bs, DMA_TO_DEVICE); in sun8i_ss_hash_run()
688 dma_unmap_single(ss->dev, addr_res, digestsize, DMA_FROM_DEVICE); in sun8i_ss_hash_run()
691 dma_unmap_sg(ss->dev, areq->src, sg_nents(areq->src), in sun8i_ss_hash_run()
699 memcpy(areq->result, result, crypto_ahash_digestsize(tfm)); in sun8i_ss_hash_run()