106f751b6SCorentin Labbe // SPDX-License-Identifier: GPL-2.0
206f751b6SCorentin Labbe /*
306f751b6SCorentin Labbe * sun8i-ce-cipher.c - hardware cryptographic offloader for
406f751b6SCorentin Labbe * Allwinner H3/A64/H5/H2+/H6/R40 SoC
506f751b6SCorentin Labbe *
606f751b6SCorentin Labbe * Copyright (C) 2016-2019 Corentin LABBE <clabbe.montjoie@gmail.com>
706f751b6SCorentin Labbe *
806f751b6SCorentin Labbe * This file add support for AES cipher with 128,192,256 bits keysize in
906f751b6SCorentin Labbe * CBC and ECB mode.
1006f751b6SCorentin Labbe *
1139db3f15SJonathan Corbet * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst
1206f751b6SCorentin Labbe */
1306f751b6SCorentin Labbe
14f75a749bSCorentin Labbe #include <linux/bottom_half.h>
1506f751b6SCorentin Labbe #include <linux/crypto.h>
1606f751b6SCorentin Labbe #include <linux/dma-mapping.h>
1706f751b6SCorentin Labbe #include <linux/io.h>
1806f751b6SCorentin Labbe #include <linux/pm_runtime.h>
1906f751b6SCorentin Labbe #include <crypto/scatterwalk.h>
2006f751b6SCorentin Labbe #include <crypto/internal/des.h>
2106f751b6SCorentin Labbe #include <crypto/internal/skcipher.h>
2206f751b6SCorentin Labbe #include "sun8i-ce.h"
2306f751b6SCorentin Labbe
sun8i_ce_cipher_need_fallback(struct skcipher_request * areq)2406f751b6SCorentin Labbe static int sun8i_ce_cipher_need_fallback(struct skcipher_request *areq)
2506f751b6SCorentin Labbe {
2606f751b6SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
2706f751b6SCorentin Labbe struct scatterlist *sg;
28aff388f7SCorentin Labbe struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
29aff388f7SCorentin Labbe struct sun8i_ce_alg_template *algt;
3042a01af3SCorentin Labbe unsigned int todo, len;
31aff388f7SCorentin Labbe
3207e34cd3SHerbert Xu algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base);
3306f751b6SCorentin Labbe
346b8309faSCorentin Labbe if (sg_nents_for_len(areq->src, areq->cryptlen) > MAX_SG ||
35aff388f7SCorentin Labbe sg_nents_for_len(areq->dst, areq->cryptlen) > MAX_SG) {
36aff388f7SCorentin Labbe algt->stat_fb_maxsg++;
3706f751b6SCorentin Labbe return true;
38aff388f7SCorentin Labbe }
3906f751b6SCorentin Labbe
40aff388f7SCorentin Labbe if (areq->cryptlen < crypto_skcipher_ivsize(tfm)) {
41aff388f7SCorentin Labbe algt->stat_fb_leniv++;
4206f751b6SCorentin Labbe return true;
43aff388f7SCorentin Labbe }
4406f751b6SCorentin Labbe
45aff388f7SCorentin Labbe if (areq->cryptlen == 0) {
46aff388f7SCorentin Labbe algt->stat_fb_len0++;
4706f751b6SCorentin Labbe return true;
48aff388f7SCorentin Labbe }
49aff388f7SCorentin Labbe
50aff388f7SCorentin Labbe if (areq->cryptlen % 16) {
51aff388f7SCorentin Labbe algt->stat_fb_mod16++;
52aff388f7SCorentin Labbe return true;
53aff388f7SCorentin Labbe }
5406f751b6SCorentin Labbe
5542a01af3SCorentin Labbe len = areq->cryptlen;
5606f751b6SCorentin Labbe sg = areq->src;
5706f751b6SCorentin Labbe while (sg) {
58aff388f7SCorentin Labbe if (!IS_ALIGNED(sg->offset, sizeof(u32))) {
59aff388f7SCorentin Labbe algt->stat_fb_srcali++;
6006f751b6SCorentin Labbe return true;
61aff388f7SCorentin Labbe }
6242a01af3SCorentin Labbe todo = min(len, sg->length);
6342a01af3SCorentin Labbe if (todo % 4) {
64aff388f7SCorentin Labbe algt->stat_fb_srclen++;
65aff388f7SCorentin Labbe return true;
66aff388f7SCorentin Labbe }
6742a01af3SCorentin Labbe len -= todo;
6806f751b6SCorentin Labbe sg = sg_next(sg);
6906f751b6SCorentin Labbe }
7042a01af3SCorentin Labbe
7142a01af3SCorentin Labbe len = areq->cryptlen;
7206f751b6SCorentin Labbe sg = areq->dst;
7306f751b6SCorentin Labbe while (sg) {
74aff388f7SCorentin Labbe if (!IS_ALIGNED(sg->offset, sizeof(u32))) {
75aff388f7SCorentin Labbe algt->stat_fb_dstali++;
7606f751b6SCorentin Labbe return true;
77aff388f7SCorentin Labbe }
7842a01af3SCorentin Labbe todo = min(len, sg->length);
7942a01af3SCorentin Labbe if (todo % 4) {
80aff388f7SCorentin Labbe algt->stat_fb_dstlen++;
81aff388f7SCorentin Labbe return true;
82aff388f7SCorentin Labbe }
8342a01af3SCorentin Labbe len -= todo;
8406f751b6SCorentin Labbe sg = sg_next(sg);
8506f751b6SCorentin Labbe }
8606f751b6SCorentin Labbe return false;
8706f751b6SCorentin Labbe }
8806f751b6SCorentin Labbe
sun8i_ce_cipher_fallback(struct skcipher_request * areq)8906f751b6SCorentin Labbe static int sun8i_ce_cipher_fallback(struct skcipher_request *areq)
9006f751b6SCorentin Labbe {
9106f751b6SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
9206f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
9306f751b6SCorentin Labbe struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
9406f751b6SCorentin Labbe int err;
9506f751b6SCorentin Labbe
9607e34cd3SHerbert Xu if (IS_ENABLED(CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG)) {
9707e34cd3SHerbert Xu struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
9807e34cd3SHerbert Xu struct sun8i_ce_alg_template *algt __maybe_unused;
9907e34cd3SHerbert Xu
10007e34cd3SHerbert Xu algt = container_of(alg, struct sun8i_ce_alg_template,
10107e34cd3SHerbert Xu alg.skcipher.base);
10207e34cd3SHerbert Xu
10307e34cd3SHerbert Xu #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
10406f751b6SCorentin Labbe algt->stat_fb++;
10506f751b6SCorentin Labbe #endif
10607e34cd3SHerbert Xu }
10706f751b6SCorentin Labbe
10831abd3ebSArd Biesheuvel skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm);
10931abd3ebSArd Biesheuvel skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags,
11031abd3ebSArd Biesheuvel areq->base.complete, areq->base.data);
11131abd3ebSArd Biesheuvel skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst,
11206f751b6SCorentin Labbe areq->cryptlen, areq->iv);
11306f751b6SCorentin Labbe if (rctx->op_dir & CE_DECRYPTION)
11431abd3ebSArd Biesheuvel err = crypto_skcipher_decrypt(&rctx->fallback_req);
11506f751b6SCorentin Labbe else
11631abd3ebSArd Biesheuvel err = crypto_skcipher_encrypt(&rctx->fallback_req);
11706f751b6SCorentin Labbe return err;
11806f751b6SCorentin Labbe }
11906f751b6SCorentin Labbe
sun8i_ce_cipher_prepare(struct crypto_engine * engine,void * async_req)1200605fa0fSCorentin Labbe static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req)
12106f751b6SCorentin Labbe {
1220605fa0fSCorentin Labbe struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base);
12306f751b6SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
12406f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
12506f751b6SCorentin Labbe struct sun8i_ce_dev *ce = op->ce;
12606f751b6SCorentin Labbe struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
12706f751b6SCorentin Labbe struct skcipher_alg *alg = crypto_skcipher_alg(tfm);
12806f751b6SCorentin Labbe struct sun8i_ce_alg_template *algt;
12906f751b6SCorentin Labbe struct sun8i_ce_flow *chan;
13006f751b6SCorentin Labbe struct ce_task *cet;
13106f751b6SCorentin Labbe struct scatterlist *sg;
13206f751b6SCorentin Labbe unsigned int todo, len, offset, ivsize;
13393c7f4d3SCorentin Labbe u32 common, sym;
13406f751b6SCorentin Labbe int flow, i;
13506f751b6SCorentin Labbe int nr_sgs = 0;
13606f751b6SCorentin Labbe int nr_sgd = 0;
13706f751b6SCorentin Labbe int err = 0;
1386b8309faSCorentin Labbe int ns = sg_nents_for_len(areq->src, areq->cryptlen);
1396b8309faSCorentin Labbe int nd = sg_nents_for_len(areq->dst, areq->cryptlen);
14006f751b6SCorentin Labbe
14107e34cd3SHerbert Xu algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base);
14206f751b6SCorentin Labbe
14306f751b6SCorentin Labbe dev_dbg(ce->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__,
14406f751b6SCorentin Labbe crypto_tfm_alg_name(areq->base.tfm),
14506f751b6SCorentin Labbe areq->cryptlen,
14606f751b6SCorentin Labbe rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm),
14706f751b6SCorentin Labbe op->keylen);
14806f751b6SCorentin Labbe
14906f751b6SCorentin Labbe #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG
15006f751b6SCorentin Labbe algt->stat_req++;
15106f751b6SCorentin Labbe #endif
15206f751b6SCorentin Labbe
15306f751b6SCorentin Labbe flow = rctx->flow;
15406f751b6SCorentin Labbe
15506f751b6SCorentin Labbe chan = &ce->chanlist[flow];
15606f751b6SCorentin Labbe
15706f751b6SCorentin Labbe cet = chan->tl;
15806f751b6SCorentin Labbe memset(cet, 0, sizeof(struct ce_task));
15906f751b6SCorentin Labbe
16093c7f4d3SCorentin Labbe cet->t_id = cpu_to_le32(flow);
16193c7f4d3SCorentin Labbe common = ce->variant->alg_cipher[algt->ce_algo_id];
16293c7f4d3SCorentin Labbe common |= rctx->op_dir | CE_COMM_INT;
16393c7f4d3SCorentin Labbe cet->t_common_ctl = cpu_to_le32(common);
16406f751b6SCorentin Labbe /* CTS and recent CE (H6) need length in bytes, in word otherwise */
1656b4f76c2SCorentin Labbe if (ce->variant->cipher_t_dlen_in_bytes)
16693c7f4d3SCorentin Labbe cet->t_dlen = cpu_to_le32(areq->cryptlen);
16793c7f4d3SCorentin Labbe else
16893c7f4d3SCorentin Labbe cet->t_dlen = cpu_to_le32(areq->cryptlen / 4);
16906f751b6SCorentin Labbe
17093c7f4d3SCorentin Labbe sym = ce->variant->op_mode[algt->ce_blockmode];
17106f751b6SCorentin Labbe len = op->keylen;
17206f751b6SCorentin Labbe switch (len) {
17306f751b6SCorentin Labbe case 128 / 8:
17493c7f4d3SCorentin Labbe sym |= CE_AES_128BITS;
17506f751b6SCorentin Labbe break;
17606f751b6SCorentin Labbe case 192 / 8:
17793c7f4d3SCorentin Labbe sym |= CE_AES_192BITS;
17806f751b6SCorentin Labbe break;
17906f751b6SCorentin Labbe case 256 / 8:
18093c7f4d3SCorentin Labbe sym |= CE_AES_256BITS;
18106f751b6SCorentin Labbe break;
18206f751b6SCorentin Labbe }
18306f751b6SCorentin Labbe
18493c7f4d3SCorentin Labbe cet->t_sym_ctl = cpu_to_le32(sym);
18506f751b6SCorentin Labbe cet->t_asym_ctl = 0;
18606f751b6SCorentin Labbe
1870605fa0fSCorentin Labbe rctx->addr_key = dma_map_single(ce->dev, op->key, op->keylen, DMA_TO_DEVICE);
1880605fa0fSCorentin Labbe if (dma_mapping_error(ce->dev, rctx->addr_key)) {
18906f751b6SCorentin Labbe dev_err(ce->dev, "Cannot DMA MAP KEY\n");
19006f751b6SCorentin Labbe err = -EFAULT;
19106f751b6SCorentin Labbe goto theend;
19206f751b6SCorentin Labbe }
193*e0740beeSAndre Przywara cet->t_key = desc_addr_val_le32(ce, rctx->addr_key);
19406f751b6SCorentin Labbe
19506f751b6SCorentin Labbe ivsize = crypto_skcipher_ivsize(tfm);
19606f751b6SCorentin Labbe if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) {
197a216f8d5SCorentin Labbe rctx->ivlen = ivsize;
19806f751b6SCorentin Labbe if (rctx->op_dir & CE_DECRYPTION) {
19906f751b6SCorentin Labbe offset = areq->cryptlen - ivsize;
20022f7c2f8SCorentin Labbe scatterwalk_map_and_copy(chan->backup_iv, areq->src,
201a216f8d5SCorentin Labbe offset, ivsize, 0);
20206f751b6SCorentin Labbe }
20322f7c2f8SCorentin Labbe memcpy(chan->bounce_iv, areq->iv, ivsize);
20422f7c2f8SCorentin Labbe rctx->addr_iv = dma_map_single(ce->dev, chan->bounce_iv, rctx->ivlen,
20593c7f4d3SCorentin Labbe DMA_TO_DEVICE);
2060605fa0fSCorentin Labbe if (dma_mapping_error(ce->dev, rctx->addr_iv)) {
20706f751b6SCorentin Labbe dev_err(ce->dev, "Cannot DMA MAP IV\n");
20806f751b6SCorentin Labbe err = -ENOMEM;
20906f751b6SCorentin Labbe goto theend_iv;
21006f751b6SCorentin Labbe }
211*e0740beeSAndre Przywara cet->t_iv = desc_addr_val_le32(ce, rctx->addr_iv);
21206f751b6SCorentin Labbe }
21306f751b6SCorentin Labbe
21406f751b6SCorentin Labbe if (areq->src == areq->dst) {
2156b8309faSCorentin Labbe nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_BIDIRECTIONAL);
21606f751b6SCorentin Labbe if (nr_sgs <= 0 || nr_sgs > MAX_SG) {
21706f751b6SCorentin Labbe dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);
21806f751b6SCorentin Labbe err = -EINVAL;
21906f751b6SCorentin Labbe goto theend_iv;
22006f751b6SCorentin Labbe }
22106f751b6SCorentin Labbe nr_sgd = nr_sgs;
22206f751b6SCorentin Labbe } else {
2236b8309faSCorentin Labbe nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE);
22406f751b6SCorentin Labbe if (nr_sgs <= 0 || nr_sgs > MAX_SG) {
22506f751b6SCorentin Labbe dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs);
22606f751b6SCorentin Labbe err = -EINVAL;
22706f751b6SCorentin Labbe goto theend_iv;
22806f751b6SCorentin Labbe }
2296b8309faSCorentin Labbe nr_sgd = dma_map_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE);
23006f751b6SCorentin Labbe if (nr_sgd <= 0 || nr_sgd > MAX_SG) {
23106f751b6SCorentin Labbe dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd);
23206f751b6SCorentin Labbe err = -EINVAL;
23306f751b6SCorentin Labbe goto theend_sgs;
23406f751b6SCorentin Labbe }
23506f751b6SCorentin Labbe }
23606f751b6SCorentin Labbe
23706f751b6SCorentin Labbe len = areq->cryptlen;
23806f751b6SCorentin Labbe for_each_sg(areq->src, sg, nr_sgs, i) {
239*e0740beeSAndre Przywara cet->t_src[i].addr = desc_addr_val_le32(ce, sg_dma_address(sg));
24006f751b6SCorentin Labbe todo = min(len, sg_dma_len(sg));
24193c7f4d3SCorentin Labbe cet->t_src[i].len = cpu_to_le32(todo / 4);
24206f751b6SCorentin Labbe dev_dbg(ce->dev, "%s total=%u SG(%d %u off=%d) todo=%u\n", __func__,
24306f751b6SCorentin Labbe areq->cryptlen, i, cet->t_src[i].len, sg->offset, todo);
24406f751b6SCorentin Labbe len -= todo;
24506f751b6SCorentin Labbe }
24606f751b6SCorentin Labbe if (len > 0) {
24706f751b6SCorentin Labbe dev_err(ce->dev, "remaining len %d\n", len);
24806f751b6SCorentin Labbe err = -EINVAL;
24906f751b6SCorentin Labbe goto theend_sgs;
25006f751b6SCorentin Labbe }
25106f751b6SCorentin Labbe
25206f751b6SCorentin Labbe len = areq->cryptlen;
25306f751b6SCorentin Labbe for_each_sg(areq->dst, sg, nr_sgd, i) {
254*e0740beeSAndre Przywara cet->t_dst[i].addr = desc_addr_val_le32(ce, sg_dma_address(sg));
25506f751b6SCorentin Labbe todo = min(len, sg_dma_len(sg));
25693c7f4d3SCorentin Labbe cet->t_dst[i].len = cpu_to_le32(todo / 4);
25706f751b6SCorentin Labbe dev_dbg(ce->dev, "%s total=%u SG(%d %u off=%d) todo=%u\n", __func__,
25806f751b6SCorentin Labbe areq->cryptlen, i, cet->t_dst[i].len, sg->offset, todo);
25906f751b6SCorentin Labbe len -= todo;
26006f751b6SCorentin Labbe }
26106f751b6SCorentin Labbe if (len > 0) {
26206f751b6SCorentin Labbe dev_err(ce->dev, "remaining len %d\n", len);
26306f751b6SCorentin Labbe err = -EINVAL;
26406f751b6SCorentin Labbe goto theend_sgs;
26506f751b6SCorentin Labbe }
26606f751b6SCorentin Labbe
26706f751b6SCorentin Labbe chan->timeout = areq->cryptlen;
2680605fa0fSCorentin Labbe rctx->nr_sgs = nr_sgs;
2690605fa0fSCorentin Labbe rctx->nr_sgd = nr_sgd;
2700605fa0fSCorentin Labbe return 0;
27106f751b6SCorentin Labbe
27206f751b6SCorentin Labbe theend_sgs:
27306f751b6SCorentin Labbe if (areq->src == areq->dst) {
2746b8309faSCorentin Labbe dma_unmap_sg(ce->dev, areq->src, ns, DMA_BIDIRECTIONAL);
27506f751b6SCorentin Labbe } else {
27606f751b6SCorentin Labbe if (nr_sgs > 0)
2776b8309faSCorentin Labbe dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE);
2786b8309faSCorentin Labbe dma_unmap_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE);
27906f751b6SCorentin Labbe }
28006f751b6SCorentin Labbe
28106f751b6SCorentin Labbe theend_iv:
28206f751b6SCorentin Labbe if (areq->iv && ivsize > 0) {
2830605fa0fSCorentin Labbe if (rctx->addr_iv)
2840605fa0fSCorentin Labbe dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE);
28506f751b6SCorentin Labbe offset = areq->cryptlen - ivsize;
28606f751b6SCorentin Labbe if (rctx->op_dir & CE_DECRYPTION) {
28722f7c2f8SCorentin Labbe memcpy(areq->iv, chan->backup_iv, ivsize);
28822f7c2f8SCorentin Labbe memzero_explicit(chan->backup_iv, ivsize);
28906f751b6SCorentin Labbe } else {
29006f751b6SCorentin Labbe scatterwalk_map_and_copy(areq->iv, areq->dst, offset,
29106f751b6SCorentin Labbe ivsize, 0);
29206f751b6SCorentin Labbe }
29322f7c2f8SCorentin Labbe memzero_explicit(chan->bounce_iv, ivsize);
29406f751b6SCorentin Labbe }
29506f751b6SCorentin Labbe
2960605fa0fSCorentin Labbe dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE);
29706f751b6SCorentin Labbe
29806f751b6SCorentin Labbe theend:
29906f751b6SCorentin Labbe return err;
30006f751b6SCorentin Labbe }
30106f751b6SCorentin Labbe
sun8i_ce_cipher_unprepare(struct crypto_engine * engine,void * async_req)3024136212aSHerbert Xu static void sun8i_ce_cipher_unprepare(struct crypto_engine *engine,
3034136212aSHerbert Xu void *async_req)
3040605fa0fSCorentin Labbe {
3050605fa0fSCorentin Labbe struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base);
3060605fa0fSCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
3070605fa0fSCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
3080605fa0fSCorentin Labbe struct sun8i_ce_dev *ce = op->ce;
3090605fa0fSCorentin Labbe struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
3100605fa0fSCorentin Labbe struct sun8i_ce_flow *chan;
3110605fa0fSCorentin Labbe struct ce_task *cet;
3120605fa0fSCorentin Labbe unsigned int ivsize, offset;
3130605fa0fSCorentin Labbe int nr_sgs = rctx->nr_sgs;
3140605fa0fSCorentin Labbe int nr_sgd = rctx->nr_sgd;
3150605fa0fSCorentin Labbe int flow;
3160605fa0fSCorentin Labbe
3170605fa0fSCorentin Labbe flow = rctx->flow;
3180605fa0fSCorentin Labbe chan = &ce->chanlist[flow];
3190605fa0fSCorentin Labbe cet = chan->tl;
3200605fa0fSCorentin Labbe ivsize = crypto_skcipher_ivsize(tfm);
3210605fa0fSCorentin Labbe
3220605fa0fSCorentin Labbe if (areq->src == areq->dst) {
3230605fa0fSCorentin Labbe dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_BIDIRECTIONAL);
3240605fa0fSCorentin Labbe } else {
3250605fa0fSCorentin Labbe if (nr_sgs > 0)
3260605fa0fSCorentin Labbe dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE);
3270605fa0fSCorentin Labbe dma_unmap_sg(ce->dev, areq->dst, nr_sgd, DMA_FROM_DEVICE);
3280605fa0fSCorentin Labbe }
3290605fa0fSCorentin Labbe
3300605fa0fSCorentin Labbe if (areq->iv && ivsize > 0) {
3310605fa0fSCorentin Labbe if (cet->t_iv)
3320605fa0fSCorentin Labbe dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE);
3330605fa0fSCorentin Labbe offset = areq->cryptlen - ivsize;
3340605fa0fSCorentin Labbe if (rctx->op_dir & CE_DECRYPTION) {
33522f7c2f8SCorentin Labbe memcpy(areq->iv, chan->backup_iv, ivsize);
33622f7c2f8SCorentin Labbe memzero_explicit(chan->backup_iv, ivsize);
3370605fa0fSCorentin Labbe } else {
3380605fa0fSCorentin Labbe scatterwalk_map_and_copy(areq->iv, areq->dst, offset,
3390605fa0fSCorentin Labbe ivsize, 0);
3400605fa0fSCorentin Labbe }
34122f7c2f8SCorentin Labbe memzero_explicit(chan->bounce_iv, ivsize);
3420605fa0fSCorentin Labbe }
3430605fa0fSCorentin Labbe
3440605fa0fSCorentin Labbe dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE);
3454136212aSHerbert Xu }
34606f751b6SCorentin Labbe
sun8i_ce_cipher_run(struct crypto_engine * engine,void * areq)34718342003SAndrey Skvortsov static void sun8i_ce_cipher_run(struct crypto_engine *engine, void *areq)
34818342003SAndrey Skvortsov {
34918342003SAndrey Skvortsov struct skcipher_request *breq = container_of(areq, struct skcipher_request, base);
35018342003SAndrey Skvortsov struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(breq);
35118342003SAndrey Skvortsov struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
35218342003SAndrey Skvortsov struct sun8i_ce_dev *ce = op->ce;
35318342003SAndrey Skvortsov struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(breq);
35418342003SAndrey Skvortsov int flow, err;
35518342003SAndrey Skvortsov
35618342003SAndrey Skvortsov flow = rctx->flow;
35718342003SAndrey Skvortsov err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(breq->base.tfm));
35818342003SAndrey Skvortsov sun8i_ce_cipher_unprepare(engine, areq);
35918342003SAndrey Skvortsov local_bh_disable();
36018342003SAndrey Skvortsov crypto_finalize_skcipher_request(engine, breq, err);
36118342003SAndrey Skvortsov local_bh_enable();
36218342003SAndrey Skvortsov }
36318342003SAndrey Skvortsov
sun8i_ce_cipher_do_one(struct crypto_engine * engine,void * areq)36407e34cd3SHerbert Xu int sun8i_ce_cipher_do_one(struct crypto_engine *engine, void *areq)
3654136212aSHerbert Xu {
3664136212aSHerbert Xu int err = sun8i_ce_cipher_prepare(engine, areq);
3674136212aSHerbert Xu
3684136212aSHerbert Xu if (err)
3694136212aSHerbert Xu return err;
3704136212aSHerbert Xu
3714136212aSHerbert Xu sun8i_ce_cipher_run(engine, areq);
37206f751b6SCorentin Labbe return 0;
37306f751b6SCorentin Labbe }
37406f751b6SCorentin Labbe
sun8i_ce_skdecrypt(struct skcipher_request * areq)37506f751b6SCorentin Labbe int sun8i_ce_skdecrypt(struct skcipher_request *areq)
37606f751b6SCorentin Labbe {
37706f751b6SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
37806f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
37906f751b6SCorentin Labbe struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
38006f751b6SCorentin Labbe struct crypto_engine *engine;
38106f751b6SCorentin Labbe int e;
38206f751b6SCorentin Labbe
38306f751b6SCorentin Labbe rctx->op_dir = CE_DECRYPTION;
38406f751b6SCorentin Labbe if (sun8i_ce_cipher_need_fallback(areq))
38506f751b6SCorentin Labbe return sun8i_ce_cipher_fallback(areq);
38606f751b6SCorentin Labbe
38706f751b6SCorentin Labbe e = sun8i_ce_get_engine_number(op->ce);
38806f751b6SCorentin Labbe rctx->flow = e;
38906f751b6SCorentin Labbe engine = op->ce->chanlist[e].engine;
39006f751b6SCorentin Labbe
39106f751b6SCorentin Labbe return crypto_transfer_skcipher_request_to_engine(engine, areq);
39206f751b6SCorentin Labbe }
39306f751b6SCorentin Labbe
sun8i_ce_skencrypt(struct skcipher_request * areq)39406f751b6SCorentin Labbe int sun8i_ce_skencrypt(struct skcipher_request *areq)
39506f751b6SCorentin Labbe {
39606f751b6SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq);
39706f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
39806f751b6SCorentin Labbe struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq);
39906f751b6SCorentin Labbe struct crypto_engine *engine;
40006f751b6SCorentin Labbe int e;
40106f751b6SCorentin Labbe
40206f751b6SCorentin Labbe rctx->op_dir = CE_ENCRYPTION;
40306f751b6SCorentin Labbe if (sun8i_ce_cipher_need_fallback(areq))
40406f751b6SCorentin Labbe return sun8i_ce_cipher_fallback(areq);
40506f751b6SCorentin Labbe
40606f751b6SCorentin Labbe e = sun8i_ce_get_engine_number(op->ce);
40706f751b6SCorentin Labbe rctx->flow = e;
40806f751b6SCorentin Labbe engine = op->ce->chanlist[e].engine;
40906f751b6SCorentin Labbe
41006f751b6SCorentin Labbe return crypto_transfer_skcipher_request_to_engine(engine, areq);
41106f751b6SCorentin Labbe }
41206f751b6SCorentin Labbe
sun8i_ce_cipher_init(struct crypto_tfm * tfm)41306f751b6SCorentin Labbe int sun8i_ce_cipher_init(struct crypto_tfm *tfm)
41406f751b6SCorentin Labbe {
41506f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
41606f751b6SCorentin Labbe struct sun8i_ce_alg_template *algt;
41706f751b6SCorentin Labbe const char *name = crypto_tfm_alg_name(tfm);
41806f751b6SCorentin Labbe struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm);
41906f751b6SCorentin Labbe struct skcipher_alg *alg = crypto_skcipher_alg(sktfm);
42006f751b6SCorentin Labbe int err;
42106f751b6SCorentin Labbe
42206f751b6SCorentin Labbe memset(op, 0, sizeof(struct sun8i_cipher_tfm_ctx));
42306f751b6SCorentin Labbe
42407e34cd3SHerbert Xu algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher.base);
42506f751b6SCorentin Labbe op->ce = algt->ce;
42606f751b6SCorentin Labbe
42731abd3ebSArd Biesheuvel op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
42806f751b6SCorentin Labbe if (IS_ERR(op->fallback_tfm)) {
42906f751b6SCorentin Labbe dev_err(op->ce->dev, "ERROR: Cannot allocate fallback for %s %ld\n",
43006f751b6SCorentin Labbe name, PTR_ERR(op->fallback_tfm));
43106f751b6SCorentin Labbe return PTR_ERR(op->fallback_tfm);
43206f751b6SCorentin Labbe }
43306f751b6SCorentin Labbe
434e9b21862SOvidiu Panait crypto_skcipher_set_reqsize(sktfm, sizeof(struct sun8i_cipher_req_ctx) +
435e9b21862SOvidiu Panait crypto_skcipher_reqsize(op->fallback_tfm));
43631abd3ebSArd Biesheuvel
437aff388f7SCorentin Labbe memcpy(algt->fbname,
438aff388f7SCorentin Labbe crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)),
439aff388f7SCorentin Labbe CRYPTO_MAX_ALG_NAME);
44006f751b6SCorentin Labbe
44106f751b6SCorentin Labbe err = pm_runtime_get_sync(op->ce->dev);
44206f751b6SCorentin Labbe if (err < 0)
44306f751b6SCorentin Labbe goto error_pm;
44406f751b6SCorentin Labbe
44506f751b6SCorentin Labbe return 0;
44606f751b6SCorentin Labbe error_pm:
4475c3a8a66SDinghao Liu pm_runtime_put_noidle(op->ce->dev);
44831abd3ebSArd Biesheuvel crypto_free_skcipher(op->fallback_tfm);
44906f751b6SCorentin Labbe return err;
45006f751b6SCorentin Labbe }
45106f751b6SCorentin Labbe
sun8i_ce_cipher_exit(struct crypto_tfm * tfm)45206f751b6SCorentin Labbe void sun8i_ce_cipher_exit(struct crypto_tfm *tfm)
45306f751b6SCorentin Labbe {
45406f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm);
45506f751b6SCorentin Labbe
456712d8069SDenis Efremov kfree_sensitive(op->key);
45731abd3ebSArd Biesheuvel crypto_free_skcipher(op->fallback_tfm);
45806f751b6SCorentin Labbe pm_runtime_put_sync_suspend(op->ce->dev);
45906f751b6SCorentin Labbe }
46006f751b6SCorentin Labbe
sun8i_ce_aes_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)46106f751b6SCorentin Labbe int sun8i_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key,
46206f751b6SCorentin Labbe unsigned int keylen)
46306f751b6SCorentin Labbe {
46406f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
46506f751b6SCorentin Labbe struct sun8i_ce_dev *ce = op->ce;
46606f751b6SCorentin Labbe
46706f751b6SCorentin Labbe switch (keylen) {
46806f751b6SCorentin Labbe case 128 / 8:
46906f751b6SCorentin Labbe break;
47006f751b6SCorentin Labbe case 192 / 8:
47106f751b6SCorentin Labbe break;
47206f751b6SCorentin Labbe case 256 / 8:
47306f751b6SCorentin Labbe break;
47406f751b6SCorentin Labbe default:
47506f751b6SCorentin Labbe dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen);
47606f751b6SCorentin Labbe return -EINVAL;
47706f751b6SCorentin Labbe }
478712d8069SDenis Efremov kfree_sensitive(op->key);
47906f751b6SCorentin Labbe op->keylen = keylen;
480c7351845SYueHaibing op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
48106f751b6SCorentin Labbe if (!op->key)
48206f751b6SCorentin Labbe return -ENOMEM;
48306f751b6SCorentin Labbe
48431abd3ebSArd Biesheuvel crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);
48531abd3ebSArd Biesheuvel crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
48606f751b6SCorentin Labbe
48731abd3ebSArd Biesheuvel return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);
48806f751b6SCorentin Labbe }
48906f751b6SCorentin Labbe
sun8i_ce_des3_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)49006f751b6SCorentin Labbe int sun8i_ce_des3_setkey(struct crypto_skcipher *tfm, const u8 *key,
49106f751b6SCorentin Labbe unsigned int keylen)
49206f751b6SCorentin Labbe {
49306f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm);
49406f751b6SCorentin Labbe int err;
49506f751b6SCorentin Labbe
49606f751b6SCorentin Labbe err = verify_skcipher_des3_key(tfm, key);
49706f751b6SCorentin Labbe if (err)
49806f751b6SCorentin Labbe return err;
49906f751b6SCorentin Labbe
500712d8069SDenis Efremov kfree_sensitive(op->key);
50106f751b6SCorentin Labbe op->keylen = keylen;
502c7351845SYueHaibing op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA);
50306f751b6SCorentin Labbe if (!op->key)
50406f751b6SCorentin Labbe return -ENOMEM;
50506f751b6SCorentin Labbe
50631abd3ebSArd Biesheuvel crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK);
50731abd3ebSArd Biesheuvel crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK);
50806f751b6SCorentin Labbe
50931abd3ebSArd Biesheuvel return crypto_skcipher_setkey(op->fallback_tfm, key, keylen);
51006f751b6SCorentin Labbe }
511