106f751b6SCorentin Labbe // SPDX-License-Identifier: GPL-2.0 206f751b6SCorentin Labbe /* 306f751b6SCorentin Labbe * sun8i-ce-cipher.c - hardware cryptographic offloader for 406f751b6SCorentin Labbe * Allwinner H3/A64/H5/H2+/H6/R40 SoC 506f751b6SCorentin Labbe * 606f751b6SCorentin Labbe * Copyright (C) 2016-2019 Corentin LABBE <clabbe.montjoie@gmail.com> 706f751b6SCorentin Labbe * 806f751b6SCorentin Labbe * This file add support for AES cipher with 128,192,256 bits keysize in 906f751b6SCorentin Labbe * CBC and ECB mode. 1006f751b6SCorentin Labbe * 1139db3f15SJonathan Corbet * You could find a link for the datasheet in Documentation/arch/arm/sunxi.rst 1206f751b6SCorentin Labbe */ 1306f751b6SCorentin Labbe 14f75a749bSCorentin Labbe #include <linux/bottom_half.h> 1506f751b6SCorentin Labbe #include <linux/crypto.h> 1606f751b6SCorentin Labbe #include <linux/dma-mapping.h> 1706f751b6SCorentin Labbe #include <linux/io.h> 1806f751b6SCorentin Labbe #include <linux/pm_runtime.h> 1906f751b6SCorentin Labbe #include <crypto/scatterwalk.h> 2006f751b6SCorentin Labbe #include <crypto/internal/des.h> 2106f751b6SCorentin Labbe #include <crypto/internal/skcipher.h> 2206f751b6SCorentin Labbe #include "sun8i-ce.h" 2306f751b6SCorentin Labbe 2406f751b6SCorentin Labbe static int sun8i_ce_cipher_need_fallback(struct skcipher_request *areq) 2506f751b6SCorentin Labbe { 2606f751b6SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); 2706f751b6SCorentin Labbe struct scatterlist *sg; 28aff388f7SCorentin Labbe struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 29aff388f7SCorentin Labbe struct sun8i_ce_alg_template *algt; 3042a01af3SCorentin Labbe unsigned int todo, len; 31aff388f7SCorentin Labbe 32aff388f7SCorentin Labbe algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher); 3306f751b6SCorentin Labbe 346b8309faSCorentin Labbe if (sg_nents_for_len(areq->src, areq->cryptlen) > MAX_SG || 35aff388f7SCorentin Labbe sg_nents_for_len(areq->dst, areq->cryptlen) > MAX_SG) { 36aff388f7SCorentin Labbe algt->stat_fb_maxsg++; 3706f751b6SCorentin Labbe return true; 38aff388f7SCorentin Labbe } 3906f751b6SCorentin Labbe 40aff388f7SCorentin Labbe if (areq->cryptlen < crypto_skcipher_ivsize(tfm)) { 41aff388f7SCorentin Labbe algt->stat_fb_leniv++; 4206f751b6SCorentin Labbe return true; 43aff388f7SCorentin Labbe } 4406f751b6SCorentin Labbe 45aff388f7SCorentin Labbe if (areq->cryptlen == 0) { 46aff388f7SCorentin Labbe algt->stat_fb_len0++; 4706f751b6SCorentin Labbe return true; 48aff388f7SCorentin Labbe } 49aff388f7SCorentin Labbe 50aff388f7SCorentin Labbe if (areq->cryptlen % 16) { 51aff388f7SCorentin Labbe algt->stat_fb_mod16++; 52aff388f7SCorentin Labbe return true; 53aff388f7SCorentin Labbe } 5406f751b6SCorentin Labbe 5542a01af3SCorentin Labbe len = areq->cryptlen; 5606f751b6SCorentin Labbe sg = areq->src; 5706f751b6SCorentin Labbe while (sg) { 58aff388f7SCorentin Labbe if (!IS_ALIGNED(sg->offset, sizeof(u32))) { 59aff388f7SCorentin Labbe algt->stat_fb_srcali++; 6006f751b6SCorentin Labbe return true; 61aff388f7SCorentin Labbe } 6242a01af3SCorentin Labbe todo = min(len, sg->length); 6342a01af3SCorentin Labbe if (todo % 4) { 64aff388f7SCorentin Labbe algt->stat_fb_srclen++; 65aff388f7SCorentin Labbe return true; 66aff388f7SCorentin Labbe } 6742a01af3SCorentin Labbe len -= todo; 6806f751b6SCorentin Labbe sg = sg_next(sg); 6906f751b6SCorentin Labbe } 7042a01af3SCorentin Labbe 7142a01af3SCorentin Labbe len = areq->cryptlen; 7206f751b6SCorentin Labbe sg = areq->dst; 7306f751b6SCorentin Labbe while (sg) { 74aff388f7SCorentin Labbe if (!IS_ALIGNED(sg->offset, sizeof(u32))) { 75aff388f7SCorentin Labbe algt->stat_fb_dstali++; 7606f751b6SCorentin Labbe return true; 77aff388f7SCorentin Labbe } 7842a01af3SCorentin Labbe todo = min(len, sg->length); 7942a01af3SCorentin Labbe if (todo % 4) { 80aff388f7SCorentin Labbe algt->stat_fb_dstlen++; 81aff388f7SCorentin Labbe return true; 82aff388f7SCorentin Labbe } 8342a01af3SCorentin Labbe len -= todo; 8406f751b6SCorentin Labbe sg = sg_next(sg); 8506f751b6SCorentin Labbe } 8606f751b6SCorentin Labbe return false; 8706f751b6SCorentin Labbe } 8806f751b6SCorentin Labbe 8906f751b6SCorentin Labbe static int sun8i_ce_cipher_fallback(struct skcipher_request *areq) 9006f751b6SCorentin Labbe { 9106f751b6SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); 9206f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); 9306f751b6SCorentin Labbe struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); 9406f751b6SCorentin Labbe int err; 9506f751b6SCorentin Labbe #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG 9606f751b6SCorentin Labbe struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 9706f751b6SCorentin Labbe struct sun8i_ce_alg_template *algt; 9806f751b6SCorentin Labbe 9906f751b6SCorentin Labbe algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher); 10006f751b6SCorentin Labbe algt->stat_fb++; 10106f751b6SCorentin Labbe #endif 10206f751b6SCorentin Labbe 10331abd3ebSArd Biesheuvel skcipher_request_set_tfm(&rctx->fallback_req, op->fallback_tfm); 10431abd3ebSArd Biesheuvel skcipher_request_set_callback(&rctx->fallback_req, areq->base.flags, 10531abd3ebSArd Biesheuvel areq->base.complete, areq->base.data); 10631abd3ebSArd Biesheuvel skcipher_request_set_crypt(&rctx->fallback_req, areq->src, areq->dst, 10706f751b6SCorentin Labbe areq->cryptlen, areq->iv); 10806f751b6SCorentin Labbe if (rctx->op_dir & CE_DECRYPTION) 10931abd3ebSArd Biesheuvel err = crypto_skcipher_decrypt(&rctx->fallback_req); 11006f751b6SCorentin Labbe else 11131abd3ebSArd Biesheuvel err = crypto_skcipher_encrypt(&rctx->fallback_req); 11206f751b6SCorentin Labbe return err; 11306f751b6SCorentin Labbe } 11406f751b6SCorentin Labbe 1150605fa0fSCorentin Labbe static int sun8i_ce_cipher_prepare(struct crypto_engine *engine, void *async_req) 11606f751b6SCorentin Labbe { 1170605fa0fSCorentin Labbe struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); 11806f751b6SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); 11906f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); 12006f751b6SCorentin Labbe struct sun8i_ce_dev *ce = op->ce; 12106f751b6SCorentin Labbe struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); 12206f751b6SCorentin Labbe struct skcipher_alg *alg = crypto_skcipher_alg(tfm); 12306f751b6SCorentin Labbe struct sun8i_ce_alg_template *algt; 12406f751b6SCorentin Labbe struct sun8i_ce_flow *chan; 12506f751b6SCorentin Labbe struct ce_task *cet; 12606f751b6SCorentin Labbe struct scatterlist *sg; 12706f751b6SCorentin Labbe unsigned int todo, len, offset, ivsize; 12893c7f4d3SCorentin Labbe u32 common, sym; 12906f751b6SCorentin Labbe int flow, i; 13006f751b6SCorentin Labbe int nr_sgs = 0; 13106f751b6SCorentin Labbe int nr_sgd = 0; 13206f751b6SCorentin Labbe int err = 0; 1336b8309faSCorentin Labbe int ns = sg_nents_for_len(areq->src, areq->cryptlen); 1346b8309faSCorentin Labbe int nd = sg_nents_for_len(areq->dst, areq->cryptlen); 13506f751b6SCorentin Labbe 13606f751b6SCorentin Labbe algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher); 13706f751b6SCorentin Labbe 13806f751b6SCorentin Labbe dev_dbg(ce->dev, "%s %s %u %x IV(%p %u) key=%u\n", __func__, 13906f751b6SCorentin Labbe crypto_tfm_alg_name(areq->base.tfm), 14006f751b6SCorentin Labbe areq->cryptlen, 14106f751b6SCorentin Labbe rctx->op_dir, areq->iv, crypto_skcipher_ivsize(tfm), 14206f751b6SCorentin Labbe op->keylen); 14306f751b6SCorentin Labbe 14406f751b6SCorentin Labbe #ifdef CONFIG_CRYPTO_DEV_SUN8I_CE_DEBUG 14506f751b6SCorentin Labbe algt->stat_req++; 14606f751b6SCorentin Labbe #endif 14706f751b6SCorentin Labbe 14806f751b6SCorentin Labbe flow = rctx->flow; 14906f751b6SCorentin Labbe 15006f751b6SCorentin Labbe chan = &ce->chanlist[flow]; 15106f751b6SCorentin Labbe 15206f751b6SCorentin Labbe cet = chan->tl; 15306f751b6SCorentin Labbe memset(cet, 0, sizeof(struct ce_task)); 15406f751b6SCorentin Labbe 15593c7f4d3SCorentin Labbe cet->t_id = cpu_to_le32(flow); 15693c7f4d3SCorentin Labbe common = ce->variant->alg_cipher[algt->ce_algo_id]; 15793c7f4d3SCorentin Labbe common |= rctx->op_dir | CE_COMM_INT; 15893c7f4d3SCorentin Labbe cet->t_common_ctl = cpu_to_le32(common); 15906f751b6SCorentin Labbe /* CTS and recent CE (H6) need length in bytes, in word otherwise */ 1606b4f76c2SCorentin Labbe if (ce->variant->cipher_t_dlen_in_bytes) 16193c7f4d3SCorentin Labbe cet->t_dlen = cpu_to_le32(areq->cryptlen); 16293c7f4d3SCorentin Labbe else 16393c7f4d3SCorentin Labbe cet->t_dlen = cpu_to_le32(areq->cryptlen / 4); 16406f751b6SCorentin Labbe 16593c7f4d3SCorentin Labbe sym = ce->variant->op_mode[algt->ce_blockmode]; 16606f751b6SCorentin Labbe len = op->keylen; 16706f751b6SCorentin Labbe switch (len) { 16806f751b6SCorentin Labbe case 128 / 8: 16993c7f4d3SCorentin Labbe sym |= CE_AES_128BITS; 17006f751b6SCorentin Labbe break; 17106f751b6SCorentin Labbe case 192 / 8: 17293c7f4d3SCorentin Labbe sym |= CE_AES_192BITS; 17306f751b6SCorentin Labbe break; 17406f751b6SCorentin Labbe case 256 / 8: 17593c7f4d3SCorentin Labbe sym |= CE_AES_256BITS; 17606f751b6SCorentin Labbe break; 17706f751b6SCorentin Labbe } 17806f751b6SCorentin Labbe 17993c7f4d3SCorentin Labbe cet->t_sym_ctl = cpu_to_le32(sym); 18006f751b6SCorentin Labbe cet->t_asym_ctl = 0; 18106f751b6SCorentin Labbe 1820605fa0fSCorentin Labbe rctx->addr_key = dma_map_single(ce->dev, op->key, op->keylen, DMA_TO_DEVICE); 1830605fa0fSCorentin Labbe if (dma_mapping_error(ce->dev, rctx->addr_key)) { 18406f751b6SCorentin Labbe dev_err(ce->dev, "Cannot DMA MAP KEY\n"); 18506f751b6SCorentin Labbe err = -EFAULT; 18606f751b6SCorentin Labbe goto theend; 18706f751b6SCorentin Labbe } 1880605fa0fSCorentin Labbe cet->t_key = cpu_to_le32(rctx->addr_key); 18906f751b6SCorentin Labbe 19006f751b6SCorentin Labbe ivsize = crypto_skcipher_ivsize(tfm); 19106f751b6SCorentin Labbe if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) { 192a216f8d5SCorentin Labbe rctx->ivlen = ivsize; 19306f751b6SCorentin Labbe if (rctx->op_dir & CE_DECRYPTION) { 19406f751b6SCorentin Labbe offset = areq->cryptlen - ivsize; 19522f7c2f8SCorentin Labbe scatterwalk_map_and_copy(chan->backup_iv, areq->src, 196a216f8d5SCorentin Labbe offset, ivsize, 0); 19706f751b6SCorentin Labbe } 19822f7c2f8SCorentin Labbe memcpy(chan->bounce_iv, areq->iv, ivsize); 19922f7c2f8SCorentin Labbe rctx->addr_iv = dma_map_single(ce->dev, chan->bounce_iv, rctx->ivlen, 20093c7f4d3SCorentin Labbe DMA_TO_DEVICE); 2010605fa0fSCorentin Labbe if (dma_mapping_error(ce->dev, rctx->addr_iv)) { 20206f751b6SCorentin Labbe dev_err(ce->dev, "Cannot DMA MAP IV\n"); 20306f751b6SCorentin Labbe err = -ENOMEM; 20406f751b6SCorentin Labbe goto theend_iv; 20506f751b6SCorentin Labbe } 2060605fa0fSCorentin Labbe cet->t_iv = cpu_to_le32(rctx->addr_iv); 20706f751b6SCorentin Labbe } 20806f751b6SCorentin Labbe 20906f751b6SCorentin Labbe if (areq->src == areq->dst) { 2106b8309faSCorentin Labbe nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_BIDIRECTIONAL); 21106f751b6SCorentin Labbe if (nr_sgs <= 0 || nr_sgs > MAX_SG) { 21206f751b6SCorentin Labbe dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); 21306f751b6SCorentin Labbe err = -EINVAL; 21406f751b6SCorentin Labbe goto theend_iv; 21506f751b6SCorentin Labbe } 21606f751b6SCorentin Labbe nr_sgd = nr_sgs; 21706f751b6SCorentin Labbe } else { 2186b8309faSCorentin Labbe nr_sgs = dma_map_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); 21906f751b6SCorentin Labbe if (nr_sgs <= 0 || nr_sgs > MAX_SG) { 22006f751b6SCorentin Labbe dev_err(ce->dev, "Invalid sg number %d\n", nr_sgs); 22106f751b6SCorentin Labbe err = -EINVAL; 22206f751b6SCorentin Labbe goto theend_iv; 22306f751b6SCorentin Labbe } 2246b8309faSCorentin Labbe nr_sgd = dma_map_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE); 22506f751b6SCorentin Labbe if (nr_sgd <= 0 || nr_sgd > MAX_SG) { 22606f751b6SCorentin Labbe dev_err(ce->dev, "Invalid sg number %d\n", nr_sgd); 22706f751b6SCorentin Labbe err = -EINVAL; 22806f751b6SCorentin Labbe goto theend_sgs; 22906f751b6SCorentin Labbe } 23006f751b6SCorentin Labbe } 23106f751b6SCorentin Labbe 23206f751b6SCorentin Labbe len = areq->cryptlen; 23306f751b6SCorentin Labbe for_each_sg(areq->src, sg, nr_sgs, i) { 23493c7f4d3SCorentin Labbe cet->t_src[i].addr = cpu_to_le32(sg_dma_address(sg)); 23506f751b6SCorentin Labbe todo = min(len, sg_dma_len(sg)); 23693c7f4d3SCorentin Labbe cet->t_src[i].len = cpu_to_le32(todo / 4); 23706f751b6SCorentin Labbe dev_dbg(ce->dev, "%s total=%u SG(%d %u off=%d) todo=%u\n", __func__, 23806f751b6SCorentin Labbe areq->cryptlen, i, cet->t_src[i].len, sg->offset, todo); 23906f751b6SCorentin Labbe len -= todo; 24006f751b6SCorentin Labbe } 24106f751b6SCorentin Labbe if (len > 0) { 24206f751b6SCorentin Labbe dev_err(ce->dev, "remaining len %d\n", len); 24306f751b6SCorentin Labbe err = -EINVAL; 24406f751b6SCorentin Labbe goto theend_sgs; 24506f751b6SCorentin Labbe } 24606f751b6SCorentin Labbe 24706f751b6SCorentin Labbe len = areq->cryptlen; 24806f751b6SCorentin Labbe for_each_sg(areq->dst, sg, nr_sgd, i) { 24993c7f4d3SCorentin Labbe cet->t_dst[i].addr = cpu_to_le32(sg_dma_address(sg)); 25006f751b6SCorentin Labbe todo = min(len, sg_dma_len(sg)); 25193c7f4d3SCorentin Labbe cet->t_dst[i].len = cpu_to_le32(todo / 4); 25206f751b6SCorentin Labbe dev_dbg(ce->dev, "%s total=%u SG(%d %u off=%d) todo=%u\n", __func__, 25306f751b6SCorentin Labbe areq->cryptlen, i, cet->t_dst[i].len, sg->offset, todo); 25406f751b6SCorentin Labbe len -= todo; 25506f751b6SCorentin Labbe } 25606f751b6SCorentin Labbe if (len > 0) { 25706f751b6SCorentin Labbe dev_err(ce->dev, "remaining len %d\n", len); 25806f751b6SCorentin Labbe err = -EINVAL; 25906f751b6SCorentin Labbe goto theend_sgs; 26006f751b6SCorentin Labbe } 26106f751b6SCorentin Labbe 26206f751b6SCorentin Labbe chan->timeout = areq->cryptlen; 2630605fa0fSCorentin Labbe rctx->nr_sgs = nr_sgs; 2640605fa0fSCorentin Labbe rctx->nr_sgd = nr_sgd; 2650605fa0fSCorentin Labbe return 0; 26606f751b6SCorentin Labbe 26706f751b6SCorentin Labbe theend_sgs: 26806f751b6SCorentin Labbe if (areq->src == areq->dst) { 2696b8309faSCorentin Labbe dma_unmap_sg(ce->dev, areq->src, ns, DMA_BIDIRECTIONAL); 27006f751b6SCorentin Labbe } else { 27106f751b6SCorentin Labbe if (nr_sgs > 0) 2726b8309faSCorentin Labbe dma_unmap_sg(ce->dev, areq->src, ns, DMA_TO_DEVICE); 2736b8309faSCorentin Labbe dma_unmap_sg(ce->dev, areq->dst, nd, DMA_FROM_DEVICE); 27406f751b6SCorentin Labbe } 27506f751b6SCorentin Labbe 27606f751b6SCorentin Labbe theend_iv: 27706f751b6SCorentin Labbe if (areq->iv && ivsize > 0) { 2780605fa0fSCorentin Labbe if (rctx->addr_iv) 2790605fa0fSCorentin Labbe dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE); 28006f751b6SCorentin Labbe offset = areq->cryptlen - ivsize; 28106f751b6SCorentin Labbe if (rctx->op_dir & CE_DECRYPTION) { 28222f7c2f8SCorentin Labbe memcpy(areq->iv, chan->backup_iv, ivsize); 28322f7c2f8SCorentin Labbe memzero_explicit(chan->backup_iv, ivsize); 28406f751b6SCorentin Labbe } else { 28506f751b6SCorentin Labbe scatterwalk_map_and_copy(areq->iv, areq->dst, offset, 28606f751b6SCorentin Labbe ivsize, 0); 28706f751b6SCorentin Labbe } 28822f7c2f8SCorentin Labbe memzero_explicit(chan->bounce_iv, ivsize); 28906f751b6SCorentin Labbe } 29006f751b6SCorentin Labbe 2910605fa0fSCorentin Labbe dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE); 29206f751b6SCorentin Labbe 29306f751b6SCorentin Labbe theend: 29406f751b6SCorentin Labbe return err; 29506f751b6SCorentin Labbe } 29606f751b6SCorentin Labbe 297*4136212aSHerbert Xu static void sun8i_ce_cipher_run(struct crypto_engine *engine, void *areq) 29806f751b6SCorentin Labbe { 29906f751b6SCorentin Labbe struct skcipher_request *breq = container_of(areq, struct skcipher_request, base); 3000605fa0fSCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(breq); 3010605fa0fSCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); 3020605fa0fSCorentin Labbe struct sun8i_ce_dev *ce = op->ce; 3030605fa0fSCorentin Labbe struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(breq); 3040605fa0fSCorentin Labbe int flow, err; 30506f751b6SCorentin Labbe 3060605fa0fSCorentin Labbe flow = rctx->flow; 3070605fa0fSCorentin Labbe err = sun8i_ce_run_task(ce, flow, crypto_tfm_alg_name(breq->base.tfm)); 308f75a749bSCorentin Labbe local_bh_disable(); 30906f751b6SCorentin Labbe crypto_finalize_skcipher_request(engine, breq, err); 310f75a749bSCorentin Labbe local_bh_enable(); 3110605fa0fSCorentin Labbe } 3120605fa0fSCorentin Labbe 313*4136212aSHerbert Xu static void sun8i_ce_cipher_unprepare(struct crypto_engine *engine, 314*4136212aSHerbert Xu void *async_req) 3150605fa0fSCorentin Labbe { 3160605fa0fSCorentin Labbe struct skcipher_request *areq = container_of(async_req, struct skcipher_request, base); 3170605fa0fSCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); 3180605fa0fSCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); 3190605fa0fSCorentin Labbe struct sun8i_ce_dev *ce = op->ce; 3200605fa0fSCorentin Labbe struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); 3210605fa0fSCorentin Labbe struct sun8i_ce_flow *chan; 3220605fa0fSCorentin Labbe struct ce_task *cet; 3230605fa0fSCorentin Labbe unsigned int ivsize, offset; 3240605fa0fSCorentin Labbe int nr_sgs = rctx->nr_sgs; 3250605fa0fSCorentin Labbe int nr_sgd = rctx->nr_sgd; 3260605fa0fSCorentin Labbe int flow; 3270605fa0fSCorentin Labbe 3280605fa0fSCorentin Labbe flow = rctx->flow; 3290605fa0fSCorentin Labbe chan = &ce->chanlist[flow]; 3300605fa0fSCorentin Labbe cet = chan->tl; 3310605fa0fSCorentin Labbe ivsize = crypto_skcipher_ivsize(tfm); 3320605fa0fSCorentin Labbe 3330605fa0fSCorentin Labbe if (areq->src == areq->dst) { 3340605fa0fSCorentin Labbe dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_BIDIRECTIONAL); 3350605fa0fSCorentin Labbe } else { 3360605fa0fSCorentin Labbe if (nr_sgs > 0) 3370605fa0fSCorentin Labbe dma_unmap_sg(ce->dev, areq->src, nr_sgs, DMA_TO_DEVICE); 3380605fa0fSCorentin Labbe dma_unmap_sg(ce->dev, areq->dst, nr_sgd, DMA_FROM_DEVICE); 3390605fa0fSCorentin Labbe } 3400605fa0fSCorentin Labbe 3410605fa0fSCorentin Labbe if (areq->iv && ivsize > 0) { 3420605fa0fSCorentin Labbe if (cet->t_iv) 3430605fa0fSCorentin Labbe dma_unmap_single(ce->dev, rctx->addr_iv, rctx->ivlen, DMA_TO_DEVICE); 3440605fa0fSCorentin Labbe offset = areq->cryptlen - ivsize; 3450605fa0fSCorentin Labbe if (rctx->op_dir & CE_DECRYPTION) { 34622f7c2f8SCorentin Labbe memcpy(areq->iv, chan->backup_iv, ivsize); 34722f7c2f8SCorentin Labbe memzero_explicit(chan->backup_iv, ivsize); 3480605fa0fSCorentin Labbe } else { 3490605fa0fSCorentin Labbe scatterwalk_map_and_copy(areq->iv, areq->dst, offset, 3500605fa0fSCorentin Labbe ivsize, 0); 3510605fa0fSCorentin Labbe } 35222f7c2f8SCorentin Labbe memzero_explicit(chan->bounce_iv, ivsize); 3530605fa0fSCorentin Labbe } 3540605fa0fSCorentin Labbe 3550605fa0fSCorentin Labbe dma_unmap_single(ce->dev, rctx->addr_key, op->keylen, DMA_TO_DEVICE); 356*4136212aSHerbert Xu } 35706f751b6SCorentin Labbe 358*4136212aSHerbert Xu static int sun8i_ce_cipher_do_one(struct crypto_engine *engine, void *areq) 359*4136212aSHerbert Xu { 360*4136212aSHerbert Xu int err = sun8i_ce_cipher_prepare(engine, areq); 361*4136212aSHerbert Xu 362*4136212aSHerbert Xu if (err) 363*4136212aSHerbert Xu return err; 364*4136212aSHerbert Xu 365*4136212aSHerbert Xu sun8i_ce_cipher_run(engine, areq); 366*4136212aSHerbert Xu sun8i_ce_cipher_unprepare(engine, areq); 36706f751b6SCorentin Labbe return 0; 36806f751b6SCorentin Labbe } 36906f751b6SCorentin Labbe 37006f751b6SCorentin Labbe int sun8i_ce_skdecrypt(struct skcipher_request *areq) 37106f751b6SCorentin Labbe { 37206f751b6SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); 37306f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); 37406f751b6SCorentin Labbe struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); 37506f751b6SCorentin Labbe struct crypto_engine *engine; 37606f751b6SCorentin Labbe int e; 37706f751b6SCorentin Labbe 37806f751b6SCorentin Labbe rctx->op_dir = CE_DECRYPTION; 37906f751b6SCorentin Labbe if (sun8i_ce_cipher_need_fallback(areq)) 38006f751b6SCorentin Labbe return sun8i_ce_cipher_fallback(areq); 38106f751b6SCorentin Labbe 38206f751b6SCorentin Labbe e = sun8i_ce_get_engine_number(op->ce); 38306f751b6SCorentin Labbe rctx->flow = e; 38406f751b6SCorentin Labbe engine = op->ce->chanlist[e].engine; 38506f751b6SCorentin Labbe 38606f751b6SCorentin Labbe return crypto_transfer_skcipher_request_to_engine(engine, areq); 38706f751b6SCorentin Labbe } 38806f751b6SCorentin Labbe 38906f751b6SCorentin Labbe int sun8i_ce_skencrypt(struct skcipher_request *areq) 39006f751b6SCorentin Labbe { 39106f751b6SCorentin Labbe struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); 39206f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); 39306f751b6SCorentin Labbe struct sun8i_cipher_req_ctx *rctx = skcipher_request_ctx(areq); 39406f751b6SCorentin Labbe struct crypto_engine *engine; 39506f751b6SCorentin Labbe int e; 39606f751b6SCorentin Labbe 39706f751b6SCorentin Labbe rctx->op_dir = CE_ENCRYPTION; 39806f751b6SCorentin Labbe if (sun8i_ce_cipher_need_fallback(areq)) 39906f751b6SCorentin Labbe return sun8i_ce_cipher_fallback(areq); 40006f751b6SCorentin Labbe 40106f751b6SCorentin Labbe e = sun8i_ce_get_engine_number(op->ce); 40206f751b6SCorentin Labbe rctx->flow = e; 40306f751b6SCorentin Labbe engine = op->ce->chanlist[e].engine; 40406f751b6SCorentin Labbe 40506f751b6SCorentin Labbe return crypto_transfer_skcipher_request_to_engine(engine, areq); 40606f751b6SCorentin Labbe } 40706f751b6SCorentin Labbe 40806f751b6SCorentin Labbe int sun8i_ce_cipher_init(struct crypto_tfm *tfm) 40906f751b6SCorentin Labbe { 41006f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); 41106f751b6SCorentin Labbe struct sun8i_ce_alg_template *algt; 41206f751b6SCorentin Labbe const char *name = crypto_tfm_alg_name(tfm); 41306f751b6SCorentin Labbe struct crypto_skcipher *sktfm = __crypto_skcipher_cast(tfm); 41406f751b6SCorentin Labbe struct skcipher_alg *alg = crypto_skcipher_alg(sktfm); 41506f751b6SCorentin Labbe int err; 41606f751b6SCorentin Labbe 41706f751b6SCorentin Labbe memset(op, 0, sizeof(struct sun8i_cipher_tfm_ctx)); 41806f751b6SCorentin Labbe 41906f751b6SCorentin Labbe algt = container_of(alg, struct sun8i_ce_alg_template, alg.skcipher); 42006f751b6SCorentin Labbe op->ce = algt->ce; 42106f751b6SCorentin Labbe 42231abd3ebSArd Biesheuvel op->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); 42306f751b6SCorentin Labbe if (IS_ERR(op->fallback_tfm)) { 42406f751b6SCorentin Labbe dev_err(op->ce->dev, "ERROR: Cannot allocate fallback for %s %ld\n", 42506f751b6SCorentin Labbe name, PTR_ERR(op->fallback_tfm)); 42606f751b6SCorentin Labbe return PTR_ERR(op->fallback_tfm); 42706f751b6SCorentin Labbe } 42806f751b6SCorentin Labbe 42931abd3ebSArd Biesheuvel sktfm->reqsize = sizeof(struct sun8i_cipher_req_ctx) + 43031abd3ebSArd Biesheuvel crypto_skcipher_reqsize(op->fallback_tfm); 43131abd3ebSArd Biesheuvel 432aff388f7SCorentin Labbe memcpy(algt->fbname, 433aff388f7SCorentin Labbe crypto_tfm_alg_driver_name(crypto_skcipher_tfm(op->fallback_tfm)), 434aff388f7SCorentin Labbe CRYPTO_MAX_ALG_NAME); 43506f751b6SCorentin Labbe 436*4136212aSHerbert Xu op->enginectx.op.do_one_request = sun8i_ce_cipher_do_one; 43706f751b6SCorentin Labbe 43806f751b6SCorentin Labbe err = pm_runtime_get_sync(op->ce->dev); 43906f751b6SCorentin Labbe if (err < 0) 44006f751b6SCorentin Labbe goto error_pm; 44106f751b6SCorentin Labbe 44206f751b6SCorentin Labbe return 0; 44306f751b6SCorentin Labbe error_pm: 4445c3a8a66SDinghao Liu pm_runtime_put_noidle(op->ce->dev); 44531abd3ebSArd Biesheuvel crypto_free_skcipher(op->fallback_tfm); 44606f751b6SCorentin Labbe return err; 44706f751b6SCorentin Labbe } 44806f751b6SCorentin Labbe 44906f751b6SCorentin Labbe void sun8i_ce_cipher_exit(struct crypto_tfm *tfm) 45006f751b6SCorentin Labbe { 45106f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_tfm_ctx(tfm); 45206f751b6SCorentin Labbe 453712d8069SDenis Efremov kfree_sensitive(op->key); 45431abd3ebSArd Biesheuvel crypto_free_skcipher(op->fallback_tfm); 45506f751b6SCorentin Labbe pm_runtime_put_sync_suspend(op->ce->dev); 45606f751b6SCorentin Labbe } 45706f751b6SCorentin Labbe 45806f751b6SCorentin Labbe int sun8i_ce_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, 45906f751b6SCorentin Labbe unsigned int keylen) 46006f751b6SCorentin Labbe { 46106f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); 46206f751b6SCorentin Labbe struct sun8i_ce_dev *ce = op->ce; 46306f751b6SCorentin Labbe 46406f751b6SCorentin Labbe switch (keylen) { 46506f751b6SCorentin Labbe case 128 / 8: 46606f751b6SCorentin Labbe break; 46706f751b6SCorentin Labbe case 192 / 8: 46806f751b6SCorentin Labbe break; 46906f751b6SCorentin Labbe case 256 / 8: 47006f751b6SCorentin Labbe break; 47106f751b6SCorentin Labbe default: 47206f751b6SCorentin Labbe dev_dbg(ce->dev, "ERROR: Invalid keylen %u\n", keylen); 47306f751b6SCorentin Labbe return -EINVAL; 47406f751b6SCorentin Labbe } 475712d8069SDenis Efremov kfree_sensitive(op->key); 47606f751b6SCorentin Labbe op->keylen = keylen; 477c7351845SYueHaibing op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); 47806f751b6SCorentin Labbe if (!op->key) 47906f751b6SCorentin Labbe return -ENOMEM; 48006f751b6SCorentin Labbe 48131abd3ebSArd Biesheuvel crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); 48231abd3ebSArd Biesheuvel crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); 48306f751b6SCorentin Labbe 48431abd3ebSArd Biesheuvel return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); 48506f751b6SCorentin Labbe } 48606f751b6SCorentin Labbe 48706f751b6SCorentin Labbe int sun8i_ce_des3_setkey(struct crypto_skcipher *tfm, const u8 *key, 48806f751b6SCorentin Labbe unsigned int keylen) 48906f751b6SCorentin Labbe { 49006f751b6SCorentin Labbe struct sun8i_cipher_tfm_ctx *op = crypto_skcipher_ctx(tfm); 49106f751b6SCorentin Labbe int err; 49206f751b6SCorentin Labbe 49306f751b6SCorentin Labbe err = verify_skcipher_des3_key(tfm, key); 49406f751b6SCorentin Labbe if (err) 49506f751b6SCorentin Labbe return err; 49606f751b6SCorentin Labbe 497712d8069SDenis Efremov kfree_sensitive(op->key); 49806f751b6SCorentin Labbe op->keylen = keylen; 499c7351845SYueHaibing op->key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); 50006f751b6SCorentin Labbe if (!op->key) 50106f751b6SCorentin Labbe return -ENOMEM; 50206f751b6SCorentin Labbe 50331abd3ebSArd Biesheuvel crypto_skcipher_clear_flags(op->fallback_tfm, CRYPTO_TFM_REQ_MASK); 50431abd3ebSArd Biesheuvel crypto_skcipher_set_flags(op->fallback_tfm, tfm->base.crt_flags & CRYPTO_TFM_REQ_MASK); 50506f751b6SCorentin Labbe 50631abd3ebSArd Biesheuvel return crypto_skcipher_setkey(op->fallback_tfm, key, keylen); 50706f751b6SCorentin Labbe } 508