1416d8220SZaibo Xu // SPDX-License-Identifier: GPL-2.0 2416d8220SZaibo Xu /* Copyright (c) 2019 HiSilicon Limited. */ 3416d8220SZaibo Xu 4416d8220SZaibo Xu #include <crypto/aes.h> 5*6c46a329SKai Ye #include <crypto/aead.h> 6416d8220SZaibo Xu #include <crypto/algapi.h> 72f072d75SZaibo Xu #include <crypto/authenc.h> 8416d8220SZaibo Xu #include <crypto/des.h> 92f072d75SZaibo Xu #include <crypto/hash.h> 102f072d75SZaibo Xu #include <crypto/internal/aead.h> 11ae6ce7b1SKai Ye #include <crypto/internal/des.h> 12a24d22b2SEric Biggers #include <crypto/sha1.h> 13a24d22b2SEric Biggers #include <crypto/sha2.h> 14416d8220SZaibo Xu #include <crypto/skcipher.h> 15416d8220SZaibo Xu #include <crypto/xts.h> 16416d8220SZaibo Xu #include <linux/crypto.h> 17416d8220SZaibo Xu #include <linux/dma-mapping.h> 18416d8220SZaibo Xu #include <linux/idr.h> 19416d8220SZaibo Xu 20416d8220SZaibo Xu #include "sec.h" 21416d8220SZaibo Xu #include "sec_crypto.h" 22416d8220SZaibo Xu 23416d8220SZaibo Xu #define SEC_PRIORITY 4001 24416d8220SZaibo Xu #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) 255652d55aSKai Ye #define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE) 26416d8220SZaibo Xu #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) 27416d8220SZaibo Xu #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE) 28416d8220SZaibo Xu #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE) 29416d8220SZaibo Xu 30416d8220SZaibo Xu /* SEC sqe(bd) bit operational relative MACRO */ 31416d8220SZaibo Xu #define SEC_DE_OFFSET 1 32416d8220SZaibo Xu #define SEC_CIPHER_OFFSET 4 33416d8220SZaibo Xu #define SEC_SCENE_OFFSET 3 34416d8220SZaibo Xu #define SEC_DST_SGL_OFFSET 2 35416d8220SZaibo Xu #define SEC_SRC_SGL_OFFSET 7 36416d8220SZaibo Xu #define SEC_CKEY_OFFSET 9 37416d8220SZaibo Xu #define SEC_CMODE_OFFSET 12 382f072d75SZaibo Xu #define SEC_AKEY_OFFSET 5 392f072d75SZaibo Xu #define SEC_AEAD_ALG_OFFSET 11 402f072d75SZaibo Xu #define SEC_AUTH_OFFSET 6 412f072d75SZaibo Xu 42adc3f65aSKai Ye #define SEC_DE_OFFSET_V3 9 43adc3f65aSKai Ye #define SEC_SCENE_OFFSET_V3 5 44adc3f65aSKai Ye #define SEC_CKEY_OFFSET_V3 13 45adc3f65aSKai Ye #define SEC_SRC_SGL_OFFSET_V3 11 46adc3f65aSKai Ye #define SEC_DST_SGL_OFFSET_V3 14 47adc3f65aSKai Ye #define SEC_CALG_OFFSET_V3 4 48adc3f65aSKai Ye #define SEC_AKEY_OFFSET_V3 9 49adc3f65aSKai Ye #define SEC_MAC_OFFSET_V3 4 50adc3f65aSKai Ye #define SEC_AUTH_ALG_OFFSET_V3 15 51adc3f65aSKai Ye #define SEC_CIPHER_AUTH_V3 0xbf 52adc3f65aSKai Ye #define SEC_AUTH_CIPHER_V3 0x40 53416d8220SZaibo Xu #define SEC_FLAG_OFFSET 7 54416d8220SZaibo Xu #define SEC_FLAG_MASK 0x0780 55416d8220SZaibo Xu #define SEC_TYPE_MASK 0x0F 56416d8220SZaibo Xu #define SEC_DONE_MASK 0x0001 57adc3f65aSKai Ye #define SEC_SQE_LEN_RATE_MASK 0x3 58416d8220SZaibo Xu 59416d8220SZaibo Xu #define SEC_TOTAL_IV_SZ (SEC_IV_SIZE * QM_Q_DEPTH) 60416d8220SZaibo Xu #define SEC_SGL_SGE_NR 128 612f072d75SZaibo Xu #define SEC_CIPHER_AUTH 0xfe 622f072d75SZaibo Xu #define SEC_AUTH_CIPHER 0x1 632f072d75SZaibo Xu #define SEC_MAX_MAC_LEN 64 642514f559SLongfang Liu #define SEC_MAX_AAD_LEN 65535 652f072d75SZaibo Xu #define SEC_TOTAL_MAC_SZ (SEC_MAX_MAC_LEN * QM_Q_DEPTH) 6674b58db8SLongfang Liu 6774b58db8SLongfang Liu #define SEC_PBUF_SZ 512 6874b58db8SLongfang Liu #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ 6974b58db8SLongfang Liu #define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE) 7074b58db8SLongfang Liu #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ 7174b58db8SLongfang Liu SEC_MAX_MAC_LEN * 2) 7274b58db8SLongfang Liu #define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG) 7374b58db8SLongfang Liu #define SEC_PBUF_PAGE_NUM (QM_Q_DEPTH / SEC_PBUF_NUM) 7474b58db8SLongfang Liu #define SEC_PBUF_LEFT_SZ (SEC_PBUF_PKG * (QM_Q_DEPTH - \ 7574b58db8SLongfang Liu SEC_PBUF_PAGE_NUM * SEC_PBUF_NUM)) 7674b58db8SLongfang Liu #define SEC_TOTAL_PBUF_SZ (PAGE_SIZE * SEC_PBUF_PAGE_NUM + \ 7774b58db8SLongfang Liu SEC_PBUF_LEFT_SZ) 7874b58db8SLongfang Liu 792f072d75SZaibo Xu #define SEC_SQE_LEN_RATE 4 80d6de2a59SZaibo Xu #define SEC_SQE_CFLAG 2 812f072d75SZaibo Xu #define SEC_SQE_AEAD_FLAG 3 82d6de2a59SZaibo Xu #define SEC_SQE_DONE 0x1 83c16a70c1SKai Ye #define MIN_MAC_LEN 4 84c16a70c1SKai Ye #define MAC_LEN_MASK 0x1U 857b44c0eeSKai Ye #define MAX_INPUT_DATA_LEN 0xFFFE00 867b44c0eeSKai Ye #define BITS_MASK 0xFF 877b44c0eeSKai Ye #define BYTE_BITS 0x8 885652d55aSKai Ye #define SEC_XTS_NAME_SZ 0x3 89c16a70c1SKai Ye #define IV_CM_CAL_NUM 2 90c16a70c1SKai Ye #define IV_CL_MASK 0x7 91c16a70c1SKai Ye #define IV_CL_MIN 2 92c16a70c1SKai Ye #define IV_CL_MID 4 93c16a70c1SKai Ye #define IV_CL_MAX 8 94c16a70c1SKai Ye #define IV_FLAGS_OFFSET 0x6 95c16a70c1SKai Ye #define IV_CM_OFFSET 0x3 96c16a70c1SKai Ye #define IV_LAST_BYTE1 1 97c16a70c1SKai Ye #define IV_LAST_BYTE2 2 98c16a70c1SKai Ye #define IV_LAST_BYTE_MASK 0xFF 99c16a70c1SKai Ye #define IV_CTR_INIT 0x1 100c16a70c1SKai Ye #define IV_BYTE_OFFSET 0x8 101416d8220SZaibo Xu 102416d8220SZaibo Xu /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ 103a181647cSZaibo Xu static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) 104416d8220SZaibo Xu { 105416d8220SZaibo Xu if (req->c_req.encrypt) 106416d8220SZaibo Xu return (u32)atomic_inc_return(&ctx->enc_qcyclic) % 107416d8220SZaibo Xu ctx->hlf_q_num; 108416d8220SZaibo Xu 109416d8220SZaibo Xu return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num + 110416d8220SZaibo Xu ctx->hlf_q_num; 111416d8220SZaibo Xu } 112416d8220SZaibo Xu 113a181647cSZaibo Xu static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req) 114416d8220SZaibo Xu { 115416d8220SZaibo Xu if (req->c_req.encrypt) 116416d8220SZaibo Xu atomic_dec(&ctx->enc_qcyclic); 117416d8220SZaibo Xu else 118416d8220SZaibo Xu atomic_dec(&ctx->dec_qcyclic); 119416d8220SZaibo Xu } 120416d8220SZaibo Xu 121416d8220SZaibo Xu static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) 122416d8220SZaibo Xu { 123416d8220SZaibo Xu int req_id; 124416d8220SZaibo Xu 125416d8220SZaibo Xu mutex_lock(&qp_ctx->req_lock); 126416d8220SZaibo Xu 127416d8220SZaibo Xu req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 128416d8220SZaibo Xu 0, QM_Q_DEPTH, GFP_ATOMIC); 129416d8220SZaibo Xu mutex_unlock(&qp_ctx->req_lock); 130b9c8d897SZaibo Xu if (unlikely(req_id < 0)) { 131a44dce50SLongfang Liu dev_err(req->ctx->dev, "alloc req id fail!\n"); 132416d8220SZaibo Xu return req_id; 133416d8220SZaibo Xu } 134416d8220SZaibo Xu 135416d8220SZaibo Xu req->qp_ctx = qp_ctx; 136416d8220SZaibo Xu qp_ctx->req_list[req_id] = req; 137633e507fSLongfang Liu 138416d8220SZaibo Xu return req_id; 139416d8220SZaibo Xu } 140416d8220SZaibo Xu 141416d8220SZaibo Xu static void sec_free_req_id(struct sec_req *req) 142416d8220SZaibo Xu { 143416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 144416d8220SZaibo Xu int req_id = req->req_id; 145416d8220SZaibo Xu 146b9c8d897SZaibo Xu if (unlikely(req_id < 0 || req_id >= QM_Q_DEPTH)) { 147a44dce50SLongfang Liu dev_err(req->ctx->dev, "free request id invalid!\n"); 148416d8220SZaibo Xu return; 149416d8220SZaibo Xu } 150416d8220SZaibo Xu 151416d8220SZaibo Xu qp_ctx->req_list[req_id] = NULL; 152416d8220SZaibo Xu req->qp_ctx = NULL; 153416d8220SZaibo Xu 154416d8220SZaibo Xu mutex_lock(&qp_ctx->req_lock); 155416d8220SZaibo Xu idr_remove(&qp_ctx->req_idr, req_id); 156416d8220SZaibo Xu mutex_unlock(&qp_ctx->req_lock); 157416d8220SZaibo Xu } 158416d8220SZaibo Xu 1592514f559SLongfang Liu static int sec_aead_verify(struct sec_req *req) 1602f072d75SZaibo Xu { 1612f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req; 1622f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 1632f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm); 1642514f559SLongfang Liu u8 *mac_out = req->aead_req.out_mac; 1652f072d75SZaibo Xu u8 *mac = mac_out + SEC_MAX_MAC_LEN; 1662f072d75SZaibo Xu struct scatterlist *sgl = aead_req->src; 1672f072d75SZaibo Xu size_t sz; 1682f072d75SZaibo Xu 1692f072d75SZaibo Xu sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac, authsize, 1702f072d75SZaibo Xu aead_req->cryptlen + aead_req->assoclen - 1712f072d75SZaibo Xu authsize); 1722f072d75SZaibo Xu if (unlikely(sz != authsize || memcmp(mac_out, mac, sz))) { 173a44dce50SLongfang Liu dev_err(req->ctx->dev, "aead verify failure!\n"); 1742f072d75SZaibo Xu return -EBADMSG; 1752f072d75SZaibo Xu } 1762f072d75SZaibo Xu 1772f072d75SZaibo Xu return 0; 1782f072d75SZaibo Xu } 1792f072d75SZaibo Xu 180adc3f65aSKai Ye static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) 181adc3f65aSKai Ye { 182adc3f65aSKai Ye struct sec_sqe *bd = resp; 183adc3f65aSKai Ye 184adc3f65aSKai Ye status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; 185adc3f65aSKai Ye status->flag = (le16_to_cpu(bd->type2.done_flag) & 186adc3f65aSKai Ye SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 187adc3f65aSKai Ye status->tag = le16_to_cpu(bd->type2.tag); 188adc3f65aSKai Ye status->err_type = bd->type2.error_type; 189adc3f65aSKai Ye 190adc3f65aSKai Ye return bd->type_cipher_auth & SEC_TYPE_MASK; 191adc3f65aSKai Ye } 192adc3f65aSKai Ye 193adc3f65aSKai Ye static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) 194adc3f65aSKai Ye { 195adc3f65aSKai Ye struct sec_sqe3 *bd3 = resp; 196adc3f65aSKai Ye 197adc3f65aSKai Ye status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK; 198adc3f65aSKai Ye status->flag = (le16_to_cpu(bd3->done_flag) & 199adc3f65aSKai Ye SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 200adc3f65aSKai Ye status->tag = le64_to_cpu(bd3->tag); 201adc3f65aSKai Ye status->err_type = bd3->error_type; 202adc3f65aSKai Ye 203adc3f65aSKai Ye return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK; 204adc3f65aSKai Ye } 205adc3f65aSKai Ye 206adc3f65aSKai Ye static int sec_cb_status_check(struct sec_req *req, 207adc3f65aSKai Ye struct bd_status *status) 208adc3f65aSKai Ye { 209adc3f65aSKai Ye struct sec_ctx *ctx = req->ctx; 210adc3f65aSKai Ye 211adc3f65aSKai Ye if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) { 212adc3f65aSKai Ye dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n", 213adc3f65aSKai Ye req->err_type, status->done); 214adc3f65aSKai Ye return -EIO; 215adc3f65aSKai Ye } 216adc3f65aSKai Ye 217adc3f65aSKai Ye if (unlikely(ctx->alg_type == SEC_SKCIPHER)) { 218adc3f65aSKai Ye if (unlikely(status->flag != SEC_SQE_CFLAG)) { 219adc3f65aSKai Ye dev_err_ratelimited(ctx->dev, "flag[%u]\n", 220adc3f65aSKai Ye status->flag); 221adc3f65aSKai Ye return -EIO; 222adc3f65aSKai Ye } 223adc3f65aSKai Ye } 224adc3f65aSKai Ye 225adc3f65aSKai Ye return 0; 226adc3f65aSKai Ye } 227adc3f65aSKai Ye 228416d8220SZaibo Xu static void sec_req_cb(struct hisi_qp *qp, void *resp) 229416d8220SZaibo Xu { 230416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = qp->qp_ctx; 2318213a1a6SKai Ye struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; 232adc3f65aSKai Ye u8 type_supported = qp_ctx->ctx->type_supported; 233adc3f65aSKai Ye struct bd_status status; 234d6de2a59SZaibo Xu struct sec_ctx *ctx; 235d6de2a59SZaibo Xu struct sec_req *req; 236adc3f65aSKai Ye int err; 237416d8220SZaibo Xu u8 type; 238416d8220SZaibo Xu 239adc3f65aSKai Ye if (type_supported == SEC_BD_TYPE2) { 240adc3f65aSKai Ye type = pre_parse_finished_bd(&status, resp); 241adc3f65aSKai Ye req = qp_ctx->req_list[status.tag]; 242adc3f65aSKai Ye } else { 243adc3f65aSKai Ye type = pre_parse_finished_bd3(&status, resp); 244adc3f65aSKai Ye req = (void *)(uintptr_t)status.tag; 245adc3f65aSKai Ye } 246adc3f65aSKai Ye 247adc3f65aSKai Ye if (unlikely(type != type_supported)) { 2488213a1a6SKai Ye atomic64_inc(&dfx->err_bd_cnt); 249416d8220SZaibo Xu pr_err("err bd type [%d]\n", type); 250416d8220SZaibo Xu return; 251416d8220SZaibo Xu } 252416d8220SZaibo Xu 2538213a1a6SKai Ye if (unlikely(!req)) { 2548213a1a6SKai Ye atomic64_inc(&dfx->invalid_req_cnt); 2559597efc3SKai Ye atomic_inc(&qp->qp_status.used); 2568213a1a6SKai Ye return; 2578213a1a6SKai Ye } 258adc3f65aSKai Ye 259adc3f65aSKai Ye req->err_type = status.err_type; 260d6de2a59SZaibo Xu ctx = req->ctx; 261adc3f65aSKai Ye err = sec_cb_status_check(req, &status); 262adc3f65aSKai Ye if (err) 2638213a1a6SKai Ye atomic64_inc(&dfx->done_flag_cnt); 2641e9bc276SZaibo Xu 2652f072d75SZaibo Xu if (ctx->alg_type == SEC_AEAD && !req->c_req.encrypt) 2662514f559SLongfang Liu err = sec_aead_verify(req); 2672f072d75SZaibo Xu 2688213a1a6SKai Ye atomic64_inc(&dfx->recv_cnt); 269416d8220SZaibo Xu 270d6de2a59SZaibo Xu ctx->req_op->buf_unmap(ctx, req); 271d6de2a59SZaibo Xu 272310ea0acSZaibo Xu ctx->req_op->callback(ctx, req, err); 273416d8220SZaibo Xu } 274416d8220SZaibo Xu 275416d8220SZaibo Xu static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) 276416d8220SZaibo Xu { 277416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 278416d8220SZaibo Xu int ret; 279416d8220SZaibo Xu 2809597efc3SKai Ye if (ctx->fake_req_limit <= 2819597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && 2829597efc3SKai Ye !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)) 2839597efc3SKai Ye return -EBUSY; 2849597efc3SKai Ye 285416d8220SZaibo Xu mutex_lock(&qp_ctx->req_lock); 286416d8220SZaibo Xu ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); 2879597efc3SKai Ye 2889597efc3SKai Ye if (ctx->fake_req_limit <= 2899597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && !ret) { 2909597efc3SKai Ye list_add_tail(&req->backlog_head, &qp_ctx->backlog); 291cb1eeb75SArnd Bergmann atomic64_inc(&ctx->sec->debug.dfx.send_cnt); 2929597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt); 2939597efc3SKai Ye mutex_unlock(&qp_ctx->req_lock); 2949597efc3SKai Ye return -EBUSY; 2959597efc3SKai Ye } 2969597efc3SKai Ye mutex_unlock(&qp_ctx->req_lock); 297416d8220SZaibo Xu 298b9c8d897SZaibo Xu if (unlikely(ret == -EBUSY)) 299416d8220SZaibo Xu return -ENOBUFS; 300416d8220SZaibo Xu 3019597efc3SKai Ye if (likely(!ret)) { 302416d8220SZaibo Xu ret = -EINPROGRESS; 3039597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.send_cnt); 3048213a1a6SKai Ye } 305416d8220SZaibo Xu 306416d8220SZaibo Xu return ret; 307416d8220SZaibo Xu } 308416d8220SZaibo Xu 3097c7d902aSZaibo Xu /* Get DMA memory resources */ 3107c7d902aSZaibo Xu static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) 3117c7d902aSZaibo Xu { 3127c7d902aSZaibo Xu int i; 3137c7d902aSZaibo Xu 3147c7d902aSZaibo Xu res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ, 3157c7d902aSZaibo Xu &res->c_ivin_dma, GFP_KERNEL); 3167c7d902aSZaibo Xu if (!res->c_ivin) 3177c7d902aSZaibo Xu return -ENOMEM; 3187c7d902aSZaibo Xu 3197c7d902aSZaibo Xu for (i = 1; i < QM_Q_DEPTH; i++) { 3207c7d902aSZaibo Xu res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE; 3217c7d902aSZaibo Xu res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE; 3227c7d902aSZaibo Xu } 3237c7d902aSZaibo Xu 3247c7d902aSZaibo Xu return 0; 3257c7d902aSZaibo Xu } 3267c7d902aSZaibo Xu 3277c7d902aSZaibo Xu static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) 3287c7d902aSZaibo Xu { 3297c7d902aSZaibo Xu if (res->c_ivin) 3307c7d902aSZaibo Xu dma_free_coherent(dev, SEC_TOTAL_IV_SZ, 3317c7d902aSZaibo Xu res->c_ivin, res->c_ivin_dma); 3327c7d902aSZaibo Xu } 3337c7d902aSZaibo Xu 334c16a70c1SKai Ye static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) 335c16a70c1SKai Ye { 336c16a70c1SKai Ye int i; 337c16a70c1SKai Ye 338c16a70c1SKai Ye res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ, 339c16a70c1SKai Ye &res->a_ivin_dma, GFP_KERNEL); 340c16a70c1SKai Ye if (!res->a_ivin) 341c16a70c1SKai Ye return -ENOMEM; 342c16a70c1SKai Ye 343c16a70c1SKai Ye for (i = 1; i < QM_Q_DEPTH; i++) { 344c16a70c1SKai Ye res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE; 345c16a70c1SKai Ye res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE; 346c16a70c1SKai Ye } 347c16a70c1SKai Ye 348c16a70c1SKai Ye return 0; 349c16a70c1SKai Ye } 350c16a70c1SKai Ye 351c16a70c1SKai Ye static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res) 352c16a70c1SKai Ye { 353c16a70c1SKai Ye if (res->a_ivin) 354c16a70c1SKai Ye dma_free_coherent(dev, SEC_TOTAL_IV_SZ, 355c16a70c1SKai Ye res->a_ivin, res->a_ivin_dma); 356c16a70c1SKai Ye } 357c16a70c1SKai Ye 3582f072d75SZaibo Xu static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) 3592f072d75SZaibo Xu { 3602f072d75SZaibo Xu int i; 3612f072d75SZaibo Xu 3622f072d75SZaibo Xu res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ << 1, 3632f072d75SZaibo Xu &res->out_mac_dma, GFP_KERNEL); 3642f072d75SZaibo Xu if (!res->out_mac) 3652f072d75SZaibo Xu return -ENOMEM; 3662f072d75SZaibo Xu 3672f072d75SZaibo Xu for (i = 1; i < QM_Q_DEPTH; i++) { 3682f072d75SZaibo Xu res[i].out_mac_dma = res->out_mac_dma + 3692f072d75SZaibo Xu i * (SEC_MAX_MAC_LEN << 1); 3702f072d75SZaibo Xu res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1); 3712f072d75SZaibo Xu } 3722f072d75SZaibo Xu 3732f072d75SZaibo Xu return 0; 3742f072d75SZaibo Xu } 3752f072d75SZaibo Xu 3762f072d75SZaibo Xu static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) 3772f072d75SZaibo Xu { 3782f072d75SZaibo Xu if (res->out_mac) 3792f072d75SZaibo Xu dma_free_coherent(dev, SEC_TOTAL_MAC_SZ << 1, 3802f072d75SZaibo Xu res->out_mac, res->out_mac_dma); 3812f072d75SZaibo Xu } 3822f072d75SZaibo Xu 38374b58db8SLongfang Liu static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) 38474b58db8SLongfang Liu { 38574b58db8SLongfang Liu if (res->pbuf) 38674b58db8SLongfang Liu dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ, 38774b58db8SLongfang Liu res->pbuf, res->pbuf_dma); 38874b58db8SLongfang Liu } 38974b58db8SLongfang Liu 39074b58db8SLongfang Liu /* 39174b58db8SLongfang Liu * To improve performance, pbuffer is used for 39274b58db8SLongfang Liu * small packets (< 512Bytes) as IOMMU translation using. 39374b58db8SLongfang Liu */ 39474b58db8SLongfang Liu static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) 39574b58db8SLongfang Liu { 39674b58db8SLongfang Liu int pbuf_page_offset; 39774b58db8SLongfang Liu int i, j, k; 39874b58db8SLongfang Liu 39974b58db8SLongfang Liu res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ, 40074b58db8SLongfang Liu &res->pbuf_dma, GFP_KERNEL); 40174b58db8SLongfang Liu if (!res->pbuf) 40274b58db8SLongfang Liu return -ENOMEM; 40374b58db8SLongfang Liu 40474b58db8SLongfang Liu /* 40574b58db8SLongfang Liu * SEC_PBUF_PKG contains data pbuf, iv and 40674b58db8SLongfang Liu * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC> 40774b58db8SLongfang Liu * Every PAGE contains six SEC_PBUF_PKG 40874b58db8SLongfang Liu * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG 40974b58db8SLongfang Liu * So we need SEC_PBUF_PAGE_NUM numbers of PAGE 41074b58db8SLongfang Liu * for the SEC_TOTAL_PBUF_SZ 41174b58db8SLongfang Liu */ 41274b58db8SLongfang Liu for (i = 0; i <= SEC_PBUF_PAGE_NUM; i++) { 41374b58db8SLongfang Liu pbuf_page_offset = PAGE_SIZE * i; 41474b58db8SLongfang Liu for (j = 0; j < SEC_PBUF_NUM; j++) { 41574b58db8SLongfang Liu k = i * SEC_PBUF_NUM + j; 41674b58db8SLongfang Liu if (k == QM_Q_DEPTH) 41774b58db8SLongfang Liu break; 41874b58db8SLongfang Liu res[k].pbuf = res->pbuf + 41974b58db8SLongfang Liu j * SEC_PBUF_PKG + pbuf_page_offset; 42074b58db8SLongfang Liu res[k].pbuf_dma = res->pbuf_dma + 42174b58db8SLongfang Liu j * SEC_PBUF_PKG + pbuf_page_offset; 42274b58db8SLongfang Liu } 42374b58db8SLongfang Liu } 424633e507fSLongfang Liu 42574b58db8SLongfang Liu return 0; 42674b58db8SLongfang Liu } 42774b58db8SLongfang Liu 4287c7d902aSZaibo Xu static int sec_alg_resource_alloc(struct sec_ctx *ctx, 4297c7d902aSZaibo Xu struct sec_qp_ctx *qp_ctx) 4307c7d902aSZaibo Xu { 4312f072d75SZaibo Xu struct sec_alg_res *res = qp_ctx->res; 432a44dce50SLongfang Liu struct device *dev = ctx->dev; 4332f072d75SZaibo Xu int ret; 4347c7d902aSZaibo Xu 4352f072d75SZaibo Xu ret = sec_alloc_civ_resource(dev, res); 4362f072d75SZaibo Xu if (ret) 4372f072d75SZaibo Xu return ret; 4382f072d75SZaibo Xu 4392f072d75SZaibo Xu if (ctx->alg_type == SEC_AEAD) { 440c16a70c1SKai Ye ret = sec_alloc_aiv_resource(dev, res); 441c16a70c1SKai Ye if (ret) 442c16a70c1SKai Ye goto alloc_aiv_fail; 443c16a70c1SKai Ye 4442f072d75SZaibo Xu ret = sec_alloc_mac_resource(dev, res); 4452f072d75SZaibo Xu if (ret) 446c16a70c1SKai Ye goto alloc_mac_fail; 4472f072d75SZaibo Xu } 44874b58db8SLongfang Liu if (ctx->pbuf_supported) { 44974b58db8SLongfang Liu ret = sec_alloc_pbuf_resource(dev, res); 45074b58db8SLongfang Liu if (ret) { 45174b58db8SLongfang Liu dev_err(dev, "fail to alloc pbuf dma resource!\n"); 45224efcec2SLongfang Liu goto alloc_pbuf_fail; 45374b58db8SLongfang Liu } 45474b58db8SLongfang Liu } 4552f072d75SZaibo Xu 4562f072d75SZaibo Xu return 0; 457633e507fSLongfang Liu 45824efcec2SLongfang Liu alloc_pbuf_fail: 45924efcec2SLongfang Liu if (ctx->alg_type == SEC_AEAD) 46024efcec2SLongfang Liu sec_free_mac_resource(dev, qp_ctx->res); 461c16a70c1SKai Ye alloc_mac_fail: 462c16a70c1SKai Ye if (ctx->alg_type == SEC_AEAD) 463c16a70c1SKai Ye sec_free_aiv_resource(dev, res); 464c16a70c1SKai Ye alloc_aiv_fail: 4652f072d75SZaibo Xu sec_free_civ_resource(dev, res); 4662f072d75SZaibo Xu return ret; 4677c7d902aSZaibo Xu } 4687c7d902aSZaibo Xu 4697c7d902aSZaibo Xu static void sec_alg_resource_free(struct sec_ctx *ctx, 4707c7d902aSZaibo Xu struct sec_qp_ctx *qp_ctx) 4717c7d902aSZaibo Xu { 472a44dce50SLongfang Liu struct device *dev = ctx->dev; 4737c7d902aSZaibo Xu 4747c7d902aSZaibo Xu sec_free_civ_resource(dev, qp_ctx->res); 4752f072d75SZaibo Xu 47674b58db8SLongfang Liu if (ctx->pbuf_supported) 47774b58db8SLongfang Liu sec_free_pbuf_resource(dev, qp_ctx->res); 4782f072d75SZaibo Xu if (ctx->alg_type == SEC_AEAD) 4792f072d75SZaibo Xu sec_free_mac_resource(dev, qp_ctx->res); 4807c7d902aSZaibo Xu } 4817c7d902aSZaibo Xu 482416d8220SZaibo Xu static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, 483416d8220SZaibo Xu int qp_ctx_id, int alg_type) 484416d8220SZaibo Xu { 485a44dce50SLongfang Liu struct device *dev = ctx->dev; 486416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx; 487416d8220SZaibo Xu struct hisi_qp *qp; 488416d8220SZaibo Xu int ret = -ENOMEM; 489416d8220SZaibo Xu 490416d8220SZaibo Xu qp_ctx = &ctx->qp_ctx[qp_ctx_id]; 4910b5e43bcSKai Ye qp = ctx->qps[qp_ctx_id]; 492416d8220SZaibo Xu qp->req_type = 0; 493416d8220SZaibo Xu qp->qp_ctx = qp_ctx; 494416d8220SZaibo Xu qp_ctx->qp = qp; 495416d8220SZaibo Xu qp_ctx->ctx = ctx; 496416d8220SZaibo Xu 497adc3f65aSKai Ye qp->req_cb = sec_req_cb; 498adc3f65aSKai Ye 499416d8220SZaibo Xu mutex_init(&qp_ctx->req_lock); 500416d8220SZaibo Xu idr_init(&qp_ctx->req_idr); 5019597efc3SKai Ye INIT_LIST_HEAD(&qp_ctx->backlog); 502416d8220SZaibo Xu 503416d8220SZaibo Xu qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, 504416d8220SZaibo Xu SEC_SGL_SGE_NR); 5058a6b8f4dSDan Carpenter if (IS_ERR(qp_ctx->c_in_pool)) { 506416d8220SZaibo Xu dev_err(dev, "fail to create sgl pool for input!\n"); 5077c7d902aSZaibo Xu goto err_destroy_idr; 508416d8220SZaibo Xu } 509416d8220SZaibo Xu 510416d8220SZaibo Xu qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, QM_Q_DEPTH, 511416d8220SZaibo Xu SEC_SGL_SGE_NR); 5128a6b8f4dSDan Carpenter if (IS_ERR(qp_ctx->c_out_pool)) { 513416d8220SZaibo Xu dev_err(dev, "fail to create sgl pool for output!\n"); 514416d8220SZaibo Xu goto err_free_c_in_pool; 515416d8220SZaibo Xu } 516416d8220SZaibo Xu 5177c7d902aSZaibo Xu ret = sec_alg_resource_alloc(ctx, qp_ctx); 518416d8220SZaibo Xu if (ret) 519416d8220SZaibo Xu goto err_free_c_out_pool; 520416d8220SZaibo Xu 521416d8220SZaibo Xu ret = hisi_qm_start_qp(qp, 0); 522416d8220SZaibo Xu if (ret < 0) 523416d8220SZaibo Xu goto err_queue_free; 524416d8220SZaibo Xu 525416d8220SZaibo Xu return 0; 526416d8220SZaibo Xu 527416d8220SZaibo Xu err_queue_free: 5287c7d902aSZaibo Xu sec_alg_resource_free(ctx, qp_ctx); 529416d8220SZaibo Xu err_free_c_out_pool: 530416d8220SZaibo Xu hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 531416d8220SZaibo Xu err_free_c_in_pool: 532416d8220SZaibo Xu hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 533416d8220SZaibo Xu err_destroy_idr: 534416d8220SZaibo Xu idr_destroy(&qp_ctx->req_idr); 535416d8220SZaibo Xu return ret; 536416d8220SZaibo Xu } 537416d8220SZaibo Xu 538416d8220SZaibo Xu static void sec_release_qp_ctx(struct sec_ctx *ctx, 539416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx) 540416d8220SZaibo Xu { 541a44dce50SLongfang Liu struct device *dev = ctx->dev; 542416d8220SZaibo Xu 543416d8220SZaibo Xu hisi_qm_stop_qp(qp_ctx->qp); 5447c7d902aSZaibo Xu sec_alg_resource_free(ctx, qp_ctx); 545416d8220SZaibo Xu 546416d8220SZaibo Xu hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 547416d8220SZaibo Xu hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 548416d8220SZaibo Xu 549416d8220SZaibo Xu idr_destroy(&qp_ctx->req_idr); 550416d8220SZaibo Xu } 551416d8220SZaibo Xu 552473a0f96SZaibo Xu static int sec_ctx_base_init(struct sec_ctx *ctx) 553416d8220SZaibo Xu { 554416d8220SZaibo Xu struct sec_dev *sec; 555416d8220SZaibo Xu int i, ret; 556416d8220SZaibo Xu 5570b5e43bcSKai Ye ctx->qps = sec_create_qps(); 5580b5e43bcSKai Ye if (!ctx->qps) { 5590b5e43bcSKai Ye pr_err("Can not create sec qps!\n"); 560416d8220SZaibo Xu return -ENODEV; 561416d8220SZaibo Xu } 5620b5e43bcSKai Ye 5630b5e43bcSKai Ye sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); 564416d8220SZaibo Xu ctx->sec = sec; 565a44dce50SLongfang Liu ctx->dev = &sec->qm.pdev->dev; 566a718cfceSZaibo Xu ctx->hlf_q_num = sec->ctx_q_num >> 1; 567416d8220SZaibo Xu 56874b58db8SLongfang Liu ctx->pbuf_supported = ctx->sec->iommu_used; 56974b58db8SLongfang Liu 570416d8220SZaibo Xu /* Half of queue depth is taken as fake requests limit in the queue. */ 571a718cfceSZaibo Xu ctx->fake_req_limit = QM_Q_DEPTH >> 1; 572416d8220SZaibo Xu ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), 573416d8220SZaibo Xu GFP_KERNEL); 57424efcec2SLongfang Liu if (!ctx->qp_ctx) { 57524efcec2SLongfang Liu ret = -ENOMEM; 57624efcec2SLongfang Liu goto err_destroy_qps; 57724efcec2SLongfang Liu } 578416d8220SZaibo Xu 579416d8220SZaibo Xu for (i = 0; i < sec->ctx_q_num; i++) { 580473a0f96SZaibo Xu ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0); 581416d8220SZaibo Xu if (ret) 582416d8220SZaibo Xu goto err_sec_release_qp_ctx; 583416d8220SZaibo Xu } 584416d8220SZaibo Xu 585416d8220SZaibo Xu return 0; 58624efcec2SLongfang Liu 587416d8220SZaibo Xu err_sec_release_qp_ctx: 588416d8220SZaibo Xu for (i = i - 1; i >= 0; i--) 589416d8220SZaibo Xu sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 590416d8220SZaibo Xu kfree(ctx->qp_ctx); 59124efcec2SLongfang Liu err_destroy_qps: 59224efcec2SLongfang Liu sec_destroy_qps(ctx->qps, sec->ctx_q_num); 593416d8220SZaibo Xu return ret; 594416d8220SZaibo Xu } 595416d8220SZaibo Xu 596473a0f96SZaibo Xu static void sec_ctx_base_uninit(struct sec_ctx *ctx) 597416d8220SZaibo Xu { 598473a0f96SZaibo Xu int i; 599416d8220SZaibo Xu 600416d8220SZaibo Xu for (i = 0; i < ctx->sec->ctx_q_num; i++) 601416d8220SZaibo Xu sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 602416d8220SZaibo Xu 6030b5e43bcSKai Ye sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num); 604416d8220SZaibo Xu kfree(ctx->qp_ctx); 605416d8220SZaibo Xu } 606416d8220SZaibo Xu 607473a0f96SZaibo Xu static int sec_cipher_init(struct sec_ctx *ctx) 608473a0f96SZaibo Xu { 609473a0f96SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 610473a0f96SZaibo Xu 611a44dce50SLongfang Liu c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 612473a0f96SZaibo Xu &c_ctx->c_key_dma, GFP_KERNEL); 613473a0f96SZaibo Xu if (!c_ctx->c_key) 614473a0f96SZaibo Xu return -ENOMEM; 615473a0f96SZaibo Xu 616473a0f96SZaibo Xu return 0; 617473a0f96SZaibo Xu } 618473a0f96SZaibo Xu 619473a0f96SZaibo Xu static void sec_cipher_uninit(struct sec_ctx *ctx) 620473a0f96SZaibo Xu { 621473a0f96SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 622473a0f96SZaibo Xu 623473a0f96SZaibo Xu memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); 624a44dce50SLongfang Liu dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 625473a0f96SZaibo Xu c_ctx->c_key, c_ctx->c_key_dma); 626473a0f96SZaibo Xu } 627473a0f96SZaibo Xu 6282f072d75SZaibo Xu static int sec_auth_init(struct sec_ctx *ctx) 6292f072d75SZaibo Xu { 6302f072d75SZaibo Xu struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 6312f072d75SZaibo Xu 632a44dce50SLongfang Liu a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 6332f072d75SZaibo Xu &a_ctx->a_key_dma, GFP_KERNEL); 6342f072d75SZaibo Xu if (!a_ctx->a_key) 6352f072d75SZaibo Xu return -ENOMEM; 6362f072d75SZaibo Xu 6372f072d75SZaibo Xu return 0; 6382f072d75SZaibo Xu } 6392f072d75SZaibo Xu 6402f072d75SZaibo Xu static void sec_auth_uninit(struct sec_ctx *ctx) 6412f072d75SZaibo Xu { 6422f072d75SZaibo Xu struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 6432f072d75SZaibo Xu 6442f072d75SZaibo Xu memzero_explicit(a_ctx->a_key, SEC_MAX_KEY_SIZE); 645a44dce50SLongfang Liu dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 6462f072d75SZaibo Xu a_ctx->a_key, a_ctx->a_key_dma); 6472f072d75SZaibo Xu } 6482f072d75SZaibo Xu 6495652d55aSKai Ye static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) 6505652d55aSKai Ye { 6515652d55aSKai Ye const char *alg = crypto_tfm_alg_name(&tfm->base); 6525652d55aSKai Ye struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 6535652d55aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 6545652d55aSKai Ye 6555652d55aSKai Ye c_ctx->fallback = false; 6565652d55aSKai Ye if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ))) 6575652d55aSKai Ye return 0; 6585652d55aSKai Ye 6595652d55aSKai Ye c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0, 6605652d55aSKai Ye CRYPTO_ALG_NEED_FALLBACK); 6615652d55aSKai Ye if (IS_ERR(c_ctx->fbtfm)) { 6625652d55aSKai Ye pr_err("failed to alloc fallback tfm!\n"); 6635652d55aSKai Ye return PTR_ERR(c_ctx->fbtfm); 6645652d55aSKai Ye } 6655652d55aSKai Ye 6665652d55aSKai Ye return 0; 6675652d55aSKai Ye } 6685652d55aSKai Ye 669473a0f96SZaibo Xu static int sec_skcipher_init(struct crypto_skcipher *tfm) 670473a0f96SZaibo Xu { 671473a0f96SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 672473a0f96SZaibo Xu int ret; 673473a0f96SZaibo Xu 6742f072d75SZaibo Xu ctx->alg_type = SEC_SKCIPHER; 675473a0f96SZaibo Xu crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req)); 676473a0f96SZaibo Xu ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); 677473a0f96SZaibo Xu if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { 6784b7aef02SLongfang Liu pr_err("get error skcipher iv size!\n"); 679473a0f96SZaibo Xu return -EINVAL; 680473a0f96SZaibo Xu } 681473a0f96SZaibo Xu 682473a0f96SZaibo Xu ret = sec_ctx_base_init(ctx); 683473a0f96SZaibo Xu if (ret) 684473a0f96SZaibo Xu return ret; 685473a0f96SZaibo Xu 686473a0f96SZaibo Xu ret = sec_cipher_init(ctx); 687473a0f96SZaibo Xu if (ret) 688473a0f96SZaibo Xu goto err_cipher_init; 689473a0f96SZaibo Xu 6905652d55aSKai Ye ret = sec_skcipher_fbtfm_init(tfm); 6915652d55aSKai Ye if (ret) 6925652d55aSKai Ye goto err_fbtfm_init; 6935652d55aSKai Ye 694473a0f96SZaibo Xu return 0; 695633e507fSLongfang Liu 6965652d55aSKai Ye err_fbtfm_init: 6975652d55aSKai Ye sec_cipher_uninit(ctx); 698473a0f96SZaibo Xu err_cipher_init: 699473a0f96SZaibo Xu sec_ctx_base_uninit(ctx); 700473a0f96SZaibo Xu return ret; 701473a0f96SZaibo Xu } 702473a0f96SZaibo Xu 703473a0f96SZaibo Xu static void sec_skcipher_uninit(struct crypto_skcipher *tfm) 704473a0f96SZaibo Xu { 705473a0f96SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 706473a0f96SZaibo Xu 7075652d55aSKai Ye if (ctx->c_ctx.fbtfm) 7085652d55aSKai Ye crypto_free_sync_skcipher(ctx->c_ctx.fbtfm); 7095652d55aSKai Ye 710473a0f96SZaibo Xu sec_cipher_uninit(ctx); 711473a0f96SZaibo Xu sec_ctx_base_uninit(ctx); 712473a0f96SZaibo Xu } 713473a0f96SZaibo Xu 714ae6ce7b1SKai Ye static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key, 715416d8220SZaibo Xu const u32 keylen, 716416d8220SZaibo Xu const enum sec_cmode c_mode) 717416d8220SZaibo Xu { 718ae6ce7b1SKai Ye struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 719ae6ce7b1SKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 720ae6ce7b1SKai Ye int ret; 721ae6ce7b1SKai Ye 722ae6ce7b1SKai Ye ret = verify_skcipher_des3_key(tfm, key); 723ae6ce7b1SKai Ye if (ret) 724ae6ce7b1SKai Ye return ret; 725ae6ce7b1SKai Ye 726416d8220SZaibo Xu switch (keylen) { 727416d8220SZaibo Xu case SEC_DES3_2KEY_SIZE: 728416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_3DES_2KEY; 729416d8220SZaibo Xu break; 730416d8220SZaibo Xu case SEC_DES3_3KEY_SIZE: 731416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_3DES_3KEY; 732416d8220SZaibo Xu break; 733416d8220SZaibo Xu default: 734416d8220SZaibo Xu return -EINVAL; 735416d8220SZaibo Xu } 736416d8220SZaibo Xu 737416d8220SZaibo Xu return 0; 738416d8220SZaibo Xu } 739416d8220SZaibo Xu 740416d8220SZaibo Xu static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, 741416d8220SZaibo Xu const u32 keylen, 742416d8220SZaibo Xu const enum sec_cmode c_mode) 743416d8220SZaibo Xu { 744416d8220SZaibo Xu if (c_mode == SEC_CMODE_XTS) { 745416d8220SZaibo Xu switch (keylen) { 746416d8220SZaibo Xu case SEC_XTS_MIN_KEY_SIZE: 747416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT; 748416d8220SZaibo Xu break; 7495652d55aSKai Ye case SEC_XTS_MID_KEY_SIZE: 7505652d55aSKai Ye c_ctx->fallback = true; 7515652d55aSKai Ye break; 752416d8220SZaibo Xu case SEC_XTS_MAX_KEY_SIZE: 753416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT; 754416d8220SZaibo Xu break; 755416d8220SZaibo Xu default: 756416d8220SZaibo Xu pr_err("hisi_sec2: xts mode key error!\n"); 757416d8220SZaibo Xu return -EINVAL; 758416d8220SZaibo Xu } 759416d8220SZaibo Xu } else { 760adc3f65aSKai Ye if (c_ctx->c_alg == SEC_CALG_SM4 && 761adc3f65aSKai Ye keylen != AES_KEYSIZE_128) { 762adc3f65aSKai Ye pr_err("hisi_sec2: sm4 key error!\n"); 763adc3f65aSKai Ye return -EINVAL; 764adc3f65aSKai Ye } else { 765416d8220SZaibo Xu switch (keylen) { 766416d8220SZaibo Xu case AES_KEYSIZE_128: 767416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT; 768416d8220SZaibo Xu break; 769416d8220SZaibo Xu case AES_KEYSIZE_192: 770416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_192BIT; 771416d8220SZaibo Xu break; 772416d8220SZaibo Xu case AES_KEYSIZE_256: 773416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT; 774416d8220SZaibo Xu break; 775416d8220SZaibo Xu default: 776416d8220SZaibo Xu pr_err("hisi_sec2: aes key error!\n"); 777416d8220SZaibo Xu return -EINVAL; 778416d8220SZaibo Xu } 779416d8220SZaibo Xu } 780adc3f65aSKai Ye } 781416d8220SZaibo Xu 782416d8220SZaibo Xu return 0; 783416d8220SZaibo Xu } 784416d8220SZaibo Xu 785416d8220SZaibo Xu static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 786416d8220SZaibo Xu const u32 keylen, const enum sec_calg c_alg, 787416d8220SZaibo Xu const enum sec_cmode c_mode) 788416d8220SZaibo Xu { 789416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 790416d8220SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 791a44dce50SLongfang Liu struct device *dev = ctx->dev; 792416d8220SZaibo Xu int ret; 793416d8220SZaibo Xu 794416d8220SZaibo Xu if (c_mode == SEC_CMODE_XTS) { 795416d8220SZaibo Xu ret = xts_verify_key(tfm, key, keylen); 796416d8220SZaibo Xu if (ret) { 797a44dce50SLongfang Liu dev_err(dev, "xts mode key err!\n"); 798416d8220SZaibo Xu return ret; 799416d8220SZaibo Xu } 800416d8220SZaibo Xu } 801416d8220SZaibo Xu 802416d8220SZaibo Xu c_ctx->c_alg = c_alg; 803416d8220SZaibo Xu c_ctx->c_mode = c_mode; 804416d8220SZaibo Xu 805416d8220SZaibo Xu switch (c_alg) { 806416d8220SZaibo Xu case SEC_CALG_3DES: 807ae6ce7b1SKai Ye ret = sec_skcipher_3des_setkey(tfm, key, keylen, c_mode); 808416d8220SZaibo Xu break; 809416d8220SZaibo Xu case SEC_CALG_AES: 810416d8220SZaibo Xu case SEC_CALG_SM4: 811416d8220SZaibo Xu ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 812416d8220SZaibo Xu break; 813416d8220SZaibo Xu default: 814416d8220SZaibo Xu return -EINVAL; 815416d8220SZaibo Xu } 816416d8220SZaibo Xu 817416d8220SZaibo Xu if (ret) { 818a44dce50SLongfang Liu dev_err(dev, "set sec key err!\n"); 819416d8220SZaibo Xu return ret; 820416d8220SZaibo Xu } 821416d8220SZaibo Xu 822416d8220SZaibo Xu memcpy(c_ctx->c_key, key, keylen); 8235652d55aSKai Ye if (c_ctx->fallback) { 8245652d55aSKai Ye ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); 8255652d55aSKai Ye if (ret) { 8265652d55aSKai Ye dev_err(dev, "failed to set fallback skcipher key!\n"); 8275652d55aSKai Ye return ret; 8285652d55aSKai Ye } 8295652d55aSKai Ye } 830416d8220SZaibo Xu return 0; 831416d8220SZaibo Xu } 832416d8220SZaibo Xu 833416d8220SZaibo Xu #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \ 834416d8220SZaibo Xu static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ 835416d8220SZaibo Xu u32 keylen) \ 836416d8220SZaibo Xu { \ 837416d8220SZaibo Xu return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \ 838416d8220SZaibo Xu } 839416d8220SZaibo Xu 840416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) 841416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) 842416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) 8437b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_ofb, SEC_CALG_AES, SEC_CMODE_OFB) 8447b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_cfb, SEC_CALG_AES, SEC_CMODE_CFB) 8457b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR) 846416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) 847416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) 848416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) 849416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) 8507b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_ofb, SEC_CALG_SM4, SEC_CMODE_OFB) 8517b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_cfb, SEC_CALG_SM4, SEC_CMODE_CFB) 8527b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR) 853416d8220SZaibo Xu 85474b58db8SLongfang Liu static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, 85574b58db8SLongfang Liu struct scatterlist *src) 85674b58db8SLongfang Liu { 857*6c46a329SKai Ye struct sec_aead_req *a_req = &req->aead_req; 858*6c46a329SKai Ye struct aead_request *aead_req = a_req->aead_req; 85974b58db8SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 86074b58db8SLongfang Liu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 861a44dce50SLongfang Liu struct device *dev = ctx->dev; 86274b58db8SLongfang Liu int copy_size, pbuf_length; 86374b58db8SLongfang Liu int req_id = req->req_id; 864*6c46a329SKai Ye struct crypto_aead *tfm; 865*6c46a329SKai Ye size_t authsize; 866*6c46a329SKai Ye u8 *mac_offset; 86774b58db8SLongfang Liu 86874b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) 86974b58db8SLongfang Liu copy_size = aead_req->cryptlen + aead_req->assoclen; 87074b58db8SLongfang Liu else 87174b58db8SLongfang Liu copy_size = c_req->c_len; 87274b58db8SLongfang Liu 87374b58db8SLongfang Liu pbuf_length = sg_copy_to_buffer(src, sg_nents(src), 874*6c46a329SKai Ye qp_ctx->res[req_id].pbuf, copy_size); 87574b58db8SLongfang Liu if (unlikely(pbuf_length != copy_size)) { 87674b58db8SLongfang Liu dev_err(dev, "copy src data to pbuf error!\n"); 87774b58db8SLongfang Liu return -EINVAL; 87874b58db8SLongfang Liu } 879*6c46a329SKai Ye if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { 880*6c46a329SKai Ye tfm = crypto_aead_reqtfm(aead_req); 881*6c46a329SKai Ye authsize = crypto_aead_authsize(tfm); 882*6c46a329SKai Ye mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize; 883*6c46a329SKai Ye memcpy(a_req->out_mac, mac_offset, authsize); 884*6c46a329SKai Ye } 88574b58db8SLongfang Liu 88674b58db8SLongfang Liu c_req->c_in_dma = qp_ctx->res[req_id].pbuf_dma; 88774b58db8SLongfang Liu c_req->c_out_dma = c_req->c_in_dma; 88874b58db8SLongfang Liu 88974b58db8SLongfang Liu return 0; 89074b58db8SLongfang Liu } 89174b58db8SLongfang Liu 89274b58db8SLongfang Liu static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, 89374b58db8SLongfang Liu struct scatterlist *dst) 89474b58db8SLongfang Liu { 89574b58db8SLongfang Liu struct aead_request *aead_req = req->aead_req.aead_req; 89674b58db8SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 89774b58db8SLongfang Liu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 898a44dce50SLongfang Liu struct device *dev = ctx->dev; 89974b58db8SLongfang Liu int copy_size, pbuf_length; 90074b58db8SLongfang Liu int req_id = req->req_id; 90174b58db8SLongfang Liu 90274b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) 90374b58db8SLongfang Liu copy_size = c_req->c_len + aead_req->assoclen; 90474b58db8SLongfang Liu else 90574b58db8SLongfang Liu copy_size = c_req->c_len; 90674b58db8SLongfang Liu 90774b58db8SLongfang Liu pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), 90874b58db8SLongfang Liu qp_ctx->res[req_id].pbuf, 90974b58db8SLongfang Liu copy_size); 91074b58db8SLongfang Liu if (unlikely(pbuf_length != copy_size)) 91174b58db8SLongfang Liu dev_err(dev, "copy pbuf data to dst error!\n"); 91274b58db8SLongfang Liu } 91374b58db8SLongfang Liu 9142514f559SLongfang Liu static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, 915416d8220SZaibo Xu struct scatterlist *src, struct scatterlist *dst) 916416d8220SZaibo Xu { 917416d8220SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 9182514f559SLongfang Liu struct sec_aead_req *a_req = &req->aead_req; 919416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 9202514f559SLongfang Liu struct sec_alg_res *res = &qp_ctx->res[req->req_id]; 921a44dce50SLongfang Liu struct device *dev = ctx->dev; 92274b58db8SLongfang Liu int ret; 9232514f559SLongfang Liu 92474b58db8SLongfang Liu if (req->use_pbuf) { 92574b58db8SLongfang Liu ret = sec_cipher_pbuf_map(ctx, req, src); 92674b58db8SLongfang Liu c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET; 92774b58db8SLongfang Liu c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET; 92874b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) { 929c16a70c1SKai Ye a_req->a_ivin = res->a_ivin; 930c16a70c1SKai Ye a_req->a_ivin_dma = res->a_ivin_dma; 93174b58db8SLongfang Liu a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET; 93274b58db8SLongfang Liu a_req->out_mac_dma = res->pbuf_dma + 93374b58db8SLongfang Liu SEC_PBUF_MAC_OFFSET; 93474b58db8SLongfang Liu } 93574b58db8SLongfang Liu 93674b58db8SLongfang Liu return ret; 93774b58db8SLongfang Liu } 9382514f559SLongfang Liu c_req->c_ivin = res->c_ivin; 9392514f559SLongfang Liu c_req->c_ivin_dma = res->c_ivin_dma; 9402514f559SLongfang Liu if (ctx->alg_type == SEC_AEAD) { 941c16a70c1SKai Ye a_req->a_ivin = res->a_ivin; 942c16a70c1SKai Ye a_req->a_ivin_dma = res->a_ivin_dma; 9432514f559SLongfang Liu a_req->out_mac = res->out_mac; 9442514f559SLongfang Liu a_req->out_mac_dma = res->out_mac_dma; 9452514f559SLongfang Liu } 946416d8220SZaibo Xu 947416d8220SZaibo Xu c_req->c_in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, 948416d8220SZaibo Xu qp_ctx->c_in_pool, 949416d8220SZaibo Xu req->req_id, 950416d8220SZaibo Xu &c_req->c_in_dma); 951416d8220SZaibo Xu 952416d8220SZaibo Xu if (IS_ERR(c_req->c_in)) { 953416d8220SZaibo Xu dev_err(dev, "fail to dma map input sgl buffers!\n"); 954416d8220SZaibo Xu return PTR_ERR(c_req->c_in); 955416d8220SZaibo Xu } 956416d8220SZaibo Xu 957416d8220SZaibo Xu if (dst == src) { 958416d8220SZaibo Xu c_req->c_out = c_req->c_in; 959416d8220SZaibo Xu c_req->c_out_dma = c_req->c_in_dma; 960416d8220SZaibo Xu } else { 961416d8220SZaibo Xu c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, 962416d8220SZaibo Xu qp_ctx->c_out_pool, 963416d8220SZaibo Xu req->req_id, 964416d8220SZaibo Xu &c_req->c_out_dma); 965416d8220SZaibo Xu 966416d8220SZaibo Xu if (IS_ERR(c_req->c_out)) { 967416d8220SZaibo Xu dev_err(dev, "fail to dma map output sgl buffers!\n"); 968416d8220SZaibo Xu hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); 969416d8220SZaibo Xu return PTR_ERR(c_req->c_out); 970416d8220SZaibo Xu } 971416d8220SZaibo Xu } 972416d8220SZaibo Xu 973416d8220SZaibo Xu return 0; 974416d8220SZaibo Xu } 975416d8220SZaibo Xu 9762514f559SLongfang Liu static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, 977a181647cSZaibo Xu struct scatterlist *src, struct scatterlist *dst) 978a181647cSZaibo Xu { 9792514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 980a44dce50SLongfang Liu struct device *dev = ctx->dev; 981a181647cSZaibo Xu 98274b58db8SLongfang Liu if (req->use_pbuf) { 98374b58db8SLongfang Liu sec_cipher_pbuf_unmap(ctx, req, dst); 98474b58db8SLongfang Liu } else { 9852514f559SLongfang Liu if (dst != src) 9862514f559SLongfang Liu hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); 9872514f559SLongfang Liu 9882514f559SLongfang Liu hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out); 989a181647cSZaibo Xu } 99074b58db8SLongfang Liu } 991a181647cSZaibo Xu 992416d8220SZaibo Xu static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 993416d8220SZaibo Xu { 994a181647cSZaibo Xu struct skcipher_request *sq = req->c_req.sk_req; 995416d8220SZaibo Xu 9962514f559SLongfang Liu return sec_cipher_map(ctx, req, sq->src, sq->dst); 997416d8220SZaibo Xu } 998416d8220SZaibo Xu 999416d8220SZaibo Xu static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 1000416d8220SZaibo Xu { 10012514f559SLongfang Liu struct skcipher_request *sq = req->c_req.sk_req; 1002416d8220SZaibo Xu 10032514f559SLongfang Liu sec_cipher_unmap(ctx, req, sq->src, sq->dst); 1004416d8220SZaibo Xu } 1005416d8220SZaibo Xu 10062f072d75SZaibo Xu static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx, 10072f072d75SZaibo Xu struct crypto_authenc_keys *keys) 10082f072d75SZaibo Xu { 10092f072d75SZaibo Xu switch (keys->enckeylen) { 10102f072d75SZaibo Xu case AES_KEYSIZE_128: 10112f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT; 10122f072d75SZaibo Xu break; 10132f072d75SZaibo Xu case AES_KEYSIZE_192: 10142f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_192BIT; 10152f072d75SZaibo Xu break; 10162f072d75SZaibo Xu case AES_KEYSIZE_256: 10172f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT; 10182f072d75SZaibo Xu break; 10192f072d75SZaibo Xu default: 10202f072d75SZaibo Xu pr_err("hisi_sec2: aead aes key error!\n"); 10212f072d75SZaibo Xu return -EINVAL; 10222f072d75SZaibo Xu } 10232f072d75SZaibo Xu memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen); 10242f072d75SZaibo Xu 10252f072d75SZaibo Xu return 0; 10262f072d75SZaibo Xu } 10272f072d75SZaibo Xu 10282f072d75SZaibo Xu static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, 10292f072d75SZaibo Xu struct crypto_authenc_keys *keys) 10302f072d75SZaibo Xu { 10312f072d75SZaibo Xu struct crypto_shash *hash_tfm = ctx->hash_tfm; 10325761498cSKai Ye int blocksize, digestsize, ret; 10332f072d75SZaibo Xu 10342f072d75SZaibo Xu if (!keys->authkeylen) { 10352f072d75SZaibo Xu pr_err("hisi_sec2: aead auth key error!\n"); 10362f072d75SZaibo Xu return -EINVAL; 10372f072d75SZaibo Xu } 10382f072d75SZaibo Xu 10392f072d75SZaibo Xu blocksize = crypto_shash_blocksize(hash_tfm); 10405761498cSKai Ye digestsize = crypto_shash_digestsize(hash_tfm); 10412f072d75SZaibo Xu if (keys->authkeylen > blocksize) { 104261c38e3aSEric Biggers ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey, 10432f072d75SZaibo Xu keys->authkeylen, ctx->a_key); 10442f072d75SZaibo Xu if (ret) { 10452203d3f7SColin Ian King pr_err("hisi_sec2: aead auth digest error!\n"); 10462f072d75SZaibo Xu return -EINVAL; 10472f072d75SZaibo Xu } 10485761498cSKai Ye ctx->a_key_len = digestsize; 10492f072d75SZaibo Xu } else { 10502f072d75SZaibo Xu memcpy(ctx->a_key, keys->authkey, keys->authkeylen); 10512f072d75SZaibo Xu ctx->a_key_len = keys->authkeylen; 10522f072d75SZaibo Xu } 10532f072d75SZaibo Xu 10542f072d75SZaibo Xu return 0; 10552f072d75SZaibo Xu } 10562f072d75SZaibo Xu 1057*6c46a329SKai Ye static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize) 1058*6c46a329SKai Ye { 1059*6c46a329SKai Ye struct crypto_tfm *tfm = crypto_aead_tfm(aead); 1060*6c46a329SKai Ye struct sec_ctx *ctx = crypto_tfm_ctx(tfm); 1061*6c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 1062*6c46a329SKai Ye 1063*6c46a329SKai Ye if (unlikely(a_ctx->fallback_aead_tfm)) 1064*6c46a329SKai Ye return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize); 1065*6c46a329SKai Ye 1066*6c46a329SKai Ye return 0; 1067*6c46a329SKai Ye } 1068*6c46a329SKai Ye 1069*6c46a329SKai Ye static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx, 1070*6c46a329SKai Ye struct crypto_aead *tfm, const u8 *key, 1071*6c46a329SKai Ye unsigned int keylen) 1072*6c46a329SKai Ye { 1073*6c46a329SKai Ye crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK); 1074*6c46a329SKai Ye crypto_aead_set_flags(a_ctx->fallback_aead_tfm, 1075*6c46a329SKai Ye crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); 1076*6c46a329SKai Ye return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen); 1077*6c46a329SKai Ye } 1078*6c46a329SKai Ye 10792f072d75SZaibo Xu static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, 10802f072d75SZaibo Xu const u32 keylen, const enum sec_hash_alg a_alg, 10812f072d75SZaibo Xu const enum sec_calg c_alg, 10822f072d75SZaibo Xu const enum sec_mac_len mac_len, 10832f072d75SZaibo Xu const enum sec_cmode c_mode) 10842f072d75SZaibo Xu { 10852f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 10862f072d75SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1087*6c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 1088a44dce50SLongfang Liu struct device *dev = ctx->dev; 10892f072d75SZaibo Xu struct crypto_authenc_keys keys; 10902f072d75SZaibo Xu int ret; 10912f072d75SZaibo Xu 10922f072d75SZaibo Xu ctx->a_ctx.a_alg = a_alg; 10932f072d75SZaibo Xu ctx->c_ctx.c_alg = c_alg; 10942f072d75SZaibo Xu ctx->a_ctx.mac_len = mac_len; 10952f072d75SZaibo Xu c_ctx->c_mode = c_mode; 10962f072d75SZaibo Xu 1097c16a70c1SKai Ye if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) { 1098c16a70c1SKai Ye ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 1099c16a70c1SKai Ye if (ret) { 1100c16a70c1SKai Ye dev_err(dev, "set sec aes ccm cipher key err!\n"); 1101c16a70c1SKai Ye return ret; 1102c16a70c1SKai Ye } 1103c16a70c1SKai Ye memcpy(c_ctx->c_key, key, keylen); 1104c16a70c1SKai Ye 1105*6c46a329SKai Ye if (unlikely(a_ctx->fallback_aead_tfm)) { 1106*6c46a329SKai Ye ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); 1107*6c46a329SKai Ye if (ret) 1108*6c46a329SKai Ye return ret; 1109*6c46a329SKai Ye } 1110*6c46a329SKai Ye 1111c16a70c1SKai Ye return 0; 1112c16a70c1SKai Ye } 1113c16a70c1SKai Ye 11142f072d75SZaibo Xu if (crypto_authenc_extractkeys(&keys, key, keylen)) 11152f072d75SZaibo Xu goto bad_key; 11162f072d75SZaibo Xu 11172f072d75SZaibo Xu ret = sec_aead_aes_set_key(c_ctx, &keys); 11182f072d75SZaibo Xu if (ret) { 1119a44dce50SLongfang Liu dev_err(dev, "set sec cipher key err!\n"); 11202f072d75SZaibo Xu goto bad_key; 11212f072d75SZaibo Xu } 11222f072d75SZaibo Xu 11232f072d75SZaibo Xu ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys); 11242f072d75SZaibo Xu if (ret) { 1125a44dce50SLongfang Liu dev_err(dev, "set sec auth key err!\n"); 11262f072d75SZaibo Xu goto bad_key; 11272f072d75SZaibo Xu } 11282f072d75SZaibo Xu 1129adc3f65aSKai Ye if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) || 1130adc3f65aSKai Ye (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) { 1131adc3f65aSKai Ye dev_err(dev, "MAC or AUTH key length error!\n"); 1132adc3f65aSKai Ye goto bad_key; 1133adc3f65aSKai Ye } 1134adc3f65aSKai Ye 11352f072d75SZaibo Xu return 0; 1136633e507fSLongfang Liu 11372f072d75SZaibo Xu bad_key: 11382f072d75SZaibo Xu memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); 11392f072d75SZaibo Xu return -EINVAL; 11402f072d75SZaibo Xu } 11412f072d75SZaibo Xu 11422f072d75SZaibo Xu 11432f072d75SZaibo Xu #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \ 11442f072d75SZaibo Xu static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \ 11452f072d75SZaibo Xu u32 keylen) \ 11462f072d75SZaibo Xu { \ 11472f072d75SZaibo Xu return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\ 11482f072d75SZaibo Xu } 11492f072d75SZaibo Xu 11502f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, 11512f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC) 11522f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, 11532f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC) 11542f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, 11552f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC) 1156c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, 1157c16a70c1SKai Ye SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) 1158c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, 1159c16a70c1SKai Ye SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) 1160c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, 1161c16a70c1SKai Ye SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) 1162c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, 1163c16a70c1SKai Ye SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) 11642f072d75SZaibo Xu 11652f072d75SZaibo Xu static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 11662f072d75SZaibo Xu { 11672f072d75SZaibo Xu struct aead_request *aq = req->aead_req.aead_req; 11682f072d75SZaibo Xu 11692514f559SLongfang Liu return sec_cipher_map(ctx, req, aq->src, aq->dst); 11702f072d75SZaibo Xu } 11712f072d75SZaibo Xu 11722f072d75SZaibo Xu static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 11732f072d75SZaibo Xu { 11742f072d75SZaibo Xu struct aead_request *aq = req->aead_req.aead_req; 11752f072d75SZaibo Xu 11762514f559SLongfang Liu sec_cipher_unmap(ctx, req, aq->src, aq->dst); 11772f072d75SZaibo Xu } 11782f072d75SZaibo Xu 1179416d8220SZaibo Xu static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) 1180416d8220SZaibo Xu { 1181416d8220SZaibo Xu int ret; 1182416d8220SZaibo Xu 1183416d8220SZaibo Xu ret = ctx->req_op->buf_map(ctx, req); 1184b9c8d897SZaibo Xu if (unlikely(ret)) 1185416d8220SZaibo Xu return ret; 1186416d8220SZaibo Xu 1187416d8220SZaibo Xu ctx->req_op->do_transfer(ctx, req); 1188416d8220SZaibo Xu 1189416d8220SZaibo Xu ret = ctx->req_op->bd_fill(ctx, req); 1190b9c8d897SZaibo Xu if (unlikely(ret)) 1191416d8220SZaibo Xu goto unmap_req_buf; 1192416d8220SZaibo Xu 1193416d8220SZaibo Xu return ret; 1194416d8220SZaibo Xu 1195416d8220SZaibo Xu unmap_req_buf: 1196416d8220SZaibo Xu ctx->req_op->buf_unmap(ctx, req); 1197416d8220SZaibo Xu return ret; 1198416d8220SZaibo Xu } 1199416d8220SZaibo Xu 1200416d8220SZaibo Xu static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req) 1201416d8220SZaibo Xu { 1202416d8220SZaibo Xu ctx->req_op->buf_unmap(ctx, req); 1203416d8220SZaibo Xu } 1204416d8220SZaibo Xu 1205416d8220SZaibo Xu static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req) 1206416d8220SZaibo Xu { 1207416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req; 12082514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1209416d8220SZaibo Xu 12102514f559SLongfang Liu memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize); 1211416d8220SZaibo Xu } 1212416d8220SZaibo Xu 1213416d8220SZaibo Xu static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 1214416d8220SZaibo Xu { 1215416d8220SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1216416d8220SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 1217416d8220SZaibo Xu struct sec_sqe *sec_sqe = &req->sec_sqe; 1218416d8220SZaibo Xu u8 scene, sa_type, da_type; 1219416d8220SZaibo Xu u8 bd_type, cipher; 12207c7d902aSZaibo Xu u8 de = 0; 1221416d8220SZaibo Xu 1222416d8220SZaibo Xu memset(sec_sqe, 0, sizeof(struct sec_sqe)); 1223416d8220SZaibo Xu 1224416d8220SZaibo Xu sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 12252514f559SLongfang Liu sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 1226416d8220SZaibo Xu sec_sqe->type2.data_src_addr = cpu_to_le64(c_req->c_in_dma); 1227416d8220SZaibo Xu sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1228416d8220SZaibo Xu 1229416d8220SZaibo Xu sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << 1230416d8220SZaibo Xu SEC_CMODE_OFFSET); 1231416d8220SZaibo Xu sec_sqe->type2.c_alg = c_ctx->c_alg; 1232416d8220SZaibo Xu sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1233416d8220SZaibo Xu SEC_CKEY_OFFSET); 1234416d8220SZaibo Xu 1235416d8220SZaibo Xu bd_type = SEC_BD_TYPE2; 1236416d8220SZaibo Xu if (c_req->encrypt) 1237416d8220SZaibo Xu cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; 1238416d8220SZaibo Xu else 1239416d8220SZaibo Xu cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; 1240416d8220SZaibo Xu sec_sqe->type_cipher_auth = bd_type | cipher; 1241416d8220SZaibo Xu 1242adc3f65aSKai Ye /* Set destination and source address type */ 1243adc3f65aSKai Ye if (req->use_pbuf) { 124474b58db8SLongfang Liu sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET; 1245adc3f65aSKai Ye da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; 1246adc3f65aSKai Ye } else { 1247416d8220SZaibo Xu sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; 1248adc3f65aSKai Ye da_type = SEC_SGL << SEC_DST_SGL_OFFSET; 1249adc3f65aSKai Ye } 1250adc3f65aSKai Ye 1251adc3f65aSKai Ye sec_sqe->sdm_addr_type |= da_type; 1252416d8220SZaibo Xu scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; 1253416d8220SZaibo Xu if (c_req->c_in_dma != c_req->c_out_dma) 1254416d8220SZaibo Xu de = 0x1 << SEC_DE_OFFSET; 1255416d8220SZaibo Xu 1256416d8220SZaibo Xu sec_sqe->sds_sa_type = (de | scene | sa_type); 1257416d8220SZaibo Xu 1258416d8220SZaibo Xu sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); 1259416d8220SZaibo Xu sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id); 1260416d8220SZaibo Xu 1261416d8220SZaibo Xu return 0; 1262416d8220SZaibo Xu } 1263416d8220SZaibo Xu 1264adc3f65aSKai Ye static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) 1265adc3f65aSKai Ye { 1266adc3f65aSKai Ye struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; 1267adc3f65aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1268adc3f65aSKai Ye struct sec_cipher_req *c_req = &req->c_req; 1269adc3f65aSKai Ye u32 bd_param = 0; 1270adc3f65aSKai Ye u16 cipher; 1271adc3f65aSKai Ye 1272adc3f65aSKai Ye memset(sec_sqe3, 0, sizeof(struct sec_sqe3)); 1273adc3f65aSKai Ye 1274adc3f65aSKai Ye sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 1275adc3f65aSKai Ye sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 1276adc3f65aSKai Ye sec_sqe3->data_src_addr = cpu_to_le64(c_req->c_in_dma); 1277adc3f65aSKai Ye sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1278adc3f65aSKai Ye 1279adc3f65aSKai Ye sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) | 1280adc3f65aSKai Ye c_ctx->c_mode; 1281adc3f65aSKai Ye sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1282adc3f65aSKai Ye SEC_CKEY_OFFSET_V3); 1283adc3f65aSKai Ye 1284adc3f65aSKai Ye if (c_req->encrypt) 1285adc3f65aSKai Ye cipher = SEC_CIPHER_ENC; 1286adc3f65aSKai Ye else 1287adc3f65aSKai Ye cipher = SEC_CIPHER_DEC; 1288adc3f65aSKai Ye sec_sqe3->c_icv_key |= cpu_to_le16(cipher); 1289adc3f65aSKai Ye 1290adc3f65aSKai Ye if (req->use_pbuf) { 1291adc3f65aSKai Ye bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3; 1292adc3f65aSKai Ye bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3; 1293adc3f65aSKai Ye } else { 1294adc3f65aSKai Ye bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3; 1295adc3f65aSKai Ye bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3; 1296adc3f65aSKai Ye } 1297adc3f65aSKai Ye 1298adc3f65aSKai Ye bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3; 1299adc3f65aSKai Ye if (c_req->c_in_dma != c_req->c_out_dma) 1300adc3f65aSKai Ye bd_param |= 0x1 << SEC_DE_OFFSET_V3; 1301adc3f65aSKai Ye 1302adc3f65aSKai Ye bd_param |= SEC_BD_TYPE3; 1303adc3f65aSKai Ye sec_sqe3->bd_param = cpu_to_le32(bd_param); 1304adc3f65aSKai Ye 1305adc3f65aSKai Ye sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len); 1306adc3f65aSKai Ye sec_sqe3->tag = cpu_to_le64(req); 1307adc3f65aSKai Ye 1308adc3f65aSKai Ye return 0; 1309adc3f65aSKai Ye } 1310adc3f65aSKai Ye 13117b44c0eeSKai Ye /* increment counter (128-bit int) */ 13127b44c0eeSKai Ye static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) 13137b44c0eeSKai Ye { 13147b44c0eeSKai Ye do { 13157b44c0eeSKai Ye --bits; 13167b44c0eeSKai Ye nums += counter[bits]; 13177b44c0eeSKai Ye counter[bits] = nums & BITS_MASK; 13187b44c0eeSKai Ye nums >>= BYTE_BITS; 13197b44c0eeSKai Ye } while (bits && nums); 13207b44c0eeSKai Ye } 13217b44c0eeSKai Ye 13222f072d75SZaibo Xu static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) 1323416d8220SZaibo Xu { 13242f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req; 1325416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req; 1326416d8220SZaibo Xu u32 iv_size = req->ctx->c_ctx.ivsize; 1327416d8220SZaibo Xu struct scatterlist *sgl; 13282f072d75SZaibo Xu unsigned int cryptlen; 1329416d8220SZaibo Xu size_t sz; 13302f072d75SZaibo Xu u8 *iv; 1331416d8220SZaibo Xu 1332416d8220SZaibo Xu if (req->c_req.encrypt) 13332f072d75SZaibo Xu sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst; 1334416d8220SZaibo Xu else 13352f072d75SZaibo Xu sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src; 1336416d8220SZaibo Xu 13372f072d75SZaibo Xu if (alg_type == SEC_SKCIPHER) { 13382f072d75SZaibo Xu iv = sk_req->iv; 13392f072d75SZaibo Xu cryptlen = sk_req->cryptlen; 13402f072d75SZaibo Xu } else { 13412f072d75SZaibo Xu iv = aead_req->iv; 13422f072d75SZaibo Xu cryptlen = aead_req->cryptlen; 13432f072d75SZaibo Xu } 13442f072d75SZaibo Xu 13457b44c0eeSKai Ye if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) { 13462f072d75SZaibo Xu sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, 13472f072d75SZaibo Xu cryptlen - iv_size); 1348b9c8d897SZaibo Xu if (unlikely(sz != iv_size)) 1349a44dce50SLongfang Liu dev_err(req->ctx->dev, "copy output iv error!\n"); 13507b44c0eeSKai Ye } else { 13517b44c0eeSKai Ye sz = cryptlen / iv_size; 13527b44c0eeSKai Ye if (cryptlen % iv_size) 13537b44c0eeSKai Ye sz += 1; 13547b44c0eeSKai Ye ctr_iv_inc(iv, iv_size, sz); 13557b44c0eeSKai Ye } 1356416d8220SZaibo Xu } 1357416d8220SZaibo Xu 13589597efc3SKai Ye static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, 13599597efc3SKai Ye struct sec_qp_ctx *qp_ctx) 13609597efc3SKai Ye { 13619597efc3SKai Ye struct sec_req *backlog_req = NULL; 13629597efc3SKai Ye 13639597efc3SKai Ye mutex_lock(&qp_ctx->req_lock); 13649597efc3SKai Ye if (ctx->fake_req_limit >= 13659597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && 13669597efc3SKai Ye !list_empty(&qp_ctx->backlog)) { 13679597efc3SKai Ye backlog_req = list_first_entry(&qp_ctx->backlog, 13689597efc3SKai Ye typeof(*backlog_req), backlog_head); 13699597efc3SKai Ye list_del(&backlog_req->backlog_head); 13709597efc3SKai Ye } 13719597efc3SKai Ye mutex_unlock(&qp_ctx->req_lock); 13729597efc3SKai Ye 13739597efc3SKai Ye return backlog_req; 13749597efc3SKai Ye } 13759597efc3SKai Ye 1376310ea0acSZaibo Xu static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, 1377310ea0acSZaibo Xu int err) 1378416d8220SZaibo Xu { 1379416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req; 1380416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 13819597efc3SKai Ye struct skcipher_request *backlog_sk_req; 13829597efc3SKai Ye struct sec_req *backlog_req; 1383416d8220SZaibo Xu 1384416d8220SZaibo Xu sec_free_req_id(req); 1385416d8220SZaibo Xu 13867b44c0eeSKai Ye /* IV output at encrypto of CBC/CTR mode */ 13877b44c0eeSKai Ye if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || 13887b44c0eeSKai Ye ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt) 13892f072d75SZaibo Xu sec_update_iv(req, SEC_SKCIPHER); 1390416d8220SZaibo Xu 13919597efc3SKai Ye while (1) { 13929597efc3SKai Ye backlog_req = sec_back_req_clear(ctx, qp_ctx); 13939597efc3SKai Ye if (!backlog_req) 13949597efc3SKai Ye break; 13959597efc3SKai Ye 13969597efc3SKai Ye backlog_sk_req = backlog_req->c_req.sk_req; 13979597efc3SKai Ye backlog_sk_req->base.complete(&backlog_sk_req->base, 13989597efc3SKai Ye -EINPROGRESS); 13999597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt); 14009597efc3SKai Ye } 14019597efc3SKai Ye 1402310ea0acSZaibo Xu sk_req->base.complete(&sk_req->base, err); 1403416d8220SZaibo Xu } 1404416d8220SZaibo Xu 1405c16a70c1SKai Ye static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) 14062f072d75SZaibo Xu { 14072f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req; 14082514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1409c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req; 1410c16a70c1SKai Ye size_t authsize = ctx->a_ctx.mac_len; 1411c16a70c1SKai Ye u32 data_size = aead_req->cryptlen; 1412c16a70c1SKai Ye u8 flage = 0; 1413c16a70c1SKai Ye u8 cm, cl; 1414c16a70c1SKai Ye 1415c16a70c1SKai Ye /* the specification has been checked in aead_iv_demension_check() */ 1416c16a70c1SKai Ye cl = c_req->c_ivin[0] + 1; 1417c16a70c1SKai Ye c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00; 1418c16a70c1SKai Ye memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl); 1419c16a70c1SKai Ye c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT; 1420c16a70c1SKai Ye 1421c16a70c1SKai Ye /* the last 3bit is L' */ 1422c16a70c1SKai Ye flage |= c_req->c_ivin[0] & IV_CL_MASK; 1423c16a70c1SKai Ye 1424c16a70c1SKai Ye /* the M' is bit3~bit5, the Flags is bit6 */ 1425c16a70c1SKai Ye cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM; 1426c16a70c1SKai Ye flage |= cm << IV_CM_OFFSET; 1427c16a70c1SKai Ye if (aead_req->assoclen) 1428c16a70c1SKai Ye flage |= 0x01 << IV_FLAGS_OFFSET; 1429c16a70c1SKai Ye 1430c16a70c1SKai Ye memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize); 1431c16a70c1SKai Ye a_req->a_ivin[0] = flage; 1432c16a70c1SKai Ye 1433c16a70c1SKai Ye /* 1434c16a70c1SKai Ye * the last 32bit is counter's initial number, 1435c16a70c1SKai Ye * but the nonce uses the first 16bit 1436c16a70c1SKai Ye * the tail 16bit fill with the cipher length 1437c16a70c1SKai Ye */ 1438c16a70c1SKai Ye if (!c_req->encrypt) 1439c16a70c1SKai Ye data_size = aead_req->cryptlen - authsize; 1440c16a70c1SKai Ye 1441c16a70c1SKai Ye a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = 1442c16a70c1SKai Ye data_size & IV_LAST_BYTE_MASK; 1443c16a70c1SKai Ye data_size >>= IV_BYTE_OFFSET; 1444c16a70c1SKai Ye a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] = 1445c16a70c1SKai Ye data_size & IV_LAST_BYTE_MASK; 1446c16a70c1SKai Ye } 1447c16a70c1SKai Ye 1448c16a70c1SKai Ye static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) 1449c16a70c1SKai Ye { 1450c16a70c1SKai Ye struct aead_request *aead_req = req->aead_req.aead_req; 1451c16a70c1SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 1452c16a70c1SKai Ye size_t authsize = crypto_aead_authsize(tfm); 1453c16a70c1SKai Ye struct sec_cipher_req *c_req = &req->c_req; 1454c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req; 14552f072d75SZaibo Xu 14562514f559SLongfang Liu memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); 1457c16a70c1SKai Ye 1458c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) { 1459c16a70c1SKai Ye /* 1460c16a70c1SKai Ye * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter}, 1461c16a70c1SKai Ye * the counter must set to 0x01 1462c16a70c1SKai Ye */ 1463c16a70c1SKai Ye ctx->a_ctx.mac_len = authsize; 1464c16a70c1SKai Ye /* CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */ 1465c16a70c1SKai Ye set_aead_auth_iv(ctx, req); 1466c16a70c1SKai Ye } 1467c16a70c1SKai Ye 1468c16a70c1SKai Ye /* GCM 12Byte Cipher_IV == Auth_IV */ 1469c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) { 1470c16a70c1SKai Ye ctx->a_ctx.mac_len = authsize; 1471c16a70c1SKai Ye memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE); 1472c16a70c1SKai Ye } 1473c16a70c1SKai Ye } 1474c16a70c1SKai Ye 1475c16a70c1SKai Ye static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir, 1476c16a70c1SKai Ye struct sec_req *req, struct sec_sqe *sec_sqe) 1477c16a70c1SKai Ye { 1478c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req; 1479c16a70c1SKai Ye struct aead_request *aq = a_req->aead_req; 1480c16a70c1SKai Ye 1481c16a70c1SKai Ye /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ 1482c16a70c1SKai Ye sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)ctx->mac_len); 1483c16a70c1SKai Ye 1484c16a70c1SKai Ye /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ 1485c16a70c1SKai Ye sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr; 1486c16a70c1SKai Ye sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); 1487c16a70c1SKai Ye sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET; 1488c16a70c1SKai Ye 1489c16a70c1SKai Ye if (dir) 1490c16a70c1SKai Ye sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 1491c16a70c1SKai Ye else 1492c16a70c1SKai Ye sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 1493c16a70c1SKai Ye 1494c16a70c1SKai Ye sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen); 1495c16a70c1SKai Ye sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0); 1496c16a70c1SKai Ye sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1497c16a70c1SKai Ye 1498c16a70c1SKai Ye sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); 1499c16a70c1SKai Ye } 1500c16a70c1SKai Ye 1501c16a70c1SKai Ye static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir, 1502c16a70c1SKai Ye struct sec_req *req, struct sec_sqe3 *sqe3) 1503c16a70c1SKai Ye { 1504c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req; 1505c16a70c1SKai Ye struct aead_request *aq = a_req->aead_req; 1506c16a70c1SKai Ye 1507c16a70c1SKai Ye /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ 1508c16a70c1SKai Ye sqe3->c_icv_key |= cpu_to_le16((u16)ctx->mac_len << SEC_MAC_OFFSET_V3); 1509c16a70c1SKai Ye 1510c16a70c1SKai Ye /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ 1511c16a70c1SKai Ye sqe3->a_key_addr = sqe3->c_key_addr; 1512c16a70c1SKai Ye sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); 1513c16a70c1SKai Ye sqe3->auth_mac_key |= SEC_NO_AUTH; 1514c16a70c1SKai Ye 1515c16a70c1SKai Ye if (dir) 1516c16a70c1SKai Ye sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; 1517c16a70c1SKai Ye else 1518c16a70c1SKai Ye sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; 1519c16a70c1SKai Ye 1520c16a70c1SKai Ye sqe3->a_len_key = cpu_to_le32(aq->assoclen); 1521c16a70c1SKai Ye sqe3->auth_src_offset = cpu_to_le16(0x0); 1522c16a70c1SKai Ye sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1523c16a70c1SKai Ye sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); 15242f072d75SZaibo Xu } 15252f072d75SZaibo Xu 15262f072d75SZaibo Xu static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, 15272f072d75SZaibo Xu struct sec_req *req, struct sec_sqe *sec_sqe) 15282f072d75SZaibo Xu { 15292f072d75SZaibo Xu struct sec_aead_req *a_req = &req->aead_req; 15302f072d75SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 15312f072d75SZaibo Xu struct aead_request *aq = a_req->aead_req; 15322f072d75SZaibo Xu 15332f072d75SZaibo Xu sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); 15342f072d75SZaibo Xu 15352f072d75SZaibo Xu sec_sqe->type2.mac_key_alg = 15362f072d75SZaibo Xu cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE); 15372f072d75SZaibo Xu 15382f072d75SZaibo Xu sec_sqe->type2.mac_key_alg |= 15392f072d75SZaibo Xu cpu_to_le32((u32)((ctx->a_key_len) / 15402f072d75SZaibo Xu SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET); 15412f072d75SZaibo Xu 15422f072d75SZaibo Xu sec_sqe->type2.mac_key_alg |= 15432f072d75SZaibo Xu cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); 15442f072d75SZaibo Xu 15452f072d75SZaibo Xu sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; 15462f072d75SZaibo Xu 15472f072d75SZaibo Xu if (dir) 15482f072d75SZaibo Xu sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 15492f072d75SZaibo Xu else 15502f072d75SZaibo Xu sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 15512f072d75SZaibo Xu 15522f072d75SZaibo Xu sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen); 15532f072d75SZaibo Xu 15542f072d75SZaibo Xu sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 15552f072d75SZaibo Xu 15562514f559SLongfang Liu sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); 15572f072d75SZaibo Xu } 15582f072d75SZaibo Xu 15592f072d75SZaibo Xu static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 15602f072d75SZaibo Xu { 15612f072d75SZaibo Xu struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 15622f072d75SZaibo Xu struct sec_sqe *sec_sqe = &req->sec_sqe; 15632f072d75SZaibo Xu int ret; 15642f072d75SZaibo Xu 15652f072d75SZaibo Xu ret = sec_skcipher_bd_fill(ctx, req); 15662f072d75SZaibo Xu if (unlikely(ret)) { 1567a44dce50SLongfang Liu dev_err(ctx->dev, "skcipher bd fill is error!\n"); 15682f072d75SZaibo Xu return ret; 15692f072d75SZaibo Xu } 15702f072d75SZaibo Xu 1571c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || 1572c16a70c1SKai Ye ctx->c_ctx.c_mode == SEC_CMODE_GCM) 1573c16a70c1SKai Ye sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe); 1574c16a70c1SKai Ye else 15752f072d75SZaibo Xu sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); 15762f072d75SZaibo Xu 15772f072d75SZaibo Xu return 0; 15782f072d75SZaibo Xu } 15792f072d75SZaibo Xu 1580adc3f65aSKai Ye static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, 1581adc3f65aSKai Ye struct sec_req *req, struct sec_sqe3 *sqe3) 1582adc3f65aSKai Ye { 1583adc3f65aSKai Ye struct sec_aead_req *a_req = &req->aead_req; 1584adc3f65aSKai Ye struct sec_cipher_req *c_req = &req->c_req; 1585adc3f65aSKai Ye struct aead_request *aq = a_req->aead_req; 1586adc3f65aSKai Ye 1587adc3f65aSKai Ye sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); 1588adc3f65aSKai Ye 1589adc3f65aSKai Ye sqe3->auth_mac_key |= 1590adc3f65aSKai Ye cpu_to_le32((u32)(ctx->mac_len / 1591adc3f65aSKai Ye SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3); 1592adc3f65aSKai Ye 1593adc3f65aSKai Ye sqe3->auth_mac_key |= 1594adc3f65aSKai Ye cpu_to_le32((u32)(ctx->a_key_len / 1595adc3f65aSKai Ye SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3); 1596adc3f65aSKai Ye 1597adc3f65aSKai Ye sqe3->auth_mac_key |= 1598adc3f65aSKai Ye cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3); 1599adc3f65aSKai Ye 1600adc3f65aSKai Ye if (dir) { 1601adc3f65aSKai Ye sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); 1602adc3f65aSKai Ye sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; 1603adc3f65aSKai Ye } else { 1604adc3f65aSKai Ye sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); 1605adc3f65aSKai Ye sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; 1606adc3f65aSKai Ye } 1607adc3f65aSKai Ye sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen); 1608adc3f65aSKai Ye 1609adc3f65aSKai Ye sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1610adc3f65aSKai Ye 1611adc3f65aSKai Ye sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); 1612adc3f65aSKai Ye } 1613adc3f65aSKai Ye 1614adc3f65aSKai Ye static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) 1615adc3f65aSKai Ye { 1616adc3f65aSKai Ye struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 1617adc3f65aSKai Ye struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; 1618adc3f65aSKai Ye int ret; 1619adc3f65aSKai Ye 1620adc3f65aSKai Ye ret = sec_skcipher_bd_fill_v3(ctx, req); 1621adc3f65aSKai Ye if (unlikely(ret)) { 1622adc3f65aSKai Ye dev_err(ctx->dev, "skcipher bd3 fill is error!\n"); 1623adc3f65aSKai Ye return ret; 1624adc3f65aSKai Ye } 1625adc3f65aSKai Ye 1626c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || 1627c16a70c1SKai Ye ctx->c_ctx.c_mode == SEC_CMODE_GCM) 1628c16a70c1SKai Ye sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt, 1629c16a70c1SKai Ye req, sec_sqe3); 1630c16a70c1SKai Ye else 1631c16a70c1SKai Ye sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt, 1632c16a70c1SKai Ye req, sec_sqe3); 1633adc3f65aSKai Ye 1634adc3f65aSKai Ye return 0; 1635adc3f65aSKai Ye } 1636adc3f65aSKai Ye 16372f072d75SZaibo Xu static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) 16382f072d75SZaibo Xu { 16392f072d75SZaibo Xu struct aead_request *a_req = req->aead_req.aead_req; 16402f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 16412514f559SLongfang Liu struct sec_aead_req *aead_req = &req->aead_req; 16422f072d75SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 16432f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm); 16442f072d75SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 16459597efc3SKai Ye struct aead_request *backlog_aead_req; 16469597efc3SKai Ye struct sec_req *backlog_req; 16472f072d75SZaibo Xu size_t sz; 16482f072d75SZaibo Xu 16492f072d75SZaibo Xu if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt) 16502f072d75SZaibo Xu sec_update_iv(req, SEC_AEAD); 16512f072d75SZaibo Xu 16522f072d75SZaibo Xu /* Copy output mac */ 16532f072d75SZaibo Xu if (!err && c_req->encrypt) { 16542f072d75SZaibo Xu struct scatterlist *sgl = a_req->dst; 16552f072d75SZaibo Xu 16562f072d75SZaibo Xu sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), 16572514f559SLongfang Liu aead_req->out_mac, 16582f072d75SZaibo Xu authsize, a_req->cryptlen + 16592f072d75SZaibo Xu a_req->assoclen); 16602f072d75SZaibo Xu 16612f072d75SZaibo Xu if (unlikely(sz != authsize)) { 1662a44dce50SLongfang Liu dev_err(c->dev, "copy out mac err!\n"); 16632f072d75SZaibo Xu err = -EINVAL; 16642f072d75SZaibo Xu } 16652f072d75SZaibo Xu } 16662f072d75SZaibo Xu 16672f072d75SZaibo Xu sec_free_req_id(req); 16682f072d75SZaibo Xu 16699597efc3SKai Ye while (1) { 16709597efc3SKai Ye backlog_req = sec_back_req_clear(c, qp_ctx); 16719597efc3SKai Ye if (!backlog_req) 16729597efc3SKai Ye break; 16739597efc3SKai Ye 16749597efc3SKai Ye backlog_aead_req = backlog_req->aead_req.aead_req; 16759597efc3SKai Ye backlog_aead_req->base.complete(&backlog_aead_req->base, 16769597efc3SKai Ye -EINPROGRESS); 16779597efc3SKai Ye atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt); 16789597efc3SKai Ye } 16792f072d75SZaibo Xu 16802f072d75SZaibo Xu a_req->base.complete(&a_req->base, err); 16812f072d75SZaibo Xu } 16822f072d75SZaibo Xu 1683416d8220SZaibo Xu static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req) 1684416d8220SZaibo Xu { 1685416d8220SZaibo Xu sec_free_req_id(req); 1686a181647cSZaibo Xu sec_free_queue_id(ctx, req); 1687416d8220SZaibo Xu } 1688416d8220SZaibo Xu 1689416d8220SZaibo Xu static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) 1690416d8220SZaibo Xu { 1691416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx; 16927c7d902aSZaibo Xu int queue_id; 1693416d8220SZaibo Xu 1694416d8220SZaibo Xu /* To load balance */ 1695a181647cSZaibo Xu queue_id = sec_alloc_queue_id(ctx, req); 1696a181647cSZaibo Xu qp_ctx = &ctx->qp_ctx[queue_id]; 1697416d8220SZaibo Xu 1698416d8220SZaibo Xu req->req_id = sec_alloc_req_id(req, qp_ctx); 1699b9c8d897SZaibo Xu if (unlikely(req->req_id < 0)) { 1700a181647cSZaibo Xu sec_free_queue_id(ctx, req); 1701416d8220SZaibo Xu return req->req_id; 1702416d8220SZaibo Xu } 1703416d8220SZaibo Xu 17047c7d902aSZaibo Xu return 0; 1705416d8220SZaibo Xu } 1706416d8220SZaibo Xu 1707416d8220SZaibo Xu static int sec_process(struct sec_ctx *ctx, struct sec_req *req) 1708416d8220SZaibo Xu { 17092514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1710416d8220SZaibo Xu int ret; 1711416d8220SZaibo Xu 1712416d8220SZaibo Xu ret = sec_request_init(ctx, req); 1713b9c8d897SZaibo Xu if (unlikely(ret)) 1714416d8220SZaibo Xu return ret; 1715416d8220SZaibo Xu 1716416d8220SZaibo Xu ret = sec_request_transfer(ctx, req); 1717b9c8d897SZaibo Xu if (unlikely(ret)) 1718416d8220SZaibo Xu goto err_uninit_req; 1719416d8220SZaibo Xu 1720416d8220SZaibo Xu /* Output IV as decrypto */ 17217b44c0eeSKai Ye if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || 17227b44c0eeSKai Ye ctx->c_ctx.c_mode == SEC_CMODE_CTR)) 17232f072d75SZaibo Xu sec_update_iv(req, ctx->alg_type); 1724416d8220SZaibo Xu 1725416d8220SZaibo Xu ret = ctx->req_op->bd_send(ctx, req); 17269597efc3SKai Ye if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) || 17279597efc3SKai Ye (ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1728a44dce50SLongfang Liu dev_err_ratelimited(ctx->dev, "send sec request failed!\n"); 1729416d8220SZaibo Xu goto err_send_req; 1730416d8220SZaibo Xu } 1731416d8220SZaibo Xu 1732416d8220SZaibo Xu return ret; 1733416d8220SZaibo Xu 1734416d8220SZaibo Xu err_send_req: 1735416d8220SZaibo Xu /* As failing, restore the IV from user */ 17362f072d75SZaibo Xu if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) { 17372f072d75SZaibo Xu if (ctx->alg_type == SEC_SKCIPHER) 17382514f559SLongfang Liu memcpy(req->c_req.sk_req->iv, c_req->c_ivin, 1739416d8220SZaibo Xu ctx->c_ctx.ivsize); 17402f072d75SZaibo Xu else 17412514f559SLongfang Liu memcpy(req->aead_req.aead_req->iv, c_req->c_ivin, 17422f072d75SZaibo Xu ctx->c_ctx.ivsize); 17432f072d75SZaibo Xu } 1744416d8220SZaibo Xu 1745416d8220SZaibo Xu sec_request_untransfer(ctx, req); 1746416d8220SZaibo Xu err_uninit_req: 1747416d8220SZaibo Xu sec_request_uninit(ctx, req); 1748416d8220SZaibo Xu return ret; 1749416d8220SZaibo Xu } 1750416d8220SZaibo Xu 1751a181647cSZaibo Xu static const struct sec_req_op sec_skcipher_req_ops = { 1752416d8220SZaibo Xu .buf_map = sec_skcipher_sgl_map, 1753416d8220SZaibo Xu .buf_unmap = sec_skcipher_sgl_unmap, 1754416d8220SZaibo Xu .do_transfer = sec_skcipher_copy_iv, 1755416d8220SZaibo Xu .bd_fill = sec_skcipher_bd_fill, 1756416d8220SZaibo Xu .bd_send = sec_bd_send, 1757416d8220SZaibo Xu .callback = sec_skcipher_callback, 1758416d8220SZaibo Xu .process = sec_process, 1759416d8220SZaibo Xu }; 1760416d8220SZaibo Xu 17612f072d75SZaibo Xu static const struct sec_req_op sec_aead_req_ops = { 17622f072d75SZaibo Xu .buf_map = sec_aead_sgl_map, 17632f072d75SZaibo Xu .buf_unmap = sec_aead_sgl_unmap, 1764c16a70c1SKai Ye .do_transfer = sec_aead_set_iv, 17652f072d75SZaibo Xu .bd_fill = sec_aead_bd_fill, 17662f072d75SZaibo Xu .bd_send = sec_bd_send, 17672f072d75SZaibo Xu .callback = sec_aead_callback, 17682f072d75SZaibo Xu .process = sec_process, 17692f072d75SZaibo Xu }; 17702f072d75SZaibo Xu 1771adc3f65aSKai Ye static const struct sec_req_op sec_skcipher_req_ops_v3 = { 1772adc3f65aSKai Ye .buf_map = sec_skcipher_sgl_map, 1773adc3f65aSKai Ye .buf_unmap = sec_skcipher_sgl_unmap, 1774adc3f65aSKai Ye .do_transfer = sec_skcipher_copy_iv, 1775adc3f65aSKai Ye .bd_fill = sec_skcipher_bd_fill_v3, 1776adc3f65aSKai Ye .bd_send = sec_bd_send, 1777adc3f65aSKai Ye .callback = sec_skcipher_callback, 1778adc3f65aSKai Ye .process = sec_process, 1779adc3f65aSKai Ye }; 1780adc3f65aSKai Ye 1781adc3f65aSKai Ye static const struct sec_req_op sec_aead_req_ops_v3 = { 1782adc3f65aSKai Ye .buf_map = sec_aead_sgl_map, 1783adc3f65aSKai Ye .buf_unmap = sec_aead_sgl_unmap, 1784c16a70c1SKai Ye .do_transfer = sec_aead_set_iv, 1785adc3f65aSKai Ye .bd_fill = sec_aead_bd_fill_v3, 1786adc3f65aSKai Ye .bd_send = sec_bd_send, 1787adc3f65aSKai Ye .callback = sec_aead_callback, 1788adc3f65aSKai Ye .process = sec_process, 1789adc3f65aSKai Ye }; 1790adc3f65aSKai Ye 1791416d8220SZaibo Xu static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) 1792416d8220SZaibo Xu { 1793416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 1794adc3f65aSKai Ye int ret; 1795416d8220SZaibo Xu 1796adc3f65aSKai Ye ret = sec_skcipher_init(tfm); 1797adc3f65aSKai Ye if (ret) 1798adc3f65aSKai Ye return ret; 1799adc3f65aSKai Ye 1800adc3f65aSKai Ye if (ctx->sec->qm.ver < QM_HW_V3) { 1801adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE2; 1802a181647cSZaibo Xu ctx->req_op = &sec_skcipher_req_ops; 1803adc3f65aSKai Ye } else { 1804adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE3; 1805adc3f65aSKai Ye ctx->req_op = &sec_skcipher_req_ops_v3; 1806adc3f65aSKai Ye } 1807416d8220SZaibo Xu 1808adc3f65aSKai Ye return ret; 1809416d8220SZaibo Xu } 1810416d8220SZaibo Xu 1811416d8220SZaibo Xu static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) 1812416d8220SZaibo Xu { 1813a181647cSZaibo Xu sec_skcipher_uninit(tfm); 1814416d8220SZaibo Xu } 1815416d8220SZaibo Xu 18162f072d75SZaibo Xu static int sec_aead_init(struct crypto_aead *tfm) 18172f072d75SZaibo Xu { 18182f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 18192f072d75SZaibo Xu int ret; 18202f072d75SZaibo Xu 18212f072d75SZaibo Xu crypto_aead_set_reqsize(tfm, sizeof(struct sec_req)); 18222f072d75SZaibo Xu ctx->alg_type = SEC_AEAD; 18232f072d75SZaibo Xu ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); 1824c16a70c1SKai Ye if (ctx->c_ctx.ivsize < SEC_AIV_SIZE || 1825c16a70c1SKai Ye ctx->c_ctx.ivsize > SEC_IV_SIZE) { 1826c16a70c1SKai Ye pr_err("get error aead iv size!\n"); 18272f072d75SZaibo Xu return -EINVAL; 18282f072d75SZaibo Xu } 18292f072d75SZaibo Xu 18302f072d75SZaibo Xu ret = sec_ctx_base_init(ctx); 18312f072d75SZaibo Xu if (ret) 18322f072d75SZaibo Xu return ret; 1833adc3f65aSKai Ye if (ctx->sec->qm.ver < QM_HW_V3) { 1834adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE2; 1835adc3f65aSKai Ye ctx->req_op = &sec_aead_req_ops; 1836adc3f65aSKai Ye } else { 1837adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE3; 1838adc3f65aSKai Ye ctx->req_op = &sec_aead_req_ops_v3; 1839adc3f65aSKai Ye } 18402f072d75SZaibo Xu 18412f072d75SZaibo Xu ret = sec_auth_init(ctx); 18422f072d75SZaibo Xu if (ret) 18432f072d75SZaibo Xu goto err_auth_init; 18442f072d75SZaibo Xu 18452f072d75SZaibo Xu ret = sec_cipher_init(ctx); 18462f072d75SZaibo Xu if (ret) 18472f072d75SZaibo Xu goto err_cipher_init; 18482f072d75SZaibo Xu 18492f072d75SZaibo Xu return ret; 18502f072d75SZaibo Xu 18512f072d75SZaibo Xu err_cipher_init: 18522f072d75SZaibo Xu sec_auth_uninit(ctx); 18532f072d75SZaibo Xu err_auth_init: 18542f072d75SZaibo Xu sec_ctx_base_uninit(ctx); 18552f072d75SZaibo Xu return ret; 18562f072d75SZaibo Xu } 18572f072d75SZaibo Xu 18582f072d75SZaibo Xu static void sec_aead_exit(struct crypto_aead *tfm) 18592f072d75SZaibo Xu { 18602f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 18612f072d75SZaibo Xu 18622f072d75SZaibo Xu sec_cipher_uninit(ctx); 18632f072d75SZaibo Xu sec_auth_uninit(ctx); 18642f072d75SZaibo Xu sec_ctx_base_uninit(ctx); 18652f072d75SZaibo Xu } 18662f072d75SZaibo Xu 18672f072d75SZaibo Xu static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) 18682f072d75SZaibo Xu { 18692f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 18702f072d75SZaibo Xu struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 18712f072d75SZaibo Xu int ret; 18722f072d75SZaibo Xu 18732f072d75SZaibo Xu ret = sec_aead_init(tfm); 18742f072d75SZaibo Xu if (ret) { 18752f072d75SZaibo Xu pr_err("hisi_sec2: aead init error!\n"); 18762f072d75SZaibo Xu return ret; 18772f072d75SZaibo Xu } 18782f072d75SZaibo Xu 18792f072d75SZaibo Xu auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); 18802f072d75SZaibo Xu if (IS_ERR(auth_ctx->hash_tfm)) { 1881a44dce50SLongfang Liu dev_err(ctx->dev, "aead alloc shash error!\n"); 18822f072d75SZaibo Xu sec_aead_exit(tfm); 18832f072d75SZaibo Xu return PTR_ERR(auth_ctx->hash_tfm); 18842f072d75SZaibo Xu } 18852f072d75SZaibo Xu 18862f072d75SZaibo Xu return 0; 18872f072d75SZaibo Xu } 18882f072d75SZaibo Xu 18892f072d75SZaibo Xu static void sec_aead_ctx_exit(struct crypto_aead *tfm) 18902f072d75SZaibo Xu { 18912f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 18922f072d75SZaibo Xu 18932f072d75SZaibo Xu crypto_free_shash(ctx->a_ctx.hash_tfm); 18942f072d75SZaibo Xu sec_aead_exit(tfm); 18952f072d75SZaibo Xu } 18962f072d75SZaibo Xu 1897c16a70c1SKai Ye static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) 1898c16a70c1SKai Ye { 1899*6c46a329SKai Ye struct aead_alg *alg = crypto_aead_alg(tfm); 1900c16a70c1SKai Ye struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1901*6c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 1902*6c46a329SKai Ye const char *aead_name = alg->base.cra_name; 1903c16a70c1SKai Ye int ret; 1904c16a70c1SKai Ye 1905c16a70c1SKai Ye ret = sec_aead_init(tfm); 1906c16a70c1SKai Ye if (ret) { 1907c16a70c1SKai Ye dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); 1908c16a70c1SKai Ye return ret; 1909c16a70c1SKai Ye } 1910c16a70c1SKai Ye 1911*6c46a329SKai Ye a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, 1912*6c46a329SKai Ye CRYPTO_ALG_NEED_FALLBACK | 1913*6c46a329SKai Ye CRYPTO_ALG_ASYNC); 1914*6c46a329SKai Ye if (IS_ERR(a_ctx->fallback_aead_tfm)) { 1915*6c46a329SKai Ye dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); 1916*6c46a329SKai Ye sec_aead_exit(tfm); 1917*6c46a329SKai Ye return PTR_ERR(a_ctx->fallback_aead_tfm); 1918*6c46a329SKai Ye } 1919*6c46a329SKai Ye a_ctx->fallback = false; 1920*6c46a329SKai Ye 1921c16a70c1SKai Ye return 0; 1922c16a70c1SKai Ye } 1923c16a70c1SKai Ye 1924c16a70c1SKai Ye static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm) 1925c16a70c1SKai Ye { 1926*6c46a329SKai Ye struct sec_ctx *ctx = crypto_aead_ctx(tfm); 1927*6c46a329SKai Ye 1928*6c46a329SKai Ye crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); 1929c16a70c1SKai Ye sec_aead_exit(tfm); 1930c16a70c1SKai Ye } 1931c16a70c1SKai Ye 19322f072d75SZaibo Xu static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) 19332f072d75SZaibo Xu { 19342f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha1"); 19352f072d75SZaibo Xu } 19362f072d75SZaibo Xu 19372f072d75SZaibo Xu static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm) 19382f072d75SZaibo Xu { 19392f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha256"); 19402f072d75SZaibo Xu } 19412f072d75SZaibo Xu 19422f072d75SZaibo Xu static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) 19432f072d75SZaibo Xu { 19442f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha512"); 19452f072d75SZaibo Xu } 19462f072d75SZaibo Xu 1947059c5342SLongfang Liu 1948059c5342SLongfang Liu static int sec_skcipher_cryptlen_ckeck(struct sec_ctx *ctx, 1949059c5342SLongfang Liu struct sec_req *sreq) 1950059c5342SLongfang Liu { 1951059c5342SLongfang Liu u32 cryptlen = sreq->c_req.sk_req->cryptlen; 1952059c5342SLongfang Liu struct device *dev = ctx->dev; 1953059c5342SLongfang Liu u8 c_mode = ctx->c_ctx.c_mode; 1954059c5342SLongfang Liu int ret = 0; 1955059c5342SLongfang Liu 1956059c5342SLongfang Liu switch (c_mode) { 1957059c5342SLongfang Liu case SEC_CMODE_XTS: 1958059c5342SLongfang Liu if (unlikely(cryptlen < AES_BLOCK_SIZE)) { 1959059c5342SLongfang Liu dev_err(dev, "skcipher XTS mode input length error!\n"); 1960059c5342SLongfang Liu ret = -EINVAL; 1961059c5342SLongfang Liu } 1962059c5342SLongfang Liu break; 1963059c5342SLongfang Liu case SEC_CMODE_ECB: 1964059c5342SLongfang Liu case SEC_CMODE_CBC: 1965059c5342SLongfang Liu if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) { 1966059c5342SLongfang Liu dev_err(dev, "skcipher AES input length error!\n"); 1967059c5342SLongfang Liu ret = -EINVAL; 1968059c5342SLongfang Liu } 1969059c5342SLongfang Liu break; 19707b44c0eeSKai Ye case SEC_CMODE_CFB: 19717b44c0eeSKai Ye case SEC_CMODE_OFB: 19727b44c0eeSKai Ye case SEC_CMODE_CTR: 19737b44c0eeSKai Ye if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) { 19747b44c0eeSKai Ye dev_err(dev, "skcipher HW version error!\n"); 19757b44c0eeSKai Ye ret = -EINVAL; 19767b44c0eeSKai Ye } 19777b44c0eeSKai Ye break; 1978059c5342SLongfang Liu default: 1979059c5342SLongfang Liu ret = -EINVAL; 1980059c5342SLongfang Liu } 1981059c5342SLongfang Liu 1982059c5342SLongfang Liu return ret; 1983059c5342SLongfang Liu } 1984059c5342SLongfang Liu 1985d6de2a59SZaibo Xu static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) 1986416d8220SZaibo Xu { 1987d6de2a59SZaibo Xu struct skcipher_request *sk_req = sreq->c_req.sk_req; 1988a44dce50SLongfang Liu struct device *dev = ctx->dev; 1989d6de2a59SZaibo Xu u8 c_alg = ctx->c_ctx.c_alg; 1990416d8220SZaibo Xu 19917b44c0eeSKai Ye if (unlikely(!sk_req->src || !sk_req->dst || 19927b44c0eeSKai Ye sk_req->cryptlen > MAX_INPUT_DATA_LEN)) { 1993416d8220SZaibo Xu dev_err(dev, "skcipher input param error!\n"); 1994416d8220SZaibo Xu return -EINVAL; 1995416d8220SZaibo Xu } 1996d6de2a59SZaibo Xu sreq->c_req.c_len = sk_req->cryptlen; 199774b58db8SLongfang Liu 199874b58db8SLongfang Liu if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) 199974b58db8SLongfang Liu sreq->use_pbuf = true; 200074b58db8SLongfang Liu else 200174b58db8SLongfang Liu sreq->use_pbuf = false; 200274b58db8SLongfang Liu 2003416d8220SZaibo Xu if (c_alg == SEC_CALG_3DES) { 2004b9c8d897SZaibo Xu if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) { 2005416d8220SZaibo Xu dev_err(dev, "skcipher 3des input length error!\n"); 2006416d8220SZaibo Xu return -EINVAL; 2007416d8220SZaibo Xu } 2008416d8220SZaibo Xu return 0; 2009416d8220SZaibo Xu } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) { 2010059c5342SLongfang Liu return sec_skcipher_cryptlen_ckeck(ctx, sreq); 2011416d8220SZaibo Xu } 2012059c5342SLongfang Liu 2013416d8220SZaibo Xu dev_err(dev, "skcipher algorithm error!\n"); 2014633e507fSLongfang Liu 2015416d8220SZaibo Xu return -EINVAL; 2016416d8220SZaibo Xu } 2017416d8220SZaibo Xu 20185652d55aSKai Ye static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, 20195652d55aSKai Ye struct skcipher_request *sreq, bool encrypt) 20205652d55aSKai Ye { 20215652d55aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 20225652d55aSKai Ye struct device *dev = ctx->dev; 20235652d55aSKai Ye int ret; 20245652d55aSKai Ye 20255652d55aSKai Ye SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm); 20265652d55aSKai Ye 20275652d55aSKai Ye if (!c_ctx->fbtfm) { 20285652d55aSKai Ye dev_err(dev, "failed to check fallback tfm\n"); 20295652d55aSKai Ye return -EINVAL; 20305652d55aSKai Ye } 20315652d55aSKai Ye 20325652d55aSKai Ye skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm); 20335652d55aSKai Ye 20345652d55aSKai Ye /* software need sync mode to do crypto */ 20355652d55aSKai Ye skcipher_request_set_callback(subreq, sreq->base.flags, 20365652d55aSKai Ye NULL, NULL); 20375652d55aSKai Ye skcipher_request_set_crypt(subreq, sreq->src, sreq->dst, 20385652d55aSKai Ye sreq->cryptlen, sreq->iv); 20395652d55aSKai Ye if (encrypt) 20405652d55aSKai Ye ret = crypto_skcipher_encrypt(subreq); 20415652d55aSKai Ye else 20425652d55aSKai Ye ret = crypto_skcipher_decrypt(subreq); 20435652d55aSKai Ye 20445652d55aSKai Ye skcipher_request_zero(subreq); 20455652d55aSKai Ye 20465652d55aSKai Ye return ret; 20475652d55aSKai Ye } 20485652d55aSKai Ye 2049416d8220SZaibo Xu static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) 2050416d8220SZaibo Xu { 2051416d8220SZaibo Xu struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); 2052416d8220SZaibo Xu struct sec_req *req = skcipher_request_ctx(sk_req); 2053416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 2054416d8220SZaibo Xu int ret; 2055416d8220SZaibo Xu 20565652d55aSKai Ye if (!sk_req->cryptlen) { 20575652d55aSKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) 20585652d55aSKai Ye return -EINVAL; 2059416d8220SZaibo Xu return 0; 20605652d55aSKai Ye } 2061416d8220SZaibo Xu 20629597efc3SKai Ye req->flag = sk_req->base.flags; 2063416d8220SZaibo Xu req->c_req.sk_req = sk_req; 2064416d8220SZaibo Xu req->c_req.encrypt = encrypt; 2065416d8220SZaibo Xu req->ctx = ctx; 2066416d8220SZaibo Xu 2067d6de2a59SZaibo Xu ret = sec_skcipher_param_check(ctx, req); 2068d6de2a59SZaibo Xu if (unlikely(ret)) 2069d6de2a59SZaibo Xu return -EINVAL; 2070d6de2a59SZaibo Xu 20715652d55aSKai Ye if (unlikely(ctx->c_ctx.fallback)) 20725652d55aSKai Ye return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); 20735652d55aSKai Ye 2074416d8220SZaibo Xu return ctx->req_op->process(ctx, req); 2075416d8220SZaibo Xu } 2076416d8220SZaibo Xu 2077416d8220SZaibo Xu static int sec_skcipher_encrypt(struct skcipher_request *sk_req) 2078416d8220SZaibo Xu { 2079416d8220SZaibo Xu return sec_skcipher_crypto(sk_req, true); 2080416d8220SZaibo Xu } 2081416d8220SZaibo Xu 2082416d8220SZaibo Xu static int sec_skcipher_decrypt(struct skcipher_request *sk_req) 2083416d8220SZaibo Xu { 2084416d8220SZaibo Xu return sec_skcipher_crypto(sk_req, false); 2085416d8220SZaibo Xu } 2086416d8220SZaibo Xu 2087416d8220SZaibo Xu #define SEC_SKCIPHER_GEN_ALG(sec_cra_name, sec_set_key, sec_min_key_size, \ 2088416d8220SZaibo Xu sec_max_key_size, ctx_init, ctx_exit, blk_size, iv_size)\ 2089416d8220SZaibo Xu {\ 2090416d8220SZaibo Xu .base = {\ 2091416d8220SZaibo Xu .cra_name = sec_cra_name,\ 2092416d8220SZaibo Xu .cra_driver_name = "hisi_sec_"sec_cra_name,\ 2093416d8220SZaibo Xu .cra_priority = SEC_PRIORITY,\ 20945652d55aSKai Ye .cra_flags = CRYPTO_ALG_ASYNC |\ 20955652d55aSKai Ye CRYPTO_ALG_ALLOCATES_MEMORY |\ 20965652d55aSKai Ye CRYPTO_ALG_NEED_FALLBACK,\ 2097416d8220SZaibo Xu .cra_blocksize = blk_size,\ 2098416d8220SZaibo Xu .cra_ctxsize = sizeof(struct sec_ctx),\ 2099416d8220SZaibo Xu .cra_module = THIS_MODULE,\ 2100416d8220SZaibo Xu },\ 2101416d8220SZaibo Xu .init = ctx_init,\ 2102416d8220SZaibo Xu .exit = ctx_exit,\ 2103416d8220SZaibo Xu .setkey = sec_set_key,\ 2104416d8220SZaibo Xu .decrypt = sec_skcipher_decrypt,\ 2105416d8220SZaibo Xu .encrypt = sec_skcipher_encrypt,\ 2106416d8220SZaibo Xu .min_keysize = sec_min_key_size,\ 2107416d8220SZaibo Xu .max_keysize = sec_max_key_size,\ 2108416d8220SZaibo Xu .ivsize = iv_size,\ 2109416d8220SZaibo Xu }, 2110416d8220SZaibo Xu 2111416d8220SZaibo Xu #define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \ 2112416d8220SZaibo Xu max_key_size, blk_size, iv_size) \ 2113416d8220SZaibo Xu SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \ 2114416d8220SZaibo Xu sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size) 2115416d8220SZaibo Xu 2116a181647cSZaibo Xu static struct skcipher_alg sec_skciphers[] = { 2117416d8220SZaibo Xu SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, 2118416d8220SZaibo Xu AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 2119416d8220SZaibo Xu AES_BLOCK_SIZE, 0) 2120416d8220SZaibo Xu 2121416d8220SZaibo Xu SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, 2122416d8220SZaibo Xu AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 2123416d8220SZaibo Xu AES_BLOCK_SIZE, AES_BLOCK_SIZE) 2124416d8220SZaibo Xu 2125416d8220SZaibo Xu SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, 2126416d8220SZaibo Xu SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MAX_KEY_SIZE, 2127416d8220SZaibo Xu AES_BLOCK_SIZE, AES_BLOCK_SIZE) 2128416d8220SZaibo Xu 2129416d8220SZaibo Xu SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, 21306161f40cSKai Ye SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, 2131416d8220SZaibo Xu DES3_EDE_BLOCK_SIZE, 0) 2132416d8220SZaibo Xu 2133416d8220SZaibo Xu SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, 21346161f40cSKai Ye SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, 2135416d8220SZaibo Xu DES3_EDE_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE) 2136416d8220SZaibo Xu 2137416d8220SZaibo Xu SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, 2138416d8220SZaibo Xu SEC_XTS_MIN_KEY_SIZE, SEC_XTS_MIN_KEY_SIZE, 2139416d8220SZaibo Xu AES_BLOCK_SIZE, AES_BLOCK_SIZE) 2140416d8220SZaibo Xu 2141416d8220SZaibo Xu SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, 2142416d8220SZaibo Xu AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, 2143416d8220SZaibo Xu AES_BLOCK_SIZE, AES_BLOCK_SIZE) 2144416d8220SZaibo Xu }; 2145416d8220SZaibo Xu 21467b44c0eeSKai Ye static struct skcipher_alg sec_skciphers_v3[] = { 21477b44c0eeSKai Ye SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb, 21487b44c0eeSKai Ye AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 21497b44c0eeSKai Ye SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) 21507b44c0eeSKai Ye 21517b44c0eeSKai Ye SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb, 21527b44c0eeSKai Ye AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 21537b44c0eeSKai Ye SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) 21547b44c0eeSKai Ye 21557b44c0eeSKai Ye SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, 21567b44c0eeSKai Ye AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, 21577b44c0eeSKai Ye SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) 21587b44c0eeSKai Ye 21597b44c0eeSKai Ye SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb, 21607b44c0eeSKai Ye AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, 21617b44c0eeSKai Ye SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) 21627b44c0eeSKai Ye 21637b44c0eeSKai Ye SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb, 21647b44c0eeSKai Ye AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, 21657b44c0eeSKai Ye SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) 21667b44c0eeSKai Ye 21677b44c0eeSKai Ye SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, 21687b44c0eeSKai Ye AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, 21697b44c0eeSKai Ye SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) 21707b44c0eeSKai Ye }; 21717b44c0eeSKai Ye 2172c16a70c1SKai Ye static int aead_iv_demension_check(struct aead_request *aead_req) 2173c16a70c1SKai Ye { 2174c16a70c1SKai Ye u8 cl; 2175c16a70c1SKai Ye 2176c16a70c1SKai Ye cl = aead_req->iv[0] + 1; 2177c16a70c1SKai Ye if (cl < IV_CL_MIN || cl > IV_CL_MAX) 2178c16a70c1SKai Ye return -EINVAL; 2179c16a70c1SKai Ye 2180c16a70c1SKai Ye if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl)) 2181c16a70c1SKai Ye return -EOVERFLOW; 2182c16a70c1SKai Ye 2183c16a70c1SKai Ye return 0; 2184c16a70c1SKai Ye } 2185c16a70c1SKai Ye 2186c16a70c1SKai Ye static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq) 2187c16a70c1SKai Ye { 2188c16a70c1SKai Ye struct aead_request *req = sreq->aead_req.aead_req; 2189c16a70c1SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2190c16a70c1SKai Ye size_t authsize = crypto_aead_authsize(tfm); 2191c16a70c1SKai Ye u8 c_mode = ctx->c_ctx.c_mode; 2192c16a70c1SKai Ye struct device *dev = ctx->dev; 2193c16a70c1SKai Ye int ret; 2194c16a70c1SKai Ye 2195c16a70c1SKai Ye if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || 2196c16a70c1SKai Ye req->assoclen > SEC_MAX_AAD_LEN)) { 2197c16a70c1SKai Ye dev_err(dev, "aead input spec error!\n"); 2198c16a70c1SKai Ye return -EINVAL; 2199c16a70c1SKai Ye } 2200c16a70c1SKai Ye 2201c16a70c1SKai Ye if (unlikely((c_mode == SEC_CMODE_GCM && authsize < DES_BLOCK_SIZE) || 2202c16a70c1SKai Ye (c_mode == SEC_CMODE_CCM && (authsize < MIN_MAC_LEN || 2203c16a70c1SKai Ye authsize & MAC_LEN_MASK)))) { 2204c16a70c1SKai Ye dev_err(dev, "aead input mac length error!\n"); 2205c16a70c1SKai Ye return -EINVAL; 2206c16a70c1SKai Ye } 2207c16a70c1SKai Ye 2208c16a70c1SKai Ye if (c_mode == SEC_CMODE_CCM) { 2209c16a70c1SKai Ye ret = aead_iv_demension_check(req); 2210c16a70c1SKai Ye if (ret) { 2211c16a70c1SKai Ye dev_err(dev, "aead input iv param error!\n"); 2212c16a70c1SKai Ye return ret; 2213c16a70c1SKai Ye } 2214c16a70c1SKai Ye } 2215c16a70c1SKai Ye 2216c16a70c1SKai Ye if (sreq->c_req.encrypt) 2217c16a70c1SKai Ye sreq->c_req.c_len = req->cryptlen; 2218c16a70c1SKai Ye else 2219c16a70c1SKai Ye sreq->c_req.c_len = req->cryptlen - authsize; 2220c16a70c1SKai Ye if (c_mode == SEC_CMODE_CBC) { 2221c16a70c1SKai Ye if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { 2222c16a70c1SKai Ye dev_err(dev, "aead crypto length error!\n"); 2223c16a70c1SKai Ye return -EINVAL; 2224c16a70c1SKai Ye } 2225c16a70c1SKai Ye } 2226c16a70c1SKai Ye 2227c16a70c1SKai Ye return 0; 2228c16a70c1SKai Ye } 2229c16a70c1SKai Ye 22302f072d75SZaibo Xu static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) 22312f072d75SZaibo Xu { 22322f072d75SZaibo Xu struct aead_request *req = sreq->aead_req.aead_req; 22332f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(req); 22342f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm); 2235a44dce50SLongfang Liu struct device *dev = ctx->dev; 2236a44dce50SLongfang Liu u8 c_alg = ctx->c_ctx.c_alg; 22372f072d75SZaibo Xu 2238c16a70c1SKai Ye if (unlikely(!req->src || !req->dst)) { 2239a44dce50SLongfang Liu dev_err(dev, "aead input param error!\n"); 22402f072d75SZaibo Xu return -EINVAL; 22412f072d75SZaibo Xu } 22422f072d75SZaibo Xu 2243c16a70c1SKai Ye if (ctx->sec->qm.ver == QM_HW_V2) { 2244c16a70c1SKai Ye if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt && 2245c16a70c1SKai Ye req->cryptlen <= authsize))) { 2246c16a70c1SKai Ye dev_err(dev, "Kunpeng920 not support 0 length!\n"); 2247*6c46a329SKai Ye ctx->a_ctx.fallback = true; 2248c16a70c1SKai Ye return -EINVAL; 2249c16a70c1SKai Ye } 2250c16a70c1SKai Ye } 2251c16a70c1SKai Ye 2252c16a70c1SKai Ye /* Support AES or SM4 */ 2253c16a70c1SKai Ye if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) { 2254c16a70c1SKai Ye dev_err(dev, "aead crypto alg error!\n"); 2255c16a70c1SKai Ye return -EINVAL; 2256c16a70c1SKai Ye } 2257c16a70c1SKai Ye 2258c16a70c1SKai Ye if (unlikely(sec_aead_spec_check(ctx, sreq))) 2259c16a70c1SKai Ye return -EINVAL; 2260c16a70c1SKai Ye 226174b58db8SLongfang Liu if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= 226274b58db8SLongfang Liu SEC_PBUF_SZ) 226374b58db8SLongfang Liu sreq->use_pbuf = true; 226474b58db8SLongfang Liu else 226574b58db8SLongfang Liu sreq->use_pbuf = false; 226674b58db8SLongfang Liu 22672f072d75SZaibo Xu return 0; 22682f072d75SZaibo Xu } 22692f072d75SZaibo Xu 2270*6c46a329SKai Ye static int sec_aead_soft_crypto(struct sec_ctx *ctx, 2271*6c46a329SKai Ye struct aead_request *aead_req, 2272*6c46a329SKai Ye bool encrypt) 2273*6c46a329SKai Ye { 2274*6c46a329SKai Ye struct aead_request *subreq = aead_request_ctx(aead_req); 2275*6c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 2276*6c46a329SKai Ye struct device *dev = ctx->dev; 2277*6c46a329SKai Ye 2278*6c46a329SKai Ye /* Kunpeng920 aead mode not support input 0 size */ 2279*6c46a329SKai Ye if (!a_ctx->fallback_aead_tfm) { 2280*6c46a329SKai Ye dev_err(dev, "aead fallbcak tfm is NULL!\n"); 2281*6c46a329SKai Ye return -EINVAL; 2282*6c46a329SKai Ye } 2283*6c46a329SKai Ye 2284*6c46a329SKai Ye aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm); 2285*6c46a329SKai Ye aead_request_set_callback(subreq, aead_req->base.flags, 2286*6c46a329SKai Ye aead_req->base.complete, aead_req->base.data); 2287*6c46a329SKai Ye aead_request_set_crypt(subreq, aead_req->src, aead_req->dst, 2288*6c46a329SKai Ye aead_req->cryptlen, aead_req->iv); 2289*6c46a329SKai Ye aead_request_set_ad(subreq, aead_req->assoclen); 2290*6c46a329SKai Ye 2291*6c46a329SKai Ye return encrypt ? crypto_aead_encrypt(subreq) : 2292*6c46a329SKai Ye crypto_aead_decrypt(subreq); 2293*6c46a329SKai Ye } 2294*6c46a329SKai Ye 22952f072d75SZaibo Xu static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) 22962f072d75SZaibo Xu { 22972f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 22982f072d75SZaibo Xu struct sec_req *req = aead_request_ctx(a_req); 22992f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 23002f072d75SZaibo Xu int ret; 23012f072d75SZaibo Xu 23029597efc3SKai Ye req->flag = a_req->base.flags; 23032f072d75SZaibo Xu req->aead_req.aead_req = a_req; 23042f072d75SZaibo Xu req->c_req.encrypt = encrypt; 23052f072d75SZaibo Xu req->ctx = ctx; 23062f072d75SZaibo Xu 23072f072d75SZaibo Xu ret = sec_aead_param_check(ctx, req); 2308*6c46a329SKai Ye if (unlikely(ret)) { 2309*6c46a329SKai Ye if (ctx->a_ctx.fallback) 2310*6c46a329SKai Ye return sec_aead_soft_crypto(ctx, a_req, encrypt); 23112f072d75SZaibo Xu return -EINVAL; 2312*6c46a329SKai Ye } 23132f072d75SZaibo Xu 23142f072d75SZaibo Xu return ctx->req_op->process(ctx, req); 23152f072d75SZaibo Xu } 23162f072d75SZaibo Xu 23172f072d75SZaibo Xu static int sec_aead_encrypt(struct aead_request *a_req) 23182f072d75SZaibo Xu { 23192f072d75SZaibo Xu return sec_aead_crypto(a_req, true); 23202f072d75SZaibo Xu } 23212f072d75SZaibo Xu 23222f072d75SZaibo Xu static int sec_aead_decrypt(struct aead_request *a_req) 23232f072d75SZaibo Xu { 23242f072d75SZaibo Xu return sec_aead_crypto(a_req, false); 23252f072d75SZaibo Xu } 23262f072d75SZaibo Xu 2327c16a70c1SKai Ye #define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\ 23282f072d75SZaibo Xu ctx_exit, blk_size, iv_size, max_authsize)\ 23292f072d75SZaibo Xu {\ 23302f072d75SZaibo Xu .base = {\ 23312f072d75SZaibo Xu .cra_name = sec_cra_name,\ 23322f072d75SZaibo Xu .cra_driver_name = "hisi_sec_"sec_cra_name,\ 23332f072d75SZaibo Xu .cra_priority = SEC_PRIORITY,\ 2334*6c46a329SKai Ye .cra_flags = CRYPTO_ALG_ASYNC |\ 2335*6c46a329SKai Ye CRYPTO_ALG_ALLOCATES_MEMORY |\ 2336*6c46a329SKai Ye CRYPTO_ALG_NEED_FALLBACK,\ 23372f072d75SZaibo Xu .cra_blocksize = blk_size,\ 23382f072d75SZaibo Xu .cra_ctxsize = sizeof(struct sec_ctx),\ 23392f072d75SZaibo Xu .cra_module = THIS_MODULE,\ 23402f072d75SZaibo Xu },\ 23412f072d75SZaibo Xu .init = ctx_init,\ 23422f072d75SZaibo Xu .exit = ctx_exit,\ 23432f072d75SZaibo Xu .setkey = sec_set_key,\ 2344*6c46a329SKai Ye .setauthsize = sec_aead_setauthsize,\ 23452f072d75SZaibo Xu .decrypt = sec_aead_decrypt,\ 23462f072d75SZaibo Xu .encrypt = sec_aead_encrypt,\ 23472f072d75SZaibo Xu .ivsize = iv_size,\ 23482f072d75SZaibo Xu .maxauthsize = max_authsize,\ 23492f072d75SZaibo Xu } 23502f072d75SZaibo Xu 23512f072d75SZaibo Xu static struct aead_alg sec_aeads[] = { 23522f072d75SZaibo Xu SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", 23532f072d75SZaibo Xu sec_setkey_aes_cbc_sha1, sec_aead_sha1_ctx_init, 2354c16a70c1SKai Ye sec_aead_ctx_exit, AES_BLOCK_SIZE, 2355c16a70c1SKai Ye AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), 23562f072d75SZaibo Xu 23572f072d75SZaibo Xu SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", 23582f072d75SZaibo Xu sec_setkey_aes_cbc_sha256, sec_aead_sha256_ctx_init, 2359c16a70c1SKai Ye sec_aead_ctx_exit, AES_BLOCK_SIZE, 2360c16a70c1SKai Ye AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), 23612f072d75SZaibo Xu 23622f072d75SZaibo Xu SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", 23632f072d75SZaibo Xu sec_setkey_aes_cbc_sha512, sec_aead_sha512_ctx_init, 2364c16a70c1SKai Ye sec_aead_ctx_exit, AES_BLOCK_SIZE, 2365c16a70c1SKai Ye AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), 2366c16a70c1SKai Ye 2367c16a70c1SKai Ye SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init, 2368c16a70c1SKai Ye sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, 2369c16a70c1SKai Ye AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2370c16a70c1SKai Ye 2371c16a70c1SKai Ye SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init, 2372c16a70c1SKai Ye sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, 2373c16a70c1SKai Ye SEC_AIV_SIZE, AES_BLOCK_SIZE) 2374c16a70c1SKai Ye }; 2375c16a70c1SKai Ye 2376c16a70c1SKai Ye static struct aead_alg sec_aeads_v3[] = { 2377c16a70c1SKai Ye SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init, 2378c16a70c1SKai Ye sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, 2379c16a70c1SKai Ye AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2380c16a70c1SKai Ye 2381c16a70c1SKai Ye SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init, 2382c16a70c1SKai Ye sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, 2383c16a70c1SKai Ye SEC_AIV_SIZE, AES_BLOCK_SIZE) 23842f072d75SZaibo Xu }; 23852f072d75SZaibo Xu 23868123455aSMeng Yu int sec_register_to_crypto(struct hisi_qm *qm) 2387416d8220SZaibo Xu { 23883d29e98dSYang Shen int ret; 2389416d8220SZaibo Xu 2390416d8220SZaibo Xu /* To avoid repeat register */ 2391a181647cSZaibo Xu ret = crypto_register_skciphers(sec_skciphers, 2392a181647cSZaibo Xu ARRAY_SIZE(sec_skciphers)); 23932f072d75SZaibo Xu if (ret) 23942f072d75SZaibo Xu return ret; 23952f072d75SZaibo Xu 23967b44c0eeSKai Ye if (qm->ver > QM_HW_V2) { 23977b44c0eeSKai Ye ret = crypto_register_skciphers(sec_skciphers_v3, 23987b44c0eeSKai Ye ARRAY_SIZE(sec_skciphers_v3)); 23997b44c0eeSKai Ye if (ret) 24007b44c0eeSKai Ye goto reg_skcipher_fail; 24017b44c0eeSKai Ye } 2402c16a70c1SKai Ye 24032f072d75SZaibo Xu ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); 24042f072d75SZaibo Xu if (ret) 24057b44c0eeSKai Ye goto reg_aead_fail; 2406c16a70c1SKai Ye if (qm->ver > QM_HW_V2) { 2407c16a70c1SKai Ye ret = crypto_register_aeads(sec_aeads_v3, ARRAY_SIZE(sec_aeads_v3)); 2408c16a70c1SKai Ye if (ret) 2409c16a70c1SKai Ye goto reg_aead_v3_fail; 2410c16a70c1SKai Ye } 24117b44c0eeSKai Ye return ret; 24127b44c0eeSKai Ye 2413c16a70c1SKai Ye reg_aead_v3_fail: 2414c16a70c1SKai Ye crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); 24157b44c0eeSKai Ye reg_aead_fail: 24167b44c0eeSKai Ye if (qm->ver > QM_HW_V2) 24177b44c0eeSKai Ye crypto_unregister_skciphers(sec_skciphers_v3, 24187b44c0eeSKai Ye ARRAY_SIZE(sec_skciphers_v3)); 24197b44c0eeSKai Ye reg_skcipher_fail: 24203d29e98dSYang Shen crypto_unregister_skciphers(sec_skciphers, 24213d29e98dSYang Shen ARRAY_SIZE(sec_skciphers)); 2422416d8220SZaibo Xu return ret; 2423416d8220SZaibo Xu } 2424416d8220SZaibo Xu 24258123455aSMeng Yu void sec_unregister_from_crypto(struct hisi_qm *qm) 2426416d8220SZaibo Xu { 24277b44c0eeSKai Ye if (qm->ver > QM_HW_V2) 2428c16a70c1SKai Ye crypto_unregister_aeads(sec_aeads_v3, 2429c16a70c1SKai Ye ARRAY_SIZE(sec_aeads_v3)); 2430c16a70c1SKai Ye crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); 2431c16a70c1SKai Ye 2432c16a70c1SKai Ye if (qm->ver > QM_HW_V2) 24337b44c0eeSKai Ye crypto_unregister_skciphers(sec_skciphers_v3, 24347b44c0eeSKai Ye ARRAY_SIZE(sec_skciphers_v3)); 2435a181647cSZaibo Xu crypto_unregister_skciphers(sec_skciphers, 2436a181647cSZaibo Xu ARRAY_SIZE(sec_skciphers)); 24372f072d75SZaibo Xu } 2438