1416d8220SZaibo Xu // SPDX-License-Identifier: GPL-2.0 2416d8220SZaibo Xu /* Copyright (c) 2019 HiSilicon Limited. */ 3416d8220SZaibo Xu 4416d8220SZaibo Xu #include <crypto/aes.h> 56c46a329SKai Ye #include <crypto/aead.h> 6416d8220SZaibo Xu #include <crypto/algapi.h> 72f072d75SZaibo Xu #include <crypto/authenc.h> 8416d8220SZaibo Xu #include <crypto/des.h> 92f072d75SZaibo Xu #include <crypto/hash.h> 102f072d75SZaibo Xu #include <crypto/internal/aead.h> 11ae6ce7b1SKai Ye #include <crypto/internal/des.h> 12a24d22b2SEric Biggers #include <crypto/sha1.h> 13a24d22b2SEric Biggers #include <crypto/sha2.h> 14416d8220SZaibo Xu #include <crypto/skcipher.h> 15416d8220SZaibo Xu #include <crypto/xts.h> 16416d8220SZaibo Xu #include <linux/crypto.h> 17416d8220SZaibo Xu #include <linux/dma-mapping.h> 18416d8220SZaibo Xu #include <linux/idr.h> 19416d8220SZaibo Xu 20416d8220SZaibo Xu #include "sec.h" 21416d8220SZaibo Xu #include "sec_crypto.h" 22416d8220SZaibo Xu 23416d8220SZaibo Xu #define SEC_PRIORITY 4001 24416d8220SZaibo Xu #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) 255652d55aSKai Ye #define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE) 26416d8220SZaibo Xu #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) 27416d8220SZaibo Xu #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE) 28416d8220SZaibo Xu #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE) 29416d8220SZaibo Xu 30416d8220SZaibo Xu /* SEC sqe(bd) bit operational relative MACRO */ 31416d8220SZaibo Xu #define SEC_DE_OFFSET 1 32416d8220SZaibo Xu #define SEC_CIPHER_OFFSET 4 33416d8220SZaibo Xu #define SEC_SCENE_OFFSET 3 34416d8220SZaibo Xu #define SEC_DST_SGL_OFFSET 2 35416d8220SZaibo Xu #define SEC_SRC_SGL_OFFSET 7 36416d8220SZaibo Xu #define SEC_CKEY_OFFSET 9 37416d8220SZaibo Xu #define SEC_CMODE_OFFSET 12 382f072d75SZaibo Xu #define SEC_AKEY_OFFSET 5 392f072d75SZaibo Xu #define SEC_AEAD_ALG_OFFSET 11 402f072d75SZaibo Xu #define SEC_AUTH_OFFSET 6 412f072d75SZaibo Xu 42adc3f65aSKai Ye #define SEC_DE_OFFSET_V3 9 43adc3f65aSKai Ye #define SEC_SCENE_OFFSET_V3 5 44adc3f65aSKai Ye #define SEC_CKEY_OFFSET_V3 13 45566f060fSKai Ye #define SEC_CTR_CNT_OFFSET 25 46566f060fSKai Ye #define SEC_CTR_CNT_ROLLOVER 2 47adc3f65aSKai Ye #define SEC_SRC_SGL_OFFSET_V3 11 48adc3f65aSKai Ye #define SEC_DST_SGL_OFFSET_V3 14 49adc3f65aSKai Ye #define SEC_CALG_OFFSET_V3 4 50adc3f65aSKai Ye #define SEC_AKEY_OFFSET_V3 9 51adc3f65aSKai Ye #define SEC_MAC_OFFSET_V3 4 52adc3f65aSKai Ye #define SEC_AUTH_ALG_OFFSET_V3 15 53adc3f65aSKai Ye #define SEC_CIPHER_AUTH_V3 0xbf 54adc3f65aSKai Ye #define SEC_AUTH_CIPHER_V3 0x40 55416d8220SZaibo Xu #define SEC_FLAG_OFFSET 7 56416d8220SZaibo Xu #define SEC_FLAG_MASK 0x0780 57416d8220SZaibo Xu #define SEC_TYPE_MASK 0x0F 58416d8220SZaibo Xu #define SEC_DONE_MASK 0x0001 59668f1ab7SKai Ye #define SEC_ICV_MASK 0x000E 60adc3f65aSKai Ye #define SEC_SQE_LEN_RATE_MASK 0x3 61416d8220SZaibo Xu 62129a9f34SWeili Qian #define SEC_TOTAL_IV_SZ(depth) (SEC_IV_SIZE * (depth)) 63416d8220SZaibo Xu #define SEC_SGL_SGE_NR 128 642f072d75SZaibo Xu #define SEC_CIPHER_AUTH 0xfe 652f072d75SZaibo Xu #define SEC_AUTH_CIPHER 0x1 662f072d75SZaibo Xu #define SEC_MAX_MAC_LEN 64 672514f559SLongfang Liu #define SEC_MAX_AAD_LEN 65535 685e340558SKai Ye #define SEC_MAX_CCM_AAD_LEN 65279 69129a9f34SWeili Qian #define SEC_TOTAL_MAC_SZ(depth) (SEC_MAX_MAC_LEN * (depth)) 7074b58db8SLongfang Liu 7174b58db8SLongfang Liu #define SEC_PBUF_SZ 512 7274b58db8SLongfang Liu #define SEC_PBUF_IV_OFFSET SEC_PBUF_SZ 7374b58db8SLongfang Liu #define SEC_PBUF_MAC_OFFSET (SEC_PBUF_SZ + SEC_IV_SIZE) 7474b58db8SLongfang Liu #define SEC_PBUF_PKG (SEC_PBUF_SZ + SEC_IV_SIZE + \ 7574b58db8SLongfang Liu SEC_MAX_MAC_LEN * 2) 7674b58db8SLongfang Liu #define SEC_PBUF_NUM (PAGE_SIZE / SEC_PBUF_PKG) 77129a9f34SWeili Qian #define SEC_PBUF_PAGE_NUM(depth) ((depth) / SEC_PBUF_NUM) 78129a9f34SWeili Qian #define SEC_PBUF_LEFT_SZ(depth) (SEC_PBUF_PKG * ((depth) - \ 79129a9f34SWeili Qian SEC_PBUF_PAGE_NUM(depth) * SEC_PBUF_NUM)) 80129a9f34SWeili Qian #define SEC_TOTAL_PBUF_SZ(depth) (PAGE_SIZE * SEC_PBUF_PAGE_NUM(depth) + \ 81129a9f34SWeili Qian SEC_PBUF_LEFT_SZ(depth)) 8274b58db8SLongfang Liu 832f072d75SZaibo Xu #define SEC_SQE_LEN_RATE 4 84d6de2a59SZaibo Xu #define SEC_SQE_CFLAG 2 852f072d75SZaibo Xu #define SEC_SQE_AEAD_FLAG 3 86d6de2a59SZaibo Xu #define SEC_SQE_DONE 0x1 87668f1ab7SKai Ye #define SEC_ICV_ERR 0x2 88c16a70c1SKai Ye #define MIN_MAC_LEN 4 89c16a70c1SKai Ye #define MAC_LEN_MASK 0x1U 907b44c0eeSKai Ye #define MAX_INPUT_DATA_LEN 0xFFFE00 917b44c0eeSKai Ye #define BITS_MASK 0xFF 927b44c0eeSKai Ye #define BYTE_BITS 0x8 935652d55aSKai Ye #define SEC_XTS_NAME_SZ 0x3 94c16a70c1SKai Ye #define IV_CM_CAL_NUM 2 95c16a70c1SKai Ye #define IV_CL_MASK 0x7 96c16a70c1SKai Ye #define IV_CL_MIN 2 97c16a70c1SKai Ye #define IV_CL_MID 4 98c16a70c1SKai Ye #define IV_CL_MAX 8 99c16a70c1SKai Ye #define IV_FLAGS_OFFSET 0x6 100c16a70c1SKai Ye #define IV_CM_OFFSET 0x3 101c16a70c1SKai Ye #define IV_LAST_BYTE1 1 102c16a70c1SKai Ye #define IV_LAST_BYTE2 2 103c16a70c1SKai Ye #define IV_LAST_BYTE_MASK 0xFF 104c16a70c1SKai Ye #define IV_CTR_INIT 0x1 105c16a70c1SKai Ye #define IV_BYTE_OFFSET 0x8 106416d8220SZaibo Xu 107*921715b6SWenkai Lin struct sec_skcipher { 108*921715b6SWenkai Lin u64 alg_msk; 109*921715b6SWenkai Lin struct skcipher_alg alg; 110*921715b6SWenkai Lin }; 111*921715b6SWenkai Lin 112*921715b6SWenkai Lin struct sec_aead { 113*921715b6SWenkai Lin u64 alg_msk; 114*921715b6SWenkai Lin struct aead_alg alg; 115*921715b6SWenkai Lin }; 116*921715b6SWenkai Lin 117416d8220SZaibo Xu /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ 118a181647cSZaibo Xu static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) 119416d8220SZaibo Xu { 120416d8220SZaibo Xu if (req->c_req.encrypt) 121416d8220SZaibo Xu return (u32)atomic_inc_return(&ctx->enc_qcyclic) % 122416d8220SZaibo Xu ctx->hlf_q_num; 123416d8220SZaibo Xu 124416d8220SZaibo Xu return (u32)atomic_inc_return(&ctx->dec_qcyclic) % ctx->hlf_q_num + 125416d8220SZaibo Xu ctx->hlf_q_num; 126416d8220SZaibo Xu } 127416d8220SZaibo Xu 128a181647cSZaibo Xu static inline void sec_free_queue_id(struct sec_ctx *ctx, struct sec_req *req) 129416d8220SZaibo Xu { 130416d8220SZaibo Xu if (req->c_req.encrypt) 131416d8220SZaibo Xu atomic_dec(&ctx->enc_qcyclic); 132416d8220SZaibo Xu else 133416d8220SZaibo Xu atomic_dec(&ctx->dec_qcyclic); 134416d8220SZaibo Xu } 135416d8220SZaibo Xu 136416d8220SZaibo Xu static int sec_alloc_req_id(struct sec_req *req, struct sec_qp_ctx *qp_ctx) 137416d8220SZaibo Xu { 138416d8220SZaibo Xu int req_id; 139416d8220SZaibo Xu 14002884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock); 141129a9f34SWeili Qian req_id = idr_alloc_cyclic(&qp_ctx->req_idr, NULL, 0, qp_ctx->qp->sq_depth, GFP_ATOMIC); 14202884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock); 143b9c8d897SZaibo Xu if (unlikely(req_id < 0)) { 144a44dce50SLongfang Liu dev_err(req->ctx->dev, "alloc req id fail!\n"); 145416d8220SZaibo Xu return req_id; 146416d8220SZaibo Xu } 147416d8220SZaibo Xu 148416d8220SZaibo Xu req->qp_ctx = qp_ctx; 149416d8220SZaibo Xu qp_ctx->req_list[req_id] = req; 150633e507fSLongfang Liu 151416d8220SZaibo Xu return req_id; 152416d8220SZaibo Xu } 153416d8220SZaibo Xu 154416d8220SZaibo Xu static void sec_free_req_id(struct sec_req *req) 155416d8220SZaibo Xu { 156416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 157416d8220SZaibo Xu int req_id = req->req_id; 158416d8220SZaibo Xu 159129a9f34SWeili Qian if (unlikely(req_id < 0 || req_id >= qp_ctx->qp->sq_depth)) { 160a44dce50SLongfang Liu dev_err(req->ctx->dev, "free request id invalid!\n"); 161416d8220SZaibo Xu return; 162416d8220SZaibo Xu } 163416d8220SZaibo Xu 164416d8220SZaibo Xu qp_ctx->req_list[req_id] = NULL; 165416d8220SZaibo Xu req->qp_ctx = NULL; 166416d8220SZaibo Xu 16702884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock); 168416d8220SZaibo Xu idr_remove(&qp_ctx->req_idr, req_id); 16902884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock); 170416d8220SZaibo Xu } 171416d8220SZaibo Xu 172adc3f65aSKai Ye static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) 173adc3f65aSKai Ye { 174adc3f65aSKai Ye struct sec_sqe *bd = resp; 175adc3f65aSKai Ye 176adc3f65aSKai Ye status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; 177668f1ab7SKai Ye status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1; 178adc3f65aSKai Ye status->flag = (le16_to_cpu(bd->type2.done_flag) & 179adc3f65aSKai Ye SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 180adc3f65aSKai Ye status->tag = le16_to_cpu(bd->type2.tag); 181adc3f65aSKai Ye status->err_type = bd->type2.error_type; 182adc3f65aSKai Ye 183adc3f65aSKai Ye return bd->type_cipher_auth & SEC_TYPE_MASK; 184adc3f65aSKai Ye } 185adc3f65aSKai Ye 186adc3f65aSKai Ye static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) 187adc3f65aSKai Ye { 188adc3f65aSKai Ye struct sec_sqe3 *bd3 = resp; 189adc3f65aSKai Ye 190adc3f65aSKai Ye status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK; 191668f1ab7SKai Ye status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1; 192adc3f65aSKai Ye status->flag = (le16_to_cpu(bd3->done_flag) & 193adc3f65aSKai Ye SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; 194adc3f65aSKai Ye status->tag = le64_to_cpu(bd3->tag); 195adc3f65aSKai Ye status->err_type = bd3->error_type; 196adc3f65aSKai Ye 197adc3f65aSKai Ye return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK; 198adc3f65aSKai Ye } 199adc3f65aSKai Ye 200adc3f65aSKai Ye static int sec_cb_status_check(struct sec_req *req, 201adc3f65aSKai Ye struct bd_status *status) 202adc3f65aSKai Ye { 203adc3f65aSKai Ye struct sec_ctx *ctx = req->ctx; 204adc3f65aSKai Ye 205adc3f65aSKai Ye if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) { 206adc3f65aSKai Ye dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n", 207adc3f65aSKai Ye req->err_type, status->done); 208adc3f65aSKai Ye return -EIO; 209adc3f65aSKai Ye } 210adc3f65aSKai Ye 211adc3f65aSKai Ye if (unlikely(ctx->alg_type == SEC_SKCIPHER)) { 212adc3f65aSKai Ye if (unlikely(status->flag != SEC_SQE_CFLAG)) { 213adc3f65aSKai Ye dev_err_ratelimited(ctx->dev, "flag[%u]\n", 214adc3f65aSKai Ye status->flag); 215adc3f65aSKai Ye return -EIO; 216adc3f65aSKai Ye } 217668f1ab7SKai Ye } else if (unlikely(ctx->alg_type == SEC_AEAD)) { 218668f1ab7SKai Ye if (unlikely(status->flag != SEC_SQE_AEAD_FLAG || 219668f1ab7SKai Ye status->icv == SEC_ICV_ERR)) { 220668f1ab7SKai Ye dev_err_ratelimited(ctx->dev, 221668f1ab7SKai Ye "flag[%u], icv[%u]\n", 222668f1ab7SKai Ye status->flag, status->icv); 223668f1ab7SKai Ye return -EBADMSG; 224668f1ab7SKai Ye } 225adc3f65aSKai Ye } 226adc3f65aSKai Ye 227adc3f65aSKai Ye return 0; 228adc3f65aSKai Ye } 229adc3f65aSKai Ye 230416d8220SZaibo Xu static void sec_req_cb(struct hisi_qp *qp, void *resp) 231416d8220SZaibo Xu { 232416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = qp->qp_ctx; 2338213a1a6SKai Ye struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; 234adc3f65aSKai Ye u8 type_supported = qp_ctx->ctx->type_supported; 235adc3f65aSKai Ye struct bd_status status; 236d6de2a59SZaibo Xu struct sec_ctx *ctx; 237d6de2a59SZaibo Xu struct sec_req *req; 238adc3f65aSKai Ye int err; 239416d8220SZaibo Xu u8 type; 240416d8220SZaibo Xu 241adc3f65aSKai Ye if (type_supported == SEC_BD_TYPE2) { 242adc3f65aSKai Ye type = pre_parse_finished_bd(&status, resp); 243adc3f65aSKai Ye req = qp_ctx->req_list[status.tag]; 244adc3f65aSKai Ye } else { 245adc3f65aSKai Ye type = pre_parse_finished_bd3(&status, resp); 246adc3f65aSKai Ye req = (void *)(uintptr_t)status.tag; 247adc3f65aSKai Ye } 248adc3f65aSKai Ye 249adc3f65aSKai Ye if (unlikely(type != type_supported)) { 2508213a1a6SKai Ye atomic64_inc(&dfx->err_bd_cnt); 25149838259SKai Ye pr_err("err bd type [%u]\n", type); 252416d8220SZaibo Xu return; 253416d8220SZaibo Xu } 254416d8220SZaibo Xu 2558213a1a6SKai Ye if (unlikely(!req)) { 2568213a1a6SKai Ye atomic64_inc(&dfx->invalid_req_cnt); 2579597efc3SKai Ye atomic_inc(&qp->qp_status.used); 2588213a1a6SKai Ye return; 2598213a1a6SKai Ye } 260adc3f65aSKai Ye 261adc3f65aSKai Ye req->err_type = status.err_type; 262d6de2a59SZaibo Xu ctx = req->ctx; 263adc3f65aSKai Ye err = sec_cb_status_check(req, &status); 264adc3f65aSKai Ye if (err) 2658213a1a6SKai Ye atomic64_inc(&dfx->done_flag_cnt); 2661e9bc276SZaibo Xu 2678213a1a6SKai Ye atomic64_inc(&dfx->recv_cnt); 268416d8220SZaibo Xu 269d6de2a59SZaibo Xu ctx->req_op->buf_unmap(ctx, req); 270d6de2a59SZaibo Xu 271310ea0acSZaibo Xu ctx->req_op->callback(ctx, req, err); 272416d8220SZaibo Xu } 273416d8220SZaibo Xu 274416d8220SZaibo Xu static int sec_bd_send(struct sec_ctx *ctx, struct sec_req *req) 275416d8220SZaibo Xu { 276416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 277416d8220SZaibo Xu int ret; 278416d8220SZaibo Xu 2799597efc3SKai Ye if (ctx->fake_req_limit <= 2809597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && 2819597efc3SKai Ye !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)) 2829597efc3SKai Ye return -EBUSY; 2839597efc3SKai Ye 28402884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock); 285416d8220SZaibo Xu ret = hisi_qp_send(qp_ctx->qp, &req->sec_sqe); 2869597efc3SKai Ye 2879597efc3SKai Ye if (ctx->fake_req_limit <= 2889597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && !ret) { 2899597efc3SKai Ye list_add_tail(&req->backlog_head, &qp_ctx->backlog); 290cb1eeb75SArnd Bergmann atomic64_inc(&ctx->sec->debug.dfx.send_cnt); 2919597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.send_busy_cnt); 29202884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock); 2939597efc3SKai Ye return -EBUSY; 2949597efc3SKai Ye } 29502884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock); 296416d8220SZaibo Xu 297b9c8d897SZaibo Xu if (unlikely(ret == -EBUSY)) 298416d8220SZaibo Xu return -ENOBUFS; 299416d8220SZaibo Xu 3009597efc3SKai Ye if (likely(!ret)) { 301416d8220SZaibo Xu ret = -EINPROGRESS; 3029597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.send_cnt); 3038213a1a6SKai Ye } 304416d8220SZaibo Xu 305416d8220SZaibo Xu return ret; 306416d8220SZaibo Xu } 307416d8220SZaibo Xu 3087c7d902aSZaibo Xu /* Get DMA memory resources */ 3097c7d902aSZaibo Xu static int sec_alloc_civ_resource(struct device *dev, struct sec_alg_res *res) 3107c7d902aSZaibo Xu { 311129a9f34SWeili Qian u16 q_depth = res->depth; 3127c7d902aSZaibo Xu int i; 3137c7d902aSZaibo Xu 314129a9f34SWeili Qian res->c_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), 3157c7d902aSZaibo Xu &res->c_ivin_dma, GFP_KERNEL); 3167c7d902aSZaibo Xu if (!res->c_ivin) 3177c7d902aSZaibo Xu return -ENOMEM; 3187c7d902aSZaibo Xu 319129a9f34SWeili Qian for (i = 1; i < q_depth; i++) { 3207c7d902aSZaibo Xu res[i].c_ivin_dma = res->c_ivin_dma + i * SEC_IV_SIZE; 3217c7d902aSZaibo Xu res[i].c_ivin = res->c_ivin + i * SEC_IV_SIZE; 3227c7d902aSZaibo Xu } 3237c7d902aSZaibo Xu 3247c7d902aSZaibo Xu return 0; 3257c7d902aSZaibo Xu } 3267c7d902aSZaibo Xu 3277c7d902aSZaibo Xu static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) 3287c7d902aSZaibo Xu { 3297c7d902aSZaibo Xu if (res->c_ivin) 330129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), 3317c7d902aSZaibo Xu res->c_ivin, res->c_ivin_dma); 3327c7d902aSZaibo Xu } 3337c7d902aSZaibo Xu 334c16a70c1SKai Ye static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) 335c16a70c1SKai Ye { 336129a9f34SWeili Qian u16 q_depth = res->depth; 337c16a70c1SKai Ye int i; 338c16a70c1SKai Ye 339129a9f34SWeili Qian res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ(q_depth), 340c16a70c1SKai Ye &res->a_ivin_dma, GFP_KERNEL); 341c16a70c1SKai Ye if (!res->a_ivin) 342c16a70c1SKai Ye return -ENOMEM; 343c16a70c1SKai Ye 344129a9f34SWeili Qian for (i = 1; i < q_depth; i++) { 345c16a70c1SKai Ye res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE; 346c16a70c1SKai Ye res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE; 347c16a70c1SKai Ye } 348c16a70c1SKai Ye 349c16a70c1SKai Ye return 0; 350c16a70c1SKai Ye } 351c16a70c1SKai Ye 352c16a70c1SKai Ye static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res) 353c16a70c1SKai Ye { 354c16a70c1SKai Ye if (res->a_ivin) 355129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_IV_SZ(res->depth), 356c16a70c1SKai Ye res->a_ivin, res->a_ivin_dma); 357c16a70c1SKai Ye } 358c16a70c1SKai Ye 3592f072d75SZaibo Xu static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) 3602f072d75SZaibo Xu { 361129a9f34SWeili Qian u16 q_depth = res->depth; 3622f072d75SZaibo Xu int i; 3632f072d75SZaibo Xu 364129a9f34SWeili Qian res->out_mac = dma_alloc_coherent(dev, SEC_TOTAL_MAC_SZ(q_depth) << 1, 3652f072d75SZaibo Xu &res->out_mac_dma, GFP_KERNEL); 3662f072d75SZaibo Xu if (!res->out_mac) 3672f072d75SZaibo Xu return -ENOMEM; 3682f072d75SZaibo Xu 369129a9f34SWeili Qian for (i = 1; i < q_depth; i++) { 3702f072d75SZaibo Xu res[i].out_mac_dma = res->out_mac_dma + 3712f072d75SZaibo Xu i * (SEC_MAX_MAC_LEN << 1); 3722f072d75SZaibo Xu res[i].out_mac = res->out_mac + i * (SEC_MAX_MAC_LEN << 1); 3732f072d75SZaibo Xu } 3742f072d75SZaibo Xu 3752f072d75SZaibo Xu return 0; 3762f072d75SZaibo Xu } 3772f072d75SZaibo Xu 3782f072d75SZaibo Xu static void sec_free_mac_resource(struct device *dev, struct sec_alg_res *res) 3792f072d75SZaibo Xu { 3802f072d75SZaibo Xu if (res->out_mac) 381129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_MAC_SZ(res->depth) << 1, 3822f072d75SZaibo Xu res->out_mac, res->out_mac_dma); 3832f072d75SZaibo Xu } 3842f072d75SZaibo Xu 38574b58db8SLongfang Liu static void sec_free_pbuf_resource(struct device *dev, struct sec_alg_res *res) 38674b58db8SLongfang Liu { 38774b58db8SLongfang Liu if (res->pbuf) 388129a9f34SWeili Qian dma_free_coherent(dev, SEC_TOTAL_PBUF_SZ(res->depth), 38974b58db8SLongfang Liu res->pbuf, res->pbuf_dma); 39074b58db8SLongfang Liu } 39174b58db8SLongfang Liu 39274b58db8SLongfang Liu /* 39374b58db8SLongfang Liu * To improve performance, pbuffer is used for 39474b58db8SLongfang Liu * small packets (< 512Bytes) as IOMMU translation using. 39574b58db8SLongfang Liu */ 39674b58db8SLongfang Liu static int sec_alloc_pbuf_resource(struct device *dev, struct sec_alg_res *res) 39774b58db8SLongfang Liu { 398129a9f34SWeili Qian u16 q_depth = res->depth; 399129a9f34SWeili Qian int size = SEC_PBUF_PAGE_NUM(q_depth); 40074b58db8SLongfang Liu int pbuf_page_offset; 40174b58db8SLongfang Liu int i, j, k; 40274b58db8SLongfang Liu 403129a9f34SWeili Qian res->pbuf = dma_alloc_coherent(dev, SEC_TOTAL_PBUF_SZ(q_depth), 40474b58db8SLongfang Liu &res->pbuf_dma, GFP_KERNEL); 40574b58db8SLongfang Liu if (!res->pbuf) 40674b58db8SLongfang Liu return -ENOMEM; 40774b58db8SLongfang Liu 40874b58db8SLongfang Liu /* 40974b58db8SLongfang Liu * SEC_PBUF_PKG contains data pbuf, iv and 41074b58db8SLongfang Liu * out_mac : <SEC_PBUF|SEC_IV|SEC_MAC> 41174b58db8SLongfang Liu * Every PAGE contains six SEC_PBUF_PKG 41274b58db8SLongfang Liu * The sec_qp_ctx contains QM_Q_DEPTH numbers of SEC_PBUF_PKG 41374b58db8SLongfang Liu * So we need SEC_PBUF_PAGE_NUM numbers of PAGE 41474b58db8SLongfang Liu * for the SEC_TOTAL_PBUF_SZ 41574b58db8SLongfang Liu */ 416129a9f34SWeili Qian for (i = 0; i <= size; i++) { 41774b58db8SLongfang Liu pbuf_page_offset = PAGE_SIZE * i; 41874b58db8SLongfang Liu for (j = 0; j < SEC_PBUF_NUM; j++) { 41974b58db8SLongfang Liu k = i * SEC_PBUF_NUM + j; 420129a9f34SWeili Qian if (k == q_depth) 42174b58db8SLongfang Liu break; 42274b58db8SLongfang Liu res[k].pbuf = res->pbuf + 42374b58db8SLongfang Liu j * SEC_PBUF_PKG + pbuf_page_offset; 42474b58db8SLongfang Liu res[k].pbuf_dma = res->pbuf_dma + 42574b58db8SLongfang Liu j * SEC_PBUF_PKG + pbuf_page_offset; 42674b58db8SLongfang Liu } 42774b58db8SLongfang Liu } 428633e507fSLongfang Liu 42974b58db8SLongfang Liu return 0; 43074b58db8SLongfang Liu } 43174b58db8SLongfang Liu 4327c7d902aSZaibo Xu static int sec_alg_resource_alloc(struct sec_ctx *ctx, 4337c7d902aSZaibo Xu struct sec_qp_ctx *qp_ctx) 4347c7d902aSZaibo Xu { 4352f072d75SZaibo Xu struct sec_alg_res *res = qp_ctx->res; 436a44dce50SLongfang Liu struct device *dev = ctx->dev; 4372f072d75SZaibo Xu int ret; 4387c7d902aSZaibo Xu 4392f072d75SZaibo Xu ret = sec_alloc_civ_resource(dev, res); 4402f072d75SZaibo Xu if (ret) 4412f072d75SZaibo Xu return ret; 4422f072d75SZaibo Xu 4432f072d75SZaibo Xu if (ctx->alg_type == SEC_AEAD) { 444c16a70c1SKai Ye ret = sec_alloc_aiv_resource(dev, res); 445c16a70c1SKai Ye if (ret) 446c16a70c1SKai Ye goto alloc_aiv_fail; 447c16a70c1SKai Ye 4482f072d75SZaibo Xu ret = sec_alloc_mac_resource(dev, res); 4492f072d75SZaibo Xu if (ret) 450c16a70c1SKai Ye goto alloc_mac_fail; 4512f072d75SZaibo Xu } 45274b58db8SLongfang Liu if (ctx->pbuf_supported) { 45374b58db8SLongfang Liu ret = sec_alloc_pbuf_resource(dev, res); 45474b58db8SLongfang Liu if (ret) { 45574b58db8SLongfang Liu dev_err(dev, "fail to alloc pbuf dma resource!\n"); 45624efcec2SLongfang Liu goto alloc_pbuf_fail; 45774b58db8SLongfang Liu } 45874b58db8SLongfang Liu } 4592f072d75SZaibo Xu 4602f072d75SZaibo Xu return 0; 461633e507fSLongfang Liu 46224efcec2SLongfang Liu alloc_pbuf_fail: 46324efcec2SLongfang Liu if (ctx->alg_type == SEC_AEAD) 46424efcec2SLongfang Liu sec_free_mac_resource(dev, qp_ctx->res); 465c16a70c1SKai Ye alloc_mac_fail: 466c16a70c1SKai Ye if (ctx->alg_type == SEC_AEAD) 467c16a70c1SKai Ye sec_free_aiv_resource(dev, res); 468c16a70c1SKai Ye alloc_aiv_fail: 4692f072d75SZaibo Xu sec_free_civ_resource(dev, res); 4702f072d75SZaibo Xu return ret; 4717c7d902aSZaibo Xu } 4727c7d902aSZaibo Xu 4737c7d902aSZaibo Xu static void sec_alg_resource_free(struct sec_ctx *ctx, 4747c7d902aSZaibo Xu struct sec_qp_ctx *qp_ctx) 4757c7d902aSZaibo Xu { 476a44dce50SLongfang Liu struct device *dev = ctx->dev; 4777c7d902aSZaibo Xu 4787c7d902aSZaibo Xu sec_free_civ_resource(dev, qp_ctx->res); 4792f072d75SZaibo Xu 48074b58db8SLongfang Liu if (ctx->pbuf_supported) 48174b58db8SLongfang Liu sec_free_pbuf_resource(dev, qp_ctx->res); 4822f072d75SZaibo Xu if (ctx->alg_type == SEC_AEAD) 4832f072d75SZaibo Xu sec_free_mac_resource(dev, qp_ctx->res); 4847c7d902aSZaibo Xu } 4857c7d902aSZaibo Xu 486129a9f34SWeili Qian static int sec_alloc_qp_ctx_resource(struct hisi_qm *qm, struct sec_ctx *ctx, 487129a9f34SWeili Qian struct sec_qp_ctx *qp_ctx) 488129a9f34SWeili Qian { 489129a9f34SWeili Qian u16 q_depth = qp_ctx->qp->sq_depth; 490129a9f34SWeili Qian struct device *dev = ctx->dev; 491129a9f34SWeili Qian int ret = -ENOMEM; 492129a9f34SWeili Qian 493129a9f34SWeili Qian qp_ctx->req_list = kcalloc(q_depth, sizeof(struct sec_req *), GFP_KERNEL); 494129a9f34SWeili Qian if (!qp_ctx->req_list) 495129a9f34SWeili Qian return ret; 496129a9f34SWeili Qian 497129a9f34SWeili Qian qp_ctx->res = kcalloc(q_depth, sizeof(struct sec_alg_res), GFP_KERNEL); 498129a9f34SWeili Qian if (!qp_ctx->res) 499129a9f34SWeili Qian goto err_free_req_list; 500129a9f34SWeili Qian qp_ctx->res->depth = q_depth; 501129a9f34SWeili Qian 502129a9f34SWeili Qian qp_ctx->c_in_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); 503129a9f34SWeili Qian if (IS_ERR(qp_ctx->c_in_pool)) { 504129a9f34SWeili Qian dev_err(dev, "fail to create sgl pool for input!\n"); 505129a9f34SWeili Qian goto err_free_res; 506129a9f34SWeili Qian } 507129a9f34SWeili Qian 508129a9f34SWeili Qian qp_ctx->c_out_pool = hisi_acc_create_sgl_pool(dev, q_depth, SEC_SGL_SGE_NR); 509129a9f34SWeili Qian if (IS_ERR(qp_ctx->c_out_pool)) { 510129a9f34SWeili Qian dev_err(dev, "fail to create sgl pool for output!\n"); 511129a9f34SWeili Qian goto err_free_c_in_pool; 512129a9f34SWeili Qian } 513129a9f34SWeili Qian 514129a9f34SWeili Qian ret = sec_alg_resource_alloc(ctx, qp_ctx); 515129a9f34SWeili Qian if (ret) 516129a9f34SWeili Qian goto err_free_c_out_pool; 517129a9f34SWeili Qian 518129a9f34SWeili Qian return 0; 519129a9f34SWeili Qian 520129a9f34SWeili Qian err_free_c_out_pool: 521129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 522129a9f34SWeili Qian err_free_c_in_pool: 523129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 524129a9f34SWeili Qian err_free_res: 525129a9f34SWeili Qian kfree(qp_ctx->res); 526129a9f34SWeili Qian err_free_req_list: 527129a9f34SWeili Qian kfree(qp_ctx->req_list); 528129a9f34SWeili Qian return ret; 529129a9f34SWeili Qian } 530129a9f34SWeili Qian 531129a9f34SWeili Qian static void sec_free_qp_ctx_resource(struct sec_ctx *ctx, struct sec_qp_ctx *qp_ctx) 532129a9f34SWeili Qian { 533129a9f34SWeili Qian struct device *dev = ctx->dev; 534129a9f34SWeili Qian 535129a9f34SWeili Qian sec_alg_resource_free(ctx, qp_ctx); 536129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_out_pool); 537129a9f34SWeili Qian hisi_acc_free_sgl_pool(dev, qp_ctx->c_in_pool); 538129a9f34SWeili Qian kfree(qp_ctx->res); 539129a9f34SWeili Qian kfree(qp_ctx->req_list); 540129a9f34SWeili Qian } 541129a9f34SWeili Qian 542416d8220SZaibo Xu static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, 543416d8220SZaibo Xu int qp_ctx_id, int alg_type) 544416d8220SZaibo Xu { 545416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx; 546416d8220SZaibo Xu struct hisi_qp *qp; 547129a9f34SWeili Qian int ret; 548416d8220SZaibo Xu 549416d8220SZaibo Xu qp_ctx = &ctx->qp_ctx[qp_ctx_id]; 5500b5e43bcSKai Ye qp = ctx->qps[qp_ctx_id]; 551416d8220SZaibo Xu qp->req_type = 0; 552416d8220SZaibo Xu qp->qp_ctx = qp_ctx; 553416d8220SZaibo Xu qp_ctx->qp = qp; 554416d8220SZaibo Xu qp_ctx->ctx = ctx; 555416d8220SZaibo Xu 556adc3f65aSKai Ye qp->req_cb = sec_req_cb; 557adc3f65aSKai Ye 55802884a4fSZhengchao Shao spin_lock_init(&qp_ctx->req_lock); 559416d8220SZaibo Xu idr_init(&qp_ctx->req_idr); 5609597efc3SKai Ye INIT_LIST_HEAD(&qp_ctx->backlog); 561416d8220SZaibo Xu 562129a9f34SWeili Qian ret = sec_alloc_qp_ctx_resource(qm, ctx, qp_ctx); 563416d8220SZaibo Xu if (ret) 564129a9f34SWeili Qian goto err_destroy_idr; 565416d8220SZaibo Xu 566416d8220SZaibo Xu ret = hisi_qm_start_qp(qp, 0); 567416d8220SZaibo Xu if (ret < 0) 568129a9f34SWeili Qian goto err_resource_free; 569416d8220SZaibo Xu 570416d8220SZaibo Xu return 0; 571416d8220SZaibo Xu 572129a9f34SWeili Qian err_resource_free: 573129a9f34SWeili Qian sec_free_qp_ctx_resource(ctx, qp_ctx); 574416d8220SZaibo Xu err_destroy_idr: 575416d8220SZaibo Xu idr_destroy(&qp_ctx->req_idr); 576416d8220SZaibo Xu return ret; 577416d8220SZaibo Xu } 578416d8220SZaibo Xu 579416d8220SZaibo Xu static void sec_release_qp_ctx(struct sec_ctx *ctx, 580416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx) 581416d8220SZaibo Xu { 582416d8220SZaibo Xu hisi_qm_stop_qp(qp_ctx->qp); 583129a9f34SWeili Qian sec_free_qp_ctx_resource(ctx, qp_ctx); 584416d8220SZaibo Xu idr_destroy(&qp_ctx->req_idr); 585416d8220SZaibo Xu } 586416d8220SZaibo Xu 587473a0f96SZaibo Xu static int sec_ctx_base_init(struct sec_ctx *ctx) 588416d8220SZaibo Xu { 589416d8220SZaibo Xu struct sec_dev *sec; 590416d8220SZaibo Xu int i, ret; 591416d8220SZaibo Xu 5920b5e43bcSKai Ye ctx->qps = sec_create_qps(); 5930b5e43bcSKai Ye if (!ctx->qps) { 5940b5e43bcSKai Ye pr_err("Can not create sec qps!\n"); 595416d8220SZaibo Xu return -ENODEV; 596416d8220SZaibo Xu } 5970b5e43bcSKai Ye 5980b5e43bcSKai Ye sec = container_of(ctx->qps[0]->qm, struct sec_dev, qm); 599416d8220SZaibo Xu ctx->sec = sec; 600a44dce50SLongfang Liu ctx->dev = &sec->qm.pdev->dev; 601a718cfceSZaibo Xu ctx->hlf_q_num = sec->ctx_q_num >> 1; 602416d8220SZaibo Xu 60374b58db8SLongfang Liu ctx->pbuf_supported = ctx->sec->iommu_used; 60474b58db8SLongfang Liu 605416d8220SZaibo Xu /* Half of queue depth is taken as fake requests limit in the queue. */ 606129a9f34SWeili Qian ctx->fake_req_limit = ctx->qps[0]->sq_depth >> 1; 607416d8220SZaibo Xu ctx->qp_ctx = kcalloc(sec->ctx_q_num, sizeof(struct sec_qp_ctx), 608416d8220SZaibo Xu GFP_KERNEL); 60924efcec2SLongfang Liu if (!ctx->qp_ctx) { 61024efcec2SLongfang Liu ret = -ENOMEM; 61124efcec2SLongfang Liu goto err_destroy_qps; 61224efcec2SLongfang Liu } 613416d8220SZaibo Xu 614416d8220SZaibo Xu for (i = 0; i < sec->ctx_q_num; i++) { 615473a0f96SZaibo Xu ret = sec_create_qp_ctx(&sec->qm, ctx, i, 0); 616416d8220SZaibo Xu if (ret) 617416d8220SZaibo Xu goto err_sec_release_qp_ctx; 618416d8220SZaibo Xu } 619416d8220SZaibo Xu 620416d8220SZaibo Xu return 0; 62124efcec2SLongfang Liu 622416d8220SZaibo Xu err_sec_release_qp_ctx: 623416d8220SZaibo Xu for (i = i - 1; i >= 0; i--) 624416d8220SZaibo Xu sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 625416d8220SZaibo Xu kfree(ctx->qp_ctx); 62624efcec2SLongfang Liu err_destroy_qps: 62724efcec2SLongfang Liu sec_destroy_qps(ctx->qps, sec->ctx_q_num); 628416d8220SZaibo Xu return ret; 629416d8220SZaibo Xu } 630416d8220SZaibo Xu 631473a0f96SZaibo Xu static void sec_ctx_base_uninit(struct sec_ctx *ctx) 632416d8220SZaibo Xu { 633473a0f96SZaibo Xu int i; 634416d8220SZaibo Xu 635416d8220SZaibo Xu for (i = 0; i < ctx->sec->ctx_q_num; i++) 636416d8220SZaibo Xu sec_release_qp_ctx(ctx, &ctx->qp_ctx[i]); 637416d8220SZaibo Xu 6380b5e43bcSKai Ye sec_destroy_qps(ctx->qps, ctx->sec->ctx_q_num); 639416d8220SZaibo Xu kfree(ctx->qp_ctx); 640416d8220SZaibo Xu } 641416d8220SZaibo Xu 642473a0f96SZaibo Xu static int sec_cipher_init(struct sec_ctx *ctx) 643473a0f96SZaibo Xu { 644473a0f96SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 645473a0f96SZaibo Xu 646a44dce50SLongfang Liu c_ctx->c_key = dma_alloc_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 647473a0f96SZaibo Xu &c_ctx->c_key_dma, GFP_KERNEL); 648473a0f96SZaibo Xu if (!c_ctx->c_key) 649473a0f96SZaibo Xu return -ENOMEM; 650473a0f96SZaibo Xu 651473a0f96SZaibo Xu return 0; 652473a0f96SZaibo Xu } 653473a0f96SZaibo Xu 654473a0f96SZaibo Xu static void sec_cipher_uninit(struct sec_ctx *ctx) 655473a0f96SZaibo Xu { 656473a0f96SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 657473a0f96SZaibo Xu 658473a0f96SZaibo Xu memzero_explicit(c_ctx->c_key, SEC_MAX_KEY_SIZE); 659a44dce50SLongfang Liu dma_free_coherent(ctx->dev, SEC_MAX_KEY_SIZE, 660473a0f96SZaibo Xu c_ctx->c_key, c_ctx->c_key_dma); 661473a0f96SZaibo Xu } 662473a0f96SZaibo Xu 6632f072d75SZaibo Xu static int sec_auth_init(struct sec_ctx *ctx) 6642f072d75SZaibo Xu { 6652f072d75SZaibo Xu struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 6662f072d75SZaibo Xu 66745f5d017SKai Ye a_ctx->a_key = dma_alloc_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, 6682f072d75SZaibo Xu &a_ctx->a_key_dma, GFP_KERNEL); 6692f072d75SZaibo Xu if (!a_ctx->a_key) 6702f072d75SZaibo Xu return -ENOMEM; 6712f072d75SZaibo Xu 6722f072d75SZaibo Xu return 0; 6732f072d75SZaibo Xu } 6742f072d75SZaibo Xu 6752f072d75SZaibo Xu static void sec_auth_uninit(struct sec_ctx *ctx) 6762f072d75SZaibo Xu { 6772f072d75SZaibo Xu struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 6782f072d75SZaibo Xu 67945f5d017SKai Ye memzero_explicit(a_ctx->a_key, SEC_MAX_AKEY_SIZE); 68045f5d017SKai Ye dma_free_coherent(ctx->dev, SEC_MAX_AKEY_SIZE, 6812f072d75SZaibo Xu a_ctx->a_key, a_ctx->a_key_dma); 6822f072d75SZaibo Xu } 6832f072d75SZaibo Xu 6845652d55aSKai Ye static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) 6855652d55aSKai Ye { 6865652d55aSKai Ye const char *alg = crypto_tfm_alg_name(&tfm->base); 6875652d55aSKai Ye struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 6885652d55aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 6895652d55aSKai Ye 6905652d55aSKai Ye c_ctx->fallback = false; 691e764d81dSKai Ye 692e764d81dSKai Ye /* Currently, only XTS mode need fallback tfm when using 192bit key */ 6935652d55aSKai Ye if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ))) 6945652d55aSKai Ye return 0; 6955652d55aSKai Ye 6965652d55aSKai Ye c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0, 6975652d55aSKai Ye CRYPTO_ALG_NEED_FALLBACK); 6985652d55aSKai Ye if (IS_ERR(c_ctx->fbtfm)) { 699e764d81dSKai Ye pr_err("failed to alloc xts mode fallback tfm!\n"); 7005652d55aSKai Ye return PTR_ERR(c_ctx->fbtfm); 7015652d55aSKai Ye } 7025652d55aSKai Ye 7035652d55aSKai Ye return 0; 7045652d55aSKai Ye } 7055652d55aSKai Ye 706473a0f96SZaibo Xu static int sec_skcipher_init(struct crypto_skcipher *tfm) 707473a0f96SZaibo Xu { 708473a0f96SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 709473a0f96SZaibo Xu int ret; 710473a0f96SZaibo Xu 7112f072d75SZaibo Xu ctx->alg_type = SEC_SKCIPHER; 712473a0f96SZaibo Xu crypto_skcipher_set_reqsize(tfm, sizeof(struct sec_req)); 713473a0f96SZaibo Xu ctx->c_ctx.ivsize = crypto_skcipher_ivsize(tfm); 714473a0f96SZaibo Xu if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { 7154b7aef02SLongfang Liu pr_err("get error skcipher iv size!\n"); 716473a0f96SZaibo Xu return -EINVAL; 717473a0f96SZaibo Xu } 718473a0f96SZaibo Xu 719473a0f96SZaibo Xu ret = sec_ctx_base_init(ctx); 720473a0f96SZaibo Xu if (ret) 721473a0f96SZaibo Xu return ret; 722473a0f96SZaibo Xu 723473a0f96SZaibo Xu ret = sec_cipher_init(ctx); 724473a0f96SZaibo Xu if (ret) 725473a0f96SZaibo Xu goto err_cipher_init; 726473a0f96SZaibo Xu 7275652d55aSKai Ye ret = sec_skcipher_fbtfm_init(tfm); 7285652d55aSKai Ye if (ret) 7295652d55aSKai Ye goto err_fbtfm_init; 7305652d55aSKai Ye 731473a0f96SZaibo Xu return 0; 732633e507fSLongfang Liu 7335652d55aSKai Ye err_fbtfm_init: 7345652d55aSKai Ye sec_cipher_uninit(ctx); 735473a0f96SZaibo Xu err_cipher_init: 736473a0f96SZaibo Xu sec_ctx_base_uninit(ctx); 737473a0f96SZaibo Xu return ret; 738473a0f96SZaibo Xu } 739473a0f96SZaibo Xu 740473a0f96SZaibo Xu static void sec_skcipher_uninit(struct crypto_skcipher *tfm) 741473a0f96SZaibo Xu { 742473a0f96SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 743473a0f96SZaibo Xu 7445652d55aSKai Ye if (ctx->c_ctx.fbtfm) 7455652d55aSKai Ye crypto_free_sync_skcipher(ctx->c_ctx.fbtfm); 7465652d55aSKai Ye 747473a0f96SZaibo Xu sec_cipher_uninit(ctx); 748473a0f96SZaibo Xu sec_ctx_base_uninit(ctx); 749473a0f96SZaibo Xu } 750473a0f96SZaibo Xu 751ae6ce7b1SKai Ye static int sec_skcipher_3des_setkey(struct crypto_skcipher *tfm, const u8 *key, 752416d8220SZaibo Xu const u32 keylen, 753416d8220SZaibo Xu const enum sec_cmode c_mode) 754416d8220SZaibo Xu { 755ae6ce7b1SKai Ye struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 756ae6ce7b1SKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 757ae6ce7b1SKai Ye int ret; 758ae6ce7b1SKai Ye 759ae6ce7b1SKai Ye ret = verify_skcipher_des3_key(tfm, key); 760ae6ce7b1SKai Ye if (ret) 761ae6ce7b1SKai Ye return ret; 762ae6ce7b1SKai Ye 763416d8220SZaibo Xu switch (keylen) { 764416d8220SZaibo Xu case SEC_DES3_2KEY_SIZE: 765416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_3DES_2KEY; 766416d8220SZaibo Xu break; 767416d8220SZaibo Xu case SEC_DES3_3KEY_SIZE: 768416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_3DES_3KEY; 769416d8220SZaibo Xu break; 770416d8220SZaibo Xu default: 771416d8220SZaibo Xu return -EINVAL; 772416d8220SZaibo Xu } 773416d8220SZaibo Xu 774416d8220SZaibo Xu return 0; 775416d8220SZaibo Xu } 776416d8220SZaibo Xu 777416d8220SZaibo Xu static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, 778416d8220SZaibo Xu const u32 keylen, 779416d8220SZaibo Xu const enum sec_cmode c_mode) 780416d8220SZaibo Xu { 781416d8220SZaibo Xu if (c_mode == SEC_CMODE_XTS) { 782416d8220SZaibo Xu switch (keylen) { 783416d8220SZaibo Xu case SEC_XTS_MIN_KEY_SIZE: 784416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT; 785416d8220SZaibo Xu break; 7865652d55aSKai Ye case SEC_XTS_MID_KEY_SIZE: 7875652d55aSKai Ye c_ctx->fallback = true; 7885652d55aSKai Ye break; 789416d8220SZaibo Xu case SEC_XTS_MAX_KEY_SIZE: 790416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT; 791416d8220SZaibo Xu break; 792416d8220SZaibo Xu default: 793416d8220SZaibo Xu pr_err("hisi_sec2: xts mode key error!\n"); 794416d8220SZaibo Xu return -EINVAL; 795416d8220SZaibo Xu } 796416d8220SZaibo Xu } else { 797adc3f65aSKai Ye if (c_ctx->c_alg == SEC_CALG_SM4 && 798adc3f65aSKai Ye keylen != AES_KEYSIZE_128) { 799adc3f65aSKai Ye pr_err("hisi_sec2: sm4 key error!\n"); 800adc3f65aSKai Ye return -EINVAL; 801adc3f65aSKai Ye } else { 802416d8220SZaibo Xu switch (keylen) { 803416d8220SZaibo Xu case AES_KEYSIZE_128: 804416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT; 805416d8220SZaibo Xu break; 806416d8220SZaibo Xu case AES_KEYSIZE_192: 807416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_192BIT; 808416d8220SZaibo Xu break; 809416d8220SZaibo Xu case AES_KEYSIZE_256: 810416d8220SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT; 811416d8220SZaibo Xu break; 812416d8220SZaibo Xu default: 813416d8220SZaibo Xu pr_err("hisi_sec2: aes key error!\n"); 814416d8220SZaibo Xu return -EINVAL; 815416d8220SZaibo Xu } 816416d8220SZaibo Xu } 817adc3f65aSKai Ye } 818416d8220SZaibo Xu 819416d8220SZaibo Xu return 0; 820416d8220SZaibo Xu } 821416d8220SZaibo Xu 822416d8220SZaibo Xu static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 823416d8220SZaibo Xu const u32 keylen, const enum sec_calg c_alg, 824416d8220SZaibo Xu const enum sec_cmode c_mode) 825416d8220SZaibo Xu { 826416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 827416d8220SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 828a44dce50SLongfang Liu struct device *dev = ctx->dev; 829416d8220SZaibo Xu int ret; 830416d8220SZaibo Xu 831416d8220SZaibo Xu if (c_mode == SEC_CMODE_XTS) { 832416d8220SZaibo Xu ret = xts_verify_key(tfm, key, keylen); 833416d8220SZaibo Xu if (ret) { 834a44dce50SLongfang Liu dev_err(dev, "xts mode key err!\n"); 835416d8220SZaibo Xu return ret; 836416d8220SZaibo Xu } 837416d8220SZaibo Xu } 838416d8220SZaibo Xu 839416d8220SZaibo Xu c_ctx->c_alg = c_alg; 840416d8220SZaibo Xu c_ctx->c_mode = c_mode; 841416d8220SZaibo Xu 842416d8220SZaibo Xu switch (c_alg) { 843416d8220SZaibo Xu case SEC_CALG_3DES: 844ae6ce7b1SKai Ye ret = sec_skcipher_3des_setkey(tfm, key, keylen, c_mode); 845416d8220SZaibo Xu break; 846416d8220SZaibo Xu case SEC_CALG_AES: 847416d8220SZaibo Xu case SEC_CALG_SM4: 848416d8220SZaibo Xu ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 849416d8220SZaibo Xu break; 850416d8220SZaibo Xu default: 851416d8220SZaibo Xu return -EINVAL; 852416d8220SZaibo Xu } 853416d8220SZaibo Xu 854416d8220SZaibo Xu if (ret) { 855a44dce50SLongfang Liu dev_err(dev, "set sec key err!\n"); 856416d8220SZaibo Xu return ret; 857416d8220SZaibo Xu } 858416d8220SZaibo Xu 859416d8220SZaibo Xu memcpy(c_ctx->c_key, key, keylen); 860e764d81dSKai Ye if (c_ctx->fallback && c_ctx->fbtfm) { 8615652d55aSKai Ye ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); 8625652d55aSKai Ye if (ret) { 8635652d55aSKai Ye dev_err(dev, "failed to set fallback skcipher key!\n"); 8645652d55aSKai Ye return ret; 8655652d55aSKai Ye } 8665652d55aSKai Ye } 867416d8220SZaibo Xu return 0; 868416d8220SZaibo Xu } 869416d8220SZaibo Xu 870416d8220SZaibo Xu #define GEN_SEC_SETKEY_FUNC(name, c_alg, c_mode) \ 871416d8220SZaibo Xu static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ 872416d8220SZaibo Xu u32 keylen) \ 873416d8220SZaibo Xu { \ 874416d8220SZaibo Xu return sec_skcipher_setkey(tfm, key, keylen, c_alg, c_mode); \ 875416d8220SZaibo Xu } 876416d8220SZaibo Xu 877416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) 878416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) 879416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) 8807b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_ofb, SEC_CALG_AES, SEC_CMODE_OFB) 8817b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_cfb, SEC_CALG_AES, SEC_CMODE_CFB) 8827b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR) 883416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) 884416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) 885416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) 886416d8220SZaibo Xu GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) 8877b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_ofb, SEC_CALG_SM4, SEC_CMODE_OFB) 8887b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_cfb, SEC_CALG_SM4, SEC_CMODE_CFB) 8897b44c0eeSKai Ye GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR) 890416d8220SZaibo Xu 89174b58db8SLongfang Liu static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, 89274b58db8SLongfang Liu struct scatterlist *src) 89374b58db8SLongfang Liu { 8946c46a329SKai Ye struct sec_aead_req *a_req = &req->aead_req; 8956c46a329SKai Ye struct aead_request *aead_req = a_req->aead_req; 89674b58db8SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 89774b58db8SLongfang Liu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 898a44dce50SLongfang Liu struct device *dev = ctx->dev; 89974b58db8SLongfang Liu int copy_size, pbuf_length; 90074b58db8SLongfang Liu int req_id = req->req_id; 9016c46a329SKai Ye struct crypto_aead *tfm; 9026c46a329SKai Ye size_t authsize; 9036c46a329SKai Ye u8 *mac_offset; 90474b58db8SLongfang Liu 90574b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) 90674b58db8SLongfang Liu copy_size = aead_req->cryptlen + aead_req->assoclen; 90774b58db8SLongfang Liu else 90874b58db8SLongfang Liu copy_size = c_req->c_len; 90974b58db8SLongfang Liu 91074b58db8SLongfang Liu pbuf_length = sg_copy_to_buffer(src, sg_nents(src), 9116c46a329SKai Ye qp_ctx->res[req_id].pbuf, copy_size); 91274b58db8SLongfang Liu if (unlikely(pbuf_length != copy_size)) { 91374b58db8SLongfang Liu dev_err(dev, "copy src data to pbuf error!\n"); 91474b58db8SLongfang Liu return -EINVAL; 91574b58db8SLongfang Liu } 9166c46a329SKai Ye if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { 9176c46a329SKai Ye tfm = crypto_aead_reqtfm(aead_req); 9186c46a329SKai Ye authsize = crypto_aead_authsize(tfm); 9196c46a329SKai Ye mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize; 9206c46a329SKai Ye memcpy(a_req->out_mac, mac_offset, authsize); 9216c46a329SKai Ye } 92274b58db8SLongfang Liu 9239039878aSKai Ye req->in_dma = qp_ctx->res[req_id].pbuf_dma; 9249039878aSKai Ye c_req->c_out_dma = req->in_dma; 92574b58db8SLongfang Liu 92674b58db8SLongfang Liu return 0; 92774b58db8SLongfang Liu } 92874b58db8SLongfang Liu 92974b58db8SLongfang Liu static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, 93074b58db8SLongfang Liu struct scatterlist *dst) 93174b58db8SLongfang Liu { 93274b58db8SLongfang Liu struct aead_request *aead_req = req->aead_req.aead_req; 93374b58db8SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 93474b58db8SLongfang Liu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 93574b58db8SLongfang Liu int copy_size, pbuf_length; 93674b58db8SLongfang Liu int req_id = req->req_id; 93774b58db8SLongfang Liu 93874b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) 93974b58db8SLongfang Liu copy_size = c_req->c_len + aead_req->assoclen; 94074b58db8SLongfang Liu else 94174b58db8SLongfang Liu copy_size = c_req->c_len; 94274b58db8SLongfang Liu 94374b58db8SLongfang Liu pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), 944668f1ab7SKai Ye qp_ctx->res[req_id].pbuf, copy_size); 94574b58db8SLongfang Liu if (unlikely(pbuf_length != copy_size)) 946668f1ab7SKai Ye dev_err(ctx->dev, "copy pbuf data to dst error!\n"); 947668f1ab7SKai Ye } 948668f1ab7SKai Ye 949668f1ab7SKai Ye static int sec_aead_mac_init(struct sec_aead_req *req) 950668f1ab7SKai Ye { 951668f1ab7SKai Ye struct aead_request *aead_req = req->aead_req; 952668f1ab7SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 953668f1ab7SKai Ye size_t authsize = crypto_aead_authsize(tfm); 954668f1ab7SKai Ye u8 *mac_out = req->out_mac; 955668f1ab7SKai Ye struct scatterlist *sgl = aead_req->src; 956668f1ab7SKai Ye size_t copy_size; 957668f1ab7SKai Ye off_t skip_size; 958668f1ab7SKai Ye 959668f1ab7SKai Ye /* Copy input mac */ 960668f1ab7SKai Ye skip_size = aead_req->assoclen + aead_req->cryptlen - authsize; 961668f1ab7SKai Ye copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out, 962668f1ab7SKai Ye authsize, skip_size); 963668f1ab7SKai Ye if (unlikely(copy_size != authsize)) 964668f1ab7SKai Ye return -EINVAL; 965668f1ab7SKai Ye 966668f1ab7SKai Ye return 0; 96774b58db8SLongfang Liu } 96874b58db8SLongfang Liu 9692514f559SLongfang Liu static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, 970416d8220SZaibo Xu struct scatterlist *src, struct scatterlist *dst) 971416d8220SZaibo Xu { 972416d8220SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 9732514f559SLongfang Liu struct sec_aead_req *a_req = &req->aead_req; 974416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 9752514f559SLongfang Liu struct sec_alg_res *res = &qp_ctx->res[req->req_id]; 976a44dce50SLongfang Liu struct device *dev = ctx->dev; 97774b58db8SLongfang Liu int ret; 9782514f559SLongfang Liu 97974b58db8SLongfang Liu if (req->use_pbuf) { 98074b58db8SLongfang Liu c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET; 98174b58db8SLongfang Liu c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET; 98274b58db8SLongfang Liu if (ctx->alg_type == SEC_AEAD) { 983c16a70c1SKai Ye a_req->a_ivin = res->a_ivin; 984c16a70c1SKai Ye a_req->a_ivin_dma = res->a_ivin_dma; 98574b58db8SLongfang Liu a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET; 98674b58db8SLongfang Liu a_req->out_mac_dma = res->pbuf_dma + 98774b58db8SLongfang Liu SEC_PBUF_MAC_OFFSET; 98874b58db8SLongfang Liu } 989668f1ab7SKai Ye ret = sec_cipher_pbuf_map(ctx, req, src); 99074b58db8SLongfang Liu 99174b58db8SLongfang Liu return ret; 99274b58db8SLongfang Liu } 9932514f559SLongfang Liu c_req->c_ivin = res->c_ivin; 9942514f559SLongfang Liu c_req->c_ivin_dma = res->c_ivin_dma; 9952514f559SLongfang Liu if (ctx->alg_type == SEC_AEAD) { 996c16a70c1SKai Ye a_req->a_ivin = res->a_ivin; 997c16a70c1SKai Ye a_req->a_ivin_dma = res->a_ivin_dma; 9982514f559SLongfang Liu a_req->out_mac = res->out_mac; 9992514f559SLongfang Liu a_req->out_mac_dma = res->out_mac_dma; 10002514f559SLongfang Liu } 1001416d8220SZaibo Xu 10029039878aSKai Ye req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, 1003416d8220SZaibo Xu qp_ctx->c_in_pool, 1004416d8220SZaibo Xu req->req_id, 10059039878aSKai Ye &req->in_dma); 10069039878aSKai Ye if (IS_ERR(req->in)) { 1007416d8220SZaibo Xu dev_err(dev, "fail to dma map input sgl buffers!\n"); 10089039878aSKai Ye return PTR_ERR(req->in); 1009416d8220SZaibo Xu } 1010416d8220SZaibo Xu 1011668f1ab7SKai Ye if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { 1012668f1ab7SKai Ye ret = sec_aead_mac_init(a_req); 1013668f1ab7SKai Ye if (unlikely(ret)) { 1014668f1ab7SKai Ye dev_err(dev, "fail to init mac data for ICV!\n"); 1015668f1ab7SKai Ye return ret; 1016668f1ab7SKai Ye } 1017668f1ab7SKai Ye } 10189039878aSKai Ye 1019416d8220SZaibo Xu if (dst == src) { 10209039878aSKai Ye c_req->c_out = req->in; 10219039878aSKai Ye c_req->c_out_dma = req->in_dma; 1022416d8220SZaibo Xu } else { 1023416d8220SZaibo Xu c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, 1024416d8220SZaibo Xu qp_ctx->c_out_pool, 1025416d8220SZaibo Xu req->req_id, 1026416d8220SZaibo Xu &c_req->c_out_dma); 1027416d8220SZaibo Xu 1028416d8220SZaibo Xu if (IS_ERR(c_req->c_out)) { 1029416d8220SZaibo Xu dev_err(dev, "fail to dma map output sgl buffers!\n"); 10309039878aSKai Ye hisi_acc_sg_buf_unmap(dev, src, req->in); 1031416d8220SZaibo Xu return PTR_ERR(c_req->c_out); 1032416d8220SZaibo Xu } 1033416d8220SZaibo Xu } 1034416d8220SZaibo Xu 1035416d8220SZaibo Xu return 0; 1036416d8220SZaibo Xu } 1037416d8220SZaibo Xu 10382514f559SLongfang Liu static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, 1039a181647cSZaibo Xu struct scatterlist *src, struct scatterlist *dst) 1040a181647cSZaibo Xu { 10412514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1042a44dce50SLongfang Liu struct device *dev = ctx->dev; 1043a181647cSZaibo Xu 104474b58db8SLongfang Liu if (req->use_pbuf) { 104574b58db8SLongfang Liu sec_cipher_pbuf_unmap(ctx, req, dst); 104674b58db8SLongfang Liu } else { 10472514f559SLongfang Liu if (dst != src) 10489039878aSKai Ye hisi_acc_sg_buf_unmap(dev, src, req->in); 10492514f559SLongfang Liu 10502514f559SLongfang Liu hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out); 1051a181647cSZaibo Xu } 105274b58db8SLongfang Liu } 1053a181647cSZaibo Xu 1054416d8220SZaibo Xu static int sec_skcipher_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 1055416d8220SZaibo Xu { 1056a181647cSZaibo Xu struct skcipher_request *sq = req->c_req.sk_req; 1057416d8220SZaibo Xu 10582514f559SLongfang Liu return sec_cipher_map(ctx, req, sq->src, sq->dst); 1059416d8220SZaibo Xu } 1060416d8220SZaibo Xu 1061416d8220SZaibo Xu static void sec_skcipher_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 1062416d8220SZaibo Xu { 10632514f559SLongfang Liu struct skcipher_request *sq = req->c_req.sk_req; 1064416d8220SZaibo Xu 10652514f559SLongfang Liu sec_cipher_unmap(ctx, req, sq->src, sq->dst); 1066416d8220SZaibo Xu } 1067416d8220SZaibo Xu 10682f072d75SZaibo Xu static int sec_aead_aes_set_key(struct sec_cipher_ctx *c_ctx, 10692f072d75SZaibo Xu struct crypto_authenc_keys *keys) 10702f072d75SZaibo Xu { 10712f072d75SZaibo Xu switch (keys->enckeylen) { 10722f072d75SZaibo Xu case AES_KEYSIZE_128: 10732f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_128BIT; 10742f072d75SZaibo Xu break; 10752f072d75SZaibo Xu case AES_KEYSIZE_192: 10762f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_192BIT; 10772f072d75SZaibo Xu break; 10782f072d75SZaibo Xu case AES_KEYSIZE_256: 10792f072d75SZaibo Xu c_ctx->c_key_len = SEC_CKEY_256BIT; 10802f072d75SZaibo Xu break; 10812f072d75SZaibo Xu default: 10822f072d75SZaibo Xu pr_err("hisi_sec2: aead aes key error!\n"); 10832f072d75SZaibo Xu return -EINVAL; 10842f072d75SZaibo Xu } 10852f072d75SZaibo Xu memcpy(c_ctx->c_key, keys->enckey, keys->enckeylen); 10862f072d75SZaibo Xu 10872f072d75SZaibo Xu return 0; 10882f072d75SZaibo Xu } 10892f072d75SZaibo Xu 10902f072d75SZaibo Xu static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, 10912f072d75SZaibo Xu struct crypto_authenc_keys *keys) 10922f072d75SZaibo Xu { 10932f072d75SZaibo Xu struct crypto_shash *hash_tfm = ctx->hash_tfm; 10945761498cSKai Ye int blocksize, digestsize, ret; 10952f072d75SZaibo Xu 10962f072d75SZaibo Xu if (!keys->authkeylen) { 10972f072d75SZaibo Xu pr_err("hisi_sec2: aead auth key error!\n"); 10982f072d75SZaibo Xu return -EINVAL; 10992f072d75SZaibo Xu } 11002f072d75SZaibo Xu 11012f072d75SZaibo Xu blocksize = crypto_shash_blocksize(hash_tfm); 11025761498cSKai Ye digestsize = crypto_shash_digestsize(hash_tfm); 11032f072d75SZaibo Xu if (keys->authkeylen > blocksize) { 110461c38e3aSEric Biggers ret = crypto_shash_tfm_digest(hash_tfm, keys->authkey, 11052f072d75SZaibo Xu keys->authkeylen, ctx->a_key); 11062f072d75SZaibo Xu if (ret) { 11072203d3f7SColin Ian King pr_err("hisi_sec2: aead auth digest error!\n"); 11082f072d75SZaibo Xu return -EINVAL; 11092f072d75SZaibo Xu } 11105761498cSKai Ye ctx->a_key_len = digestsize; 11112f072d75SZaibo Xu } else { 11122f072d75SZaibo Xu memcpy(ctx->a_key, keys->authkey, keys->authkeylen); 11132f072d75SZaibo Xu ctx->a_key_len = keys->authkeylen; 11142f072d75SZaibo Xu } 11152f072d75SZaibo Xu 11162f072d75SZaibo Xu return 0; 11172f072d75SZaibo Xu } 11182f072d75SZaibo Xu 11196c46a329SKai Ye static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize) 11206c46a329SKai Ye { 11216c46a329SKai Ye struct crypto_tfm *tfm = crypto_aead_tfm(aead); 11226c46a329SKai Ye struct sec_ctx *ctx = crypto_tfm_ctx(tfm); 11236c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 11246c46a329SKai Ye 11256c46a329SKai Ye if (unlikely(a_ctx->fallback_aead_tfm)) 11266c46a329SKai Ye return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize); 11276c46a329SKai Ye 11286c46a329SKai Ye return 0; 11296c46a329SKai Ye } 11306c46a329SKai Ye 11316c46a329SKai Ye static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx, 11326c46a329SKai Ye struct crypto_aead *tfm, const u8 *key, 11336c46a329SKai Ye unsigned int keylen) 11346c46a329SKai Ye { 11356c46a329SKai Ye crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK); 11366c46a329SKai Ye crypto_aead_set_flags(a_ctx->fallback_aead_tfm, 11376c46a329SKai Ye crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); 11386c46a329SKai Ye return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen); 11396c46a329SKai Ye } 11406c46a329SKai Ye 11412f072d75SZaibo Xu static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, 11422f072d75SZaibo Xu const u32 keylen, const enum sec_hash_alg a_alg, 11432f072d75SZaibo Xu const enum sec_calg c_alg, 11442f072d75SZaibo Xu const enum sec_mac_len mac_len, 11452f072d75SZaibo Xu const enum sec_cmode c_mode) 11462f072d75SZaibo Xu { 11472f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 11482f072d75SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 11496c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 1150a44dce50SLongfang Liu struct device *dev = ctx->dev; 11512f072d75SZaibo Xu struct crypto_authenc_keys keys; 11522f072d75SZaibo Xu int ret; 11532f072d75SZaibo Xu 11542f072d75SZaibo Xu ctx->a_ctx.a_alg = a_alg; 11552f072d75SZaibo Xu ctx->c_ctx.c_alg = c_alg; 11562f072d75SZaibo Xu ctx->a_ctx.mac_len = mac_len; 11572f072d75SZaibo Xu c_ctx->c_mode = c_mode; 11582f072d75SZaibo Xu 1159c16a70c1SKai Ye if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) { 1160c16a70c1SKai Ye ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); 1161c16a70c1SKai Ye if (ret) { 1162c16a70c1SKai Ye dev_err(dev, "set sec aes ccm cipher key err!\n"); 1163c16a70c1SKai Ye return ret; 1164c16a70c1SKai Ye } 1165c16a70c1SKai Ye memcpy(c_ctx->c_key, key, keylen); 1166c16a70c1SKai Ye 11676c46a329SKai Ye if (unlikely(a_ctx->fallback_aead_tfm)) { 11686c46a329SKai Ye ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); 11696c46a329SKai Ye if (ret) 11706c46a329SKai Ye return ret; 11716c46a329SKai Ye } 11726c46a329SKai Ye 1173c16a70c1SKai Ye return 0; 1174c16a70c1SKai Ye } 1175c16a70c1SKai Ye 11762f072d75SZaibo Xu if (crypto_authenc_extractkeys(&keys, key, keylen)) 11772f072d75SZaibo Xu goto bad_key; 11782f072d75SZaibo Xu 11792f072d75SZaibo Xu ret = sec_aead_aes_set_key(c_ctx, &keys); 11802f072d75SZaibo Xu if (ret) { 1181a44dce50SLongfang Liu dev_err(dev, "set sec cipher key err!\n"); 11822f072d75SZaibo Xu goto bad_key; 11832f072d75SZaibo Xu } 11842f072d75SZaibo Xu 11852f072d75SZaibo Xu ret = sec_aead_auth_set_key(&ctx->a_ctx, &keys); 11862f072d75SZaibo Xu if (ret) { 1187a44dce50SLongfang Liu dev_err(dev, "set sec auth key err!\n"); 11882f072d75SZaibo Xu goto bad_key; 11892f072d75SZaibo Xu } 11902f072d75SZaibo Xu 1191adc3f65aSKai Ye if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) || 1192adc3f65aSKai Ye (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) { 1193adc3f65aSKai Ye dev_err(dev, "MAC or AUTH key length error!\n"); 1194adc3f65aSKai Ye goto bad_key; 1195adc3f65aSKai Ye } 1196adc3f65aSKai Ye 11972f072d75SZaibo Xu return 0; 1198633e507fSLongfang Liu 11992f072d75SZaibo Xu bad_key: 12002f072d75SZaibo Xu memzero_explicit(&keys, sizeof(struct crypto_authenc_keys)); 12012f072d75SZaibo Xu return -EINVAL; 12022f072d75SZaibo Xu } 12032f072d75SZaibo Xu 12042f072d75SZaibo Xu 12052f072d75SZaibo Xu #define GEN_SEC_AEAD_SETKEY_FUNC(name, aalg, calg, maclen, cmode) \ 12062f072d75SZaibo Xu static int sec_setkey_##name(struct crypto_aead *tfm, const u8 *key, \ 12072f072d75SZaibo Xu u32 keylen) \ 12082f072d75SZaibo Xu { \ 12092f072d75SZaibo Xu return sec_aead_setkey(tfm, key, keylen, aalg, calg, maclen, cmode);\ 12102f072d75SZaibo Xu } 12112f072d75SZaibo Xu 12122f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha1, SEC_A_HMAC_SHA1, 12132f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA1_MAC, SEC_CMODE_CBC) 12142f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, 12152f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC) 12162f072d75SZaibo Xu GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, 12172f072d75SZaibo Xu SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC) 1218c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, 1219c16a70c1SKai Ye SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) 1220c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, 1221c16a70c1SKai Ye SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) 1222c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, 1223c16a70c1SKai Ye SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) 1224c16a70c1SKai Ye GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, 1225c16a70c1SKai Ye SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) 12262f072d75SZaibo Xu 12272f072d75SZaibo Xu static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) 12282f072d75SZaibo Xu { 12292f072d75SZaibo Xu struct aead_request *aq = req->aead_req.aead_req; 12302f072d75SZaibo Xu 12312514f559SLongfang Liu return sec_cipher_map(ctx, req, aq->src, aq->dst); 12322f072d75SZaibo Xu } 12332f072d75SZaibo Xu 12342f072d75SZaibo Xu static void sec_aead_sgl_unmap(struct sec_ctx *ctx, struct sec_req *req) 12352f072d75SZaibo Xu { 12362f072d75SZaibo Xu struct aead_request *aq = req->aead_req.aead_req; 12372f072d75SZaibo Xu 12382514f559SLongfang Liu sec_cipher_unmap(ctx, req, aq->src, aq->dst); 12392f072d75SZaibo Xu } 12402f072d75SZaibo Xu 1241416d8220SZaibo Xu static int sec_request_transfer(struct sec_ctx *ctx, struct sec_req *req) 1242416d8220SZaibo Xu { 1243416d8220SZaibo Xu int ret; 1244416d8220SZaibo Xu 1245416d8220SZaibo Xu ret = ctx->req_op->buf_map(ctx, req); 1246b9c8d897SZaibo Xu if (unlikely(ret)) 1247416d8220SZaibo Xu return ret; 1248416d8220SZaibo Xu 1249416d8220SZaibo Xu ctx->req_op->do_transfer(ctx, req); 1250416d8220SZaibo Xu 1251416d8220SZaibo Xu ret = ctx->req_op->bd_fill(ctx, req); 1252b9c8d897SZaibo Xu if (unlikely(ret)) 1253416d8220SZaibo Xu goto unmap_req_buf; 1254416d8220SZaibo Xu 1255416d8220SZaibo Xu return ret; 1256416d8220SZaibo Xu 1257416d8220SZaibo Xu unmap_req_buf: 1258416d8220SZaibo Xu ctx->req_op->buf_unmap(ctx, req); 1259416d8220SZaibo Xu return ret; 1260416d8220SZaibo Xu } 1261416d8220SZaibo Xu 1262416d8220SZaibo Xu static void sec_request_untransfer(struct sec_ctx *ctx, struct sec_req *req) 1263416d8220SZaibo Xu { 1264416d8220SZaibo Xu ctx->req_op->buf_unmap(ctx, req); 1265416d8220SZaibo Xu } 1266416d8220SZaibo Xu 1267416d8220SZaibo Xu static void sec_skcipher_copy_iv(struct sec_ctx *ctx, struct sec_req *req) 1268416d8220SZaibo Xu { 1269416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req; 12702514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1271416d8220SZaibo Xu 12722514f559SLongfang Liu memcpy(c_req->c_ivin, sk_req->iv, ctx->c_ctx.ivsize); 1273416d8220SZaibo Xu } 1274416d8220SZaibo Xu 1275416d8220SZaibo Xu static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 1276416d8220SZaibo Xu { 1277416d8220SZaibo Xu struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1278416d8220SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 1279416d8220SZaibo Xu struct sec_sqe *sec_sqe = &req->sec_sqe; 1280416d8220SZaibo Xu u8 scene, sa_type, da_type; 1281416d8220SZaibo Xu u8 bd_type, cipher; 12827c7d902aSZaibo Xu u8 de = 0; 1283416d8220SZaibo Xu 1284416d8220SZaibo Xu memset(sec_sqe, 0, sizeof(struct sec_sqe)); 1285416d8220SZaibo Xu 1286416d8220SZaibo Xu sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 12872514f559SLongfang Liu sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 12889039878aSKai Ye sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma); 1289416d8220SZaibo Xu sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1290416d8220SZaibo Xu 1291416d8220SZaibo Xu sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << 1292416d8220SZaibo Xu SEC_CMODE_OFFSET); 1293416d8220SZaibo Xu sec_sqe->type2.c_alg = c_ctx->c_alg; 1294416d8220SZaibo Xu sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1295416d8220SZaibo Xu SEC_CKEY_OFFSET); 1296416d8220SZaibo Xu 1297416d8220SZaibo Xu bd_type = SEC_BD_TYPE2; 1298416d8220SZaibo Xu if (c_req->encrypt) 1299416d8220SZaibo Xu cipher = SEC_CIPHER_ENC << SEC_CIPHER_OFFSET; 1300416d8220SZaibo Xu else 1301416d8220SZaibo Xu cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; 1302416d8220SZaibo Xu sec_sqe->type_cipher_auth = bd_type | cipher; 1303416d8220SZaibo Xu 1304adc3f65aSKai Ye /* Set destination and source address type */ 1305adc3f65aSKai Ye if (req->use_pbuf) { 130674b58db8SLongfang Liu sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET; 1307adc3f65aSKai Ye da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; 1308adc3f65aSKai Ye } else { 1309416d8220SZaibo Xu sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; 1310adc3f65aSKai Ye da_type = SEC_SGL << SEC_DST_SGL_OFFSET; 1311adc3f65aSKai Ye } 1312adc3f65aSKai Ye 1313adc3f65aSKai Ye sec_sqe->sdm_addr_type |= da_type; 1314416d8220SZaibo Xu scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; 13159039878aSKai Ye if (req->in_dma != c_req->c_out_dma) 1316416d8220SZaibo Xu de = 0x1 << SEC_DE_OFFSET; 1317416d8220SZaibo Xu 1318416d8220SZaibo Xu sec_sqe->sds_sa_type = (de | scene | sa_type); 1319416d8220SZaibo Xu 1320416d8220SZaibo Xu sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); 1321416d8220SZaibo Xu sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id); 1322416d8220SZaibo Xu 1323416d8220SZaibo Xu return 0; 1324416d8220SZaibo Xu } 1325416d8220SZaibo Xu 1326adc3f65aSKai Ye static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) 1327adc3f65aSKai Ye { 1328adc3f65aSKai Ye struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; 1329adc3f65aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 1330adc3f65aSKai Ye struct sec_cipher_req *c_req = &req->c_req; 1331adc3f65aSKai Ye u32 bd_param = 0; 1332adc3f65aSKai Ye u16 cipher; 1333adc3f65aSKai Ye 1334adc3f65aSKai Ye memset(sec_sqe3, 0, sizeof(struct sec_sqe3)); 1335adc3f65aSKai Ye 1336adc3f65aSKai Ye sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma); 1337adc3f65aSKai Ye sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); 13389039878aSKai Ye sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma); 1339adc3f65aSKai Ye sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma); 1340adc3f65aSKai Ye 1341adc3f65aSKai Ye sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) | 1342adc3f65aSKai Ye c_ctx->c_mode; 1343adc3f65aSKai Ye sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) << 1344adc3f65aSKai Ye SEC_CKEY_OFFSET_V3); 1345adc3f65aSKai Ye 1346adc3f65aSKai Ye if (c_req->encrypt) 1347adc3f65aSKai Ye cipher = SEC_CIPHER_ENC; 1348adc3f65aSKai Ye else 1349adc3f65aSKai Ye cipher = SEC_CIPHER_DEC; 1350adc3f65aSKai Ye sec_sqe3->c_icv_key |= cpu_to_le16(cipher); 1351adc3f65aSKai Ye 1352566f060fSKai Ye /* Set the CTR counter mode is 128bit rollover */ 1353566f060fSKai Ye sec_sqe3->auth_mac_key = cpu_to_le32((u32)SEC_CTR_CNT_ROLLOVER << 1354566f060fSKai Ye SEC_CTR_CNT_OFFSET); 1355566f060fSKai Ye 1356adc3f65aSKai Ye if (req->use_pbuf) { 1357adc3f65aSKai Ye bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3; 1358adc3f65aSKai Ye bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3; 1359adc3f65aSKai Ye } else { 1360adc3f65aSKai Ye bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3; 1361adc3f65aSKai Ye bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3; 1362adc3f65aSKai Ye } 1363adc3f65aSKai Ye 1364adc3f65aSKai Ye bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3; 13659039878aSKai Ye if (req->in_dma != c_req->c_out_dma) 1366adc3f65aSKai Ye bd_param |= 0x1 << SEC_DE_OFFSET_V3; 1367adc3f65aSKai Ye 1368adc3f65aSKai Ye bd_param |= SEC_BD_TYPE3; 1369adc3f65aSKai Ye sec_sqe3->bd_param = cpu_to_le32(bd_param); 1370adc3f65aSKai Ye 1371adc3f65aSKai Ye sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len); 1372adc3f65aSKai Ye sec_sqe3->tag = cpu_to_le64(req); 1373adc3f65aSKai Ye 1374adc3f65aSKai Ye return 0; 1375adc3f65aSKai Ye } 1376adc3f65aSKai Ye 13777b44c0eeSKai Ye /* increment counter (128-bit int) */ 13787b44c0eeSKai Ye static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) 13797b44c0eeSKai Ye { 13807b44c0eeSKai Ye do { 13817b44c0eeSKai Ye --bits; 13827b44c0eeSKai Ye nums += counter[bits]; 13837b44c0eeSKai Ye counter[bits] = nums & BITS_MASK; 13847b44c0eeSKai Ye nums >>= BYTE_BITS; 13857b44c0eeSKai Ye } while (bits && nums); 13867b44c0eeSKai Ye } 13877b44c0eeSKai Ye 13882f072d75SZaibo Xu static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) 1389416d8220SZaibo Xu { 13902f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req; 1391416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req; 1392416d8220SZaibo Xu u32 iv_size = req->ctx->c_ctx.ivsize; 1393416d8220SZaibo Xu struct scatterlist *sgl; 13942f072d75SZaibo Xu unsigned int cryptlen; 1395416d8220SZaibo Xu size_t sz; 13962f072d75SZaibo Xu u8 *iv; 1397416d8220SZaibo Xu 1398416d8220SZaibo Xu if (req->c_req.encrypt) 13992f072d75SZaibo Xu sgl = alg_type == SEC_SKCIPHER ? sk_req->dst : aead_req->dst; 1400416d8220SZaibo Xu else 14012f072d75SZaibo Xu sgl = alg_type == SEC_SKCIPHER ? sk_req->src : aead_req->src; 1402416d8220SZaibo Xu 14032f072d75SZaibo Xu if (alg_type == SEC_SKCIPHER) { 14042f072d75SZaibo Xu iv = sk_req->iv; 14052f072d75SZaibo Xu cryptlen = sk_req->cryptlen; 14062f072d75SZaibo Xu } else { 14072f072d75SZaibo Xu iv = aead_req->iv; 14082f072d75SZaibo Xu cryptlen = aead_req->cryptlen; 14092f072d75SZaibo Xu } 14102f072d75SZaibo Xu 14117b44c0eeSKai Ye if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) { 14122f072d75SZaibo Xu sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, 14132f072d75SZaibo Xu cryptlen - iv_size); 1414b9c8d897SZaibo Xu if (unlikely(sz != iv_size)) 1415a44dce50SLongfang Liu dev_err(req->ctx->dev, "copy output iv error!\n"); 14167b44c0eeSKai Ye } else { 14177b44c0eeSKai Ye sz = cryptlen / iv_size; 14187b44c0eeSKai Ye if (cryptlen % iv_size) 14197b44c0eeSKai Ye sz += 1; 14207b44c0eeSKai Ye ctr_iv_inc(iv, iv_size, sz); 14217b44c0eeSKai Ye } 1422416d8220SZaibo Xu } 1423416d8220SZaibo Xu 14249597efc3SKai Ye static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, 14259597efc3SKai Ye struct sec_qp_ctx *qp_ctx) 14269597efc3SKai Ye { 14279597efc3SKai Ye struct sec_req *backlog_req = NULL; 14289597efc3SKai Ye 142902884a4fSZhengchao Shao spin_lock_bh(&qp_ctx->req_lock); 14309597efc3SKai Ye if (ctx->fake_req_limit >= 14319597efc3SKai Ye atomic_read(&qp_ctx->qp->qp_status.used) && 14329597efc3SKai Ye !list_empty(&qp_ctx->backlog)) { 14339597efc3SKai Ye backlog_req = list_first_entry(&qp_ctx->backlog, 14349597efc3SKai Ye typeof(*backlog_req), backlog_head); 14359597efc3SKai Ye list_del(&backlog_req->backlog_head); 14369597efc3SKai Ye } 143702884a4fSZhengchao Shao spin_unlock_bh(&qp_ctx->req_lock); 14389597efc3SKai Ye 14399597efc3SKai Ye return backlog_req; 14409597efc3SKai Ye } 14419597efc3SKai Ye 1442310ea0acSZaibo Xu static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, 1443310ea0acSZaibo Xu int err) 1444416d8220SZaibo Xu { 1445416d8220SZaibo Xu struct skcipher_request *sk_req = req->c_req.sk_req; 1446416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 14479597efc3SKai Ye struct skcipher_request *backlog_sk_req; 14489597efc3SKai Ye struct sec_req *backlog_req; 1449416d8220SZaibo Xu 1450416d8220SZaibo Xu sec_free_req_id(req); 1451416d8220SZaibo Xu 14527b44c0eeSKai Ye /* IV output at encrypto of CBC/CTR mode */ 14537b44c0eeSKai Ye if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || 14547b44c0eeSKai Ye ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt) 14552f072d75SZaibo Xu sec_update_iv(req, SEC_SKCIPHER); 1456416d8220SZaibo Xu 14579597efc3SKai Ye while (1) { 14589597efc3SKai Ye backlog_req = sec_back_req_clear(ctx, qp_ctx); 14599597efc3SKai Ye if (!backlog_req) 14609597efc3SKai Ye break; 14619597efc3SKai Ye 14629597efc3SKai Ye backlog_sk_req = backlog_req->c_req.sk_req; 14639597efc3SKai Ye backlog_sk_req->base.complete(&backlog_sk_req->base, 14649597efc3SKai Ye -EINPROGRESS); 14659597efc3SKai Ye atomic64_inc(&ctx->sec->debug.dfx.recv_busy_cnt); 14669597efc3SKai Ye } 14679597efc3SKai Ye 1468310ea0acSZaibo Xu sk_req->base.complete(&sk_req->base, err); 1469416d8220SZaibo Xu } 1470416d8220SZaibo Xu 1471c16a70c1SKai Ye static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) 14722f072d75SZaibo Xu { 14732f072d75SZaibo Xu struct aead_request *aead_req = req->aead_req.aead_req; 14742514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1475c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req; 1476c16a70c1SKai Ye size_t authsize = ctx->a_ctx.mac_len; 1477c16a70c1SKai Ye u32 data_size = aead_req->cryptlen; 1478c16a70c1SKai Ye u8 flage = 0; 1479c16a70c1SKai Ye u8 cm, cl; 1480c16a70c1SKai Ye 1481c16a70c1SKai Ye /* the specification has been checked in aead_iv_demension_check() */ 1482c16a70c1SKai Ye cl = c_req->c_ivin[0] + 1; 1483c16a70c1SKai Ye c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00; 1484c16a70c1SKai Ye memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl); 1485c16a70c1SKai Ye c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT; 1486c16a70c1SKai Ye 1487c16a70c1SKai Ye /* the last 3bit is L' */ 1488c16a70c1SKai Ye flage |= c_req->c_ivin[0] & IV_CL_MASK; 1489c16a70c1SKai Ye 1490c16a70c1SKai Ye /* the M' is bit3~bit5, the Flags is bit6 */ 1491c16a70c1SKai Ye cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM; 1492c16a70c1SKai Ye flage |= cm << IV_CM_OFFSET; 1493c16a70c1SKai Ye if (aead_req->assoclen) 1494c16a70c1SKai Ye flage |= 0x01 << IV_FLAGS_OFFSET; 1495c16a70c1SKai Ye 1496c16a70c1SKai Ye memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize); 1497c16a70c1SKai Ye a_req->a_ivin[0] = flage; 1498c16a70c1SKai Ye 1499c16a70c1SKai Ye /* 1500c16a70c1SKai Ye * the last 32bit is counter's initial number, 1501c16a70c1SKai Ye * but the nonce uses the first 16bit 1502c16a70c1SKai Ye * the tail 16bit fill with the cipher length 1503c16a70c1SKai Ye */ 1504c16a70c1SKai Ye if (!c_req->encrypt) 1505c16a70c1SKai Ye data_size = aead_req->cryptlen - authsize; 1506c16a70c1SKai Ye 1507c16a70c1SKai Ye a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = 1508c16a70c1SKai Ye data_size & IV_LAST_BYTE_MASK; 1509c16a70c1SKai Ye data_size >>= IV_BYTE_OFFSET; 1510c16a70c1SKai Ye a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] = 1511c16a70c1SKai Ye data_size & IV_LAST_BYTE_MASK; 1512c16a70c1SKai Ye } 1513c16a70c1SKai Ye 1514c16a70c1SKai Ye static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) 1515c16a70c1SKai Ye { 1516c16a70c1SKai Ye struct aead_request *aead_req = req->aead_req.aead_req; 1517c16a70c1SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); 1518c16a70c1SKai Ye size_t authsize = crypto_aead_authsize(tfm); 1519c16a70c1SKai Ye struct sec_cipher_req *c_req = &req->c_req; 1520c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req; 15212f072d75SZaibo Xu 15222514f559SLongfang Liu memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); 1523c16a70c1SKai Ye 1524c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) { 1525c16a70c1SKai Ye /* 1526c16a70c1SKai Ye * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter}, 1527c16a70c1SKai Ye * the counter must set to 0x01 1528c16a70c1SKai Ye */ 1529c16a70c1SKai Ye ctx->a_ctx.mac_len = authsize; 1530c16a70c1SKai Ye /* CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */ 1531c16a70c1SKai Ye set_aead_auth_iv(ctx, req); 1532c16a70c1SKai Ye } 1533c16a70c1SKai Ye 1534c16a70c1SKai Ye /* GCM 12Byte Cipher_IV == Auth_IV */ 1535c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) { 1536c16a70c1SKai Ye ctx->a_ctx.mac_len = authsize; 1537c16a70c1SKai Ye memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE); 1538c16a70c1SKai Ye } 1539c16a70c1SKai Ye } 1540c16a70c1SKai Ye 1541c16a70c1SKai Ye static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir, 1542c16a70c1SKai Ye struct sec_req *req, struct sec_sqe *sec_sqe) 1543c16a70c1SKai Ye { 1544c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req; 1545c16a70c1SKai Ye struct aead_request *aq = a_req->aead_req; 1546c16a70c1SKai Ye 1547c16a70c1SKai Ye /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ 1548c16a70c1SKai Ye sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)ctx->mac_len); 1549c16a70c1SKai Ye 1550c16a70c1SKai Ye /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ 1551c16a70c1SKai Ye sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr; 1552c16a70c1SKai Ye sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); 1553c16a70c1SKai Ye sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET; 1554c16a70c1SKai Ye 1555c16a70c1SKai Ye if (dir) 1556c16a70c1SKai Ye sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 1557c16a70c1SKai Ye else 1558c16a70c1SKai Ye sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 1559c16a70c1SKai Ye 1560c16a70c1SKai Ye sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen); 1561c16a70c1SKai Ye sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0); 1562c16a70c1SKai Ye sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1563c16a70c1SKai Ye 1564c16a70c1SKai Ye sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); 1565c16a70c1SKai Ye } 1566c16a70c1SKai Ye 1567c16a70c1SKai Ye static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir, 1568c16a70c1SKai Ye struct sec_req *req, struct sec_sqe3 *sqe3) 1569c16a70c1SKai Ye { 1570c16a70c1SKai Ye struct sec_aead_req *a_req = &req->aead_req; 1571c16a70c1SKai Ye struct aead_request *aq = a_req->aead_req; 1572c16a70c1SKai Ye 1573c16a70c1SKai Ye /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ 1574c16a70c1SKai Ye sqe3->c_icv_key |= cpu_to_le16((u16)ctx->mac_len << SEC_MAC_OFFSET_V3); 1575c16a70c1SKai Ye 1576c16a70c1SKai Ye /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ 1577c16a70c1SKai Ye sqe3->a_key_addr = sqe3->c_key_addr; 1578c16a70c1SKai Ye sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); 1579c16a70c1SKai Ye sqe3->auth_mac_key |= SEC_NO_AUTH; 1580c16a70c1SKai Ye 1581c16a70c1SKai Ye if (dir) 1582c16a70c1SKai Ye sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; 1583c16a70c1SKai Ye else 1584c16a70c1SKai Ye sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; 1585c16a70c1SKai Ye 1586c16a70c1SKai Ye sqe3->a_len_key = cpu_to_le32(aq->assoclen); 1587c16a70c1SKai Ye sqe3->auth_src_offset = cpu_to_le16(0x0); 1588c16a70c1SKai Ye sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1589c16a70c1SKai Ye sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); 15902f072d75SZaibo Xu } 15912f072d75SZaibo Xu 15922f072d75SZaibo Xu static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, 15932f072d75SZaibo Xu struct sec_req *req, struct sec_sqe *sec_sqe) 15942f072d75SZaibo Xu { 15952f072d75SZaibo Xu struct sec_aead_req *a_req = &req->aead_req; 15962f072d75SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 15972f072d75SZaibo Xu struct aead_request *aq = a_req->aead_req; 15982f072d75SZaibo Xu 15992f072d75SZaibo Xu sec_sqe->type2.a_key_addr = cpu_to_le64(ctx->a_key_dma); 16002f072d75SZaibo Xu 16012f072d75SZaibo Xu sec_sqe->type2.mac_key_alg = 16022f072d75SZaibo Xu cpu_to_le32(ctx->mac_len / SEC_SQE_LEN_RATE); 16032f072d75SZaibo Xu 16042f072d75SZaibo Xu sec_sqe->type2.mac_key_alg |= 16052f072d75SZaibo Xu cpu_to_le32((u32)((ctx->a_key_len) / 16062f072d75SZaibo Xu SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET); 16072f072d75SZaibo Xu 16082f072d75SZaibo Xu sec_sqe->type2.mac_key_alg |= 16092f072d75SZaibo Xu cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); 16102f072d75SZaibo Xu 1611668f1ab7SKai Ye if (dir) { 16122f072d75SZaibo Xu sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; 16132f072d75SZaibo Xu sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; 1614668f1ab7SKai Ye } else { 1615668f1ab7SKai Ye sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET; 16162f072d75SZaibo Xu sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; 1617668f1ab7SKai Ye } 16182f072d75SZaibo Xu sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen); 16192f072d75SZaibo Xu 16202f072d75SZaibo Xu sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 16212f072d75SZaibo Xu 16222514f559SLongfang Liu sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); 16232f072d75SZaibo Xu } 16242f072d75SZaibo Xu 16252f072d75SZaibo Xu static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) 16262f072d75SZaibo Xu { 16272f072d75SZaibo Xu struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 16282f072d75SZaibo Xu struct sec_sqe *sec_sqe = &req->sec_sqe; 16292f072d75SZaibo Xu int ret; 16302f072d75SZaibo Xu 16312f072d75SZaibo Xu ret = sec_skcipher_bd_fill(ctx, req); 16322f072d75SZaibo Xu if (unlikely(ret)) { 1633a44dce50SLongfang Liu dev_err(ctx->dev, "skcipher bd fill is error!\n"); 16342f072d75SZaibo Xu return ret; 16352f072d75SZaibo Xu } 16362f072d75SZaibo Xu 1637c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || 1638c16a70c1SKai Ye ctx->c_ctx.c_mode == SEC_CMODE_GCM) 1639c16a70c1SKai Ye sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe); 1640c16a70c1SKai Ye else 16412f072d75SZaibo Xu sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); 16422f072d75SZaibo Xu 16432f072d75SZaibo Xu return 0; 16442f072d75SZaibo Xu } 16452f072d75SZaibo Xu 1646adc3f65aSKai Ye static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, 1647adc3f65aSKai Ye struct sec_req *req, struct sec_sqe3 *sqe3) 1648adc3f65aSKai Ye { 1649adc3f65aSKai Ye struct sec_aead_req *a_req = &req->aead_req; 1650adc3f65aSKai Ye struct sec_cipher_req *c_req = &req->c_req; 1651adc3f65aSKai Ye struct aead_request *aq = a_req->aead_req; 1652adc3f65aSKai Ye 1653adc3f65aSKai Ye sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); 1654adc3f65aSKai Ye 1655adc3f65aSKai Ye sqe3->auth_mac_key |= 1656adc3f65aSKai Ye cpu_to_le32((u32)(ctx->mac_len / 1657adc3f65aSKai Ye SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3); 1658adc3f65aSKai Ye 1659adc3f65aSKai Ye sqe3->auth_mac_key |= 1660adc3f65aSKai Ye cpu_to_le32((u32)(ctx->a_key_len / 1661adc3f65aSKai Ye SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3); 1662adc3f65aSKai Ye 1663adc3f65aSKai Ye sqe3->auth_mac_key |= 1664adc3f65aSKai Ye cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3); 1665adc3f65aSKai Ye 1666adc3f65aSKai Ye if (dir) { 1667adc3f65aSKai Ye sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); 1668adc3f65aSKai Ye sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; 1669adc3f65aSKai Ye } else { 167060ef3ddeSKai Ye sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE2); 1671adc3f65aSKai Ye sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; 1672adc3f65aSKai Ye } 1673adc3f65aSKai Ye sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen); 1674adc3f65aSKai Ye 1675adc3f65aSKai Ye sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); 1676adc3f65aSKai Ye 1677adc3f65aSKai Ye sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); 1678adc3f65aSKai Ye } 1679adc3f65aSKai Ye 1680adc3f65aSKai Ye static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) 1681adc3f65aSKai Ye { 1682adc3f65aSKai Ye struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 1683adc3f65aSKai Ye struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; 1684adc3f65aSKai Ye int ret; 1685adc3f65aSKai Ye 1686adc3f65aSKai Ye ret = sec_skcipher_bd_fill_v3(ctx, req); 1687adc3f65aSKai Ye if (unlikely(ret)) { 1688adc3f65aSKai Ye dev_err(ctx->dev, "skcipher bd3 fill is error!\n"); 1689adc3f65aSKai Ye return ret; 1690adc3f65aSKai Ye } 1691adc3f65aSKai Ye 1692c16a70c1SKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || 1693c16a70c1SKai Ye ctx->c_ctx.c_mode == SEC_CMODE_GCM) 1694c16a70c1SKai Ye sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt, 1695c16a70c1SKai Ye req, sec_sqe3); 1696c16a70c1SKai Ye else 1697c16a70c1SKai Ye sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt, 1698c16a70c1SKai Ye req, sec_sqe3); 1699adc3f65aSKai Ye 1700adc3f65aSKai Ye return 0; 1701adc3f65aSKai Ye } 1702adc3f65aSKai Ye 17032f072d75SZaibo Xu static void sec_aead_callback(struct sec_ctx *c, struct sec_req *req, int err) 17042f072d75SZaibo Xu { 17052f072d75SZaibo Xu struct aead_request *a_req = req->aead_req.aead_req; 17062f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 17072514f559SLongfang Liu struct sec_aead_req *aead_req = &req->aead_req; 17082f072d75SZaibo Xu struct sec_cipher_req *c_req = &req->c_req; 17092f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm); 17102f072d75SZaibo Xu struct sec_qp_ctx *qp_ctx = req->qp_ctx; 17119597efc3SKai Ye struct aead_request *backlog_aead_req; 17129597efc3SKai Ye struct sec_req *backlog_req; 17132f072d75SZaibo Xu size_t sz; 17142f072d75SZaibo Xu 17152f072d75SZaibo Xu if (!err && c->c_ctx.c_mode == SEC_CMODE_CBC && c_req->encrypt) 17162f072d75SZaibo Xu sec_update_iv(req, SEC_AEAD); 17172f072d75SZaibo Xu 17182f072d75SZaibo Xu /* Copy output mac */ 17192f072d75SZaibo Xu if (!err && c_req->encrypt) { 17202f072d75SZaibo Xu struct scatterlist *sgl = a_req->dst; 17212f072d75SZaibo Xu 17222f072d75SZaibo Xu sz = sg_pcopy_from_buffer(sgl, sg_nents(sgl), 17232514f559SLongfang Liu aead_req->out_mac, 17242f072d75SZaibo Xu authsize, a_req->cryptlen + 17252f072d75SZaibo Xu a_req->assoclen); 17262f072d75SZaibo Xu if (unlikely(sz != authsize)) { 1727a44dce50SLongfang Liu dev_err(c->dev, "copy out mac err!\n"); 17282f072d75SZaibo Xu err = -EINVAL; 17292f072d75SZaibo Xu } 17302f072d75SZaibo Xu } 17312f072d75SZaibo Xu 17322f072d75SZaibo Xu sec_free_req_id(req); 17332f072d75SZaibo Xu 17349597efc3SKai Ye while (1) { 17359597efc3SKai Ye backlog_req = sec_back_req_clear(c, qp_ctx); 17369597efc3SKai Ye if (!backlog_req) 17379597efc3SKai Ye break; 17389597efc3SKai Ye 17399597efc3SKai Ye backlog_aead_req = backlog_req->aead_req.aead_req; 17409597efc3SKai Ye backlog_aead_req->base.complete(&backlog_aead_req->base, 17419597efc3SKai Ye -EINPROGRESS); 17429597efc3SKai Ye atomic64_inc(&c->sec->debug.dfx.recv_busy_cnt); 17439597efc3SKai Ye } 17442f072d75SZaibo Xu 17452f072d75SZaibo Xu a_req->base.complete(&a_req->base, err); 17462f072d75SZaibo Xu } 17472f072d75SZaibo Xu 1748416d8220SZaibo Xu static void sec_request_uninit(struct sec_ctx *ctx, struct sec_req *req) 1749416d8220SZaibo Xu { 1750416d8220SZaibo Xu sec_free_req_id(req); 1751a181647cSZaibo Xu sec_free_queue_id(ctx, req); 1752416d8220SZaibo Xu } 1753416d8220SZaibo Xu 1754416d8220SZaibo Xu static int sec_request_init(struct sec_ctx *ctx, struct sec_req *req) 1755416d8220SZaibo Xu { 1756416d8220SZaibo Xu struct sec_qp_ctx *qp_ctx; 17577c7d902aSZaibo Xu int queue_id; 1758416d8220SZaibo Xu 1759416d8220SZaibo Xu /* To load balance */ 1760a181647cSZaibo Xu queue_id = sec_alloc_queue_id(ctx, req); 1761a181647cSZaibo Xu qp_ctx = &ctx->qp_ctx[queue_id]; 1762416d8220SZaibo Xu 1763416d8220SZaibo Xu req->req_id = sec_alloc_req_id(req, qp_ctx); 1764b9c8d897SZaibo Xu if (unlikely(req->req_id < 0)) { 1765a181647cSZaibo Xu sec_free_queue_id(ctx, req); 1766416d8220SZaibo Xu return req->req_id; 1767416d8220SZaibo Xu } 1768416d8220SZaibo Xu 17697c7d902aSZaibo Xu return 0; 1770416d8220SZaibo Xu } 1771416d8220SZaibo Xu 1772416d8220SZaibo Xu static int sec_process(struct sec_ctx *ctx, struct sec_req *req) 1773416d8220SZaibo Xu { 17742514f559SLongfang Liu struct sec_cipher_req *c_req = &req->c_req; 1775416d8220SZaibo Xu int ret; 1776416d8220SZaibo Xu 1777416d8220SZaibo Xu ret = sec_request_init(ctx, req); 1778b9c8d897SZaibo Xu if (unlikely(ret)) 1779416d8220SZaibo Xu return ret; 1780416d8220SZaibo Xu 1781416d8220SZaibo Xu ret = sec_request_transfer(ctx, req); 1782b9c8d897SZaibo Xu if (unlikely(ret)) 1783416d8220SZaibo Xu goto err_uninit_req; 1784416d8220SZaibo Xu 1785416d8220SZaibo Xu /* Output IV as decrypto */ 17867b44c0eeSKai Ye if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || 17877b44c0eeSKai Ye ctx->c_ctx.c_mode == SEC_CMODE_CTR)) 17882f072d75SZaibo Xu sec_update_iv(req, ctx->alg_type); 1789416d8220SZaibo Xu 1790416d8220SZaibo Xu ret = ctx->req_op->bd_send(ctx, req); 17919597efc3SKai Ye if (unlikely((ret != -EBUSY && ret != -EINPROGRESS) || 17929597efc3SKai Ye (ret == -EBUSY && !(req->flag & CRYPTO_TFM_REQ_MAY_BACKLOG)))) { 1793a44dce50SLongfang Liu dev_err_ratelimited(ctx->dev, "send sec request failed!\n"); 1794416d8220SZaibo Xu goto err_send_req; 1795416d8220SZaibo Xu } 1796416d8220SZaibo Xu 1797416d8220SZaibo Xu return ret; 1798416d8220SZaibo Xu 1799416d8220SZaibo Xu err_send_req: 1800416d8220SZaibo Xu /* As failing, restore the IV from user */ 18012f072d75SZaibo Xu if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) { 18022f072d75SZaibo Xu if (ctx->alg_type == SEC_SKCIPHER) 18032514f559SLongfang Liu memcpy(req->c_req.sk_req->iv, c_req->c_ivin, 1804416d8220SZaibo Xu ctx->c_ctx.ivsize); 18052f072d75SZaibo Xu else 18062514f559SLongfang Liu memcpy(req->aead_req.aead_req->iv, c_req->c_ivin, 18072f072d75SZaibo Xu ctx->c_ctx.ivsize); 18082f072d75SZaibo Xu } 1809416d8220SZaibo Xu 1810416d8220SZaibo Xu sec_request_untransfer(ctx, req); 1811416d8220SZaibo Xu err_uninit_req: 1812416d8220SZaibo Xu sec_request_uninit(ctx, req); 1813416d8220SZaibo Xu return ret; 1814416d8220SZaibo Xu } 1815416d8220SZaibo Xu 1816a181647cSZaibo Xu static const struct sec_req_op sec_skcipher_req_ops = { 1817416d8220SZaibo Xu .buf_map = sec_skcipher_sgl_map, 1818416d8220SZaibo Xu .buf_unmap = sec_skcipher_sgl_unmap, 1819416d8220SZaibo Xu .do_transfer = sec_skcipher_copy_iv, 1820416d8220SZaibo Xu .bd_fill = sec_skcipher_bd_fill, 1821416d8220SZaibo Xu .bd_send = sec_bd_send, 1822416d8220SZaibo Xu .callback = sec_skcipher_callback, 1823416d8220SZaibo Xu .process = sec_process, 1824416d8220SZaibo Xu }; 1825416d8220SZaibo Xu 18262f072d75SZaibo Xu static const struct sec_req_op sec_aead_req_ops = { 18272f072d75SZaibo Xu .buf_map = sec_aead_sgl_map, 18282f072d75SZaibo Xu .buf_unmap = sec_aead_sgl_unmap, 1829c16a70c1SKai Ye .do_transfer = sec_aead_set_iv, 18302f072d75SZaibo Xu .bd_fill = sec_aead_bd_fill, 18312f072d75SZaibo Xu .bd_send = sec_bd_send, 18322f072d75SZaibo Xu .callback = sec_aead_callback, 18332f072d75SZaibo Xu .process = sec_process, 18342f072d75SZaibo Xu }; 18352f072d75SZaibo Xu 1836adc3f65aSKai Ye static const struct sec_req_op sec_skcipher_req_ops_v3 = { 1837adc3f65aSKai Ye .buf_map = sec_skcipher_sgl_map, 1838adc3f65aSKai Ye .buf_unmap = sec_skcipher_sgl_unmap, 1839adc3f65aSKai Ye .do_transfer = sec_skcipher_copy_iv, 1840adc3f65aSKai Ye .bd_fill = sec_skcipher_bd_fill_v3, 1841adc3f65aSKai Ye .bd_send = sec_bd_send, 1842adc3f65aSKai Ye .callback = sec_skcipher_callback, 1843adc3f65aSKai Ye .process = sec_process, 1844adc3f65aSKai Ye }; 1845adc3f65aSKai Ye 1846adc3f65aSKai Ye static const struct sec_req_op sec_aead_req_ops_v3 = { 1847adc3f65aSKai Ye .buf_map = sec_aead_sgl_map, 1848adc3f65aSKai Ye .buf_unmap = sec_aead_sgl_unmap, 1849c16a70c1SKai Ye .do_transfer = sec_aead_set_iv, 1850adc3f65aSKai Ye .bd_fill = sec_aead_bd_fill_v3, 1851adc3f65aSKai Ye .bd_send = sec_bd_send, 1852adc3f65aSKai Ye .callback = sec_aead_callback, 1853adc3f65aSKai Ye .process = sec_process, 1854adc3f65aSKai Ye }; 1855adc3f65aSKai Ye 1856416d8220SZaibo Xu static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) 1857416d8220SZaibo Xu { 1858416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 1859adc3f65aSKai Ye int ret; 1860416d8220SZaibo Xu 1861adc3f65aSKai Ye ret = sec_skcipher_init(tfm); 1862adc3f65aSKai Ye if (ret) 1863adc3f65aSKai Ye return ret; 1864adc3f65aSKai Ye 1865adc3f65aSKai Ye if (ctx->sec->qm.ver < QM_HW_V3) { 1866adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE2; 1867a181647cSZaibo Xu ctx->req_op = &sec_skcipher_req_ops; 1868adc3f65aSKai Ye } else { 1869adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE3; 1870adc3f65aSKai Ye ctx->req_op = &sec_skcipher_req_ops_v3; 1871adc3f65aSKai Ye } 1872416d8220SZaibo Xu 1873adc3f65aSKai Ye return ret; 1874416d8220SZaibo Xu } 1875416d8220SZaibo Xu 1876416d8220SZaibo Xu static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) 1877416d8220SZaibo Xu { 1878a181647cSZaibo Xu sec_skcipher_uninit(tfm); 1879416d8220SZaibo Xu } 1880416d8220SZaibo Xu 18812f072d75SZaibo Xu static int sec_aead_init(struct crypto_aead *tfm) 18822f072d75SZaibo Xu { 18832f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 18842f072d75SZaibo Xu int ret; 18852f072d75SZaibo Xu 18862f072d75SZaibo Xu crypto_aead_set_reqsize(tfm, sizeof(struct sec_req)); 18872f072d75SZaibo Xu ctx->alg_type = SEC_AEAD; 18882f072d75SZaibo Xu ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); 1889c16a70c1SKai Ye if (ctx->c_ctx.ivsize < SEC_AIV_SIZE || 1890c16a70c1SKai Ye ctx->c_ctx.ivsize > SEC_IV_SIZE) { 1891c16a70c1SKai Ye pr_err("get error aead iv size!\n"); 18922f072d75SZaibo Xu return -EINVAL; 18932f072d75SZaibo Xu } 18942f072d75SZaibo Xu 18952f072d75SZaibo Xu ret = sec_ctx_base_init(ctx); 18962f072d75SZaibo Xu if (ret) 18972f072d75SZaibo Xu return ret; 1898adc3f65aSKai Ye if (ctx->sec->qm.ver < QM_HW_V3) { 1899adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE2; 1900adc3f65aSKai Ye ctx->req_op = &sec_aead_req_ops; 1901adc3f65aSKai Ye } else { 1902adc3f65aSKai Ye ctx->type_supported = SEC_BD_TYPE3; 1903adc3f65aSKai Ye ctx->req_op = &sec_aead_req_ops_v3; 1904adc3f65aSKai Ye } 19052f072d75SZaibo Xu 19062f072d75SZaibo Xu ret = sec_auth_init(ctx); 19072f072d75SZaibo Xu if (ret) 19082f072d75SZaibo Xu goto err_auth_init; 19092f072d75SZaibo Xu 19102f072d75SZaibo Xu ret = sec_cipher_init(ctx); 19112f072d75SZaibo Xu if (ret) 19122f072d75SZaibo Xu goto err_cipher_init; 19132f072d75SZaibo Xu 19142f072d75SZaibo Xu return ret; 19152f072d75SZaibo Xu 19162f072d75SZaibo Xu err_cipher_init: 19172f072d75SZaibo Xu sec_auth_uninit(ctx); 19182f072d75SZaibo Xu err_auth_init: 19192f072d75SZaibo Xu sec_ctx_base_uninit(ctx); 19202f072d75SZaibo Xu return ret; 19212f072d75SZaibo Xu } 19222f072d75SZaibo Xu 19232f072d75SZaibo Xu static void sec_aead_exit(struct crypto_aead *tfm) 19242f072d75SZaibo Xu { 19252f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 19262f072d75SZaibo Xu 19272f072d75SZaibo Xu sec_cipher_uninit(ctx); 19282f072d75SZaibo Xu sec_auth_uninit(ctx); 19292f072d75SZaibo Xu sec_ctx_base_uninit(ctx); 19302f072d75SZaibo Xu } 19312f072d75SZaibo Xu 19322f072d75SZaibo Xu static int sec_aead_ctx_init(struct crypto_aead *tfm, const char *hash_name) 19332f072d75SZaibo Xu { 19342f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 19352f072d75SZaibo Xu struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; 19362f072d75SZaibo Xu int ret; 19372f072d75SZaibo Xu 19382f072d75SZaibo Xu ret = sec_aead_init(tfm); 19392f072d75SZaibo Xu if (ret) { 19402f072d75SZaibo Xu pr_err("hisi_sec2: aead init error!\n"); 19412f072d75SZaibo Xu return ret; 19422f072d75SZaibo Xu } 19432f072d75SZaibo Xu 19442f072d75SZaibo Xu auth_ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); 19452f072d75SZaibo Xu if (IS_ERR(auth_ctx->hash_tfm)) { 1946a44dce50SLongfang Liu dev_err(ctx->dev, "aead alloc shash error!\n"); 19472f072d75SZaibo Xu sec_aead_exit(tfm); 19482f072d75SZaibo Xu return PTR_ERR(auth_ctx->hash_tfm); 19492f072d75SZaibo Xu } 19502f072d75SZaibo Xu 19512f072d75SZaibo Xu return 0; 19522f072d75SZaibo Xu } 19532f072d75SZaibo Xu 19542f072d75SZaibo Xu static void sec_aead_ctx_exit(struct crypto_aead *tfm) 19552f072d75SZaibo Xu { 19562f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 19572f072d75SZaibo Xu 19582f072d75SZaibo Xu crypto_free_shash(ctx->a_ctx.hash_tfm); 19592f072d75SZaibo Xu sec_aead_exit(tfm); 19602f072d75SZaibo Xu } 19612f072d75SZaibo Xu 1962c16a70c1SKai Ye static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) 1963c16a70c1SKai Ye { 19646c46a329SKai Ye struct aead_alg *alg = crypto_aead_alg(tfm); 1965c16a70c1SKai Ye struct sec_ctx *ctx = crypto_aead_ctx(tfm); 19666c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 19676c46a329SKai Ye const char *aead_name = alg->base.cra_name; 1968c16a70c1SKai Ye int ret; 1969c16a70c1SKai Ye 1970c16a70c1SKai Ye ret = sec_aead_init(tfm); 1971c16a70c1SKai Ye if (ret) { 1972c16a70c1SKai Ye dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); 1973c16a70c1SKai Ye return ret; 1974c16a70c1SKai Ye } 1975c16a70c1SKai Ye 19766c46a329SKai Ye a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, 19776c46a329SKai Ye CRYPTO_ALG_NEED_FALLBACK | 19786c46a329SKai Ye CRYPTO_ALG_ASYNC); 19796c46a329SKai Ye if (IS_ERR(a_ctx->fallback_aead_tfm)) { 19806c46a329SKai Ye dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); 19816c46a329SKai Ye sec_aead_exit(tfm); 19826c46a329SKai Ye return PTR_ERR(a_ctx->fallback_aead_tfm); 19836c46a329SKai Ye } 19846c46a329SKai Ye a_ctx->fallback = false; 19856c46a329SKai Ye 1986c16a70c1SKai Ye return 0; 1987c16a70c1SKai Ye } 1988c16a70c1SKai Ye 1989c16a70c1SKai Ye static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm) 1990c16a70c1SKai Ye { 19916c46a329SKai Ye struct sec_ctx *ctx = crypto_aead_ctx(tfm); 19926c46a329SKai Ye 19936c46a329SKai Ye crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); 1994c16a70c1SKai Ye sec_aead_exit(tfm); 1995c16a70c1SKai Ye } 1996c16a70c1SKai Ye 19972f072d75SZaibo Xu static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) 19982f072d75SZaibo Xu { 19992f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha1"); 20002f072d75SZaibo Xu } 20012f072d75SZaibo Xu 20022f072d75SZaibo Xu static int sec_aead_sha256_ctx_init(struct crypto_aead *tfm) 20032f072d75SZaibo Xu { 20042f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha256"); 20052f072d75SZaibo Xu } 20062f072d75SZaibo Xu 20072f072d75SZaibo Xu static int sec_aead_sha512_ctx_init(struct crypto_aead *tfm) 20082f072d75SZaibo Xu { 20092f072d75SZaibo Xu return sec_aead_ctx_init(tfm, "sha512"); 20102f072d75SZaibo Xu } 20112f072d75SZaibo Xu 2012059c5342SLongfang Liu static int sec_skcipher_cryptlen_ckeck(struct sec_ctx *ctx, 2013059c5342SLongfang Liu struct sec_req *sreq) 2014059c5342SLongfang Liu { 2015059c5342SLongfang Liu u32 cryptlen = sreq->c_req.sk_req->cryptlen; 2016059c5342SLongfang Liu struct device *dev = ctx->dev; 2017059c5342SLongfang Liu u8 c_mode = ctx->c_ctx.c_mode; 2018059c5342SLongfang Liu int ret = 0; 2019059c5342SLongfang Liu 2020059c5342SLongfang Liu switch (c_mode) { 2021059c5342SLongfang Liu case SEC_CMODE_XTS: 2022059c5342SLongfang Liu if (unlikely(cryptlen < AES_BLOCK_SIZE)) { 2023059c5342SLongfang Liu dev_err(dev, "skcipher XTS mode input length error!\n"); 2024059c5342SLongfang Liu ret = -EINVAL; 2025059c5342SLongfang Liu } 2026059c5342SLongfang Liu break; 2027059c5342SLongfang Liu case SEC_CMODE_ECB: 2028059c5342SLongfang Liu case SEC_CMODE_CBC: 2029059c5342SLongfang Liu if (unlikely(cryptlen & (AES_BLOCK_SIZE - 1))) { 2030059c5342SLongfang Liu dev_err(dev, "skcipher AES input length error!\n"); 2031059c5342SLongfang Liu ret = -EINVAL; 2032059c5342SLongfang Liu } 2033059c5342SLongfang Liu break; 20347b44c0eeSKai Ye case SEC_CMODE_CFB: 20357b44c0eeSKai Ye case SEC_CMODE_OFB: 20367b44c0eeSKai Ye case SEC_CMODE_CTR: 20377b44c0eeSKai Ye if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) { 20387b44c0eeSKai Ye dev_err(dev, "skcipher HW version error!\n"); 20397b44c0eeSKai Ye ret = -EINVAL; 20407b44c0eeSKai Ye } 20417b44c0eeSKai Ye break; 2042059c5342SLongfang Liu default: 2043059c5342SLongfang Liu ret = -EINVAL; 2044059c5342SLongfang Liu } 2045059c5342SLongfang Liu 2046059c5342SLongfang Liu return ret; 2047059c5342SLongfang Liu } 2048059c5342SLongfang Liu 2049d6de2a59SZaibo Xu static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) 2050416d8220SZaibo Xu { 2051d6de2a59SZaibo Xu struct skcipher_request *sk_req = sreq->c_req.sk_req; 2052a44dce50SLongfang Liu struct device *dev = ctx->dev; 2053d6de2a59SZaibo Xu u8 c_alg = ctx->c_ctx.c_alg; 2054416d8220SZaibo Xu 20557b44c0eeSKai Ye if (unlikely(!sk_req->src || !sk_req->dst || 20567b44c0eeSKai Ye sk_req->cryptlen > MAX_INPUT_DATA_LEN)) { 2057416d8220SZaibo Xu dev_err(dev, "skcipher input param error!\n"); 2058416d8220SZaibo Xu return -EINVAL; 2059416d8220SZaibo Xu } 2060d6de2a59SZaibo Xu sreq->c_req.c_len = sk_req->cryptlen; 206174b58db8SLongfang Liu 206274b58db8SLongfang Liu if (ctx->pbuf_supported && sk_req->cryptlen <= SEC_PBUF_SZ) 206374b58db8SLongfang Liu sreq->use_pbuf = true; 206474b58db8SLongfang Liu else 206574b58db8SLongfang Liu sreq->use_pbuf = false; 206674b58db8SLongfang Liu 2067416d8220SZaibo Xu if (c_alg == SEC_CALG_3DES) { 2068b9c8d897SZaibo Xu if (unlikely(sk_req->cryptlen & (DES3_EDE_BLOCK_SIZE - 1))) { 2069416d8220SZaibo Xu dev_err(dev, "skcipher 3des input length error!\n"); 2070416d8220SZaibo Xu return -EINVAL; 2071416d8220SZaibo Xu } 2072416d8220SZaibo Xu return 0; 2073416d8220SZaibo Xu } else if (c_alg == SEC_CALG_AES || c_alg == SEC_CALG_SM4) { 2074059c5342SLongfang Liu return sec_skcipher_cryptlen_ckeck(ctx, sreq); 2075416d8220SZaibo Xu } 2076059c5342SLongfang Liu 2077416d8220SZaibo Xu dev_err(dev, "skcipher algorithm error!\n"); 2078633e507fSLongfang Liu 2079416d8220SZaibo Xu return -EINVAL; 2080416d8220SZaibo Xu } 2081416d8220SZaibo Xu 20825652d55aSKai Ye static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, 20835652d55aSKai Ye struct skcipher_request *sreq, bool encrypt) 20845652d55aSKai Ye { 20855652d55aSKai Ye struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; 2086e764d81dSKai Ye SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm); 20875652d55aSKai Ye struct device *dev = ctx->dev; 20885652d55aSKai Ye int ret; 20895652d55aSKai Ye 20905652d55aSKai Ye if (!c_ctx->fbtfm) { 2091e764d81dSKai Ye dev_err_ratelimited(dev, "the soft tfm isn't supported in the current system.\n"); 20925652d55aSKai Ye return -EINVAL; 20935652d55aSKai Ye } 20945652d55aSKai Ye 20955652d55aSKai Ye skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm); 20965652d55aSKai Ye 20975652d55aSKai Ye /* software need sync mode to do crypto */ 20985652d55aSKai Ye skcipher_request_set_callback(subreq, sreq->base.flags, 20995652d55aSKai Ye NULL, NULL); 21005652d55aSKai Ye skcipher_request_set_crypt(subreq, sreq->src, sreq->dst, 21015652d55aSKai Ye sreq->cryptlen, sreq->iv); 21025652d55aSKai Ye if (encrypt) 21035652d55aSKai Ye ret = crypto_skcipher_encrypt(subreq); 21045652d55aSKai Ye else 21055652d55aSKai Ye ret = crypto_skcipher_decrypt(subreq); 21065652d55aSKai Ye 21075652d55aSKai Ye skcipher_request_zero(subreq); 21085652d55aSKai Ye 21095652d55aSKai Ye return ret; 21105652d55aSKai Ye } 21115652d55aSKai Ye 2112416d8220SZaibo Xu static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) 2113416d8220SZaibo Xu { 2114416d8220SZaibo Xu struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); 2115416d8220SZaibo Xu struct sec_req *req = skcipher_request_ctx(sk_req); 2116416d8220SZaibo Xu struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); 2117416d8220SZaibo Xu int ret; 2118416d8220SZaibo Xu 21195652d55aSKai Ye if (!sk_req->cryptlen) { 21205652d55aSKai Ye if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) 21215652d55aSKai Ye return -EINVAL; 2122416d8220SZaibo Xu return 0; 21235652d55aSKai Ye } 2124416d8220SZaibo Xu 21259597efc3SKai Ye req->flag = sk_req->base.flags; 2126416d8220SZaibo Xu req->c_req.sk_req = sk_req; 2127416d8220SZaibo Xu req->c_req.encrypt = encrypt; 2128416d8220SZaibo Xu req->ctx = ctx; 2129416d8220SZaibo Xu 2130d6de2a59SZaibo Xu ret = sec_skcipher_param_check(ctx, req); 2131d6de2a59SZaibo Xu if (unlikely(ret)) 2132d6de2a59SZaibo Xu return -EINVAL; 2133d6de2a59SZaibo Xu 21345652d55aSKai Ye if (unlikely(ctx->c_ctx.fallback)) 21355652d55aSKai Ye return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); 21365652d55aSKai Ye 2137416d8220SZaibo Xu return ctx->req_op->process(ctx, req); 2138416d8220SZaibo Xu } 2139416d8220SZaibo Xu 2140416d8220SZaibo Xu static int sec_skcipher_encrypt(struct skcipher_request *sk_req) 2141416d8220SZaibo Xu { 2142416d8220SZaibo Xu return sec_skcipher_crypto(sk_req, true); 2143416d8220SZaibo Xu } 2144416d8220SZaibo Xu 2145416d8220SZaibo Xu static int sec_skcipher_decrypt(struct skcipher_request *sk_req) 2146416d8220SZaibo Xu { 2147416d8220SZaibo Xu return sec_skcipher_crypto(sk_req, false); 2148416d8220SZaibo Xu } 2149416d8220SZaibo Xu 2150416d8220SZaibo Xu #define SEC_SKCIPHER_GEN_ALG(sec_cra_name, sec_set_key, sec_min_key_size, \ 2151416d8220SZaibo Xu sec_max_key_size, ctx_init, ctx_exit, blk_size, iv_size)\ 2152416d8220SZaibo Xu {\ 2153416d8220SZaibo Xu .base = {\ 2154416d8220SZaibo Xu .cra_name = sec_cra_name,\ 2155416d8220SZaibo Xu .cra_driver_name = "hisi_sec_"sec_cra_name,\ 2156416d8220SZaibo Xu .cra_priority = SEC_PRIORITY,\ 21575652d55aSKai Ye .cra_flags = CRYPTO_ALG_ASYNC |\ 21585652d55aSKai Ye CRYPTO_ALG_NEED_FALLBACK,\ 2159416d8220SZaibo Xu .cra_blocksize = blk_size,\ 2160416d8220SZaibo Xu .cra_ctxsize = sizeof(struct sec_ctx),\ 2161416d8220SZaibo Xu .cra_module = THIS_MODULE,\ 2162416d8220SZaibo Xu },\ 2163416d8220SZaibo Xu .init = ctx_init,\ 2164416d8220SZaibo Xu .exit = ctx_exit,\ 2165416d8220SZaibo Xu .setkey = sec_set_key,\ 2166416d8220SZaibo Xu .decrypt = sec_skcipher_decrypt,\ 2167416d8220SZaibo Xu .encrypt = sec_skcipher_encrypt,\ 2168416d8220SZaibo Xu .min_keysize = sec_min_key_size,\ 2169416d8220SZaibo Xu .max_keysize = sec_max_key_size,\ 2170416d8220SZaibo Xu .ivsize = iv_size,\ 2171*921715b6SWenkai Lin } 2172416d8220SZaibo Xu 2173416d8220SZaibo Xu #define SEC_SKCIPHER_ALG(name, key_func, min_key_size, \ 2174416d8220SZaibo Xu max_key_size, blk_size, iv_size) \ 2175416d8220SZaibo Xu SEC_SKCIPHER_GEN_ALG(name, key_func, min_key_size, max_key_size, \ 2176416d8220SZaibo Xu sec_skcipher_ctx_init, sec_skcipher_ctx_exit, blk_size, iv_size) 2177416d8220SZaibo Xu 2178*921715b6SWenkai Lin static struct sec_skcipher sec_skciphers[] = { 2179*921715b6SWenkai Lin { 2180*921715b6SWenkai Lin .alg_msk = BIT(0), 2181*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ecb(aes)", sec_setkey_aes_ecb, AES_MIN_KEY_SIZE, 2182*921715b6SWenkai Lin AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, 0), 2183*921715b6SWenkai Lin }, 2184*921715b6SWenkai Lin { 2185*921715b6SWenkai Lin .alg_msk = BIT(1), 2186*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cbc(aes)", sec_setkey_aes_cbc, AES_MIN_KEY_SIZE, 2187*921715b6SWenkai Lin AES_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2188*921715b6SWenkai Lin }, 2189*921715b6SWenkai Lin { 2190*921715b6SWenkai Lin .alg_msk = BIT(2), 2191*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, AES_MIN_KEY_SIZE, 2192*921715b6SWenkai Lin AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2193*921715b6SWenkai Lin }, 2194*921715b6SWenkai Lin { 2195*921715b6SWenkai Lin .alg_msk = BIT(3), 2196*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("xts(aes)", sec_setkey_aes_xts, SEC_XTS_MIN_KEY_SIZE, 2197*921715b6SWenkai Lin SEC_XTS_MAX_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2198*921715b6SWenkai Lin }, 2199*921715b6SWenkai Lin { 2200*921715b6SWenkai Lin .alg_msk = BIT(4), 2201*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb, AES_MIN_KEY_SIZE, 2202*921715b6SWenkai Lin AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2203*921715b6SWenkai Lin }, 2204*921715b6SWenkai Lin { 2205*921715b6SWenkai Lin .alg_msk = BIT(5), 2206*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb, AES_MIN_KEY_SIZE, 2207*921715b6SWenkai Lin AES_MAX_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2208*921715b6SWenkai Lin }, 2209*921715b6SWenkai Lin { 2210*921715b6SWenkai Lin .alg_msk = BIT(12), 2211*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cbc(sm4)", sec_setkey_sm4_cbc, AES_MIN_KEY_SIZE, 2212*921715b6SWenkai Lin AES_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2213*921715b6SWenkai Lin }, 2214*921715b6SWenkai Lin { 2215*921715b6SWenkai Lin .alg_msk = BIT(13), 2216*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, AES_MIN_KEY_SIZE, 2217*921715b6SWenkai Lin AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2218*921715b6SWenkai Lin }, 2219*921715b6SWenkai Lin { 2220*921715b6SWenkai Lin .alg_msk = BIT(14), 2221*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, SEC_XTS_MIN_KEY_SIZE, 2222*921715b6SWenkai Lin SEC_XTS_MIN_KEY_SIZE, AES_BLOCK_SIZE, AES_BLOCK_SIZE), 2223*921715b6SWenkai Lin }, 2224*921715b6SWenkai Lin { 2225*921715b6SWenkai Lin .alg_msk = BIT(15), 2226*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb, AES_MIN_KEY_SIZE, 2227*921715b6SWenkai Lin AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2228*921715b6SWenkai Lin }, 2229*921715b6SWenkai Lin { 2230*921715b6SWenkai Lin .alg_msk = BIT(16), 2231*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb, AES_MIN_KEY_SIZE, 2232*921715b6SWenkai Lin AES_MIN_KEY_SIZE, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE), 2233*921715b6SWenkai Lin }, 2234*921715b6SWenkai Lin { 2235*921715b6SWenkai Lin .alg_msk = BIT(23), 2236*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, SEC_DES3_3KEY_SIZE, 2237*921715b6SWenkai Lin SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0), 2238*921715b6SWenkai Lin }, 2239*921715b6SWenkai Lin { 2240*921715b6SWenkai Lin .alg_msk = BIT(24), 2241*921715b6SWenkai Lin .alg = SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, SEC_DES3_3KEY_SIZE, 2242*921715b6SWenkai Lin SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 2243*921715b6SWenkai Lin DES3_EDE_BLOCK_SIZE), 2244*921715b6SWenkai Lin }, 22457b44c0eeSKai Ye }; 22467b44c0eeSKai Ye 2247c16a70c1SKai Ye static int aead_iv_demension_check(struct aead_request *aead_req) 2248c16a70c1SKai Ye { 2249c16a70c1SKai Ye u8 cl; 2250c16a70c1SKai Ye 2251c16a70c1SKai Ye cl = aead_req->iv[0] + 1; 2252c16a70c1SKai Ye if (cl < IV_CL_MIN || cl > IV_CL_MAX) 2253c16a70c1SKai Ye return -EINVAL; 2254c16a70c1SKai Ye 2255c16a70c1SKai Ye if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl)) 2256c16a70c1SKai Ye return -EOVERFLOW; 2257c16a70c1SKai Ye 2258c16a70c1SKai Ye return 0; 2259c16a70c1SKai Ye } 2260c16a70c1SKai Ye 2261c16a70c1SKai Ye static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq) 2262c16a70c1SKai Ye { 2263c16a70c1SKai Ye struct aead_request *req = sreq->aead_req.aead_req; 2264c16a70c1SKai Ye struct crypto_aead *tfm = crypto_aead_reqtfm(req); 2265c16a70c1SKai Ye size_t authsize = crypto_aead_authsize(tfm); 2266c16a70c1SKai Ye u8 c_mode = ctx->c_ctx.c_mode; 2267c16a70c1SKai Ye struct device *dev = ctx->dev; 2268c16a70c1SKai Ye int ret; 2269c16a70c1SKai Ye 2270c16a70c1SKai Ye if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || 2271c16a70c1SKai Ye req->assoclen > SEC_MAX_AAD_LEN)) { 2272c16a70c1SKai Ye dev_err(dev, "aead input spec error!\n"); 2273c16a70c1SKai Ye return -EINVAL; 2274c16a70c1SKai Ye } 2275c16a70c1SKai Ye 2276c16a70c1SKai Ye if (unlikely((c_mode == SEC_CMODE_GCM && authsize < DES_BLOCK_SIZE) || 2277c16a70c1SKai Ye (c_mode == SEC_CMODE_CCM && (authsize < MIN_MAC_LEN || 2278c16a70c1SKai Ye authsize & MAC_LEN_MASK)))) { 2279c16a70c1SKai Ye dev_err(dev, "aead input mac length error!\n"); 2280c16a70c1SKai Ye return -EINVAL; 2281c16a70c1SKai Ye } 2282c16a70c1SKai Ye 2283c16a70c1SKai Ye if (c_mode == SEC_CMODE_CCM) { 22845e340558SKai Ye if (unlikely(req->assoclen > SEC_MAX_CCM_AAD_LEN)) { 22855e340558SKai Ye dev_err_ratelimited(dev, "CCM input aad parameter is too long!\n"); 22865e340558SKai Ye return -EINVAL; 22875e340558SKai Ye } 2288c16a70c1SKai Ye ret = aead_iv_demension_check(req); 2289c16a70c1SKai Ye if (ret) { 2290c16a70c1SKai Ye dev_err(dev, "aead input iv param error!\n"); 2291c16a70c1SKai Ye return ret; 2292c16a70c1SKai Ye } 2293c16a70c1SKai Ye } 2294c16a70c1SKai Ye 2295c16a70c1SKai Ye if (sreq->c_req.encrypt) 2296c16a70c1SKai Ye sreq->c_req.c_len = req->cryptlen; 2297c16a70c1SKai Ye else 2298c16a70c1SKai Ye sreq->c_req.c_len = req->cryptlen - authsize; 2299c16a70c1SKai Ye if (c_mode == SEC_CMODE_CBC) { 2300c16a70c1SKai Ye if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { 2301c16a70c1SKai Ye dev_err(dev, "aead crypto length error!\n"); 2302c16a70c1SKai Ye return -EINVAL; 2303c16a70c1SKai Ye } 2304c16a70c1SKai Ye } 2305c16a70c1SKai Ye 2306c16a70c1SKai Ye return 0; 2307c16a70c1SKai Ye } 2308c16a70c1SKai Ye 23092f072d75SZaibo Xu static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) 23102f072d75SZaibo Xu { 23112f072d75SZaibo Xu struct aead_request *req = sreq->aead_req.aead_req; 23122f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(req); 23132f072d75SZaibo Xu size_t authsize = crypto_aead_authsize(tfm); 2314a44dce50SLongfang Liu struct device *dev = ctx->dev; 2315a44dce50SLongfang Liu u8 c_alg = ctx->c_ctx.c_alg; 23162f072d75SZaibo Xu 2317c16a70c1SKai Ye if (unlikely(!req->src || !req->dst)) { 2318a44dce50SLongfang Liu dev_err(dev, "aead input param error!\n"); 23192f072d75SZaibo Xu return -EINVAL; 23202f072d75SZaibo Xu } 23212f072d75SZaibo Xu 2322c16a70c1SKai Ye if (ctx->sec->qm.ver == QM_HW_V2) { 2323c16a70c1SKai Ye if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt && 2324c16a70c1SKai Ye req->cryptlen <= authsize))) { 23256c46a329SKai Ye ctx->a_ctx.fallback = true; 2326c16a70c1SKai Ye return -EINVAL; 2327c16a70c1SKai Ye } 2328c16a70c1SKai Ye } 2329c16a70c1SKai Ye 2330c16a70c1SKai Ye /* Support AES or SM4 */ 2331c16a70c1SKai Ye if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) { 2332c16a70c1SKai Ye dev_err(dev, "aead crypto alg error!\n"); 2333c16a70c1SKai Ye return -EINVAL; 2334c16a70c1SKai Ye } 2335c16a70c1SKai Ye 2336c16a70c1SKai Ye if (unlikely(sec_aead_spec_check(ctx, sreq))) 2337c16a70c1SKai Ye return -EINVAL; 2338c16a70c1SKai Ye 233974b58db8SLongfang Liu if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= 234074b58db8SLongfang Liu SEC_PBUF_SZ) 234174b58db8SLongfang Liu sreq->use_pbuf = true; 234274b58db8SLongfang Liu else 234374b58db8SLongfang Liu sreq->use_pbuf = false; 234474b58db8SLongfang Liu 23452f072d75SZaibo Xu return 0; 23462f072d75SZaibo Xu } 23472f072d75SZaibo Xu 23486c46a329SKai Ye static int sec_aead_soft_crypto(struct sec_ctx *ctx, 23496c46a329SKai Ye struct aead_request *aead_req, 23506c46a329SKai Ye bool encrypt) 23516c46a329SKai Ye { 23526c46a329SKai Ye struct sec_auth_ctx *a_ctx = &ctx->a_ctx; 23536c46a329SKai Ye struct device *dev = ctx->dev; 23540a2a464fSKai Ye struct aead_request *subreq; 23550a2a464fSKai Ye int ret; 23566c46a329SKai Ye 23576c46a329SKai Ye /* Kunpeng920 aead mode not support input 0 size */ 23586c46a329SKai Ye if (!a_ctx->fallback_aead_tfm) { 235984c2c729SColin Ian King dev_err(dev, "aead fallback tfm is NULL!\n"); 23606c46a329SKai Ye return -EINVAL; 23616c46a329SKai Ye } 23626c46a329SKai Ye 23630a2a464fSKai Ye subreq = aead_request_alloc(a_ctx->fallback_aead_tfm, GFP_KERNEL); 23640a2a464fSKai Ye if (!subreq) 23650a2a464fSKai Ye return -ENOMEM; 23660a2a464fSKai Ye 23676c46a329SKai Ye aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm); 23686c46a329SKai Ye aead_request_set_callback(subreq, aead_req->base.flags, 23696c46a329SKai Ye aead_req->base.complete, aead_req->base.data); 23706c46a329SKai Ye aead_request_set_crypt(subreq, aead_req->src, aead_req->dst, 23716c46a329SKai Ye aead_req->cryptlen, aead_req->iv); 23726c46a329SKai Ye aead_request_set_ad(subreq, aead_req->assoclen); 23736c46a329SKai Ye 23740a2a464fSKai Ye if (encrypt) 23750a2a464fSKai Ye ret = crypto_aead_encrypt(subreq); 23760a2a464fSKai Ye else 23770a2a464fSKai Ye ret = crypto_aead_decrypt(subreq); 23780a2a464fSKai Ye aead_request_free(subreq); 23790a2a464fSKai Ye 23800a2a464fSKai Ye return ret; 23816c46a329SKai Ye } 23826c46a329SKai Ye 23832f072d75SZaibo Xu static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) 23842f072d75SZaibo Xu { 23852f072d75SZaibo Xu struct crypto_aead *tfm = crypto_aead_reqtfm(a_req); 23862f072d75SZaibo Xu struct sec_req *req = aead_request_ctx(a_req); 23872f072d75SZaibo Xu struct sec_ctx *ctx = crypto_aead_ctx(tfm); 23882f072d75SZaibo Xu int ret; 23892f072d75SZaibo Xu 23909597efc3SKai Ye req->flag = a_req->base.flags; 23912f072d75SZaibo Xu req->aead_req.aead_req = a_req; 23922f072d75SZaibo Xu req->c_req.encrypt = encrypt; 23932f072d75SZaibo Xu req->ctx = ctx; 23942f072d75SZaibo Xu 23952f072d75SZaibo Xu ret = sec_aead_param_check(ctx, req); 23966c46a329SKai Ye if (unlikely(ret)) { 23976c46a329SKai Ye if (ctx->a_ctx.fallback) 23986c46a329SKai Ye return sec_aead_soft_crypto(ctx, a_req, encrypt); 23992f072d75SZaibo Xu return -EINVAL; 24006c46a329SKai Ye } 24012f072d75SZaibo Xu 24022f072d75SZaibo Xu return ctx->req_op->process(ctx, req); 24032f072d75SZaibo Xu } 24042f072d75SZaibo Xu 24052f072d75SZaibo Xu static int sec_aead_encrypt(struct aead_request *a_req) 24062f072d75SZaibo Xu { 24072f072d75SZaibo Xu return sec_aead_crypto(a_req, true); 24082f072d75SZaibo Xu } 24092f072d75SZaibo Xu 24102f072d75SZaibo Xu static int sec_aead_decrypt(struct aead_request *a_req) 24112f072d75SZaibo Xu { 24122f072d75SZaibo Xu return sec_aead_crypto(a_req, false); 24132f072d75SZaibo Xu } 24142f072d75SZaibo Xu 2415c16a70c1SKai Ye #define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\ 24162f072d75SZaibo Xu ctx_exit, blk_size, iv_size, max_authsize)\ 24172f072d75SZaibo Xu {\ 24182f072d75SZaibo Xu .base = {\ 24192f072d75SZaibo Xu .cra_name = sec_cra_name,\ 24202f072d75SZaibo Xu .cra_driver_name = "hisi_sec_"sec_cra_name,\ 24212f072d75SZaibo Xu .cra_priority = SEC_PRIORITY,\ 24226c46a329SKai Ye .cra_flags = CRYPTO_ALG_ASYNC |\ 24236c46a329SKai Ye CRYPTO_ALG_NEED_FALLBACK,\ 24242f072d75SZaibo Xu .cra_blocksize = blk_size,\ 24252f072d75SZaibo Xu .cra_ctxsize = sizeof(struct sec_ctx),\ 24262f072d75SZaibo Xu .cra_module = THIS_MODULE,\ 24272f072d75SZaibo Xu },\ 24282f072d75SZaibo Xu .init = ctx_init,\ 24292f072d75SZaibo Xu .exit = ctx_exit,\ 24302f072d75SZaibo Xu .setkey = sec_set_key,\ 24316c46a329SKai Ye .setauthsize = sec_aead_setauthsize,\ 24322f072d75SZaibo Xu .decrypt = sec_aead_decrypt,\ 24332f072d75SZaibo Xu .encrypt = sec_aead_encrypt,\ 24342f072d75SZaibo Xu .ivsize = iv_size,\ 24352f072d75SZaibo Xu .maxauthsize = max_authsize,\ 24362f072d75SZaibo Xu } 24372f072d75SZaibo Xu 2438*921715b6SWenkai Lin static struct sec_aead sec_aeads[] = { 2439*921715b6SWenkai Lin { 2440*921715b6SWenkai Lin .alg_msk = BIT(6), 2441*921715b6SWenkai Lin .alg = SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init, 2442*921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, 2443*921715b6SWenkai Lin AES_BLOCK_SIZE), 2444*921715b6SWenkai Lin }, 2445*921715b6SWenkai Lin { 2446*921715b6SWenkai Lin .alg_msk = BIT(7), 2447*921715b6SWenkai Lin .alg = SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init, 2448*921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, 2449*921715b6SWenkai Lin AES_BLOCK_SIZE), 2450*921715b6SWenkai Lin }, 2451*921715b6SWenkai Lin { 2452*921715b6SWenkai Lin .alg_msk = BIT(17), 2453*921715b6SWenkai Lin .alg = SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init, 2454*921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE, 2455*921715b6SWenkai Lin AES_BLOCK_SIZE), 2456*921715b6SWenkai Lin }, 2457*921715b6SWenkai Lin { 2458*921715b6SWenkai Lin .alg_msk = BIT(18), 2459*921715b6SWenkai Lin .alg = SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init, 2460*921715b6SWenkai Lin sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, SEC_AIV_SIZE, 2461*921715b6SWenkai Lin AES_BLOCK_SIZE), 2462*921715b6SWenkai Lin }, 2463*921715b6SWenkai Lin { 2464*921715b6SWenkai Lin .alg_msk = BIT(43), 2465*921715b6SWenkai Lin .alg = SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1, 2466*921715b6SWenkai Lin sec_aead_sha1_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, 2467c16a70c1SKai Ye AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), 2468*921715b6SWenkai Lin }, 2469*921715b6SWenkai Lin { 2470*921715b6SWenkai Lin .alg_msk = BIT(44), 2471*921715b6SWenkai Lin .alg = SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256, 2472*921715b6SWenkai Lin sec_aead_sha256_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, 2473c16a70c1SKai Ye AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), 2474*921715b6SWenkai Lin }, 2475*921715b6SWenkai Lin { 2476*921715b6SWenkai Lin .alg_msk = BIT(45), 2477*921715b6SWenkai Lin .alg = SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512, 2478*921715b6SWenkai Lin sec_aead_sha512_ctx_init, sec_aead_ctx_exit, AES_BLOCK_SIZE, 2479c16a70c1SKai Ye AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), 2480*921715b6SWenkai Lin }, 2481c16a70c1SKai Ye }; 2482c16a70c1SKai Ye 2483*921715b6SWenkai Lin static void sec_unregister_skcipher(u64 alg_mask, int end) 2484*921715b6SWenkai Lin { 2485*921715b6SWenkai Lin int i; 2486c16a70c1SKai Ye 2487*921715b6SWenkai Lin for (i = 0; i < end; i++) 2488*921715b6SWenkai Lin if (sec_skciphers[i].alg_msk & alg_mask) 2489*921715b6SWenkai Lin crypto_unregister_skcipher(&sec_skciphers[i].alg); 2490*921715b6SWenkai Lin } 2491*921715b6SWenkai Lin 2492*921715b6SWenkai Lin static int sec_register_skcipher(u64 alg_mask) 2493*921715b6SWenkai Lin { 2494*921715b6SWenkai Lin int i, ret, count; 2495*921715b6SWenkai Lin 2496*921715b6SWenkai Lin count = ARRAY_SIZE(sec_skciphers); 2497*921715b6SWenkai Lin 2498*921715b6SWenkai Lin for (i = 0; i < count; i++) { 2499*921715b6SWenkai Lin if (!(sec_skciphers[i].alg_msk & alg_mask)) 2500*921715b6SWenkai Lin continue; 2501*921715b6SWenkai Lin 2502*921715b6SWenkai Lin ret = crypto_register_skcipher(&sec_skciphers[i].alg); 2503*921715b6SWenkai Lin if (ret) 2504*921715b6SWenkai Lin goto err; 2505*921715b6SWenkai Lin } 2506*921715b6SWenkai Lin 2507*921715b6SWenkai Lin return 0; 2508*921715b6SWenkai Lin 2509*921715b6SWenkai Lin err: 2510*921715b6SWenkai Lin sec_unregister_skcipher(alg_mask, i); 2511*921715b6SWenkai Lin 2512*921715b6SWenkai Lin return ret; 2513*921715b6SWenkai Lin } 2514*921715b6SWenkai Lin 2515*921715b6SWenkai Lin static void sec_unregister_aead(u64 alg_mask, int end) 2516*921715b6SWenkai Lin { 2517*921715b6SWenkai Lin int i; 2518*921715b6SWenkai Lin 2519*921715b6SWenkai Lin for (i = 0; i < end; i++) 2520*921715b6SWenkai Lin if (sec_aeads[i].alg_msk & alg_mask) 2521*921715b6SWenkai Lin crypto_unregister_aead(&sec_aeads[i].alg); 2522*921715b6SWenkai Lin } 2523*921715b6SWenkai Lin 2524*921715b6SWenkai Lin static int sec_register_aead(u64 alg_mask) 2525*921715b6SWenkai Lin { 2526*921715b6SWenkai Lin int i, ret, count; 2527*921715b6SWenkai Lin 2528*921715b6SWenkai Lin count = ARRAY_SIZE(sec_aeads); 2529*921715b6SWenkai Lin 2530*921715b6SWenkai Lin for (i = 0; i < count; i++) { 2531*921715b6SWenkai Lin if (!(sec_aeads[i].alg_msk & alg_mask)) 2532*921715b6SWenkai Lin continue; 2533*921715b6SWenkai Lin 2534*921715b6SWenkai Lin ret = crypto_register_aead(&sec_aeads[i].alg); 2535*921715b6SWenkai Lin if (ret) 2536*921715b6SWenkai Lin goto err; 2537*921715b6SWenkai Lin } 2538*921715b6SWenkai Lin 2539*921715b6SWenkai Lin return 0; 2540*921715b6SWenkai Lin 2541*921715b6SWenkai Lin err: 2542*921715b6SWenkai Lin sec_unregister_aead(alg_mask, i); 2543*921715b6SWenkai Lin 2544*921715b6SWenkai Lin return ret; 2545*921715b6SWenkai Lin } 25462f072d75SZaibo Xu 25478123455aSMeng Yu int sec_register_to_crypto(struct hisi_qm *qm) 2548416d8220SZaibo Xu { 2549*921715b6SWenkai Lin u64 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH, SEC_DRV_ALG_BITMAP_LOW); 25503d29e98dSYang Shen int ret; 2551416d8220SZaibo Xu 2552*921715b6SWenkai Lin ret = sec_register_skcipher(alg_mask); 25532f072d75SZaibo Xu if (ret) 25542f072d75SZaibo Xu return ret; 25552f072d75SZaibo Xu 2556*921715b6SWenkai Lin ret = sec_register_aead(alg_mask); 25577b44c0eeSKai Ye if (ret) 2558*921715b6SWenkai Lin sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); 2559c16a70c1SKai Ye 2560416d8220SZaibo Xu return ret; 2561416d8220SZaibo Xu } 2562416d8220SZaibo Xu 25638123455aSMeng Yu void sec_unregister_from_crypto(struct hisi_qm *qm) 2564416d8220SZaibo Xu { 2565*921715b6SWenkai Lin u64 alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH, SEC_DRV_ALG_BITMAP_LOW); 2566c16a70c1SKai Ye 2567*921715b6SWenkai Lin sec_unregister_aead(alg_mask, ARRAY_SIZE(sec_aeads)); 2568*921715b6SWenkai Lin sec_unregister_skcipher(alg_mask, ARRAY_SIZE(sec_skciphers)); 25692f072d75SZaibo Xu } 2570